@backstage/plugin-catalog-backend 1.3.1 → 1.4.0-next.2

package/CHANGELOG.md

@@ -1,5 +1,56 @@
 # @backstage/plugin-catalog-backend
 
+## 1.4.0-next.2
+
+### Patch Changes
+
+- eadf56bbbf: Bump `git-url-parse` version to `^13.0.0`
+- 667d917488: Updated dependency `msw` to `^0.47.0`.
+- 87ec2ba4d6: Updated dependency `msw` to `^0.46.0`.
+- 06e2b077a1: Limit the length of error messages that get written to the database and logs - to prevent performance issues
+- Updated dependencies
+  - @backstage/backend-plugin-api@0.1.2-next.1
+  - @backstage/plugin-catalog-node@1.0.2-next.1
+  - @backstage/backend-common@0.15.1-next.2
+  - @backstage/integration@1.3.1-next.1
+  - @backstage/catalog-client@1.0.5-next.1
+  - @backstage/plugin-permission-common@0.6.4-next.1
+  - @backstage/plugin-permission-node@0.6.5-next.2
+
+## 1.4.0-next.1
+
+### Minor Changes
+
+- dd395335bc: Allow unknown typed location from being registered via the location service by configuration settings
+- 651c9d6800: The search index now does retain fields that have a very long value, but in the form of just a null. This makes it possible to at least filter for their existence.
+
+### Patch Changes
+
+- ce77e78c93: Fixes a bug to be able to utilize refresh keys after the entity is loaded from cache
+- 679f7c5e95: Include entity ref into error message when catalog policies fail
+- Updated dependencies
+  - @backstage/plugin-permission-node@0.6.5-next.1
+  - @backstage/backend-common@0.15.1-next.1
+
+## 1.3.2-next.0
+
+### Patch Changes
+
+- 243533ecdc: Added support to mysql on some raw queries
+- bf5e9030eb: Updated dependency `msw` to `^0.45.0`.
+- 62788b2ee8: The experimental `CatalogProcessingExtensionPoint` now accepts multiple providers and processors at once.
+- Updated dependencies
+  - @backstage/backend-common@0.15.1-next.0
+  - @backstage/backend-plugin-api@0.1.2-next.0
+  - @backstage/catalog-client@1.0.5-next.0
+  - @backstage/integration@1.3.1-next.0
+  - @backstage/plugin-permission-common@0.6.4-next.0
+  - @backstage/plugin-permission-node@0.6.5-next.0
+  - @backstage/plugin-scaffolder-common@1.2.0-next.0
+  - @backstage/plugin-catalog-node@1.0.2-next.0
+  - @backstage/plugin-catalog-common@1.0.6-next.0
+  - @backstage/plugin-search-common@1.0.1-next.0
+
 ## 1.3.1
 
 ### Patch Changes

package/alpha/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@backstage/plugin-catalog-backend",
-  "version": "1.3.1",
+  "version": "1.4.0-next.2",
   "main": "../dist/index.cjs.js",
   "types": "../dist/index.alpha.d.ts"
 }

package/dist/index.alpha.d.ts

@@ -171,7 +171,8 @@ export declare class CatalogBuilder {
     private onProcessingError?;
     private processingInterval;
     private locationAnalyzer;
-    private permissionRules;
+    private readonly permissionRules;
+    private allowedLocationType;
     /**
      * Creates a catalog builder.
      */
@@ -293,6 +294,12 @@ export declare class CatalogBuilder {
      * @alpha
      */
     addPermissionRules(...permissionRules: Array<CatalogPermissionRule | Array<CatalogPermissionRule>>): void;
+    /**
+     * Sets up the allowed location types from being registered via the location service.
+     *
+     * @param allowedLocationTypes - the allowed location types
+     */
+    setAllowedLocationTypes(allowedLocationTypes: string[]): CatalogBuilder;
     /**
      * Wires up and returns all of the component parts of the catalog
      */
@@ -349,7 +356,7 @@ export declare type CatalogPermissionRule<TParams extends unknown[] = unknown[]>
  * Catalog plugin
  * @alpha
  */
-export declare const catalogPlugin: (options?: unknown) => BackendFeature;
+export declare const catalogPlugin: (options?: undefined) => BackendFeature;
 
 /** @public */
 export declare interface CatalogProcessingEngine {

package/dist/index.beta.d.ts

@@ -171,7 +171,8 @@ export declare class CatalogBuilder {
     private onProcessingError?;
     private processingInterval;
     private locationAnalyzer;
-    private permissionRules;
+    private readonly permissionRules;
+    private allowedLocationType;
     /**
      * Creates a catalog builder.
      */
@@ -285,6 +286,12 @@ export declare class CatalogBuilder {
      */
     setEntityDataParser(parser: CatalogProcessorParser): CatalogBuilder;
     /* Excluded from this release type: addPermissionRules */
+    /**
+     * Sets up the allowed location types from being registered via the location service.
+     *
+     * @param allowedLocationTypes - the allowed location types
+     */
+    setAllowedLocationTypes(allowedLocationTypes: string[]): CatalogBuilder;
     /**
      * Wires up and returns all of the component parts of the catalog
      */

package/dist/index.cjs.js

@@ -749,11 +749,15 @@ class UrlReaderProcessor {
       emit(pluginCatalogNode.processingResult.refresh(`${location.type}:${location.target}`));
     } catch (error) {
       errors.assertError(error);
-      const message = `Unable to read ${location.type}, ${error}`;
+      const message = `Unable to read ${location.type}, ${error}`.substring(
+        0,
+        5e3
+      );
       if (error.name === "NotModifiedError" && cacheItem) {
         for (const parseResult of cacheItem.value) {
           emit(parseResult);
         }
+        emit(pluginCatalogNode.processingResult.refresh(`${location.type}:${location.target}`));
       } else if (error.name === "NotFoundError") {
         if (!optional) {
           emit(pluginCatalogNode.processingResult.notFoundError(location, message));
@@ -1506,7 +1510,7 @@ class DefaultProcessingDatabase {
       sourceEntityRef
     });
     let previousRelationRows;
-    if (tx.client.config.client.includes("sqlite3")) {
+    if (tx.client.config.client.includes("sqlite3") || tx.client.config.client.includes("mysql")) {
       previousRelationRows = await tx("relations").select("*").where({ originating_entity_id: id });
       await tx("relations").where({ originating_entity_id: id }).delete();
     } else {
@@ -1558,6 +1562,12 @@ class DefaultProcessingDatabase {
     const { toAdd, toUpsert, toRemove } = await this.createDelta(tx, options);
     if (toRemove.length) {
       let removedCount = 0;
+      const rootId = () => {
+        if (tx.client.config.client.includes("mysql")) {
+          return tx.raw("CAST(NULL as UNSIGNED INT)", []);
+        }
+        return tx.raw("CAST(NULL as INT)", []);
+      };
       for (const refs of lodash__default["default"].chunk(toRemove, 1e3)) {
         removedCount += await tx("refresh_state").whereIn("entity_ref", function orphanedEntityRefs(orphans) {
           return orphans.withRecursive("descendants", function descendants(outer) {
@@ -1571,7 +1581,7 @@ class DefaultProcessingDatabase {
             });
           }).withRecursive("ancestors", function ancestors(outer) {
             return outer.select({
-              root_id: tx.raw("CAST(NULL as INT)", []),
+              root_id: rootId(),
               via_entity_ref: "entity_ref",
               to_entity_ref: "entity_ref"
             }).from("descendants").union(function recursive(inner) {
@@ -1695,11 +1705,20 @@ class DefaultProcessingDatabase {
     }
     const items = await itemsQuery.where("next_update_at", "<=", tx.fn.now()).limit(request.processBatchSize).orderBy("next_update_at", "asc");
     const interval = this.options.refreshInterval();
+    const nextUpdateAt = (refreshInterval) => {
+      if (tx.client.config.client.includes("sqlite3")) {
+        return tx.raw(`datetime('now', ?)`, [`${refreshInterval} seconds`]);
+      }
+      if (tx.client.config.client.includes("mysql")) {
+        return tx.raw(`now() + interval ${refreshInterval} second`);
+      }
+      return tx.raw(`now() + interval '${refreshInterval} seconds'`);
+    };
     await tx("refresh_state").whereIn(
       "entity_ref",
       items.map((i) => i.entity_ref)
     ).update({
-      next_update_at: tx.client.config.client.includes("sqlite3") ? tx.raw(`datetime('now', ?)`, [`${interval} seconds`]) : tx.raw(`now() + interval '${interval} seconds'`)
+      next_update_at: nextUpdateAt(interval)
     });
     return {
       items: items.map(
@@ -2246,13 +2265,20 @@ function progressTracker() {
 }
 
 class DefaultLocationService {
-  constructor(store, orchestrator) {
+  constructor(store, orchestrator, options = {
+    allowedLocationTypes: ["url"]
+  }) {
     this.store = store;
     this.orchestrator = orchestrator;
+    this.options = options;
   }
   async createLocation(input, dryRun) {
-    if (input.type !== "url") {
-      throw new errors.InputError(`Registered locations must be of type 'url'`);
+    if (!this.options.allowedLocationTypes.includes(input.type)) {
+      throw new errors.InputError(
+        `Registered locations must be of an allowed type ${JSON.stringify(
+          this.options.allowedLocationTypes
+        )}`
+      );
     }
     if (dryRun) {
       return this.dryRunCreateLocation(input);
@@ -2808,10 +2834,17 @@ class DefaultCatalogProcessingOrchestrator {
     try {
       policyEnforcedEntity = await this.options.policy.enforce(entity);
     } catch (e) {
-      throw new errors.InputError("Policy check failed", e);
+      throw new errors.InputError(
+        `Policy check failed for ${catalogModel.stringifyEntityRef(entity)}`,
+        e
+      );
     }
     if (!policyEnforcedEntity) {
-      throw new Error("Policy unexpectedly returned no data");
+      throw new Error(
+        `Policy unexpectedly returned no data for ${catalogModel.stringifyEntityRef(
+          entity
+        )}`
+      );
     }
     return policyEnforcedEntity;
   }
@@ -2981,8 +3014,12 @@ function mapToRows(input, entityId) {
       result.push({ entity_id: entityId, key, value: null });
     } else {
       const value = String(rawValue).toLocaleLowerCase("en-US");
-      if (key.length <= MAX_KEY_LENGTH && value.length <= MAX_VALUE_LENGTH) {
-        result.push({ entity_id: entityId, key, value });
+      if (key.length <= MAX_KEY_LENGTH) {
+        result.push({
+          entity_id: entityId,
+          key,
+          value: value.length <= MAX_VALUE_LENGTH ? value : null
+        });
       }
     }
   }
@@ -3905,6 +3942,7 @@ class CatalogBuilder {
     this.processorsReplace = false;
     this.parser = void 0;
     this.permissionRules = Object.values(permissionRules);
+    this.allowedLocationType = ["url"];
   }
   static create(env) {
     return new CatalogBuilder(env);
@@ -3971,6 +4009,10 @@ class CatalogBuilder {
   addPermissionRules(...permissionRules) {
     this.permissionRules.push(...permissionRules.flat());
   }
+  setAllowedLocationTypes(allowedLocationTypes) {
+    this.allowedLocationType = allowedLocationTypes;
+    return this;
+  }
   async build() {
     var _a, _b;
     const { config, database, logger, permissions } = this.env;
@@ -4056,7 +4098,9 @@ class CatalogBuilder {
     );
     const locationAnalyzer = (_b = this.locationAnalyzer) != null ? _b : new RepoLocationAnalyzer(logger, integrations);
     const locationService = new AuthorizedLocationService(
-      new DefaultLocationService(locationStore, orchestrator),
+      new DefaultLocationService(locationStore, orchestrator, {
+        allowedLocationTypes: this.allowedLocationType
+      }),
       permissionEvaluator
     );
     const refreshService = new AuthorizedRefreshService(
@@ -4234,11 +4278,11 @@ class CatalogExtensionPointImpl {
     __privateAdd(this, _processors, new Array());
     __privateAdd(this, _entityProviders, new Array());
   }
-  addProcessor(processor) {
-    __privateGet(this, _processors).push(processor);
+  addProcessor(...processors) {
+    __privateGet(this, _processors).push(...processors.flat());
   }
-  addEntityProvider(provider) {
-    __privateGet(this, _entityProviders).push(provider);
+  addEntityProvider(...providers) {
+    __privateGet(this, _entityProviders).push(...providers.flat());
   }
   get processors() {
     return __privateGet(this, _processors);