@backstage/plugin-catalog-backend 1.6.0-next.3 → 1.7.0-next.0

package/CHANGELOG.md

@@ -1,5 +1,97 @@
 # @backstage/plugin-catalog-backend
 
+## 1.7.0-next.0
+
+### Minor Changes
+
+- f75bf76330: Implemented server side ordering in the entities endpoint
+
+### Patch Changes
+
+- d136793ff0: Fixed an issue where internal references in the catalog would stick around for longer than expected, causing entities to not be deleted or orphaned as expected.
+- Updated dependencies
+  - @backstage/catalog-model@1.1.5-next.0
+  - @backstage/plugin-scaffolder-common@1.2.4-next.0
+  - @backstage/catalog-client@1.3.0-next.0
+  - @backstage/backend-common@0.17.0
+  - @backstage/backend-plugin-api@0.2.0
+  - @backstage/config@1.0.5
+  - @backstage/errors@1.1.4
+  - @backstage/integration@1.4.1
+  - @backstage/types@1.0.2
+  - @backstage/plugin-catalog-common@1.0.10-next.0
+  - @backstage/plugin-catalog-node@1.3.1-next.0
+  - @backstage/plugin-permission-common@0.7.2
+  - @backstage/plugin-permission-node@0.7.2
+  - @backstage/plugin-search-common@1.2.0
+
+## 1.6.0
+
+### Minor Changes
+
+- 16891a212c: Added new `POST /entities/by-refs` endpoint, which allows you to efficiently
+  batch-fetch entities by their entity ref. This can be useful e.g. in graphql
+  resolvers or similar contexts where you need to fetch many entities at the same
+  time.
+- 273ba3a77f: Deprecated Prometheus metrics in favour of OpenTelemtry metrics.
+- c395abb5b2: The catalog no longer stops after the first processor `validateEntityKind`
+  method returns `true` when validating entity kind shapes. Instead, it continues
+  through all registered processors that have this method, and requires that _at
+  least one_ of them returned true.
+
+  The old behavior of stopping early made it harder to extend existing core kinds
+  with additional fields, since the `BuiltinKindsEntityProcessor` is always
+  present at the top of the processing chain and ensures that your additional
+  validation code would never be run.
+
+  This is technically a breaking change, although it should not affect anybody
+  under normal circumstances, except if you had problematic validation code that
+  you were unaware that it was not being run. That code may now start to exhibit
+  those problems.
+
+  If you need to disable this new behavior, `CatalogBuilder` as used in your
+  `packages/backend/src/plugins/catalog.ts` file now has a
+  `useLegacySingleProcessorValidation()` method to go back to the old behavior.
+
+  ```diff
+   const builder = await CatalogBuilder.create(env);
+  +builder.useLegacySingleProcessorValidation();
+  ```
+
+- 3072ebfdd7: The search table also holds the original entity value now and the facets endpoint fetches the filtered entity data from the search table.
+
+### Patch Changes
+
+- ba13ff663c: Added a new `catalog.rules[].location` configuration that makes it possible to configure catalog rules to only apply to specific locations, either via exact match or a glob pattern.
+- d8593ce0e6: Do not use deprecated `LocationSpec` from the `@backstage/plugin-catalog-node` package
+- c507aee8a2: Ensured typescript type checks in migration files.
+- 2a8e3cc0b5: Optimize `Stitcher` process to be more memory efficient
+- 884d749b14: Refactored to use `coreServices` from `@backstage/backend-plugin-api`.
+- eacc8e2b55: Make it possible for entity providers to supply only entity refs, instead of full entities, in `delta` mutation deletions.
+- b05dcd5530: Move the `zod` dependency to a version that does not collide with other libraries
+- 5b3e2afa45: Fixed deprecated use of `substr` into `substring`.
+- 71147d5c16: Internal code reorganization.
+- 93870e4df1: Track the last time the final entity changed with new timestamp "last updated at" data in final entities database, which gets updated with the time when final entity is updated.
+- 20a5161f04: Adds MySQL support for the catalog-backend
+- 3280711113: Updated dependency `msw` to `^0.49.0`.
+- e982f77fe3: Registered shutdown hook in experimental catalog plugin.
+- b3fac9c107: Ignore attempts at emitting the current entity as a child of itself.
+- Updated dependencies
+  - @backstage/catalog-client@1.2.0
+  - @backstage/backend-common@0.17.0
+  - @backstage/plugin-catalog-node@1.3.0
+  - @backstage/plugin-permission-common@0.7.2
+  - @backstage/plugin-permission-node@0.7.2
+  - @backstage/errors@1.1.4
+  - @backstage/backend-plugin-api@0.2.0
+  - @backstage/integration@1.4.1
+  - @backstage/types@1.0.2
+  - @backstage/plugin-search-common@1.2.0
+  - @backstage/catalog-model@1.1.4
+  - @backstage/config@1.0.5
+  - @backstage/plugin-catalog-common@1.0.9
+  - @backstage/plugin-scaffolder-common@1.2.3
+
 ## 1.6.0-next.3
 
 ### Patch Changes

package/alpha/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@backstage/plugin-catalog-backend",
-  "version": "1.6.0-next.3",
+  "version": "1.7.0-next.0",
   "main": "../dist/index.cjs.js",
   "types": "../dist/index.alpha.d.ts"
 }

package/dist/index.cjs.js

@@ -28,6 +28,7 @@ var backendCommon = require('@backstage/backend-common');
 var luxon = require('luxon');
 var promClient = require('prom-client');
 var stableStringify = require('fast-json-stable-stringify');
+var api = require('@opentelemetry/api');
 var express = require('express');
 var Router = require('express-promise-router');
 var yn = require('yn');
@@ -1724,7 +1725,6 @@ class DefaultProcessingDatabase {
   async addUnprocessedEntities(txOpaque, options) {
     const tx = txOpaque;
     const stateReferences = new Array();
-    const conflictingStateReferences = new Array();
     for (const { entity, locationKey } of options.entities) {
       const entityRef = catalogModel.stringifyEntityRef(entity);
       const hash = generateStableHash$1(entity);
@@ -1758,10 +1758,9 @@ class DefaultProcessingDatabase {
         this.options.logger.warn(
           `Detected conflicting entityRef ${entityRef} already referenced by ${conflictingKey} and now also ${locationKey}`
         );
-        conflictingStateReferences.push(entityRef);
       }
     }
-    await tx("refresh_state_references").whereNotIn("target_entity_ref", conflictingStateReferences).andWhere({ source_entity_ref: options.sourceEntityRef }).delete();
+    await tx("refresh_state_references").andWhere({ source_entity_ref: options.sourceEntityRef }).delete();
     await tx.batchInsert(
       "refresh_state_references",
       stateReferences.map((entityRef) => ({
@@ -2008,54 +2007,104 @@ class DefaultCatalogProcessingEngine {
   }
 }
 function progressTracker() {
-  const stitchedEntities = createCounterMetric({
+  const promStitchedEntities = createCounterMetric({
     name: "catalog_stitched_entities_count",
-    help: "Amount of entities stitched"
+    help: "Amount of entities stitched. DEPRECATED, use OpenTelemetry metrics instead"
   });
-  const processedEntities = createCounterMetric({
+  const promProcessedEntities = createCounterMetric({
     name: "catalog_processed_entities_count",
-    help: "Amount of entities processed",
+    help: "Amount of entities processed, DEPRECATED, use OpenTelemetry metrics instead",
     labelNames: ["result"]
   });
-  const processingDuration = createSummaryMetric({
+  const promProcessingDuration = createSummaryMetric({
     name: "catalog_processing_duration_seconds",
-    help: "Time spent executing the full processing flow",
+    help: "Time spent executing the full processing flow, DEPRECATED, use OpenTelemetry metrics instead",
     labelNames: ["result"]
   });
-  const processorsDuration = createSummaryMetric({
+  const promProcessorsDuration = createSummaryMetric({
     name: "catalog_processors_duration_seconds",
-    help: "Time spent executing catalog processors",
+    help: "Time spent executing catalog processors, DEPRECATED, use OpenTelemetry metrics instead",
     labelNames: ["result"]
   });
-  const processingQueueDelay = createSummaryMetric({
+  const promProcessingQueueDelay = createSummaryMetric({
     name: "catalog_processing_queue_delay_seconds",
-    help: "The amount of delay between being scheduled for processing, and the start of actually being processed"
+    help: "The amount of delay between being scheduled for processing, and the start of actually being processed, DEPRECATED, use OpenTelemetry metrics instead"
   });
+  const meter = api.metrics.getMeter("default");
+  const stitchedEntities = meter.createCounter(
+    "catalog.stitched.entities.count",
+    {
+      description: "Amount of entities stitched"
+    }
+  );
+  const processedEntities = meter.createCounter(
+    "catalog.processed.entities.count",
+    { description: "Amount of entities processed" }
+  );
+  const processingDuration = meter.createHistogram(
+    "catalog.processing.duration",
+    {
+      description: "Time spent executing the full processing flow",
+      unit: "seconds"
+    }
+  );
+  const processorsDuration = meter.createHistogram(
+    "catalog.processors.duration",
+    {
+      description: "Time spent executing catalog processors",
+      unit: "seconds"
+    }
+  );
+  const processingQueueDelay = meter.createHistogram(
+    "catalog.processing.queue.delay",
+    {
+      description: "The amount of delay between being scheduled for processing, and the start of actually being processed",
+      unit: "seconds"
+    }
+  );
   function processStart(item, logger) {
+    const startTime = process.hrtime();
+    const endOverallTimer = promProcessingDuration.startTimer();
+    const endProcessorsTimer = promProcessorsDuration.startTimer();
     logger.debug(`Processing ${item.entityRef}`);
     if (item.nextUpdateAt) {
-      processingQueueDelay.observe(-item.nextUpdateAt.diffNow().as("seconds"));
+      const seconds = -item.nextUpdateAt.diffNow().as("seconds");
+      promProcessingQueueDelay.observe(seconds);
+      processingQueueDelay.record(seconds);
+    }
+    function endTime() {
+      const delta = process.hrtime(startTime);
+      return delta[0] + delta[1] / 1e9;
     }
-    const endOverallTimer = processingDuration.startTimer();
-    const endProcessorsTimer = processorsDuration.startTimer();
     function markProcessorsCompleted(result) {
       endProcessorsTimer({ result: result.ok ? "ok" : "failed" });
+      processorsDuration.record(endTime(), {
+        result: result.ok ? "ok" : "failed"
+      });
     }
     function markSuccessfulWithNoChanges() {
       endOverallTimer({ result: "unchanged" });
-      processedEntities.inc({ result: "unchanged" }, 1);
+      promProcessedEntities.inc({ result: "unchanged" }, 1);
+      processingDuration.record(endTime(), { result: "unchanged" });
+      processedEntities.add(1, { result: "unchanged" });
     }
     function markSuccessfulWithErrors() {
       endOverallTimer({ result: "errors" });
-      processedEntities.inc({ result: "errors" }, 1);
+      promProcessedEntities.inc({ result: "errors" }, 1);
+      processingDuration.record(endTime(), { result: "errors" });
+      processedEntities.add(1, { result: "errors" });
     }
     function markSuccessfulWithChanges(stitchedCount) {
       endOverallTimer({ result: "changed" });
-      stitchedEntities.inc(stitchedCount);
-      processedEntities.inc({ result: "changed" }, 1);
+      promStitchedEntities.inc(stitchedCount);
+      promProcessedEntities.inc({ result: "changed" }, 1);
+      processingDuration.record(endTime(), { result: "changed" });
+      stitchedEntities.add(stitchedCount);
+      processedEntities.add(1, { result: "changed" });
     }
     function markFailed(error) {
-      processedEntities.inc({ result: "failed" }, 1);
+      promProcessedEntities.inc({ result: "failed" }, 1);
+      processedEntities.add(1, { result: "failed" });
       logger.warn(`Processing of ${item.entityRef} failed`, error);
     }
     return {
@@ -2198,7 +2247,7 @@ function stringifyPagination(input) {
   return base64;
 }
 function addCondition(queryBuilder, db, filter, negate = false, entityIdField = "entity_id") {
-  const matchQuery = db("search").select(entityIdField).where({ key: filter.key.toLowerCase() }).andWhere(function keyFilter() {
+  const matchQuery = db("search").select("search.entity_id").where({ key: filter.key.toLowerCase() }).andWhere(function keyFilter() {
     if (filter.values) {
       if (filter.values.length === 1) {
         this.where({ value: filter.values[0].toLowerCase() });
@@ -2254,12 +2303,41 @@ class DefaultEntitiesCatalog {
     this.stitcher = stitcher;
   }
   async entities(request) {
+    var _a, _b;
     const db = this.database;
     let entitiesQuery = db("final_entities").select("final_entities.*");
+    (_a = request == null ? void 0 : request.order) == null ? void 0 : _a.forEach(({ field }, index) => {
+      const alias = `order_${index}`;
+      entitiesQuery = entitiesQuery.leftOuterJoin(
+        { [alias]: "search" },
+        function search(inner) {
+          inner.on(`${alias}.entity_id`, "final_entities.entity_id").andOn(`${alias}.key`, db.raw("?", [field]));
+        }
+      );
+    });
+    entitiesQuery = entitiesQuery.whereNotNull("final_entities.final_entity");
     if (request == null ? void 0 : request.filter) {
-      entitiesQuery = parseFilter(request.filter, entitiesQuery, db);
+      entitiesQuery = parseFilter(
+        request.filter,
+        entitiesQuery,
+        db,
+        false,
+        "final_entities.entity_id"
+      );
     }
-    entitiesQuery = entitiesQuery.whereNotNull("final_entities.final_entity").orderBy("entity_id", "asc");
+    (_b = request == null ? void 0 : request.order) == null ? void 0 : _b.forEach(({ order }, index) => {
+      if (db.client.config.client === "pg") {
+        entitiesQuery = entitiesQuery.orderBy([
+          { column: `order_${index}.value`, order, nulls: "last" }
+        ]);
+      } else {
+        entitiesQuery = entitiesQuery.orderBy([
+          { column: `order_${index}.value`, order: void 0, nulls: "last" },
+          { column: `order_${index}.value`, order }
+        ]);
+      }
+    });
+    entitiesQuery = entitiesQuery.orderBy("final_entities.entity_id", "asc");
     const { limit, offset } = parsePagination(request == null ? void 0 : request.pagination);
     if (limit !== void 0) {
       entitiesQuery = entitiesQuery.limit(limit + 1);
@@ -2439,8 +2517,14 @@ class ProcessorOutputCollector {
     this.refreshKeys = new Array();
     this.done = false;
   }
-  get onEmit() {
-    return (i) => this.receive(i);
+  generic() {
+    return (i) => this.receive(this.logger, i);
+  }
+  forProcessor(processor) {
+    const logger = this.logger.child({
+      processor: processor.getProcessorName()
+    });
+    return (i) => this.receive(logger, i);
   }
   results() {
     this.done = true;
@@ -2451,9 +2535,9 @@ class ProcessorOutputCollector {
       deferredEntities: this.deferredEntities
     };
   }
-  receive(i) {
+  receive(logger, i) {
     if (this.done) {
-      this.logger.warn(
+      logger.warn(
         `Item of type "${i.type}" was emitted after processing had completed. Stack trace: ${new Error().stack}`
       );
       return;
@@ -2465,10 +2549,17 @@ class ProcessorOutputCollector {
         entity = validateEntityEnvelope(i.entity);
       } catch (e) {
         errors.assertError(e);
-        this.logger.debug(`Envelope validation failed at ${location}, ${e}`);
+        logger.debug(`Envelope validation failed at ${location}, ${e}`);
         this.errors.push(e);
         return;
       }
+      const entityRef = catalogModel.stringifyEntityRef(entity);
+      if (entityRef === catalogModel.stringifyEntityRef(this.parentEntity)) {
+        logger.warn(
+          `Ignored emitted entity ${entityRef} whose ref was identical to the one being processed. This commonly indicates mistakenly emitting the input entity instead of returning it.`
+        );
+        return;
+      }
       const annotations = entity.metadata.annotations || {};
       if (typeof annotations === "object" && !Array.isArray(annotations)) {
         const originLocation = getEntityOriginLocationRef(this.parentEntity);
@@ -2666,7 +2757,7 @@ class DefaultCatalogProcessingOrchestrator {
           res = await processor.preProcessEntity(
             res,
             context.location,
-            context.collector.onEmit,
+            context.collector.forProcessor(processor),
             context.originLocation,
             context.cache.forProcessor(processor)
           );
@@ -2749,7 +2840,7 @@ class DefaultCatalogProcessingOrchestrator {
     }
     for (const maybeRelativeTarget of targets) {
       if (type === "file" && maybeRelativeTarget.endsWith(path__default["default"].sep)) {
-        context.collector.onEmit(
+        context.collector.generic()(
           pluginCatalogNode.processingResult.inputError(
             context.location,
             `LocationEntityProcessor cannot handle ${type} type location with target ${context.location.target} that ends with a path separator`
@@ -2774,7 +2865,7 @@ class DefaultCatalogProcessingOrchestrator {
                 presence
               },
               presence === "optional",
-              context.collector.onEmit,
+              context.collector.forProcessor(processor),
               this.options.parser,
               context.cache.forProcessor(processor, target)
             );
@@ -2805,7 +2896,7 @@ class DefaultCatalogProcessingOrchestrator {
           res = await processor.postProcessEntity(
             res,
             context.location,
-            context.collector.onEmit,
+            context.collector.forProcessor(processor),
             context.cache.forProcessor(processor)
           );
         } catch (e) {
@@ -3041,8 +3132,9 @@ class Stitcher {
       "final_entities"
     ).update({
       final_entity: JSON.stringify(entity),
-      hash
-    }).where("entity_id", entityId).where("stitch_ticket", ticket).onConflict("entity_id").merge(["final_entity", "hash"]);
+      hash,
+      last_updated_at: this.database.fn.now()
+    }).where("entity_id", entityId).where("stitch_ticket", ticket).onConflict("entity_id").merge(["final_entity", "hash", "last_updated_at"]);
     if (amountOfRowsChanged === 0) {
       this.logger.debug(
         `Entity ${entityRef} is already processed, skipping write.`
@@ -3217,6 +3309,22 @@ function parseEntityFacetParams(params) {
   throw new errors.InputError("Missing facet parameter");
 }
 
+function parseEntityOrderParams(params) {
+  var _a;
+  return (_a = parseStringsParam(params.order, "order")) == null ? void 0 : _a.map((item) => {
+    const match = item.match(/^(asc|desc):(.+)$/);
+    if (!match) {
+      throw new errors.InputError(
+        `Invalid order parameter "${item}", expected "<asc or desc>:<field name>"`
+      );
+    }
+    return {
+      order: match[1],
+      field: match[2]
+    };
+  });
+}
+
 async function requireRequestBody(req) {
   const contentType = req.header("content-type");
   if (!contentType) {
@@ -3288,6 +3396,7 @@ async function createRouter(options) {
       const { entities, pageInfo } = await entitiesCatalog.entities({
         filter: parseEntityFilterParams(req.query),
         fields: parseEntityTransformParams(req.query),
+        order: parseEntityOrderParams(req.query),
         pagination: parseEntityPaginationParams(req.query),
         authorizationToken: getBearerToken(req.header("authorization"))
       });
@@ -4024,6 +4133,7 @@ class DefaultProviderDatabase {
               logger: this.options.logger
             });
           }
+          await tx("refresh_state_references").where("target_entity_ref", entityRef).andWhere({ source_key: options.sourceKey }).delete();
           if (ok) {
             await tx(
               "refresh_state_references"