@backstage/plugin-catalog-backend 1.6.0-next.3 → 1.6.0

package/CHANGELOG.md

@@ -1,5 +1,72 @@
 # @backstage/plugin-catalog-backend
 
+## 1.6.0
+
+### Minor Changes
+
+- 16891a212c: Added new `POST /entities/by-refs` endpoint, which allows you to efficiently
+  batch-fetch entities by their entity ref. This can be useful e.g. in graphql
+  resolvers or similar contexts where you need to fetch many entities at the same
+  time.
+- 273ba3a77f: Deprecated Prometheus metrics in favour of OpenTelemtry metrics.
+- c395abb5b2: The catalog no longer stops after the first processor `validateEntityKind`
+  method returns `true` when validating entity kind shapes. Instead, it continues
+  through all registered processors that have this method, and requires that _at
+  least one_ of them returned true.
+
+  The old behavior of stopping early made it harder to extend existing core kinds
+  with additional fields, since the `BuiltinKindsEntityProcessor` is always
+  present at the top of the processing chain and ensures that your additional
+  validation code would never be run.
+
+  This is technically a breaking change, although it should not affect anybody
+  under normal circumstances, except if you had problematic validation code that
+  you were unaware that it was not being run. That code may now start to exhibit
+  those problems.
+
+  If you need to disable this new behavior, `CatalogBuilder` as used in your
+  `packages/backend/src/plugins/catalog.ts` file now has a
+  `useLegacySingleProcessorValidation()` method to go back to the old behavior.
+
+  ```diff
+   const builder = await CatalogBuilder.create(env);
+  +builder.useLegacySingleProcessorValidation();
+  ```
+
+- 3072ebfdd7: The search table also holds the original entity value now and the facets endpoint fetches the filtered entity data from the search table.
+
+### Patch Changes
+
+- ba13ff663c: Added a new `catalog.rules[].location` configuration that makes it possible to configure catalog rules to only apply to specific locations, either via exact match or a glob pattern.
+- d8593ce0e6: Do not use deprecated `LocationSpec` from the `@backstage/plugin-catalog-node` package
+- c507aee8a2: Ensured typescript type checks in migration files.
+- 2a8e3cc0b5: Optimize `Stitcher` process to be more memory efficient
+- 884d749b14: Refactored to use `coreServices` from `@backstage/backend-plugin-api`.
+- eacc8e2b55: Make it possible for entity providers to supply only entity refs, instead of full entities, in `delta` mutation deletions.
+- b05dcd5530: Move the `zod` dependency to a version that does not collide with other libraries
+- 5b3e2afa45: Fixed deprecated use of `substr` into `substring`.
+- 71147d5c16: Internal code reorganization.
+- 93870e4df1: Track the last time the final entity changed with new timestamp "last updated at" data in final entities database, which gets updated with the time when final entity is updated.
+- 20a5161f04: Adds MySQL support for the catalog-backend
+- 3280711113: Updated dependency `msw` to `^0.49.0`.
+- e982f77fe3: Registered shutdown hook in experimental catalog plugin.
+- b3fac9c107: Ignore attempts at emitting the current entity as a child of itself.
+- Updated dependencies
+  - @backstage/catalog-client@1.2.0
+  - @backstage/backend-common@0.17.0
+  - @backstage/plugin-catalog-node@1.3.0
+  - @backstage/plugin-permission-common@0.7.2
+  - @backstage/plugin-permission-node@0.7.2
+  - @backstage/errors@1.1.4
+  - @backstage/backend-plugin-api@0.2.0
+  - @backstage/integration@1.4.1
+  - @backstage/types@1.0.2
+  - @backstage/plugin-search-common@1.2.0
+  - @backstage/catalog-model@1.1.4
+  - @backstage/config@1.0.5
+  - @backstage/plugin-catalog-common@1.0.9
+  - @backstage/plugin-scaffolder-common@1.2.3
+
 ## 1.6.0-next.3
 
 ### Patch Changes

package/alpha/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@backstage/plugin-catalog-backend",
-  "version": "1.6.0-next.3",
+  "version": "1.6.0",
   "main": "../dist/index.cjs.js",
   "types": "../dist/index.alpha.d.ts"
 }

package/dist/index.cjs.js

@@ -28,6 +28,7 @@ var backendCommon = require('@backstage/backend-common');
 var luxon = require('luxon');
 var promClient = require('prom-client');
 var stableStringify = require('fast-json-stable-stringify');
+var api = require('@opentelemetry/api');
 var express = require('express');
 var Router = require('express-promise-router');
 var yn = require('yn');
@@ -2008,54 +2009,104 @@ class DefaultCatalogProcessingEngine {
   }
 }
 function progressTracker() {
-  const stitchedEntities = createCounterMetric({
+  const promStitchedEntities = createCounterMetric({
     name: "catalog_stitched_entities_count",
-    help: "Amount of entities stitched"
+    help: "Amount of entities stitched. DEPRECATED, use OpenTelemetry metrics instead"
   });
-  const processedEntities = createCounterMetric({
+  const promProcessedEntities = createCounterMetric({
     name: "catalog_processed_entities_count",
-    help: "Amount of entities processed",
+    help: "Amount of entities processed, DEPRECATED, use OpenTelemetry metrics instead",
     labelNames: ["result"]
   });
-  const processingDuration = createSummaryMetric({
+  const promProcessingDuration = createSummaryMetric({
     name: "catalog_processing_duration_seconds",
-    help: "Time spent executing the full processing flow",
+    help: "Time spent executing the full processing flow, DEPRECATED, use OpenTelemetry metrics instead",
     labelNames: ["result"]
   });
-  const processorsDuration = createSummaryMetric({
+  const promProcessorsDuration = createSummaryMetric({
     name: "catalog_processors_duration_seconds",
-    help: "Time spent executing catalog processors",
+    help: "Time spent executing catalog processors, DEPRECATED, use OpenTelemetry metrics instead",
     labelNames: ["result"]
   });
-  const processingQueueDelay = createSummaryMetric({
+  const promProcessingQueueDelay = createSummaryMetric({
     name: "catalog_processing_queue_delay_seconds",
-    help: "The amount of delay between being scheduled for processing, and the start of actually being processed"
+    help: "The amount of delay between being scheduled for processing, and the start of actually being processed, DEPRECATED, use OpenTelemetry metrics instead"
   });
+  const meter = api.metrics.getMeter("default");
+  const stitchedEntities = meter.createCounter(
+    "catalog.stitched.entities.count",
+    {
+      description: "Amount of entities stitched"
+    }
+  );
+  const processedEntities = meter.createCounter(
+    "catalog.processed.entities.count",
+    { description: "Amount of entities processed" }
+  );
+  const processingDuration = meter.createHistogram(
+    "catalog.processing.duration",
+    {
+      description: "Time spent executing the full processing flow",
+      unit: "seconds"
+    }
+  );
+  const processorsDuration = meter.createHistogram(
+    "catalog.processors.duration",
+    {
+      description: "Time spent executing catalog processors",
+      unit: "seconds"
+    }
+  );
+  const processingQueueDelay = meter.createHistogram(
+    "catalog.processing.queue.delay",
+    {
+      description: "The amount of delay between being scheduled for processing, and the start of actually being processed",
+      unit: "seconds"
+    }
+  );
   function processStart(item, logger) {
+    const startTime = process.hrtime();
+    const endOverallTimer = promProcessingDuration.startTimer();
+    const endProcessorsTimer = promProcessorsDuration.startTimer();
     logger.debug(`Processing ${item.entityRef}`);
     if (item.nextUpdateAt) {
-      processingQueueDelay.observe(-item.nextUpdateAt.diffNow().as("seconds"));
+      const seconds = -item.nextUpdateAt.diffNow().as("seconds");
+      promProcessingQueueDelay.observe(seconds);
+      processingQueueDelay.record(seconds);
+    }
+    function endTime() {
+      const delta = process.hrtime(startTime);
+      return delta[0] + delta[1] / 1e9;
     }
-    const endOverallTimer = processingDuration.startTimer();
-    const endProcessorsTimer = processorsDuration.startTimer();
     function markProcessorsCompleted(result) {
       endProcessorsTimer({ result: result.ok ? "ok" : "failed" });
+      processorsDuration.record(endTime(), {
+        result: result.ok ? "ok" : "failed"
+      });
     }
     function markSuccessfulWithNoChanges() {
       endOverallTimer({ result: "unchanged" });
-      processedEntities.inc({ result: "unchanged" }, 1);
+      promProcessedEntities.inc({ result: "unchanged" }, 1);
+      processingDuration.record(endTime(), { result: "unchanged" });
+      processedEntities.add(1, { result: "unchanged" });
     }
     function markSuccessfulWithErrors() {
       endOverallTimer({ result: "errors" });
-      processedEntities.inc({ result: "errors" }, 1);
+      promProcessedEntities.inc({ result: "errors" }, 1);
+      processingDuration.record(endTime(), { result: "errors" });
+      processedEntities.add(1, { result: "errors" });
     }
     function markSuccessfulWithChanges(stitchedCount) {
       endOverallTimer({ result: "changed" });
-      stitchedEntities.inc(stitchedCount);
-      processedEntities.inc({ result: "changed" }, 1);
+      promStitchedEntities.inc(stitchedCount);
+      promProcessedEntities.inc({ result: "changed" }, 1);
+      processingDuration.record(endTime(), { result: "changed" });
+      stitchedEntities.add(stitchedCount);
+      processedEntities.add(1, { result: "changed" });
     }
     function markFailed(error) {
-      processedEntities.inc({ result: "failed" }, 1);
+      promProcessedEntities.inc({ result: "failed" }, 1);
+      processedEntities.add(1, { result: "failed" });
       logger.warn(`Processing of ${item.entityRef} failed`, error);
     }
     return {
@@ -2439,8 +2490,14 @@ class ProcessorOutputCollector {
     this.refreshKeys = new Array();
     this.done = false;
   }
-  get onEmit() {
-    return (i) => this.receive(i);
+  generic() {
+    return (i) => this.receive(this.logger, i);
+  }
+  forProcessor(processor) {
+    const logger = this.logger.child({
+      processor: processor.getProcessorName()
+    });
+    return (i) => this.receive(logger, i);
   }
   results() {
     this.done = true;
@@ -2451,9 +2508,9 @@ class ProcessorOutputCollector {
       deferredEntities: this.deferredEntities
     };
   }
-  receive(i) {
+  receive(logger, i) {
     if (this.done) {
-      this.logger.warn(
+      logger.warn(
         `Item of type "${i.type}" was emitted after processing had completed. Stack trace: ${new Error().stack}`
       );
       return;
@@ -2465,10 +2522,17 @@ class ProcessorOutputCollector {
         entity = validateEntityEnvelope(i.entity);
       } catch (e) {
         errors.assertError(e);
-        this.logger.debug(`Envelope validation failed at ${location}, ${e}`);
+        logger.debug(`Envelope validation failed at ${location}, ${e}`);
         this.errors.push(e);
         return;
       }
+      const entityRef = catalogModel.stringifyEntityRef(entity);
+      if (entityRef === catalogModel.stringifyEntityRef(this.parentEntity)) {
+        logger.warn(
+          `Ignored emitted entity ${entityRef} whose ref was identical to the one being processed. This commonly indicates mistakenly emitting the input entity instead of returning it.`
+        );
+        return;
+      }
       const annotations = entity.metadata.annotations || {};
       if (typeof annotations === "object" && !Array.isArray(annotations)) {
         const originLocation = getEntityOriginLocationRef(this.parentEntity);
@@ -2666,7 +2730,7 @@ class DefaultCatalogProcessingOrchestrator {
           res = await processor.preProcessEntity(
             res,
             context.location,
-            context.collector.onEmit,
+            context.collector.forProcessor(processor),
             context.originLocation,
             context.cache.forProcessor(processor)
           );
@@ -2749,7 +2813,7 @@ class DefaultCatalogProcessingOrchestrator {
     }
     for (const maybeRelativeTarget of targets) {
       if (type === "file" && maybeRelativeTarget.endsWith(path__default["default"].sep)) {
-        context.collector.onEmit(
+        context.collector.generic()(
           pluginCatalogNode.processingResult.inputError(
             context.location,
             `LocationEntityProcessor cannot handle ${type} type location with target ${context.location.target} that ends with a path separator`
@@ -2774,7 +2838,7 @@ class DefaultCatalogProcessingOrchestrator {
                 presence
               },
               presence === "optional",
-              context.collector.onEmit,
+              context.collector.forProcessor(processor),
               this.options.parser,
               context.cache.forProcessor(processor, target)
             );
@@ -2805,7 +2869,7 @@ class DefaultCatalogProcessingOrchestrator {
           res = await processor.postProcessEntity(
             res,
             context.location,
-            context.collector.onEmit,
+            context.collector.forProcessor(processor),
             context.cache.forProcessor(processor)
           );
         } catch (e) {
@@ -3041,8 +3105,9 @@ class Stitcher {
       "final_entities"
     ).update({
       final_entity: JSON.stringify(entity),
-      hash
-    }).where("entity_id", entityId).where("stitch_ticket", ticket).onConflict("entity_id").merge(["final_entity", "hash"]);
+      hash,
+      last_updated_at: this.database.fn.now()
+    }).where("entity_id", entityId).where("stitch_ticket", ticket).onConflict("entity_id").merge(["final_entity", "hash", "last_updated_at"]);
     if (amountOfRowsChanged === 0) {
       this.logger.debug(
         `Entity ${entityRef} is already processed, skipping write.`