@backstage/plugin-catalog-backend 1.7.0-next.1 → 1.7.0

package/CHANGELOG.md

@@ -1,5 +1,56 @@
 # @backstage/plugin-catalog-backend
 
+## 1.7.0
+
+### Minor Changes
+
+- f75bf76330: Implemented server side ordering in the entities endpoint
+
+### Patch Changes
+
+- e23f13a573: Enable the `by-refs` endpoint to receive `fields` through the POST body as well as through query parameters.
+- f23eef3aa2: Updated dependency `better-sqlite3` to `^8.0.0`.
+- d136793ff0: Fixed an issue where internal references in the catalog would stick around for longer than expected, causing entities to not be deleted or orphaned as expected.
+- 8e06f3cf00: Switched imports of `loggerToWinstonLogger` to `@backstage/backend-common`.
+- Updated dependencies
+  - @backstage/backend-plugin-api@0.3.0
+  - @backstage/backend-common@0.18.0
+  - @backstage/catalog-model@1.1.5
+  - @backstage/plugin-scaffolder-common@1.2.4
+  - @backstage/catalog-client@1.3.0
+  - @backstage/plugin-catalog-node@1.3.1
+  - @backstage/config@1.0.6
+  - @backstage/errors@1.1.4
+  - @backstage/integration@1.4.2
+  - @backstage/types@1.0.2
+  - @backstage/plugin-catalog-common@1.0.10
+  - @backstage/plugin-permission-common@0.7.3
+  - @backstage/plugin-permission-node@0.7.3
+  - @backstage/plugin-search-common@1.2.1
+
+## 1.7.0-next.2
+
+### Patch Changes
+
+- e23f13a573: Enable the `by-refs` endpoint to receive `fields` through the POST body as well as through query parameters.
+- f23eef3aa2: Updated dependency `better-sqlite3` to `^8.0.0`.
+- 8e06f3cf00: Switched imports of `loggerToWinstonLogger` to `@backstage/backend-common`.
+- Updated dependencies
+  - @backstage/backend-plugin-api@0.3.0-next.1
+  - @backstage/backend-common@0.18.0-next.1
+  - @backstage/catalog-client@1.3.0-next.2
+  - @backstage/plugin-catalog-node@1.3.1-next.2
+  - @backstage/plugin-permission-node@0.7.3-next.1
+  - @backstage/catalog-model@1.1.5-next.1
+  - @backstage/config@1.0.6-next.0
+  - @backstage/errors@1.1.4
+  - @backstage/integration@1.4.2-next.0
+  - @backstage/types@1.0.2
+  - @backstage/plugin-catalog-common@1.0.10-next.1
+  - @backstage/plugin-permission-common@0.7.3-next.0
+  - @backstage/plugin-scaffolder-common@1.2.4-next.1
+  - @backstage/plugin-search-common@1.2.1-next.0
+
 ## 1.7.0-next.1
 
 ### Patch Changes

package/alpha/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@backstage/plugin-catalog-backend",
-  "version": "1.7.0-next.1",
+  "version": "1.7.0",
   "main": "../dist/index.cjs.js",
   "types": "../dist/index.alpha.d.ts"
 }

package/dist/index.alpha.d.ts

@@ -390,7 +390,7 @@ export declare type CatalogPermissionRule<TParams extends PermissionRuleParams =
  * Catalog plugin
  * @alpha
  */
-export declare const catalogPlugin: (options?: undefined) => BackendFeature;
+export declare const catalogPlugin: () => BackendFeature;
 
 /**
  * Represents the engine that drives the processing loops. Some backend

package/dist/index.cjs.js

@@ -97,8 +97,11 @@ function normalizeCodeOwner(owner) {
 
 const CODEOWNERS = "CODEOWNERS";
 const scmCodeOwnersPaths = {
+  // https://mibexsoftware.atlassian.net/wiki/spaces/CODEOWNERS/pages/222822413/Usage
   bitbucket: [CODEOWNERS, `.bitbucket/${CODEOWNERS}`],
+  // https://docs.gitlab.com/ee/user/project/code_owners.html#how-to-set-up-code-owners
   gitlab: [CODEOWNERS, `.gitlab/${CODEOWNERS}`, `docs/${CODEOWNERS}`],
+  // https://docs.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners#codeowners-file-location
   github: [CODEOWNERS, `.github/${CODEOWNERS}`, `docs/${CODEOWNERS}`]
 };
 
@@ -1549,6 +1552,9 @@ async function updateUnprocessedEntity(options) {
     unprocessed_hash: hash,
     location_key: locationKey,
     last_discovery_at: tx.fn.now(),
+    // We only get to this point if a processed entity actually had any changes, or
+    // if an entity provider requested this mutation, meaning that we can safely
+    // bump the deferred entities to the front of the queue for immediate processing.
     next_update_at: tx.fn.now()
   }).where("entity_ref", entityRef).andWhere((inner) => {
     if (!locationKey) {
@@ -1707,6 +1713,7 @@ class DefaultProcessingDatabase {
           result = await fn(tx);
         },
         {
+          // If we explicitly trigger a rollback, don't fail.
           doNotRejectOnRollback: true
         }
       );
@@ -1722,6 +1729,10 @@ class DefaultProcessingDatabase {
       (r) => `${r.source_entity_ref}:${r.target_entity_ref}:${r.type}`
     );
   }
+  /**
+   * Add a set of deferred entities for processing.
+   * The entities will be added at the front of the processing queue.
+   */
   async addUnprocessedEntities(txOpaque, options) {
     const tx = txOpaque;
     const stateReferences = new Array();
@@ -2159,6 +2170,7 @@ class DefaultLocationService {
       const processed = await this.orchestrator.process({
         entity: currentEntity.entity,
         state: {}
+        // we process without the existing cache
       });
       if (processed.ok) {
         if (entities.some(
@@ -2749,6 +2761,8 @@ class DefaultCatalogProcessingOrchestrator {
       };
     }
   }
+  // Pre-process phase, used to populate entities with data that is required
+  // during the main processing step
   async runPreProcessStep(entity, context) {
     let res = entity;
     for (const processor of this.options.processors) {
@@ -2771,6 +2785,9 @@ class DefaultCatalogProcessingOrchestrator {
     }
     return res;
   }
+  /**
+   * Enforce entity policies making sure that entities conform to a general schema
+   */
   async runPolicyStep(entity) {
     let policyEnforcedEntity;
     try {
@@ -2790,6 +2807,9 @@ class DefaultCatalogProcessingOrchestrator {
     }
     return policyEnforcedEntity;
   }
+  /**
+   * Validate the given entity
+   */
   async runValidateStep(entity, context) {
     if (catalogModel.stringifyEntityRef(entity) !== context.entityRef) {
       throw new errors.ConflictError(
@@ -2829,6 +2849,9 @@ class DefaultCatalogProcessingOrchestrator {
       );
     }
   }
+  /**
+   * Backwards compatible processing of location entities
+   */
   async runSpecialLocationStep(entity, context) {
     const { type = context.location.type, presence = "required" } = entity.spec;
     const targets = new Array();
@@ -2888,6 +2911,9 @@ class DefaultCatalogProcessingOrchestrator {
       }
     }
   }
+  /**
+   * Main processing step of the entity
+   */
   async runPostProcessStep(entity, context) {
     let res = entity;
     for (const processor of this.options.processors) {
@@ -3108,7 +3134,10 @@ class Stitcher {
         }));
       }
     }
-    entity.relations = relationsResult.filter((row) => row.relationType).map((row) => ({
+    entity.relations = relationsResult.filter(
+      (row) => row.relationType
+      /* exclude null row, if relevant */
+    ).map((row) => ({
       type: row.relationType,
       targetRef: row.relationTarget
     }));
@@ -3147,7 +3176,8 @@ class Stitcher {
 }
 
 const schema = zod.z.object({
-  entityRefs: zod.z.array(zod.z.string())
+  entityRefs: zod.z.array(zod.z.string()),
+  fields: zod.z.array(zod.z.string()).optional()
 });
 function entitiesBatchRequest(req) {
   try {
@@ -3274,17 +3304,22 @@ function getPathArrayAndValue(input, field) {
     [[], input]
   );
 }
-function parseEntityTransformParams(params) {
-  const fieldsStrings = parseStringsParam(params.fields, "fields");
-  if (!fieldsStrings) {
-    return void 0;
-  }
-  const fields = fieldsStrings.map((s) => s.split(",")).flat().map((s) => s.trim()).filter(Boolean);
+function parseEntityTransformParams(params, extra) {
+  var _a;
+  const queryFields = parseStringsParam(params.fields, "fields");
+  const fields = Array.from(
+    new Set(
+      [...extra != null ? extra : [], ...(_a = queryFields == null ? void 0 : queryFields.map((s) => s.split(","))) != null ? _a : []].flat().map((s) => s.trim()).filter(Boolean)
+    )
+  );
   if (!fields.length) {
     return void 0;
   }
-  if (fields.some((f) => f.includes("["))) {
-    throw new errors.InputError("invalid fields, array type fields are not supported");
+  const arrayTypeField = fields.find((f) => f.includes("["));
+  if (arrayTypeField) {
+    throw new errors.InputError(
+      `Invalid field "${arrayTypeField}", array type fields are not supported`
+    );
   }
   return (input) => {
     const output = {};
@@ -3454,7 +3489,7 @@ async function createRouter(options) {
       const token = getBearerToken(req.header("authorization"));
       const response = await entitiesCatalog.entitiesBatch({
         entityRefs: request.entityRefs,
-        fields: parseEntityTransformParams(req.query),
+        fields: parseEntityTransformParams(req.query, request.fields),
         authorizationToken: token
       });
       res.status(200).json(response);
@@ -3618,6 +3653,44 @@ const _DefaultCatalogRulesEnforcer = class {
   constructor(rules) {
     this.rules = rules;
   }
+  /**
+   * Loads catalog rules from config.
+   *
+   * This reads `catalog.rules` and defaults to the default rules if no value is present.
+   * The value of the config should be a list of config objects, each with a single `allow`
+   * field which in turn is a list of entity kinds to allow.
+   *
+   * If there is no matching rule to allow an ingested entity, it will be rejected by the catalog.
+   *
+   * It also reads in rules from `catalog.locations`, where each location can have a list
+   * of rules for that specific location, specified in a `rules` field.
+   *
+   * For example:
+   *
+   * ```yaml
+   * catalog:
+   *   rules:
+   *   - allow: [Component, API]
+   *   - allow: [Template]
+   *     locations:
+   *       - type: url
+   *         pattern: https://github.com/org/*\/blob/master/template.yaml
+   *   - allow: [Location]
+   *     locations:
+   *       - type: url
+   *         pattern: https://github.com/org/repo/blob/master/location.yaml
+   *
+   *   locations:
+   *   - type: url
+   *     target: https://github.com/org/repo/blob/master/users.yaml
+   *     rules:
+   *       - allow: [User, Group]
+   *   - type: url
+   *     target: https://github.com/org/repo/blob/master/systems.yaml
+   *     rules:
+   *       - allow: [System]
+   * ```
+   */
   static fromConfig(config) {
     const rules = new Array();
     if (config.has("catalog.rules")) {
@@ -3660,6 +3733,10 @@ const _DefaultCatalogRulesEnforcer = class {
     }
     return new _DefaultCatalogRulesEnforcer(rules);
   }
+  /**
+   * Checks whether a specific entity/location combination is allowed
+   * according to the configured rules.
+   */
   isAllowed(entity, location) {
     for (const rule of this.rules) {
       if (!this.matchLocation(location, rule.locations)) {
@@ -3704,6 +3781,11 @@ const _DefaultCatalogRulesEnforcer = class {
   }
 };
 let DefaultCatalogRulesEnforcer = _DefaultCatalogRulesEnforcer;
+/**
+ * Default rules used by the catalog.
+ *
+ * Denies any location from specifying user or group entities.
+ */
 DefaultCatalogRulesEnforcer.defaultRules = [
   {
     allow: ["Component", "API", "Location"].map((kind) => ({ kind }))
@@ -4052,6 +4134,7 @@ class DefaultProviderDatabase {
           result = await fn(tx);
         },
         {
+          // If we explicitly trigger a rollback, don't fail.
           doNotRejectOnRollback: true
         }
       );
@@ -4233,6 +4316,7 @@ class DefaultCatalogDatabase {
           result = await fn(tx);
         },
         {
+          // If we explicitly trigger a rollback, don't fail.
           doNotRejectOnRollback: true
         }
       );
@@ -4302,13 +4386,33 @@ class CatalogBuilder {
     this.permissionRules = Object.values(permissionRules);
     this.allowedLocationType = ["url"];
   }
+  /**
+   * Creates a catalog builder.
+   */
   static create(env) {
     return new CatalogBuilder(env);
   }
+  /**
+   * Adds policies that are used to validate entities between the pre-
+   * processing and post-processing stages. All such policies must pass for the
+   * entity to be considered valid.
+   *
+   * If what you want to do is to replace the rules for what format is allowed
+   * in various core entity fields (such as metadata.name), you may want to use
+   * {@link CatalogBuilder#setFieldFormatValidators} instead.
+   *
+   * @param policies - One or more policies
+   */
   addEntityPolicy(...policies) {
     this.entityPolicies.push(...policies.flat());
     return this;
   }
+  /**
+   * Processing interval determines how often entities should be processed.
+   * Seconds provided will be multiplied by 1.5
+   * The default processing interval is 100-150 seconds.
+   * setting this too low will potentially deplete request quotas to upstream services.
+   */
   setProcessingIntervalSeconds(seconds) {
     this.processingInterval = createRandomProcessingInterval({
       minSeconds: seconds,
@@ -4316,40 +4420,109 @@ class CatalogBuilder {
     });
     return this;
   }
+  /**
+   * Overwrites the default processing interval function used to spread
+   * entity updates in the catalog.
+   */
   setProcessingInterval(processingInterval) {
     this.processingInterval = processingInterval;
     return this;
   }
+  /**
+   * Overwrites the default location analyzer.
+   */
   setLocationAnalyzer(locationAnalyzer) {
     this.locationAnalyzer = locationAnalyzer;
     return this;
   }
+  /**
+   * Sets what policies to use for validation of entities between the pre-
+   * processing and post-processing stages. All such policies must pass for the
+   * entity to be considered valid.
+   *
+   * If what you want to do is to replace the rules for what format is allowed
+   * in various core entity fields (such as metadata.name), you may want to use
+   * {@link CatalogBuilder#setFieldFormatValidators} instead.
+   *
+   * This function replaces the default set of policies; use with care.
+   *
+   * @param policies - One or more policies
+   */
   replaceEntityPolicies(policies) {
     this.entityPolicies = [...policies];
     this.entityPoliciesReplace = true;
     return this;
   }
+  /**
+   * Adds, or overwrites, a handler for placeholders (e.g. $file) in entity
+   * definition files.
+   *
+   * @param key - The key that identifies the placeholder, e.g. "file"
+   * @param resolver - The resolver that gets values for this placeholder
+   */
   setPlaceholderResolver(key, resolver) {
     this.placeholderResolvers[key] = resolver;
     return this;
   }
+  /**
+   * Sets the validator function to use for one or more special fields of an
+   * entity. This is useful if the default rules for formatting of fields are
+   * not sufficient.
+   *
+   * This function has no effect if used together with
+   * {@link CatalogBuilder#replaceEntityPolicies}.
+   *
+   * @param validators - The (subset of) validators to set
+   */
   setFieldFormatValidators(validators) {
     lodash__default["default"].merge(this.fieldFormatValidators, validators);
     return this;
   }
+  /**
+   * Adds or replaces entity providers. These are responsible for bootstrapping
+   * the list of entities out of original data sources. For example, there is
+   * one entity source for the config locations, and one for the database
+   * stored locations. If you ingest entities out of a third party system, you
+   * may want to implement that in terms of an entity provider as well.
+   *
+   * @param providers - One or more entity providers
+   */
   addEntityProvider(...providers) {
     this.entityProviders.push(...providers.flat());
     return this;
   }
+  /**
+   * Adds entity processors. These are responsible for reading, parsing, and
+   * processing entities before they are persisted in the catalog.
+   *
+   * @param processors - One or more processors
+   */
   addProcessor(...processors) {
     this.processors.push(...processors.flat());
     return this;
   }
+  /**
+   * Sets what entity processors to use. These are responsible for reading,
+   * parsing, and processing entities before they are persisted in the catalog.
+   *
+   * This function replaces the default set of processors, consider using with
+   * {@link CatalogBuilder#getDefaultProcessors}; use with care.
+   *
+   * @param processors - One or more processors
+   */
   replaceProcessors(processors) {
     this.processors = [...processors];
     this.processorsReplace = true;
     return this;
   }
+  /**
+   * Returns the default list of entity processors. These are responsible for reading,
+   * parsing, and processing entities before they are persisted in the catalog. Changing
+   * the order of processing can give more control to custom processors.
+   *
+   * Consider using with {@link CatalogBuilder#replaceProcessors}
+   *
+   */
   getDefaultProcessors() {
     const { config, logger, reader } = this.env;
     const integrations = integration.ScmIntegrations.fromConfig(config);
@@ -4360,26 +4533,63 @@ class CatalogBuilder {
       new AnnotateLocationEntityProcessor({ integrations })
     ];
   }
+  /**
+   * Adds Location Analyzers. These are responsible for analyzing
+   * repositories when onboarding them into the catalog, by finding
+   * catalog-info.yaml files and other artifacts that can help automatically
+   * register or create catalog data on the user's behalf.
+   *
+   * @param locationAnalyzers - One or more location analyzers
+   */
   addLocationAnalyzers(...analyzers) {
     this.locationAnalyzers.push(...analyzers.flat());
     return this;
   }
+  /**
+   * Sets up the catalog to use a custom parser for entity data.
+   *
+   * This is the function that gets called immediately after some raw entity
+   * specification data has been read from a remote source, and needs to be
+   * parsed and emitted as structured data.
+   *
+   * @param parser - The custom parser
+   */
   setEntityDataParser(parser) {
     this.parser = parser;
     return this;
   }
+  /**
+   * Adds additional permission rules. Permission rules are used to evaluate
+   * catalog resources against queries. See
+   * {@link @backstage/plugin-permission-node#PermissionRule}.
+   *
+   * @param permissionRules - Additional permission rules
+   * @alpha
+   */
   addPermissionRules(...permissionRules) {
     this.permissionRules.push(...permissionRules.flat());
     return this;
   }
+  /**
+   * Sets up the allowed location types from being registered via the location service.
+   *
+   * @param allowedLocationTypes - the allowed location types
+   */
   setAllowedLocationTypes(allowedLocationTypes) {
     this.allowedLocationType = allowedLocationTypes;
     return this;
   }
+  /**
+   * Enables the legacy behaviour of canceling validation early whenever only a
+   * single processor declares an entity kind to be valid.
+   */
   useLegacySingleProcessorValidation() {
     this.legacySingleProcessorValidation = true;
     return this;
   }
+  /**
+   * Wires up and returns all of the component parts of the catalog
+   */
   async build() {
     var _a, _b;
     const { config, database, logger, permissions } = this.env;
@@ -4542,6 +4752,8 @@ class CatalogBuilder {
     this.checkMissingExternalProcessors(processors);
     return processors;
   }
+  // TODO(Rugvip): These old processors are removed, for a while we'll be throwing
+  //               errors here to make sure people know where to move the config
   checkDeprecatedReaderProcessors() {
     const pc = this.env.config.getOptionalConfig("catalog.processors");
     if (pc == null ? void 0 : pc.has("github")) {
@@ -4565,6 +4777,7 @@ class CatalogBuilder {
       );
     }
   }
+  // TODO(freben): This can be removed no sooner than June 2022, after adopters have had some time to adapt to the new package structure
   checkMissingExternalProcessors(processors) {
     var _a, _b;
     const skipCheckVarName = "BACKSTAGE_CATALOG_SKIP_MISSING_PROCESSORS_CHECK";
@@ -4699,7 +4912,7 @@ const catalogPlugin = backendPluginApi.createBackendPlugin({
         httpRouter,
         lifecycle
       }) {
-        const winstonLogger = backendPluginApi.loggerToWinstonLogger(logger);
+        const winstonLogger = backendCommon.loggerToWinstonLogger(logger);
         const builder = await CatalogBuilder.create({
           config,
           reader,