@backstage/plugin-catalog-backend 0.21.3 → 0.21.4

package/CHANGELOG.md

@@ -1,5 +1,45 @@
 # @backstage/plugin-catalog-backend
 
+## 0.21.4
+
+### Patch Changes
+
+- 379da9fb1d: The following processors now properly accept an `ScmIntegrationRegistry` (an
+  interface) instead of an `ScmIntegrations` (which is a concrete class).
+
+  - `AzureDevOpsDiscoveryProcessor`
+  - `CodeOwnersProcessor`
+  - `GitLabDiscoveryProcessor`
+  - `GithubDiscoveryProcessor`
+  - `GithubMultiOrgReaderProcessor`
+  - `GithubOrgReaderProcessor`
+
+- 1ed305728b: Bump `node-fetch` to version 2.6.7 and `cross-fetch` to version 3.1.5
+- c77c5c7eb6: Added `backstage.role` to `package.json`
+- 538ca90790: Use updated type names from `@backstage/catalog-client`
+- ca1d6c1788: Support "dependencyOf" relation in Resource entities
+- 244d24ebc4: Import `Location` from the `@backstage/catalog-client` package.
+- e483dd6c72: Update internal `Location` validation.
+- 216725b434: Updated to use new names for `parseLocationRef` and `stringifyLocationRef`
+- e72d371296: Use `TemplateEntityV1beta2` from `@backstage/plugin-scaffolder-common` instead
+  of `@backstage/catalog-model`.
+- 27eccab216: Replaces use of deprecated catalog-model constants.
+- 7aeb491394: Replace use of deprecated `ENTITY_DEFAULT_NAMESPACE` constant with `DEFAULT_NAMESPACE`.
+- b590e9b58d: Optimized entity provider mutations with large numbers of new additions, such as big initial startup commits
+- Updated dependencies
+  - @backstage/plugin-scaffolder-common@0.2.0
+  - @backstage/backend-common@0.10.8
+  - @backstage/catalog-client@0.7.0
+  - @backstage/errors@0.2.1
+  - @backstage/integration@0.7.3
+  - @backstage/plugin-permission-common@0.5.0
+  - @backstage/catalog-model@0.10.0
+  - @backstage/config@0.1.14
+  - @backstage/search-common@0.2.3
+  - @backstage/types@0.1.2
+  - @backstage/plugin-catalog-common@0.1.3
+  - @backstage/plugin-permission-node@0.5.0
+
 ## 0.21.3
 
 ### Patch Changes

package/dist/index.cjs.js

@@ -11,6 +11,7 @@ var parseGitUrl = require('git-url-parse');
 var AWS = require('aws-sdk');
 var limiterFactory = require('p-limit');
 var fetch = require('node-fetch');
+var pluginScaffolderCommon = require('@backstage/plugin-scaffolder-common');
 require('core-js/features/promise');
 var codeowners = require('codeowners-utils');
 var fp = require('lodash/fp');
@@ -26,6 +27,7 @@ var crypto = require('crypto');
 var express = require('express');
 var Router = require('express-promise-router');
 var yn = require('yn');
+var zod = require('zod');
 var uuid = require('uuid');
 var luxon = require('luxon');
 var promClient = require('prom-client');
@@ -206,7 +208,7 @@ class AnnotateLocationEntityProcessor {
         base: location.target
       });
       if (sourceUrl) {
-        sourceLocation = catalogModel.stringifyLocationReference({
+        sourceLocation = catalogModel.stringifyLocationRef({
           type: "url",
           target: sourceUrl
         });
@@ -215,11 +217,11 @@ class AnnotateLocationEntityProcessor {
     return lodash.merge({
       metadata: {
         annotations: lodash.pickBy({
-          [catalogModel.LOCATION_ANNOTATION]: catalogModel.stringifyLocationReference(location),
-          [catalogModel.ORIGIN_LOCATION_ANNOTATION]: catalogModel.stringifyLocationReference(originLocation),
-          [catalogModel.VIEW_URL_ANNOTATION]: viewUrl,
-          [catalogModel.EDIT_URL_ANNOTATION]: editUrl,
-          [catalogModel.SOURCE_LOCATION_ANNOTATION]: sourceLocation
+          [catalogModel.ANNOTATION_LOCATION]: catalogModel.stringifyLocationRef(location),
+          [catalogModel.ANNOTATION_ORIGIN_LOCATION]: catalogModel.stringifyLocationRef(originLocation),
+          [catalogModel.ANNOTATION_VIEW_URL]: viewUrl,
+          [catalogModel.ANNOTATION_EDIT_URL]: editUrl,
+          [catalogModel.ANNOTATION_SOURCE_LOCATION]: sourceLocation
         }, lodash.identity)
       }
     }, entity);
@@ -650,7 +652,7 @@ class BuiltinKindsEntityProcessor {
       catalogModel.resourceEntityV1alpha1Validator,
       catalogModel.groupEntityV1alpha1Validator,
       catalogModel.locationEntityV1alpha1Validator,
-      catalogModel.templateEntityV1beta2Validator,
+      pluginScaffolderCommon.templateEntityV1beta2Validator,
       catalogModel.userEntityV1alpha1Validator,
       catalogModel.systemEntityV1alpha1Validator,
       catalogModel.domainEntityV1alpha1Validator
@@ -718,6 +720,7 @@ class BuiltinKindsEntityProcessor {
       const resource = entity;
       doEmit(resource.spec.owner, { defaultKind: "Group", defaultNamespace: selfRef.namespace }, catalogModel.RELATION_OWNED_BY, catalogModel.RELATION_OWNER_OF);
       doEmit(resource.spec.dependsOn, { defaultNamespace: selfRef.namespace }, catalogModel.RELATION_DEPENDS_ON, catalogModel.RELATION_DEPENDENCY_OF);
+      doEmit(resource.spec.dependencyOf, { defaultNamespace: selfRef.namespace }, catalogModel.RELATION_DEPENDENCY_OF, catalogModel.RELATION_DEPENDS_ON);
       doEmit(resource.spec.system, { defaultKind: "System", defaultNamespace: selfRef.namespace }, catalogModel.RELATION_PART_OF, catalogModel.RELATION_HAS_PART);
     }
     if (entity.kind === "User") {
@@ -1814,14 +1817,14 @@ function* parseEntityYaml(data, location) {
   try {
     documents = yaml__default["default"].parseAllDocuments(data.toString("utf8")).filter((d) => d);
   } catch (e) {
-    const loc = catalogModel.stringifyLocationReference(location);
+    const loc = catalogModel.stringifyLocationRef(location);
     const message = `Failed to parse YAML at ${loc}, ${e}`;
     yield generalError(location, message);
     return;
   }
   for (const document of documents) {
     if ((_a = document.errors) == null ? void 0 : _a.length) {
-      const loc = catalogModel.stringifyLocationReference(location);
+      const loc = catalogModel.stringifyLocationRef(location);
       const message = `YAML error at ${loc}, ${document.errors[0]}`;
       yield generalError(location, message);
     } else {
@@ -1920,8 +1923,8 @@ function withLocations(baseUrl, org, entity) {
   return lodash.merge({
     metadata: {
       annotations: {
-        [catalogModel.LOCATION_ANNOTATION]: location,
-        [catalogModel.ORIGIN_LOCATION_ANNOTATION]: location
+        [catalogModel.ANNOTATION_LOCATION]: location,
+        [catalogModel.ANNOTATION_ORIGIN_LOCATION]: location
       }
     }
   }, entity);
@@ -2037,18 +2040,18 @@ function locationSpecToLocationEntity(location, parentEntity) {
   let ownLocation;
   let originLocation;
   if (parentEntity) {
-    const maybeOwnLocation = (_a = parentEntity.metadata.annotations) == null ? void 0 : _a[catalogModel.LOCATION_ANNOTATION];
+    const maybeOwnLocation = (_a = parentEntity.metadata.annotations) == null ? void 0 : _a[catalogModel.ANNOTATION_LOCATION];
     if (!maybeOwnLocation) {
-      throw new Error(`Parent entity '${catalogModel.stringifyEntityRef(parentEntity)}' of location '${catalogModel.stringifyLocationReference(location)}' does not have a location annotation`);
+      throw new Error(`Parent entity '${catalogModel.stringifyEntityRef(parentEntity)}' of location '${catalogModel.stringifyLocationRef(location)}' does not have a location annotation`);
     }
     ownLocation = maybeOwnLocation;
-    const maybeOriginLocation = (_b = parentEntity.metadata.annotations) == null ? void 0 : _b[catalogModel.ORIGIN_LOCATION_ANNOTATION];
+    const maybeOriginLocation = (_b = parentEntity.metadata.annotations) == null ? void 0 : _b[catalogModel.ANNOTATION_ORIGIN_LOCATION];
     if (!maybeOriginLocation) {
-      throw new Error(`Parent entity '${catalogModel.stringifyEntityRef(parentEntity)}' of location '${catalogModel.stringifyLocationReference(location)}' does not have an origin location annotation`);
+      throw new Error(`Parent entity '${catalogModel.stringifyEntityRef(parentEntity)}' of location '${catalogModel.stringifyLocationRef(location)}' does not have an origin location annotation`);
     }
     originLocation = maybeOriginLocation;
   } else {
-    ownLocation = catalogModel.stringifyLocationReference(location);
+    ownLocation = catalogModel.stringifyLocationRef(location);
     originLocation = ownLocation;
   }
   const result = {
@@ -2057,8 +2060,8 @@ function locationSpecToLocationEntity(location, parentEntity) {
     metadata: {
       name: locationSpecToMetadataName(location),
       annotations: {
-        [catalogModel.LOCATION_ANNOTATION]: ownLocation,
-        [catalogModel.ORIGIN_LOCATION_ANNOTATION]: originLocation
+        [catalogModel.ANNOTATION_LOCATION]: ownLocation,
+        [catalogModel.ANNOTATION_ORIGIN_LOCATION]: originLocation
       }
     },
     spec: {
@@ -2075,7 +2078,7 @@ function isLocationEntity(entity) {
 }
 function getEntityLocationRef(entity) {
   var _a;
-  const ref = (_a = entity.metadata.annotations) == null ? void 0 : _a[catalogModel.LOCATION_ANNOTATION];
+  const ref = (_a = entity.metadata.annotations) == null ? void 0 : _a[catalogModel.ANNOTATION_LOCATION];
   if (!ref) {
     const entityRef = catalogModel.stringifyEntityRef(entity);
     throw new errors.InputError(`Entity '${entityRef}' does not have a location`);
@@ -2084,7 +2087,7 @@ function getEntityLocationRef(entity) {
 }
 function getEntityOriginLocationRef(entity) {
   var _a;
-  const ref = (_a = entity.metadata.annotations) == null ? void 0 : _a[catalogModel.ORIGIN_LOCATION_ANNOTATION];
+  const ref = (_a = entity.metadata.annotations) == null ? void 0 : _a[catalogModel.ANNOTATION_ORIGIN_LOCATION];
   if (!ref) {
     const entityRef = catalogModel.stringifyEntityRef(entity);
     throw new errors.InputError(`Entity '${entityRef}' does not have an origin location`);
@@ -2150,7 +2153,7 @@ class ProcessorOutputCollector {
         this.errors.push(e);
         return;
       }
-      const location = catalogModel.stringifyLocationReference(i.location);
+      const location = catalogModel.stringifyLocationRef(i.location);
       const annotations = entity.metadata.annotations || {};
       if (typeof annotations === "object" && !Array.isArray(annotations)) {
         const originLocation = getEntityOriginLocationRef(this.parentEntity);
@@ -2160,8 +2163,8 @@ class ProcessorOutputCollector {
             ...entity.metadata,
             annotations: {
               ...annotations,
-              [catalogModel.ORIGIN_LOCATION_ANNOTATION]: originLocation,
-              [catalogModel.LOCATION_ANNOTATION]: location
+              [catalogModel.ANNOTATION_ORIGIN_LOCATION]: originLocation,
+              [catalogModel.ANNOTATION_LOCATION]: location
             }
           }
         };
@@ -2282,8 +2285,8 @@ class DefaultCatalogProcessingOrchestrator {
       }
       const context = {
         entityRef: catalogModel.stringifyEntityRef(entity),
-        location: catalogModel.parseLocationReference(getEntityLocationRef(entity)),
-        originLocation: catalogModel.parseLocationReference(getEntityOriginLocationRef(entity)),
+        location: catalogModel.parseLocationRef(getEntityLocationRef(entity)),
+        originLocation: catalogModel.parseLocationRef(getEntityOriginLocationRef(entity)),
         cache,
         collector
       };
@@ -2297,7 +2300,7 @@ class DefaultCatalogProcessingOrchestrator {
       const collectorResults = context.collector.results();
       for (const deferredEntity of collectorResults.deferredEntities) {
         if (!this.options.rulesEnforcer.isAllowed(deferredEntity.entity, context.originLocation)) {
-          throw new errors.NotAllowedError(`Entity ${catalogModel.stringifyEntityRef(deferredEntity.entity)} at ${catalogModel.stringifyLocationReference(context.location)}, originated at ${catalogModel.stringifyLocationReference(context.originLocation)}, is not of an allowed kind for that location`);
+          throw new errors.NotAllowedError(`Entity ${catalogModel.stringifyEntityRef(deferredEntity.entity)} at ${catalogModel.stringifyLocationRef(context.location)}, originated at ${catalogModel.stringifyLocationRef(context.originLocation)}, is not of an allowed kind for that location`);
         }
       }
       return {
@@ -2565,14 +2568,18 @@ async function requireRequestBody(req) {
   }
   return body;
 }
+const locationSpec = zod.z.object({
+  type: zod.z.string(),
+  target: zod.z.string(),
+  presence: zod.z.literal("required").or(zod.z.literal("optional")).optional()
+}).strict();
 async function validateRequestBody(req, schema) {
   const body = await requireRequestBody(req);
   try {
-    await schema.validate(body, { strict: true });
+    return await schema.parse(body);
   } catch (e) {
     throw new errors.InputError(`Malformed request: ${e}`);
   }
-  return body;
 }
 function disallowReadonlyMode(readonly) {
   if (readonly) {
@@ -2661,12 +2668,12 @@ async function createRouter(options) {
   }
   if (locationService) {
     router.post("/locations", async (req, res) => {
-      const input = await validateRequestBody(req, catalogModel.locationSpecSchema);
+      const location = await validateRequestBody(req, locationSpec);
       const dryRun = yn__default["default"](req.query.dryRun, { default: false });
       if (!dryRun) {
         disallowReadonlyMode(readonlyEnabled);
       }
-      const output = await locationService.createLocation(input, dryRun, {
+      const output = await locationService.createLocation(location, dryRun, {
         authorizationToken: getBearerToken(req.header("authorization"))
       });
       res.status(201).json(output);
@@ -2692,8 +2699,9 @@ async function createRouter(options) {
   }
   if (locationAnalyzer) {
     router.post("/analyze-location", async (req, res) => {
-      const input = await validateRequestBody(req, catalogModel.analyzeLocationSchema);
-      const output = await locationAnalyzer.analyzeLocation(input);
+      const body = await validateRequestBody(req, zod.z.object({ location: locationSpec }));
+      const schema = zod.z.object({ location: locationSpec });
+      const output = await locationAnalyzer.analyzeLocation(schema.parse(body));
       res.status(200).json(output);
     });
   }
@@ -2939,7 +2947,7 @@ class DefaultProcessingDatabase {
   }
   async replaceUnprocessedEntities(txOpaque, options) {
     const tx = txOpaque;
-    const { toUpsert, toRemove } = await this.createDelta(tx, options);
+    const { toAdd, toUpsert, toRemove } = await this.createDelta(tx, options);
     if (toRemove.length) {
       const removedCount = await tx("refresh_state").whereIn("entity_ref", function orphanedEntityRefs(orphans) {
         return orphans.withRecursive("descendants", function descendants(outer) {
@@ -2974,6 +2982,33 @@ class DefaultProcessingDatabase {
       await tx("refresh_state_references").where("source_key", "=", options.sourceKey).whereIn("target_entity_ref", toRemove).delete();
       this.options.logger.debug(`removed, ${removedCount} entities: ${JSON.stringify(toRemove)}`);
     }
+    if (toAdd.length) {
+      for (const chunk of lodash__default["default"].chunk(toAdd, 50)) {
+        try {
+          await tx.batchInsert("refresh_state", chunk.map((item) => ({
+            entity_id: uuid.v4(),
+            entity_ref: catalogModel.stringifyEntityRef(item.deferred.entity),
+            unprocessed_entity: JSON.stringify(item.deferred.entity),
+            unprocessed_hash: item.hash,
+            errors: "",
+            location_key: item.deferred.locationKey,
+            next_update_at: tx.fn.now(),
+            last_discovery_at: tx.fn.now()
+          })), BATCH_SIZE$1);
+          await tx.batchInsert("refresh_state_references", chunk.map((item) => ({
+            source_key: options.sourceKey,
+            target_entity_ref: catalogModel.stringifyEntityRef(item.deferred.entity)
+          })), BATCH_SIZE$1);
+        } catch (error) {
+          if (!backendCommon.isDatabaseConflictError(error)) {
+            throw error;
+          } else {
+            this.options.logger.debug(`Fast insert path failed, falling back to slow path, ${error}`);
+            toUpsert.push(...chunk);
+          }
+        }
+      }
+    }
     if (toUpsert.length) {
       for (const {
         deferred: { entity, locationKey },
@@ -3139,6 +3174,7 @@ class DefaultProcessingDatabase {
   async createDelta(tx, options) {
     if (options.type === "delta") {
       return {
+        toAdd: [],
         toUpsert: options.added.map((e) => ({
           deferred: e,
           hash: generateStableHash$1(e.entity)
@@ -3166,21 +3202,22 @@ class DefaultProcessingDatabase {
       }
     ]));
     const newRefsSet = new Set(items.map((item) => item.ref));
+    const toAdd = new Array();
     const toUpsert = new Array();
     const toRemove = oldRefs.map((row) => row.target_entity_ref).filter((ref) => !newRefsSet.has(ref));
     for (const item of items) {
       const oldRef = oldRefsSet.get(item.ref);
       const upsertItem = { deferred: item.deferred, hash: item.hash };
       if (!oldRef) {
-        toUpsert.push(upsertItem);
+        toAdd.push(upsertItem);
       } else if (oldRef.locationKey !== item.deferred.locationKey) {
         toRemove.push(item.ref);
-        toUpsert.push(upsertItem);
+        toAdd.push(upsertItem);
       } else if (oldRef.oldEntityHash !== item.hash) {
         toUpsert.push(upsertItem);
       }
     }
-    return { toUpsert, toRemove };
+    return { toAdd, toUpsert, toRemove };
   }
   async addUnprocessedEntities(txOpaque, options) {
     const tx = txOpaque;
@@ -3516,8 +3553,8 @@ class DefaultLocationService {
         }),
         namespace: "default",
         annotations: {
-          [catalogModel.LOCATION_ANNOTATION]: `${spec.type}:${spec.target}`,
-          [catalogModel.ORIGIN_LOCATION_ANNOTATION]: `${spec.type}:${spec.target}`
+          [catalogModel.ANNOTATION_LOCATION]: `${spec.type}:${spec.target}`,
+          [catalogModel.ANNOTATION_ORIGIN_LOCATION]: `${spec.type}:${spec.target}`
         }
       },
       spec: {
@@ -3856,7 +3893,7 @@ function buildEntitySearch(entityId, entity) {
   raw.push({ key: "metadata.namespace", value: entity.metadata.namespace });
   raw.push({ key: "metadata.uid", value: entity.metadata.uid });
   if (!entity.metadata.namespace) {
-    raw.push({ key: "metadata.namespace", value: catalogModel.ENTITY_DEFAULT_NAMESPACE });
+    raw.push({ key: "metadata.namespace", value: catalogModel.DEFAULT_NAMESPACE });
   }
   for (const relation of (_a = entity.relations) != null ? _a : []) {
     raw.push({