@backstage/plugin-catalog-backend 0.17.3 → 0.17.4

package/CHANGELOG.md

@@ -1,5 +1,21 @@
 # @backstage/plugin-catalog-backend
 
+## 0.17.4
+
+### Patch Changes
+
+- 5d2a7303bd: This fixes a bug where locations couldn't be added unless the processing engine is started.
+  It's now possible to run the catalog backend without starting the processing engine and still allowing locations registrations.
+
+  This is done by refactor the `EntityProvider.connect` to happen outside the engine.
+
+- 06934f2f52: Adjust entity query construction to ensure sub-queries are always isolated from one another.
+- b90fc74d70: adds getDefaultProcessor method to CatalogBuilder
+- Updated dependencies
+  - @backstage/catalog-client@0.5.2
+  - @backstage/catalog-model@0.9.7
+  - @backstage/backend-common@0.9.10
+
 ## 0.17.3
 
 ### Patch Changes

package/dist/index.cjs.js

@@ -3705,49 +3705,9 @@ function startTaskPipeline(options) {
 }
 
 const CACHE_TTL = 5;
-class Connection {
-  constructor(config) {
-    this.config = config;
-    this.validateEntityEnvelope = catalogModel.entityEnvelopeSchemaValidator();
-  }
-  async applyMutation(mutation) {
-    const db = this.config.processingDatabase;
-    if (mutation.type === "full") {
-      this.check(mutation.entities.map((e) => e.entity));
-      await db.transaction(async (tx) => {
-        await db.replaceUnprocessedEntities(tx, {
-          sourceKey: this.config.id,
-          type: "full",
-          items: mutation.entities
-        });
-      });
-    } else if (mutation.type === "delta") {
-      this.check(mutation.added.map((e) => e.entity));
-      this.check(mutation.removed.map((e) => e.entity));
-      await db.transaction(async (tx) => {
-        await db.replaceUnprocessedEntities(tx, {
-          sourceKey: this.config.id,
-          type: "delta",
-          added: mutation.added,
-          removed: mutation.removed
-        });
-      });
-    }
-  }
-  check(entities) {
-    for (const entity of entities) {
-      try {
-        this.validateEntityEnvelope(entity);
-      } catch (e) {
-        throw new TypeError(`Malformed entity envelope, ${e}`);
-      }
-    }
-  }
-}
 class DefaultCatalogProcessingEngine {
-  constructor(logger, entityProviders, processingDatabase, orchestrator, stitcher, createHash, pollingIntervalMs = 1e3) {
+  constructor(logger, processingDatabase, orchestrator, stitcher, createHash, pollingIntervalMs = 1e3) {
     this.logger = logger;
-    this.entityProviders = entityProviders;
     this.processingDatabase = processingDatabase;
     this.orchestrator = orchestrator;
     this.stitcher = stitcher;
@@ -3759,12 +3719,6 @@ class DefaultCatalogProcessingEngine {
     if (this.stopFunc) {
       throw new Error("Processing engine is already started");
     }
-    for (const provider of this.entityProviders) {
-      await provider.connect(new Connection({
-        id: provider.getProviderName(),
-        processingDatabase: this.processingDatabase
-      }));
-    }
     this.stopFunc = startTaskPipeline({
       lowWatermark: 5,
       highWatermark: 10,
@@ -4157,25 +4111,26 @@ function isOrEntityFilter(filter) {
   return filter.hasOwnProperty("anyOf");
 }
 function parseFilter(filter, query, db) {
-  var _a, _b;
   if (isEntitiesSearchFilter(filter)) {
-    return query.where(function filterFunction() {
+    return query.andWhere(function filterFunction() {
       addCondition(this, db, filter);
     });
   }
   if (isOrEntityFilter(filter)) {
-    let cumulativeQuery = query;
-    for (const subFilter of (_a = filter.anyOf) != null ? _a : []) {
-      cumulativeQuery = cumulativeQuery.orWhere((subQuery) => parseFilter(subFilter, subQuery, db));
-    }
-    return cumulativeQuery;
+    return query.andWhere(function filterFunction() {
+      var _a;
+      for (const subFilter of (_a = filter.anyOf) != null ? _a : []) {
+        this.orWhere((subQuery) => parseFilter(subFilter, subQuery, db));
+      }
+    });
   }
   if (isAndEntityFilter(filter)) {
-    let cumulativeQuery = query;
-    for (const subFilter of (_b = filter.allOf) != null ? _b : []) {
-      cumulativeQuery = cumulativeQuery.andWhere((subQuery) => parseFilter(subFilter, subQuery, db));
-    }
-    return cumulativeQuery;
+    return query.andWhere(function filterFunction() {
+      var _a;
+      for (const subFilter of (_a = filter.allOf) != null ? _a : []) {
+        this.andWhere((subQuery) => parseFilter(subFilter, subQuery, db));
+      }
+    });
   }
   return query;
 }
@@ -4790,6 +4745,55 @@ class DefaultRefreshService {
   }
 }
 
+class Connection {
+  constructor(config) {
+    this.config = config;
+    this.validateEntityEnvelope = catalogModel.entityEnvelopeSchemaValidator();
+  }
+  async applyMutation(mutation) {
+    const db = this.config.processingDatabase;
+    if (mutation.type === "full") {
+      this.check(mutation.entities.map((e) => e.entity));
+      await db.transaction(async (tx) => {
+        await db.replaceUnprocessedEntities(tx, {
+          sourceKey: this.config.id,
+          type: "full",
+          items: mutation.entities
+        });
+      });
+    } else if (mutation.type === "delta") {
+      this.check(mutation.added.map((e) => e.entity));
+      this.check(mutation.removed.map((e) => e.entity));
+      await db.transaction(async (tx) => {
+        await db.replaceUnprocessedEntities(tx, {
+          sourceKey: this.config.id,
+          type: "delta",
+          added: mutation.added,
+          removed: mutation.removed
+        });
+      });
+    }
+  }
+  check(entities) {
+    for (const entity of entities) {
+      try {
+        this.validateEntityEnvelope(entity);
+      } catch (e) {
+        throw new TypeError(`Malformed entity envelope, ${e}`);
+      }
+    }
+  }
+}
+async function connectEntityProviders(db, providers) {
+  await Promise.all(providers.map(async (provider) => {
+    const connection = new Connection({
+      id: provider.getProviderName(),
+      processingDatabase: db
+    });
+    return provider.connect(connection);
+  }));
+}
+
 class NextCatalogBuilder {
   constructor(env) {
     this.refreshInterval = createRandomRefreshInterval({
@@ -4852,6 +4856,20 @@ class NextCatalogBuilder {
     this.processorsReplace = true;
     return this;
   }
+  getDefaultProcessors() {
+    const {config, logger, reader} = this.env;
+    const integrations = integration.ScmIntegrations.fromConfig(config);
+    return [
+      new FileReaderProcessor(),
+      BitbucketDiscoveryProcessor.fromConfig(config, {logger}),
+      GithubDiscoveryProcessor.fromConfig(config, {logger}),
+      GithubOrgReaderProcessor.fromConfig(config, {logger}),
+      GitLabDiscoveryProcessor.fromConfig(config, {logger}),
+      new UrlReaderProcessor({reader, logger}),
+      CodeOwnersProcessor.fromConfig(config, {logger, reader}),
+      new AnnotateLocationEntityProcessor({integrations})
+    ];
+  }
   setEntityDataParser(parser) {
     this.parser = parser;
     return this;
@@ -4885,7 +4903,7 @@ class NextCatalogBuilder {
     const locationStore = new DefaultLocationStore(dbClient);
     const configLocationProvider = new ConfigLocationEntityProvider(config);
     const entityProviders = lodash__default['default'].uniqBy([...this.entityProviders, locationStore, configLocationProvider], (provider) => provider.getProviderName());
-    const processingEngine = new DefaultCatalogProcessingEngine(logger, entityProviders, processingDatabase, orchestrator, stitcher, () => crypto.createHash("sha1"));
+    const processingEngine = new DefaultCatalogProcessingEngine(logger, processingDatabase, orchestrator, stitcher, () => crypto.createHash("sha1"));
     const locationsCatalog = new DatabaseLocationsCatalog(db);
     const locationAnalyzer = (_a = this.locationAnalyzer) != null ? _a : new RepoLocationAnalyzer(logger, integrations);
     const locationService = new DefaultLocationService(locationStore, orchestrator);
@@ -4900,6 +4918,7 @@ class NextCatalogBuilder {
       logger,
       config
     });
+    await connectEntityProviders(processingDatabase, entityProviders);
     return {
       entitiesCatalog,
       locationsCatalog,
@@ -4920,7 +4939,7 @@ class NextCatalogBuilder {
     return catalogModel.EntityPolicies.allOf(entityPolicies);
   }
   buildProcessors() {
-    const {config, logger, reader} = this.env;
+    const {config, reader} = this.env;
     const integrations = integration.ScmIntegrations.fromConfig(config);
     this.checkDeprecatedReaderProcessors();
     const placeholderResolvers = {
@@ -4938,7 +4957,7 @@ class NextCatalogBuilder {
       new BuiltinKindsEntityProcessor()
     ];
     if (!this.processorsReplace) {
-      processors.push(new FileReaderProcessor(), BitbucketDiscoveryProcessor.fromConfig(config, {logger}), GithubDiscoveryProcessor.fromConfig(config, {logger}), GithubOrgReaderProcessor.fromConfig(config, {logger}), GitLabDiscoveryProcessor.fromConfig(config, {logger}), new UrlReaderProcessor({reader, logger}), CodeOwnersProcessor.fromConfig(config, {logger, reader}), new AnnotateLocationEntityProcessor({integrations}));
+      processors.push(...this.getDefaultProcessors());
     }
     processors.push(...this.processors);
     return processors;