@backstage/plugin-catalog-backend 0.17.4 → 0.18.0

package/CHANGELOG.md

@@ -1,5 +1,72 @@
 # @backstage/plugin-catalog-backend
 
+## 0.18.0
+
+### Minor Changes
+
+- 7f82ce9f51: **BREAKING** EntitiesSearchFilter fields have changed.
+
+  EntitiesSearchFilter now has only two fields: `key` and `value`. The `matchValueIn` and `matchValueExists` fields are no longer are supported. Previous filters written using the `matchValueIn` and `matchValueExists` fields can be rewritten as follows:
+
+  Filtering by existence of key only:
+
+  ```diff
+    filter: {
+      {
+        key: 'abc',
+  -     matchValueExists: true,
+      },
+    }
+  ```
+
+  Filtering by key and values:
+
+  ```diff
+    filter: {
+      {
+        key: 'abc',
+  -     matchValueExists: true,
+  -     matchValueIn: ['xyz'],
+  +     values: ['xyz'],
+      },
+    }
+  ```
+
+  Negation of filters can now be achieved through a `not` object:
+
+  ```
+  filter: {
+    not: {
+      key: 'abc',
+      values: ['xyz'],
+    },
+  }
+  ```
+
+### Patch Changes
+
+- 740f958290: Providing an empty values array in an EntityFilter will now return no matches.
+- bab752e2b3: Change default port of backend from 7000 to 7007.
+
+  This is due to the AirPlay Receiver process occupying port 7000 and preventing local Backstage instances on MacOS to start.
+
+  You can change the port back to 7000 or any other value by providing an `app-config.yaml` with the following values:
+
+  ```
+  backend:
+    listen: 0.0.0.0:7123
+    baseUrl: http://localhost:7123
+  ```
+
+  More information can be found here: https://backstage.io/docs/conf/writing
+
+- eddb82ab7c: Index User entities by displayName to be able to search by full name. Added displayName (if present) to the 'text' field in the indexed document.
+- 563b039f0b: Added Azure DevOps discovery processor
+- 8866b62f3d: Detect a duplicate entities when adding locations through dry run
+- Updated dependencies
+  - @backstage/errors@0.1.5
+  - @backstage/backend-common@0.9.11
+
 ## 0.17.4
 
 ### Patch Changes

package/dist/index.cjs.js

@@ -542,7 +542,7 @@ class BitbucketDiscoveryProcessor {
   }
   async processOrganisationRepositories(options) {
     const {client, location, integration, emit} = options;
-    const {catalogPath: requestedCatalogPath} = parseUrl$2(location.target);
+    const {catalogPath: requestedCatalogPath} = parseUrl$3(location.target);
     const catalogPath = requestedCatalogPath === EMPTY_CATALOG_LOCATION ? DEFAULT_CATALOG_LOCATION : requestedCatalogPath;
     const result = await readBitbucketOrg(client, location.target);
     for (const repository of result.matches) {
@@ -561,7 +561,7 @@ class BitbucketDiscoveryProcessor {
   }
 }
 async function readBitbucketOrg(client, target) {
-  const {projectSearchPath, repoSearchPath} = parseUrl$2(target);
+  const {projectSearchPath, repoSearchPath} = parseUrl$3(target);
   const projects = paginated$1((options) => client.listProjects(options));
   const result = {
     scanned: 0,
@@ -603,7 +603,7 @@ async function readBitbucketCloud(client, target) {
   }
   return result;
 }
-function parseUrl$2(urlString) {
+function parseUrl$3(urlString) {
   const url = new URL(urlString);
   const path = url.pathname.substr(1).split("/");
   if (path.length > 3 && path[1].length && path[3].length) {
@@ -1115,7 +1115,7 @@ class GithubDiscoveryProcessor {
     if (!gitHubConfig) {
       throw new Error(`There is no GitHub integration that matches ${location$1.target}. Please add a configuration entry for it under integrations.github`);
     }
-    const {org, repoSearchPath, catalogPath, branch, host} = parseUrl$1(location$1.target);
+    const {org, repoSearchPath, catalogPath, branch, host} = parseUrl$2(location$1.target);
     const orgUrl = `https://${host}/${org}`;
     const {headers} = await integration.GithubCredentialsProvider.create(gitHubConfig).getCredentials({url: orgUrl});
     const client = graphql.graphql.defaults({
@@ -1143,7 +1143,7 @@ class GithubDiscoveryProcessor {
     return true;
   }
 }
-function parseUrl$1(urlString) {
+function parseUrl$2(urlString) {
   const url = new URL(urlString);
   const path = url.pathname.substr(1).split("/");
   if (path.length > 2 && path[0].length && path[1].length) {
@@ -1169,6 +1169,93 @@ function escapeRegExp(str) {
   return new RegExp(`^${str.replace(/\*/g, ".*")}$`);
 }
 
+const isCloud = (host) => host === "dev.azure.com";
+const PAGE_SIZE = 1e3;
+async function codeSearch(azureConfig, org, project, repo, path) {
+  const searchBaseUrl = isCloud(azureConfig.host) ? "https://almsearch.dev.azure.com" : `https://${azureConfig.host}`;
+  const searchUrl = `${searchBaseUrl}/${org}/${project}/_apis/search/codesearchresults?api-version=6.0-preview.1`;
+  let items = [];
+  let hasMorePages = true;
+  do {
+    const response = await fetch__default['default'](searchUrl, {
+      ...integration.getAzureRequestOptions(azureConfig, {
+        "Content-Type": "application/json"
+      }),
+      method: "POST",
+      body: JSON.stringify({
+        searchText: `path:${path} repo:${repo || "*"}`,
+        $skip: items.length,
+        $top: PAGE_SIZE
+      })
+    });
+    if (response.status !== 200) {
+      throw new Error(`Azure DevOps search failed with response status ${response.status}`);
+    }
+    const body = await response.json();
+    items = [...items, ...body.results];
+    hasMorePages = body.count > items.length;
+  } while (hasMorePages);
+  return items;
+}
+
+class AzureDevOpsDiscoveryProcessor {
+  static fromConfig(config, options) {
+    const integrations = integration.ScmIntegrations.fromConfig(config);
+    return new AzureDevOpsDiscoveryProcessor({
+      ...options,
+      integrations
+    });
+  }
+  constructor(options) {
+    this.integrations = options.integrations;
+    this.logger = options.logger;
+  }
+  async readLocation(location$1, _optional, emit) {
+    var _a;
+    if (location$1.type !== "azure-discovery") {
+      return false;
+    }
+    const azureConfig = (_a = this.integrations.azure.byUrl(location$1.target)) == null ? void 0 : _a.config;
+    if (!azureConfig) {
+      throw new Error(`There is no Azure integration that matches ${location$1.target}. Please add a configuration entry for it under integrations.azure`);
+    }
+    const {baseUrl, org, project, repo, catalogPath} = parseUrl$1(location$1.target);
+    this.logger.info(`Reading Azure DevOps repositories from ${location$1.target}`);
+    const files = await codeSearch(azureConfig, org, project, repo, catalogPath);
+    this.logger.debug(`Found ${files.length} files in Azure DevOps from ${location$1.target}.`);
+    for (const file of files) {
+      emit(location({
+        type: "url",
+        target: `${baseUrl}/${org}/${project}/_git/${file.repository.name}?path=${file.path}`
+      }, true));
+    }
+    return true;
+  }
+}
+function parseUrl$1(urlString) {
+  const url = new URL(urlString);
+  const path = url.pathname.substr(1).split("/");
+  const catalogPath = url.searchParams.get("path") || "/catalog-info.yaml";
+  if (path.length === 2 && path[0].length && path[1].length) {
+    return {
+      baseUrl: url.origin,
+      org: decodeURIComponent(path[0]),
+      project: decodeURIComponent(path[1]),
+      repo: "",
+      catalogPath
+    };
+  } else if (path.length === 4 && path[0].length && path[1].length && path[2].length && path[3].length) {
+    return {
+      baseUrl: url.origin,
+      org: decodeURIComponent(path[0]),
+      project: decodeURIComponent(path[1]),
+      repo: decodeURIComponent(path[3]),
+      catalogPath
+    };
+  }
+  throw new Error(`Failed to parse ${urlString}`);
+}
+
 function buildOrgHierarchy(groups) {
   const groupsByName = new Map(groups.map((g) => [g.metadata.name, g]));
   for (const group of groups) {
@@ -1829,8 +1916,8 @@ function basicEntityFilter(items) {
   const filtersByKey = {};
   for (const [key, value] of Object.entries(items)) {
     const values = [value].flat();
-    const f = key in filtersByKey ? filtersByKey[key] : filtersByKey[key] = {key, matchValueIn: []};
-    f.matchValueIn.push(...values);
+    const f = key in filtersByKey ? filtersByKey[key] : filtersByKey[key] = {key, values: []};
+    f.values.push(...values);
   }
   return {anyOf: [{allOf: Object.values(filtersByKey)}]};
 }
@@ -1893,11 +1980,9 @@ function parseEntityFilterString(filterString) {
       throw new errors.InputError(`Invalid filter, '${statement}' is not a valid statement (expected a string on the form a=b or a= or a)`);
     }
     const f = key in filtersByKey ? filtersByKey[key] : filtersByKey[key] = {key};
-    if (value === void 0) {
-      f.matchValueExists = true;
-    } else {
-      f.matchValueIn = f.matchValueIn || [];
-      f.matchValueIn.push(value);
+    if (value !== void 0) {
+      f.values = f.values || [];
+      f.values.push(value);
     }
   }
   return Object.values(filtersByKey);
@@ -2633,27 +2718,27 @@ class CommonDatabase {
     var _a, _b;
     const tx = txOpaque;
     let entitiesQuery = tx("entities");
-    if ((request == null ? void 0 : request.filter) && (request.filter.hasOwnProperty("key") || request.filter.hasOwnProperty("allOf"))) {
+    if ((request == null ? void 0 : request.filter) && (request.filter.hasOwnProperty("key") || request.filter.hasOwnProperty("allOf") || request.filter.hasOwnProperty("not"))) {
       throw new Error("Filters for the legacy CommonDatabase must obey the { anyOf: [{ allOf: [] }] } format.");
     }
     for (const singleFilter of (_b = (_a = request == null ? void 0 : request.filter) == null ? void 0 : _a.anyOf) != null ? _b : []) {
       entitiesQuery = entitiesQuery.orWhere(function singleFilterFn() {
         for (const filter of singleFilter.allOf) {
-          if (filter.hasOwnProperty("anyOf") || filter.hasOwnProperty("allOf")) {
+          if (filter.hasOwnProperty("anyOf") || filter.hasOwnProperty("allOf") || filter.hasOwnProperty("not")) {
             throw new Error("Nested filters are not supported in the legacy CommonDatabase");
           }
-          const {key, matchValueIn, matchValueExists} = filter;
+          const {key, values} = filter;
           const matchQuery = tx("entities_search").select("entity_id").where(function keyFilter() {
             this.andWhere({key: key.toLowerCase()});
-            if (matchValueExists !== false && matchValueIn) {
-              if (matchValueIn.length === 1) {
-                this.andWhere({value: matchValueIn[0].toLowerCase()});
-              } else if (matchValueIn.length > 1) {
-                this.andWhere("value", "in", matchValueIn.map((v) => v.toLowerCase()));
+            if (values) {
+              if (values.length === 1) {
+                this.andWhere({value: values[0].toLowerCase()});
+              } else if (values.length > 1) {
+                this.andWhere("value", "in", values.map((v) => v.toLowerCase()));
               }
             }
           });
-          this.andWhere("id", matchValueExists === false ? "not in" : "in", matchQuery);
+          this.andWhere("id", "in", matchQuery);
         }
       });
     }
@@ -3919,6 +4004,29 @@ class DefaultLocationService {
   deleteLocation(id) {
     return this.store.deleteLocation(id);
   }
+  async processEntities(unprocessedEntities) {
+    const entities = [];
+    while (unprocessedEntities.length) {
+      const currentEntity = unprocessedEntities.pop();
+      if (!currentEntity) {
+        continue;
+      }
+      const processed = await this.orchestrator.process({
+        entity: currentEntity.entity,
+        state: {}
+      });
+      if (processed.ok) {
+        if (entities.some((e) => catalogModel.stringifyEntityRef(e) === catalogModel.stringifyEntityRef(processed.completedEntity))) {
+          throw new Error(`Duplicate nested entity: ${catalogModel.stringifyEntityRef(processed.completedEntity)}`);
+        }
+        unprocessedEntities.push(...processed.deferredEntities);
+        entities.push(processed.completedEntity);
+      } else {
+        throw Error(processed.errors.map(String).join(", "));
+      }
+    }
+    return entities;
+  }
   async dryRunCreateLocation(spec) {
     const existsPromise = this.store.listLocations().then((locations) => locations.some((l) => l.type === spec.type && l.target === spec.target));
     const entity = {
@@ -3943,23 +4051,7 @@ class DefaultLocationService {
     const unprocessedEntities = [
       {entity, locationKey: `${spec.type}:${spec.target}`}
     ];
-    const entities = [];
-    while (unprocessedEntities.length) {
-      const currentEntity = unprocessedEntities.pop();
-      if (!currentEntity) {
-        continue;
-      }
-      const processed = await this.orchestrator.process({
-        entity: currentEntity.entity,
-        state: {}
-      });
-      if (processed.ok) {
-        unprocessedEntities.push(...processed.deferredEntities);
-        entities.push(processed.completedEntity);
-      } else {
-        throw Error(processed.errors.map(String).join(", "));
-      }
-    }
+    const entities = await this.processEntities(unprocessedEntities);
     return {
       exists: await existsPromise,
       location: {...spec, id: `${spec.type}:${spec.target}`},
@@ -4088,51 +4180,48 @@ function stringifyPagination(input) {
   const base64 = Buffer.from(json, "utf8").toString("base64");
   return base64;
 }
-function addCondition(queryBuilder, db, {key, matchValueIn, matchValueExists}) {
-  const matchQuery = db("search").select("entity_id").where(function keyFilter() {
-    this.andWhere({key: key.toLowerCase()});
-    if (matchValueExists !== false && matchValueIn) {
-      if (matchValueIn.length === 1) {
-        this.andWhere({value: matchValueIn[0].toLowerCase()});
-      } else if (matchValueIn.length > 1) {
-        this.andWhere("value", "in", matchValueIn.map((v) => v.toLowerCase()));
+function addCondition(queryBuilder, db, filter, negate = false) {
+  const matchQuery = db("search").select("entity_id").where({key: filter.key.toLowerCase()}).andWhere(function keyFilter() {
+    if (filter.values) {
+      if (filter.values.length === 1) {
+        this.where({value: filter.values[0].toLowerCase()});
+      } else {
+        this.andWhere("value", "in", filter.values.map((v) => v.toLowerCase()));
       }
     }
   });
-  queryBuilder.andWhere("entity_id", matchValueExists === false ? "not in" : "in", matchQuery);
+  queryBuilder.andWhere("entity_id", negate ? "not in" : "in", matchQuery);
 }
 function isEntitiesSearchFilter(filter) {
   return filter.hasOwnProperty("key");
 }
-function isAndEntityFilter(filter) {
-  return filter.hasOwnProperty("allOf");
-}
 function isOrEntityFilter(filter) {
   return filter.hasOwnProperty("anyOf");
 }
-function parseFilter(filter, query, db) {
+function isNegationEntityFilter(filter) {
+  return filter.hasOwnProperty("not");
+}
+function parseFilter(filter, query, db, negate = false) {
   if (isEntitiesSearchFilter(filter)) {
     return query.andWhere(function filterFunction() {
-      addCondition(this, db, filter);
+      addCondition(this, db, filter, negate);
     });
   }
-  if (isOrEntityFilter(filter)) {
-    return query.andWhere(function filterFunction() {
-      var _a;
+  if (isNegationEntityFilter(filter)) {
+    return parseFilter(filter.not, query, db, !negate);
+  }
+  return query[negate ? "andWhereNot" : "andWhere"](function filterFunction() {
+    var _a, _b;
+    if (isOrEntityFilter(filter)) {
       for (const subFilter of (_a = filter.anyOf) != null ? _a : []) {
         this.orWhere((subQuery) => parseFilter(subFilter, subQuery, db));
       }
-    });
-  }
-  if (isAndEntityFilter(filter)) {
-    return query.andWhere(function filterFunction() {
-      var _a;
-      for (const subFilter of (_a = filter.allOf) != null ? _a : []) {
+    } else {
+      for (const subFilter of (_b = filter.allOf) != null ? _b : []) {
         this.andWhere((subQuery) => parseFilter(subFilter, subQuery, db));
       }
-    });
-  }
-  return query;
+    }
+  });
 }
 class NextEntitiesCatalog {
   constructor(database) {
@@ -4862,6 +4951,7 @@ class NextCatalogBuilder {
     return [
       new FileReaderProcessor(),
       BitbucketDiscoveryProcessor.fromConfig(config, {logger}),
+      AzureDevOpsDiscoveryProcessor.fromConfig(config, {logger}),
       GithubDiscoveryProcessor.fromConfig(config, {logger}),
       GithubOrgReaderProcessor.fromConfig(config, {logger}),
       GitLabDiscoveryProcessor.fromConfig(config, {logger}),
@@ -5080,7 +5170,7 @@ class CatalogBuilder {
       new BuiltinKindsEntityProcessor()
     ];
     if (!this.processorsReplace) {
-      processors.push(new FileReaderProcessor(), BitbucketDiscoveryProcessor.fromConfig(config, {logger}), GithubDiscoveryProcessor.fromConfig(config, {logger}), GithubOrgReaderProcessor.fromConfig(config, {logger}), GitLabDiscoveryProcessor.fromConfig(config, {logger}), new UrlReaderProcessor({reader, logger}), CodeOwnersProcessor.fromConfig(config, {logger, reader}), new LocationEntityProcessor({integrations}), new AnnotateLocationEntityProcessor({integrations}));
+      processors.push(new FileReaderProcessor(), BitbucketDiscoveryProcessor.fromConfig(config, {logger}), GithubDiscoveryProcessor.fromConfig(config, {logger}), AzureDevOpsDiscoveryProcessor.fromConfig(config, {logger}), GithubOrgReaderProcessor.fromConfig(config, {logger}), GitLabDiscoveryProcessor.fromConfig(config, {logger}), new UrlReaderProcessor({reader, logger}), CodeOwnersProcessor.fromConfig(config, {logger, reader}), new LocationEntityProcessor({integrations}), new AnnotateLocationEntityProcessor({integrations}));
     }
     processors.push(...this.processors);
     return processors;
@@ -5267,6 +5357,22 @@ class DefaultCatalogCollator {
     }
     return formatted.toLowerCase();
   }
+  isUserEntity(entity) {
+    return entity.kind.toLocaleUpperCase("en-US") === "USER";
+  }
+  getDocumentText(entity) {
+    var _a, _b, _c, _d, _e, _f;
+    let documentText = entity.metadata.description || "";
+    if (this.isUserEntity(entity)) {
+      if (((_b = (_a = entity.spec) == null ? void 0 : _a.profile) == null ? void 0 : _b.displayName) && documentText) {
+        const displayName = (_d = (_c = entity.spec) == null ? void 0 : _c.profile) == null ? void 0 : _d.displayName;
+        documentText = displayName.concat(" : ", documentText);
+      } else {
+        documentText = ((_f = (_e = entity.spec) == null ? void 0 : _e.profile) == null ? void 0 : _f.displayName) || documentText;
+      }
+    }
+    return documentText;
+  }
   async execute() {
     const response = await this.catalogClient.getEntities({
       filter: this.filter
@@ -5280,7 +5386,7 @@ class DefaultCatalogCollator {
           kind: entity.kind,
           name: entity.metadata.name
         }),
-        text: entity.metadata.description || "",
+        text: this.getDocumentText(entity),
         componentType: ((_c = (_b = entity.spec) == null ? void 0 : _b.type) == null ? void 0 : _c.toString()) || "other",
         namespace: entity.metadata.namespace || "default",
         kind: entity.kind,
@@ -5295,6 +5401,7 @@ exports.AnnotateLocationEntityProcessor = AnnotateLocationEntityProcessor;
 exports.AnnotateScmSlugEntityProcessor = AnnotateScmSlugEntityProcessor;
 exports.AwsOrganizationCloudAccountProcessor = AwsOrganizationCloudAccountProcessor;
 exports.AwsS3DiscoveryProcessor = AwsS3DiscoveryProcessor;
+exports.AzureDevOpsDiscoveryProcessor = AzureDevOpsDiscoveryProcessor;
 exports.BitbucketDiscoveryProcessor = BitbucketDiscoveryProcessor;
 exports.BuiltinKindsEntityProcessor = BuiltinKindsEntityProcessor;
 exports.CatalogBuilder = CatalogBuilder;