@backstage/plugin-catalog-backend 1.26.1-next.0 → 1.26.2-next.1

package/CHANGELOG.md

@@ -1,5 +1,27 @@
 # @backstage/plugin-catalog-backend
 
+## 1.26.2-next.1
+
+### Patch Changes
+
+- 62747f8: Fixed a bug where the concurrency limiter for URL reading was not honored
+- 8206f49: Fix a bug where etags were expiring too soon in the URL reader
+- Updated dependencies
+  - @backstage/integration@1.15.1-next.0
+  - @backstage/backend-openapi-utils@0.1.19-next.0
+  - @backstage/backend-plugin-api@1.0.1-next.0
+  - @backstage/catalog-client@1.7.0
+  - @backstage/catalog-model@1.7.0
+  - @backstage/config@1.2.0
+  - @backstage/errors@1.2.4
+  - @backstage/types@1.1.1
+  - @backstage/plugin-catalog-common@1.1.0
+  - @backstage/plugin-catalog-node@1.13.1-next.0
+  - @backstage/plugin-events-node@0.4.1-next.0
+  - @backstage/plugin-permission-common@0.8.1
+  - @backstage/plugin-permission-node@0.8.4-next.0
+  - @backstage/plugin-search-backend-module-catalog@0.2.3-next.1
+
 ## 1.26.1-next.0
 
 ### Patch Changes

package/alpha/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@backstage/plugin-catalog-backend__alpha",
-  "version": "1.26.1-next.0",
+  "version": "1.26.2-next.1",
   "main": "../dist/alpha.cjs.js",
   "types": "../dist/alpha.d.ts"
 }

package/dist/alpha.cjs.js

@@ -4,7 +4,7 @@ Object.defineProperty(exports, '__esModule', { value: true });
 
 var alpha = require('@backstage/plugin-catalog-common/alpha');
 var pluginPermissionNode = require('@backstage/plugin-permission-node');
-var CatalogBuilder = require('./cjs/CatalogBuilder-CGSl8LEN.cjs.js');
+var CatalogBuilder = require('./cjs/CatalogBuilder-BBOOqsyi.cjs.js');
 var backendPluginApi = require('@backstage/backend-plugin-api');
 var pluginEventsNode = require('@backstage/plugin-events-node');
 var alpha$1 = require('@backstage/plugin-catalog-node/alpha');
@@ -14,10 +14,10 @@ require('@backstage/backend-common');
 require('@backstage/catalog-model');
 require('@backstage/integration');
 require('crypto');
-require('core-js/features/promise');
-require('codeowners-utils');
 require('git-url-parse');
 require('@backstage/plugin-catalog-node');
+require('core-js/features/promise');
+require('codeowners-utils');
 require('fs-extra');
 require('glob');
 require('path');

package/dist/cjs/{CatalogBuilder-CGSl8LEN.cjs.js → CatalogBuilder-BBOOqsyi.cjs.js}

@@ -5,11 +5,11 @@ var catalogModel = require('@backstage/catalog-model');
 var integration = require('@backstage/integration');
 var crypto = require('crypto');
 var lodash = require('lodash');
+var parseGitUrl = require('git-url-parse');
+var pluginCatalogNode = require('@backstage/plugin-catalog-node');
 var errors = require('@backstage/errors');
 require('core-js/features/promise');
 var codeowners = require('codeowners-utils');
-var parseGitUrl = require('git-url-parse');
-var pluginCatalogNode = require('@backstage/plugin-catalog-node');
 var fs = require('fs-extra');
 var g = require('glob');
 var path = require('path');
@@ -56,8 +56,8 @@ function _interopNamespaceCompat(e) {
 }
 
 var lodash__default = /*#__PURE__*/_interopDefaultCompat(lodash);
-var codeowners__namespace = /*#__PURE__*/_interopNamespaceCompat(codeowners);
 var parseGitUrl__default = /*#__PURE__*/_interopDefaultCompat(parseGitUrl);
+var codeowners__namespace = /*#__PURE__*/_interopNamespaceCompat(codeowners);
 var fs__default = /*#__PURE__*/_interopDefaultCompat(fs);
 var g__default = /*#__PURE__*/_interopDefaultCompat(g);
 var path__default = /*#__PURE__*/_interopDefaultCompat(path);
@@ -68,118 +68,6 @@ var uniq__default = /*#__PURE__*/_interopDefaultCompat(uniq);
 var splitToChunks__default = /*#__PURE__*/_interopDefaultCompat(splitToChunks);
 var yn__default = /*#__PURE__*/_interopDefaultCompat(yn);
 
-const USER_PATTERN = /^@.*/;
-const GROUP_PATTERN = /^@.*\/.*/;
-const EMAIL_PATTERN = /^.*@.*\..*$/;
-function resolveCodeOwner(contents, catalogInfoFileUrl) {
-  const codeOwnerEntries = codeowners__namespace.parse(contents);
-  const { filepath } = parseGitUrl__default.default(catalogInfoFileUrl);
-  const match = codeowners__namespace.matchFile(filepath, codeOwnerEntries);
-  return match ? normalizeCodeOwner(match.owners[0]) : void 0;
-}
-function normalizeCodeOwner(owner) {
-  if (owner.match(GROUP_PATTERN)) {
-    return owner.split("/")[1];
-  } else if (owner.match(USER_PATTERN)) {
-    return `User:${owner.substring(1)}`;
-  } else if (owner.match(EMAIL_PATTERN)) {
-    return owner.split("@")[0];
-  }
-  return owner;
-}
-
-const CODEOWNERS = "CODEOWNERS";
-const scmCodeOwnersPaths = {
-  // https://mibexsoftware.atlassian.net/wiki/spaces/CODEOWNERS/pages/222822413/Usage
-  bitbucket: [CODEOWNERS, `.bitbucket/${CODEOWNERS}`],
-  // https://docs.gitlab.com/ee/user/project/code_owners.html#how-to-set-up-code-owners
-  gitlab: [CODEOWNERS, `.gitlab/${CODEOWNERS}`, `docs/${CODEOWNERS}`],
-  // https://docs.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners#codeowners-file-location
-  github: [CODEOWNERS, `.github/${CODEOWNERS}`, `docs/${CODEOWNERS}`]
-};
-
-async function readCodeOwners(reader, sourceUrl, codeownersPaths) {
-  const readOwnerLocation = async (path) => {
-    const url = `${sourceUrl}${path}`;
-    const data = await reader.readUrl(url);
-    const buffer = await data.buffer();
-    return buffer.toString();
-  };
-  const candidates = codeownersPaths.map(readOwnerLocation);
-  return Promise.any(candidates).catch((aggregateError) => {
-    const hardError = aggregateError.errors.find(
-      (error) => !(error instanceof errors.NotFoundError)
-    );
-    if (hardError) {
-      throw hardError;
-    }
-    return void 0;
-  });
-}
-async function findCodeOwnerByTarget(reader, targetUrl, scmIntegration) {
-  const codeownersPaths = scmCodeOwnersPaths[scmIntegration?.type ?? ""];
-  const sourceUrl = scmIntegration?.resolveUrl({
-    url: "/",
-    base: targetUrl
-  });
-  if (!sourceUrl || !codeownersPaths) {
-    return void 0;
-  }
-  const contents = await readCodeOwners(reader, sourceUrl, codeownersPaths);
-  if (!contents) {
-    return void 0;
-  }
-  const owner = resolveCodeOwner(contents, targetUrl);
-  return owner;
-}
-
-const ALLOWED_KINDS = ["API", "Component", "Domain", "Resource", "System"];
-const ALLOWED_LOCATION_TYPES = ["url"];
-class CodeOwnersProcessor {
-  integrations;
-  logger;
-  reader;
-  static fromConfig(config, options) {
-    const integrations = integration.ScmIntegrations.fromConfig(config);
-    return new CodeOwnersProcessor({
-      ...options,
-      integrations
-    });
-  }
-  constructor(options) {
-    this.integrations = options.integrations;
-    this.logger = options.logger;
-    this.reader = options.reader;
-  }
-  getProcessorName() {
-    return "CodeOwnersProcessor";
-  }
-  async preProcessEntity(entity, location) {
-    if (!entity || !ALLOWED_KINDS.includes(entity.kind) || !ALLOWED_LOCATION_TYPES.includes(location.type) || entity.spec && entity.spec.owner) {
-      return entity;
-    }
-    const scmIntegration = this.integrations.byUrl(location.target);
-    if (!scmIntegration) {
-      return entity;
-    }
-    const owner = await findCodeOwnerByTarget(
-      this.reader,
-      location.target,
-      scmIntegration
-    );
-    if (!owner) {
-      this.logger.debug(
-        `CodeOwnerProcessor could not resolve owner for ${location.target}`
-      );
-      return entity;
-    }
-    return {
-      ...entity,
-      spec: { ...entity.spec, owner }
-    };
-  }
-}
-
 const commitHashRegExp = /\b[0-9a-f]{40,}\b/;
 class AnnotateLocationEntityProcessor {
   constructor(options) {
@@ -436,6 +324,118 @@ class BuiltinKindsEntityProcessor {
   }
 }
 
+const USER_PATTERN = /^@.*/;
+const GROUP_PATTERN = /^@.*\/.*/;
+const EMAIL_PATTERN = /^.*@.*\..*$/;
+function resolveCodeOwner(contents, catalogInfoFileUrl) {
+  const codeOwnerEntries = codeowners__namespace.parse(contents);
+  const { filepath } = parseGitUrl__default.default(catalogInfoFileUrl);
+  const match = codeowners__namespace.matchFile(filepath, codeOwnerEntries);
+  return match ? normalizeCodeOwner(match.owners[0]) : void 0;
+}
+function normalizeCodeOwner(owner) {
+  if (owner.match(GROUP_PATTERN)) {
+    return owner.split("/")[1];
+  } else if (owner.match(USER_PATTERN)) {
+    return `User:${owner.substring(1)}`;
+  } else if (owner.match(EMAIL_PATTERN)) {
+    return owner.split("@")[0];
+  }
+  return owner;
+}
+
+const CODEOWNERS = "CODEOWNERS";
+const scmCodeOwnersPaths = {
+  // https://mibexsoftware.atlassian.net/wiki/spaces/CODEOWNERS/pages/222822413/Usage
+  bitbucket: [CODEOWNERS, `.bitbucket/${CODEOWNERS}`],
+  // https://docs.gitlab.com/ee/user/project/code_owners.html#how-to-set-up-code-owners
+  gitlab: [CODEOWNERS, `.gitlab/${CODEOWNERS}`, `docs/${CODEOWNERS}`],
+  // https://docs.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners#codeowners-file-location
+  github: [CODEOWNERS, `.github/${CODEOWNERS}`, `docs/${CODEOWNERS}`]
+};
+
+async function readCodeOwners(reader, sourceUrl, codeownersPaths) {
+  const readOwnerLocation = async (path) => {
+    const url = `${sourceUrl}${path}`;
+    const data = await reader.readUrl(url);
+    const buffer = await data.buffer();
+    return buffer.toString();
+  };
+  const candidates = codeownersPaths.map(readOwnerLocation);
+  return Promise.any(candidates).catch((aggregateError) => {
+    const hardError = aggregateError.errors.find(
+      (error) => !(error instanceof errors.NotFoundError)
+    );
+    if (hardError) {
+      throw hardError;
+    }
+    return void 0;
+  });
+}
+async function findCodeOwnerByTarget(reader, targetUrl, scmIntegration) {
+  const codeownersPaths = scmCodeOwnersPaths[scmIntegration?.type ?? ""];
+  const sourceUrl = scmIntegration?.resolveUrl({
+    url: "/",
+    base: targetUrl
+  });
+  if (!sourceUrl || !codeownersPaths) {
+    return void 0;
+  }
+  const contents = await readCodeOwners(reader, sourceUrl, codeownersPaths);
+  if (!contents) {
+    return void 0;
+  }
+  const owner = resolveCodeOwner(contents, targetUrl);
+  return owner;
+}
+
+const ALLOWED_KINDS = ["API", "Component", "Domain", "Resource", "System"];
+const ALLOWED_LOCATION_TYPES = ["url"];
+class CodeOwnersProcessor {
+  integrations;
+  logger;
+  reader;
+  static fromConfig(config, options) {
+    const integrations = integration.ScmIntegrations.fromConfig(config);
+    return new CodeOwnersProcessor({
+      ...options,
+      integrations
+    });
+  }
+  constructor(options) {
+    this.integrations = options.integrations;
+    this.logger = options.logger;
+    this.reader = options.reader;
+  }
+  getProcessorName() {
+    return "CodeOwnersProcessor";
+  }
+  async preProcessEntity(entity, location) {
+    if (!entity || !ALLOWED_KINDS.includes(entity.kind) || !ALLOWED_LOCATION_TYPES.includes(location.type) || entity.spec && entity.spec.owner) {
+      return entity;
+    }
+    const scmIntegration = this.integrations.byUrl(location.target);
+    if (!scmIntegration) {
+      return entity;
+    }
+    const owner = await findCodeOwnerByTarget(
+      this.reader,
+      location.target,
+      scmIntegration
+    );
+    if (!owner) {
+      this.logger.debug(
+        `CodeOwnerProcessor could not resolve owner for ${location.target}`
+      );
+      return entity;
+    }
+    return {
+      ...entity,
+      spec: { ...entity.spec, owner }
+    };
+  }
+}
+
 const glob = util.promisify(g__default.default);
 const LOCATION_TYPE = "file";
 class FileReaderProcessor {
@@ -612,7 +612,11 @@ const CACHE_KEY = "v1";
 class UrlReaderProcessor {
   constructor(options) {
     this.options = options;
+    this.#limiter = limiterFactory__default.default(5);
   }
+  // This limiter is used for only consuming a limited number of read streams
+  // concurrently.
+  #limiter;
   getProcessorName() {
     return "url-reader";
   }
@@ -655,6 +659,7 @@ class UrlReaderProcessor {
           emit(parseResult);
         }
         emit(pluginCatalogNode.processingResult.refresh(`${location.type}:${location.target}`));
+        await cache.set(CACHE_KEY, cacheItem);
       } else if (error.name === "NotFoundError") {
         if (!optional) {
           emit(pluginCatalogNode.processingResult.notFoundError(location, message));
@@ -668,11 +673,10 @@ class UrlReaderProcessor {
   async doRead(location, etag) {
     const { filepath } = parseGitUrl__default.default(location);
     if (filepath?.match(/[*?]/)) {
-      const limiter = limiterFactory__default.default(5);
       const response = await this.options.reader.search(location, { etag });
       const output = response.files.map(async (file) => ({
         url: file.url,
-        data: await limiter(file.content)
+        data: await this.#limiter(file.content)
       }));
       return { response: await Promise.all(output), etag: response.etag };
     }
@@ -684,38 +688,6 @@ class UrlReaderProcessor {
   }
 }
 
-function* parseEntityYaml(data, location) {
-  let documents;
-  try {
-    documents = yaml__default.default.parseAllDocuments(data.toString("utf8")).filter((d) => d);
-  } catch (e) {
-    const loc = catalogModel.stringifyLocationRef(location);
-    const message = `Failed to parse YAML at ${loc}, ${e}`;
-    yield pluginCatalogNode.processingResult.generalError(location, message);
-    return;
-  }
-  for (const document of documents) {
-    if (document.errors?.length) {
-      const loc = catalogModel.stringifyLocationRef(location);
-      const message = `YAML error at ${loc}, ${document.errors[0]}`;
-      yield pluginCatalogNode.processingResult.generalError(location, message);
-    } else {
-      const json = document.toJSON();
-      if (lodash__default.default.isPlainObject(json)) {
-        yield pluginCatalogNode.processingResult.entity(location, json);
-      } else if (json === null) ; else {
-        const message = `Expected object at root, got ${typeof json}`;
-        yield pluginCatalogNode.processingResult.generalError(location, message);
-      }
-    }
-  }
-}
-const defaultEntityDataParser = async function* defaultEntityDataParser2({ data, location }) {
-  for (const e of parseEntityYaml(data, location)) {
-    yield e;
-  }
-};
-
 function createRandomProcessingInterval(options) {
   const { minSeconds, maxSeconds } = options;
   return () => {
@@ -1072,6 +1044,38 @@ class AuthorizedLocationAnalyzer {
   }
 }
 
+function* parseEntityYaml(data, location) {
+  let documents;
+  try {
+    documents = yaml__default.default.parseAllDocuments(data.toString("utf8")).filter((d) => d);
+  } catch (e) {
+    const loc = catalogModel.stringifyLocationRef(location);
+    const message = `Failed to parse YAML at ${loc}, ${e}`;
+    yield pluginCatalogNode.processingResult.generalError(location, message);
+    return;
+  }
+  for (const document of documents) {
+    if (document.errors?.length) {
+      const loc = catalogModel.stringifyLocationRef(location);
+      const message = `YAML error at ${loc}, ${document.errors[0]}`;
+      yield pluginCatalogNode.processingResult.generalError(location, message);
+    } else {
+      const json = document.toJSON();
+      if (lodash__default.default.isPlainObject(json)) {
+        yield pluginCatalogNode.processingResult.entity(location, json);
+      } else if (json === null) ; else {
+        const message = `Expected object at root, got ${typeof json}`;
+        yield pluginCatalogNode.processingResult.generalError(location, message);
+      }
+    }
+  }
+}
+const defaultEntityDataParser = async function* defaultEntityDataParser2({ data, location }) {
+  for (const e of parseEntityYaml(data, location)) {
+    yield e;
+  }
+};
+
 function timestampToDateTime(input) {
   try {
     if (typeof input === "object") {
@@ -7372,4 +7376,4 @@ exports.createCatalogPermissionRule = createCatalogPermissionRule;
 exports.createRandomProcessingInterval = createRandomProcessingInterval;
 exports.parseEntityYaml = parseEntityYaml;
 exports.permissionRules = permissionRules;
-//# sourceMappingURL=CatalogBuilder-CGSl8LEN.cjs.js.map
+//# sourceMappingURL=CatalogBuilder-BBOOqsyi.cjs.js.map