@backstage/plugin-catalog-backend-module-github 0.2.7-next.0 → 0.2.7-next.1

package/CHANGELOG.md

@@ -1,5 +1,24 @@
 # @backstage/plugin-catalog-backend-module-github
 
+## 0.2.7-next.1
+
+### Patch Changes
+
+- Updated dependencies
+  - @backstage/plugin-catalog-backend@1.8.1-next.1
+  - @backstage/backend-tasks@0.5.1-next.1
+  - @backstage/integration@1.4.4-next.0
+  - @backstage/backend-common@0.18.4-next.1
+  - @backstage/backend-plugin-api@0.5.1-next.1
+  - @backstage/catalog-client@1.4.0
+  - @backstage/catalog-model@1.2.1
+  - @backstage/config@1.0.7
+  - @backstage/errors@1.1.5
+  - @backstage/types@1.0.2
+  - @backstage/plugin-catalog-common@1.0.13-next.0
+  - @backstage/plugin-catalog-node@1.3.5-next.1
+  - @backstage/plugin-events-node@0.2.5-next.1
+
 ## 0.2.7-next.0
 
 ### Patch Changes

package/alpha/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@backstage/plugin-catalog-backend-module-github",
-  "version": "0.2.7-next.0",
+  "version": "0.2.7-next.1",
   "main": "../dist/alpha.cjs.js",
   "types": "../dist/alpha.d.ts"
 }

package/dist/alpha.cjs.js

@@ -5,488 +5,15 @@ Object.defineProperty(exports, '__esModule', { value: true });
 var backendPluginApi = require('@backstage/backend-plugin-api');
 var backendCommon = require('@backstage/backend-common');
 var alpha = require('@backstage/plugin-catalog-node/alpha');
-var integration = require('@backstage/integration');
-var pluginCatalogNode = require('@backstage/plugin-catalog-node');
-var graphql = require('@octokit/graphql');
-var uuid = require('uuid');
-var backendTasks = require('@backstage/backend-tasks');
+var GithubEntityProvider = require('./cjs/GithubEntityProvider-15bc2fa6.cjs.js');
+require('@backstage/integration');
+require('@backstage/plugin-catalog-node');
+require('@octokit/graphql');
+require('uuid');
+require('@backstage/backend-tasks');
 require('@backstage/catalog-model');
 require('lodash');
-var minimatch = require('minimatch');
-
-function _interopNamespace(e) {
-  if (e && e.__esModule) return e;
-  var n = Object.create(null);
-  if (e) {
-    Object.keys(e).forEach(function (k) {
-      if (k !== 'default') {
-        var d = Object.getOwnPropertyDescriptor(e, k);
-        Object.defineProperty(n, k, d.get ? d : {
-          enumerable: true,
-          get: function () { return e[k]; }
-        });
-      }
-    });
-  }
-  n["default"] = e;
-  return Object.freeze(n);
-}
-
-var uuid__namespace = /*#__PURE__*/_interopNamespace(uuid);
-
-const DEFAULT_CATALOG_PATH = "/catalog-info.yaml";
-const DEFAULT_PROVIDER_ID = "default";
-function readProviderConfigs(config) {
-  const providersConfig = config.getOptionalConfig("catalog.providers.github");
-  if (!providersConfig) {
-    return [];
-  }
-  if (providersConfig.has("organization")) {
-    return [readProviderConfig(DEFAULT_PROVIDER_ID, providersConfig)];
-  }
-  return providersConfig.keys().map((id) => {
-    const providerConfig = providersConfig.getConfig(id);
-    return readProviderConfig(id, providerConfig);
-  });
-}
-function readProviderConfig(id, config) {
-  var _a, _b, _c, _d;
-  const organization = config.getString("organization");
-  const catalogPath = (_a = config.getOptionalString("catalogPath")) != null ? _a : DEFAULT_CATALOG_PATH;
-  const host = (_b = config.getOptionalString("host")) != null ? _b : "github.com";
-  const repositoryPattern = config.getOptionalString("filters.repository");
-  const branchPattern = config.getOptionalString("filters.branch");
-  const allowForks = (_c = config.getOptionalBoolean("filters.allowForks")) != null ? _c : true;
-  const topicFilterInclude = config == null ? void 0 : config.getOptionalStringArray(
-    "filters.topic.include"
-  );
-  const topicFilterExclude = config == null ? void 0 : config.getOptionalStringArray(
-    "filters.topic.exclude"
-  );
-  const validateLocationsExist = (_d = config == null ? void 0 : config.getOptionalBoolean("validateLocationsExist")) != null ? _d : false;
-  const catalogPathContainsWildcard = catalogPath.includes("*");
-  if (validateLocationsExist && catalogPathContainsWildcard) {
-    throw Error(
-      `Error while processing GitHub provider config. The catalog path ${catalogPath} contains a wildcard, which is incompatible with validation of locations existing before emitting them. Ensure that validateLocationsExist is set to false.`
-    );
-  }
-  const schedule = config.has("schedule") ? backendTasks.readTaskScheduleDefinitionFromConfig(config.getConfig("schedule")) : void 0;
-  return {
-    id,
-    catalogPath,
-    organization,
-    host,
-    filters: {
-      repository: repositoryPattern ? compileRegExp(repositoryPattern) : void 0,
-      branch: branchPattern || void 0,
-      allowForks,
-      topic: {
-        include: topicFilterInclude,
-        exclude: topicFilterExclude
-      }
-    },
-    schedule,
-    validateLocationsExist
-  };
-}
-function compileRegExp(pattern) {
-  let fullLinePattern = pattern;
-  if (!fullLinePattern.startsWith("^")) {
-    fullLinePattern = `^${fullLinePattern}`;
-  }
-  if (!fullLinePattern.endsWith("$")) {
-    fullLinePattern = `${fullLinePattern}$`;
-  }
-  return new RegExp(fullLinePattern);
-}
-
-function satisfiesTopicFilter(topics, topicFilter) {
-  var _a, _b, _c, _d;
-  if (!topicFilter)
-    return true;
-  if (!topicFilter.include && !topicFilter.exclude)
-    return true;
-  if (!((_a = topicFilter.include) == null ? void 0 : _a.length) && !((_b = topicFilter.exclude) == null ? void 0 : _b.length))
-    return true;
-  if (((_c = topicFilter.include) == null ? void 0 : _c.length) && !topicFilter.exclude) {
-    for (const topic of topics) {
-      if (topicFilter.include.includes(topic))
-        return true;
-    }
-    return false;
-  }
-  if (!topicFilter.include && ((_d = topicFilter.exclude) == null ? void 0 : _d.length)) {
-    if (!topics.length)
-      return true;
-    for (const topic of topics) {
-      if (topicFilter.exclude.includes(topic))
-        return false;
-    }
-    return true;
-  }
-  if (topicFilter.include && topicFilter.exclude) {
-    const matchesInclude = satisfiesTopicFilter(topics, {
-      include: topicFilter.include
-    });
-    const matchesExclude = !satisfiesTopicFilter(topics, {
-      exclude: topicFilter.exclude
-    });
-    if (matchesExclude)
-      return false;
-    return matchesInclude;
-  }
-  return true;
-}
-function satisfiesForkFilter(allowForks, isFork) {
-  if (!allowForks && isFork)
-    return false;
-  return true;
-}
-
-async function getOrganizationRepositories(client, org, catalogPath) {
-  let relativeCatalogPathRef;
-  if (catalogPath.startsWith("/")) {
-    relativeCatalogPathRef = catalogPath.substring(1);
-  } else {
-    relativeCatalogPathRef = catalogPath;
-  }
-  const catalogPathRef = `HEAD:${relativeCatalogPathRef}`;
-  const query = `
-    query repositories($org: String!, $catalogPathRef: String!, $cursor: String) {
-      repositoryOwner(login: $org) {
-        login
-        repositories(first: 100, after: $cursor) {
-          nodes {
-            name
-            catalogInfoFile: object(expression: $catalogPathRef) {
-              __typename
-              ... on Blob {
-                id
-                text
-              }
-            }
-            url
-            isArchived
-            isFork
-            repositoryTopics(first: 100) {
-              nodes {
-                ... on RepositoryTopic {
-                  topic {
-                    name
-                  }
-                }
-              }
-            }
-            defaultBranchRef {
-              name
-            }
-          }
-          pageInfo {
-            hasNextPage
-            endCursor
-          }
-        }
-      }
-    }`;
-  const repositories = await queryWithPaging(
-    client,
-    query,
-    org,
-    (r) => {
-      var _a;
-      return (_a = r.repositoryOwner) == null ? void 0 : _a.repositories;
-    },
-    async (x) => x,
-    { org, catalogPathRef }
-  );
-  return { repositories };
-}
-async function queryWithPaging(client, query, org, connection, transformer, variables) {
-  const result = [];
-  let cursor = void 0;
-  for (let j = 0; j < 1e3; ++j) {
-    const response = await client(query, {
-      ...variables,
-      cursor
-    });
-    const conn = connection(response);
-    if (!conn) {
-      throw new Error(`Found no match for ${JSON.stringify(variables)}`);
-    }
-    for (const node of conn.nodes) {
-      const transformedNode = await transformer(node, {
-        client,
-        query,
-        org
-      });
-      if (transformedNode) {
-        result.push(transformedNode);
-      }
-    }
-    if (!conn.pageInfo.hasNextPage) {
-      break;
-    } else {
-      cursor = conn.pageInfo.endCursor;
-    }
-  }
-  return result;
-}
-
-const TOPIC_REPO_PUSH = "github.push";
-class GithubEntityProvider {
-  static fromConfig(config, options) {
-    if (!options.schedule && !options.scheduler) {
-      throw new Error("Either schedule or scheduler must be provided.");
-    }
-    const integrations = integration.ScmIntegrations.fromConfig(config);
-    return readProviderConfigs(config).map((providerConfig) => {
-      var _a;
-      const integrationHost = providerConfig.host;
-      const integration = integrations.github.byHost(integrationHost);
-      if (!integration) {
-        throw new Error(
-          `There is no GitHub config that matches host ${integrationHost}. Please add a configuration entry for it under integrations.github`
-        );
-      }
-      if (!options.schedule && !providerConfig.schedule) {
-        throw new Error(
-          `No schedule provided neither via code nor config for github-provider:${providerConfig.id}.`
-        );
-      }
-      const taskRunner = (_a = options.schedule) != null ? _a : options.scheduler.createScheduledTaskRunner(providerConfig.schedule);
-      return new GithubEntityProvider(
-        providerConfig,
-        integration,
-        options.logger,
-        taskRunner
-      );
-    });
-  }
-  constructor(config, integration$1, logger, taskRunner) {
-    this.config = config;
-    this.integration = integration$1.config;
-    this.logger = logger.child({
-      target: this.getProviderName()
-    });
-    this.scheduleFn = this.createScheduleFn(taskRunner);
-    this.githubCredentialsProvider = integration.SingleInstanceGithubCredentialsProvider.create(integration$1.config);
-  }
-  /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.getProviderName} */
-  getProviderName() {
-    return `github-provider:${this.config.id}`;
-  }
-  /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.connect} */
-  async connect(connection) {
-    this.connection = connection;
-    return await this.scheduleFn();
-  }
-  createScheduleFn(taskRunner) {
-    return async () => {
-      const taskId = `${this.getProviderName()}:refresh`;
-      return taskRunner.run({
-        id: taskId,
-        fn: async () => {
-          const logger = this.logger.child({
-            class: GithubEntityProvider.prototype.constructor.name,
-            taskId,
-            taskInstanceId: uuid__namespace.v4()
-          });
-          try {
-            await this.refresh(logger);
-          } catch (error) {
-            logger.error(`${this.getProviderName()} refresh failed`, error);
-          }
-        }
-      });
-    };
-  }
-  async refresh(logger) {
-    if (!this.connection) {
-      throw new Error("Not initialized");
-    }
-    const targets = await this.findCatalogFiles();
-    const matchingTargets = this.matchesFilters(targets);
-    const entities = matchingTargets.map((repository) => this.createLocationUrl(repository)).map(GithubEntityProvider.toLocationSpec).map((location) => {
-      return {
-        locationKey: this.getProviderName(),
-        entity: pluginCatalogNode.locationSpecToLocationEntity({ location })
-      };
-    });
-    await this.connection.applyMutation({
-      type: "full",
-      entities
-    });
-    logger.info(
-      `Read ${targets.length} GitHub repositories (${entities.length} matching the pattern)`
-    );
-  }
-  // go to the server and get all of the repositories
-  async findCatalogFiles() {
-    const organization = this.config.organization;
-    const host = this.integration.host;
-    const catalogPath = this.config.catalogPath;
-    const orgUrl = `https://${host}/${organization}`;
-    const { headers } = await this.githubCredentialsProvider.getCredentials({
-      url: orgUrl
-    });
-    const client = graphql.graphql.defaults({
-      baseUrl: this.integration.apiBaseUrl,
-      headers
-    });
-    const { repositories: repositoriesFromGithub } = await getOrganizationRepositories(client, organization, catalogPath);
-    const repositories = repositoriesFromGithub.map((r) => {
-      var _a, _b;
-      return {
-        url: r.url,
-        name: r.name,
-        defaultBranchRef: (_a = r.defaultBranchRef) == null ? void 0 : _a.name,
-        repositoryTopics: r.repositoryTopics.nodes.map((t) => t.topic.name),
-        isArchived: r.isArchived,
-        isFork: r.isFork,
-        isCatalogInfoFilePresent: ((_b = r.catalogInfoFile) == null ? void 0 : _b.__typename) === "Blob" && r.catalogInfoFile.text !== ""
-      };
-    });
-    if (this.config.validateLocationsExist) {
-      return repositories.filter(
-        (repository) => repository.isCatalogInfoFilePresent
-      );
-    }
-    return repositories;
-  }
-  matchesFilters(repositories) {
-    var _a, _b, _c, _d;
-    const repositoryFilter = (_a = this.config.filters) == null ? void 0 : _a.repository;
-    const topicFilters = (_b = this.config.filters) == null ? void 0 : _b.topic;
-    const allowForks = (_d = (_c = this.config.filters) == null ? void 0 : _c.allowForks) != null ? _d : true;
-    const matchingRepositories = repositories.filter((r) => {
-      const repoTopics = r.repositoryTopics;
-      return !r.isArchived && (!repositoryFilter || repositoryFilter.test(r.name)) && satisfiesTopicFilter(repoTopics, topicFilters) && satisfiesForkFilter(allowForks, r.isFork) && r.defaultBranchRef;
-    });
-    return matchingRepositories;
-  }
-  createLocationUrl(repository) {
-    var _a;
-    const branch = ((_a = this.config.filters) == null ? void 0 : _a.branch) || repository.defaultBranchRef || "-";
-    const catalogFile = this.config.catalogPath.startsWith("/") ? this.config.catalogPath.substring(1) : this.config.catalogPath;
-    return `${repository.url}/blob/${branch}/${catalogFile}`;
-  }
-  static toLocationSpec(target) {
-    return {
-      type: "url",
-      target,
-      presence: "optional"
-    };
-  }
-  /** {@inheritdoc @backstage/plugin-events-node#EventSubscriber.onEvent} */
-  async onEvent(params) {
-    this.logger.debug(`Received event from ${params.topic}`);
-    if (params.topic !== TOPIC_REPO_PUSH) {
-      return;
-    }
-    await this.onRepoPush(params.eventPayload);
-  }
-  /** {@inheritdoc @backstage/plugin-events-node#EventSubscriber.supportsEventTopics} */
-  supportsEventTopics() {
-    return [TOPIC_REPO_PUSH];
-  }
-  async onRepoPush(event) {
-    var _a;
-    if (!this.connection) {
-      throw new Error("Not initialized");
-    }
-    const repoName = event.repository.name;
-    const repoUrl = event.repository.url;
-    this.logger.debug(`handle github:push event for ${repoName} - ${repoUrl}`);
-    const branch = ((_a = this.config.filters) == null ? void 0 : _a.branch) || event.repository.default_branch;
-    if (!event.ref.includes(branch)) {
-      this.logger.debug(`skipping push event from ref ${event.ref}`);
-      return;
-    }
-    const repository = {
-      url: event.repository.url,
-      name: event.repository.name,
-      defaultBranchRef: event.repository.default_branch,
-      repositoryTopics: event.repository.topics,
-      isArchived: event.repository.archived,
-      isFork: event.repository.fork,
-      // we can consider this file present because
-      // only the catalog file will be recovered from the commits
-      isCatalogInfoFilePresent: true
-    };
-    const matchingTargets = this.matchesFilters([repository]);
-    if (matchingTargets.length === 0) {
-      this.logger.debug(
-        `skipping push event from repository ${repoName} because didn't match provider filters`
-      );
-      return;
-    }
-    const added = this.collectDeferredEntitiesFromCommit(
-      event.repository.url,
-      branch,
-      event.commits,
-      (commit) => [...commit.added]
-    );
-    const removed = this.collectDeferredEntitiesFromCommit(
-      event.repository.url,
-      branch,
-      event.commits,
-      (commit) => [...commit.removed]
-    );
-    const modified = this.collectFilesFromCommit(
-      event.commits,
-      (commit) => [...commit.modified]
-    );
-    if (modified.length > 0) {
-      await this.connection.refresh({
-        keys: [
-          ...modified.map(
-            (filePath) => `url:${event.repository.url}/tree/${branch}/${filePath}`
-          ),
-          ...modified.map(
-            (filePath) => `url:${event.repository.url}/blob/${branch}/${filePath}`
-          )
-        ]
-      });
-    }
-    if (added.length > 0 || removed.length > 0) {
-      await this.connection.applyMutation({
-        type: "delta",
-        added,
-        removed
-      });
-    }
-    this.logger.info(
-      `Processed Github push event: added ${added.length} - removed ${removed.length} - modified ${modified.length}`
-    );
-  }
-  collectDeferredEntitiesFromCommit(repositoryUrl, branch, commits, transformOperation) {
-    const catalogFiles = this.collectFilesFromCommit(
-      commits,
-      transformOperation
-    );
-    return this.toDeferredEntities(
-      catalogFiles.map(
-        (filePath) => `${repositoryUrl}/blob/${branch}/${filePath}`
-      )
-    );
-  }
-  collectFilesFromCommit(commits, transformOperation) {
-    const catalogFile = this.config.catalogPath.startsWith("/") ? this.config.catalogPath.substring(1) : this.config.catalogPath;
-    const matcher = new minimatch.Minimatch(catalogFile);
-    return commits.map(transformOperation).flat().filter((file) => matcher.match(file));
-  }
-  toDeferredEntities(targets) {
-    return targets.map((target) => {
-      const location = GithubEntityProvider.toLocationSpec(target);
-      return pluginCatalogNode.locationSpecToLocationEntity({ location });
-    }).map((entity) => {
-      return {
-        locationKey: this.getProviderName(),
-        entity
-      };
-    });
-  }
-}
+require('minimatch');
 
 const catalogModuleGithubEntityProvider = backendPluginApi.createBackendModule({
   pluginId: "catalog",
@@ -501,7 +28,7 @@ const catalogModuleGithubEntityProvider = backendPluginApi.createBackendModule({
       },
       async init({ catalog, config, logger, scheduler }) {
         catalog.addEntityProvider(
-          GithubEntityProvider.fromConfig(config, {
+          GithubEntityProvider.GithubEntityProvider.fromConfig(config, {
             logger: backendCommon.loggerToWinstonLogger(logger),
             scheduler
           })