@backstage/plugin-catalog-backend-module-github 0.0.0-nightly-20220311022539

package/CHANGELOG.md

@@ -0,0 +1,22 @@
+# @backstage/plugin-catalog-backend-module-github
+
+## 0.0.0-nightly-20220311022539
+
+### Patch Changes
+
+- Updated dependencies
+  - @backstage/catalog-model@0.0.0-nightly-20220311022539
+  - @backstage/plugin-catalog-backend@0.0.0-nightly-20220311022539
+
+## 0.1.0
+
+### Minor Changes
+
+- d4934e19b1: Added package, moving out GitHub specific functionality from the catalog-backend
+
+### Patch Changes
+
+- Updated dependencies
+  - @backstage/backend-common@0.13.0
+  - @backstage/plugin-catalog-backend@0.24.0
+  - @backstage/catalog-model@0.13.0

package/README.md

@@ -0,0 +1,8 @@
+# Catalog Backend Module for GitHub
+
+This is an extension module to the plugin-catalog-backend plugin, providing extensions targeted at GitHub offerings.
+
+## Getting started
+
+See [Backstage documentation](https://backstage.io/docs/integrations/github/discovery) for details on how to install
+and configure the plugin.

package/dist/index.cjs.js

@@ -0,0 +1,571 @@
+'use strict';
+
+Object.defineProperty(exports, '__esModule', { value: true });
+
+var integration = require('@backstage/integration');
+var pluginCatalogBackend = require('@backstage/plugin-catalog-backend');
+var graphql = require('@octokit/graphql');
+var catalogModel = require('@backstage/catalog-model');
+var lodash = require('lodash');
+
+function readGithubMultiOrgConfig(config) {
+  var _a;
+  const orgConfigs = (_a = config.getOptionalConfigArray("orgs")) != null ? _a : [];
+  return orgConfigs.map((c) => {
+    var _a2, _b;
+    return {
+      name: c.getString("name"),
+      groupNamespace: ((_a2 = c.getOptionalString("groupNamespace")) != null ? _a2 : c.getString("name")).toLowerCase(),
+      userNamespace: (_b = c.getOptionalString("userNamespace")) != null ? _b : void 0
+    };
+  });
+}
+
+async function getOrganizationUsers(client, org, tokenType, userNamespace) {
+  const query = `
+    query users($org: String!, $email: Boolean!, $cursor: String) {
+      organization(login: $org) {
+        membersWithRole(first: 100, after: $cursor) {
+          pageInfo { hasNextPage, endCursor }
+          nodes {
+            avatarUrl,
+            bio,
+            email @include(if: $email),
+            login,
+            name
+          }
+        }
+      }
+    }`;
+  const mapper = (user) => {
+    const entity = {
+      apiVersion: "backstage.io/v1alpha1",
+      kind: "User",
+      metadata: {
+        name: user.login,
+        annotations: {
+          "github.com/user-login": user.login
+        }
+      },
+      spec: {
+        profile: {},
+        memberOf: []
+      }
+    };
+    if (userNamespace)
+      entity.metadata.namespace = userNamespace;
+    if (user.bio)
+      entity.metadata.description = user.bio;
+    if (user.name)
+      entity.spec.profile.displayName = user.name;
+    if (user.email)
+      entity.spec.profile.email = user.email;
+    if (user.avatarUrl)
+      entity.spec.profile.picture = user.avatarUrl;
+    return entity;
+  };
+  const users = await queryWithPaging(client, query, (r) => {
+    var _a;
+    return (_a = r.organization) == null ? void 0 : _a.membersWithRole;
+  }, mapper, { org, email: tokenType === "token" });
+  return { users };
+}
+async function getOrganizationTeams(client, org, orgNamespace) {
+  const query = `
+    query teams($org: String!, $cursor: String) {
+      organization(login: $org) {
+        teams(first: 100, after: $cursor) {
+          pageInfo { hasNextPage, endCursor }
+          nodes {
+            slug
+            combinedSlug
+            name
+            description
+            avatarUrl
+            parentTeam { slug }
+            members(first: 100, membership: IMMEDIATE) {
+              pageInfo { hasNextPage }
+              nodes { login }
+            }
+          }
+        }
+      }
+    }`;
+  const groupMemberUsers = /* @__PURE__ */ new Map();
+  const mapper = async (team) => {
+    const entity = {
+      apiVersion: "backstage.io/v1alpha1",
+      kind: "Group",
+      metadata: {
+        name: team.slug,
+        annotations: {
+          "github.com/team-slug": team.combinedSlug
+        }
+      },
+      spec: {
+        type: "team",
+        profile: {},
+        children: []
+      }
+    };
+    if (orgNamespace) {
+      entity.metadata.namespace = orgNamespace;
+    }
+    if (team.description) {
+      entity.metadata.description = team.description;
+    }
+    if (team.name) {
+      entity.spec.profile.displayName = team.name;
+    }
+    if (team.avatarUrl) {
+      entity.spec.profile.picture = team.avatarUrl;
+    }
+    if (team.parentTeam) {
+      entity.spec.parent = team.parentTeam.slug;
+    }
+    const memberNames = [];
+    const groupKey = orgNamespace ? `${orgNamespace}/${team.slug}` : team.slug;
+    groupMemberUsers.set(groupKey, memberNames);
+    if (!team.members.pageInfo.hasNextPage) {
+      for (const user of team.members.nodes) {
+        memberNames.push(user.login);
+      }
+    } else {
+      const { members } = await getTeamMembers(client, org, team.slug);
+      for (const userLogin of members) {
+        memberNames.push(userLogin);
+      }
+    }
+    return entity;
+  };
+  const groups = await queryWithPaging(client, query, (r) => {
+    var _a;
+    return (_a = r.organization) == null ? void 0 : _a.teams;
+  }, mapper, { org });
+  return { groups, groupMemberUsers };
+}
+async function getOrganizationRepositories(client, org) {
+  const query = `
+    query repositories($org: String!, $cursor: String) {
+      repositoryOwner(login: $org) {
+        login
+        repositories(first: 100, after: $cursor) {
+          nodes {
+            name
+            url
+            isArchived
+            defaultBranchRef {
+              name
+            }
+          }
+          pageInfo {
+            hasNextPage
+            endCursor
+          }
+        }
+      }
+    }`;
+  const repositories = await queryWithPaging(client, query, (r) => {
+    var _a;
+    return (_a = r.repositoryOwner) == null ? void 0 : _a.repositories;
+  }, (x) => x, { org });
+  return { repositories };
+}
+async function getTeamMembers(client, org, teamSlug) {
+  const query = `
+    query members($org: String!, $teamSlug: String!, $cursor: String) {
+      organization(login: $org) {
+        team(slug: $teamSlug) {
+          members(first: 100, after: $cursor, membership: IMMEDIATE) {
+            pageInfo { hasNextPage, endCursor }
+            nodes { login }
+          }
+        }
+      }
+    }`;
+  const members = await queryWithPaging(client, query, (r) => {
+    var _a, _b;
+    return (_b = (_a = r.organization) == null ? void 0 : _a.team) == null ? void 0 : _b.members;
+  }, (user) => user.login, { org, teamSlug });
+  return { members };
+}
+async function queryWithPaging(client, query, connection, mapper, variables) {
+  const result = [];
+  let cursor = void 0;
+  for (let j = 0; j < 1e3; ++j) {
+    const response = await client(query, {
+      ...variables,
+      cursor
+    });
+    const conn = connection(response);
+    if (!conn) {
+      throw new Error(`Found no match for ${JSON.stringify(variables)}`);
+    }
+    for (const node of conn.nodes) {
+      result.push(await mapper(node));
+    }
+    if (!conn.pageInfo.hasNextPage) {
+      break;
+    } else {
+      cursor = conn.pageInfo.endCursor;
+    }
+  }
+  return result;
+}
+
+function buildOrgHierarchy(groups) {
+  const groupsByName = new Map(groups.map((g) => [g.metadata.name, g]));
+  for (const group of groups) {
+    const selfName = group.metadata.name;
+    const parentName = group.spec.parent;
+    if (parentName) {
+      const parent = groupsByName.get(parentName);
+      if (parent && !parent.spec.children.includes(selfName)) {
+        parent.spec.children.push(selfName);
+      }
+    }
+  }
+  for (const group of groups) {
+    const selfName = group.metadata.name;
+    for (const childName of group.spec.children) {
+      const child = groupsByName.get(childName);
+      if (child && !child.spec.parent) {
+        child.spec.parent = selfName;
+      }
+    }
+  }
+}
+function assignGroupsToUsers(users, groupMemberUsers) {
+  const usersByName = new Map(users.map((u) => [u.metadata.name, u]));
+  for (const [groupName, userNames] of groupMemberUsers.entries()) {
+    for (const userName of userNames) {
+      const user = usersByName.get(userName);
+      if (user && !user.spec.memberOf.includes(groupName)) {
+        user.spec.memberOf.push(groupName);
+      }
+    }
+  }
+}
+
+function parseGitHubOrgUrl(urlString) {
+  const path = new URL(urlString).pathname.substr(1).split("/");
+  if (path.length === 1 && path[0].length) {
+    return { org: decodeURIComponent(path[0]) };
+  }
+  throw new Error(`Expected a URL pointing to /<org>`);
+}
+
+class GithubDiscoveryProcessor {
+  static fromConfig(config, options) {
+    const integrations = integration.ScmIntegrations.fromConfig(config);
+    return new GithubDiscoveryProcessor({
+      ...options,
+      integrations
+    });
+  }
+  constructor(options) {
+    this.integrations = options.integrations;
+    this.logger = options.logger;
+    this.githubCredentialsProvider = options.githubCredentialsProvider || integration.DefaultGithubCredentialsProvider.fromIntegrations(this.integrations);
+  }
+  getProcessorName() {
+    return "GithubDiscoveryProcessor";
+  }
+  async readLocation(location, _optional, emit) {
+    var _a, _b;
+    if (location.type !== "github-discovery") {
+      return false;
+    }
+    const gitHubConfig = (_a = this.integrations.github.byUrl(location.target)) == null ? void 0 : _a.config;
+    if (!gitHubConfig) {
+      throw new Error(`There is no GitHub integration that matches ${location.target}. Please add a configuration entry for it under integrations.github`);
+    }
+    const { org, repoSearchPath, catalogPath, branch, host } = parseUrl(location.target);
+    const orgUrl = `https://${host}/${org}`;
+    const { headers } = await this.githubCredentialsProvider.getCredentials({
+      url: orgUrl
+    });
+    const client = graphql.graphql.defaults({
+      baseUrl: gitHubConfig.apiBaseUrl,
+      headers
+    });
+    const startTimestamp = Date.now();
+    this.logger.info(`Reading GitHub repositories from ${location.target}`);
+    const { repositories } = await getOrganizationRepositories(client, org);
+    const matching = repositories.filter((r) => !r.isArchived && repoSearchPath.test(r.name));
+    const duration = ((Date.now() - startTimestamp) / 1e3).toFixed(1);
+    this.logger.debug(`Read ${repositories.length} GitHub repositories (${matching.length} matching the pattern) in ${duration} seconds`);
+    for (const repository of matching) {
+      const branchName = branch === "-" ? (_b = repository.defaultBranchRef) == null ? void 0 : _b.name : branch;
+      if (!branchName) {
+        this.logger.info(`the repository ${repository.url} does not have a default branch, skipping`);
+        continue;
+      }
+      const path = `/blob/${branchName}${catalogPath}`;
+      emit(pluginCatalogBackend.processingResult.location({
+        type: "url",
+        target: `${repository.url}${path}`,
+        presence: "optional"
+      }));
+    }
+    return true;
+  }
+}
+function parseUrl(urlString) {
+  const url = new URL(urlString);
+  const path = url.pathname.substr(1).split("/");
+  if (path.length > 2 && path[0].length && path[1].length) {
+    return {
+      org: decodeURIComponent(path[0]),
+      repoSearchPath: escapeRegExp(decodeURIComponent(path[1])),
+      branch: decodeURIComponent(path[3]),
+      catalogPath: `/${decodeURIComponent(path.slice(4).join("/"))}`,
+      host: url.host
+    };
+  } else if (path.length === 1 && path[0].length) {
+    return {
+      org: decodeURIComponent(path[0]),
+      host: url.host,
+      repoSearchPath: escapeRegExp("*"),
+      catalogPath: "/catalog-info.yaml",
+      branch: "-"
+    };
+  }
+  throw new Error(`Failed to parse ${urlString}`);
+}
+function escapeRegExp(str) {
+  return new RegExp(`^${str.replace(/\*/g, ".*")}$`);
+}
+
+class GithubMultiOrgReaderProcessor {
+  static fromConfig(config, options) {
+    const c = config.getOptionalConfig("catalog.processors.githubMultiOrg");
+    const integrations = integration.ScmIntegrations.fromConfig(config);
+    return new GithubMultiOrgReaderProcessor({
+      ...options,
+      integrations,
+      orgs: c ? readGithubMultiOrgConfig(c) : []
+    });
+  }
+  constructor(options) {
+    this.integrations = options.integrations;
+    this.logger = options.logger;
+    this.orgs = options.orgs;
+    this.githubCredentialsProvider = options.githubCredentialsProvider || integration.DefaultGithubCredentialsProvider.fromIntegrations(this.integrations);
+  }
+  getProcessorName() {
+    return "GithubMultiOrgReaderProcessor";
+  }
+  async readLocation(location, _optional, emit) {
+    var _a, _b;
+    if (location.type !== "github-multi-org") {
+      return false;
+    }
+    const gitHubConfig = (_a = this.integrations.github.byUrl(location.target)) == null ? void 0 : _a.config;
+    if (!gitHubConfig) {
+      throw new Error(`There is no GitHub integration that matches ${location.target}. Please add a configuration entry for it under integrations.github`);
+    }
+    const allUsersMap = /* @__PURE__ */ new Map();
+    const baseUrl = new URL(location.target).origin;
+    const orgsToProcess = this.orgs.length ? this.orgs : await this.getAllOrgs(gitHubConfig);
+    for (const orgConfig of orgsToProcess) {
+      try {
+        const { headers, type: tokenType } = await this.githubCredentialsProvider.getCredentials({
+          url: `${baseUrl}/${orgConfig.name}`
+        });
+        const client = graphql.graphql.defaults({
+          baseUrl: gitHubConfig.apiBaseUrl,
+          headers
+        });
+        const startTimestamp = Date.now();
+        this.logger.info(`Reading GitHub users and teams for org: ${orgConfig.name}`);
+        const { users } = await getOrganizationUsers(client, orgConfig.name, tokenType, orgConfig.userNamespace);
+        const { groups, groupMemberUsers } = await getOrganizationTeams(client, orgConfig.name, orgConfig.groupNamespace);
+        const duration = ((Date.now() - startTimestamp) / 1e3).toFixed(1);
+        this.logger.debug(`Read ${users.length} GitHub users and ${groups.length} GitHub teams from ${orgConfig.name} in ${duration} seconds`);
+        let prefix = (_b = orgConfig.userNamespace) != null ? _b : "";
+        if (prefix.length > 0)
+          prefix += "/";
+        users.forEach((u) => {
+          if (!allUsersMap.has(prefix + u.metadata.name)) {
+            allUsersMap.set(prefix + u.metadata.name, u);
+          }
+        });
+        for (const [groupName, userNames] of groupMemberUsers.entries()) {
+          for (const userName of userNames) {
+            const user = allUsersMap.get(prefix + userName);
+            if (user && !user.spec.memberOf.includes(groupName)) {
+              user.spec.memberOf.push(groupName);
+            }
+          }
+        }
+        buildOrgHierarchy(groups);
+        for (const group of groups) {
+          emit(pluginCatalogBackend.processingResult.entity(location, group));
+        }
+      } catch (e) {
+        this.logger.error(`Failed to read GitHub org data for ${orgConfig.name}: ${e}`);
+      }
+    }
+    const allUsers = Array.from(allUsersMap.values());
+    for (const user of allUsers) {
+      emit(pluginCatalogBackend.processingResult.entity(location, user));
+    }
+    return true;
+  }
+  async getAllOrgs(gitHubConfig) {
+    const githubAppMux = new integration.GithubAppCredentialsMux(gitHubConfig);
+    const installs = await githubAppMux.getAllInstallations();
+    return installs.map((install) => install.target_type === "Organization" && install.account && install.account.login ? {
+      name: install.account.login,
+      groupNamespace: install.account.login.toLowerCase()
+    } : void 0).filter(Boolean);
+  }
+}
+
+class GitHubOrgEntityProvider {
+  constructor(options) {
+    this.options = options;
+    this.githubCredentialsProvider = options.githubCredentialsProvider || integration.SingleInstanceGithubCredentialsProvider.create(options.gitHubConfig);
+  }
+  static fromConfig(config, options) {
+    var _a;
+    const integrations = integration.ScmIntegrations.fromConfig(config);
+    const gitHubConfig = (_a = integrations.github.byUrl(options.orgUrl)) == null ? void 0 : _a.config;
+    if (!gitHubConfig) {
+      throw new Error(`There is no GitHub Org provider that matches ${options.orgUrl}. Please add a configuration for an integration.`);
+    }
+    const logger = options.logger.child({
+      target: options.orgUrl
+    });
+    return new GitHubOrgEntityProvider({
+      id: options.id,
+      orgUrl: options.orgUrl,
+      logger,
+      gitHubConfig,
+      githubCredentialsProvider: options.githubCredentialsProvider || integration.DefaultGithubCredentialsProvider.fromIntegrations(integrations)
+    });
+  }
+  getProviderName() {
+    return `GitHubOrgEntityProvider:${this.options.id}`;
+  }
+  async connect(connection) {
+    this.connection = connection;
+  }
+  async read() {
+    if (!this.connection) {
+      throw new Error("Not initialized");
+    }
+    const { markReadComplete } = trackProgress(this.options.logger);
+    const { headers, type: tokenType } = await this.githubCredentialsProvider.getCredentials({
+      url: this.options.orgUrl
+    });
+    const client = graphql.graphql.defaults({
+      baseUrl: this.options.gitHubConfig.apiBaseUrl,
+      headers
+    });
+    const { org } = parseGitHubOrgUrl(this.options.orgUrl);
+    const { users } = await getOrganizationUsers(client, org, tokenType);
+    const { groups, groupMemberUsers } = await getOrganizationTeams(client, org);
+    assignGroupsToUsers(users, groupMemberUsers);
+    buildOrgHierarchy(groups);
+    const { markCommitComplete } = markReadComplete({ users, groups });
+    await this.connection.applyMutation({
+      type: "full",
+      entities: [...users, ...groups].map((entity) => ({
+        locationKey: `github-org-provider:${this.options.id}`,
+        entity: withLocations(`https://${this.options.gitHubConfig.host}`, org, entity)
+      }))
+    });
+    markCommitComplete();
+  }
+}
+function trackProgress(logger) {
+  let timestamp = Date.now();
+  let summary;
+  logger.info("Reading GitHub users and groups");
+  function markReadComplete(read) {
+    summary = `${read.users.length} GitHub users and ${read.groups.length} GitHub groups`;
+    const readDuration = ((Date.now() - timestamp) / 1e3).toFixed(1);
+    timestamp = Date.now();
+    logger.info(`Read ${summary} in ${readDuration} seconds. Committing...`);
+    return { markCommitComplete };
+  }
+  function markCommitComplete() {
+    const commitDuration = ((Date.now() - timestamp) / 1e3).toFixed(1);
+    logger.info(`Committed ${summary} in ${commitDuration} seconds.`);
+  }
+  return { markReadComplete };
+}
+function withLocations(baseUrl, org, entity) {
+  const location = entity.kind === "Group" ? `url:${baseUrl}/orgs/${org}/teams/${entity.metadata.name}` : `url:${baseUrl}/${entity.metadata.name}`;
+  return lodash.merge({
+    metadata: {
+      annotations: {
+        [catalogModel.ANNOTATION_LOCATION]: location,
+        [catalogModel.ANNOTATION_ORIGIN_LOCATION]: location
+      }
+    }
+  }, entity);
+}
+
+class GithubOrgReaderProcessor {
+  static fromConfig(config, options) {
+    const integrations = integration.ScmIntegrations.fromConfig(config);
+    return new GithubOrgReaderProcessor({
+      ...options,
+      integrations
+    });
+  }
+  constructor(options) {
+    this.integrations = options.integrations;
+    this.githubCredentialsProvider = options.githubCredentialsProvider || integration.DefaultGithubCredentialsProvider.fromIntegrations(this.integrations);
+    this.logger = options.logger;
+  }
+  getProcessorName() {
+    return "GithubOrgReaderProcessor";
+  }
+  async readLocation(location, _optional, emit) {
+    if (location.type !== "github-org") {
+      return false;
+    }
+    const { client, tokenType } = await this.createClient(location.target);
+    const { org } = parseGitHubOrgUrl(location.target);
+    const startTimestamp = Date.now();
+    this.logger.info("Reading GitHub users and groups");
+    const { users } = await getOrganizationUsers(client, org, tokenType);
+    const { groups, groupMemberUsers } = await getOrganizationTeams(client, org);
+    const duration = ((Date.now() - startTimestamp) / 1e3).toFixed(1);
+    this.logger.debug(`Read ${users.length} GitHub users and ${groups.length} GitHub groups in ${duration} seconds`);
+    assignGroupsToUsers(users, groupMemberUsers);
+    buildOrgHierarchy(groups);
+    for (const group of groups) {
+      emit(pluginCatalogBackend.processingResult.entity(location, group));
+    }
+    for (const user of users) {
+      emit(pluginCatalogBackend.processingResult.entity(location, user));
+    }
+    return true;
+  }
+  async createClient(orgUrl) {
+    var _a;
+    const gitHubConfig = (_a = this.integrations.github.byUrl(orgUrl)) == null ? void 0 : _a.config;
+    if (!gitHubConfig) {
+      throw new Error(`There is no GitHub Org provider that matches ${orgUrl}. Please add a configuration for an integration.`);
+    }
+    const { headers, type: tokenType } = await this.githubCredentialsProvider.getCredentials({
+      url: orgUrl
+    });
+    const client = graphql.graphql.defaults({
+      baseUrl: gitHubConfig.apiBaseUrl,
+      headers
+    });
+    return { client, tokenType };
+  }
+}
+
+exports.GitHubOrgEntityProvider = GitHubOrgEntityProvider;
+exports.GithubDiscoveryProcessor = GithubDiscoveryProcessor;
+exports.GithubMultiOrgReaderProcessor = GithubMultiOrgReaderProcessor;
+exports.GithubOrgReaderProcessor = GithubOrgReaderProcessor;
+//# sourceMappingURL=index.cjs.js.map

package/dist/index.cjs.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.cjs.js","sources":["../src/lib/config.ts","../src/lib/github.ts","../src/lib/org.ts","../src/lib/util.ts","../src/GithubDiscoveryProcessor.ts","../src/GithubMultiOrgReaderProcessor.ts","../src/GitHubOrgEntityProvider.ts","../src/GithubOrgReaderProcessor.ts"],"sourcesContent":["/*\n * Copyright 2020 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Config } from '@backstage/config';\n\n/**\n * The configuration parameters for a multi-org GitHub processor.\n * @public\n */\nexport type GithubMultiOrgConfig = Array<{\n  /**\n   * The name of the GitHub org to process.\n   */\n  name: string;\n  /**\n   * The namespace of the group created for this org.\n   */\n  groupNamespace: string;\n  /**\n   * The namespace of the users created for this org. If not specified defaults to undefined.\n   */\n  userNamespace: string | undefined;\n}>;\n\nexport function readGithubMultiOrgConfig(config: Config): GithubMultiOrgConfig {\n  const orgConfigs = config.getOptionalConfigArray('orgs') ?? [];\n  return orgConfigs.map(c => ({\n    name: c.getString('name'),\n    groupNamespace: (\n      c.getOptionalString('groupNamespace') ?? c.getString('name')\n    ).toLowerCase(),\n    userNamespace: c.getOptionalString('userNamespace') ?? undefined,\n  }));\n}\n","/*\n * Copyright 2020 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { GroupEntity, UserEntity } from '@backstage/catalog-model';\nimport { GithubCredentialType } from '@backstage/integration';\nimport { graphql } from '@octokit/graphql';\n\n// Graphql types\n\nexport type QueryResponse = {\n  organization?: Organization;\n  repositoryOwner?: Organization | User;\n};\n\nexport type Organization = {\n  membersWithRole?: Connection<User>;\n  team?: Team;\n  teams?: Connection<Team>;\n  repositories?: Connection<Repository>;\n};\n\nexport type PageInfo = {\n  hasNextPage: boolean;\n  endCursor?: string;\n};\n\nexport type User = {\n  login: string;\n  bio?: string;\n  avatarUrl?: string;\n  email?: string;\n  name?: string;\n  repositories?: Connection<Repository>;\n};\n\nexport type Team = {\n  slug: string;\n  combinedSlug: string;\n  name?: string;\n  description?: string;\n  avatarUrl?: string;\n  parentTeam?: Team;\n  members: Connection<User>;\n};\n\nexport type Repository = {\n  name: string;\n  url: string;\n  isArchived: boolean;\n  defaultBranchRef: {\n    name: string;\n  } | null;\n};\n\nexport type Connection<T> = {\n  pageInfo: PageInfo;\n  nodes: T[];\n};\n\n/**\n * Gets all the users out of a GitHub organization.\n *\n * Note that the users will not have their memberships filled in.\n *\n * @param client - An octokit graphql client\n * @param org - The slug of the org to read\n */\nexport async function getOrganizationUsers(\n  client: typeof graphql,\n  org: string,\n  tokenType: GithubCredentialType,\n  userNamespace?: string,\n): Promise<{ users: UserEntity[] }> {\n  const query = `\n    query users($org: String!, $email: Boolean!, $cursor: String) {\n      organization(login: $org) {\n        membersWithRole(first: 100, after: $cursor) {\n          pageInfo { hasNextPage, endCursor }\n          nodes {\n            avatarUrl,\n            bio,\n            email @include(if: $email),\n            login,\n            name\n          }\n        }\n      }\n    }`;\n\n  // There is no user -> teams edge, so we leave the memberships empty for\n  // now and let the team iteration handle it instead\n  const mapper = (user: User) => {\n    const entity: UserEntity = {\n      apiVersion: 'backstage.io/v1alpha1',\n      kind: 'User',\n      metadata: {\n        name: user.login,\n        annotations: {\n          'github.com/user-login': user.login,\n        },\n      },\n      spec: {\n        profile: {},\n        memberOf: [],\n      },\n    };\n\n    if (userNamespace) entity.metadata.namespace = userNamespace;\n    if (user.bio) entity.metadata.description = user.bio;\n    if (user.name) entity.spec.profile!.displayName = user.name;\n    if (user.email) entity.spec.profile!.email = user.email;\n    if (user.avatarUrl) entity.spec.profile!.picture = user.avatarUrl;\n\n    return entity;\n  };\n\n  const users = await queryWithPaging(\n    client,\n    query,\n    r => r.organization?.membersWithRole,\n    mapper,\n    { org, email: tokenType === 'token' },\n  );\n\n  return { users };\n}\n\n/**\n * Gets all the teams out of a GitHub organization.\n *\n * Note that the teams will not have any relations apart from parent filled in.\n *\n * @param client - An octokit graphql client\n * @param org - The slug of the org to read\n */\nexport async function getOrganizationTeams(\n  client: typeof graphql,\n  org: string,\n  orgNamespace?: string,\n): Promise<{\n  groups: GroupEntity[];\n  groupMemberUsers: Map<string, string[]>;\n}> {\n  const query = `\n    query teams($org: String!, $cursor: String) {\n      organization(login: $org) {\n        teams(first: 100, after: $cursor) {\n          pageInfo { hasNextPage, endCursor }\n          nodes {\n            slug\n            combinedSlug\n            name\n            description\n            avatarUrl\n            parentTeam { slug }\n            members(first: 100, membership: IMMEDIATE) {\n              pageInfo { hasNextPage }\n              nodes { login }\n            }\n          }\n        }\n      }\n    }`;\n\n  // Gets populated inside the mapper below\n  const groupMemberUsers = new Map<string, string[]>();\n\n  const mapper = async (team: Team) => {\n    const entity: GroupEntity = {\n      apiVersion: 'backstage.io/v1alpha1',\n      kind: 'Group',\n      metadata: {\n        name: team.slug,\n        annotations: {\n          'github.com/team-slug': team.combinedSlug,\n        },\n      },\n      spec: {\n        type: 'team',\n        profile: {},\n        children: [],\n      },\n    };\n\n    if (orgNamespace) {\n      entity.metadata.namespace = orgNamespace;\n    }\n\n    if (team.description) {\n      entity.metadata.description = team.description;\n    }\n    if (team.name) {\n      entity.spec.profile!.displayName = team.name;\n    }\n    if (team.avatarUrl) {\n      entity.spec.profile!.picture = team.avatarUrl;\n    }\n    if (team.parentTeam) {\n      entity.spec.parent = team.parentTeam.slug;\n    }\n\n    const memberNames: string[] = [];\n    const groupKey = orgNamespace ? `${orgNamespace}/${team.slug}` : team.slug;\n    groupMemberUsers.set(groupKey, memberNames);\n\n    if (!team.members.pageInfo.hasNextPage) {\n      // We got all the members in one go, run the fast path\n      for (const user of team.members.nodes) {\n        memberNames.push(user.login);\n      }\n    } else {\n      // There were more than a hundred immediate members - run the slow\n      // path of fetching them explicitly\n      const { members } = await getTeamMembers(client, org, team.slug);\n      for (const userLogin of members) {\n        memberNames.push(userLogin);\n      }\n    }\n\n    return entity;\n  };\n\n  const groups = await queryWithPaging(\n    client,\n    query,\n    r => r.organization?.teams,\n    mapper,\n    { org },\n  );\n\n  return { groups, groupMemberUsers };\n}\n\nexport async function getOrganizationRepositories(\n  client: typeof graphql,\n  org: string,\n): Promise<{ repositories: Repository[] }> {\n  const query = `\n    query repositories($org: String!, $cursor: String) {\n      repositoryOwner(login: $org) {\n        login\n        repositories(first: 100, after: $cursor) {\n          nodes {\n            name\n            url\n            isArchived\n            defaultBranchRef {\n              name\n            }\n          }\n          pageInfo {\n            hasNextPage\n            endCursor\n          }\n        }\n      }\n    }`;\n\n  const repositories = await queryWithPaging(\n    client,\n    query,\n    r => r.repositoryOwner?.repositories,\n    x => x,\n    { org },\n  );\n\n  return { repositories };\n}\n\n/**\n * Gets all the users out of a GitHub organization.\n *\n * Note that the users will not have their memberships filled in.\n *\n * @param client - An octokit graphql client\n * @param org - The slug of the org to read\n * @param teamSlug - The slug of the team to read\n */\nexport async function getTeamMembers(\n  client: typeof graphql,\n  org: string,\n  teamSlug: string,\n): Promise<{ members: string[] }> {\n  const query = `\n    query members($org: String!, $teamSlug: String!, $cursor: String) {\n      organization(login: $org) {\n        team(slug: $teamSlug) {\n          members(first: 100, after: $cursor, membership: IMMEDIATE) {\n            pageInfo { hasNextPage, endCursor }\n            nodes { login }\n          }\n        }\n      }\n    }`;\n\n  const members = await queryWithPaging(\n    client,\n    query,\n    r => r.organization?.team?.members,\n    user => user.login,\n    { org, teamSlug },\n  );\n\n  return { members };\n}\n\n//\n// Helpers\n//\n\n/**\n * Assists in repeatedly executing a query with a paged response.\n *\n * Requires that the query accepts a $cursor variable.\n *\n * @param client - The octokit client\n * @param query - The query to execute\n * @param connection - A function that, given the response, picks out the actual\n *                   Connection object that's being iterated\n * @param mapper - A function that, given one of the nodes in the Connection,\n *               returns the model mapped form of it\n * @param variables - The variable values that the query needs, minus the cursor\n */\nexport async function queryWithPaging<\n  GraphqlType,\n  OutputType,\n  Variables extends {},\n  Response = QueryResponse,\n>(\n  client: typeof graphql,\n  query: string,\n  connection: (response: Response) => Connection<GraphqlType> | undefined,\n  mapper: (item: GraphqlType) => Promise<OutputType> | OutputType,\n  variables: Variables,\n): Promise<OutputType[]> {\n  const result: OutputType[] = [];\n\n  let cursor: string | undefined = undefined;\n  for (let j = 0; j < 1000 /* just for sanity */; ++j) {\n    const response: Response = await client(query, {\n      ...variables,\n      cursor,\n    });\n\n    const conn = connection(response);\n    if (!conn) {\n      throw new Error(`Found no match for ${JSON.stringify(variables)}`);\n    }\n\n    for (const node of conn.nodes) {\n      result.push(await mapper(node));\n    }\n\n    if (!conn.pageInfo.hasNextPage) {\n      break;\n    } else {\n      cursor = conn.pageInfo.endCursor;\n    }\n  }\n\n  return result;\n}\n","/*\n * Copyright 2020 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { GroupEntity, UserEntity } from '@backstage/catalog-model';\n\nexport function buildOrgHierarchy(groups: GroupEntity[]) {\n  const groupsByName = new Map(groups.map(g => [g.metadata.name, g]));\n\n  //\n  // Make sure that g.parent.children contain g\n  //\n\n  for (const group of groups) {\n    const selfName = group.metadata.name;\n    const parentName = group.spec.parent;\n    if (parentName) {\n      const parent = groupsByName.get(parentName);\n      if (parent && !parent.spec.children.includes(selfName)) {\n        parent.spec.children.push(selfName);\n      }\n    }\n  }\n\n  //\n  // Make sure that g.children.parent is g\n  //\n\n  for (const group of groups) {\n    const selfName = group.metadata.name;\n    for (const childName of group.spec.children) {\n      const child = groupsByName.get(childName);\n      if (child && !child.spec.parent) {\n        child.spec.parent = selfName;\n      }\n    }\n  }\n}\n\n// Ensure that users have their direct group memberships.\nexport function assignGroupsToUsers(\n  users: UserEntity[],\n  groupMemberUsers: Map<string, string[]>,\n) {\n  const usersByName = new Map(users.map(u => [u.metadata.name, u]));\n  for (const [groupName, userNames] of groupMemberUsers.entries()) {\n    for (const userName of userNames) {\n      const user = usersByName.get(userName);\n      if (user && !user.spec.memberOf.includes(groupName)) {\n        user.spec.memberOf.push(groupName);\n      }\n    }\n  }\n}\n\n// Ensure that users have their transitive group memberships. Requires that\n// the groups were previously processed with buildOrgHierarchy()\nexport function buildMemberOf(groups: GroupEntity[], users: UserEntity[]) {\n  const groupsByName = new Map(groups.map(g => [g.metadata.name, g]));\n\n  users.forEach(user => {\n    const transitiveMemberOf = new Set<string>();\n\n    const todo = [\n      ...user.spec.memberOf,\n      ...groups\n        .filter(g => g.spec.members?.includes(user.metadata.name))\n        .map(g => g.metadata.name),\n    ];\n\n    for (;;) {\n      const current = todo.pop();\n      if (!current) {\n        break;\n      }\n\n      if (!transitiveMemberOf.has(current)) {\n        transitiveMemberOf.add(current);\n        const group = groupsByName.get(current);\n        if (group?.spec.parent) {\n          todo.push(group.spec.parent);\n        }\n      }\n    }\n\n    user.spec.memberOf = [...transitiveMemberOf];\n  });\n}\n","/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nexport function parseGitHubOrgUrl(urlString: string): { org: string } {\n  const path = new URL(urlString).pathname.substr(1).split('/');\n\n  // /backstage\n  if (path.length === 1 && path[0].length) {\n    return { org: decodeURIComponent(path[0]) };\n  }\n\n  throw new Error(`Expected a URL pointing to /<org>`);\n}\n","/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Config } from '@backstage/config';\nimport {\n  DefaultGithubCredentialsProvider,\n  GithubCredentialsProvider,\n  ScmIntegrationRegistry,\n  ScmIntegrations,\n} from '@backstage/integration';\nimport {\n  CatalogProcessor,\n  CatalogProcessorEmit,\n  LocationSpec,\n  processingResult,\n} from '@backstage/plugin-catalog-backend';\nimport { graphql } from '@octokit/graphql';\nimport { Logger } from 'winston';\nimport { getOrganizationRepositories } from './lib';\n\n/**\n * Extracts repositories out of a GitHub org.\n *\n * The following will create locations for all projects which have a catalog-info.yaml\n * on the default branch. The first is shorthand for the second.\n *\n *    target: \"https://github.com/backstage\"\n *    or\n *    target: https://github.com/backstage/*\\/blob/-/catalog-info.yaml\n *\n * You may also explicitly specify the source branch:\n *\n *    target: https://github.com/backstage/*\\/blob/main/catalog-info.yaml\n *\n * @public\n **/\nexport class GithubDiscoveryProcessor implements CatalogProcessor {\n  private readonly integrations: ScmIntegrationRegistry;\n  private readonly logger: Logger;\n  private readonly githubCredentialsProvider: GithubCredentialsProvider;\n\n  static fromConfig(\n    config: Config,\n    options: {\n      logger: Logger;\n      githubCredentialsProvider?: GithubCredentialsProvider;\n    },\n  ) {\n    const integrations = ScmIntegrations.fromConfig(config);\n\n    return new GithubDiscoveryProcessor({\n      ...options,\n      integrations,\n    });\n  }\n\n  constructor(options: {\n    integrations: ScmIntegrationRegistry;\n    logger: Logger;\n    githubCredentialsProvider?: GithubCredentialsProvider;\n  }) {\n    this.integrations = options.integrations;\n    this.logger = options.logger;\n    this.githubCredentialsProvider =\n      options.githubCredentialsProvider ||\n      DefaultGithubCredentialsProvider.fromIntegrations(this.integrations);\n  }\n  getProcessorName(): string {\n    return 'GithubDiscoveryProcessor';\n  }\n\n  async readLocation(\n    location: LocationSpec,\n    _optional: boolean,\n    emit: CatalogProcessorEmit,\n  ): Promise<boolean> {\n    if (location.type !== 'github-discovery') {\n      return false;\n    }\n\n    const gitHubConfig = this.integrations.github.byUrl(\n      location.target,\n    )?.config;\n    if (!gitHubConfig) {\n      throw new Error(\n        `There is no GitHub integration that matches ${location.target}. Please add a configuration entry for it under integrations.github`,\n      );\n    }\n\n    const { org, repoSearchPath, catalogPath, branch, host } = parseUrl(\n      location.target,\n    );\n\n    // Building the org url here so that the github creds provider doesn't need to know\n    // about how to handle the wild card which is special for this processor.\n    const orgUrl = `https://${host}/${org}`;\n\n    const { headers } = await this.githubCredentialsProvider.getCredentials({\n      url: orgUrl,\n    });\n\n    const client = graphql.defaults({\n      baseUrl: gitHubConfig.apiBaseUrl,\n      headers,\n    });\n\n    // Read out all of the raw data\n    const startTimestamp = Date.now();\n    this.logger.info(`Reading GitHub repositories from ${location.target}`);\n\n    const { repositories } = await getOrganizationRepositories(client, org);\n    const matching = repositories.filter(\n      r => !r.isArchived && repoSearchPath.test(r.name),\n    );\n\n    const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1);\n    this.logger.debug(\n      `Read ${repositories.length} GitHub repositories (${matching.length} matching the pattern) in ${duration} seconds`,\n    );\n\n    for (const repository of matching) {\n      const branchName =\n        branch === '-' ? repository.defaultBranchRef?.name : branch;\n\n      if (!branchName) {\n        this.logger.info(\n          `the repository ${repository.url} does not have a default branch, skipping`,\n        );\n        continue;\n      }\n\n      const path = `/blob/${branchName}${catalogPath}`;\n\n      emit(\n        processingResult.location({\n          type: 'url',\n          target: `${repository.url}${path}`,\n          // Not all locations may actually exist, since the user defined them as a wildcard pattern.\n          // Thus, we emit them as optional and let the downstream processor find them while not outputting\n          // an error if it couldn't.\n          presence: 'optional',\n        }),\n      );\n    }\n\n    return true;\n  }\n}\n\n/*\n * Helpers\n */\n\nexport function parseUrl(urlString: string): {\n  org: string;\n  repoSearchPath: RegExp;\n  catalogPath: string;\n  branch: string;\n  host: string;\n} {\n  const url = new URL(urlString);\n  const path = url.pathname.substr(1).split('/');\n\n  // /backstage/techdocs-*/blob/master/catalog-info.yaml\n  // can also be\n  // /backstage\n  if (path.length > 2 && path[0].length && path[1].length) {\n    return {\n      org: decodeURIComponent(path[0]),\n      repoSearchPath: escapeRegExp(decodeURIComponent(path[1])),\n      branch: decodeURIComponent(path[3]),\n      catalogPath: `/${decodeURIComponent(path.slice(4).join('/'))}`,\n      host: url.host,\n    };\n  } else if (path.length === 1 && path[0].length) {\n    return {\n      org: decodeURIComponent(path[0]),\n      host: url.host,\n      repoSearchPath: escapeRegExp('*'),\n      catalogPath: '/catalog-info.yaml',\n      branch: '-',\n    };\n  }\n\n  throw new Error(`Failed to parse ${urlString}`);\n}\n\nexport function escapeRegExp(str: string): RegExp {\n  return new RegExp(`^${str.replace(/\\*/g, '.*')}$`);\n}\n","/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Config } from '@backstage/config';\nimport {\n  DefaultGithubCredentialsProvider,\n  GithubAppCredentialsMux,\n  GithubCredentialsProvider,\n  GitHubIntegrationConfig,\n  ScmIntegrationRegistry,\n  ScmIntegrations,\n} from '@backstage/integration';\nimport {\n  CatalogProcessor,\n  CatalogProcessorEmit,\n  LocationSpec,\n  processingResult,\n} from '@backstage/plugin-catalog-backend';\nimport { graphql } from '@octokit/graphql';\nimport { Logger } from 'winston';\nimport {\n  buildOrgHierarchy,\n  getOrganizationTeams,\n  getOrganizationUsers,\n  GithubMultiOrgConfig,\n  readGithubMultiOrgConfig,\n} from './lib';\n\n/**\n * Extracts teams and users out of a multiple GitHub orgs namespaced per org.\n *\n * Be aware that this processor may not be compatible with future org structures in the catalog.\n *\n * @public\n */\nexport class GithubMultiOrgReaderProcessor implements CatalogProcessor {\n  private readonly integrations: ScmIntegrationRegistry;\n  private readonly orgs: GithubMultiOrgConfig;\n  private readonly logger: Logger;\n  private readonly githubCredentialsProvider: GithubCredentialsProvider;\n\n  static fromConfig(\n    config: Config,\n    options: {\n      logger: Logger;\n      githubCredentialsProvider?: GithubCredentialsProvider;\n    },\n  ) {\n    const c = config.getOptionalConfig('catalog.processors.githubMultiOrg');\n    const integrations = ScmIntegrations.fromConfig(config);\n\n    return new GithubMultiOrgReaderProcessor({\n      ...options,\n      integrations,\n      orgs: c ? readGithubMultiOrgConfig(c) : [],\n    });\n  }\n\n  constructor(options: {\n    integrations: ScmIntegrationRegistry;\n    logger: Logger;\n    orgs: GithubMultiOrgConfig;\n    githubCredentialsProvider?: GithubCredentialsProvider;\n  }) {\n    this.integrations = options.integrations;\n    this.logger = options.logger;\n    this.orgs = options.orgs;\n    this.githubCredentialsProvider =\n      options.githubCredentialsProvider ||\n      DefaultGithubCredentialsProvider.fromIntegrations(this.integrations);\n  }\n  getProcessorName(): string {\n    return 'GithubMultiOrgReaderProcessor';\n  }\n\n  async readLocation(\n    location: LocationSpec,\n    _optional: boolean,\n    emit: CatalogProcessorEmit,\n  ): Promise<boolean> {\n    if (location.type !== 'github-multi-org') {\n      return false;\n    }\n\n    const gitHubConfig = this.integrations.github.byUrl(\n      location.target,\n    )?.config;\n    if (!gitHubConfig) {\n      throw new Error(\n        `There is no GitHub integration that matches ${location.target}. Please add a configuration entry for it under integrations.github`,\n      );\n    }\n\n    const allUsersMap = new Map();\n    const baseUrl = new URL(location.target).origin;\n\n    const orgsToProcess = this.orgs.length\n      ? this.orgs\n      : await this.getAllOrgs(gitHubConfig);\n\n    for (const orgConfig of orgsToProcess) {\n      try {\n        const { headers, type: tokenType } =\n          await this.githubCredentialsProvider.getCredentials({\n            url: `${baseUrl}/${orgConfig.name}`,\n          });\n        const client = graphql.defaults({\n          baseUrl: gitHubConfig.apiBaseUrl,\n          headers,\n        });\n\n        const startTimestamp = Date.now();\n        this.logger.info(\n          `Reading GitHub users and teams for org: ${orgConfig.name}`,\n        );\n        const { users } = await getOrganizationUsers(\n          client,\n          orgConfig.name,\n          tokenType,\n          orgConfig.userNamespace,\n        );\n        const { groups, groupMemberUsers } = await getOrganizationTeams(\n          client,\n          orgConfig.name,\n          orgConfig.groupNamespace,\n        );\n\n        const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1);\n        this.logger.debug(\n          `Read ${users.length} GitHub users and ${groups.length} GitHub teams from ${orgConfig.name} in ${duration} seconds`,\n        );\n\n        let prefix: string = orgConfig.userNamespace ?? '';\n        if (prefix.length > 0) prefix += '/';\n\n        users.forEach(u => {\n          if (!allUsersMap.has(prefix + u.metadata.name)) {\n            allUsersMap.set(prefix + u.metadata.name, u);\n          }\n        });\n\n        for (const [groupName, userNames] of groupMemberUsers.entries()) {\n          for (const userName of userNames) {\n            const user = allUsersMap.get(prefix + userName);\n            if (user && !user.spec.memberOf.includes(groupName)) {\n              user.spec.memberOf.push(groupName);\n            }\n          }\n        }\n        buildOrgHierarchy(groups);\n\n        for (const group of groups) {\n          emit(processingResult.entity(location, group));\n        }\n      } catch (e) {\n        this.logger.error(\n          `Failed to read GitHub org data for ${orgConfig.name}: ${e}`,\n        );\n      }\n    }\n\n    const allUsers = Array.from(allUsersMap.values());\n    for (const user of allUsers) {\n      emit(processingResult.entity(location, user));\n    }\n\n    return true;\n  }\n\n  // Note: Does not support usage of PATs\n  private async getAllOrgs(\n    gitHubConfig: GitHubIntegrationConfig,\n  ): Promise<GithubMultiOrgConfig> {\n    const githubAppMux = new GithubAppCredentialsMux(gitHubConfig);\n    const installs = await githubAppMux.getAllInstallations();\n\n    return installs\n      .map(install =>\n        install.target_type === 'Organization' &&\n        install.account &&\n        install.account.login\n          ? {\n              name: install.account.login,\n              groupNamespace: install.account.login.toLowerCase(),\n            }\n          : undefined,\n      )\n      .filter(Boolean) as GithubMultiOrgConfig;\n  }\n}\n","/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport {\n  ANNOTATION_LOCATION,\n  ANNOTATION_ORIGIN_LOCATION,\n  Entity,\n} from '@backstage/catalog-model';\nimport { Config } from '@backstage/config';\nimport {\n  DefaultGithubCredentialsProvider,\n  GithubCredentialsProvider,\n  GitHubIntegrationConfig,\n  ScmIntegrations,\n  SingleInstanceGithubCredentialsProvider,\n} from '@backstage/integration';\nimport {\n  EntityProvider,\n  EntityProviderConnection,\n} from '@backstage/plugin-catalog-backend';\nimport { graphql } from '@octokit/graphql';\nimport { merge } from 'lodash';\nimport { Logger } from 'winston';\nimport {\n  assignGroupsToUsers,\n  buildOrgHierarchy,\n  getOrganizationTeams,\n  getOrganizationUsers,\n  parseGitHubOrgUrl,\n} from './lib';\n\n// TODO: Consider supporting an (optional) webhook that reacts on org changes\n/** @public */\nexport class GitHubOrgEntityProvider implements EntityProvider {\n  private connection?: EntityProviderConnection;\n  private githubCredentialsProvider: GithubCredentialsProvider;\n\n  static fromConfig(\n    config: Config,\n    options: {\n      id: string;\n      orgUrl: string;\n      logger: Logger;\n      githubCredentialsProvider?: GithubCredentialsProvider;\n    },\n  ) {\n    const integrations = ScmIntegrations.fromConfig(config);\n    const gitHubConfig = integrations.github.byUrl(options.orgUrl)?.config;\n\n    if (!gitHubConfig) {\n      throw new Error(\n        `There is no GitHub Org provider that matches ${options.orgUrl}. Please add a configuration for an integration.`,\n      );\n    }\n\n    const logger = options.logger.child({\n      target: options.orgUrl,\n    });\n\n    return new GitHubOrgEntityProvider({\n      id: options.id,\n      orgUrl: options.orgUrl,\n      logger,\n      gitHubConfig,\n      githubCredentialsProvider:\n        options.githubCredentialsProvider ||\n        DefaultGithubCredentialsProvider.fromIntegrations(integrations),\n    });\n  }\n\n  constructor(\n    private options: {\n      id: string;\n      orgUrl: string;\n      gitHubConfig: GitHubIntegrationConfig;\n      logger: Logger;\n      githubCredentialsProvider?: GithubCredentialsProvider;\n    },\n  ) {\n    this.githubCredentialsProvider =\n      options.githubCredentialsProvider ||\n      SingleInstanceGithubCredentialsProvider.create(options.gitHubConfig);\n  }\n\n  getProviderName() {\n    return `GitHubOrgEntityProvider:${this.options.id}`;\n  }\n\n  async connect(connection: EntityProviderConnection) {\n    this.connection = connection;\n  }\n\n  async read() {\n    if (!this.connection) {\n      throw new Error('Not initialized');\n    }\n\n    const { markReadComplete } = trackProgress(this.options.logger);\n\n    const { headers, type: tokenType } =\n      await this.githubCredentialsProvider.getCredentials({\n        url: this.options.orgUrl,\n      });\n    const client = graphql.defaults({\n      baseUrl: this.options.gitHubConfig.apiBaseUrl,\n      headers,\n    });\n\n    const { org } = parseGitHubOrgUrl(this.options.orgUrl);\n    const { users } = await getOrganizationUsers(client, org, tokenType);\n    const { groups, groupMemberUsers } = await getOrganizationTeams(\n      client,\n      org,\n    );\n    assignGroupsToUsers(users, groupMemberUsers);\n    buildOrgHierarchy(groups);\n\n    const { markCommitComplete } = markReadComplete({ users, groups });\n\n    await this.connection.applyMutation({\n      type: 'full',\n      entities: [...users, ...groups].map(entity => ({\n        locationKey: `github-org-provider:${this.options.id}`,\n        entity: withLocations(\n          `https://${this.options.gitHubConfig.host}`,\n          org,\n          entity,\n        ),\n      })),\n    });\n\n    markCommitComplete();\n  }\n}\n\n// Helps wrap the timing and logging behaviors\nfunction trackProgress(logger: Logger) {\n  let timestamp = Date.now();\n  let summary: string;\n\n  logger.info('Reading GitHub users and groups');\n\n  function markReadComplete(read: { users: unknown[]; groups: unknown[] }) {\n    summary = `${read.users.length} GitHub users and ${read.groups.length} GitHub groups`;\n    const readDuration = ((Date.now() - timestamp) / 1000).toFixed(1);\n    timestamp = Date.now();\n    logger.info(`Read ${summary} in ${readDuration} seconds. Committing...`);\n    return { markCommitComplete };\n  }\n\n  function markCommitComplete() {\n    const commitDuration = ((Date.now() - timestamp) / 1000).toFixed(1);\n    logger.info(`Committed ${summary} in ${commitDuration} seconds.`);\n  }\n\n  return { markReadComplete };\n}\n\n// Makes sure that emitted entities have a proper location\nexport function withLocations(\n  baseUrl: string,\n  org: string,\n  entity: Entity,\n): Entity {\n  const location =\n    entity.kind === 'Group'\n      ? `url:${baseUrl}/orgs/${org}/teams/${entity.metadata.name}`\n      : `url:${baseUrl}/${entity.metadata.name}`;\n  return merge(\n    {\n      metadata: {\n        annotations: {\n          [ANNOTATION_LOCATION]: location,\n          [ANNOTATION_ORIGIN_LOCATION]: location,\n        },\n      },\n    },\n    entity,\n  ) as Entity;\n}\n","/*\n * Copyright 2020 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Config } from '@backstage/config';\nimport {\n  DefaultGithubCredentialsProvider,\n  GithubCredentialsProvider,\n  GithubCredentialType,\n  ScmIntegrationRegistry,\n  ScmIntegrations,\n} from '@backstage/integration';\nimport {\n  CatalogProcessor,\n  CatalogProcessorEmit,\n  LocationSpec,\n  processingResult,\n} from '@backstage/plugin-catalog-backend';\nimport { graphql } from '@octokit/graphql';\nimport { Logger } from 'winston';\nimport {\n  assignGroupsToUsers,\n  buildOrgHierarchy,\n  getOrganizationTeams,\n  getOrganizationUsers,\n  parseGitHubOrgUrl,\n} from './lib';\n\ntype GraphQL = typeof graphql;\n\n/**\n * Extracts teams and users out of a GitHub org.\n * @public\n */\nexport class GithubOrgReaderProcessor implements CatalogProcessor {\n  private readonly integrations: ScmIntegrationRegistry;\n  private readonly logger: Logger;\n  private readonly githubCredentialsProvider: GithubCredentialsProvider;\n\n  static fromConfig(\n    config: Config,\n    options: {\n      logger: Logger;\n      githubCredentialsProvider?: GithubCredentialsProvider;\n    },\n  ) {\n    const integrations = ScmIntegrations.fromConfig(config);\n\n    return new GithubOrgReaderProcessor({\n      ...options,\n      integrations,\n    });\n  }\n\n  constructor(options: {\n    integrations: ScmIntegrationRegistry;\n    logger: Logger;\n    githubCredentialsProvider?: GithubCredentialsProvider;\n  }) {\n    this.integrations = options.integrations;\n    this.githubCredentialsProvider =\n      options.githubCredentialsProvider ||\n      DefaultGithubCredentialsProvider.fromIntegrations(this.integrations);\n    this.logger = options.logger;\n  }\n  getProcessorName(): string {\n    return 'GithubOrgReaderProcessor';\n  }\n\n  async readLocation(\n    location: LocationSpec,\n    _optional: boolean,\n    emit: CatalogProcessorEmit,\n  ): Promise<boolean> {\n    if (location.type !== 'github-org') {\n      return false;\n    }\n\n    const { client, tokenType } = await this.createClient(location.target);\n    const { org } = parseGitHubOrgUrl(location.target);\n\n    // Read out all of the raw data\n    const startTimestamp = Date.now();\n    this.logger.info('Reading GitHub users and groups');\n\n    const { users } = await getOrganizationUsers(client, org, tokenType);\n    const { groups, groupMemberUsers } = await getOrganizationTeams(\n      client,\n      org,\n    );\n\n    const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1);\n    this.logger.debug(\n      `Read ${users.length} GitHub users and ${groups.length} GitHub groups in ${duration} seconds`,\n    );\n\n    assignGroupsToUsers(users, groupMemberUsers);\n    buildOrgHierarchy(groups);\n\n    // Done!\n    for (const group of groups) {\n      emit(processingResult.entity(location, group));\n    }\n    for (const user of users) {\n      emit(processingResult.entity(location, user));\n    }\n\n    return true;\n  }\n\n  private async createClient(\n    orgUrl: string,\n  ): Promise<{ client: GraphQL; tokenType: GithubCredentialType }> {\n    const gitHubConfig = this.integrations.github.byUrl(orgUrl)?.config;\n\n    if (!gitHubConfig) {\n      throw new Error(\n        `There is no GitHub Org provider that matches ${orgUrl}. Please add a configuration for an integration.`,\n      );\n    }\n\n    const { headers, type: tokenType } =\n      await this.githubCredentialsProvider.getCredentials({\n        url: orgUrl,\n      });\n\n    const client = graphql.defaults({\n      baseUrl: gitHubConfig.apiBaseUrl,\n      headers,\n    });\n\n    return { client, tokenType };\n  }\n}\n"],"names":["ScmIntegrations","DefaultGithubCredentialsProvider","graphql","processingResult","GithubAppCredentialsMux","SingleInstanceGithubCredentialsProvider","merge","ANNOTATION_LOCATION","ANNOTATION_ORIGIN_LOCATION"],"mappings":";;;;;;;;;;kCAqCyC,QAAsC;AArC/E;AAsCE,QAAM,aAAa,aAAO,uBAAuB,YAA9B,YAAyC;AAC5D,SAAO,WAAW,IAAI,OAAE;AAvC1B;AAuC8B;AAAA,MAC1B,MAAM,EAAE,UAAU;AAAA,MAClB,gBACE,UAAE,kBAAkB,sBAApB,aAAyC,EAAE,UAAU,SACrD;AAAA,MACF,eAAe,QAAE,kBAAkB,qBAApB,YAAwC;AAAA;AAAA;AAAA;;oCCqCzD,QACA,KACA,WACA,eACkC;AAClC,QAAM,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAkBd,QAAM,SAAS,CAAC,SAAe;AAC7B,UAAM,SAAqB;AAAA,MACzB,YAAY;AAAA,MACZ,MAAM;AAAA,MACN,UAAU;AAAA,QACR,MAAM,KAAK;AAAA,QACX,aAAa;AAAA,UACX,yBAAyB,KAAK;AAAA;AAAA;AAAA,MAGlC,MAAM;AAAA,QACJ,SAAS;AAAA,QACT,UAAU;AAAA;AAAA;AAId,QAAI;AAAe,aAAO,SAAS,YAAY;AAC/C,QAAI,KAAK;AAAK,aAAO,SAAS,cAAc,KAAK;AACjD,QAAI,KAAK;AAAM,aAAO,KAAK,QAAS,cAAc,KAAK;AACvD,QAAI,KAAK;AAAO,aAAO,KAAK,QAAS,QAAQ,KAAK;AAClD,QAAI,KAAK;AAAW,aAAO,KAAK,QAAS,UAAU,KAAK;AAExD,WAAO;AAAA;AAGT,QAAM,QAAQ,MAAM,gBAClB,QACA,OACA,OAAE;AApIN;AAoIS,mBAAE,iBAAF,mBAAgB;AAAA,KACrB,QACA,EAAE,KAAK,OAAO,cAAc;AAG9B,SAAO,EAAE;AAAA;oCAYT,QACA,KACA,cAIC;AACD,QAAM,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAsBd,QAAM,uCAAuB;AAE7B,QAAM,SAAS,OAAO,SAAe;AACnC,UAAM,SAAsB;AAAA,MAC1B,YAAY;AAAA,MACZ,MAAM;AAAA,MACN,UAAU;AAAA,QACR,MAAM,KAAK;AAAA,QACX,aAAa;AAAA,UACX,wBAAwB,KAAK;AAAA;AAAA;AAAA,MAGjC,MAAM;AAAA,QACJ,MAAM;AAAA,QACN,SAAS;AAAA,QACT,UAAU;AAAA;AAAA;AAId,QAAI,cAAc;AAChB,aAAO,SAAS,YAAY;AAAA;AAG9B,QAAI,KAAK,aAAa;AACpB,aAAO,SAAS,cAAc,KAAK;AAAA;AAErC,QAAI,KAAK,MAAM;AACb,aAAO,KAAK,QAAS,cAAc,KAAK;AAAA;AAE1C,QAAI,KAAK,WAAW;AAClB,aAAO,KAAK,QAAS,UAAU,KAAK;AAAA;AAEtC,QAAI,KAAK,YAAY;AACnB,aAAO,KAAK,SAAS,KAAK,WAAW;AAAA;AAGvC,UAAM,cAAwB;AAC9B,UAAM,WAAW,eAAe,GAAG,gBAAgB,KAAK,SAAS,KAAK;AACtE,qBAAiB,IAAI,UAAU;AAE/B,QAAI,CAAC,KAAK,QAAQ,SAAS,aAAa;AAEtC,iBAAW,QAAQ,KAAK,QAAQ,OAAO;AACrC,oBAAY,KAAK,KAAK;AAAA;AAAA,WAEnB;AAGL,YAAM,EAAE,YAAY,MAAM,eAAe,QAAQ,KAAK,KAAK;AAC3D,iBAAW,aAAa,SAAS;AAC/B,oBAAY,KAAK;AAAA;AAAA;AAIrB,WAAO;AAAA;AAGT,QAAM,SAAS,MAAM,gBACnB,QACA,OACA,OAAE;AA9ON;AA8OS,mBAAE,iBAAF,mBAAgB;AAAA,KACrB,QACA,EAAE;AAGJ,SAAO,EAAE,QAAQ;AAAA;2CAIjB,QACA,KACyC;AACzC,QAAM,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAqBd,QAAM,eAAe,MAAM,gBACzB,QACA,OACA,OAAE;AAlRN;AAkRS,mBAAE,oBAAF,mBAAmB;AAAA,KACxB,OAAK,GACL,EAAE;AAGJ,SAAO,EAAE;AAAA;8BAaT,QACA,KACA,UACgC;AAChC,QAAM,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAYd,QAAM,UAAU,MAAM,gBACpB,QACA,OACA,OAAE;AAvTN;AAuTS,yBAAE,iBAAF,mBAAgB,SAAhB,mBAAsB;AAAA,KAC3B,UAAQ,KAAK,OACb,EAAE,KAAK;AAGT,SAAO,EAAE;AAAA;+BA0BT,QACA,OACA,YACA,QACA,WACuB;AACvB,QAAM,SAAuB;AAE7B,MAAI,SAA6B;AACjC,WAAS,IAAI,GAAG,IAAI,KAA4B,EAAE,GAAG;AACnD,UAAM,WAAqB,MAAM,OAAO,OAAO;AAAA,SAC1C;AAAA,MACH;AAAA;AAGF,UAAM,OAAO,WAAW;AACxB,QAAI,CAAC,MAAM;AACT,YAAM,IAAI,MAAM,sBAAsB,KAAK,UAAU;AAAA;AAGvD,eAAW,QAAQ,KAAK,OAAO;AAC7B,aAAO,KAAK,MAAM,OAAO;AAAA;AAG3B,QAAI,CAAC,KAAK,SAAS,aAAa;AAC9B;AAAA,WACK;AACL,eAAS,KAAK,SAAS;AAAA;AAAA;AAI3B,SAAO;AAAA;;2BCnWyB,QAAuB;AACvD,QAAM,eAAe,IAAI,IAAI,OAAO,IAAI,OAAK,CAAC,EAAE,SAAS,MAAM;AAM/D,aAAW,SAAS,QAAQ;AAC1B,UAAM,WAAW,MAAM,SAAS;AAChC,UAAM,aAAa,MAAM,KAAK;AAC9B,QAAI,YAAY;AACd,YAAM,SAAS,aAAa,IAAI;AAChC,UAAI,UAAU,CAAC,OAAO,KAAK,SAAS,SAAS,WAAW;AACtD,eAAO,KAAK,SAAS,KAAK;AAAA;AAAA;AAAA;AAShC,aAAW,SAAS,QAAQ;AAC1B,UAAM,WAAW,MAAM,SAAS;AAChC,eAAW,aAAa,MAAM,KAAK,UAAU;AAC3C,YAAM,QAAQ,aAAa,IAAI;AAC/B,UAAI,SAAS,CAAC,MAAM,KAAK,QAAQ;AAC/B,cAAM,KAAK,SAAS;AAAA;AAAA;AAAA;AAAA;6BAQ1B,OACA,kBACA;AACA,QAAM,cAAc,IAAI,IAAI,MAAM,IAAI,OAAK,CAAC,EAAE,SAAS,MAAM;AAC7D,aAAW,CAAC,WAAW,cAAc,iBAAiB,WAAW;AAC/D,eAAW,YAAY,WAAW;AAChC,YAAM,OAAO,YAAY,IAAI;AAC7B,UAAI,QAAQ,CAAC,KAAK,KAAK,SAAS,SAAS,YAAY;AACnD,aAAK,KAAK,SAAS,KAAK;AAAA;AAAA;AAAA;AAAA;;2BC7CE,WAAoC;AACpE,QAAM,OAAO,IAAI,IAAI,WAAW,SAAS,OAAO,GAAG,MAAM;AAGzD,MAAI,KAAK,WAAW,KAAK,KAAK,GAAG,QAAQ;AACvC,WAAO,EAAE,KAAK,mBAAmB,KAAK;AAAA;AAGxC,QAAM,IAAI,MAAM;AAAA;;+BCyBgD;AAAA,SAKzD,WACL,QACA,SAIA;AACA,UAAM,eAAeA,4BAAgB,WAAW;AAEhD,WAAO,IAAI,yBAAyB;AAAA,SAC/B;AAAA,MACH;AAAA;AAAA;AAAA,EAIJ,YAAY,SAIT;AACD,SAAK,eAAe,QAAQ;AAC5B,SAAK,SAAS,QAAQ;AACtB,SAAK,4BACH,QAAQ,6BACRC,6CAAiC,iBAAiB,KAAK;AAAA;AAAA,EAE3D,mBAA2B;AACzB,WAAO;AAAA;AAAA,QAGH,aACJ,UACA,WACA,MACkB;AAxFtB;AAyFI,QAAI,SAAS,SAAS,oBAAoB;AACxC,aAAO;AAAA;AAGT,UAAM,eAAe,WAAK,aAAa,OAAO,MAC5C,SAAS,YADU,mBAElB;AACH,QAAI,CAAC,cAAc;AACjB,YAAM,IAAI,MACR,+CAA+C,SAAS;AAAA;AAI5D,UAAM,EAAE,KAAK,gBAAgB,aAAa,QAAQ,SAAS,SACzD,SAAS;AAKX,UAAM,SAAS,WAAW,QAAQ;AAElC,UAAM,EAAE,YAAY,MAAM,KAAK,0BAA0B,eAAe;AAAA,MACtE,KAAK;AAAA;AAGP,UAAM,SAASC,gBAAQ,SAAS;AAAA,MAC9B,SAAS,aAAa;AAAA,MACtB;AAAA;AAIF,UAAM,iBAAiB,KAAK;AAC5B,SAAK,OAAO,KAAK,oCAAoC,SAAS;AAE9D,UAAM,EAAE,iBAAiB,MAAM,4BAA4B,QAAQ;AACnE,UAAM,WAAW,aAAa,OAC5B,OAAK,CAAC,EAAE,cAAc,eAAe,KAAK,EAAE;AAG9C,UAAM,WAAa,OAAK,QAAQ,kBAAkB,KAAM,QAAQ;AAChE,SAAK,OAAO,MACV,QAAQ,aAAa,+BAA+B,SAAS,mCAAmC;AAGlG,eAAW,cAAc,UAAU;AACjC,YAAM,aACJ,WAAW,MAAM,iBAAW,qBAAX,mBAA6B,OAAO;AAEvD,UAAI,CAAC,YAAY;AACf,aAAK,OAAO,KACV,kBAAkB,WAAW;AAE/B;AAAA;AAGF,YAAM,OAAO,SAAS,aAAa;AAEnC,WACEC,sCAAiB,SAAS;AAAA,QACxB,MAAM;AAAA,QACN,QAAQ,GAAG,WAAW,MAAM;AAAA,QAI5B,UAAU;AAAA;AAAA;AAKhB,WAAO;AAAA;AAAA;kBAQc,WAMvB;AACA,QAAM,MAAM,IAAI,IAAI;AACpB,QAAM,OAAO,IAAI,SAAS,OAAO,GAAG,MAAM;AAK1C,MAAI,KAAK,SAAS,KAAK,KAAK,GAAG,UAAU,KAAK,GAAG,QAAQ;AACvD,WAAO;AAAA,MACL,KAAK,mBAAmB,KAAK;AAAA,MAC7B,gBAAgB,aAAa,mBAAmB,KAAK;AAAA,MACrD,QAAQ,mBAAmB,KAAK;AAAA,MAChC,aAAa,IAAI,mBAAmB,KAAK,MAAM,GAAG,KAAK;AAAA,MACvD,MAAM,IAAI;AAAA;AAAA,aAEH,KAAK,WAAW,KAAK,KAAK,GAAG,QAAQ;AAC9C,WAAO;AAAA,MACL,KAAK,mBAAmB,KAAK;AAAA,MAC7B,MAAM,IAAI;AAAA,MACV,gBAAgB,aAAa;AAAA,MAC7B,aAAa;AAAA,MACb,QAAQ;AAAA;AAAA;AAIZ,QAAM,IAAI,MAAM,mBAAmB;AAAA;sBAGR,KAAqB;AAChD,SAAO,IAAI,OAAO,IAAI,IAAI,QAAQ,OAAO;AAAA;;oCCzJ4B;AAAA,SAM9D,WACL,QACA,SAIA;AACA,UAAM,IAAI,OAAO,kBAAkB;AACnC,UAAM,eAAeH,4BAAgB,WAAW;AAEhD,WAAO,IAAI,8BAA8B;AAAA,SACpC;AAAA,MACH;AAAA,MACA,MAAM,IAAI,yBAAyB,KAAK;AAAA;AAAA;AAAA,EAI5C,YAAY,SAKT;AACD,SAAK,eAAe,QAAQ;AAC5B,SAAK,SAAS,QAAQ;AACtB,SAAK,OAAO,QAAQ;AACpB,SAAK,4BACH,QAAQ,6BACRC,6CAAiC,iBAAiB,KAAK;AAAA;AAAA,EAE3D,mBAA2B;AACzB,WAAO;AAAA;AAAA,QAGH,aACJ,UACA,WACA,MACkB;AA5FtB;AA6FI,QAAI,SAAS,SAAS,oBAAoB;AACxC,aAAO;AAAA;AAGT,UAAM,eAAe,WAAK,aAAa,OAAO,MAC5C,SAAS,YADU,mBAElB;AACH,QAAI,CAAC,cAAc;AACjB,YAAM,IAAI,MACR,+CAA+C,SAAS;AAAA;AAI5D,UAAM,kCAAkB;AACxB,UAAM,UAAU,IAAI,IAAI,SAAS,QAAQ;AAEzC,UAAM,gBAAgB,KAAK,KAAK,SAC5B,KAAK,OACL,MAAM,KAAK,WAAW;AAE1B,eAAW,aAAa,eAAe;AACrC,UAAI;AACF,cAAM,EAAE,SAAS,MAAM,cACrB,MAAM,KAAK,0BAA0B,eAAe;AAAA,UAClD,KAAK,GAAG,WAAW,UAAU;AAAA;AAEjC,cAAM,SAASC,gBAAQ,SAAS;AAAA,UAC9B,SAAS,aAAa;AAAA,UACtB;AAAA;AAGF,cAAM,iBAAiB,KAAK;AAC5B,aAAK,OAAO,KACV,2CAA2C,UAAU;AAEvD,cAAM,EAAE,UAAU,MAAM,qBACtB,QACA,UAAU,MACV,WACA,UAAU;AAEZ,cAAM,EAAE,QAAQ,qBAAqB,MAAM,qBACzC,QACA,UAAU,MACV,UAAU;AAGZ,cAAM,WAAa,OAAK,QAAQ,kBAAkB,KAAM,QAAQ;AAChE,aAAK,OAAO,MACV,QAAQ,MAAM,2BAA2B,OAAO,4BAA4B,UAAU,WAAW;AAGnG,YAAI,SAAiB,gBAAU,kBAAV,YAA2B;AAChD,YAAI,OAAO,SAAS;AAAG,oBAAU;AAEjC,cAAM,QAAQ,OAAK;AACjB,cAAI,CAAC,YAAY,IAAI,SAAS,EAAE,SAAS,OAAO;AAC9C,wBAAY,IAAI,SAAS,EAAE,SAAS,MAAM;AAAA;AAAA;AAI9C,mBAAW,CAAC,WAAW,cAAc,iBAAiB,WAAW;AAC/D,qBAAW,YAAY,WAAW;AAChC,kBAAM,OAAO,YAAY,IAAI,SAAS;AACtC,gBAAI,QAAQ,CAAC,KAAK,KAAK,SAAS,SAAS,YAAY;AACnD,mBAAK,KAAK,SAAS,KAAK;AAAA;AAAA;AAAA;AAI9B,0BAAkB;AAElB,mBAAW,SAAS,QAAQ;AAC1B,eAAKC,sCAAiB,OAAO,UAAU;AAAA;AAAA,eAElC,GAAP;AACA,aAAK,OAAO,MACV,sCAAsC,UAAU,SAAS;AAAA;AAAA;AAK/D,UAAM,WAAW,MAAM,KAAK,YAAY;AACxC,eAAW,QAAQ,UAAU;AAC3B,WAAKA,sCAAiB,OAAO,UAAU;AAAA;AAGzC,WAAO;AAAA;AAAA,QAIK,WACZ,cAC+B;AAC/B,UAAM,eAAe,IAAIC,oCAAwB;AACjD,UAAM,WAAW,MAAM,aAAa;AAEpC,WAAO,SACJ,IAAI,aACH,QAAQ,gBAAgB,kBACxB,QAAQ,WACR,QAAQ,QAAQ,QACZ;AAAA,MACE,MAAM,QAAQ,QAAQ;AAAA,MACtB,gBAAgB,QAAQ,QAAQ,MAAM;AAAA,QAExC,QAEL,OAAO;AAAA;AAAA;;8BC1JiD;AAAA,EAqC7D,YACU,SAOR;AAPQ;AAQR,SAAK,4BACH,QAAQ,6BACRC,oDAAwC,OAAO,QAAQ;AAAA;AAAA,SA5CpD,WACL,QACA,SAMA;AA1DJ;AA2DI,UAAM,eAAeL,4BAAgB,WAAW;AAChD,UAAM,eAAe,mBAAa,OAAO,MAAM,QAAQ,YAAlC,mBAA2C;AAEhE,QAAI,CAAC,cAAc;AACjB,YAAM,IAAI,MACR,gDAAgD,QAAQ;AAAA;AAI5D,UAAM,SAAS,QAAQ,OAAO,MAAM;AAAA,MAClC,QAAQ,QAAQ;AAAA;AAGlB,WAAO,IAAI,wBAAwB;AAAA,MACjC,IAAI,QAAQ;AAAA,MACZ,QAAQ,QAAQ;AAAA,MAChB;AAAA,MACA;AAAA,MACA,2BACE,QAAQ,6BACRC,6CAAiC,iBAAiB;AAAA;AAAA;AAAA,EAkBxD,kBAAkB;AAChB,WAAO,2BAA2B,KAAK,QAAQ;AAAA;AAAA,QAG3C,QAAQ,YAAsC;AAClD,SAAK,aAAa;AAAA;AAAA,QAGd,OAAO;AACX,QAAI,CAAC,KAAK,YAAY;AACpB,YAAM,IAAI,MAAM;AAAA;AAGlB,UAAM,EAAE,qBAAqB,cAAc,KAAK,QAAQ;AAExD,UAAM,EAAE,SAAS,MAAM,cACrB,MAAM,KAAK,0BAA0B,eAAe;AAAA,MAClD,KAAK,KAAK,QAAQ;AAAA;AAEtB,UAAM,SAASC,gBAAQ,SAAS;AAAA,MAC9B,SAAS,KAAK,QAAQ,aAAa;AAAA,MACnC;AAAA;AAGF,UAAM,EAAE,QAAQ,kBAAkB,KAAK,QAAQ;AAC/C,UAAM,EAAE,UAAU,MAAM,qBAAqB,QAAQ,KAAK;AAC1D,UAAM,EAAE,QAAQ,qBAAqB,MAAM,qBACzC,QACA;AAEF,wBAAoB,OAAO;AAC3B,sBAAkB;AAElB,UAAM,EAAE,uBAAuB,iBAAiB,EAAE,OAAO;AAEzD,UAAM,KAAK,WAAW,cAAc;AAAA,MAClC,MAAM;AAAA,MACN,UAAU,CAAC,GAAG,OAAO,GAAG,QAAQ,IAAI;AAAW,QAC7C,aAAa,uBAAuB,KAAK,QAAQ;AAAA,QACjD,QAAQ,cACN,WAAW,KAAK,QAAQ,aAAa,QACrC,KACA;AAAA;AAAA;AAKN;AAAA;AAAA;AAKJ,uBAAuB,QAAgB;AACrC,MAAI,YAAY,KAAK;AACrB,MAAI;AAEJ,SAAO,KAAK;AAEZ,4BAA0B,MAA+C;AACvE,cAAU,GAAG,KAAK,MAAM,2BAA2B,KAAK,OAAO;AAC/D,UAAM,eAAiB,OAAK,QAAQ,aAAa,KAAM,QAAQ;AAC/D,gBAAY,KAAK;AACjB,WAAO,KAAK,QAAQ,cAAc;AAClC,WAAO,EAAE;AAAA;AAGX,gCAA8B;AAC5B,UAAM,iBAAmB,OAAK,QAAQ,aAAa,KAAM,QAAQ;AACjE,WAAO,KAAK,aAAa,cAAc;AAAA;AAGzC,SAAO,EAAE;AAAA;uBAKT,SACA,KACA,QACQ;AACR,QAAM,WACJ,OAAO,SAAS,UACZ,OAAO,gBAAgB,aAAa,OAAO,SAAS,SACpD,OAAO,WAAW,OAAO,SAAS;AACxC,SAAOI,aACL;AAAA,IACE,UAAU;AAAA,MACR,aAAa;AAAA,SACVC,mCAAsB;AAAA,SACtBC,0CAA6B;AAAA;AAAA;AAAA,KAIpC;AAAA;;+BChJ8D;AAAA,SAKzD,WACL,QACA,SAIA;AACA,UAAM,eAAeR,4BAAgB,WAAW;AAEhD,WAAO,IAAI,yBAAyB;AAAA,SAC/B;AAAA,MACH;AAAA;AAAA;AAAA,EAIJ,YAAY,SAIT;AACD,SAAK,eAAe,QAAQ;AAC5B,SAAK,4BACH,QAAQ,6BACRC,6CAAiC,iBAAiB,KAAK;AACzD,SAAK,SAAS,QAAQ;AAAA;AAAA,EAExB,mBAA2B;AACzB,WAAO;AAAA;AAAA,QAGH,aACJ,UACA,WACA,MACkB;AAClB,QAAI,SAAS,SAAS,cAAc;AAClC,aAAO;AAAA;AAGT,UAAM,EAAE,QAAQ,cAAc,MAAM,KAAK,aAAa,SAAS;AAC/D,UAAM,EAAE,QAAQ,kBAAkB,SAAS;AAG3C,UAAM,iBAAiB,KAAK;AAC5B,SAAK,OAAO,KAAK;AAEjB,UAAM,EAAE,UAAU,MAAM,qBAAqB,QAAQ,KAAK;AAC1D,UAAM,EAAE,QAAQ,qBAAqB,MAAM,qBACzC,QACA;AAGF,UAAM,WAAa,OAAK,QAAQ,kBAAkB,KAAM,QAAQ;AAChE,SAAK,OAAO,MACV,QAAQ,MAAM,2BAA2B,OAAO,2BAA2B;AAG7E,wBAAoB,OAAO;AAC3B,sBAAkB;AAGlB,eAAW,SAAS,QAAQ;AAC1B,WAAKE,sCAAiB,OAAO,UAAU;AAAA;AAEzC,eAAW,QAAQ,OAAO;AACxB,WAAKA,sCAAiB,OAAO,UAAU;AAAA;AAGzC,WAAO;AAAA;AAAA,QAGK,aACZ,QAC+D;AA5HnE;AA6HI,UAAM,eAAe,WAAK,aAAa,OAAO,MAAM,YAA/B,mBAAwC;AAE7D,QAAI,CAAC,cAAc;AACjB,YAAM,IAAI,MACR,gDAAgD;AAAA;AAIpD,UAAM,EAAE,SAAS,MAAM,cACrB,MAAM,KAAK,0BAA0B,eAAe;AAAA,MAClD,KAAK;AAAA;AAGT,UAAM,SAASD,gBAAQ,SAAS;AAAA,MAC9B,SAAS,aAAa;AAAA,MACtB;AAAA;AAGF,WAAO,EAAE,QAAQ;AAAA;AAAA;;;;;;;"}

package/dist/index.d.ts

@@ -0,0 +1,130 @@
+import { Config } from '@backstage/config';
+import { GithubCredentialsProvider, ScmIntegrationRegistry, GitHubIntegrationConfig } from '@backstage/integration';
+import { CatalogProcessor, LocationSpec, CatalogProcessorEmit, EntityProvider, EntityProviderConnection } from '@backstage/plugin-catalog-backend';
+import { Logger } from 'winston';
+
+/**
+ * Extracts repositories out of a GitHub org.
+ *
+ * The following will create locations for all projects which have a catalog-info.yaml
+ * on the default branch. The first is shorthand for the second.
+ *
+ *    target: "https://github.com/backstage"
+ *    or
+ *    target: https://github.com/backstage/*\/blob/-/catalog-info.yaml
+ *
+ * You may also explicitly specify the source branch:
+ *
+ *    target: https://github.com/backstage/*\/blob/main/catalog-info.yaml
+ *
+ * @public
+ **/
+declare class GithubDiscoveryProcessor implements CatalogProcessor {
+    private readonly integrations;
+    private readonly logger;
+    private readonly githubCredentialsProvider;
+    static fromConfig(config: Config, options: {
+        logger: Logger;
+        githubCredentialsProvider?: GithubCredentialsProvider;
+    }): GithubDiscoveryProcessor;
+    constructor(options: {
+        integrations: ScmIntegrationRegistry;
+        logger: Logger;
+        githubCredentialsProvider?: GithubCredentialsProvider;
+    });
+    getProcessorName(): string;
+    readLocation(location: LocationSpec, _optional: boolean, emit: CatalogProcessorEmit): Promise<boolean>;
+}
+
+/**
+ * The configuration parameters for a multi-org GitHub processor.
+ * @public
+ */
+declare type GithubMultiOrgConfig = Array<{
+    /**
+     * The name of the GitHub org to process.
+     */
+    name: string;
+    /**
+     * The namespace of the group created for this org.
+     */
+    groupNamespace: string;
+    /**
+     * The namespace of the users created for this org. If not specified defaults to undefined.
+     */
+    userNamespace: string | undefined;
+}>;
+
+/**
+ * Extracts teams and users out of a multiple GitHub orgs namespaced per org.
+ *
+ * Be aware that this processor may not be compatible with future org structures in the catalog.
+ *
+ * @public
+ */
+declare class GithubMultiOrgReaderProcessor implements CatalogProcessor {
+    private readonly integrations;
+    private readonly orgs;
+    private readonly logger;
+    private readonly githubCredentialsProvider;
+    static fromConfig(config: Config, options: {
+        logger: Logger;
+        githubCredentialsProvider?: GithubCredentialsProvider;
+    }): GithubMultiOrgReaderProcessor;
+    constructor(options: {
+        integrations: ScmIntegrationRegistry;
+        logger: Logger;
+        orgs: GithubMultiOrgConfig;
+        githubCredentialsProvider?: GithubCredentialsProvider;
+    });
+    getProcessorName(): string;
+    readLocation(location: LocationSpec, _optional: boolean, emit: CatalogProcessorEmit): Promise<boolean>;
+    private getAllOrgs;
+}
+
+/** @public */
+declare class GitHubOrgEntityProvider implements EntityProvider {
+    private options;
+    private connection?;
+    private githubCredentialsProvider;
+    static fromConfig(config: Config, options: {
+        id: string;
+        orgUrl: string;
+        logger: Logger;
+        githubCredentialsProvider?: GithubCredentialsProvider;
+    }): GitHubOrgEntityProvider;
+    constructor(options: {
+        id: string;
+        orgUrl: string;
+        gitHubConfig: GitHubIntegrationConfig;
+        logger: Logger;
+        githubCredentialsProvider?: GithubCredentialsProvider;
+    });
+    getProviderName(): string;
+    connect(connection: EntityProviderConnection): Promise<void>;
+    read(): Promise<void>;
+}
+
+/**
+ * Extracts teams and users out of a GitHub org.
+ * @public
+ */
+declare class GithubOrgReaderProcessor implements CatalogProcessor {
+    private readonly integrations;
+    private readonly logger;
+    private readonly githubCredentialsProvider;
+    static fromConfig(config: Config, options: {
+        logger: Logger;
+        githubCredentialsProvider?: GithubCredentialsProvider;
+    }): GithubOrgReaderProcessor;
+    constructor(options: {
+        integrations: ScmIntegrationRegistry;
+        logger: Logger;
+        githubCredentialsProvider?: GithubCredentialsProvider;
+    });
+    getProcessorName(): string;
+    readLocation(location: LocationSpec, _optional: boolean, emit: CatalogProcessorEmit): Promise<boolean>;
+    private createClient;
+}
+
+export { GitHubOrgEntityProvider, GithubDiscoveryProcessor, GithubMultiOrgConfig, GithubMultiOrgReaderProcessor, GithubOrgReaderProcessor };

package/package.json

@@ -0,0 +1,57 @@
+{
+  "name": "@backstage/plugin-catalog-backend-module-github",
+  "description": "A Backstage catalog backend module that helps integrate towards GitHub",
+  "version": "0.0.0-nightly-20220311022539",
+  "main": "dist/index.cjs.js",
+  "types": "dist/index.d.ts",
+  "license": "Apache-2.0",
+  "private": false,
+  "publishConfig": {
+    "access": "public",
+    "main": "dist/index.cjs.js",
+    "types": "dist/index.d.ts"
+  },
+  "backstage": {
+    "role": "backend-plugin-module"
+  },
+  "homepage": "https://backstage.io",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/backstage/backstage",
+    "directory": "plugins/catalog-backend-module-github"
+  },
+  "keywords": [
+    "backstage"
+  ],
+  "scripts": {
+    "build": "backstage-cli package build",
+    "lint": "backstage-cli package lint",
+    "test": "backstage-cli package test",
+    "prepack": "backstage-cli package prepack",
+    "postpack": "backstage-cli package postpack",
+    "clean": "backstage-cli package clean",
+    "start": "backstage-cli package start"
+  },
+  "dependencies": {
+    "@backstage/backend-common": "^0.13.0",
+    "@backstage/catalog-model": "^0.0.0-nightly-20220311022539",
+    "@backstage/config": "^0.1.15",
+    "@backstage/errors": "^0.2.2",
+    "@backstage/integration": "^0.8.0",
+    "@backstage/plugin-catalog-backend": "^0.0.0-nightly-20220311022539",
+    "@backstage/types": "^0.1.3",
+    "@octokit/graphql": "^4.5.8",
+    "lodash": "^4.17.21",
+    "msw": "^0.35.0",
+    "node-fetch": "^2.6.7",
+    "winston": "^3.2.1"
+  },
+  "devDependencies": {
+    "@backstage/backend-test-utils": "^0.1.21",
+    "@backstage/cli": "^0.15.2",
+    "@types/lodash": "^4.14.151"
+  },
+  "files": [
+    "dist"
+  ]
+}