@backstage/plugin-catalog-backend-module-github 0.1.6-next.2 → 0.1.7-next.1

package/CHANGELOG.md

@@ -1,5 +1,46 @@
 # @backstage/plugin-catalog-backend-module-github
 
+## 0.1.7-next.1
+
+### Patch Changes
+
+- 287a64bf97: Added the ability to configure the host for the `GitHubEntityProvider` to use against GitHub Enterprise
+- Updated dependencies
+  - @backstage/backend-common@0.15.1-next.1
+  - @backstage/plugin-catalog-backend@1.4.0-next.1
+
+## 0.1.7-next.0
+
+### Patch Changes
+
+- 3c4a388537: New experimental alpha exports for use with the upcoming backend system.
+- bf5e9030eb: Updated dependency `msw` to `^0.45.0`.
+- Updated dependencies
+  - @backstage/backend-common@0.15.1-next.0
+  - @backstage/backend-tasks@0.3.5-next.0
+  - @backstage/plugin-catalog-backend@1.3.2-next.0
+  - @backstage/backend-plugin-api@0.1.2-next.0
+  - @backstage/integration@1.3.1-next.0
+  - @backstage/plugin-catalog-node@1.0.2-next.0
+
+## 0.1.6
+
+### Patch Changes
+
+- f48950e34b: Github Entity Provider functionality for adding entities to the catalog.
+
+  This provider replaces the GithubDiscoveryProcessor functionality as providers offer more flexibility with scheduling ingestion, removing and preventing orphaned entities.
+
+  More information can be found on the [GitHub Discovery](https://backstage.io/docs/integrations/github/discovery) page.
+
+- c59d1ce487: Fixed bug where repository filter was including all archived repositories
+- 97f0a37378: Improved support for wildcards in `catalogPath`
+- Updated dependencies
+  - @backstage/backend-common@0.15.0
+  - @backstage/integration@1.3.0
+  - @backstage/backend-tasks@0.3.4
+  - @backstage/plugin-catalog-backend@1.3.1
+
 ## 0.1.6-next.2
 
 ### Patch Changes

package/alpha/package.json

@@ -0,0 +1,6 @@
+{
+  "name": "@backstage/plugin-catalog-backend-module-github",
+  "version": "0.1.7-next.1",
+  "main": "../dist/index.cjs.js",
+  "types": "../dist/index.alpha.d.ts"
+}

package/dist/index.alpha.d.ts

@@ -0,0 +1,249 @@
+/**
+ * A Backstage catalog backend module that helps integrate towards GitHub
+ *
+ * @packageDocumentation
+ */
+
+import { BackendFeature } from '@backstage/backend-plugin-api';
+import { CatalogProcessor } from '@backstage/plugin-catalog-backend';
+import { CatalogProcessorEmit } from '@backstage/plugin-catalog-backend';
+import { Config } from '@backstage/config';
+import { EntityProvider } from '@backstage/plugin-catalog-backend';
+import { EntityProviderConnection } from '@backstage/plugin-catalog-backend';
+import { GithubCredentialsProvider } from '@backstage/integration';
+import { GitHubIntegrationConfig } from '@backstage/integration';
+import { LocationSpec } from '@backstage/plugin-catalog-backend';
+import { Logger } from 'winston';
+import { ScmIntegrationRegistry } from '@backstage/integration';
+import { TaskRunner } from '@backstage/backend-tasks';
+import { TaskScheduleDefinition } from '@backstage/backend-tasks';
+
+/**
+ * Extracts repositories out of a GitHub org.
+ *
+ * The following will create locations for all projects which have a catalog-info.yaml
+ * on the default branch. The first is shorthand for the second.
+ *
+ *    target: "https://github.com/backstage"
+ *    or
+ *    target: https://github.com/backstage/*\/blob/-/catalog-info.yaml
+ *
+ * You may also explicitly specify the source branch:
+ *
+ *    target: https://github.com/backstage/*\/blob/main/catalog-info.yaml
+ *
+ * @public
+ **/
+export declare class GithubDiscoveryProcessor implements CatalogProcessor {
+    private readonly integrations;
+    private readonly logger;
+    private readonly githubCredentialsProvider;
+    static fromConfig(config: Config, options: {
+        logger: Logger;
+        githubCredentialsProvider?: GithubCredentialsProvider;
+    }): GithubDiscoveryProcessor;
+    constructor(options: {
+        integrations: ScmIntegrationRegistry;
+        logger: Logger;
+        githubCredentialsProvider?: GithubCredentialsProvider;
+    });
+    getProcessorName(): string;
+    readLocation(location: LocationSpec, _optional: boolean, emit: CatalogProcessorEmit): Promise<boolean>;
+}
+
+/**
+ * Discovers catalog files located in [GitHub](https://github.com).
+ * The provider will search your GitHub account and register catalog files matching the configured path
+ * as Location entity and via following processing steps add all contained catalog entities.
+ * This can be useful as an alternative to static locations or manually adding things to the catalog.
+ *
+ * @public
+ */
+export declare class GitHubEntityProvider implements EntityProvider {
+    private readonly config;
+    private readonly logger;
+    private readonly integration;
+    private readonly scheduleFn;
+    private connection?;
+    private readonly githubCredentialsProvider;
+    static fromConfig(config: Config, options: {
+        logger: Logger;
+        schedule: TaskRunner;
+    }): GitHubEntityProvider[];
+    private constructor();
+    /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.getProviderName} */
+    getProviderName(): string;
+    /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.connect} */
+    connect(connection: EntityProviderConnection): Promise<void>;
+    private createScheduleFn;
+    refresh(logger: Logger): Promise<void>;
+    private findCatalogFiles;
+    private matchesFilters;
+    private createLocationUrl;
+    private static toLocationSpec;
+}
+
+/**
+ * Registers the GitHubEntityProvider with the catalog processing extension point.
+ *
+ * @alpha
+ */
+export declare const githubEntityProviderCatalogModule: (options?: GithubEntityProviderCatalogModuleOptions | undefined) => BackendFeature;
+
+/**
+ * Options for {@link githubEntityProviderCatalogModule}.
+ *
+ * @alpha
+ */
+export declare type GithubEntityProviderCatalogModuleOptions = {
+    schedule?: TaskScheduleDefinition;
+};
+
+/**
+ * The configuration parameters for a multi-org GitHub processor.
+ * @public
+ */
+export declare type GithubMultiOrgConfig = Array<{
+    /**
+     * The name of the GitHub org to process.
+     */
+    name: string;
+    /**
+     * The namespace of the group created for this org.
+     */
+    groupNamespace: string;
+    /**
+     * The namespace of the users created for this org. If not specified defaults to undefined.
+     */
+    userNamespace: string | undefined;
+}>;
+
+/**
+ * Extracts teams and users out of a multiple GitHub orgs namespaced per org.
+ *
+ * Be aware that this processor may not be compatible with future org structures in the catalog.
+ *
+ * @public
+ */
+export declare class GithubMultiOrgReaderProcessor implements CatalogProcessor {
+    private readonly integrations;
+    private readonly orgs;
+    private readonly logger;
+    private readonly githubCredentialsProvider;
+    static fromConfig(config: Config, options: {
+        logger: Logger;
+        githubCredentialsProvider?: GithubCredentialsProvider;
+    }): GithubMultiOrgReaderProcessor;
+    constructor(options: {
+        integrations: ScmIntegrationRegistry;
+        logger: Logger;
+        orgs: GithubMultiOrgConfig;
+        githubCredentialsProvider?: GithubCredentialsProvider;
+    });
+    getProcessorName(): string;
+    readLocation(location: LocationSpec, _optional: boolean, emit: CatalogProcessorEmit): Promise<boolean>;
+    private getAllOrgs;
+}
+
+/**
+ * Ingests org data (users and groups) from GitHub.
+ *
+ * @public
+ */
+export declare class GitHubOrgEntityProvider implements EntityProvider {
+    private options;
+    private readonly credentialsProvider;
+    private connection?;
+    private scheduleFn?;
+    static fromConfig(config: Config, options: GitHubOrgEntityProviderOptions): GitHubOrgEntityProvider;
+    constructor(options: {
+        id: string;
+        orgUrl: string;
+        gitHubConfig: GitHubIntegrationConfig;
+        logger: Logger;
+        githubCredentialsProvider?: GithubCredentialsProvider;
+    });
+    /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.getProviderName} */
+    getProviderName(): string;
+    /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.connect} */
+    connect(connection: EntityProviderConnection): Promise<void>;
+    /**
+     * Runs one single complete ingestion. This is only necessary if you use
+     * manual scheduling.
+     */
+    read(options?: {
+        logger?: Logger;
+    }): Promise<void>;
+    private schedule;
+}
+
+/**
+ * Options for {@link GitHubOrgEntityProvider}.
+ *
+ * @public
+ */
+export declare interface GitHubOrgEntityProviderOptions {
+    /**
+     * A unique, stable identifier for this provider.
+     *
+     * @example "production"
+     */
+    id: string;
+    /**
+     * The target that this provider should consume.
+     *
+     * @example "https://github.com/backstage"
+     */
+    orgUrl: string;
+    /**
+     * The refresh schedule to use.
+     *
+     * @defaultValue "manual"
+     * @remarks
+     *
+     * If you pass in 'manual', you are responsible for calling the `read` method
+     * manually at some interval.
+     *
+     * But more commonly you will pass in the result of
+     * {@link @backstage/backend-tasks#PluginTaskScheduler.createScheduledTaskRunner}
+     * to enable automatic scheduling of tasks.
+     */
+    schedule?: 'manual' | TaskRunner;
+    /**
+     * The logger to use.
+     */
+    logger: Logger;
+    /**
+     * Optionally supply a custom credentials provider, replacing the default one.
+     */
+    githubCredentialsProvider?: GithubCredentialsProvider;
+}
+
+/**
+ * Extracts teams and users out of a GitHub org.
+ *
+ * @remarks
+ *
+ * Consider using {@link GitHubOrgEntityProvider} instead.
+ *
+ * @public
+ */
+export declare class GithubOrgReaderProcessor implements CatalogProcessor {
+    private readonly integrations;
+    private readonly logger;
+    private readonly githubCredentialsProvider;
+    static fromConfig(config: Config, options: {
+        logger: Logger;
+        githubCredentialsProvider?: GithubCredentialsProvider;
+    }): GithubOrgReaderProcessor;
+    constructor(options: {
+        integrations: ScmIntegrationRegistry;
+        logger: Logger;
+        githubCredentialsProvider?: GithubCredentialsProvider;
+    });
+    getProcessorName(): string;
+    readLocation(location: LocationSpec, _optional: boolean, emit: CatalogProcessorEmit): Promise<boolean>;
+    private createClient;
+}
+
+export { }

package/dist/index.beta.d.ts

@@ -0,0 +1,237 @@
+/**
+ * A Backstage catalog backend module that helps integrate towards GitHub
+ *
+ * @packageDocumentation
+ */
+
+import { BackendFeature } from '@backstage/backend-plugin-api';
+import { CatalogProcessor } from '@backstage/plugin-catalog-backend';
+import { CatalogProcessorEmit } from '@backstage/plugin-catalog-backend';
+import { Config } from '@backstage/config';
+import { EntityProvider } from '@backstage/plugin-catalog-backend';
+import { EntityProviderConnection } from '@backstage/plugin-catalog-backend';
+import { GithubCredentialsProvider } from '@backstage/integration';
+import { GitHubIntegrationConfig } from '@backstage/integration';
+import { LocationSpec } from '@backstage/plugin-catalog-backend';
+import { Logger } from 'winston';
+import { ScmIntegrationRegistry } from '@backstage/integration';
+import { TaskRunner } from '@backstage/backend-tasks';
+import { TaskScheduleDefinition } from '@backstage/backend-tasks';
+
+/**
+ * Extracts repositories out of a GitHub org.
+ *
+ * The following will create locations for all projects which have a catalog-info.yaml
+ * on the default branch. The first is shorthand for the second.
+ *
+ *    target: "https://github.com/backstage"
+ *    or
+ *    target: https://github.com/backstage/*\/blob/-/catalog-info.yaml
+ *
+ * You may also explicitly specify the source branch:
+ *
+ *    target: https://github.com/backstage/*\/blob/main/catalog-info.yaml
+ *
+ * @public
+ **/
+export declare class GithubDiscoveryProcessor implements CatalogProcessor {
+    private readonly integrations;
+    private readonly logger;
+    private readonly githubCredentialsProvider;
+    static fromConfig(config: Config, options: {
+        logger: Logger;
+        githubCredentialsProvider?: GithubCredentialsProvider;
+    }): GithubDiscoveryProcessor;
+    constructor(options: {
+        integrations: ScmIntegrationRegistry;
+        logger: Logger;
+        githubCredentialsProvider?: GithubCredentialsProvider;
+    });
+    getProcessorName(): string;
+    readLocation(location: LocationSpec, _optional: boolean, emit: CatalogProcessorEmit): Promise<boolean>;
+}
+
+/**
+ * Discovers catalog files located in [GitHub](https://github.com).
+ * The provider will search your GitHub account and register catalog files matching the configured path
+ * as Location entity and via following processing steps add all contained catalog entities.
+ * This can be useful as an alternative to static locations or manually adding things to the catalog.
+ *
+ * @public
+ */
+export declare class GitHubEntityProvider implements EntityProvider {
+    private readonly config;
+    private readonly logger;
+    private readonly integration;
+    private readonly scheduleFn;
+    private connection?;
+    private readonly githubCredentialsProvider;
+    static fromConfig(config: Config, options: {
+        logger: Logger;
+        schedule: TaskRunner;
+    }): GitHubEntityProvider[];
+    private constructor();
+    /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.getProviderName} */
+    getProviderName(): string;
+    /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.connect} */
+    connect(connection: EntityProviderConnection): Promise<void>;
+    private createScheduleFn;
+    refresh(logger: Logger): Promise<void>;
+    private findCatalogFiles;
+    private matchesFilters;
+    private createLocationUrl;
+    private static toLocationSpec;
+}
+
+/* Excluded from this release type: githubEntityProviderCatalogModule */
+
+/* Excluded from this release type: GithubEntityProviderCatalogModuleOptions */
+
+/**
+ * The configuration parameters for a multi-org GitHub processor.
+ * @public
+ */
+export declare type GithubMultiOrgConfig = Array<{
+    /**
+     * The name of the GitHub org to process.
+     */
+    name: string;
+    /**
+     * The namespace of the group created for this org.
+     */
+    groupNamespace: string;
+    /**
+     * The namespace of the users created for this org. If not specified defaults to undefined.
+     */
+    userNamespace: string | undefined;
+}>;
+
+/**
+ * Extracts teams and users out of a multiple GitHub orgs namespaced per org.
+ *
+ * Be aware that this processor may not be compatible with future org structures in the catalog.
+ *
+ * @public
+ */
+export declare class GithubMultiOrgReaderProcessor implements CatalogProcessor {
+    private readonly integrations;
+    private readonly orgs;
+    private readonly logger;
+    private readonly githubCredentialsProvider;
+    static fromConfig(config: Config, options: {
+        logger: Logger;
+        githubCredentialsProvider?: GithubCredentialsProvider;
+    }): GithubMultiOrgReaderProcessor;
+    constructor(options: {
+        integrations: ScmIntegrationRegistry;
+        logger: Logger;
+        orgs: GithubMultiOrgConfig;
+        githubCredentialsProvider?: GithubCredentialsProvider;
+    });
+    getProcessorName(): string;
+    readLocation(location: LocationSpec, _optional: boolean, emit: CatalogProcessorEmit): Promise<boolean>;
+    private getAllOrgs;
+}
+
+/**
+ * Ingests org data (users and groups) from GitHub.
+ *
+ * @public
+ */
+export declare class GitHubOrgEntityProvider implements EntityProvider {
+    private options;
+    private readonly credentialsProvider;
+    private connection?;
+    private scheduleFn?;
+    static fromConfig(config: Config, options: GitHubOrgEntityProviderOptions): GitHubOrgEntityProvider;
+    constructor(options: {
+        id: string;
+        orgUrl: string;
+        gitHubConfig: GitHubIntegrationConfig;
+        logger: Logger;
+        githubCredentialsProvider?: GithubCredentialsProvider;
+    });
+    /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.getProviderName} */
+    getProviderName(): string;
+    /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.connect} */
+    connect(connection: EntityProviderConnection): Promise<void>;
+    /**
+     * Runs one single complete ingestion. This is only necessary if you use
+     * manual scheduling.
+     */
+    read(options?: {
+        logger?: Logger;
+    }): Promise<void>;
+    private schedule;
+}
+
+/**
+ * Options for {@link GitHubOrgEntityProvider}.
+ *
+ * @public
+ */
+export declare interface GitHubOrgEntityProviderOptions {
+    /**
+     * A unique, stable identifier for this provider.
+     *
+     * @example "production"
+     */
+    id: string;
+    /**
+     * The target that this provider should consume.
+     *
+     * @example "https://github.com/backstage"
+     */
+    orgUrl: string;
+    /**
+     * The refresh schedule to use.
+     *
+     * @defaultValue "manual"
+     * @remarks
+     *
+     * If you pass in 'manual', you are responsible for calling the `read` method
+     * manually at some interval.
+     *
+     * But more commonly you will pass in the result of
+     * {@link @backstage/backend-tasks#PluginTaskScheduler.createScheduledTaskRunner}
+     * to enable automatic scheduling of tasks.
+     */
+    schedule?: 'manual' | TaskRunner;
+    /**
+     * The logger to use.
+     */
+    logger: Logger;
+    /**
+     * Optionally supply a custom credentials provider, replacing the default one.
+     */
+    githubCredentialsProvider?: GithubCredentialsProvider;
+}
+
+/**
+ * Extracts teams and users out of a GitHub org.
+ *
+ * @remarks
+ *
+ * Consider using {@link GitHubOrgEntityProvider} instead.
+ *
+ * @public
+ */
+export declare class GithubOrgReaderProcessor implements CatalogProcessor {
+    private readonly integrations;
+    private readonly logger;
+    private readonly githubCredentialsProvider;
+    static fromConfig(config: Config, options: {
+        logger: Logger;
+        githubCredentialsProvider?: GithubCredentialsProvider;
+    }): GithubOrgReaderProcessor;
+    constructor(options: {
+        integrations: ScmIntegrationRegistry;
+        logger: Logger;
+        githubCredentialsProvider?: GithubCredentialsProvider;
+    });
+    getProcessorName(): string;
+    readLocation(location: LocationSpec, _optional: boolean, emit: CatalogProcessorEmit): Promise<boolean>;
+    private createClient;
+}
+
+export { }

package/dist/index.cjs.js

@@ -8,6 +8,8 @@ var graphql = require('@octokit/graphql');
 var uuid = require('uuid');
 var catalogModel = require('@backstage/catalog-model');
 var lodash = require('lodash');
+var backendPluginApi = require('@backstage/backend-plugin-api');
+var pluginCatalogNode = require('@backstage/plugin-catalog-node');
 
 function _interopNamespace(e) {
   if (e && e.__esModule) return e;
@@ -590,15 +592,17 @@ function readProviderConfigs(config) {
   });
 }
 function readProviderConfig(id, config) {
-  var _a;
+  var _a, _b;
   const organization = config.getString("organization");
   const catalogPath = (_a = config.getOptionalString("catalogPath")) != null ? _a : DEFAULT_CATALOG_PATH;
+  const host = (_b = config.getOptionalString("host")) != null ? _b : "github.com";
   const repositoryPattern = config.getOptionalString("filters.repository");
   const branchPattern = config.getOptionalString("filters.branch");
   return {
     id,
     catalogPath,
     organization,
+    host,
     filters: {
       repository: repositoryPattern ? compileRegExp(repositoryPattern) : void 0,
       branch: branchPattern || void 0
@@ -619,20 +623,21 @@ function compileRegExp(pattern) {
 class GitHubEntityProvider {
   static fromConfig(config, options) {
     const integrations = integration.ScmIntegrations.fromConfig(config);
-    const integration$1 = integrations.github.byHost("github.com");
-    if (!integration$1) {
-      throw new Error(
-        `There is no GitHub config that matches github. Please add a configuration entry for it under integrations.github`
-      );
-    }
-    return readProviderConfigs(config).map(
-      (providerConfig) => new GitHubEntityProvider(
+    return readProviderConfigs(config).map((providerConfig) => {
+      const integrationHost = providerConfig.host;
+      const integration = integrations.github.byHost(integrationHost);
+      if (!integration) {
+        throw new Error(
+          `There is no GitHub config that matches host ${integrationHost}. Please add a configuration entry for it under integrations.github`
+        );
+      }
+      return new GitHubEntityProvider(
         providerConfig,
-        integration$1,
+        integration,
         options.logger,
         options.schedule
-      )
-    );
+      );
+    });
   }
   constructor(config, integration$1, logger, schedule) {
     this.config = config;
@@ -713,7 +718,7 @@ class GitHubEntityProvider {
     const repositoryFilter = (_a = this.config.filters) == null ? void 0 : _a.repository;
     const matchingRepositories = repositories.filter((r) => {
       var _a2;
-      return !r.isArchived && repositoryFilter ? repositoryFilter == null ? void 0 : repositoryFilter.test(r.name) : (_a2 = r.defaultBranchRef) == null ? void 0 : _a2.name;
+      return !r.isArchived && (!repositoryFilter || repositoryFilter.test(r.name)) && ((_a2 = r.defaultBranchRef) == null ? void 0 : _a2.name);
     });
     return matchingRepositories;
   }
@@ -859,9 +864,39 @@ function withLocations(baseUrl, org, entity) {
   );
 }
 
+const githubEntityProviderCatalogModule = backendPluginApi.createBackendModule({
+  pluginId: "catalog",
+  moduleId: "github-entity-provider",
+  register(env, options) {
+    env.registerInit({
+      deps: {
+        config: backendPluginApi.configServiceRef,
+        catalog: pluginCatalogNode.catalogProcessingExtensionPoint,
+        logger: backendPluginApi.loggerServiceRef,
+        scheduler: backendPluginApi.schedulerServiceRef
+      },
+      async init({ config, catalog, logger, scheduler }) {
+        var _a;
+        const scheduleDef = (_a = options == null ? void 0 : options.schedule) != null ? _a : {
+          frequency: { seconds: 600 },
+          timeout: { seconds: 900 },
+          initialDelay: { seconds: 3 }
+        };
+        catalog.addEntityProvider(
+          GitHubEntityProvider.fromConfig(config, {
+            logger: backendPluginApi.loggerToWinstonLogger(logger),
+            schedule: scheduler.createScheduledTaskRunner(scheduleDef)
+          })
+        );
+      }
+    });
+  }
+});
+
 exports.GitHubEntityProvider = GitHubEntityProvider;
 exports.GitHubOrgEntityProvider = GitHubOrgEntityProvider;
 exports.GithubDiscoveryProcessor = GithubDiscoveryProcessor;
 exports.GithubMultiOrgReaderProcessor = GithubMultiOrgReaderProcessor;
 exports.GithubOrgReaderProcessor = GithubOrgReaderProcessor;
+exports.githubEntityProviderCatalogModule = githubEntityProviderCatalogModule;
 //# sourceMappingURL=index.cjs.js.map