@backstage/plugin-catalog-backend-module-github 0.1.6-next.1 → 0.1.7-next.0

package/CHANGELOG.md

@@ -1,5 +1,45 @@
 # @backstage/plugin-catalog-backend-module-github
 
+## 0.1.7-next.0
+
+### Patch Changes
+
+- 3c4a388537: New experimental alpha exports for use with the upcoming backend system.
+- bf5e9030eb: Updated dependency `msw` to `^0.45.0`.
+- Updated dependencies
+  - @backstage/backend-common@0.15.1-next.0
+  - @backstage/backend-tasks@0.3.5-next.0
+  - @backstage/plugin-catalog-backend@1.3.2-next.0
+  - @backstage/backend-plugin-api@0.1.2-next.0
+  - @backstage/integration@1.3.1-next.0
+  - @backstage/plugin-catalog-node@1.0.2-next.0
+
+## 0.1.6
+
+### Patch Changes
+
+- f48950e34b: Github Entity Provider functionality for adding entities to the catalog.
+
+  This provider replaces the GithubDiscoveryProcessor functionality as providers offer more flexibility with scheduling ingestion, removing and preventing orphaned entities.
+
+  More information can be found on the [GitHub Discovery](https://backstage.io/docs/integrations/github/discovery) page.
+
+- c59d1ce487: Fixed bug where repository filter was including all archived repositories
+- 97f0a37378: Improved support for wildcards in `catalogPath`
+- Updated dependencies
+  - @backstage/backend-common@0.15.0
+  - @backstage/integration@1.3.0
+  - @backstage/backend-tasks@0.3.4
+  - @backstage/plugin-catalog-backend@1.3.1
+
+## 0.1.6-next.2
+
+### Patch Changes
+
+- 97f0a37378: Improved support for wildcards in `catalogPath`
+- Updated dependencies
+  - @backstage/plugin-catalog-backend@1.3.1-next.2
+
 ## 0.1.6-next.1
 
 ### Patch Changes

package/alpha/package.json

@@ -0,0 +1,6 @@
+{
+  "name": "@backstage/plugin-catalog-backend-module-github",
+  "version": "0.1.7-next.0",
+  "main": "../dist/index.cjs.js",
+  "types": "../dist/index.alpha.d.ts"
+}

package/dist/index.alpha.d.ts

@@ -0,0 +1,249 @@
+/**
+ * A Backstage catalog backend module that helps integrate towards GitHub
+ *
+ * @packageDocumentation
+ */
+
+import { BackendFeature } from '@backstage/backend-plugin-api';
+import { CatalogProcessor } from '@backstage/plugin-catalog-backend';
+import { CatalogProcessorEmit } from '@backstage/plugin-catalog-backend';
+import { Config } from '@backstage/config';
+import { EntityProvider } from '@backstage/plugin-catalog-backend';
+import { EntityProviderConnection } from '@backstage/plugin-catalog-backend';
+import { GithubCredentialsProvider } from '@backstage/integration';
+import { GitHubIntegrationConfig } from '@backstage/integration';
+import { LocationSpec } from '@backstage/plugin-catalog-backend';
+import { Logger } from 'winston';
+import { ScmIntegrationRegistry } from '@backstage/integration';
+import { TaskRunner } from '@backstage/backend-tasks';
+import { TaskScheduleDefinition } from '@backstage/backend-tasks';
+
+/**
+ * Extracts repositories out of a GitHub org.
+ *
+ * The following will create locations for all projects which have a catalog-info.yaml
+ * on the default branch. The first is shorthand for the second.
+ *
+ *    target: "https://github.com/backstage"
+ *    or
+ *    target: https://github.com/backstage/*\/blob/-/catalog-info.yaml
+ *
+ * You may also explicitly specify the source branch:
+ *
+ *    target: https://github.com/backstage/*\/blob/main/catalog-info.yaml
+ *
+ * @public
+ **/
+export declare class GithubDiscoveryProcessor implements CatalogProcessor {
+    private readonly integrations;
+    private readonly logger;
+    private readonly githubCredentialsProvider;
+    static fromConfig(config: Config, options: {
+        logger: Logger;
+        githubCredentialsProvider?: GithubCredentialsProvider;
+    }): GithubDiscoveryProcessor;
+    constructor(options: {
+        integrations: ScmIntegrationRegistry;
+        logger: Logger;
+        githubCredentialsProvider?: GithubCredentialsProvider;
+    });
+    getProcessorName(): string;
+    readLocation(location: LocationSpec, _optional: boolean, emit: CatalogProcessorEmit): Promise<boolean>;
+}
+
+/**
+ * Discovers catalog files located in [GitHub](https://github.com).
+ * The provider will search your GitHub account and register catalog files matching the configured path
+ * as Location entity and via following processing steps add all contained catalog entities.
+ * This can be useful as an alternative to static locations or manually adding things to the catalog.
+ *
+ * @public
+ */
+export declare class GitHubEntityProvider implements EntityProvider {
+    private readonly config;
+    private readonly logger;
+    private readonly integration;
+    private readonly scheduleFn;
+    private connection?;
+    private readonly githubCredentialsProvider;
+    static fromConfig(config: Config, options: {
+        logger: Logger;
+        schedule: TaskRunner;
+    }): GitHubEntityProvider[];
+    private constructor();
+    /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.getProviderName} */
+    getProviderName(): string;
+    /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.connect} */
+    connect(connection: EntityProviderConnection): Promise<void>;
+    private createScheduleFn;
+    refresh(logger: Logger): Promise<void>;
+    private findCatalogFiles;
+    private matchesFilters;
+    private createLocationUrl;
+    private static toLocationSpec;
+}
+
+/**
+ * Registers the GitHubEntityProvider with the catalog processing extension point.
+ *
+ * @alpha
+ */
+export declare const githubEntityProviderCatalogModule: (options?: GithubEntityProviderCatalogModuleOptions | undefined) => BackendFeature;
+
+/**
+ * Options for {@link githubEntityProviderCatalogModule}.
+ *
+ * @alpha
+ */
+export declare type GithubEntityProviderCatalogModuleOptions = {
+    schedule?: TaskScheduleDefinition;
+};
+
+/**
+ * The configuration parameters for a multi-org GitHub processor.
+ * @public
+ */
+export declare type GithubMultiOrgConfig = Array<{
+    /**
+     * The name of the GitHub org to process.
+     */
+    name: string;
+    /**
+     * The namespace of the group created for this org.
+     */
+    groupNamespace: string;
+    /**
+     * The namespace of the users created for this org. If not specified defaults to undefined.
+     */
+    userNamespace: string | undefined;
+}>;
+
+/**
+ * Extracts teams and users out of a multiple GitHub orgs namespaced per org.
+ *
+ * Be aware that this processor may not be compatible with future org structures in the catalog.
+ *
+ * @public
+ */
+export declare class GithubMultiOrgReaderProcessor implements CatalogProcessor {
+    private readonly integrations;
+    private readonly orgs;
+    private readonly logger;
+    private readonly githubCredentialsProvider;
+    static fromConfig(config: Config, options: {
+        logger: Logger;
+        githubCredentialsProvider?: GithubCredentialsProvider;
+    }): GithubMultiOrgReaderProcessor;
+    constructor(options: {
+        integrations: ScmIntegrationRegistry;
+        logger: Logger;
+        orgs: GithubMultiOrgConfig;
+        githubCredentialsProvider?: GithubCredentialsProvider;
+    });
+    getProcessorName(): string;
+    readLocation(location: LocationSpec, _optional: boolean, emit: CatalogProcessorEmit): Promise<boolean>;
+    private getAllOrgs;
+}
+
+/**
+ * Ingests org data (users and groups) from GitHub.
+ *
+ * @public
+ */
+export declare class GitHubOrgEntityProvider implements EntityProvider {
+    private options;
+    private readonly credentialsProvider;
+    private connection?;
+    private scheduleFn?;
+    static fromConfig(config: Config, options: GitHubOrgEntityProviderOptions): GitHubOrgEntityProvider;
+    constructor(options: {
+        id: string;
+        orgUrl: string;
+        gitHubConfig: GitHubIntegrationConfig;
+        logger: Logger;
+        githubCredentialsProvider?: GithubCredentialsProvider;
+    });
+    /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.getProviderName} */
+    getProviderName(): string;
+    /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.connect} */
+    connect(connection: EntityProviderConnection): Promise<void>;
+    /**
+     * Runs one single complete ingestion. This is only necessary if you use
+     * manual scheduling.
+     */
+    read(options?: {
+        logger?: Logger;
+    }): Promise<void>;
+    private schedule;
+}
+
+/**
+ * Options for {@link GitHubOrgEntityProvider}.
+ *
+ * @public
+ */
+export declare interface GitHubOrgEntityProviderOptions {
+    /**
+     * A unique, stable identifier for this provider.
+     *
+     * @example "production"
+     */
+    id: string;
+    /**
+     * The target that this provider should consume.
+     *
+     * @example "https://github.com/backstage"
+     */
+    orgUrl: string;
+    /**
+     * The refresh schedule to use.
+     *
+     * @defaultValue "manual"
+     * @remarks
+     *
+     * If you pass in 'manual', you are responsible for calling the `read` method
+     * manually at some interval.
+     *
+     * But more commonly you will pass in the result of
+     * {@link @backstage/backend-tasks#PluginTaskScheduler.createScheduledTaskRunner}
+     * to enable automatic scheduling of tasks.
+     */
+    schedule?: 'manual' | TaskRunner;
+    /**
+     * The logger to use.
+     */
+    logger: Logger;
+    /**
+     * Optionally supply a custom credentials provider, replacing the default one.
+     */
+    githubCredentialsProvider?: GithubCredentialsProvider;
+}
+
+/**
+ * Extracts teams and users out of a GitHub org.
+ *
+ * @remarks
+ *
+ * Consider using {@link GitHubOrgEntityProvider} instead.
+ *
+ * @public
+ */
+export declare class GithubOrgReaderProcessor implements CatalogProcessor {
+    private readonly integrations;
+    private readonly logger;
+    private readonly githubCredentialsProvider;
+    static fromConfig(config: Config, options: {
+        logger: Logger;
+        githubCredentialsProvider?: GithubCredentialsProvider;
+    }): GithubOrgReaderProcessor;
+    constructor(options: {
+        integrations: ScmIntegrationRegistry;
+        logger: Logger;
+        githubCredentialsProvider?: GithubCredentialsProvider;
+    });
+    getProcessorName(): string;
+    readLocation(location: LocationSpec, _optional: boolean, emit: CatalogProcessorEmit): Promise<boolean>;
+    private createClient;
+}
+
+export { }

package/dist/index.beta.d.ts

@@ -0,0 +1,237 @@
+/**
+ * A Backstage catalog backend module that helps integrate towards GitHub
+ *
+ * @packageDocumentation
+ */
+
+import { BackendFeature } from '@backstage/backend-plugin-api';
+import { CatalogProcessor } from '@backstage/plugin-catalog-backend';
+import { CatalogProcessorEmit } from '@backstage/plugin-catalog-backend';
+import { Config } from '@backstage/config';
+import { EntityProvider } from '@backstage/plugin-catalog-backend';
+import { EntityProviderConnection } from '@backstage/plugin-catalog-backend';
+import { GithubCredentialsProvider } from '@backstage/integration';
+import { GitHubIntegrationConfig } from '@backstage/integration';
+import { LocationSpec } from '@backstage/plugin-catalog-backend';
+import { Logger } from 'winston';
+import { ScmIntegrationRegistry } from '@backstage/integration';
+import { TaskRunner } from '@backstage/backend-tasks';
+import { TaskScheduleDefinition } from '@backstage/backend-tasks';
+
+/**
+ * Extracts repositories out of a GitHub org.
+ *
+ * The following will create locations for all projects which have a catalog-info.yaml
+ * on the default branch. The first is shorthand for the second.
+ *
+ *    target: "https://github.com/backstage"
+ *    or
+ *    target: https://github.com/backstage/*\/blob/-/catalog-info.yaml
+ *
+ * You may also explicitly specify the source branch:
+ *
+ *    target: https://github.com/backstage/*\/blob/main/catalog-info.yaml
+ *
+ * @public
+ **/
+export declare class GithubDiscoveryProcessor implements CatalogProcessor {
+    private readonly integrations;
+    private readonly logger;
+    private readonly githubCredentialsProvider;
+    static fromConfig(config: Config, options: {
+        logger: Logger;
+        githubCredentialsProvider?: GithubCredentialsProvider;
+    }): GithubDiscoveryProcessor;
+    constructor(options: {
+        integrations: ScmIntegrationRegistry;
+        logger: Logger;
+        githubCredentialsProvider?: GithubCredentialsProvider;
+    });
+    getProcessorName(): string;
+    readLocation(location: LocationSpec, _optional: boolean, emit: CatalogProcessorEmit): Promise<boolean>;
+}
+
+/**
+ * Discovers catalog files located in [GitHub](https://github.com).
+ * The provider will search your GitHub account and register catalog files matching the configured path
+ * as Location entity and via following processing steps add all contained catalog entities.
+ * This can be useful as an alternative to static locations or manually adding things to the catalog.
+ *
+ * @public
+ */
+export declare class GitHubEntityProvider implements EntityProvider {
+    private readonly config;
+    private readonly logger;
+    private readonly integration;
+    private readonly scheduleFn;
+    private connection?;
+    private readonly githubCredentialsProvider;
+    static fromConfig(config: Config, options: {
+        logger: Logger;
+        schedule: TaskRunner;
+    }): GitHubEntityProvider[];
+    private constructor();
+    /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.getProviderName} */
+    getProviderName(): string;
+    /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.connect} */
+    connect(connection: EntityProviderConnection): Promise<void>;
+    private createScheduleFn;
+    refresh(logger: Logger): Promise<void>;
+    private findCatalogFiles;
+    private matchesFilters;
+    private createLocationUrl;
+    private static toLocationSpec;
+}
+
+/* Excluded from this release type: githubEntityProviderCatalogModule */
+
+/* Excluded from this release type: GithubEntityProviderCatalogModuleOptions */
+
+/**
+ * The configuration parameters for a multi-org GitHub processor.
+ * @public
+ */
+export declare type GithubMultiOrgConfig = Array<{
+    /**
+     * The name of the GitHub org to process.
+     */
+    name: string;
+    /**
+     * The namespace of the group created for this org.
+     */
+    groupNamespace: string;
+    /**
+     * The namespace of the users created for this org. If not specified defaults to undefined.
+     */
+    userNamespace: string | undefined;
+}>;
+
+/**
+ * Extracts teams and users out of a multiple GitHub orgs namespaced per org.
+ *
+ * Be aware that this processor may not be compatible with future org structures in the catalog.
+ *
+ * @public
+ */
+export declare class GithubMultiOrgReaderProcessor implements CatalogProcessor {
+    private readonly integrations;
+    private readonly orgs;
+    private readonly logger;
+    private readonly githubCredentialsProvider;
+    static fromConfig(config: Config, options: {
+        logger: Logger;
+        githubCredentialsProvider?: GithubCredentialsProvider;
+    }): GithubMultiOrgReaderProcessor;
+    constructor(options: {
+        integrations: ScmIntegrationRegistry;
+        logger: Logger;
+        orgs: GithubMultiOrgConfig;
+        githubCredentialsProvider?: GithubCredentialsProvider;
+    });
+    getProcessorName(): string;
+    readLocation(location: LocationSpec, _optional: boolean, emit: CatalogProcessorEmit): Promise<boolean>;
+    private getAllOrgs;
+}
+
+/**
+ * Ingests org data (users and groups) from GitHub.
+ *
+ * @public
+ */
+export declare class GitHubOrgEntityProvider implements EntityProvider {
+    private options;
+    private readonly credentialsProvider;
+    private connection?;
+    private scheduleFn?;
+    static fromConfig(config: Config, options: GitHubOrgEntityProviderOptions): GitHubOrgEntityProvider;
+    constructor(options: {
+        id: string;
+        orgUrl: string;
+        gitHubConfig: GitHubIntegrationConfig;
+        logger: Logger;
+        githubCredentialsProvider?: GithubCredentialsProvider;
+    });
+    /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.getProviderName} */
+    getProviderName(): string;
+    /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.connect} */
+    connect(connection: EntityProviderConnection): Promise<void>;
+    /**
+     * Runs one single complete ingestion. This is only necessary if you use
+     * manual scheduling.
+     */
+    read(options?: {
+        logger?: Logger;
+    }): Promise<void>;
+    private schedule;
+}
+
+/**
+ * Options for {@link GitHubOrgEntityProvider}.
+ *
+ * @public
+ */
+export declare interface GitHubOrgEntityProviderOptions {
+    /**
+     * A unique, stable identifier for this provider.
+     *
+     * @example "production"
+     */
+    id: string;
+    /**
+     * The target that this provider should consume.
+     *
+     * @example "https://github.com/backstage"
+     */
+    orgUrl: string;
+    /**
+     * The refresh schedule to use.
+     *
+     * @defaultValue "manual"
+     * @remarks
+     *
+     * If you pass in 'manual', you are responsible for calling the `read` method
+     * manually at some interval.
+     *
+     * But more commonly you will pass in the result of
+     * {@link @backstage/backend-tasks#PluginTaskScheduler.createScheduledTaskRunner}
+     * to enable automatic scheduling of tasks.
+     */
+    schedule?: 'manual' | TaskRunner;
+    /**
+     * The logger to use.
+     */
+    logger: Logger;
+    /**
+     * Optionally supply a custom credentials provider, replacing the default one.
+     */
+    githubCredentialsProvider?: GithubCredentialsProvider;
+}
+
+/**
+ * Extracts teams and users out of a GitHub org.
+ *
+ * @remarks
+ *
+ * Consider using {@link GitHubOrgEntityProvider} instead.
+ *
+ * @public
+ */
+export declare class GithubOrgReaderProcessor implements CatalogProcessor {
+    private readonly integrations;
+    private readonly logger;
+    private readonly githubCredentialsProvider;
+    static fromConfig(config: Config, options: {
+        logger: Logger;
+        githubCredentialsProvider?: GithubCredentialsProvider;
+    }): GithubOrgReaderProcessor;
+    constructor(options: {
+        integrations: ScmIntegrationRegistry;
+        logger: Logger;
+        githubCredentialsProvider?: GithubCredentialsProvider;
+    });
+    getProcessorName(): string;
+    readLocation(location: LocationSpec, _optional: boolean, emit: CatalogProcessorEmit): Promise<boolean>;
+    private createClient;
+}
+
+export { }

package/dist/index.cjs.js

@@ -8,6 +8,8 @@ var graphql = require('@octokit/graphql');
 var uuid = require('uuid');
 var catalogModel = require('@backstage/catalog-model');
 var lodash = require('lodash');
+var backendPluginApi = require('@backstage/backend-plugin-api');
+var pluginCatalogNode = require('@backstage/plugin-catalog-node');
 
 function _interopNamespace(e) {
   if (e && e.__esModule) return e;
@@ -713,16 +715,14 @@ class GitHubEntityProvider {
     const repositoryFilter = (_a = this.config.filters) == null ? void 0 : _a.repository;
     const matchingRepositories = repositories.filter((r) => {
       var _a2;
-      return !r.isArchived && repositoryFilter ? repositoryFilter == null ? void 0 : repositoryFilter.test(r.name) : (_a2 = r.defaultBranchRef) == null ? void 0 : _a2.name;
+      return !r.isArchived && (!repositoryFilter || repositoryFilter.test(r.name)) && ((_a2 = r.defaultBranchRef) == null ? void 0 : _a2.name);
     });
     return matchingRepositories;
   }
   createLocationUrl(repository) {
     var _a, _b;
     const branch = ((_a = this.config.filters) == null ? void 0 : _a.branch) || ((_b = repository.defaultBranchRef) == null ? void 0 : _b.name) || "-";
-    const catalogFile = this.config.catalogPath.substring(
-      this.config.catalogPath.lastIndexOf("/") + 1
-    );
+    const catalogFile = this.config.catalogPath.startsWith("/") ? this.config.catalogPath.substring(1) : this.config.catalogPath;
     return `${repository.url}/blob/${branch}/${catalogFile}`;
   }
   static toLocationSpec(target) {
@@ -861,9 +861,39 @@ function withLocations(baseUrl, org, entity) {
   );
 }
 
+const githubEntityProviderCatalogModule = backendPluginApi.createBackendModule({
+  pluginId: "catalog",
+  moduleId: "github-entity-provider",
+  register(env, options) {
+    env.registerInit({
+      deps: {
+        config: backendPluginApi.configServiceRef,
+        catalog: pluginCatalogNode.catalogProcessingExtensionPoint,
+        logger: backendPluginApi.loggerServiceRef,
+        scheduler: backendPluginApi.schedulerServiceRef
+      },
+      async init({ config, catalog, logger, scheduler }) {
+        var _a;
+        const scheduleDef = (_a = options == null ? void 0 : options.schedule) != null ? _a : {
+          frequency: { seconds: 600 },
+          timeout: { seconds: 900 },
+          initialDelay: { seconds: 3 }
+        };
+        catalog.addEntityProvider(
+          GitHubEntityProvider.fromConfig(config, {
+            logger: backendPluginApi.loggerToWinstonLogger(logger),
+            schedule: scheduler.createScheduledTaskRunner(scheduleDef)
+          })
+        );
+      }
+    });
+  }
+});
+
 exports.GitHubEntityProvider = GitHubEntityProvider;
 exports.GitHubOrgEntityProvider = GitHubOrgEntityProvider;
 exports.GithubDiscoveryProcessor = GithubDiscoveryProcessor;
 exports.GithubMultiOrgReaderProcessor = GithubMultiOrgReaderProcessor;
 exports.GithubOrgReaderProcessor = GithubOrgReaderProcessor;
+exports.githubEntityProviderCatalogModule = githubEntityProviderCatalogModule;
 //# sourceMappingURL=index.cjs.js.map