@backstage/backend-defaults 0.3.0-next.2 → 0.3.0-next.3

package/dist/urlReader.d.ts

@@ -1,7 +1,419 @@
+/// <reference types="node" />
 import * as _backstage_backend_plugin_api from '@backstage/backend-plugin-api';
-import * as _backstage_backend_common from '@backstage/backend-common';
+import { UrlReaderService, LoggerService, UrlReaderReadTreeResponse, UrlReaderReadUrlOptions, UrlReaderReadUrlResponse, UrlReaderReadTreeOptions, UrlReaderSearchOptions, UrlReaderSearchResponse } from '@backstage/backend-plugin-api';
+import { AzureIntegration, AzureDevOpsCredentialsProvider, BitbucketCloudIntegration, BitbucketIntegration, BitbucketServerIntegration, GerritIntegration, GithubIntegration, GithubCredentialsProvider, GitLabIntegration, GiteaIntegration, HarnessIntegration, AwsS3Integration } from '@backstage/integration';
+import { Readable } from 'stream';
+import { Config } from '@backstage/config';
+import { AwsCredentialsManager } from '@backstage/integration-aws-node';
+
+/**
+ * A predicate that decides whether a specific {@link @backstage/backend-plugin-api#UrlReaderService} can handle a
+ * given URL.
+ *
+ * @public
+ */
+type UrlReaderPredicateTuple = {
+    predicate: (url: URL) => boolean;
+    reader: UrlReaderService;
+};
+/**
+ * A factory function that can read config to construct zero or more
+ * {@link @backstage/backend-plugin-api#UrlReaderService}s along with a predicate for when it should be used.
+ *
+ * @public
+ */
+type ReaderFactory = (options: {
+    config: Config;
+    logger: LoggerService;
+    treeResponseFactory: ReadTreeResponseFactory;
+}) => UrlReaderPredicateTuple[];
+/**
+ * An options object for {@link ReadUrlResponseFactory} factory methods.
+ *
+ * @public
+ */
+type ReadUrlResponseFactoryFromStreamOptions = {
+    etag?: string;
+    lastModifiedAt?: Date;
+};
+/**
+ * Options that control execution of {@link ReadTreeResponseFactory} methods.
+ *
+ * @public
+ */
+type ReadTreeResponseFactoryOptions = {
+    stream: Readable;
+    subpath?: string;
+    etag: string;
+    filter?: (path: string, info?: {
+        size: number;
+    }) => boolean;
+};
+/**
+ * Options that control {@link ReadTreeResponseFactory.fromReadableArray}
+ * execution.
+ *
+ * @public
+ */
+type FromReadableArrayOptions = Array<{
+    /**
+     * The raw data itself.
+     */
+    data: Readable;
+    /**
+     * The filepath of the data.
+     */
+    path: string;
+    /**
+     * Last modified date of the file contents.
+     */
+    lastModifiedAt?: Date;
+}>;
+/**
+ * A factory for response factories that handle the unpacking and inspection of
+ * complex responses such as archive data.
+ *
+ * @public
+ */
+interface ReadTreeResponseFactory {
+    fromTarArchive(options: ReadTreeResponseFactoryOptions & {
+        /**
+         * Strip the first parent directory of a tar archive.
+         * Defaults to true.
+         */
+        stripFirstDirectory?: boolean;
+    }): Promise<UrlReaderReadTreeResponse>;
+    fromZipArchive(options: ReadTreeResponseFactoryOptions): Promise<UrlReaderReadTreeResponse>;
+    fromReadableArray(options: FromReadableArrayOptions): Promise<UrlReaderReadTreeResponse>;
+}
+
+/**
+ * Implements a {@link @backstage/backend-plugin-api#UrlReaderService} for Azure repos.
+ *
+ * @public
+ */
+declare class AzureUrlReader implements UrlReaderService {
+    private readonly integration;
+    private readonly deps;
+    static factory: ReaderFactory;
+    constructor(integration: AzureIntegration, deps: {
+        treeResponseFactory: ReadTreeResponseFactory;
+        credentialsProvider: AzureDevOpsCredentialsProvider;
+    });
+    read(url: string): Promise<Buffer>;
+    readUrl(url: string, options?: UrlReaderReadUrlOptions): Promise<UrlReaderReadUrlResponse>;
+    readTree(url: string, options?: UrlReaderReadTreeOptions): Promise<UrlReaderReadTreeResponse>;
+    search(url: string, options?: UrlReaderSearchOptions): Promise<UrlReaderSearchResponse>;
+    toString(): string;
+}
+
+/**
+ * Implements a {@link @backstage/backend-plugin-api#UrlReaderService} for files from Bitbucket Cloud.
+ *
+ * @public
+ */
+declare class BitbucketCloudUrlReader implements UrlReaderService {
+    private readonly integration;
+    private readonly deps;
+    static factory: ReaderFactory;
+    constructor(integration: BitbucketCloudIntegration, deps: {
+        treeResponseFactory: ReadTreeResponseFactory;
+    });
+    read(url: string): Promise<Buffer>;
+    readUrl(url: string, options?: UrlReaderReadUrlOptions): Promise<UrlReaderReadUrlResponse>;
+    readTree(url: string, options?: UrlReaderReadTreeOptions): Promise<UrlReaderReadTreeResponse>;
+    search(url: string, options?: UrlReaderSearchOptions): Promise<UrlReaderSearchResponse>;
+    toString(): string;
+    private getLastCommitShortHash;
+}
+
+/**
+ * Implements a {@link @backstage/backend-plugin-api#UrlReaderService} for files from Bitbucket v1 and v2 APIs, such
+ * as the one exposed by Bitbucket Cloud itself.
+ *
+ * @public
+ * @deprecated in favor of BitbucketCloudUrlReader and BitbucketServerUrlReader
+ */
+declare class BitbucketUrlReader implements UrlReaderService {
+    private readonly integration;
+    private readonly deps;
+    static factory: ReaderFactory;
+    constructor(integration: BitbucketIntegration, logger: LoggerService, deps: {
+        treeResponseFactory: ReadTreeResponseFactory;
+    });
+    read(url: string): Promise<Buffer>;
+    readUrl(url: string, options?: UrlReaderReadUrlOptions): Promise<UrlReaderReadUrlResponse>;
+    readTree(url: string, options?: UrlReaderReadTreeOptions): Promise<UrlReaderReadTreeResponse>;
+    search(url: string, options?: UrlReaderSearchOptions): Promise<UrlReaderSearchResponse>;
+    toString(): string;
+    private getLastCommitShortHash;
+}
+
+/**
+ * Implements a {@link @backstage/backend-plugin-api#UrlReaderService} for files from Bitbucket Server APIs.
+ *
+ * @public
+ */
+declare class BitbucketServerUrlReader implements UrlReaderService {
+    private readonly integration;
+    private readonly deps;
+    static factory: ReaderFactory;
+    constructor(integration: BitbucketServerIntegration, deps: {
+        treeResponseFactory: ReadTreeResponseFactory;
+    });
+    read(url: string): Promise<Buffer>;
+    readUrl(url: string, options?: UrlReaderReadUrlOptions): Promise<UrlReaderReadUrlResponse>;
+    readTree(url: string, options?: UrlReaderReadTreeOptions): Promise<UrlReaderReadTreeResponse>;
+    search(url: string, options?: UrlReaderSearchOptions): Promise<UrlReaderSearchResponse>;
+    toString(): string;
+    private getLastCommitShortHash;
+}
+
+/**
+ * Implements a {@link @backstage/backend-plugin-api#UrlReaderService} for files in Gerrit.
+ *
+ * @remarks
+ * To be able to link to Git contents for Gerrit providers in a user friendly
+ * way we are depending on that there is a Gitiles installation somewhere
+ * that we can link to. It is perfectly possible to integrate Gerrit with
+ * Backstage without Gitiles since all API calls goes directly to Gerrit.
+ * However if Gitiles is configured, readTree will use it to fetch
+ * an archive instead of cloning the repository.
+ *
+ * The "host" variable in the config is the Gerrit host. The address where
+ * Gitiles is installed may be on the same host but it could be on a
+ * separate host. For example a Gerrit instance could be hosted on
+ * "gerrit-review.company.com" but the repos could be browsable on a separate
+ * host, e.g. "gerrit.company.com" and the human readable URL would then
+ * not point to the API host.
+ *
+ * @public
+ */
+declare class GerritUrlReader implements UrlReaderService {
+    private readonly integration;
+    private readonly deps;
+    private readonly workDir;
+    static factory: ReaderFactory;
+    constructor(integration: GerritIntegration, deps: {
+        treeResponseFactory: ReadTreeResponseFactory;
+    }, workDir: string);
+    read(url: string): Promise<Buffer>;
+    readUrl(url: string, options?: UrlReaderReadUrlOptions): Promise<UrlReaderReadUrlResponse>;
+    readTree(url: string, options?: UrlReaderReadTreeOptions): Promise<UrlReaderReadTreeResponse>;
+    search(): Promise<UrlReaderSearchResponse>;
+    toString(): string;
+    private readTreeFromGitClone;
+    private readTreeFromGitiles;
+}
+
+/**
+ * Implements a {@link @backstage/backend-plugin-api#UrlReaderService} for files through the GitHub v3 APIs, such as
+ * the one exposed by GitHub itself.
+ *
+ * @public
+ */
+declare class GithubUrlReader implements UrlReaderService {
+    private readonly integration;
+    private readonly deps;
+    static factory: ReaderFactory;
+    constructor(integration: GithubIntegration, deps: {
+        treeResponseFactory: ReadTreeResponseFactory;
+        credentialsProvider: GithubCredentialsProvider;
+    });
+    read(url: string): Promise<Buffer>;
+    private getCredentials;
+    readUrl(url: string, options?: UrlReaderReadUrlOptions): Promise<UrlReaderReadUrlResponse>;
+    readTree(url: string, options?: UrlReaderReadTreeOptions): Promise<UrlReaderReadTreeResponse>;
+    search(url: string, options?: UrlReaderSearchOptions): Promise<UrlReaderSearchResponse>;
+    toString(): string;
+    private doReadTree;
+    private doSearch;
+    private getRepoDetails;
+    private getDefaultBranch;
+    private fetchResponse;
+    private fetchJson;
+}
+
+/**
+ * Implements a {@link @backstage/backend-plugin-api#UrlReaderService} for files on GitLab.
+ *
+ * @public
+ */
+declare class GitlabUrlReader implements UrlReaderService {
+    private readonly integration;
+    private readonly deps;
+    static factory: ReaderFactory;
+    constructor(integration: GitLabIntegration, deps: {
+        treeResponseFactory: ReadTreeResponseFactory;
+    });
+    read(url: string): Promise<Buffer>;
+    readUrl(url: string, options?: UrlReaderReadUrlOptions): Promise<UrlReaderReadUrlResponse>;
+    readTree(url: string, options?: UrlReaderReadTreeOptions): Promise<UrlReaderReadTreeResponse>;
+    search(url: string, options?: UrlReaderSearchOptions): Promise<UrlReaderSearchResponse>;
+    /**
+     * This function splits the input globPattern string into segments using the  path separator /. It then iterates over
+     * the segments from the end of the array towards the beginning, checking if the concatenated string up to that
+     * segment matches the original globPattern using the minimatch function. If a match is found, it continues iterating.
+     * If no match is found, it returns the concatenated string up to the current segment, which is the static part of the
+     * glob pattern.
+     *
+     * E.g. `catalog/foo/*.yaml` will return `catalog/foo`.
+     *
+     * @param globPattern the glob pattern
+     * @private
+     */
+    private getStaticPart;
+    toString(): string;
+    private getGitlabFetchUrl;
+    private getGitlabArtifactFetchUrl;
+    private resolveProjectToId;
+}
+
+/**
+ * Implements a {@link @backstage/backend-plugin-api#UrlReaderService} for the Gitea v1 api.
+ *
+ * @public
+ */
+declare class GiteaUrlReader implements UrlReaderService {
+    private readonly integration;
+    private readonly deps;
+    static factory: ReaderFactory;
+    constructor(integration: GiteaIntegration, deps: {
+        treeResponseFactory: ReadTreeResponseFactory;
+    });
+    read(url: string): Promise<Buffer>;
+    readUrl(url: string, options?: UrlReaderReadUrlOptions): Promise<UrlReaderReadUrlResponse>;
+    readTree(url: string, options?: UrlReaderReadTreeOptions): Promise<UrlReaderReadTreeResponse>;
+    search(): Promise<UrlReaderSearchResponse>;
+    toString(): string;
+    private getLastCommitHash;
+}
+
+/**
+ * Implements a {@link @backstage/backend-plugin-api#UrlReaderService} for the Harness code v1 api.
+ *
+ *
+ * @public
+ */
+declare class HarnessUrlReader implements UrlReaderService {
+    private readonly integration;
+    private readonly deps;
+    static factory: ReaderFactory;
+    constructor(integration: HarnessIntegration, deps: {
+        treeResponseFactory: ReadTreeResponseFactory;
+    });
+    read(url: string): Promise<Buffer>;
+    readUrl(url: string, options?: UrlReaderReadUrlOptions): Promise<UrlReaderReadUrlResponse>;
+    readTree(url: string, options?: UrlReaderReadTreeOptions): Promise<UrlReaderReadTreeResponse>;
+    search(): Promise<UrlReaderSearchResponse>;
+    toString(): string;
+    private getLastCommitHash;
+}
+
+/**
+ * Implements a {@link @backstage/backend-plugin-api#UrlReaderService} for AWS S3 buckets.
+ *
+ * @public
+ */
+declare class AwsS3UrlReader implements UrlReaderService {
+    private readonly credsManager;
+    private readonly integration;
+    private readonly deps;
+    static factory: ReaderFactory;
+    constructor(credsManager: AwsCredentialsManager, integration: AwsS3Integration, deps: {
+        treeResponseFactory: ReadTreeResponseFactory;
+    });
+    /**
+     * If accessKeyId and secretAccessKey are missing, the standard credentials provider chain will be used:
+     * https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/DefaultAWSCredentialsProviderChain.html
+     */
+    private static buildStaticCredentials;
+    private static buildCredentials;
+    private buildS3Client;
+    private retrieveS3ObjectData;
+    read(url: string): Promise<Buffer>;
+    readUrl(url: string, options?: UrlReaderReadUrlOptions): Promise<UrlReaderReadUrlResponse>;
+    readTree(url: string, options?: UrlReaderReadTreeOptions): Promise<UrlReaderReadTreeResponse>;
+    search(): Promise<UrlReaderSearchResponse>;
+    toString(): string;
+}
+
+/**
+ * A {@link @backstage/backend-plugin-api#UrlReaderService} that does a plain fetch of the URL.
+ *
+ * @public
+ */
+declare class FetchUrlReader implements UrlReaderService {
+    /**
+     * The factory creates a single reader that will be used for reading any URL that's listed
+     * in configuration at `backend.reading.allow`. The allow list contains a list of objects describing
+     * targets to allow, containing the following fields:
+     *
+     * `host`:
+     *   Either full hostnames to match, or subdomain wildcard matchers with a leading '*'.
+     *   For example 'example.com' and '*.example.com' are valid values, 'prod.*.example.com' is not.
+     *
+     * `paths`:
+     *   An optional list of paths which are allowed. If the list is omitted all paths are allowed.
+     */
+    static factory: ReaderFactory;
+    read(url: string): Promise<Buffer>;
+    readUrl(url: string, options?: UrlReaderReadUrlOptions): Promise<UrlReaderReadUrlResponse>;
+    readTree(): Promise<UrlReaderReadTreeResponse>;
+    search(): Promise<UrlReaderSearchResponse>;
+    toString(): string;
+}
+
+/**
+ * Utility class for UrlReader implementations to create valid ReadUrlResponse
+ * instances from common response primitives.
+ *
+ * @public
+ */
+declare class ReadUrlResponseFactory {
+    /**
+     * Resolves a ReadUrlResponse from a Readable stream.
+     */
+    static fromReadable(stream: Readable, options?: ReadUrlResponseFactoryFromStreamOptions): Promise<UrlReaderReadUrlResponse>;
+    /**
+     * Resolves a ReadUrlResponse from an old-style NodeJS.ReadableStream.
+     */
+    static fromNodeJSReadable(oldStyleStream: NodeJS.ReadableStream, options?: ReadUrlResponseFactoryFromStreamOptions): Promise<UrlReaderReadUrlResponse>;
+}
+
+/**
+ * Creation options for {@link @backstage/backend-plugin-api#UrlReaderService}.
+ *
+ * @public
+ */
+type UrlReadersOptions = {
+    /** Root config object */
+    config: Config;
+    /** Logger used by all the readers */
+    logger: LoggerService;
+    /** A list of factories used to construct individual readers that match on URLs */
+    factories?: ReaderFactory[];
+};
+/**
+ * Helps construct {@link @backstage/backend-plugin-api#UrlReaderService}s.
+ *
+ * @public
+ */
+declare class UrlReaders {
+    /**
+     * Creates a custom {@link @backstage/backend-plugin-api#UrlReaderService} wrapper for your own set of factories.
+     */
+    static create(options: UrlReadersOptions): UrlReaderService;
+    /**
+     * Creates a {@link @backstage/backend-plugin-api#UrlReaderService} wrapper that includes all the default factories
+     * from this package.
+     *
+     * Any additional factories passed will be loaded before the default ones.
+     */
+    static default(options: UrlReadersOptions): UrlReaderService;
+}
 
 /** @public */
-declare const urlReaderServiceFactory: () => _backstage_backend_plugin_api.ServiceFactory<_backstage_backend_common.UrlReader, "plugin">;
+declare const urlReaderServiceFactory: () => _backstage_backend_plugin_api.ServiceFactory<_backstage_backend_plugin_api.UrlReaderService, "plugin">;
 
-export { urlReaderServiceFactory };
+export { AwsS3UrlReader, AzureUrlReader, BitbucketCloudUrlReader, BitbucketServerUrlReader, BitbucketUrlReader, FetchUrlReader, type FromReadableArrayOptions, GerritUrlReader, GiteaUrlReader, GithubUrlReader, GitlabUrlReader, HarnessUrlReader, type ReadTreeResponseFactory, type ReadTreeResponseFactoryOptions, ReadUrlResponseFactory, type ReadUrlResponseFactoryFromStreamOptions, type ReaderFactory, type UrlReaderPredicateTuple, UrlReaders, type UrlReadersOptions, urlReaderServiceFactory };

package/lifecycle/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@backstage/backend-defaults",
-  "version": "0.3.0-next.2",
+  "version": "0.3.0-next.3",
   "main": "../dist/lifecycle.cjs.js",
   "types": "../dist/lifecycle.d.ts"
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@backstage/backend-defaults",
-  "version": "0.3.0-next.2",
+  "version": "0.3.0-next.3",
   "description": "Backend defaults used by Backstage backend apps",
   "backstage": {
     "role": "node-library"
@@ -97,38 +97,60 @@
     "test": "backstage-cli package test"
   },
   "dependencies": {
-    "@backstage/backend-app-api": "^0.7.6-next.2",
-    "@backstage/backend-common": "^0.23.0-next.2",
+    "@aws-sdk/abort-controller": "^3.347.0",
+    "@aws-sdk/client-codecommit": "^3.350.0",
+    "@aws-sdk/client-s3": "^3.350.0",
+    "@aws-sdk/credential-providers": "^3.350.0",
+    "@aws-sdk/types": "^3.347.0",
+    "@backstage/backend-app-api": "^0.7.6-next.3",
+    "@backstage/backend-common": "^0.23.0-next.3",
     "@backstage/backend-dev-utils": "^0.1.4",
-    "@backstage/backend-plugin-api": "^0.6.19-next.2",
+    "@backstage/backend-plugin-api": "^0.6.19-next.3",
     "@backstage/config": "^1.2.0",
-    "@backstage/config-loader": "^1.8.0",
+    "@backstage/config-loader": "^1.8.1-next.0",
     "@backstage/errors": "^1.2.4",
-    "@backstage/plugin-events-node": "^0.3.5-next.1",
-    "@backstage/plugin-permission-node": "^0.7.30-next.2",
+    "@backstage/integration": "^1.12.0-next.1",
+    "@backstage/integration-aws-node": "^0.1.12",
+    "@backstage/plugin-events-node": "^0.3.5-next.2",
+    "@backstage/plugin-permission-node": "^0.7.30-next.3",
     "@backstage/types": "^1.1.1",
+    "@google-cloud/storage": "^7.0.0",
     "@keyv/memcache": "^1.3.5",
     "@keyv/redis": "^2.5.3",
+    "@octokit/rest": "^19.0.3",
     "@opentelemetry/api": "^1.3.0",
+    "archiver": "^6.0.0",
+    "base64-stream": "^1.0.0",
     "better-sqlite3": "^9.0.0",
+    "concat-stream": "^2.0.0",
     "cron": "^3.0.0",
     "fs-extra": "^11.2.0",
+    "git-url-parse": "^14.0.0",
+    "isomorphic-git": "^1.23.0",
     "keyv": "^4.5.2",
     "knex": "^3.0.0",
     "lodash": "^4.17.21",
     "luxon": "^3.0.0",
+    "minimatch": "^9.0.0",
     "mysql2": "^3.0.0",
+    "node-fetch": "^2.6.7",
     "p-limit": "^3.1.0",
     "pg": "^8.11.3",
     "pg-connection-string": "^2.3.0",
+    "raw-body": "^2.4.1",
+    "tar": "^6.1.12",
     "uuid": "^9.0.0",
+    "yauzl": "^3.0.0",
     "yn": "^4.0.0",
     "zod": "^3.22.4"
   },
   "devDependencies": {
-    "@backstage/backend-plugin-api": "^0.6.19-next.2",
-    "@backstage/backend-test-utils": "^0.4.0-next.2",
-    "@backstage/cli": "^0.26.7-next.2",
+    "@aws-sdk/util-stream-node": "^3.350.0",
+    "@backstage/backend-plugin-api": "^0.6.19-next.3",
+    "@backstage/backend-test-utils": "^0.4.0-next.3",
+    "@backstage/cli": "^0.26.7-next.3",
+    "aws-sdk-client-mock": "^4.0.0",
+    "msw": "^1.0.0",
     "wait-for-expect": "^3.0.2"
   },
   "configSchema": "config.d.ts"

package/permissions/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@backstage/backend-defaults",
-  "version": "0.3.0-next.2",
+  "version": "0.3.0-next.3",
   "main": "../dist/permissions.cjs.js",
   "types": "../dist/permissions.d.ts"
 }

package/rootConfig/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@backstage/backend-defaults",
-  "version": "0.3.0-next.2",
+  "version": "0.3.0-next.3",
   "main": "../dist/rootConfig.cjs.js",
   "types": "../dist/rootConfig.d.ts"
 }

package/rootLifecycle/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@backstage/backend-defaults",
-  "version": "0.3.0-next.2",
+  "version": "0.3.0-next.3",
   "main": "../dist/rootLifecycle.cjs.js",
   "types": "../dist/rootLifecycle.d.ts"
 }

package/scheduler/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@backstage/backend-defaults",
-  "version": "0.3.0-next.2",
+  "version": "0.3.0-next.3",
   "main": "../dist/scheduler.cjs.js",
   "types": "../dist/scheduler.d.ts"
 }

package/urlReader/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@backstage/backend-defaults",
-  "version": "0.3.0-next.2",
+  "version": "0.3.0-next.3",
   "main": "../dist/urlReader.cjs.js",
   "types": "../dist/urlReader.d.ts"
 }