@aws-cdk-testing/cli-integ 3.5.0 → 3.7.0

package/lib/staging/codeartifact.ts

@@ -1,387 +0,0 @@
-import {
-  AssociateExternalConnectionCommand,
-  CodeartifactClient,
-  CreateDomainCommand,
-  CreateRepositoryCommand,
-  DeleteRepositoryCommand,
-  DescribeDomainCommand,
-  DescribeRepositoryCommand,
-  GetAuthorizationTokenCommand,
-  GetRepositoryEndpointCommand,
-  ListPackagesCommand,
-  ListPackagesRequest,
-  ListRepositoriesCommand,
-  ListTagsForResourceCommand,
-  PutPackageOriginConfigurationCommand,
-} from '@aws-sdk/client-codeartifact';
-import { sleep } from '../aws';
-
-const COLLECT_BY_TAG = 'collect-by';
-const REPO_LIFETIME_MS = 24 * 3600 * 1000; // One day
-
-export class TestRepository {
-  public static readonly DEFAULT_DOMAIN = 'test-cdk';
-
-  public static async newRandom() {
-    const qualifier = Math.random()
-      .toString(36)
-      .replace(/[^a-z0-9]+/g, '');
-
-    const repo = new TestRepository(`test-${qualifier}`);
-    await repo.prepare();
-    return repo;
-  }
-
-  public static async newWithName(name: string) {
-    const repo = new TestRepository(name);
-    await repo.prepare();
-    return repo;
-  }
-
-  public static existing(repositoryName: string) {
-    return new TestRepository(repositoryName);
-  }
-
-  /**
-   * Garbage collect repositories
-   */
-  public static async gc() {
-    if (!(await TestRepository.existing('*dummy*').domainExists())) {
-      return;
-    }
-
-    const codeArtifact = new CodeartifactClient();
-
-    let nextToken: string | undefined;
-    do {
-      const page = await codeArtifact.send(
-        new ListRepositoriesCommand({
-          nextToken: nextToken,
-        }),
-      );
-
-      for (const repo of page.repositories ?? []) {
-        const tags = await codeArtifact.send(
-          new ListTagsForResourceCommand({
-            resourceArn: repo.arn!,
-          }),
-        );
-        const collectable = tags?.tags?.find((t) => t.key === COLLECT_BY_TAG && Number(t.value) < Date.now());
-        if (collectable) {
-          // eslint-disable-next-line no-console
-          console.log('Deleting', repo.name);
-          await codeArtifact.send(
-            new DeleteRepositoryCommand({
-              domain: repo.domainName!,
-              repository: repo.name!,
-            }),
-          );
-        }
-      }
-
-      nextToken = page.nextToken;
-    } while (nextToken);
-  }
-
-  public readonly npmUpstream = 'npm-upstream';
-  public readonly pypiUpstream = 'pypi-upstream';
-  public readonly nugetUpstream = 'nuget-upstream';
-  public readonly mavenUpstream = 'maven-upstream';
-  public readonly domain = TestRepository.DEFAULT_DOMAIN;
-
-  private readonly codeArtifact = new CodeartifactClient();
-
-  private _loginInformation: LoginInformation | undefined;
-
-  private constructor(public readonly repositoryName: string) {}
-
-  public async prepare() {
-    await this.ensureDomain();
-    await this.ensureUpstreams();
-
-    await this.ensureRepository(this.repositoryName, {
-      description: 'Testing repository',
-      upstreams: [this.npmUpstream, this.pypiUpstream, this.nugetUpstream, this.mavenUpstream],
-      tags: {
-        [COLLECT_BY_TAG]: `${Date.now() + REPO_LIFETIME_MS}`,
-      },
-    });
-  }
-
-  public async loginInformation(): Promise<LoginInformation> {
-    if (this._loginInformation) {
-      return this._loginInformation;
-    }
-
-    this._loginInformation = {
-      authToken: (
-        await this.codeArtifact.send(
-          new GetAuthorizationTokenCommand({
-            domain: this.domain,
-            durationSeconds: 12 * 3600,
-          }),
-        )
-      ).authorizationToken!,
-      repositoryName: this.repositoryName,
-
-      npmEndpoint: (
-        await this.codeArtifact.send(
-          new GetRepositoryEndpointCommand({
-            domain: this.domain,
-            repository: this.repositoryName,
-            format: 'npm',
-          }),
-        )
-      ).repositoryEndpoint!,
-
-      mavenEndpoint: (
-        await this.codeArtifact.send(
-          new GetRepositoryEndpointCommand({
-            domain: this.domain,
-            repository: this.repositoryName,
-            format: 'maven',
-          }),
-        )
-      ).repositoryEndpoint!,
-
-      nugetEndpoint: (
-        await this.codeArtifact.send(
-          new GetRepositoryEndpointCommand({
-            domain: this.domain,
-            repository: this.repositoryName,
-            format: 'nuget',
-          }),
-        )
-      ).repositoryEndpoint!,
-
-      pypiEndpoint: (
-        await this.codeArtifact.send(
-          new GetRepositoryEndpointCommand({
-            domain: this.domain,
-            repository: this.repositoryName,
-            format: 'pypi',
-          }),
-        )
-      ).repositoryEndpoint!,
-    };
-    return this._loginInformation;
-  }
-
-  public async delete() {
-    try {
-      await this.codeArtifact.send(
-        new DeleteRepositoryCommand({
-          domain: this.domain,
-          repository: this.repositoryName,
-        }),
-      );
-
-      // eslint-disable-next-line no-console
-      console.log('Deleted', this.repositoryName);
-    } catch (e: any) {
-      if (e.name !== 'ResourceNotFoundException') {
-        throw e;
-      }
-      // Okay
-    }
-  }
-
-  /**
-   * List all packages and mark them as "allow upstream versions".
-   *
-   * If we don't do this and we publish `foo@2.3.4-rc.0`, then we can't
-   * download `foo@2.3.0` anymore because by default CodeArtifact will
-   * block different versions from the same package.
-   */
-  public async markAllUpstreamAllow() {
-    for await (const pkg of this.listPackages({ upstream: 'BLOCK' })) {
-      await retryThrottled(() =>
-        this.codeArtifact.send(
-          new PutPackageOriginConfigurationCommand({
-            domain: this.domain,
-            repository: this.repositoryName,
-
-            format: pkg.format!,
-            package: pkg.package!,
-            namespace: pkg.namespace!,
-            restrictions: {
-              publish: 'ALLOW',
-              upstream: 'ALLOW',
-            },
-          }),
-        ),
-      );
-    }
-  }
-
-  private async ensureDomain() {
-    if (await this.domainExists()) {
-      return;
-    }
-    await this.codeArtifact.send(
-      new CreateDomainCommand({
-        domain: this.domain,
-        tags: [{ key: 'testing', value: 'true' }],
-      }),
-    );
-  }
-
-  private async ensureUpstreams() {
-    await this.ensureRepository(this.npmUpstream, {
-      description: 'The upstream repository for NPM',
-      external: 'public:npmjs',
-    });
-    await this.ensureRepository(this.mavenUpstream, {
-      description: 'The upstream repository for Maven',
-      external: 'public:maven-central',
-    });
-    await this.ensureRepository(this.nugetUpstream, {
-      description: 'The upstream repository for NuGet',
-      external: 'public:nuget-org',
-    });
-    await this.ensureRepository(this.pypiUpstream, {
-      description: 'The upstream repository for PyPI',
-      external: 'public:pypi',
-    });
-  }
-
-  private async ensureRepository(
-    name: string,
-    options?: {
-      readonly description?: string;
-      readonly external?: string;
-      readonly upstreams?: string[];
-      readonly tags?: Record<string, string>;
-    },
-  ) {
-    if (await this.repositoryExists(name)) {
-      return;
-    }
-
-    await this.codeArtifact.send(
-      new CreateRepositoryCommand({
-        domain: this.domain,
-        repository: name,
-        description: options?.description,
-        upstreams: options?.upstreams?.map((repositoryName) => ({ repositoryName })),
-        tags: options?.tags ? Object.entries(options.tags).map(([key, value]) => ({ key, value })) : undefined,
-      }),
-    );
-
-    if (options?.external) {
-      const externalConnection = options.external;
-      await retry(() =>
-        this.codeArtifact.send(
-          new AssociateExternalConnectionCommand({
-            domain: this.domain,
-            repository: name,
-            externalConnection,
-          }),
-        ),
-      );
-    }
-  }
-
-  private async domainExists() {
-    try {
-      await this.codeArtifact.send(new DescribeDomainCommand({ domain: this.domain }));
-      return true;
-    } catch (e: any) {
-      if (e.name !== 'ResourceNotFoundException') {
-        throw e;
-      }
-      return false;
-    }
-  }
-
-  private async repositoryExists(name: string) {
-    try {
-      await this.codeArtifact.send(new DescribeRepositoryCommand({ domain: this.domain, repository: name }));
-      return true;
-    } catch (e: any) {
-      if (e.name !== 'ResourceNotFoundException') {
-        throw e;
-      }
-      return false;
-    }
-  }
-
-  private async *listPackages(filter: Pick<ListPackagesRequest, 'upstream' | 'publish' | 'format'> = {}) {
-    let response = await retryThrottled(() =>
-      this.codeArtifact.send(
-        new ListPackagesCommand({
-          domain: this.domain,
-          repository: this.repositoryName,
-          ...filter,
-        }),
-      ),
-    );
-
-    while (true) {
-      for (const p of response.packages ?? []) {
-        yield p;
-      }
-
-      if (!response.nextToken) {
-        break;
-      }
-
-      response = await retryThrottled(() =>
-        this.codeArtifact.send(
-          new ListPackagesCommand({
-            domain: this.domain,
-            repository: this.repositoryName,
-            ...filter,
-            nextToken: response.nextToken,
-          }),
-        ),
-      );
-    }
-  }
-}
-
-async function retry<A>(block: () => Promise<A>) {
-  let attempts = 3;
-  while (true) {
-    try {
-      return await block();
-    } catch (e: any) {
-      if (attempts-- === 0) {
-        throw e;
-      }
-      // eslint-disable-next-line no-console
-      console.debug(e.message);
-      await sleep(500);
-    }
-  }
-}
-
-async function retryThrottled<A>(block: () => Promise<A>) {
-  let time = 100;
-  let attempts = 15;
-  while (true) {
-    try {
-      return await block();
-    } catch (e: any) {
-      // eslint-disable-next-line no-console
-      console.debug(e.message);
-      if (e.name !== 'ThrottlingException') {
-        throw e;
-      }
-      if (attempts-- === 0) {
-        throw e;
-      }
-      await sleep(Math.floor(Math.random() * time));
-      time *= 2;
-    }
-  }
-}
-
-export interface LoginInformation {
-  readonly authToken: string;
-  readonly repositoryName: string;
-  readonly npmEndpoint: string;
-  readonly mavenEndpoint: string;
-  readonly nugetEndpoint: string;
-  readonly pypiEndpoint: string;
-}

package/lib/staging/maven.ts

@@ -1,95 +0,0 @@
-/* eslint-disable no-console */
-import * as path from 'path';
-import { pathExists } from 'fs-extra';
-import { LoginInformation } from './codeartifact';
-import { parallelShell } from './parallel-shell';
-import { UsageDir } from './usage-dir';
-import { writeFile } from '../files';
-import { shell } from '../shell';
-
-// Do not try to JIT the Maven binary
-const NO_JIT = '-XX:+TieredCompilation -XX:TieredStopAtLevel=1';
-
-export async function mavenLogin(login: LoginInformation, usageDir: UsageDir) {
-  await writeMavenSettingsFile(settingsFile(usageDir), login);
-
-  // Write env var
-  // Twiddle JVM settings a bit to make Maven survive running on a CodeBuild box.
-  await usageDir.addToEnv({
-    MAVEN_OPTS: `-Duser.home=${usageDir.directory} ${NO_JIT} ${process.env.MAVEN_OPTS ?? ''}`.trim(),
-  });
-}
-
-function settingsFile(usageDir: UsageDir) {
-  // If we configure usageDir as a fake home directory Maven will find this file.
-  // (No other way to configure the settings file as part of the environment).
-  return path.join(usageDir.directory, '.m2', 'settings.xml');
-}
-
-export async function uploadJavaPackages(packages: string[], login: LoginInformation, usageDir: UsageDir) {
-  await parallelShell(packages, async (pkg, output) => {
-    console.log(`⏳ ${pkg}`);
-
-    const sourcesFile = pkg.replace(/.pom$/, '-sources.jar');
-    const javadocFile = pkg.replace(/.pom$/, '-javadoc.jar');
-
-    await shell(['mvn',
-      `--settings=${settingsFile(usageDir)}`,
-      'org.apache.maven.plugins:maven-deploy-plugin:3.0.0:deploy-file',
-      `-Durl=${login.mavenEndpoint}`,
-      '-DrepositoryId=codeartifact',
-      `-DpomFile=${pkg}`,
-      `-Dfile=${pkg.replace(/.pom$/, '.jar')}`,
-      ...await pathExists(sourcesFile) ? [`-Dsources=${sourcesFile}`] : [],
-      ...await pathExists(javadocFile) ? [`-Djavadoc=${javadocFile}`] : []], {
-      outputs: [output],
-      modEnv: {
-        // Do not try to JIT the Maven binary
-        MAVEN_OPTS: `${NO_JIT} ${process.env.MAVEN_OPTS ?? ''}`.trim(),
-      },
-    });
-
-    console.log(`✅ ${pkg}`);
-  },
-  (pkg, output) => {
-    if (output.toString().includes('409 Conflict')) {
-      console.log(`❌ ${pkg}: already exists. Skipped.`);
-      return 'skip';
-    }
-    if (output.toString().includes('Too Many Requests')) {
-      console.log(`♻️ ${pkg}: Too many requests. Retrying.`);
-      return 'retry';
-    }
-    return 'fail';
-  });
-}
-
-export async function writeMavenSettingsFile(filename: string, login: LoginInformation) {
-  await writeFile(filename, `<?xml version="1.0" encoding="UTF-8" ?>
-  <settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
-            xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-            xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0
-                                http://maven.apache.org/xsd/settings-1.0.0.xsd">
-    <servers>
-      <server>
-        <id>codeartifact</id>
-        <username>aws</username>
-        <password>${login.authToken}</password>
-      </server>
-    </servers>
-    <profiles>
-      <profile>
-        <id>default</id>
-        <repositories>
-          <repository>
-            <id>codeartifact</id>
-            <url>${login.mavenEndpoint}</url>
-          </repository>
-        </repositories>
-      </profile>
-    </profiles>
-    <activeProfiles>
-      <activeProfile>default</activeProfile>
-    </activeProfiles>
-  </settings>`);
-}

package/lib/staging/npm.ts

@@ -1,62 +0,0 @@
-/* eslint-disable no-console */
-import * as path from 'path';
-import { LoginInformation } from './codeartifact';
-import { parallelShell } from './parallel-shell';
-import { UsageDir } from './usage-dir';
-import { updateIniKey, loadLines, writeLines } from '../files';
-import { shell } from '../shell';
-
-export async function npmLogin(login: LoginInformation, usageDir: UsageDir) {
-  // Creating an ~/.npmrc that references an envvar is what you're supposed to do. (https://docs.npmjs.com/private-modules/ci-server-config)
-  await writeNpmLoginToken(usageDir, login.npmEndpoint, '${NPM_TOKEN}');
-
-  // Add variables to env file
-  await usageDir.addToEnv(npmEnv(usageDir, login));
-}
-
-function npmEnv(usageDir: UsageDir, login: LoginInformation) {
-  return {
-    npm_config_userconfig: path.join(usageDir.directory, '.npmrc'),
-    npm_config_registry: login.npmEndpoint,
-    npm_config_always_auth: 'true', // Necessary for NPM 6, otherwise it will sometimes not pass the token
-    NPM_TOKEN: login.authToken,
-  };
-}
-
-export async function uploadNpmPackages(packages: string[], login: LoginInformation, usageDir: UsageDir) {
-  await parallelShell(packages, async (pkg, output) => {
-    console.log(`⏳ ${pkg}`);
-
-    // path.resolve() is required -- if the filename ends up looking like `js/bla.tgz` then NPM thinks it's a short form GitHub name.
-    await shell(['node', require.resolve('npm'), 'publish', path.resolve(pkg)], {
-      modEnv: npmEnv(usageDir, login),
-      show: 'error',
-      outputs: [output],
-    });
-
-    console.log(`✅ ${pkg}`);
-  }, (pkg, output) => {
-    if (output.toString().includes('code EPUBLISHCONFLICT')) {
-      console.log(`❌ ${pkg}: already exists. Skipped.`);
-      return 'skip';
-    }
-    if (output.toString().includes('code EPRIVATE')) {
-      console.log(`❌ ${pkg}: is private. Skipped.`);
-      return 'skip';
-    }
-    return 'fail';
-  });
-}
-
-async function writeNpmLoginToken(usageDir: UsageDir, endpoint: string, token: string) {
-  const rcFile = path.join(usageDir.directory, '.npmrc');
-  const lines = await loadLines(rcFile);
-
-  const key = `${endpoint.replace(/^https:/, '')}:_authToken`;
-  updateIniKey(lines, key, token);
-
-  await writeLines(rcFile, lines);
-  return rcFile;
-}
-
-// Environment variable, .npmrc in same directory as package.json or in home dir

package/lib/staging/nuget.ts

@@ -1,75 +0,0 @@
-/* eslint-disable no-console */
-import { LoginInformation } from './codeartifact';
-import { parallelShell } from './parallel-shell';
-import { UsageDir } from './usage-dir';
-import { writeFile } from '../files';
-import { shell } from '../shell';
-
-export async function nugetLogin(login: LoginInformation, usageDir: UsageDir) {
-  // NuGet.Config MUST live in the current directory or in the home directory, and there is no environment
-  // variable to configure its location.
-  await writeNuGetConfigFile(usageDir.cwdFile('NuGet.Config'), login);
-}
-
-export async function uploadDotnetPackages(packages: string[], usageDir: UsageDir) {
-  await usageDir.copyCwdFileHere('NuGet.Config');
-
-  await parallelShell(packages, async (pkg, output) => {
-    console.log(`⏳ ${pkg}`);
-
-    await shell(['dotnet', 'nuget', 'push',
-      pkg,
-      '--source', 'CodeArtifact',
-      '--no-symbols',
-      '--force-english-output',
-      '--disable-buffering',
-      '--timeout', '600',
-      '--skip-duplicate'], {
-      outputs: [output],
-    });
-
-    console.log(`✅ ${pkg}`);
-  },
-  (pkg, output) => {
-    if (output.toString().includes('Conflict')) {
-      console.log(`❌ ${pkg}: already exists. Skipped.`);
-      return 'skip';
-    }
-    if (output.includes('System.Threading.AbandonedMutexException')) {
-      console.log(`♻️ ${pkg}: AbandonedMutexException. Probably a sign of throttling, retrying.`);
-      return 'retry';
-    }
-    if (output.includes('Too Many Requests')) {
-      console.log(`♻️ ${pkg}: Too many requests. Retrying.`);
-      return 'retry';
-    }
-    if (output.includes('System.IO.IOException: The system cannot open the device or file specified.')) {
-      console.log(`♻️ ${pkg}: Some error that we've seen before as a result of throttling. Retrying.`);
-      return 'retry';
-    }
-    return 'fail';
-  });
-}
-
-async function writeNuGetConfigFile(filename: string, login: LoginInformation) {
-  // `dotnet nuget push` has an `--api-key` parameter, but CodeArtifact
-  // does not support that. We must authenticate with Basic auth.
-  await writeFile(filename, `<?xml version="1.0" encoding="utf-8"?>
-<configuration>
-  <packageSources>
-    <add key="nuget.org" value="https://api.nuget.org/v3/index.json" protocolVersion="3" />
-    <add key="CodeArtifact" value="${login.nugetEndpoint}v3/index.json" />
-  </packageSources>
-  <activePackageSource>
-    <add key="CodeArtifact" value="${login.nugetEndpoint}v3/index.json" />
-  </activePackageSource>
-  <packageSourceCredentials>
-    <CodeArtifact>
-        <add key="Username" value="aws" />
-        <add key="ClearTextPassword" value="${login.authToken}" />
-      </CodeArtifact>
-  </packageSourceCredentials>
-</configuration>`);
-}
-
-// NuGet.Config in current directory

package/lib/staging/parallel-shell.ts

@@ -1,51 +0,0 @@
-import PQueue from 'p-queue';
-import { sleep } from '../aws';
-import { MemoryStream } from '../corking';
-
-export type ErrorResponse = 'fail' | 'skip' | 'retry';
-
-/**
- * Run a function in parallel with cached output
- */
-export async function parallelShell<A>(
-  inputs: A[],
-  block: (x: A, output: NodeJS.WritableStream) => Promise<void>,
-  swallowError?: (x: A, output: string) => ErrorResponse,
-) {
-  // Limit to 10 for now, too many instances of Maven exhaust the CodeBuild instance memory
-  const q = new PQueue({ concurrency: Number(process.env.CONCURRENCY) || 10 });
-  await q.addAll(inputs.map(input => async () => {
-    let attempts = 10;
-    let sleepMs = 500;
-    while (true) {
-      const output = new MemoryStream();
-      try {
-        await block(input, output);
-        return;
-      } catch (e) {
-        switch (swallowError?.(input, output.toString())) {
-          case 'skip':
-            return;
-
-          case 'retry':
-            if (--attempts > 0) {
-              await sleep(Math.floor(Math.random() * sleepMs));
-              sleepMs *= 2;
-              continue;
-            }
-            break;
-
-          case 'fail':
-          case undefined:
-            break;
-        }
-
-        // eslint-disable-next-line no-console
-        console.error(output.toString());
-        throw e;
-      }
-    }
-  }));
-
-  await q.onEmpty();
-}