@aws-cdk-testing/cli-integ 3.4.5 → 3.6.0

package/lib/staging/maven.ts

@@ -1,95 +0,0 @@
-/* eslint-disable no-console */
-import * as path from 'path';
-import { pathExists } from 'fs-extra';
-import { LoginInformation } from './codeartifact';
-import { parallelShell } from './parallel-shell';
-import { UsageDir } from './usage-dir';
-import { writeFile } from '../files';
-import { shell } from '../shell';
-
-// Do not try to JIT the Maven binary
-const NO_JIT = '-XX:+TieredCompilation -XX:TieredStopAtLevel=1';
-
-export async function mavenLogin(login: LoginInformation, usageDir: UsageDir) {
-  await writeMavenSettingsFile(settingsFile(usageDir), login);
-
-  // Write env var
-  // Twiddle JVM settings a bit to make Maven survive running on a CodeBuild box.
-  await usageDir.addToEnv({
-    MAVEN_OPTS: `-Duser.home=${usageDir.directory} ${NO_JIT} ${process.env.MAVEN_OPTS ?? ''}`.trim(),
-  });
-}
-
-function settingsFile(usageDir: UsageDir) {
-  // If we configure usageDir as a fake home directory Maven will find this file.
-  // (No other way to configure the settings file as part of the environment).
-  return path.join(usageDir.directory, '.m2', 'settings.xml');
-}
-
-export async function uploadJavaPackages(packages: string[], login: LoginInformation, usageDir: UsageDir) {
-  await parallelShell(packages, async (pkg, output) => {
-    console.log(`⏳ ${pkg}`);
-
-    const sourcesFile = pkg.replace(/.pom$/, '-sources.jar');
-    const javadocFile = pkg.replace(/.pom$/, '-javadoc.jar');
-
-    await shell(['mvn',
-      `--settings=${settingsFile(usageDir)}`,
-      'org.apache.maven.plugins:maven-deploy-plugin:3.0.0:deploy-file',
-      `-Durl=${login.mavenEndpoint}`,
-      '-DrepositoryId=codeartifact',
-      `-DpomFile=${pkg}`,
-      `-Dfile=${pkg.replace(/.pom$/, '.jar')}`,
-      ...await pathExists(sourcesFile) ? [`-Dsources=${sourcesFile}`] : [],
-      ...await pathExists(javadocFile) ? [`-Djavadoc=${javadocFile}`] : []], {
-      outputs: [output],
-      modEnv: {
-        // Do not try to JIT the Maven binary
-        MAVEN_OPTS: `${NO_JIT} ${process.env.MAVEN_OPTS ?? ''}`.trim(),
-      },
-    });
-
-    console.log(`✅ ${pkg}`);
-  },
-  (pkg, output) => {
-    if (output.toString().includes('409 Conflict')) {
-      console.log(`❌ ${pkg}: already exists. Skipped.`);
-      return 'skip';
-    }
-    if (output.toString().includes('Too Many Requests')) {
-      console.log(`♻️ ${pkg}: Too many requests. Retrying.`);
-      return 'retry';
-    }
-    return 'fail';
-  });
-}
-
-export async function writeMavenSettingsFile(filename: string, login: LoginInformation) {
-  await writeFile(filename, `<?xml version="1.0" encoding="UTF-8" ?>
-  <settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
-            xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-            xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0
-                                http://maven.apache.org/xsd/settings-1.0.0.xsd">
-    <servers>
-      <server>
-        <id>codeartifact</id>
-        <username>aws</username>
-        <password>${login.authToken}</password>
-      </server>
-    </servers>
-    <profiles>
-      <profile>
-        <id>default</id>
-        <repositories>
-          <repository>
-            <id>codeartifact</id>
-            <url>${login.mavenEndpoint}</url>
-          </repository>
-        </repositories>
-      </profile>
-    </profiles>
-    <activeProfiles>
-      <activeProfile>default</activeProfile>
-    </activeProfiles>
-  </settings>`);
-}

package/lib/staging/npm.ts

@@ -1,62 +0,0 @@
-/* eslint-disable no-console */
-import * as path from 'path';
-import { LoginInformation } from './codeartifact';
-import { parallelShell } from './parallel-shell';
-import { UsageDir } from './usage-dir';
-import { updateIniKey, loadLines, writeLines } from '../files';
-import { shell } from '../shell';
-
-export async function npmLogin(login: LoginInformation, usageDir: UsageDir) {
-  // Creating an ~/.npmrc that references an envvar is what you're supposed to do. (https://docs.npmjs.com/private-modules/ci-server-config)
-  await writeNpmLoginToken(usageDir, login.npmEndpoint, '${NPM_TOKEN}');
-
-  // Add variables to env file
-  await usageDir.addToEnv(npmEnv(usageDir, login));
-}
-
-function npmEnv(usageDir: UsageDir, login: LoginInformation) {
-  return {
-    npm_config_userconfig: path.join(usageDir.directory, '.npmrc'),
-    npm_config_registry: login.npmEndpoint,
-    npm_config_always_auth: 'true', // Necessary for NPM 6, otherwise it will sometimes not pass the token
-    NPM_TOKEN: login.authToken,
-  };
-}
-
-export async function uploadNpmPackages(packages: string[], login: LoginInformation, usageDir: UsageDir) {
-  await parallelShell(packages, async (pkg, output) => {
-    console.log(`⏳ ${pkg}`);
-
-    // path.resolve() is required -- if the filename ends up looking like `js/bla.tgz` then NPM thinks it's a short form GitHub name.
-    await shell(['node', require.resolve('npm'), 'publish', path.resolve(pkg)], {
-      modEnv: npmEnv(usageDir, login),
-      show: 'error',
-      outputs: [output],
-    });
-
-    console.log(`✅ ${pkg}`);
-  }, (pkg, output) => {
-    if (output.toString().includes('code EPUBLISHCONFLICT')) {
-      console.log(`❌ ${pkg}: already exists. Skipped.`);
-      return 'skip';
-    }
-    if (output.toString().includes('code EPRIVATE')) {
-      console.log(`❌ ${pkg}: is private. Skipped.`);
-      return 'skip';
-    }
-    return 'fail';
-  });
-}
-
-async function writeNpmLoginToken(usageDir: UsageDir, endpoint: string, token: string) {
-  const rcFile = path.join(usageDir.directory, '.npmrc');
-  const lines = await loadLines(rcFile);
-
-  const key = `${endpoint.replace(/^https:/, '')}:_authToken`;
-  updateIniKey(lines, key, token);
-
-  await writeLines(rcFile, lines);
-  return rcFile;
-}
-
-// Environment variable, .npmrc in same directory as package.json or in home dir

package/lib/staging/nuget.ts

@@ -1,75 +0,0 @@
-/* eslint-disable no-console */
-import { LoginInformation } from './codeartifact';
-import { parallelShell } from './parallel-shell';
-import { UsageDir } from './usage-dir';
-import { writeFile } from '../files';
-import { shell } from '../shell';
-
-export async function nugetLogin(login: LoginInformation, usageDir: UsageDir) {
-  // NuGet.Config MUST live in the current directory or in the home directory, and there is no environment
-  // variable to configure its location.
-  await writeNuGetConfigFile(usageDir.cwdFile('NuGet.Config'), login);
-}
-
-export async function uploadDotnetPackages(packages: string[], usageDir: UsageDir) {
-  await usageDir.copyCwdFileHere('NuGet.Config');
-
-  await parallelShell(packages, async (pkg, output) => {
-    console.log(`⏳ ${pkg}`);
-
-    await shell(['dotnet', 'nuget', 'push',
-      pkg,
-      '--source', 'CodeArtifact',
-      '--no-symbols',
-      '--force-english-output',
-      '--disable-buffering',
-      '--timeout', '600',
-      '--skip-duplicate'], {
-      outputs: [output],
-    });
-
-    console.log(`✅ ${pkg}`);
-  },
-  (pkg, output) => {
-    if (output.toString().includes('Conflict')) {
-      console.log(`❌ ${pkg}: already exists. Skipped.`);
-      return 'skip';
-    }
-    if (output.includes('System.Threading.AbandonedMutexException')) {
-      console.log(`♻️ ${pkg}: AbandonedMutexException. Probably a sign of throttling, retrying.`);
-      return 'retry';
-    }
-    if (output.includes('Too Many Requests')) {
-      console.log(`♻️ ${pkg}: Too many requests. Retrying.`);
-      return 'retry';
-    }
-    if (output.includes('System.IO.IOException: The system cannot open the device or file specified.')) {
-      console.log(`♻️ ${pkg}: Some error that we've seen before as a result of throttling. Retrying.`);
-      return 'retry';
-    }
-    return 'fail';
-  });
-}
-
-async function writeNuGetConfigFile(filename: string, login: LoginInformation) {
-  // `dotnet nuget push` has an `--api-key` parameter, but CodeArtifact
-  // does not support that. We must authenticate with Basic auth.
-  await writeFile(filename, `<?xml version="1.0" encoding="utf-8"?>
-<configuration>
-  <packageSources>
-    <add key="nuget.org" value="https://api.nuget.org/v3/index.json" protocolVersion="3" />
-    <add key="CodeArtifact" value="${login.nugetEndpoint}v3/index.json" />
-  </packageSources>
-  <activePackageSource>
-    <add key="CodeArtifact" value="${login.nugetEndpoint}v3/index.json" />
-  </activePackageSource>
-  <packageSourceCredentials>
-    <CodeArtifact>
-        <add key="Username" value="aws" />
-        <add key="ClearTextPassword" value="${login.authToken}" />
-      </CodeArtifact>
-  </packageSourceCredentials>
-</configuration>`);
-}
-
-// NuGet.Config in current directory

package/lib/staging/parallel-shell.ts

@@ -1,51 +0,0 @@
-import PQueue from 'p-queue';
-import { sleep } from '../aws';
-import { MemoryStream } from '../corking';
-
-export type ErrorResponse = 'fail' | 'skip' | 'retry';
-
-/**
- * Run a function in parallel with cached output
- */
-export async function parallelShell<A>(
-  inputs: A[],
-  block: (x: A, output: NodeJS.WritableStream) => Promise<void>,
-  swallowError?: (x: A, output: string) => ErrorResponse,
-) {
-  // Limit to 10 for now, too many instances of Maven exhaust the CodeBuild instance memory
-  const q = new PQueue({ concurrency: Number(process.env.CONCURRENCY) || 10 });
-  await q.addAll(inputs.map(input => async () => {
-    let attempts = 10;
-    let sleepMs = 500;
-    while (true) {
-      const output = new MemoryStream();
-      try {
-        await block(input, output);
-        return;
-      } catch (e) {
-        switch (swallowError?.(input, output.toString())) {
-          case 'skip':
-            return;
-
-          case 'retry':
-            if (--attempts > 0) {
-              await sleep(Math.floor(Math.random() * sleepMs));
-              sleepMs *= 2;
-              continue;
-            }
-            break;
-
-          case 'fail':
-          case undefined:
-            break;
-        }
-
-        // eslint-disable-next-line no-console
-        console.error(output.toString());
-        throw e;
-      }
-    }
-  }));
-
-  await q.onEmpty();
-}

package/lib/staging/pypi.ts

@@ -1,50 +0,0 @@
-/* eslint-disable no-console */
-import * as path from 'path';
-import { LoginInformation } from './codeartifact';
-import { parallelShell } from './parallel-shell';
-import { UsageDir } from './usage-dir';
-import { writeFile } from '../files';
-import { shell } from '../shell';
-
-export async function pypiLogin(login: LoginInformation, usageDir: UsageDir) {
-  // Write pip config file and set environment var
-  await writeFile(path.join(usageDir.directory, 'pip.conf'), [
-    '[global]',
-    `index-url = https://aws:${login.authToken}@${login.pypiEndpoint.replace(/^https:\/\//, '')}simple/`,
-  ].join('\n'));
-  await usageDir.addToEnv({
-    PIP_CONFIG_FILE: `${usageDir.directory}/pip.conf`,
-  });
-}
-
-export async function uploadPythonPackages(packages: string[], login: LoginInformation) {
-  await shell(['pip', 'install', 'twine'], { show: 'error' });
-
-  // Even though twine supports uploading all packages in one go, we have to upload them
-  // individually since CodeArtifact does not support Twine's `--skip-existing`. Fun beans.
-  await parallelShell(packages, async (pkg, output) => {
-    console.log(`⏳ ${pkg}`);
-
-    await shell(['twine', 'upload', '--verbose', pkg], {
-      modEnv: {
-        TWINE_USERNAME: 'aws',
-        TWINE_PASSWORD: login.authToken,
-        TWINE_REPOSITORY_URL: login.pypiEndpoint,
-      },
-      show: 'error',
-      outputs: [output],
-    });
-
-    console.log(`✅ ${pkg}`);
-  }, (pkg, output) => {
-    if (output.toString().includes('This package is configured to block new versions') || output.toString().includes('409 Conflict')) {
-      console.log(`❌ ${pkg}: already exists. Skipped.`);
-      return 'skip';
-    }
-    if (output.includes('429 Too Many Requests ')) {
-      console.log(`♻️ ${pkg}: 429 Too Many Requests. Retrying.`);
-      return 'retry';
-    }
-    return 'fail';
-  });
-}

package/lib/staging/usage-dir.ts

@@ -1,99 +0,0 @@
-import * as path from 'path';
-import * as fs from 'fs-extra';
-import { copyDirectoryContents, homeDir, loadLines, updateIniKey, writeLines } from '../files';
-
-export const DEFAULT_USAGE_DIR = path.join(homeDir(), '.codeartifact/usage');
-
-/**
- * The usage directory is where we write per-session config files to access the CodeArtifact repository.
- *
- * Some config files may be written in a system-global location, but they will not be active unless the
- * contents of this directory have been sourced/copied into the current terminal.
- *
- * CONTRACT
- *
- * There are two special entries:
- *
- * - `env`, a file with `key=value` entries for environment variables to  set.
- * - `cwd/`, a directory with files that need to be copied into the current directory before each command.
- *
- * Other than these, code may write tempfiles to this directory if it wants, but there is no meaning
- * implied for other files.
- */
-export class UsageDir {
-  public static default() {
-    return new UsageDir(DEFAULT_USAGE_DIR);
-  }
-
-  public readonly envFile: string;
-  public readonly cwdDir: string;
-
-  private constructor(public readonly directory: string) {
-    this.envFile = path.join(this.directory, 'env');
-    this.cwdDir = path.join(this.directory, 'cwd');
-  }
-
-  public async clean() {
-    await fs.rm(this.directory, { recursive: true, force: true });
-    await fs.mkdirp(path.join(this.directory, 'cwd'));
-    await fs.writeFile(path.join(this.directory, 'env'), '', { encoding: 'utf-8' });
-
-    await this.addToEnv({
-      CWD_FILES_DIR: path.join(this.directory, 'cwd'),
-    });
-
-    // Write a bash helper to load these settings
-    await fs.writeFile(path.join(this.directory, 'activate.bash'), [
-      `while read -u10 line; do [[ -z $line ]] || export "$line"; done 10<${this.directory}/env`,
-      'cp -R $CWD_FILES_DIR/ .', // Copy files from directory even if it is empty
-    ].join('\n'), { encoding: 'utf-8' });
-  }
-
-  public async addToEnv(settings: Record<string, string>) {
-    const lines = await loadLines(this.envFile);
-    for (const [k, v] of Object.entries(settings)) {
-      updateIniKey(lines, k, v);
-    }
-    await writeLines(this.envFile, lines);
-  }
-
-  public async currentEnv(): Promise<Record<string, string>> {
-    const lines = await loadLines(this.envFile);
-
-    const splitter = /^([a-zA-Z0-9_-]+)\s*=\s*(.*)$/;
-
-    const ret: Record<string, string> = {};
-    for (const line of lines) {
-      const m = line.match(splitter);
-      if (m) {
-        ret[m[1]] = m[2];
-      }
-    }
-    return ret;
-  }
-
-  public cwdFile(filename: string) {
-    return path.join(this.cwdDir, filename);
-  }
-
-  public async activateInCurrentProcess() {
-    for (const [k, v] of Object.entries(await this.currentEnv())) {
-      process.env[k] = v;
-    }
-
-    await copyDirectoryContents(this.cwdDir, '.');
-  }
-
-  public async copyCwdFileHere(...filenames: string[]) {
-    for (const file of filenames) {
-      await fs.copyFile(path.join(this.cwdDir, file), file);
-    }
-  }
-
-  public advertise() {
-    // eslint-disable-next-line no-console
-    console.log('To activate these settings in the current terminal:');
-    // eslint-disable-next-line no-console
-    console.log(`    source ${this.directory}/activate.bash`);
-  }
-}

package/lib/with-aws.ts

@@ -1,111 +0,0 @@
-import { AtmosphereClient } from '@cdklabs/cdk-atmosphere-client';
-import { AwsClients } from './aws';
-import { TestContext } from './integ-test';
-import { ResourcePool } from './resource-pool';
-import { DisableBootstrapContext } from './with-cdk-app';
-
-export function atmosphereEnabled(): boolean {
-  const enabled = process.env.CDK_INTEG_ATMOSPHERE_ENABLED;
-  return enabled === 'true' || enabled === '1';
-}
-
-export function atmosphereEndpoint(): string {
-  const value = process.env.CDK_INTEG_ATMOSPHERE_ENDPOINT;
-  if (!value) {
-    throw new Error('CDK_INTEG_ATMOSPHERE_ENDPOINT is not defined');
-  }
-  return value;
-}
-
-export function atmospherePool() {
-  const value = process.env.CDK_INTEG_ATMOSPHERE_POOL;
-  if (!value) {
-    throw new Error('CDK_INTEG_ATMOSPHERE_POOL is not defined');
-  }
-  return value;
-}
-
-export type AwsContext = { readonly aws: AwsClients };
-
-/**
- * Higher order function to execute a block with an AWS client setup
- *
- * Allocate the next region from the REGION pool and dispose it afterwards.
- */
-export function withAws<A extends TestContext>(
-  block: (context: A & AwsContext & DisableBootstrapContext) => Promise<void>,
-  disableBootstrap: boolean = false,
-): (context: A) => Promise<void> {
-  return async (context: A) => {
-    if (atmosphereEnabled()) {
-      const atmosphere = new AtmosphereClient(atmosphereEndpoint());
-      const allocation = await atmosphere.acquire({ pool: atmospherePool(), requester: context.name, timeoutSeconds: 60 * 30 });
-      const aws = await AwsClients.forIdentity(allocation.environment.region, {
-        accessKeyId: allocation.credentials.accessKeyId,
-        secretAccessKey: allocation.credentials.secretAccessKey,
-        sessionToken: allocation.credentials.sessionToken,
-        accountId: allocation.environment.account,
-      }, context.output);
-
-      await sanityCheck(aws);
-
-      let outcome = 'success';
-      try {
-        return await block({ ...context, disableBootstrap, aws });
-      } catch (e: any) {
-        outcome = 'failure';
-        throw e;
-      } finally {
-        await atmosphere.release(allocation.id, outcome);
-      }
-    } else {
-      return regionPool().using(async (region) => {
-        const aws = await AwsClients.forRegion(region, context.output);
-        await sanityCheck(aws);
-
-        return block({ ...context, disableBootstrap, aws });
-      });
-    }
-  };
-}
-
-let _regionPool: undefined | ResourcePool;
-export function regionPool(): ResourcePool {
-  if (_regionPool !== undefined) {
-    return _regionPool;
-  }
-
-  const REGIONS = process.env.AWS_REGIONS
-    ? process.env.AWS_REGIONS.split(',')
-    : [process.env.AWS_REGION ?? process.env.AWS_DEFAULT_REGION ?? 'us-east-1'];
-
-  _regionPool = ResourcePool.withResources('aws_regions', REGIONS);
-  return _regionPool;
-}
-
-/**
- * Perform a one-time quick sanity check that the AWS clients have properly configured credentials
- *
- * If we don't do this, calls are going to fail and they'll be retried and everything will take
- * forever before the user notices a simple misconfiguration.
- *
- * We can't check for the presence of environment variables since credentials could come from
- * anywhere, so do simple account retrieval.
- *
- * Only do it once per process.
- */
-async function sanityCheck(aws: AwsClients) {
-  if (sanityChecked === undefined) {
-    try {
-      await aws.account();
-      sanityChecked = true;
-    } catch (e: any) {
-      sanityChecked = false;
-      throw new Error(`AWS credentials probably not configured, got error: ${e.message}`);
-    }
-  }
-  if (!sanityChecked) {
-    throw new Error('AWS credentials probably not configured, see previous error');
-  }
-}
-let sanityChecked: boolean | undefined;