@pwrdrvr/microapps-publish 0.3.5 → 0.4.0-alpha.10

package/src/commands/publish.ts

@@ -4,14 +4,19 @@ import * as lambda from '@aws-sdk/client-lambda';
 import * as s3 from '@aws-sdk/client-s3';
 import * as sts from '@aws-sdk/client-sts';
 import { Command, flags as flagsParser } from '@oclif/command';
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const pMap = require('p-map');
 import * as path from 'path';
 import { pathExists, createReadStream } from 'fs-extra';
 import { Listr, ListrTask } from 'listr2';
 import { createVersions, IVersions } from '../lib/Versions';
 import { Config, IConfig } from '../config/Config';
-import DeployClient, { IDeployVersionPreflightResult } from '../lib/DeployClient';
-import S3Uploader from '../lib/S3Uploader';
-import S3TransferUtility from '../lib/S3TransferUtility';
+import DeployClient, {
+  DeployVersionArgs,
+  IDeployVersionPreflightResult,
+} from '../lib/DeployClient';
+import { S3Uploader } from '../lib/S3Uploader';
+import { S3TransferUtility } from '../lib/S3TransferUtility';
 import { Upload } from '@aws-sdk/lib-storage';
 import { contentType } from 'mime-types';
 import { TaskWrapper } from 'listr2/dist/lib/task-wrapper';
@@ -22,8 +27,6 @@ const lambdaClient = new lambda.LambdaClient({
   maxAttempts: 8,
 });
 
-type DeployVersionArgs = Parameters<typeof DeployClient.DeployVersionLite>[0];
-
 interface IContext {
   preflightResult: IDeployVersionPreflightResult;
   files: string[];
@@ -66,38 +69,78 @@ export class PublishCommand extends Command {
       char: 'v',
     }),
     help: flagsParser.help(),
+    // Deprecated
     deployerLambdaName: flagsParser.string({
+      multiple: false,
+      required: false,
+      hidden: true,
+    }),
+    'deployer-lambda-name': flagsParser.string({
       char: 'd',
       multiple: false,
-      required: true,
+      exactlyOne: ['deployer-lambda-name', 'deployerLambdaName'],
       description: 'Name of the deployer lambda function',
     }),
+    // Deprecated
     newVersion: flagsParser.string({
+      multiple: false,
+      required: false,
+      hidden: true,
+    }),
+    'new-version': flagsParser.string({
       char: 'n',
       multiple: false,
-      required: true,
+      exactlyOne: ['new-version', 'newVersion'],
       description: 'New semantic version to apply',
     }),
+    // Deprecated
     appLambdaName: flagsParser.string({
+      multiple: false,
+      required: false,
+      hidden: true,
+      exclusive: ['app-lambda-name'],
+    }),
+    'app-lambda-name': flagsParser.string({
       char: 'l',
       multiple: false,
       required: false,
       description: 'ARN of lambda version, alias, or function (name or ARN) to deploy',
     }),
+    // Deprecated
     appName: flagsParser.string({
-      char: 'a',
       multiple: false,
       required: false,
+      hidden: true,
+      exclusive: ['app-name'],
+    }),
+    'app-name': flagsParser.string({
+      char: 'a',
+      multiple: false,
+      exactlyOne: ['app-name', 'appName'],
       description: 'MicroApps app name (this becomes the path the app is rooted at)',
     }),
+    // Deprecated
     staticAssetsPath: flagsParser.string({
+      multiple: false,
+      required: false,
+      hidden: true,
+      exclusive: ['static-assets-path'],
+    }),
+    'static-assets-path': flagsParser.string({
       char: 's',
       multiple: false,
       required: false,
       description:
         'Path to files to be uploaded to S3 static bucket at app/version/ path.  Do include app/version/ in path if files are already "rooted" under that path locally.',
     }),
+    // Deprecated
     defaultFile: flagsParser.string({
+      multiple: false,
+      required: false,
+      hidden: true,
+      exclusive: ['default-file'],
+    }),
+    'default-file': flagsParser.string({
       char: 'i',
       multiple: false,
       required: false,
@@ -111,7 +154,13 @@ export class PublishCommand extends Command {
       description:
         'Allow overwrite - Warn but do not fail if version exists. Discouraged outside of test envs if cacheable static files have changed.',
     }),
+    // Deprecated
     noCache: flagsParser.boolean({
+      required: false,
+      default: false,
+      hidden: true,
+    }),
+    'no-cache': flagsParser.boolean({
       required: false,
       default: false,
       description: 'Force revalidation of CloudFront and browser caching of static assets',
@@ -149,14 +198,22 @@ export class PublishCommand extends Command {
     const RUNNING = ''; //chalk.reset.inverse.yellow.bold(RUNNING_TEXT) + ' ';
 
     const { flags: parsedFlags } = this.parse(PublishCommand);
-    const appLambdaName = parsedFlags.appLambdaName ?? config.app.lambdaName;
-    const appName = parsedFlags.appName ?? config.app.name;
-    const deployerLambdaName = parsedFlags.deployerLambdaName ?? config.deployer.lambdaName;
-    const semVer = parsedFlags.newVersion ?? config.app.semVer;
-    const staticAssetsPath = parsedFlags.staticAssetsPath ?? config.app.staticAssetsPath;
-    const defaultFile = parsedFlags.defaultFile ?? config.app.defaultFile;
+    const appLambdaName =
+      parsedFlags.appLambdaName ?? parsedFlags['app-lambda-name'] ?? config.app.lambdaName;
+    const appName = parsedFlags.appName ?? parsedFlags['app-name'] ?? config.app.name;
+    const deployerLambdaName =
+      parsedFlags.deployerLambdaName ??
+      parsedFlags['deployer-lambda-name'] ??
+      config.deployer.lambdaName;
+    const semVer = parsedFlags.newVersion ?? parsedFlags['new-version'] ?? config.app.semVer;
+    const staticAssetsPath =
+      parsedFlags.staticAssetsPath ??
+      parsedFlags['static-assets-path'] ??
+      config.app.staticAssetsPath;
+    const defaultFile =
+      parsedFlags.defaultFile ?? parsedFlags['default-file'] ?? config.app.defaultFile;
     const overwrite = parsedFlags.overwrite;
-    const noCache = parsedFlags.noCache;
+    const noCache = parsedFlags.noCache ?? parsedFlags['no-cache'];
 
     // Override the config value
     config.deployer.lambdaName = deployerLambdaName;
@@ -331,11 +388,11 @@ export class PublishCommand extends Command {
         {
           enabled: () => !!config.app.staticAssetsPath,
           title: 'Enumerate Files to Upload to S3',
-          task: async (ctx, task) => {
+          task: (ctx, task) => {
             const origTitle = task.title;
             task.title = RUNNING + origTitle;
 
-            ctx.files = (await S3TransferUtility.GetFiles(S3Uploader.TempDir)) as string[];
+            ctx.files = S3TransferUtility.GetFiles(S3Uploader.TempDir);
 
             task.title = origTitle;
           },
@@ -343,7 +400,7 @@ export class PublishCommand extends Command {
         {
           enabled: () => !!config.app.staticAssetsPath,
           title: 'Upload Static Files to S3',
-          task: (ctx, task) => {
+          task: async (ctx, task) => {
             const origTitle = task.title;
             task.title = RUNNING + origTitle;
 
@@ -363,50 +420,88 @@ export class PublishCommand extends Command {
 
             // Setup caching on static assets
             // NoCache - Only used for test deploys, requires browser and CloudFront to refetch every time
-            // Overwrite - Reduces default cache time period from 24 hours to 15 minutes
             // Default - 24 hours
             const CacheControl = noCache
               ? 'max-age=0, must-revalidate, public'
-              : overwrite
-              ? `max-age=${15 * 60}, public`
               : `max-age=${24 * 60 * 60}, public`;
 
             const pathWithoutAppAndVer = path.join(S3Uploader.TempDir, destinationPrefix);
 
-            const tasks: ListrTask<IContext>[] = ctx.files.map((filePath) => ({
-              task: async (ctx: IContext, subtask) => {
-                const relFilePath = path.relative(pathWithoutAppAndVer, filePath);
-
-                const origTitle = relFilePath;
-                subtask.title = RUNNING + origTitle;
-
-                const upload = new Upload({
-                  client: s3Client,
-                  leavePartsOnError: false,
-                  params: {
-                    Bucket: bucketName,
-                    Key: path.relative(S3Uploader.TempDir, filePath),
-                    Body: createReadStream(filePath),
-                    ContentType: contentType(path.basename(filePath)) || 'application/octet-stream',
-                    CacheControl,
-                  },
-                });
-                await upload.done();
-
-                subtask.title = origTitle;
-              },
-            }));
-
-            task.title = origTitle;
-
-            return task.newListr(tasks, {
-              concurrent: 8,
-              rendererOptions: {
-                clearOutput: false,
-                showErrorMessage: true,
-                showTimer: true,
-              },
-            });
+            // Listr causes OOM if passes a list of, say, 5,000 to 20,000 files
+            if (ctx.files.length > 200) {
+              const fileCountMsgInterval = Math.floor(ctx.files.length / 10);
+              let filesPublished = 0;
+
+              await pMap(
+                ctx.files,
+                async (filePath: string) => {
+                  // Can't use tasks for each file
+                  const relFilePath = path.relative(pathWithoutAppAndVer, filePath);
+
+                  if (
+                    ctx.files.length > 1000 &&
+                    (filesPublished % fileCountMsgInterval === 0 ||
+                      filesPublished === ctx.files.length)
+                  ) {
+                    task.output = `Uploaded ${filesPublished} of ${ctx.files.length} files`;
+                  } else if (ctx.files.length <= 1000) {
+                    task.output = `Uploading ${relFilePath}`;
+                  }
+
+                  const upload = new Upload({
+                    client: s3Client,
+                    leavePartsOnError: false,
+                    params: {
+                      Bucket: bucketName,
+                      Key: path.relative(S3Uploader.TempDir, filePath),
+                      Body: createReadStream(filePath),
+                      ContentType:
+                        contentType(path.basename(filePath)) || 'application/octet-stream',
+                      CacheControl,
+                    },
+                  });
+                  await upload.done();
+                  filesPublished++;
+                },
+                { concurrency: 40 },
+              );
+            } else {
+              const tasks: ListrTask<IContext>[] = ctx.files.map((filePath) => ({
+                task: async (ctx: IContext, subtask) => {
+                  const relFilePath = path.relative(pathWithoutAppAndVer, filePath);
+
+                  const origTitle = relFilePath;
+                  subtask.title = RUNNING + origTitle;
+
+                  const upload = new Upload({
+                    client: s3Client,
+                    leavePartsOnError: false,
+                    params: {
+                      Bucket: bucketName,
+                      Key: path.relative(S3Uploader.TempDir, filePath),
+                      Body: createReadStream(filePath),
+                      ContentType:
+                        contentType(path.basename(filePath)) || 'application/octet-stream',
+                      CacheControl,
+                    },
+                  });
+                  await upload.done();
+
+                  subtask.title = origTitle;
+                },
+              }));
+
+              task.title = origTitle;
+
+              return task.newListr(tasks, {
+                concurrent: 20,
+                rendererOptions: {
+                  clearOutput: false,
+                  showErrorMessage: true,
+                  showTimer: true,
+                },
+              });
+            }
           },
         },
         {

package/src/lib/DeployClient.ts

@@ -19,7 +19,7 @@ export interface IDeployVersionPreflightResult {
   response: IDeployVersionPreflightResponse;
 }
 
-export type DeployVersionArgs = Parameters<typeof DeployClient.DeployVersionLite>;
+export type DeployVersionArgs = Parameters<typeof DeployClient.DeployVersionLite>[0];
 
 export default class DeployClient {
   static readonly _client = new lambda.LambdaClient({

package/src/lib/S3TransferUtility.spec.ts

@@ -0,0 +1,15 @@
+import { S3TransferUtility } from './S3TransferUtility';
+import path from 'path';
+
+describe('S3TransferUtility', () => {
+  it('collects the files', () => {
+    const testFilesRoot = path.join(__dirname, '../../tests/files');
+    const files = S3TransferUtility.GetFiles(path.join(__dirname, '../../tests/files'));
+
+    expect(files).toBeDefined();
+    expect(files).toHaveLength(6);
+
+    const relativeFiles = files.map((file) => path.relative(testFilesRoot, file));
+    expect(relativeFiles).toMatchSnapshot();
+  });
+});

package/src/lib/S3TransferUtility.ts

@@ -2,7 +2,7 @@
 // From: https://stackoverflow.com/a/65862128/878903
 //
 
-import { promises as fs, createReadStream } from 'fs';
+import { createReadStream, readdirSync } from 'fs';
 import * as path from 'path';
 import * as s3 from '@aws-sdk/client-s3';
 import { Upload } from '@aws-sdk/lib-storage';
@@ -10,7 +10,7 @@ import type { IDeployVersionPreflightResponse } from '@pwrdrvr/microapps-deploye
 import { contentType } from 'mime-types';
 import pMap from 'p-map';
 
-export default class S3TransferUtility {
+export class S3TransferUtility {
   /**
    * @deprecated 2021-11-27
    *
@@ -36,7 +36,7 @@ export default class S3TransferUtility {
     });
 
     // console.log('Uploading files to S3');
-    const files = (await S3TransferUtility.GetFiles(s3Path)) as string[];
+    const files = S3TransferUtility.GetFiles(s3Path);
     // const pathWithoutAppAndVer = path.join(s3Path, destPrefixPath);
     // for (const filePath of files) {
     //   const relFilePath = path.relative(pathWithoutAppAndVer, filePath);
@@ -82,14 +82,32 @@ export default class S3TransferUtility {
    * @param dir
    * @returns
    */
-  public static async GetFiles(dir: string): Promise<string | string[]> {
-    const dirents = await fs.readdir(dir, { withFileTypes: true });
-    const files = await Promise.all(
-      dirents.map((dirent) => {
-        const res = path.resolve(dir, dirent.name);
-        return dirent.isDirectory() ? S3TransferUtility.GetFiles(res) : res;
-      }),
-    );
-    return Array.prototype.concat(...files);
+  public static GetFiles(dir: string): string[] {
+    const dirents = readdirSync(dir, { withFileTypes: true });
+    const files: string[] = [];
+    const toScan = dirents.map((dirent) => ({
+      path: path.resolve(dir, dirent.name),
+      isDirectory: dirent.isDirectory(),
+    }));
+
+    // Iteratively collect all the files in the directories
+    // Do not use function call recursion
+    while (toScan.length > 0) {
+      const dirOrFile = toScan.pop();
+
+      if (dirOrFile.isDirectory) {
+        const direntsChild = readdirSync(dirOrFile.path, { withFileTypes: true });
+        const toScanChild = direntsChild.map((dirent) => ({
+          path: path.resolve(dirOrFile.path, dirent.name),
+          isDirectory: dirent.isDirectory(),
+        }));
+        Array.prototype.push.apply(toScan, toScanChild);
+      } else {
+        files.push(dirOrFile.path);
+      }
+    }
+
+    // Reverse the array so that the files are in the same order as the original
+    return files.reverse();
   }
 }

package/src/lib/S3Uploader.ts

@@ -2,9 +2,9 @@ import * as path from 'path';
 import type { IDeployVersionPreflightResponse } from '@pwrdrvr/microapps-deployer-lib';
 import * as fs from 'fs-extra';
 import { IConfig } from '../config/Config';
-import S3TransferUtility from './S3TransferUtility';
+import { S3TransferUtility } from './S3TransferUtility';
 
-export default class S3Uploader {
+export class S3Uploader {
   /**
    * Copy files to local upload directory
    * @param config

package/src/lib/__snapshots__/S3TransferUtility.spec.ts.snap

@@ -0,0 +1,12 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`S3TransferUtility collects the files 1`] = `
+[
+  "dir1/.touch",
+  "dir2/dir21/dir211/some211-1.json",
+  "dir2/dir21/dir211/some211-2.json",
+  "dir2/dir21/some21.json",
+  "dir2/dir22/some22.json",
+  "some.json",
+]
+`;