@diquattro/cfnassets 0.6.1

package/src/core/rollup/rollupPackageEntries.ts

@@ -0,0 +1,46 @@
+import { MergedRollupOptions, rollup } from 'rollup';
+import { BatchWarnings } from 'rollup/loadConfigFile';
+import { ZipAssetEntry } from '../zip/ZipAssetEntry.js';
+
+const SourceMapUrl = 'sourceMappingURL';
+
+export async function* rollupPackageEntries(
+  options: MergedRollupOptions[],
+  warnings: BatchWarnings,
+): AsyncIterableIterator<ZipAssetEntry> {
+  for (const inputOptions of options) {
+    const bundle = await rollup(inputOptions);
+
+    for (const outputOptions of inputOptions.output) {
+      const output = await bundle.generate(outputOptions);
+
+      for (const chunkOrAsset of output.output) {
+        let content: Buffer;
+
+        if (chunkOrAsset.type === 'asset') {
+          content = Buffer.from(chunkOrAsset.source);
+        } else if (chunkOrAsset.type === 'chunk') {
+          let code = chunkOrAsset.code;
+
+          if (chunkOrAsset.map) {
+            const url = `${chunkOrAsset.fileName}.map`;
+
+            yield {
+              archivePath: url,
+              content: chunkOrAsset.map.toString(),
+            };
+            code += `//# ${SourceMapUrl}=${url}\n`;
+          }
+
+          content = Buffer.from(code);
+        } else {
+          continue;
+        }
+
+        yield { archivePath: chunkOrAsset.fileName, content };
+        warnings?.flush();
+      }
+    }
+  }
+  warnings?.flush();
+}

package/src/core/zip/ZipAssetEntry.ts

@@ -0,0 +1,11 @@
+import { Readable } from 'stream';
+
+export type EntryContent = Readable | string | Buffer;
+
+export interface ZipAssetEntry {
+  archivePath: string;
+  content:
+    | EntryContent
+    | (() => EntryContent)
+    | (() => PromiseLike<EntryContent>);
+}

package/src/core/zip/getFolderEntries.ts

@@ -0,0 +1,47 @@
+import { createReadStream } from 'fs';
+import { readdir } from 'fs/promises';
+import ignore from 'ignore';
+import { join, relative, resolve } from 'path';
+import { ZipAssetEntry } from './ZipAssetEntry.js';
+
+export interface FolderEntriesOptions {
+  archivePath?: string;
+  ignore?: string[];
+  source: string;
+}
+
+export async function* getFolderEntries({
+  archivePath: archiveBasePath = '/',
+  source,
+  ignore: ignorePaths,
+}: FolderEntriesOptions): AsyncIterableIterator<ZipAssetEntry> {
+  const work = [resolve(source)];
+  const ig = ignore.default().add(ignorePaths || []);
+
+  while (work.length) {
+    const curr = work.pop() as string;
+
+    const entries = await readdir(curr, { withFileTypes: true });
+
+    for (const entry of entries) {
+      const entryPath = join(curr, entry.name);
+
+      let archivePath = relative(source, entryPath);
+      if (entry.isDirectory()) {
+        archivePath += '/';
+      }
+      if (ig.ignores(archivePath)) {
+        continue;
+      }
+
+      if (entry.isDirectory()) {
+        work.push(entryPath);
+      } else if (entry.isFile()) {
+        yield {
+          archivePath: join(archiveBasePath, archivePath),
+          content: () => createReadStream(entryPath),
+        };
+      }
+    }
+  }
+}

package/src/core/zip/getPackageEntries.ts

@@ -0,0 +1,104 @@
+import chalk from 'chalk';
+import childProc from 'child_process';
+import { copyFile, readFile, writeFile } from 'fs/promises';
+import { basename, join } from 'path';
+import { temporaryDirectory } from 'tempy';
+import { getFolderEntries } from './getFolderEntries.js';
+import { ZipAssetEntry } from './ZipAssetEntry.js';
+
+export interface PackageEntriesOptions {
+  archivePath?: string;
+  ignorePaths?: string[];
+  packageArch?: string;
+  packageFilePath: string;
+  packagePlatform?: string;
+  packageLockPath: string;
+  packageNames: string[];
+}
+
+export async function* getPackageEntries({
+  archivePath = 'node_modules',
+  ignorePaths,
+  packageLockPath,
+  packageFilePath,
+  packageArch,
+  packagePlatform,
+  packageNames,
+}: PackageEntriesOptions): AsyncIterableIterator<ZipAssetEntry> {
+  let exec: string[];
+  const npmConfig: string[] = [];
+
+  const lockBasename = basename(packageLockPath);
+  if (lockBasename === 'package-lock.json') {
+    exec = ['npm', 'ci'];
+  } else if (lockBasename === 'yarn.lock') {
+    exec = ['yarn', '--frozen-lockfile'];
+  } else if (lockBasename === 'pnpm-lock.yaml') {
+    exec = ['pnpm', 'install', '--frozen-lockfile'];
+  } else {
+    throw new Error(`unknown lockfile type for path '${packageLockPath}'`);
+  }
+
+  const pkg = JSON.parse(await readFile(packageFilePath, 'utf-8'));
+
+  const newPackageJson = {
+    name: 'build',
+    private: true,
+    dependencies: {} as Record<string, string>,
+  };
+
+  for (const dep of packageNames) {
+    const version =
+      (pkg.dependencies && pkg.dependencies[dep]) ||
+      (pkg.devDependencies && pkg.devDependencies[dep]);
+
+    if (!version) {
+      throw new Error(`cannot find dependency ${dep} in ${packageFilePath}`);
+    }
+
+    newPackageJson.dependencies[dep] = version;
+  }
+
+  if (packageArch) {
+    npmConfig.push(`arch=${packageArch}`);
+  }
+  if (packagePlatform) {
+    npmConfig.push(`platform=${packagePlatform}`);
+  }
+
+  const outDir = temporaryDirectory();
+  await writeFile(join(outDir, 'package.json'), JSON.stringify(newPackageJson));
+  await copyFile(packageLockPath, join(outDir, lockBasename));
+
+  if (npmConfig.length) {
+    await writeFile(join(outDir, '.npmrc'), npmConfig.join('\n') + '\n');
+  }
+
+  const [cmd, ...args] = exec;
+
+  const flags = npmConfig.join(', ');
+  console.log(`\n${chalk.cyan.bold(`${cmd} install`)} ${chalk.gray(flags)}`);
+
+  const proc = childProc.spawn(cmd, args, {
+    cwd: outDir,
+    stdio: 'inherit',
+  });
+
+  await new Promise<void>((resolve, reject) => {
+    proc.on('close', (code) => {
+      if (code === 0) {
+        resolve();
+      } else {
+        reject(new Error(`npm exited with non-zero error code ${code}`));
+      }
+    });
+  });
+
+  console.log(`\n`);
+
+  yield* getFolderEntries({
+    source: join(outDir, 'node_modules'),
+    archivePath,
+    ignore: ignorePaths,
+  });
+}

package/src/core/zip/makeZipPackageStream.ts

@@ -0,0 +1,41 @@
+import archiver from 'archiver';
+import stream from 'stream';
+import { ZipAssetEntry } from './ZipAssetEntry.js';
+
+export async function makeZipPackageStream(
+  entries: AsyncIterable<ZipAssetEntry> | Iterable<ZipAssetEntry>,
+): Promise<stream.Readable> {
+  const zip = archiver('zip', { zlib: { level: 9 } });
+  let error: unknown;
+
+  zip.on('warning', (err: unknown) => {
+    console.error(`rollupPackage: WARN: `, err);
+  });
+
+  zip.on('error', (err: unknown) => {
+    error = err || new Error(`unknown error occurred`);
+  });
+
+  const sortedEntries: ZipAssetEntry[] = [];
+  for await (const entry of entries) {
+    sortedEntries.push(entry);
+  }
+  sortedEntries.sort((a, b) => a.archivePath.localeCompare(b.archivePath));
+
+  for await (const entry of sortedEntries) {
+    if (error) {
+      throw error;
+    }
+
+    const content =
+      typeof entry.content === 'function'
+        ? await entry.content()
+        : entry.content;
+
+    zip.append(content, { name: entry.archivePath, date: new Date(0) });
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-floating-promises
+  zip.finalize();
+  return zip;
+}

package/src/core/zip/zipDir.ts

@@ -0,0 +1,43 @@
+import fs from 'fs';
+import { mkdir } from 'fs/promises';
+import path from 'path';
+import { pipeline } from 'stream/promises';
+import { addBundleInfoToPackageJson } from '../../internal/addBundleInfoToPackageJson.js';
+import { readDotIgnoreForFolder } from '../../internal/readDotIgnoreForFolder.js';
+import { getFolderEntries } from './getFolderEntries.js';
+import { makeZipPackageStream } from './makeZipPackageStream.js';
+
+export interface ZipDirOptions {
+  bundleName?: string;
+  ignorePaths?: string[];
+  outputPath?: string;
+  packagePath?: string;
+}
+
+export async function zipDir(
+  dirname: string,
+  opts?: ZipDirOptions,
+): Promise<void> {
+  const outputPath =
+    opts?.outputPath || `dist/${opts?.bundleName || 'bundle'}.zip`;
+  const fullOutputPath = path.resolve(outputPath);
+
+  const ignorePaths = opts?.ignorePaths || [];
+  ignorePaths.push(
+    ...(await readDotIgnoreForFolder(opts?.packagePath || dirname)),
+  );
+
+  const zip = await makeZipPackageStream(
+    getFolderEntries({ source: dirname, ignore: ignorePaths }),
+  );
+
+  await mkdir(path.dirname(fullOutputPath), { recursive: true });
+  await pipeline(zip, fs.createWriteStream(fullOutputPath));
+
+  if (opts?.packagePath) {
+    await addBundleInfoToPackageJson(opts?.packagePath, {
+      name: opts?.bundleName,
+      path: outputPath,
+    });
+  }
+}

package/src/index.ts

@@ -0,0 +1,25 @@
+// AUTO-GENERATED node ../makeIndex.js
+export * from './core/rollup/makeRollupPackageStream.js';
+export * from './core/rollup/rollupPackageDir.js';
+export * from './core/rollup/rollupPackageEntries.js';
+export * from './core/zip/getFolderEntries.js';
+export * from './core/zip/getPackageEntries.js';
+export * from './core/zip/makeZipPackageStream.js';
+export * from './core/zip/ZipAssetEntry.js';
+export * from './core/zip/zipDir.js';
+export * from './template/AssetAttributes.js';
+export * from './template/AssetBase.js';
+export * from './template/AssetBuilder.js';
+export * from './template/AssetContext.js';
+export * from './template/AssetDefinition.js';
+export * from './template/AssetGenerator.js';
+export * from './template/AssetManifest.js';
+export * from './template/AssetOutput.js';
+export * from './template/AssetRef.js';
+export * from './template/CustomAsset.js';
+export * from './template/DeploymentBundle.js';
+export * from './template/FileSystemDeploymentBundle.js';
+export * from './template/makeAsset.js';
+export * from './template/makeAssetFromPackage.js';
+export * from './template/processAssets.js';
+export * from './template/ProgressStats.js';

package/src/internal/HashStream.ts

@@ -0,0 +1,36 @@
+import { BinaryToTextEncoding, createHash, Hash } from 'crypto';
+import { Transform, TransformCallback, TransformOptions } from 'stream';
+
+export class HashStream extends Transform {
+  private readonly hash: Hash;
+
+  constructor(algorithm?: string | Hash, options?: TransformOptions) {
+    super(options);
+
+    if (!algorithm) {
+      algorithm = 'sha1';
+    }
+    this.hash =
+      typeof algorithm === 'string' ? createHash(algorithm) : algorithm;
+  }
+
+  public digest(): Buffer;
+  public digest(encoding: BinaryToTextEncoding): string;
+  public digest(encoding?: BinaryToTextEncoding): Buffer | string {
+    return encoding ? this.hash.digest(encoding) : this.hash.digest();
+  }
+
+  public override _transform(
+    // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
+    chunk: any,
+    encoding: BufferEncoding,
+    callback: TransformCallback,
+  ): void {
+    try {
+      this.hash.update(chunk);
+      callback(null, chunk);
+    } catch (err: any) {
+      callback(err);
+    }
+  }
+}

package/src/internal/ProgressStream.ts

@@ -0,0 +1,32 @@
+import stream from 'stream';
+
+export class ProgressStream extends stream.Transform {
+  private _transferred = 0;
+
+  constructor(listener?: (total: number, delta: number) => void) {
+    super();
+    if (listener) {
+      this.on('progress', listener);
+    }
+  }
+
+  public get transferred(): number {
+    return this._transferred;
+  }
+
+  public override _transform(
+    // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
+    chunk: any,
+    encoding: BufferEncoding,
+    callback: stream.TransformCallback,
+  ): void {
+    const delta = this.writableObjectMode ? 1 : chunk.length;
+    this._transferred += delta;
+    this.emit('progress', this._transferred, delta);
+    callback(null, chunk);
+  }
+
+  public override eventNames(): (string | symbol)[] {
+    return [...super.eventNames(), 'progress'];
+  }
+}

package/src/internal/addBundleInfoToPackageJson.ts

@@ -0,0 +1,52 @@
+import { readFile, writeFile } from 'fs/promises';
+import { basename, resolve } from 'path';
+
+const PackageJsonName = 'package.json';
+
+export interface BundleInfo {
+  name?: string;
+  path: string;
+}
+
+export async function addBundleInfoToPackageJson(
+  packagePath: string,
+  info: BundleInfo,
+): Promise<void> {
+  const packageFilePath =
+    basename(packagePath) !== PackageJsonName
+      ? resolve(packagePath, PackageJsonName)
+      : packagePath;
+
+  const pkg = JSON.parse(await readFile(packageFilePath, 'utf8'));
+
+  if (info.name) {
+    if (
+      pkg.bundles &&
+      pkg.bundles[info.name] &&
+      pkg.bundles[info.name].path === info.path
+    ) {
+      // don't touch the file if it isn't going to change.
+      return;
+    }
+
+    if (!pkg.bundles || typeof pkg.bundles !== 'object') {
+      pkg.bundles = {};
+    }
+    if (!pkg.bundles[info.name] || typeof pkg.bundles[info.name] !== 'object') {
+      pkg.bundles[info.name] = {};
+    }
+    pkg.bundles[info.name].path = info.path;
+  } else {
+    if (pkg.bundle?.path === info.path) {
+      // don't touch the file if it isn't going to change.
+      return;
+    }
+
+    if (!pkg.bundle || typeof pkg.bundle !== 'object') {
+      pkg.bundle = {};
+    }
+    pkg.bundle.path = info.path;
+  }
+
+  await writeFile(packageFilePath, JSON.stringify(pkg, null, 2) + '\n');
+}

package/src/internal/getPackagePath.ts

@@ -0,0 +1,13 @@
+import path from 'path';
+import { tryStat } from './tryStat.js';
+
+export async function getPackagePath(
+  search: string,
+): Promise<string | undefined> {
+  if (await tryStat(path.resolve(search, 'package.json'))) {
+    return search;
+  }
+  if (await tryStat(path.resolve(process.cwd(), 'package.json'))) {
+    return process.cwd();
+  }
+}

package/src/internal/hash.ts

@@ -0,0 +1,9 @@
+import { createHash } from 'crypto';
+
+export function hash(data: Iterable<string | Buffer>): string {
+  const sha1 = createHash('sha1');
+  for (const part of data) {
+    sha1.update(part);
+  }
+  return sha1.digest('hex');
+}

package/src/internal/readDotIgnoreFile.ts

@@ -0,0 +1,15 @@
+import { readFile } from 'fs/promises';
+
+export async function readDotIgnoreFile(
+  filename: string,
+  ignoreMissing = false,
+): Promise<string[]> {
+  try {
+    return (await readFile(filename, 'utf8')).split('\n');
+  } catch (err: any) {
+    if (!ignoreMissing || err.code !== 'ENOENT') {
+      throw err;
+    }
+    return [];
+  }
+}

package/src/internal/readDotIgnoreForFolder.ts

@@ -0,0 +1,8 @@
+import path from 'path';
+import { readDotIgnoreFile } from './readDotIgnoreFile.js';
+
+export async function readDotIgnoreForFolder(
+  dirname: string,
+): Promise<string[]> {
+  return await readDotIgnoreFile(path.resolve(dirname, '.assetignore'), true);
+}

package/src/internal/tryStat.ts

@@ -0,0 +1,12 @@
+import fs from 'fs';
+
+export async function tryStat(path: string): Promise<fs.Stats | undefined> {
+  try {
+    return await fs.promises.stat(path);
+  } catch (err: any) {
+    if (err.code === 'ENOENT') {
+      return;
+    }
+    throw err;
+  }
+}

package/src/template/AssetAttributes.ts

@@ -0,0 +1,5 @@
+import { AssetRef } from './AssetRef.js';
+
+export interface AssetAttributes {
+  ref: AssetRef;
+}

package/src/template/AssetBase.ts

@@ -0,0 +1,60 @@
+import {
+  BuilderContext,
+  makeParameter,
+  Template,
+  TemplateBuilder,
+  TemplateFragment,
+} from '@awboost/cfntemplate';
+import { AssetBuilder } from './AssetBuilder.js';
+import { AssetContext } from './AssetContext.js';
+import { AssetOutput } from './AssetOutput.js';
+import { AssetRef } from './AssetRef.js';
+
+export abstract class AssetBase implements AssetBuilder {
+  public readonly name: string;
+  public readonly parameters: AssetRef;
+  public readonly ref: AssetRef;
+
+  private readonly paramBuilder: TemplateBuilder;
+
+  constructor(name: string) {
+    this.name = name;
+
+    const [bucketParamBuilder, bucketParam] = makeParameter(
+      `${name}BucketName`,
+      {
+        Type: 'String',
+      },
+    );
+    const [objectParamBuilder, objectParam] = makeParameter(
+      `${name}ObjectKey`,
+      {
+        Type: 'String',
+      },
+    );
+
+    this.paramBuilder = TemplateFragment.compose(
+      bucketParamBuilder,
+      objectParamBuilder,
+    );
+
+    this.parameters = {
+      S3Bucket: bucketParam.name,
+      S3Key: objectParam.name,
+    };
+    this.ref = {
+      S3Bucket: bucketParam.ref,
+      S3Key: objectParam.ref,
+    };
+  }
+
+  public build(template: Template, ctx: BuilderContext): Template {
+    const assets = ctx.get(AssetContext);
+    if (!assets.addAsset(this.name, this)) {
+      return template;
+    }
+    return this.paramBuilder.build(template, ctx);
+  }
+
+  public abstract generate(): PromiseLike<AssetOutput> | AssetOutput;
+}

package/src/template/AssetBuilder.ts

@@ -0,0 +1,8 @@
+import { TemplateBuilder } from '@awboost/cfntemplate';
+import { AssetAttributes } from './AssetAttributes.js';
+import { AssetDefinition } from './AssetDefinition.js';
+
+export interface AssetBuilder
+  extends AssetDefinition,
+    TemplateBuilder,
+    AssetAttributes {}

package/src/template/AssetContext.ts

@@ -0,0 +1,15 @@
+import { AssetDefinition } from './AssetDefinition.js';
+
+export class AssetContext {
+  public static readonly ContextKey = 'AssetContext';
+
+  public readonly assets: AssetDefinition[] = [];
+
+  public addAsset(name: string, value: AssetDefinition): boolean {
+    const add = !this.assets.find((x) => x.name === name);
+    if (add) {
+      this.assets.push(value);
+    }
+    return add;
+  }
+}

package/src/template/AssetDefinition.ts

@@ -0,0 +1,8 @@
+import { AssetGenerator } from './AssetGenerator.js';
+import { AssetRef } from './AssetRef.js';
+
+export interface AssetDefinition {
+  generate: AssetGenerator;
+  name: string;
+  parameters: AssetRef;
+}

package/src/template/AssetGenerator.ts

@@ -0,0 +1,5 @@
+import { AssetOutput } from './AssetOutput.js';
+
+export interface AssetGenerator {
+  (): PromiseLike<AssetOutput> | AssetOutput;
+}

package/src/template/AssetManifest.ts

@@ -0,0 +1,25 @@
+import { array, object, text } from '@fmtk/decoders';
+
+export const AssetManifestMetadataKey = 'DeployAssetManifest';
+
+export interface AssetDescriptor {
+  name: string;
+  key: string;
+  bucketParam: string;
+  keyParam: string;
+}
+
+export interface AssetManifest {
+  assets: AssetDescriptor[];
+}
+
+export const decodeAssetDescriptor = object<AssetDescriptor>({
+  name: text,
+  key: text,
+  bucketParam: text,
+  keyParam: text,
+});
+
+export const decodeAssetManifest = object<AssetManifest>({
+  assets: array(decodeAssetDescriptor),
+});

package/src/template/AssetOutput.ts

@@ -0,0 +1,6 @@
+import { Readable } from 'stream';
+
+export interface AssetOutput {
+  content: Readable;
+  fileName: string;
+}

package/src/template/AssetRef.ts

@@ -0,0 +1,4 @@
+export interface AssetRef {
+  S3Bucket: string;
+  S3Key: string;
+}

package/src/template/CustomAsset.ts

@@ -0,0 +1,11 @@
+import { AssetBase } from './AssetBase.js';
+import { AssetGenerator } from './AssetGenerator.js';
+
+export class CustomAsset extends AssetBase {
+  constructor(
+    name: string,
+    public readonly generate: AssetGenerator,
+  ) {
+    super(name);
+  }
+}