@journeyapps/cloudcode-build-agent 0.0.0-dev.1377dae → 0.0.0-dev.2b3e822

package/src/builder.ts

@@ -1,9 +1,8 @@
 import * as path from 'node:path';
 import * as jetpack from 'fs-jetpack';
 import * as semver from 'semver';
-
-import { ensureCustomNodeVersion } from './installer';
-import { detectTasks, DetectedTask } from './detect_tasks';
+import * as utils from './installer';
+import { detectTasks } from './detect_tasks';
 import { runCommand } from './run';
 
 import * as _ from 'lodash';
@@ -15,104 +14,81 @@ export type GeneralTaskConfig = {
   backend_url: string;
 };
 
-export type BuildOptions = {
-  only?: string;
-  custom_node_installation_path?: string;
-};
-
-type ToolPaths = { bin_path: string; node_bin: string; npm_bin: string };
+export async function buildTasks(project_path: string, dest: string, config: GeneralTaskConfig, only?: string) {
+  const tasks = await detectTasks(project_path, only);
 
-export async function buildTasks(
-  project_path: string,
-  dest: string,
-  config: GeneralTaskConfig,
-  options?: BuildOptions
-) {
-  const tasks = await detectTasks(project_path, options?.only);
+  const required_node_versions = _.compact(tasks.map((t) => t.required_node_version));
+  const install_node_versions = _.uniq(required_node_versions);
 
+  // FIXME: Maybe refactor this section into an ensureNodeVersion (or something) that returns the relevant toolpaths?
+  const tool_paths: Record<string, { bin_path: string; node_bin: string; npm_bin: string }> = {};
   // Use the version of nodejs that's running this script as the default.
   const default_tool_paths = {
     bin_path: path.dirname(process.execPath),
     node_bin: process.execPath,
     npm_bin: path.join(path.dirname(process.execPath), 'npm')
   };
-
-  for (const task of tasks) {
-    let custom_tool_paths;
-    if (task.required_node_version) {
-      // FIXME: when defaulting to project_path, the custom node installation is copied into the builder's workdir as well.
-      const custom_node_installation_path = options?.custom_node_installation_path ?? project_path;
-      const custom_node_path = path.resolve(
-        custom_node_installation_path,
-        `node-${semver.major(task.required_node_version)}`
-      );
-      custom_tool_paths = await ensureCustomNodeVersion(task.required_node_version, custom_node_path);
+  tool_paths[process.version] = default_tool_paths;
+
+  for (const required_node_version of install_node_versions) {
+    // FIXME: put this in another directory to avoid leaking into the builder
+    const custom_node_path = path.resolve(`node-${semver.major(required_node_version)}`);
+
+    if (!jetpack.exists(path.join(custom_node_path, 'bin/node'))) {
+      console.debug(`installing to ${custom_node_path}`);
+      await jetpack.dirAsync(custom_node_path);
+      await utils.downloadAndInstallNode(required_node_version, custom_node_path);
+    } else {
+      console.debug(`already installed in ${custom_node_path}`);
     }
-    const node_version = task.required_node_version ?? process.version;
-
-    await buildTask(task, project_path, dest, config, {
-      node_version,
-      ...(custom_tool_paths ?? default_tool_paths)
-    });
-  }
-}
-
-export async function buildTask(
-  task: DetectedTask,
-  project_path: string,
-  dest: string,
-  config: GeneralTaskConfig,
-  node_context: { node_version: string } & ToolPaths
-) {
-  const builder_package = task.builder_package;
-  const builder_script = task.builder_script;
-  const { bin_path, node_bin, npm_bin } = node_context;
-  const builder_bin = path.resolve(bin_path, builder_script);
-
-  if (!jetpack.exists(node_bin)) {
-    throw new Error(`Node binary not found: ${node_bin}`);
+    tool_paths[required_node_version] = utils.nodePaths(custom_node_path);
   }
+  console.debug('----');
 
-  console.debug(
-    `[${task.task_name}] Installing builder script "${builder_package}" for node ${node_context.node_version}`
-  );
+  for (const task of tasks) {
+    const debug = require('debug')(`${task.task_name}`);
+    const node_version = task.required_node_version ?? process.version;
+    const builder_package = task.builder_package;
+    const builder_script = task.builder_script;
+    const { bin_path, node_bin, npm_bin } = tool_paths[node_version];
+    const builder_bin = path.resolve(bin_path, builder_script);
 
-  const stream1 = runCommand(node_bin, [npm_bin, '--global', 'install', builder_package], {
-    cwd: project_path,
-    env: process.env
-  });
-  for await (let event of stream1) {
-    const log = event.stdout ?? event.stderr;
-    if (log) {
-      console.log(`[${task.task_name} - install]`, log.trimRight());
+    if (!jetpack.exists(node_bin)) {
+      throw new Error(`Node binary not found: ${node_bin}`);
     }
-  }
-
-  if (!jetpack.exists(builder_bin)) {
-    console.error(`[${task.task_name}] ${builder_bin} not found`);
-    throw new Error(`Builder script not found for task ${task.task_name}`);
-  }
-
-  const builder_args: string[] = [];
-  builder_args.push(builder_bin);
-  builder_args.push('--src', project_path);
-  builder_args.push(`--out`, path.join(dest, task.task_name));
-  builder_args.push(`--zip`, path.join(dest, `${task.task_name}.zip`));
-  builder_args.push(`--task`, `${task.task_name}`);
-  builder_args.push(`--appId`, config.app_id);
-  builder_args.push(`--env`, config.env);
-  builder_args.push(`--backendId`, config.backend_id);
-  builder_args.push(`--backendUrl`, config.backend_url);
-  builder_args.push(`--runInstallScripts`, 'true');
-
-  console.debug(`[${task.task_name}] Trying: ${node_bin} ${builder_args.join(' ')}`);
 
-  const stream2 = runCommand(node_bin, builder_args, { cwd: project_path, env: process.env });
-
-  for await (let event of stream2) {
-    const log = event.stdout ?? event.stderr;
-    if (log) {
-      console.log(`[${task.task_name} - build]`, log.trimRight());
+    // console.debug(`[${task.task_name}] Installing builder script "${builder_package}" for node ${node_version}`);
+    debug(`Installing builder script "${builder_package}" for node ${node_version}`);
+    // TODO: This may need to be replaced with the something like the spawn-stream stuff in build-agent? OR what to do with the stdout/stderr and errorhandling?
+    await runCommand(
+      node_bin,
+      [npm_bin, '--global', 'install', builder_package],
+      { cwd: project_path },
+      { task_name: task.task_name }
+    );
+
+    if (!jetpack.exists(builder_bin)) {
+      console.error(`[${task.task_name}] ${builder_bin} not found`);
+      throw new Error(`Builder script not found for task ${task.task_name}`);
     }
+
+    const builder_args: string[] = [];
+    // --src ./ --out ./dist/echo --task echo --appId 'appid' --env 'testing' --backendId 'backendid' --backendUrl 'http://run.journeyapps.test'
+    builder_args.push(builder_bin);
+    builder_args.push('--src', project_path);
+    builder_args.push(`--out`, path.join(dest, task.task_name));
+    builder_args.push(`--zip`, path.join(dest, `${task.task_name}.zip`));
+    builder_args.push(`--task`, `${task.task_name}`);
+    builder_args.push(`--appId`, config.app_id);
+    builder_args.push(`--env`, config.env);
+    builder_args.push(`--backendId`, config.backend_id);
+    builder_args.push(`--backendUrl`, config.backend_url);
+    // builder_args.push(`--runInstallScripts`, 'true'); // TODO: handle this from feature flags somehow
+
+    // console.debug(`[${task.task_name}] Trying: ${node_bin} ${builder_args.join(' ')}`);
+    debug(`Trying: ${node_bin} ${builder_args.join(' ')}`);
+
+    await runCommand(node_bin, builder_args, { cwd: project_path }, { task_name: task.task_name });
+    console.debug('----');
   }
 }

package/src/cli.ts

@@ -5,8 +5,7 @@ import * as yargs from 'yargs';
 import { buildTasks } from './';
 
 const DEFAULT_SRC_PATH = './';
-const DEFAULT_OUT_PATH = './dist/cloudcode';
-const DEFAULT_TMP_PATH = '/tmp/cloudcode-build-agent/node';
+const DEFAULT_OUT_PATH = './dist';
 
 yargs
   .command(
@@ -18,7 +17,6 @@ yargs
           .option('only', { string: true })
           .option('path', { default: DEFAULT_SRC_PATH })
           .option('out', { default: DEFAULT_OUT_PATH })
-          .option('node_work_dir', { default: DEFAULT_TMP_PATH })
           // TODO: investigate yargs.config() for reading task config blob from a file. But how to ensure required args?
           .option('appId', { string: true, demandOption: true, describe: "CloudCode's reference id" })
           .option('env', { string: true, demandOption: true })
@@ -33,10 +31,7 @@ yargs
         backend_id: argv.backendId,
         backend_url: argv.backendUrl
       };
-      await buildTasks(argv.path, argv.out, config, {
-        only: argv.only,
-        custom_node_installation_path: argv.node_work_dir
-      });
+      await buildTasks(argv.path, argv.out, config, argv.only);
     })
   )
   .option('verbose', {

package/src/defs.ts

@@ -1,34 +1,45 @@
 // TODO: maybe publish (some of) this from the CC service?
 export const SUPPORTED_VERSIONS = [
-  // {
-  //   // proposed
-  //   version: '1.13.0',
-  //   node: '16.19.1',
-  //   builder_package: '@journeyapps/cloudcode-build@1.13.0',
-  //   builder_script: 'cloudcode-build'
-  // },
+  {
+    // proposed
+    version: '1.13.0',
+    node: '16.19.1',
+    builder_package: '@journeyapps/cloudcode-build@1.13.0',
+    builder_script: 'cloudcode-build'
+  },
   {
     version: '1.12.0',
-    node: '16.19.1', // FIXME: maybe the very specific node version here is too brittle? Should we just specify the major version?
-    builder_package: '@journeyapps/cloudcode-build-legacy@1.12.0',
+    node: '16.19.1', // FIXME: maybe the very specific node version here is too brittle?
+    // builder_package: '@journeyapps/cloudcode-build-legacy@1.12.0'
+    // DEV HACK:
+    builder_package: '~/src/journey-cloudcode/packages/cloudcode-build-legacy',
     builder_script: 'cloudcode-build-legacy'
   },
   {
     version: '1.11.2',
-    node: '14.21.3',
-    builder_package: '@journeyapps/cloudcode-build-legacy@1.12.0',
+    // node: '14.21.3',
+    // builder_package: '@journeyapps/cloudcode-build-legacy@1.12.0'
+    // DEV HACK:
+    node: '16.19.1', // because local dev on darwin/arm64 and there's no darwin-arm64 builds for node14 available.
+    builder_package: '~/src/journey-cloudcode/packages/cloudcode-build-legacy',
     builder_script: 'cloudcode-build-legacy'
   },
   {
     version: '1.11.1',
-    node: '14.21.3',
-    builder_package: '@journeyapps/cloudcode-build-legacy@1.12.0',
+    // node: '14.21.3',
+    // builder_package: '@journeyapps/cloudcode-build-legacy@1.12.0'
+    // DEV HACK:
+    node: '16.19.1', // because local dev on darwin/arm64 and there's no darwin-arm64 builds for node14 available.
+    builder_package: '~/src/journey-cloudcode/packages/cloudcode-build-legacy',
     builder_script: 'cloudcode-build-legacy'
   },
   {
     version: '1.11.0',
-    node: '14.21.3',
-    builder_package: '@journeyapps/cloudcode-build-legacy@1.12.0',
+    // node: '14.21.3',
+    // builder_package: '@journeyapps/cloudcode-build-legacy@1.12.0'
+    // DEV HACK:
+    node: '16.19.1', // because local dev on darwin/arm64 and there's no darwin-arm64 builds for node14 available.
+    builder_package: '~/src/journey-cloudcode/packages/cloudcode-build-legacy',
     builder_script: 'cloudcode-build-legacy'
   }
 ];

package/src/detect_tasks.ts

@@ -4,6 +4,7 @@ import * as jetpack from 'fs-jetpack';
 import * as semver from 'semver';
 import * as defs from './defs';
 
+// TODO: validations for cloudcode specific keys and structure?
 async function readPackage(path: string) {
   try {
     const content = await fs.promises.readFile(path, { encoding: 'utf-8' });
@@ -16,16 +17,7 @@ async function readPackage(path: string) {
   }
 }
 
-export type DetectedTask = {
-  pkg_path: string;
-  task_name: string;
-  task_version: string;
-  required_node_version?: string;
-  builder_package: string;
-  builder_script: string;
-};
-
-export async function detectTasks(project_path: string, only?: string): Promise<DetectedTask[]> {
+export async function detectTasks(project_path: string, only?: string) {
   const package_files = await jetpack.findAsync(path.join(project_path, 'cloudcode'), { matching: '**/package.json' });
   const filtered_package_files = package_files.filter((pkg_path) => {
     // FIXME: this is kinda clunky.
@@ -63,8 +55,9 @@ export async function detectTasks(project_path: string, only?: string): Promise<
 
       const running_node_version = process.versions.node;
       let required_node_version;
-      if (matching?.node && semver.major(matching.node) !== semver.major(running_node_version)) {
-        console.debug(`Task requires different node version: v${matching.node}. Running: ${running_node_version}`);
+      // if (matching?.node && semver.major(matching.node) !== semver.major(running_node_version)) {
+      if (matching?.node && semver.parse(matching.node) !== semver.parse(running_node_version)) {
+        console.debug(`Task requires different node version: v${matching.node}`);
 
         required_node_version = matching.node;
       }

package/src/installer.ts

@@ -1,22 +1,10 @@
-import * as jetpack from 'fs-jetpack';
-import * as path from 'node:path';
-import * as URL from 'node:url';
 import fetch from 'node-fetch';
+import * as path from 'node:path';
 import * as fs from 'node:fs';
 import * as os from 'node:os';
+import * as URL from 'node:url';
 import * as tar from 'tar';
 
-export async function ensureCustomNodeVersion(node_version: string, install_path: string) {
-  if (!jetpack.exists(path.join(install_path, 'bin/node'))) {
-    console.debug(`[node ${node_version}] Installing to ${install_path}`);
-    await jetpack.dirAsync(install_path);
-    await downloadAndInstallNode(node_version, install_path);
-  } else {
-    console.debug(`[node ${node_version}] Already installed in ${install_path}`);
-  }
-  return nodePaths(install_path);
-}
-
 /* Basically this, but for different dirs.
     curl https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-${PLATFORM}64.tar.gz -L | tar -xzC /node-dedicated && \
     mv /node-dedicated/node-v${NODE_VERSION}-linux-${PLATFORM}64/bin/node /node-dedicated/node 
@@ -24,11 +12,17 @@ export async function ensureCustomNodeVersion(node_version: string, install_path
 // TODO: feature to find the latest minor and patch for a given major version?
 export async function downloadAndInstallNode(node_version: string, destination: string) {
   // eg. https://nodejs.org/dist/v18.14.2/node-v18.14.2-linux-x64.tar.xz
+
+  // const PLATFORM = 'x';
+  // const node_download_url = `https://nodejs.org/dist/v${node_version}/node-v${node_version}-linux-${PLATFORM}64.tar.gz`;
+
+  // TODO: this is just for dev for now. Find a better fix.
   const ARCH = os.arch();
   const PLATFORM = os.platform();
   const node_download_url = `https://nodejs.org/dist/v${node_version}/node-v${node_version}-${PLATFORM}-${ARCH}.tar.gz`;
 
   await downloadAndExtractTarball(node_download_url, destination);
+  // TODO: move binaries into local bin path?
   return nodePaths(destination);
 }
 
@@ -61,12 +55,15 @@ export async function downloadAndExtractTarball(url: string, dest: string) {
       throw new Error(`Failed to download: ${errorBody}`);
     }
 
+    // FIXME: replace with webstreams? but according to the types package webstreams aren't available in node 16?!??
     const file = fs.createWriteStream(download); // Sink the download stream into a file, so it can be extracted.
     response.body.pipe(file);
 
     file.on('finish', async () => {
       try {
-        console.debug('Extracting...', download);
+        console.log('Extracting...', download);
+
+        // const comp = fs.createReadStream(download);
 
         tar.extract({
           cwd: tmpdir,
@@ -82,29 +79,28 @@ export async function downloadAndExtractTarball(url: string, dest: string) {
         //   fs.rmSync(dist, { recursive: true });
         // }
         if (fs.existsSync(uncomp)) {
-          console.debug(`Moving extracted files from ${uncomp} to ${dist}`);
+          console.log(`Moving extracted files from ${uncomp} to ${dist}`);
           fs.renameSync(uncomp, dist);
         } else {
-          console.debug(`Can't find extract dir ${uncomp}`);
+          console.log(`Can't find extract dir ${uncomp}`);
         }
 
+        // FIXME: this seems to sometimes cause errors
         /*
-          FIXME: this seems to sometimes cause errors: eg.
-          ```
-            node:events:505
-                  throw er; // Unhandled 'error' event
-                  ^
-
-            Error: ENOENT: no such file or directory, open '/var/folders/kc/h6m4zpmd23v13s63mygvkslm0000gn/T/node-v16.19.1-darwin-arm64.tar.gz'
-            Emitted 'error' event on ReadStream instance at:
-                at emitErrorNT (node:internal/streams/destroy:157:8)
-                at emitErrorCloseNT (node:internal/streams/destroy:122:3)
-                at processTicksAndRejections (node:internal/process/task_queues:83:21) {
-              errno: -2,
-              code: 'ENOENT',
-              syscall: 'open',
-              path: '/var/folders/kc/h6m4zpmd23v13s63mygvkslm0000gn/T/node-v16.19.1-darwin-arm64.tar.gz'
-          ```
+        ----
+node:events:505
+      throw er; // Unhandled 'error' event
+      ^
+
+Error: ENOENT: no such file or directory, open '/var/folders/kc/h6m4zpmd23v13s63mygvkslm0000gn/T/node-v16.19.1-darwin-arm64.tar.gz'
+Emitted 'error' event on ReadStream instance at:
+    at emitErrorNT (node:internal/streams/destroy:157:8)
+    at emitErrorCloseNT (node:internal/streams/destroy:122:3)
+    at processTicksAndRejections (node:internal/process/task_queues:83:21) {
+  errno: -2,
+  code: 'ENOENT',
+  syscall: 'open',
+  path: '/var/folders/kc/h6m4zpmd23v13s63mygvkslm0000gn/T/node-v16.19.1-darwin-arm64.tar.gz'
         */
         // fs.unlinkSync(file.path);
 

package/src/run.ts

@@ -1,23 +1,112 @@
-import { spawnStream, SpawnStreamEvent } from './spawn-stream';
-import { SpawnOptions } from 'child_process';
+import { spawn, Options } from 'child-process-promise';
+import type { SpawnOptions } from 'child_process';
 
-export async function* runCommand(
+export async function runCommand(
   command: string,
-  args: string[],
-  options?: Partial<SpawnOptions>
-): AsyncIterable<SpawnStreamEvent> {
-  const { childProcess, stream } = spawnStream(command, args, { cwd: options?.cwd, splitLines: true, ...options });
+  commandArgs: string[],
+  options: RunCommandOpts = {},
+  context: { task_name: string }
+) {
+  const debug = require('debug')(`${context.task_name} build`);
+  const errors: string[] = [];
 
-  let exitCode: number | null = null;
-
-  for await (let event of stream) {
-    yield event;
-    if (event.exitCode != null) {
-      exitCode = event.exitCode;
+  const defaultLogLine = (isStderr: boolean, line: string, childProcess: any) => {
+    if (isStderr) {
+      if (line.startsWith('warning package.json: No license field')) {
+        // This is a noisy yarn warning, ignore it.
+        return;
+      }
+      errors.push(line);
+    }
+  };
+  let result;
+  try {
+    result = await spawnPromise(command, commandArgs, { logLine: defaultLogLine, ...options }, context);
+  } catch (err) {
+    if (errors.length > 0) {
+      // Handled below
+    } else {
+      // Exit code without any logged errors.
+      throw err;
     }
   }
 
-  if (exitCode != 0) {
-    throw new Error(`Command failed with code ${exitCode}`);
+  if (result) {
+    debug('exit code:', result.code);
+  }
+
+  if (errors.length > 0) {
+    // We consider this a failure, regardless of exit code.
+    // In general, we expect errors to be accompanied by an error code of 1 (and vice versa),
+    // but we do not depend on that behaviour.
+    throw new Error('Error: ' + errors.join('\n'));
+  }
+
+  if (result?.code === 0) {
+    return result;
+  } else {
+    throw new Error(`Error code '${result?.code}' while running command`);
+  }
+}
+
+type RunCommandOpts = {
+  logPrefix?: string;
+} & Readonly<Options & SpawnOptions>;
+
+type MySpawnOpts = RunCommandOpts & { receivedLines?: string; logLine?: Function };
+
+/**
+ * Run a command and log the output. Ported from stack-versioned
+ */
+async function spawnPromise(
+  command: string,
+  commandArgs: string[],
+  options: MySpawnOpts = {},
+  context: { task_name: string }
+) {
+  const debug = require('debug')(`${context.task_name} build`);
+  let logPrefix = options.logPrefix || '';
+  delete options.logPrefix;
+  if (logPrefix) {
+    logPrefix += ' ';
+  }
+
+  const logLine = options.logLine;
+  delete options.receivedLines;
+
+  let logBuffer = '';
+  const log = (isStderr: boolean, data: any) => {
+    data = data.toString();
+
+    let trimmed = data.replace(/[\n|\r|\r\n]$/, '');
+    if (trimmed.length == data.length) {
+      logBuffer += trimmed;
+      return;
+    }
+
+    if (logBuffer.length > 0) {
+      trimmed = logBuffer + trimmed;
+      logBuffer = '';
+    }
+
+    trimmed.split('\n').forEach((line: string) => {
+      debug(`${logPrefix}${line}`);
+      if (logLine) {
+        logLine(isStderr, line, spawnPromise.childProcess);
+      }
+    });
+  };
+
+  debug(`${logPrefix}(${options.cwd || '.'}) \$ ${commandArgs.join(' ')}`);
+  let spawnPromise = spawn(command, commandArgs, options);
+
+  spawnPromise.childProcess.stdout!.on('data', log.bind(null, false));
+  spawnPromise.childProcess.stderr!.on('data', log.bind(null, true));
+
+  try {
+    return await spawnPromise;
+  } finally {
+    spawnPromise.childProcess.stdout!.removeAllListeners('data');
+    spawnPromise.childProcess.stderr!.removeAllListeners('data');
   }
 }

package/tsconfig.json

@@ -3,6 +3,7 @@
   "compilerOptions": {
     "outDir": "dist",
     "declarationDir": "dist",
-    "rootDir": "src"
+    "rootDir": "src",
+    "lib": ["es2019", "dom"],
   },
 }