@journeyapps/cloudcode-build-agent 0.0.0-dev.2b3e822 → 0.0.0-dev.55699c3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@journeyapps/cloudcode-build-agent",
-  "version": "0.0.0-dev.2b3e822",
+  "version": "0.0.0-dev.55699c3",
   "description": "",
   "main": "./dist/index",
   "types": "./dist/index",
@@ -9,8 +9,6 @@
     "cloudcode-build-agent": "./bin.js"
   },
   "dependencies": {
-    "child-process-promise": "^2.2.1",
-    "debug": "^2.6.0",
     "fs-jetpack": "5.1.0",
     "lodash": "4.17.21",
     "node-fetch": "<3",
@@ -19,9 +17,8 @@
     "yargs": "17.6.2"
   },
   "devDependencies": {
-    "@types/child-process-promise": "2.2.2",
     "@types/lodash": "4.14.191",
-    "@types/node": "14.18.36",
+    "@types/node": "16.11.7",
     "@types/node-fetch": "2.6.2",
     "@types/semver": "7.3.13",
     "@types/tar": "6.1.4",

package/src/builder.ts

@@ -1,94 +1,11 @@
-import * as path from 'node:path';
-import * as jetpack from 'fs-jetpack';
-import * as semver from 'semver';
-import * as utils from './installer';
+import { spawnSync } from 'child_process';
 import { detectTasks } from './detect_tasks';
-import { runCommand } from './run';
 
-import * as _ from 'lodash';
-
-export type GeneralTaskConfig = {
-  app_id: string;
-  env: string;
-  backend_id: string;
-  backend_url: string;
-};
-
-export async function buildTasks(project_path: string, dest: string, config: GeneralTaskConfig, only?: string) {
+// Detect tasks in given path and build them.
+export async function buildTasks(project_path: string, dest: string, config: any, only?: string) {
   const tasks = await detectTasks(project_path, only);
 
-  const required_node_versions = _.compact(tasks.map((t) => t.required_node_version));
-  const install_node_versions = _.uniq(required_node_versions);
-
-  // FIXME: Maybe refactor this section into an ensureNodeVersion (or something) that returns the relevant toolpaths?
-  const tool_paths: Record<string, { bin_path: string; node_bin: string; npm_bin: string }> = {};
-  // Use the version of nodejs that's running this script as the default.
-  const default_tool_paths = {
-    bin_path: path.dirname(process.execPath),
-    node_bin: process.execPath,
-    npm_bin: path.join(path.dirname(process.execPath), 'npm')
-  };
-  tool_paths[process.version] = default_tool_paths;
-
-  for (const required_node_version of install_node_versions) {
-    // FIXME: put this in another directory to avoid leaking into the builder
-    const custom_node_path = path.resolve(`node-${semver.major(required_node_version)}`);
-
-    if (!jetpack.exists(path.join(custom_node_path, 'bin/node'))) {
-      console.debug(`installing to ${custom_node_path}`);
-      await jetpack.dirAsync(custom_node_path);
-      await utils.downloadAndInstallNode(required_node_version, custom_node_path);
-    } else {
-      console.debug(`already installed in ${custom_node_path}`);
-    }
-    tool_paths[required_node_version] = utils.nodePaths(custom_node_path);
-  }
-  console.debug('----');
-
-  for (const task of tasks) {
-    const debug = require('debug')(`${task.task_name}`);
-    const node_version = task.required_node_version ?? process.version;
-    const builder_package = task.builder_package;
-    const builder_script = task.builder_script;
-    const { bin_path, node_bin, npm_bin } = tool_paths[node_version];
-    const builder_bin = path.resolve(bin_path, builder_script);
-
-    if (!jetpack.exists(node_bin)) {
-      throw new Error(`Node binary not found: ${node_bin}`);
-    }
-
-    // console.debug(`[${task.task_name}] Installing builder script "${builder_package}" for node ${node_version}`);
-    debug(`Installing builder script "${builder_package}" for node ${node_version}`);
-    // TODO: This may need to be replaced with the something like the spawn-stream stuff in build-agent? OR what to do with the stdout/stderr and errorhandling?
-    await runCommand(
-      node_bin,
-      [npm_bin, '--global', 'install', builder_package],
-      { cwd: project_path },
-      { task_name: task.task_name }
-    );
-
-    if (!jetpack.exists(builder_bin)) {
-      console.error(`[${task.task_name}] ${builder_bin} not found`);
-      throw new Error(`Builder script not found for task ${task.task_name}`);
-    }
-
-    const builder_args: string[] = [];
-    // --src ./ --out ./dist/echo --task echo --appId 'appid' --env 'testing' --backendId 'backendid' --backendUrl 'http://run.journeyapps.test'
-    builder_args.push(builder_bin);
-    builder_args.push('--src', project_path);
-    builder_args.push(`--out`, path.join(dest, task.task_name));
-    builder_args.push(`--zip`, path.join(dest, `${task.task_name}.zip`));
-    builder_args.push(`--task`, `${task.task_name}`);
-    builder_args.push(`--appId`, config.app_id);
-    builder_args.push(`--env`, config.env);
-    builder_args.push(`--backendId`, config.backend_id);
-    builder_args.push(`--backendUrl`, config.backend_url);
-    // builder_args.push(`--runInstallScripts`, 'true'); // TODO: handle this from feature flags somehow
-
-    // console.debug(`[${task.task_name}] Trying: ${node_bin} ${builder_args.join(' ')}`);
-    debug(`Trying: ${node_bin} ${builder_args.join(' ')}`);
-
-    await runCommand(node_bin, builder_args, { cwd: project_path }, { task_name: task.task_name });
-    console.debug('----');
+  for (const t of tasks) {
+    // await t.drink()
   }
 }

package/src/cli.ts

@@ -2,36 +2,40 @@
 
 import * as yargs from 'yargs';
 
-import { buildTasks } from './';
+import { ProcessError, buildTasks, prepareBuildEnvForTasks } from './';
 
 const DEFAULT_SRC_PATH = './';
-const DEFAULT_OUT_PATH = './dist';
+const DEFAULT_OUT_PATH = '';
 
 yargs
   .command(
     ['build', '$0'], // $0 means this is the default command
     'build tasks',
     (yargs) => {
-      return (
-        yargs
-          .option('only', { string: true })
-          .option('path', { default: DEFAULT_SRC_PATH })
-          .option('out', { default: DEFAULT_OUT_PATH })
-          // TODO: investigate yargs.config() for reading task config blob from a file. But how to ensure required args?
-          .option('appId', { string: true, demandOption: true, describe: "CloudCode's reference id" })
-          .option('env', { string: true, demandOption: true })
-          .option('backendId', { string: true, demandOption: true })
-          .option('backendUrl', { string: true, demandOption: true })
-      );
+      return yargs
+        .option('only', { string: true })
+        .option('path', { default: DEFAULT_SRC_PATH })
+        .option('out', { default: DEFAULT_OUT_PATH });
     },
     handled(async (argv) => {
-      const config = {
-        app_id: argv.appId,
-        env: argv.env,
-        backend_id: argv.backendId,
-        backend_url: argv.backendUrl
-      };
-      await buildTasks(argv.path, argv.out, config, argv.only);
+      // iterate over task directories and run: 'yarn cloudcode-build'.
+      // optionally filtered by `argv.only` to build a single task.
+
+      await buildTasks(argv.path, argv.out, argv.only);
+    })
+  )
+  .command(
+    'prepare-env',
+    'Install node environments and packages for each task',
+    (yargs) => {
+      return yargs.option('only', { string: true }).option('path', { default: DEFAULT_SRC_PATH });
+    },
+    handled(async (argv) => {
+      // iterate over task directories:
+      //   ensure required node version for task CC version is installed - What if too old - error?
+      //   run yarn install
+
+      await prepareBuildEnvForTasks(argv.path, argv.only);
     })
   )
   .option('verbose', {
@@ -44,8 +48,13 @@ function handled<T>(fn: (argv: T) => Promise<void>): (argv: T) => Promise<void>
     try {
       await fn(argv);
     } catch (err) {
-      console.error(err.stack);
-      process.exit(1);
+      if (err instanceof ProcessError) {
+        console.error(err.message);
+        process.exit(err.status);
+      } else {
+        console.error(err.stack);
+        process.exit(1);
+      }
     }
   };
 }

package/src/detect_tasks.ts

@@ -40,7 +40,6 @@ export async function detectTasks(project_path: string, only?: string) {
 
       const task_name = task_package.name; // CAVEAT: the pkg name _must_ match the dir.
       const task_version = task_package?.cloudcode?.runtime;
-      // FIXME: Do we want to filter out disabled tasks from the build process?
 
       console.debug(`Detected task version ${task_version}`);
 

package/src/errors.ts

@@ -0,0 +1,22 @@
+import { SpawnSyncReturns } from 'child_process';
+
+export class ProcessError extends Error {
+  cause?: Error;
+  status: number;
+
+  constructor(public result: SpawnSyncReturns<string>) {
+    super(constructMessage(result));
+
+    if (result.error) {
+      this.cause = result.error;
+    }
+    this.status = result.status || 1;
+  }
+}
+
+function constructMessage(result: SpawnSyncReturns<string>): string {
+  if (result.error) {
+    return result.error.message;
+  }
+  return `${result.stdout}${result.stderr}`.trim();
+}

package/src/index.ts

@@ -1 +1,3 @@
+export * from './errors';
 export * from './builder';
+export * from './installer';

package/src/installer.ts

@@ -1,113 +1,82 @@
-import fetch from 'node-fetch';
-import * as path from 'node:path';
 import * as fs from 'node:fs';
-import * as os from 'node:os';
-import * as URL from 'node:url';
-import * as tar from 'tar';
-
-/* Basically this, but for different dirs.
-    curl https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-${PLATFORM}64.tar.gz -L | tar -xzC /node-dedicated && \
-    mv /node-dedicated/node-v${NODE_VERSION}-linux-${PLATFORM}64/bin/node /node-dedicated/node 
-*/
-// TODO: feature to find the latest minor and patch for a given major version?
-export async function downloadAndInstallNode(node_version: string, destination: string) {
-  // eg. https://nodejs.org/dist/v18.14.2/node-v18.14.2-linux-x64.tar.xz
-
-  // const PLATFORM = 'x';
-  // const node_download_url = `https://nodejs.org/dist/v${node_version}/node-v${node_version}-linux-${PLATFORM}64.tar.gz`;
-
-  // TODO: this is just for dev for now. Find a better fix.
-  const ARCH = os.arch();
-  const PLATFORM = os.platform();
-  const node_download_url = `https://nodejs.org/dist/v${node_version}/node-v${node_version}-${PLATFORM}-${ARCH}.tar.gz`;
-
-  await downloadAndExtractTarball(node_download_url, destination);
-  // TODO: move binaries into local bin path?
-  return nodePaths(destination);
-}
-
-export function nodePaths(destination: string) {
-  return {
-    bin_path: path.resolve(destination, 'bin'),
-    node_bin: path.resolve(destination, 'bin', 'node'),
-    npm_bin: path.resolve(destination, 'bin', 'npm')
-  };
-}
-
-export async function downloadAndExtractTarball(url: string, dest: string) {
-  const tmpdir = os.tmpdir();
-
-  const filename = path.basename(URL.parse(url).pathname as string);
-  const base = path.basename(filename, '.tar.gz');
-  const download = path.join(tmpdir, filename);
-
-  if (fs.existsSync(download)) {
-    console.debug(`deleting old ${download}`);
-    fs.unlinkSync(download);
-  }
-
-  await new Promise(async (resolve, reject) => {
-    console.log(`fetching ${url} into ${download}`);
-
-    const response = await fetch(url);
-    if (!response.ok) {
-      const errorBody = await response.statusText;
-      throw new Error(`Failed to download: ${errorBody}`);
+import * as path from 'node:path';
+import * as jetpack from 'fs-jetpack';
+import * as semver from 'semver';
+import * as utils from './utils';
+import { spawnSync } from 'child_process';
+import { detectTasks } from './detect_tasks';
+
+import * as _ from 'lodash';
+
+export async function prepareBuildEnvForTasks(project_path: string, only?: string) {
+  const tasks = await detectTasks(project_path, only);
+
+  const required_node_versions = _.compact(tasks.map((t) => t.required_node_version));
+  const install_node_versions = _.uniq(required_node_versions);
+
+  const tool_paths: Record<string, { bin_path: string; node_bin: string; npm_bin: string }> = {};
+  for (const required_node_version of install_node_versions) {
+    // FIXME: maybe a path prefix.
+    const custom_node_path = path.resolve(`node-${semver.major(required_node_version)}`);
+
+    if (!jetpack.exists(custom_node_path)) {
+      console.log(`installing to ${custom_node_path}`);
+      await jetpack.dirAsync(custom_node_path);
+      await utils.downloadAndInstallNode(required_node_version, custom_node_path);
+    } else {
+      console.log(`already installed in ${custom_node_path}`);
     }
-
-    // FIXME: replace with webstreams? but according to the types package webstreams aren't available in node 16?!??
-    const file = fs.createWriteStream(download); // Sink the download stream into a file, so it can be extracted.
-    response.body.pipe(file);
-
-    file.on('finish', async () => {
-      try {
-        console.log('Extracting...', download);
-
-        // const comp = fs.createReadStream(download);
-
-        tar.extract({
-          cwd: tmpdir,
-          file: download,
-          sync: true
-        });
-
-        const uncomp = path.join(tmpdir, base);
-
-        const dist = path.resolve(dest);
-        // FIXME: dangerous. Add protection or replace.
-        // if (fs.existsSync(dist)) {
-        //   fs.rmSync(dist, { recursive: true });
-        // }
-        if (fs.existsSync(uncomp)) {
-          console.log(`Moving extracted files from ${uncomp} to ${dist}`);
-          fs.renameSync(uncomp, dist);
-        } else {
-          console.log(`Can't find extract dir ${uncomp}`);
-        }
-
-        // FIXME: this seems to sometimes cause errors
-        /*
-        ----
-node:events:505
-      throw er; // Unhandled 'error' event
-      ^
-
-Error: ENOENT: no such file or directory, open '/var/folders/kc/h6m4zpmd23v13s63mygvkslm0000gn/T/node-v16.19.1-darwin-arm64.tar.gz'
-Emitted 'error' event on ReadStream instance at:
-    at emitErrorNT (node:internal/streams/destroy:157:8)
-    at emitErrorCloseNT (node:internal/streams/destroy:122:3)
-    at processTicksAndRejections (node:internal/process/task_queues:83:21) {
-  errno: -2,
-  code: 'ENOENT',
-  syscall: 'open',
-  path: '/var/folders/kc/h6m4zpmd23v13s63mygvkslm0000gn/T/node-v16.19.1-darwin-arm64.tar.gz'
-        */
-        // fs.unlinkSync(file.path);
-
-        resolve(null);
-      } catch (err) {
-        reject(err);
+    tool_paths[required_node_version] = utils.nodePaths(custom_node_path);
+  }
+  console.log('----');
+
+  for (const task of tasks) {
+    const node_version = task.required_node_version ?? process.version;
+    const builder_package = task.builder_package;
+    const builder_script = task.builder_script;
+    console.log(`[${task.task_name}] Installing builder script "${builder_package}" for node ${node_version}`);
+    const { bin_path, node_bin, npm_bin } = tool_paths[node_version];
+    const builder_bin = path.resolve(bin_path, builder_script);
+
+    // TODO: This may need to be replaced with the something like the spawn-stream stuff in build-agent? OR what to do with the stdout/stderr and errorhandling?
+    const spawnResult = spawnSync(node_bin, [npm_bin, '--global', 'install', builder_package], { cwd: project_path });
+    if (spawnResult.status == 0 && jetpack.exists(builder_bin)) {
+      // DEV HACK: this needs to move to the build thingie.
+
+      const builder_args: string[] = [];
+      // --src ~/src/cool-app-bro-master --out ~/src/dist --task echo --appId 'appid' --env 'testing' --backendId 'backendid' --backendUrl 'http://run.journeyapps.test'
+      builder_args.push(builder_bin);
+      // // builder_args.push('--version')
+      builder_args.push('--src', './');
+      builder_args.push(`--out`, `./dist/${task.task_name}`);
+      builder_args.push(`--task`, `${task.task_name}`);
+      builder_args.push(
+        `--appId`,
+        'appid',
+        `--env`,
+        'testing',
+        `--backendId`,
+        'backendid',
+        `--backendUrl`,
+        'http://run.journeyapps.test'
+      );
+
+      console.log(`[${task.task_name}] Trying: ${node_bin} ${builder_args.join(' ')}`);
+
+      const builderSpawnResult = spawnSync(node_bin, builder_args, { cwd: project_path });
+
+      if (builderSpawnResult.status !== 0) {
+        console.log(`[${task.task_name}] FAILED TO RUN BUILDER SCRIPT`);
+        console.log(`[${task.task_name}] OUTPUT: ${builderSpawnResult.stdout}`);
+        console.log(`[${task.task_name}] ERROR: ${builderSpawnResult.stderr}`);
+      } else {
+        console.log(`[${task.task_name}] SCRIPT SUCCESS?`);
+        console.log(`[${task.task_name}] OUTPUT: ${builderSpawnResult.stdout}`);
+        console.log(`[${task.task_name}] ERROR: ${builderSpawnResult.stderr}`);
       }
-    });
-  });
+      console.log('----');
+    } else {
+      console.log(`[${task.task_name}] failed to install ${builder_package}`);
+    }
+  }
 }

package/src/utils.ts

@@ -0,0 +1,82 @@
+import fetch from 'node-fetch';
+import * as path from 'node:path';
+import * as fs from 'node:fs';
+import * as os from 'node:os';
+import * as URL from 'node:url';
+import * as tar from 'tar';
+
+/* Basically this, but for different dirs.
+    curl https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-${PLATFORM}64.tar.gz -L | tar -xzC /node-dedicated && \
+    mv /node-dedicated/node-v${NODE_VERSION}-linux-${PLATFORM}64/bin/node /node-dedicated/node 
+*/
+// TODO: feature to find the latest minor and patch for a given major version?
+export async function downloadAndInstallNode(node_version: string, destination: string) {
+  // eg. https://nodejs.org/dist/v18.14.2/node-v18.14.2-linux-x64.tar.xz
+
+  // const PLATFORM = 'x';
+  // const node_download_url = `https://nodejs.org/dist/v${node_version}/node-v${node_version}-linux-${PLATFORM}64.tar.gz`;
+
+  // TODO: this is just for dev for now. Find a better fix.
+  const ARCH = os.arch();
+  const PLATFORM = os.platform();
+  const node_download_url = `https://nodejs.org/dist/v${node_version}/node-v${node_version}-${PLATFORM}-${ARCH}.tar.gz`;
+
+  await downloadAndExtractTarball(node_download_url, destination);
+  // TODO: move binaries into local bin path?
+  return nodePaths(destination);
+}
+
+export function nodePaths(destination: string) {
+  return {
+    bin_path: path.resolve(destination, 'bin'),
+    node_bin: path.resolve(destination, 'bin', 'node'),
+    npm_bin: path.resolve(destination, 'bin', 'npm')
+  };
+}
+
+export async function downloadAndExtractTarball(url: string, dest: string) {
+  const tmpdir = os.tmpdir();
+
+  const filename = path.basename(URL.parse(url).pathname as string);
+  const base = path.basename(filename, '.tar.gz');
+  const download = path.join(tmpdir, filename);
+
+  console.log(`fetching ${url}`);
+
+  const file = fs.createWriteStream(download);
+  const response = await fetch(url);
+  if (!response.ok) {
+    const errorBody = await response.statusText;
+    throw new Error(`Failed to download: ${errorBody}`);
+  }
+
+  // FIXME: replace with webstreams?
+  response.body.pipe(file);
+
+  file.on('finish', async () => {
+    console.log('Extracting...', download);
+
+    const comp = fs.createReadStream(download);
+
+    tar.extract({
+      cwd: tmpdir,
+      file: comp.path as string,
+      sync: true
+    });
+
+    const uncomp = path.join(tmpdir, base);
+
+    const dist = path.resolve(dest);
+    // FIXME: dangerous. Add protection or replace.
+    // if (fs.existsSync(dist)) {
+    //   fs.rmSync(dist, { recursive: true });
+    // }
+    if (fs.existsSync(uncomp)) {
+      console.log(`Moving extracted files from ${uncomp} to ${dist}`);
+      fs.renameSync(uncomp, dist);
+    }
+
+    // FIXME: this seems to sometimes cause errors
+    // fs.unlinkSync(file.path);
+  });
+}