@journeyapps/cloudcode-build-agent 0.0.0-dev.8ebefb0 → 0.0.0-dev.90a9f01

package/src/builder.ts

@@ -1,105 +1,7 @@
-import * as path from 'node:path';
-import * as jetpack from 'fs-jetpack';
-import * as semver from 'semver';
-import * as utils from './installer';
-import { detectTasks } from './detect_tasks';
-import { runCommand } from './run';
+import { spawnSync } from 'child_process';
 
-import * as _ from 'lodash';
+// Detect tasks in given path and build them.
+export async function buildTasks(src: string, dest: string, only?: string) {}
 
-export type GeneralTaskConfig = {
-  app_id: string;
-  env: string;
-  backend_id: string;
-  backend_url: string;
-};
-
-export async function buildTasks(project_path: string, dest: string, config: GeneralTaskConfig, only?: string) {
-  const tasks = await detectTasks(project_path, only);
-
-  const required_node_versions = _.compact(tasks.map((t) => t.required_node_version));
-  const install_node_versions = _.uniq(required_node_versions);
-
-  // FIXME: Maybe refactor this section into an ensureNodeVersion (or something) that returns the relevant toolpaths?
-  const tool_paths: Record<string, { bin_path: string; node_bin: string; npm_bin: string }> = {};
-  // Use the version of nodejs that's running this script as the default.
-  const default_tool_paths = {
-    bin_path: path.dirname(process.execPath),
-    node_bin: process.execPath,
-    npm_bin: path.join(path.dirname(process.execPath), 'npm')
-  };
-  tool_paths[process.version] = default_tool_paths;
-
-  for (const required_node_version of install_node_versions) {
-    // FIXME: put this in another directory to avoid leaking into the builder
-    const custom_node_path = path.resolve(`node-${semver.major(required_node_version)}`);
-
-    if (!jetpack.exists(path.join(custom_node_path, 'bin/node'))) {
-      console.debug(`installing to ${custom_node_path}`);
-      await jetpack.dirAsync(custom_node_path);
-      await utils.downloadAndInstallNode(required_node_version, custom_node_path);
-    } else {
-      console.debug(`already installed in ${custom_node_path}`);
-    }
-    tool_paths[required_node_version] = utils.nodePaths(custom_node_path);
-  }
-  // console.debug('----');
-
-  for (const task of tasks) {
-    const node_version = task.required_node_version ?? process.version;
-    const builder_package = task.builder_package;
-    const builder_script = task.builder_script;
-    const { bin_path, node_bin, npm_bin } = tool_paths[node_version];
-    const builder_bin = path.resolve(bin_path, builder_script);
-
-    if (!jetpack.exists(node_bin)) {
-      throw new Error(`Node binary not found: ${node_bin}`);
-    }
-
-    console.debug(`[${task.task_name}] Installing builder script "${builder_package}" for node ${node_version}`);
-    // debug(`Installing builder script "${builder_package}" for node ${node_version}`);
-
-    const stream1 = runCommand(node_bin, [npm_bin, '--global', 'install', builder_package], {
-      cwd: project_path,
-      env: process.env
-    });
-    for await (let event of stream1) {
-      const log = event.stdout ?? event.stderr;
-      if (log) {
-        console.log(`[${task.task_name} - install]`, log.trimRight());
-      }
-    }
-
-    if (!jetpack.exists(builder_bin)) {
-      console.error(`[${task.task_name}] ${builder_bin} not found`);
-      throw new Error(`Builder script not found for task ${task.task_name}`);
-    }
-
-    const builder_args: string[] = [];
-    // --src ./ --out ./dist/echo --task echo --appId 'appid' --env 'testing' --backendId 'backendid' --backendUrl 'http://run.journeyapps.test'
-    builder_args.push(builder_bin);
-    builder_args.push('--src', project_path);
-    builder_args.push(`--out`, path.join(dest, task.task_name));
-    builder_args.push(`--zip`, path.join(dest, `${task.task_name}.zip`));
-    builder_args.push(`--task`, `${task.task_name}`);
-    builder_args.push(`--appId`, config.app_id);
-    builder_args.push(`--env`, config.env);
-    builder_args.push(`--backendId`, config.backend_id);
-    builder_args.push(`--backendUrl`, config.backend_url);
-    // builder_args.push(`--runInstallScripts`, 'true'); // TODO: handle this from feature flags somehow
-
-    console.debug(`[${task.task_name}] Trying: ${node_bin} ${builder_args.join(' ')}`);
-    // debug(`Trying: ${node_bin} ${builder_args.join(' ')}`);
-
-    const stream2 = runCommand(node_bin, builder_args, { cwd: project_path, env: process.env });
-
-    for await (let event of stream2) {
-      const log = event.stdout ?? event.stderr;
-      if (log) {
-        console.log(`[${task.task_name} - build]`, log.trimRight());
-      }
-    }
-
-    // console.debug('----');
-  }
-}
+// Build a single task in the specified subdirectory.
+async function buildTask(src: string, dest: string, config: any) {}

package/src/cli.ts

@@ -2,36 +2,40 @@
 
 import * as yargs from 'yargs';
 
-import { buildTasks } from './';
+import { ProcessError, buildTasks, prepareBuildEnvForTasks } from './';
 
 const DEFAULT_SRC_PATH = './';
-const DEFAULT_OUT_PATH = './dist';
+const DEFAULT_OUT_PATH = '';
 
 yargs
   .command(
     ['build', '$0'], // $0 means this is the default command
     'build tasks',
     (yargs) => {
-      return (
-        yargs
-          .option('only', { string: true })
-          .option('path', { default: DEFAULT_SRC_PATH })
-          .option('out', { default: DEFAULT_OUT_PATH })
-          // TODO: investigate yargs.config() for reading task config blob from a file. But how to ensure required args?
-          .option('appId', { string: true, demandOption: true, describe: "CloudCode's reference id" })
-          .option('env', { string: true, demandOption: true })
-          .option('backendId', { string: true, demandOption: true })
-          .option('backendUrl', { string: true, demandOption: true })
-      );
+      return yargs
+        .option('only', { string: true })
+        .option('path', { default: DEFAULT_SRC_PATH })
+        .option('out', { default: DEFAULT_OUT_PATH });
     },
     handled(async (argv) => {
-      const config = {
-        app_id: argv.appId,
-        env: argv.env,
-        backend_id: argv.backendId,
-        backend_url: argv.backendUrl
-      };
-      await buildTasks(argv.path, argv.out, config, argv.only);
+      // iterate over task directories and run: 'yarn cloudcode-build'.
+      // optionally filtered by `argv.only` to build a single task.
+
+      await buildTasks(argv.path, argv.out, argv.only);
+    })
+  )
+  .command(
+    'prepare-env',
+    'Install node environments and packages for each task',
+    (yargs) => {
+      return yargs.option('only', { string: true }).option('path', { default: DEFAULT_SRC_PATH });
+    },
+    handled(async (argv) => {
+      // iterate over task directories:
+      //   ensure required node version for task CC version is installed - What if too old - error?
+      //   run yarn install
+
+      await prepareBuildEnvForTasks(argv.path, argv.only);
     })
   )
   .option('verbose', {
@@ -44,8 +48,13 @@ function handled<T>(fn: (argv: T) => Promise<void>): (argv: T) => Promise<void>
     try {
       await fn(argv);
     } catch (err) {
-      console.error(err.stack);
-      process.exit(1);
+      if (err instanceof ProcessError) {
+        console.error(err.message);
+        process.exit(err.status);
+      } else {
+        console.error(err.stack);
+        process.exit(1);
+      }
     }
   };
 }

package/src/errors.ts

@@ -0,0 +1,22 @@
+import { SpawnSyncReturns } from 'child_process';
+
+export class ProcessError extends Error {
+  cause?: Error;
+  status: number;
+
+  constructor(public result: SpawnSyncReturns<string>) {
+    super(constructMessage(result));
+
+    if (result.error) {
+      this.cause = result.error;
+    }
+    this.status = result.status || 1;
+  }
+}
+
+function constructMessage(result: SpawnSyncReturns<string>): string {
+  if (result.error) {
+    return result.error.message;
+  }
+  return `${result.stdout}${result.stderr}`.trim();
+}

package/src/index.ts

@@ -1 +1,3 @@
+export * from './errors';
 export * from './builder';
+export * from './installer';

package/src/installer.ts

@@ -1,112 +1,88 @@
-import fetch from 'node-fetch';
-import * as path from 'node:path';
+import { spawnSync } from 'child_process';
 import * as fs from 'node:fs';
-import * as os from 'node:os';
-import * as URL from 'node:url';
-import * as tar from 'tar';
-
-/* Basically this, but for different dirs.
-    curl https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-${PLATFORM}64.tar.gz -L | tar -xzC /node-dedicated && \
-    mv /node-dedicated/node-v${NODE_VERSION}-linux-${PLATFORM}64/bin/node /node-dedicated/node 
-*/
-// TODO: feature to find the latest minor and patch for a given major version?
-export async function downloadAndInstallNode(node_version: string, destination: string) {
-  // eg. https://nodejs.org/dist/v18.14.2/node-v18.14.2-linux-x64.tar.xz
-
-  // const PLATFORM = 'x';
-  // const node_download_url = `https://nodejs.org/dist/v${node_version}/node-v${node_version}-linux-${PLATFORM}64.tar.gz`;
-
-  // TODO: this is just for dev for now. Find a better fix.
-  const ARCH = os.arch();
-  const PLATFORM = os.platform();
-  const node_download_url = `https://nodejs.org/dist/v${node_version}/node-v${node_version}-${PLATFORM}-${ARCH}.tar.gz`;
-
-  await downloadAndExtractTarball(node_download_url, destination);
-  // TODO: move binaries into local bin path?
-  return nodePaths(destination);
-}
-
-export function nodePaths(destination: string) {
-  return {
-    bin_path: path.resolve(destination, 'bin'),
-    node_bin: path.resolve(destination, 'bin', 'node'),
-    npm_bin: path.resolve(destination, 'bin', 'npm')
-  };
-}
-
-export async function downloadAndExtractTarball(url: string, dest: string) {
-  const tmpdir = os.tmpdir();
-
-  const filename = path.basename(URL.parse(url).pathname as string);
-  const base = path.basename(filename, '.tar.gz');
-  const download = path.join(tmpdir, filename);
-
-  if (fs.existsSync(download)) {
-    console.debug(`deleting old ${download}`);
-    fs.unlinkSync(download);
+import * as path from 'node:path';
+import * as jetpack from 'fs-jetpack';
+import * as semver from 'semver';
+import * as utils from './utils';
+
+const SUPPORTED_VERSIONS = [
+  {
+    version: '1.12.0',
+    node: '16.19.1' // FIXME: this is maybe brittle?
+  },
+  {
+    version: '1.11.2',
+    node: '14.21.3'
+  },
+  {
+    version: '1.11.1',
+    node: '14.21.3'
+  },
+  {
+    version: '1.11.0',
+    node: '14.21.3'
   }
-
-  await new Promise(async (resolve, reject) => {
-    console.log(`fetching ${url} into ${download}`);
-
-    const response = await fetch(url);
-    if (!response.ok) {
-      const errorBody = await response.statusText;
-      throw new Error(`Failed to download: ${errorBody}`);
+];
+
+export async function prepareBuildEnvForTasks(project_path: string, only?: string) {
+  const package_files = await jetpack.findAsync(path.join(project_path, 'cloudcode'), { matching: '**/package.json' });
+  const filtered_package_files = package_files.filter((pkg_path) => {
+    // FIXME: this is kinda clunky.
+    const pm = /^\/?cloudcode\/([^\/]+)\/package\.json$/.exec(pkg_path);
+    if (!pm) {
+      return false;
     }
+    const taskName = pm[1];
 
-    const file = fs.createWriteStream(download); // Sink the download stream into a file, so it can be extracted.
-    response.body.pipe(file);
+    if (only != null && only != taskName) {
+      // !(only.indexOf(taskName) >= 0)
+      return false;
+    }
+    return true;
+  });
 
-    file.on('finish', async () => {
-      try {
-        console.log('Extracting...', download);
+  // FIXME: Would it be better to map out required node_versions and subsequently install them, or is this in-line approach okay?
+  /* TODO:
+       How to install and execute custom node version in a portable way?
+          - install to predictable path inside project dir, which is writeable in the container. 
+              But how to run it from inside task dirs? local bin? ENV?
+              What about yarn for each node version?
+  */
+
+  for (const pkg_path of filtered_package_files) {
+    try {
+      const content = await fs.promises.readFile(pkg_path, { encoding: 'utf-8' });
+      const task_package = JSON.parse(content);
+      const task_version = task_package?.cloudcode?.runtime;
+
+      // check task version against supported versions, install relevant node version and yarn
+      console.log(`Detected task version ${task_version}`);
+
+      const matching = SUPPORTED_VERSIONS.find((v) => {
+        return semver.satisfies(v.version, task_version);
+      });
+      if (!matching) {
+        console.error('FIXME: unsupported version');
+      }
 
-        // const comp = fs.createReadStream(download);
+      console.log(`Matching versions: ${JSON.stringify(matching)}`);
+      const running_node_version = process.versions.node;
+      if (matching?.node && semver.major(matching.node) !== semver.major(running_node_version)) {
+        console.log(`Task requires different node version: v${matching.node}`);
 
-        tar.extract({
-          cwd: tmpdir,
-          file: download,
-          sync: true
-        });
+        const NODE_VERSION = matching.node;
 
-        const uncomp = path.join(tmpdir, base);
+        const custom_node_path = path.resolve(`node-${semver.major(NODE_VERSION)}`);
 
-        const dist = path.resolve(dest);
-        // FIXME: dangerous. Add protection or replace.
-        // if (fs.existsSync(dist)) {
-        //   fs.rmSync(dist, { recursive: true });
-        // }
-        if (fs.existsSync(uncomp)) {
-          console.log(`Moving extracted files from ${uncomp} to ${dist}`);
-          fs.renameSync(uncomp, dist);
-        } else {
-          console.log(`Can't find extract dir ${uncomp}`);
+        if (!jetpack.exists(custom_node_path)) {
+          await jetpack.dirAsync(custom_node_path);
+          await utils.downloadAndInstallNode(NODE_VERSION, custom_node_path);
         }
-
-        // FIXME: this seems to sometimes cause errors
-        /*
-        ----
-node:events:505
-      throw er; // Unhandled 'error' event
-      ^
-
-Error: ENOENT: no such file or directory, open '/var/folders/kc/h6m4zpmd23v13s63mygvkslm0000gn/T/node-v16.19.1-darwin-arm64.tar.gz'
-Emitted 'error' event on ReadStream instance at:
-    at emitErrorNT (node:internal/streams/destroy:157:8)
-    at emitErrorCloseNT (node:internal/streams/destroy:122:3)
-    at processTicksAndRejections (node:internal/process/task_queues:83:21) {
-  errno: -2,
-  code: 'ENOENT',
-  syscall: 'open',
-  path: '/var/folders/kc/h6m4zpmd23v13s63mygvkslm0000gn/T/node-v16.19.1-darwin-arm64.tar.gz'
-        */
-        // fs.unlinkSync(file.path);
-
-        resolve(null);
-      } catch (err) {
-        reject(err);
       }
-    });
-  });
+    } catch (err) {
+      console.error(`ERROR: ${err}`);
+      // todo: check for enoent and skip?
+      // todo: if json error, throw a CC build error?
+    }
+  }
 }

package/src/utils.ts

@@ -0,0 +1,63 @@
+import fetch from 'node-fetch';
+import * as path from 'node:path';
+import * as fs from 'node:fs';
+import * as os from 'node:os';
+import * as URL from 'node:url';
+import * as tar from 'tar';
+
+/* Basically this, but for different dirs.
+    curl https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-${PLATFORM}64.tar.gz -L | tar -xzC /node-dedicated && \
+    mv /node-dedicated/node-v${NODE_VERSION}-linux-${PLATFORM}64/bin/node /node-dedicated/node 
+*/
+export async function downloadAndInstallNode(node_version: string, destination: string) {
+  // eg. https://nodejs.org/dist/v18.14.2/node-v18.14.2-linux-x64.tar.xz
+  const PLATFORM = 'x';
+  const node_download_url = `https://nodejs.org/dist/v${node_version}/node-v${node_version}-linux-${PLATFORM}64.tar.gz`;
+
+  await downloadAndExtractTarball(node_download_url, destination);
+  // TODO: move binaries into local bin path?
+}
+
+export async function downloadAndExtractTarball(url: string, dest: string) {
+  const tmpdir = os.tmpdir();
+
+  const filename = path.basename(URL.parse(url).pathname as string);
+  const base = path.basename(filename, '.tar.gz');
+  const download = path.join(tmpdir, filename);
+
+  const file = fs.createWriteStream(download);
+  const response = await fetch(url);
+  if (!response.ok) {
+    const errorBody = await response.text();
+    throw new Error(`Failed to download: ${errorBody}`);
+  }
+
+  // FIXME: replace with webstreams?
+  response.body.pipe(file);
+
+  file.on('finish', async () => {
+    console.log('Extracting...', download);
+
+    const comp = fs.createReadStream(download);
+
+    tar.extract({
+      cwd: tmpdir,
+      file: comp.path as string,
+      sync: true
+    });
+
+    const uncomp = path.join(tmpdir, base);
+
+    const dist = path.resolve(dest);
+    // FIXME: dangerous. Add protection or replace.
+    // if (fs.existsSync(dist)) {
+    //   fs.rmSync(dist, { recursive: true });
+    // }
+    if (fs.existsSync(uncomp)) {
+      console.log(`Moving extracted files from ${uncomp} to ${dist}`);
+      fs.renameSync(uncomp, dist);
+    }
+
+    fs.unlinkSync(file.path);
+  });
+}

package/tsconfig.json

@@ -3,6 +3,7 @@
   "compilerOptions": {
     "outDir": "dist",
     "declarationDir": "dist",
-    "rootDir": "src"
+    "rootDir": "src",
+    "lib": ["es2019", "dom"],
   },
 }