libdragon 10.6.0 → 10.7.0

package/CHANGELOG.md

@@ -1,5 +1,52 @@
 # Change Log
 
+## [10.7.0] - 2022-04-17
+
+### Fixed
+
+- Logs properly goes to stderr now. Previously they were written to stdout. This
+means the id output of the `start` action is now written to stdout while we can
+also display other information on the terminal. This allowed enabling the docker
+logs for a more responsive experience. The output of `help` still goes to stdout.
+- Skip reading the project information for `version` and `help` commands. This
+was breaking things when these were run with a stopped docker service.
+- Resolved a potential failure case where the host mount path is not reported
+as a native path. That would prevent proper container discovery. This was not
+being an issue previously but I realized it was not working for Windows now.
+- Do not throw if the output stream is not finalized. This was previously causing
+issues when the long output of a command is piped to another process like `less`.
+Fixes #46
+- Produce a correct error if an invalid flag with a singe `-` is provided.
+- Sekeleton project's Makefile `clean` recipe now works properly. Fixes #56.
+- A case where it is impossible to recover the container is fixed. This was
+happening when the host does not have git and the cached container file is
+missing. In that case the cli was attempting to run git without the containerId.
+- `destroy` can now cleanup any remaining container even if the project init was
+failed. This is also useful to clean up old folders that were once libdragon
+projects.
+
+### Added
+
+- Stdin consumption support. Now it is possible to pipe anything to `exec` and
+it will pass it through to the target. In case of no running container, it will
+keep a copy of the stdin stream until the docker process is ready. This enables
+piping in data from the host if ever needed for some reason. This enables usages
+like `cat file.txt | libdragon exec cat - | less`.
+- Automatically convert host paths into posix format so that the user can use
+the host's path autocompletion. It will also convert absolute host paths into
+relative container paths automatically. Previously all paths were assumed to be
+container paths relative to the location corresponding to the host cwd.
+Closes #24.
+
+### Changed
+
+- Refactored process spawns.
+- Refactored main flow and separated parsing logic.
+- Reorder actions & correction on flag usage for help output.
+- Setting `--verbose` for `start` does not guarantee the only-id output anymore.
+- Refactored parameter parsing.
+- Update submdule for local development.
+
 ## [10.6.0] - 2022-04-09
 ### Fixed
 

package/index.js

@@ -1,8 +1,7 @@
 #!/usr/bin/env node
 
-const chalk = require('chalk');
+const chalk = require('chalk').stderr;
 
-const actions = require('./modules/actions');
 const { fn: printUsage } = require('./modules/actions/help');
 const {
   STATUS_OK,
@@ -17,140 +16,24 @@ const {
   ValidationError,
   log,
 } = require('./modules/helpers');
+const { parseParameters } = require('./modules/parameters');
 const { readProjectInfo, writeProjectInfo } = require('./modules/project-info');
 
-let options = {},
-  currentAction;
-
-for (let i = 2; i < process.argv.length; i++) {
-  const val = process.argv[i];
-
-  // Allow console here
-  /* eslint-disable no-console */
-
-  if (['--verbose', '-v'].includes(val)) {
-    options.VERBOSE = true;
-    globals.verbose = true;
-    continue;
-  }
-
-  // TODO: we might move these to actions as well.
-  if (['--image', '-i'].includes(val)) {
-    options.DOCKER_IMAGE = process.argv[++i];
-    continue;
-  } else if (val.indexOf('--image=') === 0) {
-    options.DOCKER_IMAGE = val.split('=')[1];
-    continue;
-  }
-
-  if (['--directory', '-d'].includes(val)) {
-    options.VENDOR_DIR = process.argv[++i];
-    continue;
-  } else if (val.indexOf('--directory=') === 0) {
-    options.VENDOR_DIR = val.split('=')[1];
-    continue;
-  }
-
-  if (['--strategy', '-s'].includes(val)) {
-    options.VENDOR_STRAT = process.argv[++i];
-    continue;
-  } else if (val.indexOf('--strategy=') === 0) {
-    options.VENDOR_STRAT = val.split('=')[1];
-    continue;
-  }
-
-  if (['--file', '-f'].includes(val)) {
-    options.FILE = process.argv[++i];
-    continue;
-  } else if (val.indexOf('--file=') === 0) {
-    options.FILE = val.split('=')[1];
-    continue;
-  }
-
-  if (val.indexOf('--') >= 0) {
-    console.error(chalk.red(`Invalid flag \`${val}\``));
-    printUsage();
-    process.exit(STATUS_BAD_PARAM);
-  }
-
-  if (currentAction) {
-    console.error(
-      chalk.red(`Expected only a single action, found: \`${val}\``)
-    );
-    printUsage();
-    process.exit(STATUS_BAD_PARAM);
-  }
-
-  currentAction = actions[val];
-
-  if (!currentAction) {
-    console.error(chalk.red(`Invalid action \`${val}\``));
-    printUsage();
-    process.exit(STATUS_BAD_PARAM);
-  }
-
-  if (currentAction.forwardsRestParams) {
-    options.PARAMS = process.argv.slice(i + 1);
-    break;
-  }
-}
-
-if (!currentAction) {
-  console.error(chalk.red('No action provided'));
-  printUsage();
-  process.exit(STATUS_BAD_PARAM);
-}
-
-if (currentAction === actions.exec && options.PARAMS.length === 0) {
-  console.error(chalk.red('You should provide a command to exec'));
-  printUsage(undefined, [currentAction.name]);
-  process.exit(STATUS_BAD_PARAM);
-}
-
-if (
-  ![actions.init, actions.install, actions.update].includes(currentAction) &&
-  options.DOCKER_IMAGE
-) {
-  console.error(chalk.red('Invalid flag: image'));
-  printUsage(undefined, [currentAction.name]);
-  process.exit(STATUS_BAD_PARAM);
-}
-
-if (
-  options.VENDOR_STRAT &&
-  !['submodule', 'subtree', 'manual'].includes(options.VENDOR_STRAT)
-) {
-  console.error(chalk.red(`Invalid strategy \`${options.VENDOR_STRAT}\``));
-  printUsage();
-  process.exit(STATUS_BAD_PARAM);
-}
-
-if (![actions.disasm].includes(currentAction) && options.FILE) {
-  console.error(chalk.red('Invalid flag: file'));
-  printUsage(undefined, [currentAction.name]);
-  process.exit(STATUS_BAD_PARAM);
-}
-
-readProjectInfo()
-  .then((info) =>
-    currentAction.fn(
-      {
-        ...info,
-        options,
-        ...currentAction,
-      },
-      options.PARAMS ?? []
-    )
-  )
+parseParameters(process.argv)
+  .then(readProjectInfo)
+  .then((info) => info.options.CURRENT_ACTION.fn(info))
   .catch((e) => {
     if (e instanceof ParameterError) {
-      console.error(chalk.red(e.message));
-      printUsage(undefined, [currentAction.name]);
+      log(chalk.red(e.message));
+      printUsage({
+        options: {
+          EXTRA_PARAMS: [e.actionName],
+        },
+      });
       process.exit(STATUS_BAD_PARAM);
     }
-
     if (e instanceof ValidationError) {
-      console.error(chalk.red(e.message));
+      log(chalk.red(e.message));
       process.exit(STATUS_VALIDATION_ERROR);
     }
 
@@ -158,7 +41,7 @@ readProjectInfo()
 
     // Show additional information to user if verbose or we did a mistake
     if (globals.verbose || !userTargetedError) {
-      console.error(chalk.red(globals.verbose ? e.stack : e.message));
+      log(chalk.red(globals.verbose ? e.stack : e.message));
     }
 
     // Print the underlying error out only if not verbose and we did a mistake

package/modules/actions/destroy.js

@@ -1,13 +1,12 @@
 const fs = require('fs/promises');
 const path = require('path');
 
-const { mustHaveProject, destroyContainer } = require('./utils');
+const { destroyContainer } = require('./utils');
 const { CONFIG_FILE, LIBDRAGON_PROJECT_MANIFEST } = require('../constants');
-const { fileExists, dirExists } = require('../helpers');
+const { fileExists, dirExists, log } = require('../helpers');
+const chalk = require('chalk');
 
 const destroy = async (libdragonInfo) => {
-  await mustHaveProject(libdragonInfo);
-
   await destroyContainer(libdragonInfo);
 
   const projectPath = path.join(libdragonInfo.root, LIBDRAGON_PROJECT_MANIFEST);
@@ -16,20 +15,20 @@ const destroy = async (libdragonInfo) => {
   if (await fileExists(configPath)) {
     await fs.rm(configPath);
   }
-  if (dirExists(projectPath)) {
+  if (await dirExists(projectPath)) {
     await fs.rmdir(projectPath);
   }
+
+  log(chalk.green('Done cleanup.'));
 };
 
 module.exports = {
   name: 'destroy',
   fn: destroy,
-  showStatus: true,
+  mustHaveProject: false,
   usage: {
     name: 'destroy',
     summary: 'Do clean-up for current project.',
-    description: `Removes libdragon configuration from current project and removes any known containers but will not touch previously vendored files. \`libdragon\` will not work anymore for this project.
-
-      Must be run in an initialized libdragon project.`,
+    description: `Removes libdragon configuration from current project and removes any known containers but will not touch previously vendored files. \`libdragon\` will not work anymore for this project.`,
   },
 };

package/modules/actions/disasm.js

@@ -1,7 +1,6 @@
 const path = require('path');
 const fs = require('fs/promises');
 
-const { mustHaveProject } = require('./utils');
 const { fn: exec } = require('./exec');
 const {
   ValidationError,
@@ -44,49 +43,55 @@ const findElf = async (stop, start = '.') => {
   }
 };
 
-const disasm = async (libdragonInfo, extraArgs) => {
-  await mustHaveProject(libdragonInfo);
-
+const disasm = async (info) => {
   let elfPath;
-  if (libdragonInfo.options.FILE) {
-    if (
-      path
-        .relative(libdragonInfo.root, libdragonInfo.options.FILE)
-        .startsWith('..')
-    ) {
+  if (info.options.FILE) {
+    if (path.relative(info.root, info.options.FILE).startsWith('..')) {
       throw new ParameterError(
-        `Provided file ${libdragonInfo.options.FILE} is outside the project directory.`
+        `Provided file ${info.options.FILE} is outside the project directory.`,
+        info.options.CURRENT_ACTION.name
       );
     }
-    if (!(await fileExists(libdragonInfo.options.FILE)))
+    if (!(await fileExists(info.options.FILE)))
       throw new ParameterError(
-        `Provided file ${libdragonInfo.options.FILE} does not exist`
+        `Provided file ${info.options.FILE} does not exist`,
+        info.options.CURRENT_ACTION.name
       );
-    elfPath = libdragonInfo.options.FILE;
+    elfPath = info.options.FILE;
   }
-  elfPath = elfPath ?? (await findElf(libdragonInfo.root));
+  elfPath = elfPath ?? (await findElf(info.root));
 
-  const haveSymbol = extraArgs.length > 0 && !extraArgs[0].startsWith('-');
+  const haveSymbol =
+    info.options.EXTRA_PARAMS.length > 0 &&
+    !info.options.EXTRA_PARAMS[0].startsWith('-');
 
   const finalArgs = haveSymbol
-    ? [`--disassemble=${extraArgs[0]}`, ...extraArgs.slice(1)]
-    : extraArgs;
+    ? [
+        `--disassemble=${info.options.EXTRA_PARAMS[0]}`,
+        ...info.options.EXTRA_PARAMS.slice(1),
+      ]
+    : info.options.EXTRA_PARAMS;
 
   const intermixSourceParams =
-    extraArgs.length === 0 || haveSymbol ? ['-S'] : [];
-
-  return await exec(libdragonInfo, [
-    'mips64-elf-objdump',
-    ...finalArgs,
-    ...intermixSourceParams,
-    toPosixPath(path.relative(process.cwd(), elfPath)),
-  ]);
+    info.options.EXTRA_PARAMS.length === 0 || haveSymbol ? ['-S'] : [];
+
+  return await exec({
+    ...info,
+    options: {
+      ...info.options,
+      EXTRA_PARAMS: [
+        'mips64-elf-objdump',
+        ...finalArgs,
+        ...intermixSourceParams,
+        toPosixPath(path.relative(process.cwd(), elfPath)),
+      ],
+    },
+  });
 };
 
 module.exports = {
   name: 'disasm',
   fn: disasm,
-  showStatus: true,
   forwardsRestParams: true,
   usage: {
     name: 'disasm [symbol|flags]',

package/modules/actions/exec.js

@@ -1,11 +1,18 @@
 const path = require('path');
+const { PassThrough } = require('stream');
 
 const { CONTAINER_TARGET_PATH } = require('../constants');
-const { log, dockerExec, toPosixPath } = require('../helpers');
+const {
+  log,
+  dockerExec,
+  toPosixPath,
+  fileExists,
+  dirExists,
+} = require('../helpers');
 
 const { start } = require('./start');
 const { dockerHostUserParams } = require('./docker-utils');
-const { installDependencies, mustHaveProject } = require('./utils');
+const { installDependencies } = require('./utils');
 
 function dockerRelativeWorkdir(libdragonInfo) {
   return (
@@ -19,72 +26,114 @@ function dockerRelativeWorkdirParams(libdragonInfo) {
   return ['--workdir', dockerRelativeWorkdir(libdragonInfo)];
 }
 
-const exec = async (libdragonInfo, commandAndParams) => {
-  await mustHaveProject(libdragonInfo);
-
+const exec = async (info) => {
+  const parameters = info.options.EXTRA_PARAMS.slice(1);
   log(
-    `Running ${commandAndParams[0]} at ${dockerRelativeWorkdir(
-      libdragonInfo
-    )} with [${commandAndParams.slice(1)}]`,
+    `Running ${info.options.EXTRA_PARAMS[0]} at ${dockerRelativeWorkdir(
+      info
+    )} with [${parameters}]`,
     true
   );
 
-  const tryCmd = (libdragonInfo) =>
-    libdragonInfo.containerId &&
-    dockerExec(
-      libdragonInfo,
-      [
-        ...dockerRelativeWorkdirParams(libdragonInfo),
-        ...dockerHostUserParams(libdragonInfo),
-      ],
-      commandAndParams,
-      true,
-      true // Cannot use "full" here, we need to know if the container is alive
+  const stdin = new PassThrough();
+
+  const paramsWithConvertedPaths = parameters.map((item) => {
+    if (item.startsWith('-')) {
+      return item;
+    }
+    if (item.includes(path.sep) && (fileExists(item) || dirExists(item))) {
+      return toPosixPath(
+        path.isAbsolute(item) ? path.relative(process.cwd(), item) : item
+      );
+    }
+    return item;
+  });
+
+  const tryCmd = (libdragonInfo, opts = {}) => {
+    const enableTTY = Boolean(process.stdout.isTTY && process.stdin.isTTY);
+
+    return (
+      libdragonInfo.containerId &&
+      dockerExec(
+        libdragonInfo,
+        [
+          ...dockerRelativeWorkdirParams(libdragonInfo),
+          ...dockerHostUserParams(libdragonInfo),
+        ],
+        [libdragonInfo.options.EXTRA_PARAMS[0], ...paramsWithConvertedPaths],
+        {
+          userCommand: true,
+          // Inherit stdin/out in tandem if we are going to disable TTY o/w the input
+          // stream remains inherited by the node process while the output pipe is
+          // waiting data from stdout and it behaves like we are still controlling
+          // the spawned process while the terminal is actually displaying say for
+          // example `less`.
+          inheritStdout: enableTTY,
+          inheritStdin: enableTTY,
+          // spawnProcess defaults does not apply to dockerExec so we need to
+          // provide these explicitly here.
+          inheritStderr: true,
+          ...opts,
+        }
+      )
     );
+  };
 
   let started = false;
-  const startOnceAndCmd = async () => {
+  const startOnceAndCmd = async (stdin) => {
     if (!started) {
-      const newId = await start(libdragonInfo);
+      const newId = await start(info);
+      const newInfo = {
+        ...info,
+        containerId: newId,
+      };
       started = true;
 
       // Re-install vendors on new container if one was created upon start
       // Ideally we would want the consumer to handle dependencies and rebuild
       // libdragon if necessary. Currently this saves the day with a little bit
       // extra waiting when the container is deleted.
-      if (libdragonInfo.containerId !== newId) {
-        await installDependencies({
-          ...libdragonInfo,
-          containerId: newId,
-        });
+      if (info.containerId !== newId) {
+        await installDependencies(newInfo);
       }
-      await tryCmd({
-        ...libdragonInfo,
-        containerId: newId,
-      });
+      await tryCmd(newInfo, { stdin });
       return newId;
     }
   };
 
-  if (!libdragonInfo.containerId) {
+  if (!info.containerId) {
     log(`Container does not exist for sure, restart`, true);
     await startOnceAndCmd();
-    return libdragonInfo;
+    return info;
   }
 
   try {
-    await tryCmd(libdragonInfo);
+    // Start collecting stdin data on an auxiliary stream such that we can pipe
+    // it back to the container process if this fails the first time. Then the
+    // initial failed docker process would eat up the input stream. Here, we pass
+    // it to the target process eventually via startOnceAndCmd. If the input
+    // stream is from a TTY, spawnProcess will already inherit it. Listening
+    // to the stream here causes problems for unknown reasons.
+    !process.stdin.isTTY && process.stdin.pipe(stdin);
+    await tryCmd(info, {
+      // Disable the error tty to be able to read the error message in case
+      // the container is not running
+      inheritStderr: false,
+      // In the first run, pass the stdin to the process if it is not a TTY
+      // o/w we loose a user input unnecesarily somehow.
+      stdin: !process.stdin.isTTY && process.stdin,
+    });
   } catch (e) {
     if (
       !e.out ||
       // TODO: is there a better way?
-      !e.out.toString().includes(libdragonInfo.containerId)
+      !e.out.toString().includes(info.containerId)
     ) {
       throw e;
     }
-    await startOnceAndCmd();
+    await startOnceAndCmd(stdin);
   }
-  return libdragonInfo;
+  return info;
 };
 
 module.exports = {

package/modules/actions/help.js

@@ -1,9 +1,9 @@
 const chalk = require('chalk');
 const commandLineUsage = require('command-line-usage');
 
-const { log } = require('../helpers');
+const { print } = require('../helpers');
 
-const printUsage = (_, actionArr) => {
+const printUsage = (info) => {
   const actions = require('./');
   const globalOptionDefinitions = [
     {
@@ -42,22 +42,24 @@ const printUsage = (_, actionArr) => {
         You can always switch to manual by re-running \`init\` with \`--strategy manual\`, though you will be responsible for managing the existing submodule/subtree. Also it is not possible to automatically switch back.
 
         With the \`manual\` strategy, it is still recommended to have a git repository at project root such that container actions can execute faster by caching the container id inside the \`.git\` folder.\n`,
-      alias: 'v',
+      alias: 's',
       typeLabel: '<strategy>',
       group: 'vendoring',
     },
   ];
 
-  const actionsToShow = actionArr
-    ?.filter((action) => Object.keys(actions).includes(action))
-    .filter((action) => !['help'].includes(action));
+  const actionsToShow =
+    info?.options.EXTRA_PARAMS?.filter(
+      (action) =>
+        Object.keys(actions).includes(action) && !['help'].includes(action)
+    ) ?? (info ? [info.options.CURRENT_ACTION.name] : []);
 
   const sections = [
     {
       header: chalk.green('Usage:'),
       content: `libdragon [flags] <action>
 
-      For string flags valid syntax are: \`-i <value>\` or \`--image=<value>\``,
+      Joining flags not supported, provide them separately like \`-v -i\` `,
     },
     ...(actionsToShow?.length
       ? actionsToShow.flatMap((action) => [
@@ -91,14 +93,14 @@ const printUsage = (_, actionArr) => {
     },
   ];
   const usage = commandLineUsage(sections);
-  log(usage);
+  print(usage);
 };
 
 module.exports = {
   name: 'help',
   fn: printUsage,
-  showStatus: true,
   forwardsRestParams: true,
+  mustHaveProject: false,
   usage: {
     name: 'help [action]',
     summary: 'Display this help information or details for the given action.',

package/modules/actions/index.js

@@ -1,15 +1,14 @@
 module.exports = {
-  start: require('./start'),
-  stop: require('./stop'),
   init: require('./init'),
-
-  exec: require('./exec'),
   make: require('./make'),
 
+  update: require('./update'),
+  exec: require('./exec'),
   disasm: require('./disasm'),
-
   install: require('./install'),
-  update: require('./update'),
+
+  start: require('./start'),
+  stop: require('./stop'),
 
   help: require('./help'),
   version: require('./version'),