libdragon 10.4.2 → 10.7.0

package/CHANGELOG.md

@@ -1,5 +1,80 @@
 # Change Log
 
+## [10.7.0] - 2022-04-17
+
+### Fixed
+
+- Logs properly goes to stderr now. Previously they were written to stdout. This
+means the id output of the `start` action is now written to stdout while we can
+also display other information on the terminal. This allowed enabling the docker
+logs for a more responsive experience. The output of `help` still goes to stdout.
+- Skip reading the project information for `version` and `help` commands. This
+was breaking things when these were run with a stopped docker service.
+- Resolved a potential failure case where the host mount path is not reported
+as a native path. That would prevent proper container discovery. This was not
+being an issue previously but I realized it was not working for Windows now.
+- Do not throw if the output stream is not finalized. This was previously causing
+issues when the long output of a command is piped to another process like `less`.
+Fixes #46
+- Produce a correct error if an invalid flag with a singe `-` is provided.
+- Sekeleton project's Makefile `clean` recipe now works properly. Fixes #56.
+- A case where it is impossible to recover the container is fixed. This was
+happening when the host does not have git and the cached container file is
+missing. In that case the cli was attempting to run git without the containerId.
+- `destroy` can now cleanup any remaining container even if the project init was
+failed. This is also useful to clean up old folders that were once libdragon
+projects.
+
+### Added
+
+- Stdin consumption support. Now it is possible to pipe anything to `exec` and
+it will pass it through to the target. In case of no running container, it will
+keep a copy of the stdin stream until the docker process is ready. This enables
+piping in data from the host if ever needed for some reason. This enables usages
+like `cat file.txt | libdragon exec cat - | less`.
+- Automatically convert host paths into posix format so that the user can use
+the host's path autocompletion. It will also convert absolute host paths into
+relative container paths automatically. Previously all paths were assumed to be
+container paths relative to the location corresponding to the host cwd.
+Closes #24.
+
+### Changed
+
+- Refactored process spawns.
+- Refactored main flow and separated parsing logic.
+- Reorder actions & correction on flag usage for help output.
+- Setting `--verbose` for `start` does not guarantee the only-id output anymore.
+- Refactored parameter parsing.
+- Update submdule for local development.
+
+## [10.6.0] - 2022-04-09
+### Fixed
+
+- Fix a path bug that would cause incorrect behaviour when the command is run
+deeper than a single level in the project folder.
+- Fix a potential issue where `build.sh`might be incorrectly found inexistant
+if the OS is picky about the paths to have native separators.
+- Only save project information when necessary. Previously actions like `help`
+were saving project info mistakenly.
+
+### Added
+
+- `disasm` action to simplify disassembling ELF files generated by the toolchain.
+- `version` action to display current version.
+- `destroy` action to remove the libdragon project.
+- Additional documentation for flags.
+- Print duration information when verbose.
+
+### Changed
+
+- Refactored out NPM related functions.
+- Moved usage parameters to respective actions files as a refactor.
+- It is possible to provide an absolute path to init `--directory` as long as it
+is inside the project directory. Previously it was possible to provide somewhere
+outside the project, but it would fail with an unexpected error.
+- Simplify saving mechanism. Each action now internally resolves into the data
+to save if any.
+
 ## [10.4.2] - 2022-04-03
 
 ### Fixed

package/index.js

@@ -1,8 +1,7 @@
 #!/usr/bin/env node
 
-const chalk = require('chalk');
+const chalk = require('chalk').stderr;
 
-const actions = require('./modules/actions');
 const { fn: printUsage } = require('./modules/actions/help');
 const {
   STATUS_OK,
@@ -15,126 +14,26 @@ const {
   CommandError,
   ParameterError,
   ValidationError,
+  log,
 } = require('./modules/helpers');
+const { parseParameters } = require('./modules/parameters');
 const { readProjectInfo, writeProjectInfo } = require('./modules/project-info');
 
-let options = {},
-  currentAction;
-
-for (let i = 2; i < process.argv.length; i++) {
-  const val = process.argv[i];
-
-  // Allow console here
-  /* eslint-disable no-console */
-
-  if (['--verbose', '-v'].includes(val)) {
-    options.VERBOSE = true;
-    globals.verbose = true;
-    continue;
-  }
-
-  if (['--image', '-i'].includes(val)) {
-    options.DOCKER_IMAGE = process.argv[++i];
-    continue;
-  } else if (val.indexOf('--image=') === 0) {
-    options.DOCKER_IMAGE = val.split('=')[1];
-    continue;
-  }
-
-  if (['--directory', '-d'].includes(val)) {
-    options.VENDOR_DIR = process.argv[++i];
-    continue;
-  } else if (val.indexOf('--directory=') === 0) {
-    options.VENDOR_DIR = val.split('=')[1];
-    continue;
-  }
-
-  if (['--strategy', '-s'].includes(val)) {
-    options.VENDOR_STRAT = process.argv[++i];
-    continue;
-  } else if (val.indexOf('--strategy=') === 0) {
-    options.VENDOR_STRAT = val.split('=')[1];
-    continue;
-  }
-
-  if (val.indexOf('--') >= 0) {
-    console.error(chalk.red(`Invalid flag \`${val}\``));
-    printUsage();
-    process.exit(STATUS_BAD_PARAM);
-  }
-
-  if (currentAction) {
-    console.error(
-      chalk.red(`Expected only a single action, found: \`${val}\``)
-    );
-    printUsage();
-    process.exit(STATUS_BAD_PARAM);
-  }
-
-  currentAction = actions[val];
-
-  if (!currentAction) {
-    console.error(chalk.red(`Invalid action \`${val}\``));
-    printUsage();
-    process.exit(STATUS_BAD_PARAM);
-  }
-
-  if (currentAction.forwardsRestParams) {
-    options.PARAMS = process.argv.slice(i + 1);
-    break;
-  }
-}
-
-if (!currentAction) {
-  console.error(chalk.red('No action provided'));
-  printUsage();
-  process.exit(STATUS_BAD_PARAM);
-}
-
-if (currentAction === actions.exec && options.PARAMS.length === 0) {
-  console.error(chalk.red('You should provide a command to exec'));
-  printUsage(undefined, [currentAction.name]);
-  process.exit(STATUS_BAD_PARAM);
-}
-
-if (
-  ![actions.init, actions.install, actions.update].includes(currentAction) &&
-  options.DOCKER_IMAGE
-) {
-  console.error(chalk.red('Invalid flag: image'));
-  printUsage(undefined, [currentAction.name]);
-  process.exit(STATUS_BAD_PARAM);
-}
-
-if (
-  options.VENDOR_STRAT &&
-  !['submodule', 'subtree', 'manual'].includes(options.VENDOR_STRAT)
-) {
-  console.error(chalk.red(`Invalid strategy \`${options.VENDOR_STRAT}\``));
-  printUsage();
-  process.exit(STATUS_BAD_PARAM);
-}
-
-readProjectInfo()
-  .then((info) =>
-    currentAction.fn(
-      {
-        ...info,
-        options,
-        ...currentAction,
-      },
-      options.PARAMS ?? []
-    )
-  )
+parseParameters(process.argv)
+  .then(readProjectInfo)
+  .then((info) => info.options.CURRENT_ACTION.fn(info))
   .catch((e) => {
     if (e instanceof ParameterError) {
-      console.error(chalk.red(e.message));
-      printUsage(undefined, [currentAction.name]);
+      log(chalk.red(e.message));
+      printUsage({
+        options: {
+          EXTRA_PARAMS: [e.actionName],
+        },
+      });
       process.exit(STATUS_BAD_PARAM);
     }
-
     if (e instanceof ValidationError) {
-      console.error(chalk.red(e.message));
+      log(chalk.red(e.message));
       process.exit(STATUS_VALIDATION_ERROR);
     }
 
@@ -142,7 +41,7 @@ readProjectInfo()
 
     // Show additional information to user if verbose or we did a mistake
     if (globals.verbose || !userTargetedError) {
-      console.error(chalk.red(globals.verbose ? e.stack : e.message));
+      log(chalk.red(globals.verbose ? e.stack : e.message));
     }
 
     // Print the underlying error out only if not verbose and we did a mistake
@@ -160,10 +59,9 @@ readProjectInfo()
     // We don't have a user targeted error anymore, we did a mistake for sure
     process.exit(STATUS_ERROR);
   })
-  .then(() => {
-    // Everything was done, update the configuration file if not exiting early
-    return writeProjectInfo();
-  })
+  // Everything was done, update the configuration file if not exiting early
+  .then(writeProjectInfo)
   .finally(() => {
+    log(`Took: ${process.uptime()}s`, true);
     process.exit(STATUS_OK);
   });

package/modules/actions/destroy.js

@@ -0,0 +1,34 @@
+const fs = require('fs/promises');
+const path = require('path');
+
+const { destroyContainer } = require('./utils');
+const { CONFIG_FILE, LIBDRAGON_PROJECT_MANIFEST } = require('../constants');
+const { fileExists, dirExists, log } = require('../helpers');
+const chalk = require('chalk');
+
+const destroy = async (libdragonInfo) => {
+  await destroyContainer(libdragonInfo);
+
+  const projectPath = path.join(libdragonInfo.root, LIBDRAGON_PROJECT_MANIFEST);
+  const configPath = path.join(projectPath, CONFIG_FILE);
+
+  if (await fileExists(configPath)) {
+    await fs.rm(configPath);
+  }
+  if (await dirExists(projectPath)) {
+    await fs.rmdir(projectPath);
+  }
+
+  log(chalk.green('Done cleanup.'));
+};
+
+module.exports = {
+  name: 'destroy',
+  fn: destroy,
+  mustHaveProject: false,
+  usage: {
+    name: 'destroy',
+    summary: 'Do clean-up for current project.',
+    description: `Removes libdragon configuration from current project and removes any known containers but will not touch previously vendored files. \`libdragon\` will not work anymore for this project.`,
+  },
+};

package/modules/actions/disasm.js

@@ -0,0 +1,121 @@
+const path = require('path');
+const fs = require('fs/promises');
+
+const { fn: exec } = require('./exec');
+const {
+  ValidationError,
+  toPosixPath,
+  dirExists,
+  fileExists,
+  ParameterError,
+} = require('../helpers');
+
+const findElf = async (stop, start = '.') => {
+  start = path.resolve(start);
+
+  const files = await fs.readdir(start);
+
+  const buildDir = path.join(start, 'build');
+  if (await dirExists(buildDir)) {
+    const elfFile = await findElf(buildDir, buildDir);
+    if (elfFile) return elfFile;
+  }
+
+  const elfFiles = files.filter((name) => name.endsWith('.elf'));
+
+  if (elfFiles.length > 1) {
+    throw new ValidationError(
+      `Multiple ELF files found in ${path.resolve(
+        start
+      )}. Use --file to specify.`
+    );
+  }
+
+  if (elfFiles.length === 1) {
+    return path.join(start, elfFiles[0]);
+  }
+
+  const parent = path.join(start, '..');
+  if (start !== stop) {
+    return await findElf(stop, parent);
+  } else {
+    throw new ValidationError(`No ELF files found. Use --file to specify.`);
+  }
+};
+
+const disasm = async (info) => {
+  let elfPath;
+  if (info.options.FILE) {
+    if (path.relative(info.root, info.options.FILE).startsWith('..')) {
+      throw new ParameterError(
+        `Provided file ${info.options.FILE} is outside the project directory.`,
+        info.options.CURRENT_ACTION.name
+      );
+    }
+    if (!(await fileExists(info.options.FILE)))
+      throw new ParameterError(
+        `Provided file ${info.options.FILE} does not exist`,
+        info.options.CURRENT_ACTION.name
+      );
+    elfPath = info.options.FILE;
+  }
+  elfPath = elfPath ?? (await findElf(info.root));
+
+  const haveSymbol =
+    info.options.EXTRA_PARAMS.length > 0 &&
+    !info.options.EXTRA_PARAMS[0].startsWith('-');
+
+  const finalArgs = haveSymbol
+    ? [
+        `--disassemble=${info.options.EXTRA_PARAMS[0]}`,
+        ...info.options.EXTRA_PARAMS.slice(1),
+      ]
+    : info.options.EXTRA_PARAMS;
+
+  const intermixSourceParams =
+    info.options.EXTRA_PARAMS.length === 0 || haveSymbol ? ['-S'] : [];
+
+  return await exec({
+    ...info,
+    options: {
+      ...info.options,
+      EXTRA_PARAMS: [
+        'mips64-elf-objdump',
+        ...finalArgs,
+        ...intermixSourceParams,
+        toPosixPath(path.relative(process.cwd(), elfPath)),
+      ],
+    },
+  });
+};
+
+module.exports = {
+  name: 'disasm',
+  fn: disasm,
+  forwardsRestParams: true,
+  usage: {
+    name: 'disasm [symbol|flags]',
+    summary: 'Disassemble the nearest *.elf file.',
+    description: `Executes \`objdump\` for the nearest *.elf file starting from the working directory, going up. If there is a \`build\` directory in the searched paths, checks inside it as well. Any extra flags are passed down to \`objdump\` before the filename.
+
+    If a symbol name is provided after the action, it is converted to \`--disassembly=<symbol>\` and intermixed source* (\`-S\`) is automatically applied. This allows disassembling a single symbol by running;
+
+    \`libdragon disasm main\`
+
+    Again, any following flags are forwarded down. If run without extra symbol/flags, disassembles whole ELF with \`-S\` by default.
+
+    Must be run in an initialized libdragon project.
+
+    * Note that to be able to see the source, the code must be built with \`D=1\``,
+
+    optionList: [
+      {
+        name: 'file',
+        description:
+          'Provide a specific ELF file relative to current working directory and inside the libdragon project.',
+        alias: 'f',
+        typeLabel: ' ',
+      },
+    ],
+  },
+};

package/modules/actions/docker-utils.js

@@ -0,0 +1,8 @@
+function dockerHostUserParams(libdragonInfo) {
+  const { uid, gid } = libdragonInfo.userInfo;
+  return ['-u', `${uid >= 0 ? uid : ''}:${gid >= 0 ? gid : ''}`];
+}
+
+module.exports = {
+  dockerHostUserParams,
+};

package/modules/actions/exec.js

@@ -1,14 +1,18 @@
 const path = require('path');
+const { PassThrough } = require('stream');
 
 const { CONTAINER_TARGET_PATH } = require('../constants');
-const { log, dockerExec, toPosixPath } = require('../helpers');
+const {
+  log,
+  dockerExec,
+  toPosixPath,
+  fileExists,
+  dirExists,
+} = require('../helpers');
 
 const { start } = require('./start');
-const {
-  dockerHostUserParams,
-  installDependencies,
-  mustHaveProject,
-} = require('./utils');
+const { dockerHostUserParams } = require('./docker-utils');
+const { installDependencies } = require('./utils');
 
 function dockerRelativeWorkdir(libdragonInfo) {
   return (
@@ -22,71 +26,114 @@ function dockerRelativeWorkdirParams(libdragonInfo) {
   return ['--workdir', dockerRelativeWorkdir(libdragonInfo)];
 }
 
-const exec = async (libdragonInfo, commandAndParams) => {
-  await mustHaveProject(libdragonInfo);
-
+const exec = async (info) => {
+  const parameters = info.options.EXTRA_PARAMS.slice(1);
   log(
-    `Running ${commandAndParams[0]} at ${dockerRelativeWorkdir(
-      libdragonInfo
-    )} with [${commandAndParams.slice(1)}]`,
+    `Running ${info.options.EXTRA_PARAMS[0]} at ${dockerRelativeWorkdir(
+      info
+    )} with [${parameters}]`,
     true
   );
 
-  const tryCmd = (libdragonInfo) =>
-    libdragonInfo.containerId &&
-    dockerExec(
-      libdragonInfo,
-      [
-        ...dockerRelativeWorkdirParams(libdragonInfo),
-        ...dockerHostUserParams(libdragonInfo),
-      ],
-      commandAndParams,
-      true,
-      true // Cannot use "full" here, we need to know if the container is alive
+  const stdin = new PassThrough();
+
+  const paramsWithConvertedPaths = parameters.map((item) => {
+    if (item.startsWith('-')) {
+      return item;
+    }
+    if (item.includes(path.sep) && (fileExists(item) || dirExists(item))) {
+      return toPosixPath(
+        path.isAbsolute(item) ? path.relative(process.cwd(), item) : item
+      );
+    }
+    return item;
+  });
+
+  const tryCmd = (libdragonInfo, opts = {}) => {
+    const enableTTY = Boolean(process.stdout.isTTY && process.stdin.isTTY);
+
+    return (
+      libdragonInfo.containerId &&
+      dockerExec(
+        libdragonInfo,
+        [
+          ...dockerRelativeWorkdirParams(libdragonInfo),
+          ...dockerHostUserParams(libdragonInfo),
+        ],
+        [libdragonInfo.options.EXTRA_PARAMS[0], ...paramsWithConvertedPaths],
+        {
+          userCommand: true,
+          // Inherit stdin/out in tandem if we are going to disable TTY o/w the input
+          // stream remains inherited by the node process while the output pipe is
+          // waiting data from stdout and it behaves like we are still controlling
+          // the spawned process while the terminal is actually displaying say for
+          // example `less`.
+          inheritStdout: enableTTY,
+          inheritStdin: enableTTY,
+          // spawnProcess defaults does not apply to dockerExec so we need to
+          // provide these explicitly here.
+          inheritStderr: true,
+          ...opts,
+        }
+      )
     );
+  };
 
   let started = false;
-  const startOnceAndCmd = async () => {
+  const startOnceAndCmd = async (stdin) => {
     if (!started) {
-      const newId = await start(libdragonInfo);
+      const newId = await start(info);
+      const newInfo = {
+        ...info,
+        containerId: newId,
+      };
       started = true;
 
       // Re-install vendors on new container if one was created upon start
       // Ideally we would want the consumer to handle dependencies and rebuild
       // libdragon if necessary. Currently this saves the day with a little bit
       // extra waiting when the container is deleted.
-      if (libdragonInfo.containerId !== newId) {
-        await installDependencies({
-          ...libdragonInfo,
-          containerId: newId,
-        });
+      if (info.containerId !== newId) {
+        await installDependencies(newInfo);
       }
-      await tryCmd({
-        ...libdragonInfo,
-        containerId: newId,
-      });
+      await tryCmd(newInfo, { stdin });
       return newId;
     }
   };
 
-  if (!libdragonInfo.containerId) {
+  if (!info.containerId) {
     log(`Container does not exist for sure, restart`, true);
     await startOnceAndCmd();
-    return;
+    return info;
   }
 
   try {
-    await tryCmd(libdragonInfo);
+    // Start collecting stdin data on an auxiliary stream such that we can pipe
+    // it back to the container process if this fails the first time. Then the
+    // initial failed docker process would eat up the input stream. Here, we pass
+    // it to the target process eventually via startOnceAndCmd. If the input
+    // stream is from a TTY, spawnProcess will already inherit it. Listening
+    // to the stream here causes problems for unknown reasons.
+    !process.stdin.isTTY && process.stdin.pipe(stdin);
+    await tryCmd(info, {
+      // Disable the error tty to be able to read the error message in case
+      // the container is not running
+      inheritStderr: false,
+      // In the first run, pass the stdin to the process if it is not a TTY
+      // o/w we loose a user input unnecesarily somehow.
+      stdin: !process.stdin.isTTY && process.stdin,
+    });
   } catch (e) {
     if (
       !e.out ||
       // TODO: is there a better way?
-      !e.out.toString().includes(libdragonInfo.containerId)
+      !e.out.toString().includes(info.containerId)
     ) {
       throw e;
     }
-    await startOnceAndCmd();
+    await startOnceAndCmd(stdin);
   }
+  return info;
 };
 
 module.exports = {
@@ -94,4 +141,15 @@ module.exports = {
   fn: exec,
   forwardsRestParams: true,
   showStatus: true,
+  usage: {
+    name: 'exec <command>',
+    summary: 'Execute given command in the current directory.',
+    description: `Executes the given command in the container passing down any arguments provided. If you change your host working directory, the command will be executed in the corresponding folder in the container as well.
+
+    This action will first try to execute the command in the container and if the container is not accessible, it will attempt a complete \`start\` cycle.
+
+    This will properly passthrough your TTY if you have one. So by running \`libdragon exec bash\` you can start an interactive bash session with full TTY support.
+    
+    Must be run in an initialized libdragon project.`,
+  },
 };