@openfn/cli 0.0.26 → 0.0.28

package/README.md

@@ -50,7 +50,9 @@ If output.json and state.json are not passed, the CLI will look for them next to
 
 The CLI can auto-install language adaptors to its own privately maintained repo. Run `openfn repo list` to see where the repo is, and what's in it. Set the `OPENFN_REPO_DIR` env var to specify the repo folder. When autoinstalling, the CLI will check to see if a matching version is found in the repo.
 
-You can specify adaptors with a shorthand (`http`) or use the full package name (`@openfn/language-http`). You can add a specific version like `http@2.0.0`. You can pass a path to a locally installed adaptor like `http=/repo/openfn/adaptors/my-http-build`. Set the OPENFN_ADAPTORS_REPO env var to load adaptors straight out of the monorepo (pass the `--no-adaptors-repo` flag to disable this for a single run).
+You can specify adaptors with a shorthand (`http`) or use the full package name (`@openfn/language-http`). You can add a specific version like `http@2.0.0`. You can pass a path to a locally installed adaptor like `http=/repo/openfn/adaptors/my-http-build`.
+
+If you have the adaptors monorepo set up on your machine, you can also run from that. Pass the `-m` flag to load from the monorepo. Set the monorepo location by setting the OPENFN_ADAPTORS_REPO env var to a valid path. This runs from the built package, so remember to build an adaptor before running!
 
 You can pass `--log info` to get more feedback about what's happening, or `--log debug` for more details than you could ever use.
 
@@ -68,6 +70,40 @@ If no command is specified, execute will run.
 
 To get more information about a command, including usage examples, run `openfn <command> help`, ie, `openfn compile help`.
 
+## Logging 
+
+The CLI is actually a collection of packages, each of which will log with slightly different rules. To help understand where logs are coming from, each package prints a namespace or prefix at the start of its log.
+
+* [CLI] - the CLI itself, responsible for parsing and validating user input, reading and writing to disk, and executing the correct functionality.
+* [CMP] - the Compiler will parse openfn jobs into executable Javascript, changing your code
+* [R/T] - the Runtime executes your job code in a secure sandboxed environment, one operation at a time
+* [JOB] - the actual job code that your wrote. Any console.log statements in your job will appear under this namespace.
+
+The CLI will log information at three different levels of verbosity: `default`, `info` and `debug` (`none` is also supported).
+
+To set the log level, pass `--log info` into your command. You can configure this for individual packages, ie `--log cmp=debug` will run the compiler with debug logging but leave everything else at default.
+
+Note that, unless explicitly overriden, jobs will always report at debug verbosity (meaning job logging will always be shown).
+
+If something unexpected happens during a command, your first step should be to re-run with info-level logging.
+
+`default` logging is designed to give high-level feedback about what you absolutely need to know. It will show any errors or warnings, as well as high-level reporting about what the command has actually done.
+
+`info` level logging is suitable for most developers. It is more verbose than default but still aims to provide high-level information about a command. It includes version numbers, key paths, and simple reporting about how the compiler changes your code (see below).
+
+`debug` level logging is highly verbose and aims to tell you everything that's going on under-the hood. This is aimed mostly at CLI/runtime developers and can be very useful for debugging problems.
+
+## Structred/JSON logging
+
+By default all logs will be printed as human-readable strings.
+
+For a more structured output, you can emit logs as JSON objects with `level`, `name` and `message` properties:
+```
+{ level: 'info', name: 'CLI', message: ['Loaded adaptor'] }
+```
+
+Pass `--log-json` to the CLI to do this. You can also set the OPENFN_LOG_JSON env var (and use `--no-log-json` to disable).
+
 ## Compilation
 
 The CLI will attempt to compile your job code into normalized Javascript. It will do a number of things to make your code robust and portable:

package/dist/index.js

@@ -95,6 +95,10 @@ var executeCommand = {
     return applyExecuteOptions(yargs2).option("immutable", {
       boolean: true,
       description: "Treat state as immutable"
+    }).option("use-adaptors-monorepo", {
+      alias: "m",
+      boolean: true,
+      description: "Load adaptors from the monorepo. The OPENFN_ADAPTORS_REPO env var must be set to a valid path"
     }).option("autoinstall", {
       alias: "i",
       boolean: true,
@@ -105,6 +109,9 @@ var executeCommand = {
     }).option("state-stdin", {
       alias: "S",
       description: "Read state from stdin (instead of a file)"
+    }).option("skip-adaptor-validation", {
+      boolean: true,
+      description: "Skip adaptor validation warnings"
     }).option("timeout", {
       alias: "-t",
       description: "Set the timeout duration in MS"
@@ -113,7 +120,7 @@ var executeCommand = {
       description: "Skip compilation"
     }).option("no-strict-output", {
       boolean: true,
-      description: "Allow properties other than data to be returned in the output."
+      description: "Allow properties other than data to be returned in the output"
     }).example(
       "openfn foo/job.js",
       "Reads foo/job.js, looks for state and output in foo"
@@ -138,7 +145,7 @@ var applyExecuteOptions = (yargs2) => yargs2.positional("path", {
   description: "Print output to stdout (instead of a file)"
 }).option("adaptors", {
   alias: ["a", "adaptor"],
-  description: "A language adaptor to use for the job. Short-form names are allowed. Can include an explicit path to a local adaptor build.",
+  description: "A language adaptor to use for the job. Short-form names are allowed. Can include an explicit path to a local adaptor build",
   array: true
 }).option("no-expand", {
   description: "Don	 attempt to auto-expand adaptor shorthand names",
@@ -206,6 +213,9 @@ var cmd = yargs(hideBin(process.argv)).command(command_default).command(command_
   alias: ["l"],
   description: "Set the default log level to none, default, info or debug",
   array: true
+}).option("log-json", {
+  description: "Output all logs as JSON objects",
+  boolean: true
 }).example("openfn execute help", "Show documentation for the execute command").example(
   "openfn docs @openfn/language-common each",
   "Get more help on the common.each command"

package/dist/process/runner.js

@@ -12,8 +12,11 @@ var namespaces = {
   [JOB]: "JOB"
 };
 var createLogger = (name = "", options) => {
-  const logOptions = options.log || {};
+  const logOptions = options.log || { json: true };
   let level = logOptions[name] || logOptions.default || "default";
+  if (options.logJson) {
+    logOptions.json = true;
+  }
   return actualCreateLogger(namespaces[name] || name, {
     level,
     ...logOptions
@@ -78,22 +81,27 @@ function ensureOpts(basePath = ".", opts) {
     adaptors: opts.adaptors || [],
     autoinstall: opts.autoinstall,
     command: opts.command,
+    expand: opts.expand !== false,
     force: opts.force || false,
-    repoDir: opts.repoDir || process.env.OPENFN_REPO_DIR || DEFAULT_REPO_DIR,
+    immutable: opts.immutable || false,
+    logJson: typeof opts.logJson == "boolean" ? opts.logJson : Boolean(process.env.OPENFN_LOG_JSON),
     noCompile: Boolean(opts.noCompile),
-    expand: opts.expand !== false,
-    outputStdout: Boolean(opts.outputStdout),
     operation: opts.operation,
+    outputStdout: Boolean(opts.outputStdout),
     packages: opts.packages,
-    stateStdin: opts.stateStdin,
-    timeout: opts.timeout,
+    repoDir: opts.repoDir || process.env.OPENFN_REPO_DIR || DEFAULT_REPO_DIR,
+    skipAdaptorValidation: opts.skipAdaptorValidation ?? false,
     specifier: opts.specifier,
+    stateStdin: opts.stateStdin,
     strictOutput: opts.strictOutput ?? true,
-    immutable: opts.immutable || false
+    timeout: opts.timeout
   };
   const set = (key, value) => {
     newOpts[key] = opts.hasOwnProperty(key) ? opts[key] : value;
   };
+  if (opts.useAdaptorsMonorepo) {
+    newOpts.monorepoPath = process.env.OPENFN_ADAPTORS_REPO || "ERR";
+  }
   let baseDir = basePath;
   if (basePath.endsWith(".js")) {
     baseDir = path.dirname(basePath);
@@ -163,11 +171,11 @@ var execute_default = (code, state, opts) => {
 function parseAdaptors(opts) {
   const adaptors = {};
   opts.adaptors.reduce((obj, exp) => {
-    const [module, path3] = exp.split("=");
+    const [module, path4] = exp.split("=");
     const { name, version } = getNameAndVersion(module);
     const info = {};
-    if (path3) {
-      info.path = path3;
+    if (path4) {
+      info.path = path4;
     }
     if (version) {
       info.version = version;
@@ -209,10 +217,10 @@ var stripVersionSpecifier = (specifier) => {
   return specifier;
 };
 var resolveSpecifierPath = async (pattern, repoDir, log) => {
-  const [specifier, path3] = pattern.split("=");
-  if (path3) {
-    log.debug(`Resolved ${specifier} to path: ${path3}`);
-    return path3;
+  const [specifier, path4] = pattern.split("=");
+  if (path4) {
+    log.debug(`Resolved ${specifier} to path: ${path4}`);
+    return path4;
   }
   const repoPath = await getModulePath(specifier, repoDir, log);
   if (repoPath) {
@@ -229,15 +237,15 @@ var loadTransformOptions = async (opts, log) => {
     const [pattern] = opts.adaptors;
     const [specifier] = pattern.split("=");
     log.debug(`Attempting to preload typedefs for ${specifier}`);
-    const path3 = await resolveSpecifierPath(pattern, opts.repoDir, log);
-    if (path3) {
+    const path4 = await resolveSpecifierPath(pattern, opts.repoDir, log);
+    if (path4) {
       try {
-        exports = await preloadAdaptorExports(path3);
+        exports = await preloadAdaptorExports(path4);
         if (exports) {
           log.info(`Loaded typedefs for ${specifier}`);
         }
       } catch (e) {
-        log.error(`Failed to load adaptor typedefs from path ${path3}`);
+        log.error(`Failed to load adaptor typedefs from path ${path4}`);
         log.error(e);
       }
     }
@@ -299,7 +307,7 @@ var expand_adaptors_default = (names, log = nullLogger) => names?.map((name) =>
     return name;
   }
   const expanded = `@openfn/language-${name}`;
-  log.info(`Expanded adaptor ${name} to ${expanded}`);
+  log.debug(`Expanded adaptor ${name} to ${expanded}`);
   return expanded;
 });
 
@@ -370,20 +378,44 @@ var list = async (options, logger) => {
   logger.success("Installed packages:\n\n" + treeify.asTree(output));
 };
 
+// src/util/validate-adaptors.ts
+var validateAdaptors = async (options, logger) => {
+  if (options.skipAdaptorValidation) {
+    return;
+  }
+  if (!options.adaptors || options.adaptors.length === 0) {
+    logger.warn("WARNING: No adaptor provided!");
+    logger.warn(
+      "This job will probably fail. Pass an adaptor with the -a flag, eg:"
+    );
+    logger.break();
+    logger.print("          openfn job.js -a common");
+    logger.break();
+  }
+};
+var validate_adaptors_default = validateAdaptors;
+
 // src/execute/handler.ts
 var getAutoinstallTargets = (options) => {
-  if (options.autoinstall && options.adaptors) {
+  if (options.adaptors) {
     return options.adaptors?.filter((a) => !/=/.test(a));
   }
   return [];
 };
 var executeHandler = async (options, logger) => {
   const start = new Date().getTime();
-  const autoInstallTargets = getAutoinstallTargets(options);
-  if (autoInstallTargets.length) {
-    const { repoDir } = options;
-    logger.info("Auto-installing language adaptors");
-    await install({ packages: autoInstallTargets, repoDir }, logger);
+  await validate_adaptors_default(options, logger);
+  const { repoDir, monorepoPath, autoinstall } = options;
+  if (autoinstall) {
+    if (monorepoPath) {
+      logger.warn("Skipping auto-install as monorepo is being used");
+    } else {
+      const autoInstallTargets = getAutoinstallTargets(options);
+      if (autoInstallTargets.length) {
+        logger.info("Auto-installing language adaptors");
+        await install({ packages: autoInstallTargets, repoDir }, logger);
+      }
+    }
   }
   const state = await load_state_default(options, logger);
   const code = await compile_default(options, logger);
@@ -447,20 +479,20 @@ var RETRY_COUNT = 20;
 var TIMEOUT_MS = 1e3 * 60;
 var actualDocGen = (specifier) => describePackage(specifier, {});
 var ensurePath = (filePath) => mkdirSync(path2.dirname(filePath), { recursive: true });
-var generatePlaceholder = (path3) => {
-  writeFileSync(path3, `{ "loading": true, "timestamp": ${Date.now()}}`);
+var generatePlaceholder = (path4) => {
+  writeFileSync(path4, `{ "loading": true, "timestamp": ${Date.now()}}`);
 };
 var finish = (logger, resultPath) => {
   logger.success("Done! Docs can be found at:\n");
   logger.print(`  ${path2.resolve(resultPath)}`);
 };
-var generateDocs = async (specifier, path3, docgen, logger) => {
+var generateDocs = async (specifier, path4, docgen, logger) => {
   const result = await docgen(specifier);
-  await writeFile3(path3, JSON.stringify(result, null, 2));
-  finish(logger, path3);
-  return path3;
+  await writeFile3(path4, JSON.stringify(result, null, 2));
+  finish(logger, path4);
+  return path4;
 };
-var waitForDocs = async (docs, path3, logger, retryDuration = RETRY_DURATION) => {
+var waitForDocs = async (docs, path4, logger, retryDuration = RETRY_DURATION) => {
   try {
     if (docs.hasOwnProperty("loading")) {
       logger.info("Docs are being loaded by another process. Waiting.");
@@ -472,19 +504,19 @@ var waitForDocs = async (docs, path3, logger, retryDuration = RETRY_DURATION) =>
             clearInterval(i);
             reject(new Error("Timed out waiting for docs to load"));
           }
-          const updated = JSON.parse(readFileSync(path3, "utf8"));
+          const updated = JSON.parse(readFileSync(path4, "utf8"));
           if (!updated.hasOwnProperty("loading")) {
             logger.info("Docs found!");
             clearInterval(i);
-            resolve(path3);
+            resolve(path4);
           }
           count++;
         }, retryDuration);
       });
     } else {
-      logger.info(`Docs already written to cache at ${path3}`);
-      finish(logger, path3);
-      return path3;
+      logger.info(`Docs already written to cache at ${path4}`);
+      finish(logger, path4);
+      return path4;
     }
   } catch (e) {
     logger.error("Existing doc JSON corrupt. Aborting");
@@ -501,28 +533,28 @@ var docgenHandler = (options, logger, docgen = actualDocGen, retryDuration = RET
     process.exit(9);
   }
   logger.success(`Generating docs for ${specifier}`);
-  const path3 = `${repoDir}/docs/${specifier}.json`;
-  ensurePath(path3);
+  const path4 = `${repoDir}/docs/${specifier}.json`;
+  ensurePath(path4);
   const handleError = () => {
     logger.info("Removing placeholder");
-    rmSync(path3);
+    rmSync(path4);
   };
   try {
-    const existing = readFileSync(path3, "utf8");
+    const existing = readFileSync(path4, "utf8");
     const json = JSON.parse(existing);
     if (json && json.timeout && Date.now() - json.timeout >= TIMEOUT_MS) {
       logger.info(`Expired placeholder found. Removing.`);
-      rmSync(path3);
+      rmSync(path4);
       throw new Error("TIMEOUT");
     }
-    return waitForDocs(json, path3, logger, retryDuration);
+    return waitForDocs(json, path4, logger, retryDuration);
   } catch (e) {
     if (e.message !== "TIMEOUT") {
-      logger.info(`Docs JSON not found at ${path3}`);
+      logger.info(`Docs JSON not found at ${path4}`);
     }
     logger.debug("Generating placeholder");
-    generatePlaceholder(path3);
-    return generateDocs(specifier, path3, docgen, logger).catch((e2) => {
+    generatePlaceholder(path4);
+    return generateDocs(specifier, path4, docgen, logger).catch((e2) => {
       logger.error("Error generating documentation");
       logger.error(e2);
       handleError();
@@ -570,15 +602,15 @@ var docsHandler = async (options, logger) => {
     logger.success(`Showing docs for ${adaptorName} v${version}`);
   }
   logger.info("Generating/loading documentation...");
-  const path3 = await handler_default4(
+  const path4 = await handler_default4(
     {
       specifier: `${name}@${version}`,
       repoDir
     },
     createNullLogger()
   );
-  if (path3) {
-    const source = await readFile(path3, "utf8");
+  if (path4) {
+    const source = await readFile(path4, "utf8");
     const data = JSON.parse(source);
     let desc;
     if (operation) {
@@ -599,19 +631,91 @@ var docsHandler = async (options, logger) => {
 };
 var handler_default5 = docsHandler;
 
+// src/util/use-adaptors-repo.ts
+import { readFile as readFile2 } from "node:fs/promises";
+import path3 from "node:path";
+import assert from "node:assert";
+import { getNameAndVersion as getNameAndVersion4 } from "@openfn/runtime";
+var validateMonoRepo = async (repoPath, log) => {
+  try {
+    const raw = await readFile2(`${repoPath}/package.json`, "utf8");
+    const pkg = JSON.parse(raw);
+    assert(pkg.name === "adaptors");
+  } catch (e) {
+    log.error(`ERROR: Adaptors Monorepo not found at ${repoPath}`);
+    process.exit(9);
+  }
+};
+var updatePath = (adaptor, repoPath, log) => {
+  if (adaptor.match("=")) {
+    return adaptor;
+  }
+  const { name, version } = getNameAndVersion4(adaptor);
+  if (version) {
+    log.warn(
+      `Warning: Ignoring version specifier on ${adaptor} as loading from the adaptors monorepo`
+    );
+  }
+  const shortName = name.replace("@openfn/language-", "");
+  const abspath = path3.resolve(repoPath, "packages", shortName);
+  return `${name}=${abspath}`;
+};
+var useAdaptorsRepo = async (adaptors, repoPath, log) => {
+  await validateMonoRepo(repoPath, log);
+  log.success(`Loading adaptors from monorepo at ${repoPath}`);
+  const updatedAdaptors = adaptors.map((a) => {
+    const p = updatePath(a, repoPath, log);
+    log.info(`Mapped adaptor ${a} to monorepo: ${p.split("=")[1]}`);
+    return p;
+  });
+  return updatedAdaptors;
+};
+var use_adaptors_repo_default = useAdaptorsRepo;
+
 // src/util/print-versions.ts
 import { mainSymbols } from "figures";
+import { getNameAndVersion as getNameAndVersion5 } from "@openfn/runtime";
 var { triangleRightSmall: t } = mainSymbols;
-var printVersions = async (logger) => {
+var printVersions = async (logger, options = {}) => {
+  const prefix = (str) => `         ${t} ${str.padEnd(options.adaptors ? 16 : 8, " ")}`;
   const pkg = await import("../../package.json", { assert: { type: "json" } });
   const { version, dependencies } = pkg.default;
   const compilerVersion = dependencies["@openfn/compiler"];
   const runtimeVersion = dependencies["@openfn/runtime"];
-  logger.info(`Versions:
-        ${t} Node.js   ${process.version.substring(1)}
-        ${t} cli       ${version}
-        ${t} runtime   ${runtimeVersion}
-        ${t} compiler  ${compilerVersion}`);
+  const { adaptors } = options;
+  let adaptorName, adaptorVersion;
+  if (adaptors && adaptors.length === 1) {
+    const [a] = adaptors;
+    const { name, version: version2 } = getNameAndVersion5(a);
+    adaptorName = name.replace(/^@openfn\/language-/, "");
+    adaptorVersion = version2 || "latest";
+  }
+  let output;
+  if (options.logJson) {
+    output = {
+      versions: {
+        "node.js": process.version.substring(1),
+        cli: version,
+        runtime: runtimeVersion,
+        compiler: compilerVersion
+      }
+    };
+    if (adaptorName) {
+      output.versions.adaptor = {
+        name: adaptorName,
+        version: adaptorVersion
+      };
+    }
+  } else {
+    const adaptorVersionString = adaptorName ? `
+${prefix("adaptor " + adaptorName)}${adaptorVersion}` : "";
+    output = `Versions:
+${prefix("node.js")}${process.version.substring(1)}
+${prefix("cli")}${version}
+${prefix("runtime")}${runtimeVersion}
+${prefix("compiler")}${compilerVersion}${adaptorVersionString}`;
+  }
+  logger.info(output);
 };
 var print_versions_default = printVersions;
 
@@ -626,15 +730,28 @@ var handlers = {
   ["repo-install"]: install,
   ["repo-pwd"]: pwd,
   ["repo-list"]: list,
-  version: async (_opts, logger) => print_versions_default(logger)
+  version: async (opts, logger) => print_versions_default(logger, opts)
 };
 var parse = async (basePath, options, log) => {
   const opts = ensureOpts(basePath, options);
   const logger = log || logger_default(CLI, opts);
   if (opts.command === "execute" || opts.command === "test") {
-    await print_versions_default(logger);
-  }
-  if (opts.adaptors && opts.expand) {
+    await print_versions_default(logger, opts);
+  }
+  if (opts.monorepoPath) {
+    if (opts.monorepoPath === "ERR") {
+      logger.error(
+        "ERROR: --use-adaptors-monorepo was passed, but OPENFN_ADAPTORS_REPO env var is undefined"
+      );
+      logger.error("Set OPENFN_ADAPTORS_REPO to a path pointing to the repo");
+      process.exit(9);
+    }
+    opts.adaptors = await use_adaptors_repo_default(
+      opts.adaptors,
+      opts.monorepoPath,
+      logger
+    );
+  } else if (opts.adaptors && opts.expand) {
     opts.adaptors = expand_adaptors_default(opts.adaptors, logger);
   }
   if (/^(test|version)$/.test(opts.command) && !opts.repoDir) {
@@ -650,10 +767,20 @@ var parse = async (basePath, options, log) => {
     assertPath(basePath);
   }
   if (!handler) {
-    logger.error(`Unrecognise command: ${options.command}`);
+    logger.error(`Unrecognised command: ${options.command}`);
     process.exit(1);
   }
-  return handler(opts, logger);
+  try {
+    const result = await handler(opts, logger);
+    return result;
+  } catch (e) {
+    if (!process.exitCode) {
+      process.exitCode = e.exitCode || 1;
+    }
+    logger.break();
+    logger.error("Command failed!");
+    logger.error(e);
+  }
 };
 var commands_default = parse;
 var assertPath = (basePath) => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/cli",
-  "version": "0.0.26",
+  "version": "0.0.28",
   "description": "CLI devtools for the openfn toolchain.",
   "engines": {
     "node": ">=18",
@@ -36,10 +36,10 @@
     "typescript": "^4.7.4"
   },
   "dependencies": {
-    "@openfn/compiler": "0.0.23",
+    "@openfn/compiler": "0.0.24",
     "@openfn/describe-package": "0.0.14",
-    "@openfn/logger": "0.0.8",
-    "@openfn/runtime": "0.0.16",
+    "@openfn/logger": "0.0.9",
+    "@openfn/runtime": "0.0.18",
     "fast-safe-stringify": "^2.1.1",
     "figures": "^5.0.0",
     "rimraf": "^3.0.2",