@openfn/cli 0.0.27 → 0.0.28

package/README.md

@@ -50,7 +50,9 @@ If output.json and state.json are not passed, the CLI will look for them next to
 
 The CLI can auto-install language adaptors to its own privately maintained repo. Run `openfn repo list` to see where the repo is, and what's in it. Set the `OPENFN_REPO_DIR` env var to specify the repo folder. When autoinstalling, the CLI will check to see if a matching version is found in the repo.
 
-You can specify adaptors with a shorthand (`http`) or use the full package name (`@openfn/language-http`). You can add a specific version like `http@2.0.0`. You can pass a path to a locally installed adaptor like `http=/repo/openfn/adaptors/my-http-build`. Set the OPENFN_ADAPTORS_REPO env var to load adaptors straight out of the monorepo (pass the `--no-adaptors-repo` flag to disable this for a single run).
+You can specify adaptors with a shorthand (`http`) or use the full package name (`@openfn/language-http`). You can add a specific version like `http@2.0.0`. You can pass a path to a locally installed adaptor like `http=/repo/openfn/adaptors/my-http-build`.
+
+If you have the adaptors monorepo set up on your machine, you can also run from that. Pass the `-m` flag to load from the monorepo. Set the monorepo location by setting the OPENFN_ADAPTORS_REPO env var to a valid path. This runs from the built package, so remember to build an adaptor before running!
 
 You can pass `--log info` to get more feedback about what's happening, or `--log debug` for more details than you could ever use.
 
@@ -91,6 +93,16 @@ If something unexpected happens during a command, your first step should be to r
 
 `debug` level logging is highly verbose and aims to tell you everything that's going on under-the hood. This is aimed mostly at CLI/runtime developers and can be very useful for debugging problems.
 
+## Structred/JSON logging
+
+By default all logs will be printed as human-readable strings.
+
+For a more structured output, you can emit logs as JSON objects with `level`, `name` and `message` properties:
+```
+{ level: 'info', name: 'CLI', message: ['Loaded adaptor'] }
+```
+
+Pass `--log-json` to the CLI to do this. You can also set the OPENFN_LOG_JSON env var (and use `--no-log-json` to disable).
 
 ## Compilation
 

package/dist/index.js

@@ -95,6 +95,10 @@ var executeCommand = {
     return applyExecuteOptions(yargs2).option("immutable", {
       boolean: true,
       description: "Treat state as immutable"
+    }).option("use-adaptors-monorepo", {
+      alias: "m",
+      boolean: true,
+      description: "Load adaptors from the monorepo. The OPENFN_ADAPTORS_REPO env var must be set to a valid path"
     }).option("autoinstall", {
       alias: "i",
       boolean: true,
@@ -116,7 +120,7 @@ var executeCommand = {
       description: "Skip compilation"
     }).option("no-strict-output", {
       boolean: true,
-      description: "Allow properties other than data to be returned in the output."
+      description: "Allow properties other than data to be returned in the output"
     }).example(
       "openfn foo/job.js",
       "Reads foo/job.js, looks for state and output in foo"
@@ -141,7 +145,7 @@ var applyExecuteOptions = (yargs2) => yargs2.positional("path", {
   description: "Print output to stdout (instead of a file)"
 }).option("adaptors", {
   alias: ["a", "adaptor"],
-  description: "A language adaptor to use for the job. Short-form names are allowed. Can include an explicit path to a local adaptor build.",
+  description: "A language adaptor to use for the job. Short-form names are allowed. Can include an explicit path to a local adaptor build",
   array: true
 }).option("no-expand", {
   description: "Don	 attempt to auto-expand adaptor shorthand names",
@@ -209,6 +213,9 @@ var cmd = yargs(hideBin(process.argv)).command(command_default).command(command_
   alias: ["l"],
   description: "Set the default log level to none, default, info or debug",
   array: true
+}).option("log-json", {
+  description: "Output all logs as JSON objects",
+  boolean: true
 }).example("openfn execute help", "Show documentation for the execute command").example(
   "openfn docs @openfn/language-common each",
   "Get more help on the common.each command"

package/dist/process/runner.js

@@ -12,8 +12,11 @@ var namespaces = {
   [JOB]: "JOB"
 };
 var createLogger = (name = "", options) => {
-  const logOptions = options.log || {};
+  const logOptions = options.log || { json: true };
   let level = logOptions[name] || logOptions.default || "default";
+  if (options.logJson) {
+    logOptions.json = true;
+  }
   return actualCreateLogger(namespaces[name] || name, {
     level,
     ...logOptions
@@ -78,23 +81,27 @@ function ensureOpts(basePath = ".", opts) {
     adaptors: opts.adaptors || [],
     autoinstall: opts.autoinstall,
     command: opts.command,
+    expand: opts.expand !== false,
     force: opts.force || false,
-    repoDir: opts.repoDir || process.env.OPENFN_REPO_DIR || DEFAULT_REPO_DIR,
+    immutable: opts.immutable || false,
+    logJson: typeof opts.logJson == "boolean" ? opts.logJson : Boolean(process.env.OPENFN_LOG_JSON),
     noCompile: Boolean(opts.noCompile),
-    expand: opts.expand !== false,
-    outputStdout: Boolean(opts.outputStdout),
     operation: opts.operation,
+    outputStdout: Boolean(opts.outputStdout),
     packages: opts.packages,
-    stateStdin: opts.stateStdin,
-    timeout: opts.timeout,
+    repoDir: opts.repoDir || process.env.OPENFN_REPO_DIR || DEFAULT_REPO_DIR,
+    skipAdaptorValidation: opts.skipAdaptorValidation ?? false,
     specifier: opts.specifier,
+    stateStdin: opts.stateStdin,
     strictOutput: opts.strictOutput ?? true,
-    skipAdaptorValidation: opts.skipAdaptorValidation ?? false,
-    immutable: opts.immutable || false
+    timeout: opts.timeout
   };
   const set = (key, value) => {
     newOpts[key] = opts.hasOwnProperty(key) ? opts[key] : value;
   };
+  if (opts.useAdaptorsMonorepo) {
+    newOpts.monorepoPath = process.env.OPENFN_ADAPTORS_REPO || "ERR";
+  }
   let baseDir = basePath;
   if (basePath.endsWith(".js")) {
     baseDir = path.dirname(basePath);
@@ -164,11 +171,11 @@ var execute_default = (code, state, opts) => {
 function parseAdaptors(opts) {
   const adaptors = {};
   opts.adaptors.reduce((obj, exp) => {
-    const [module, path3] = exp.split("=");
+    const [module, path4] = exp.split("=");
     const { name, version } = getNameAndVersion(module);
     const info = {};
-    if (path3) {
-      info.path = path3;
+    if (path4) {
+      info.path = path4;
     }
     if (version) {
       info.version = version;
@@ -210,10 +217,10 @@ var stripVersionSpecifier = (specifier) => {
   return specifier;
 };
 var resolveSpecifierPath = async (pattern, repoDir, log) => {
-  const [specifier, path3] = pattern.split("=");
-  if (path3) {
-    log.debug(`Resolved ${specifier} to path: ${path3}`);
-    return path3;
+  const [specifier, path4] = pattern.split("=");
+  if (path4) {
+    log.debug(`Resolved ${specifier} to path: ${path4}`);
+    return path4;
   }
   const repoPath = await getModulePath(specifier, repoDir, log);
   if (repoPath) {
@@ -230,15 +237,15 @@ var loadTransformOptions = async (opts, log) => {
     const [pattern] = opts.adaptors;
     const [specifier] = pattern.split("=");
     log.debug(`Attempting to preload typedefs for ${specifier}`);
-    const path3 = await resolveSpecifierPath(pattern, opts.repoDir, log);
-    if (path3) {
+    const path4 = await resolveSpecifierPath(pattern, opts.repoDir, log);
+    if (path4) {
       try {
-        exports = await preloadAdaptorExports(path3);
+        exports = await preloadAdaptorExports(path4);
         if (exports) {
           log.info(`Loaded typedefs for ${specifier}`);
         }
       } catch (e) {
-        log.error(`Failed to load adaptor typedefs from path ${path3}`);
+        log.error(`Failed to load adaptor typedefs from path ${path4}`);
         log.error(e);
       }
     }
@@ -390,7 +397,7 @@ var validate_adaptors_default = validateAdaptors;
 
 // src/execute/handler.ts
 var getAutoinstallTargets = (options) => {
-  if (options.autoinstall && options.adaptors) {
+  if (options.adaptors) {
     return options.adaptors?.filter((a) => !/=/.test(a));
   }
   return [];
@@ -398,11 +405,17 @@ var getAutoinstallTargets = (options) => {
 var executeHandler = async (options, logger) => {
   const start = new Date().getTime();
   await validate_adaptors_default(options, logger);
-  const autoInstallTargets = getAutoinstallTargets(options);
-  if (autoInstallTargets.length) {
-    const { repoDir } = options;
-    logger.info("Auto-installing language adaptors");
-    await install({ packages: autoInstallTargets, repoDir }, logger);
+  const { repoDir, monorepoPath, autoinstall } = options;
+  if (autoinstall) {
+    if (monorepoPath) {
+      logger.warn("Skipping auto-install as monorepo is being used");
+    } else {
+      const autoInstallTargets = getAutoinstallTargets(options);
+      if (autoInstallTargets.length) {
+        logger.info("Auto-installing language adaptors");
+        await install({ packages: autoInstallTargets, repoDir }, logger);
+      }
+    }
   }
   const state = await load_state_default(options, logger);
   const code = await compile_default(options, logger);
@@ -466,20 +479,20 @@ var RETRY_COUNT = 20;
 var TIMEOUT_MS = 1e3 * 60;
 var actualDocGen = (specifier) => describePackage(specifier, {});
 var ensurePath = (filePath) => mkdirSync(path2.dirname(filePath), { recursive: true });
-var generatePlaceholder = (path3) => {
-  writeFileSync(path3, `{ "loading": true, "timestamp": ${Date.now()}}`);
+var generatePlaceholder = (path4) => {
+  writeFileSync(path4, `{ "loading": true, "timestamp": ${Date.now()}}`);
 };
 var finish = (logger, resultPath) => {
   logger.success("Done! Docs can be found at:\n");
   logger.print(`  ${path2.resolve(resultPath)}`);
 };
-var generateDocs = async (specifier, path3, docgen, logger) => {
+var generateDocs = async (specifier, path4, docgen, logger) => {
   const result = await docgen(specifier);
-  await writeFile3(path3, JSON.stringify(result, null, 2));
-  finish(logger, path3);
-  return path3;
+  await writeFile3(path4, JSON.stringify(result, null, 2));
+  finish(logger, path4);
+  return path4;
 };
-var waitForDocs = async (docs, path3, logger, retryDuration = RETRY_DURATION) => {
+var waitForDocs = async (docs, path4, logger, retryDuration = RETRY_DURATION) => {
   try {
     if (docs.hasOwnProperty("loading")) {
       logger.info("Docs are being loaded by another process. Waiting.");
@@ -491,19 +504,19 @@ var waitForDocs = async (docs, path3, logger, retryDuration = RETRY_DURATION) =>
             clearInterval(i);
             reject(new Error("Timed out waiting for docs to load"));
           }
-          const updated = JSON.parse(readFileSync(path3, "utf8"));
+          const updated = JSON.parse(readFileSync(path4, "utf8"));
           if (!updated.hasOwnProperty("loading")) {
             logger.info("Docs found!");
             clearInterval(i);
-            resolve(path3);
+            resolve(path4);
           }
           count++;
         }, retryDuration);
       });
     } else {
-      logger.info(`Docs already written to cache at ${path3}`);
-      finish(logger, path3);
-      return path3;
+      logger.info(`Docs already written to cache at ${path4}`);
+      finish(logger, path4);
+      return path4;
     }
   } catch (e) {
     logger.error("Existing doc JSON corrupt. Aborting");
@@ -520,28 +533,28 @@ var docgenHandler = (options, logger, docgen = actualDocGen, retryDuration = RET
     process.exit(9);
   }
   logger.success(`Generating docs for ${specifier}`);
-  const path3 = `${repoDir}/docs/${specifier}.json`;
-  ensurePath(path3);
+  const path4 = `${repoDir}/docs/${specifier}.json`;
+  ensurePath(path4);
   const handleError = () => {
     logger.info("Removing placeholder");
-    rmSync(path3);
+    rmSync(path4);
   };
   try {
-    const existing = readFileSync(path3, "utf8");
+    const existing = readFileSync(path4, "utf8");
     const json = JSON.parse(existing);
     if (json && json.timeout && Date.now() - json.timeout >= TIMEOUT_MS) {
       logger.info(`Expired placeholder found. Removing.`);
-      rmSync(path3);
+      rmSync(path4);
       throw new Error("TIMEOUT");
     }
-    return waitForDocs(json, path3, logger, retryDuration);
+    return waitForDocs(json, path4, logger, retryDuration);
   } catch (e) {
     if (e.message !== "TIMEOUT") {
-      logger.info(`Docs JSON not found at ${path3}`);
+      logger.info(`Docs JSON not found at ${path4}`);
     }
     logger.debug("Generating placeholder");
-    generatePlaceholder(path3);
-    return generateDocs(specifier, path3, docgen, logger).catch((e2) => {
+    generatePlaceholder(path4);
+    return generateDocs(specifier, path4, docgen, logger).catch((e2) => {
       logger.error("Error generating documentation");
       logger.error(e2);
       handleError();
@@ -589,15 +602,15 @@ var docsHandler = async (options, logger) => {
     logger.success(`Showing docs for ${adaptorName} v${version}`);
   }
   logger.info("Generating/loading documentation...");
-  const path3 = await handler_default4(
+  const path4 = await handler_default4(
     {
       specifier: `${name}@${version}`,
       repoDir
     },
     createNullLogger()
   );
-  if (path3) {
-    const source = await readFile(path3, "utf8");
+  if (path4) {
+    const source = await readFile(path4, "utf8");
     const data = JSON.parse(source);
     let desc;
     if (operation) {
@@ -618,9 +631,50 @@ var docsHandler = async (options, logger) => {
 };
 var handler_default5 = docsHandler;
 
+// src/util/use-adaptors-repo.ts
+import { readFile as readFile2 } from "node:fs/promises";
+import path3 from "node:path";
+import assert from "node:assert";
+import { getNameAndVersion as getNameAndVersion4 } from "@openfn/runtime";
+var validateMonoRepo = async (repoPath, log) => {
+  try {
+    const raw = await readFile2(`${repoPath}/package.json`, "utf8");
+    const pkg = JSON.parse(raw);
+    assert(pkg.name === "adaptors");
+  } catch (e) {
+    log.error(`ERROR: Adaptors Monorepo not found at ${repoPath}`);
+    process.exit(9);
+  }
+};
+var updatePath = (adaptor, repoPath, log) => {
+  if (adaptor.match("=")) {
+    return adaptor;
+  }
+  const { name, version } = getNameAndVersion4(adaptor);
+  if (version) {
+    log.warn(
+      `Warning: Ignoring version specifier on ${adaptor} as loading from the adaptors monorepo`
+    );
+  }
+  const shortName = name.replace("@openfn/language-", "");
+  const abspath = path3.resolve(repoPath, "packages", shortName);
+  return `${name}=${abspath}`;
+};
+var useAdaptorsRepo = async (adaptors, repoPath, log) => {
+  await validateMonoRepo(repoPath, log);
+  log.success(`Loading adaptors from monorepo at ${repoPath}`);
+  const updatedAdaptors = adaptors.map((a) => {
+    const p = updatePath(a, repoPath, log);
+    log.info(`Mapped adaptor ${a} to monorepo: ${p.split("=")[1]}`);
+    return p;
+  });
+  return updatedAdaptors;
+};
+var use_adaptors_repo_default = useAdaptorsRepo;
+
 // src/util/print-versions.ts
 import { mainSymbols } from "figures";
-import { getNameAndVersion as getNameAndVersion4 } from "@openfn/runtime";
+import { getNameAndVersion as getNameAndVersion5 } from "@openfn/runtime";
 var { triangleRightSmall: t } = mainSymbols;
 var printVersions = async (logger, options = {}) => {
   const prefix = (str) => `         ${t} ${str.padEnd(options.adaptors ? 16 : 8, " ")}`;
@@ -629,20 +683,39 @@ var printVersions = async (logger, options = {}) => {
   const compilerVersion = dependencies["@openfn/compiler"];
   const runtimeVersion = dependencies["@openfn/runtime"];
   const { adaptors } = options;
-  let adaptorVersionString = "";
+  let adaptorName, adaptorVersion;
   if (adaptors && adaptors.length === 1) {
     const [a] = adaptors;
-    const { name, version: version2 } = getNameAndVersion4(a);
-    adaptorVersionString = `
-${prefix(
-      "adaptor " + name.replace(/^@openfn\/language-/, "")
-    )}${version2 || "latest"}`;
-  }
-  logger.info(`Versions:
+    const { name, version: version2 } = getNameAndVersion5(a);
+    adaptorName = name.replace(/^@openfn\/language-/, "");
+    adaptorVersion = version2 || "latest";
+  }
+  let output;
+  if (options.logJson) {
+    output = {
+      versions: {
+        "node.js": process.version.substring(1),
+        cli: version,
+        runtime: runtimeVersion,
+        compiler: compilerVersion
+      }
+    };
+    if (adaptorName) {
+      output.versions.adaptor = {
+        name: adaptorName,
+        version: adaptorVersion
+      };
+    }
+  } else {
+    const adaptorVersionString = adaptorName ? `
+${prefix("adaptor " + adaptorName)}${adaptorVersion}` : "";
+    output = `Versions:
 ${prefix("node.js")}${process.version.substring(1)}
 ${prefix("cli")}${version}
 ${prefix("runtime")}${runtimeVersion}
-${prefix("compiler")}${compilerVersion}${adaptorVersionString}`);
+${prefix("compiler")}${compilerVersion}${adaptorVersionString}`;
+  }
+  logger.info(output);
 };
 var print_versions_default = printVersions;
 
@@ -657,7 +730,7 @@ var handlers = {
   ["repo-install"]: install,
   ["repo-pwd"]: pwd,
   ["repo-list"]: list,
-  version: async (_opts, logger) => print_versions_default(logger)
+  version: async (opts, logger) => print_versions_default(logger, opts)
 };
 var parse = async (basePath, options, log) => {
   const opts = ensureOpts(basePath, options);
@@ -665,7 +738,20 @@ var parse = async (basePath, options, log) => {
   if (opts.command === "execute" || opts.command === "test") {
     await print_versions_default(logger, opts);
   }
-  if (opts.adaptors && opts.expand) {
+  if (opts.monorepoPath) {
+    if (opts.monorepoPath === "ERR") {
+      logger.error(
+        "ERROR: --use-adaptors-monorepo was passed, but OPENFN_ADAPTORS_REPO env var is undefined"
+      );
+      logger.error("Set OPENFN_ADAPTORS_REPO to a path pointing to the repo");
+      process.exit(9);
+    }
+    opts.adaptors = await use_adaptors_repo_default(
+      opts.adaptors,
+      opts.monorepoPath,
+      logger
+    );
+  } else if (opts.adaptors && opts.expand) {
     opts.adaptors = expand_adaptors_default(opts.adaptors, logger);
   }
   if (/^(test|version)$/.test(opts.command) && !opts.repoDir) {
@@ -681,7 +767,7 @@ var parse = async (basePath, options, log) => {
     assertPath(basePath);
   }
   if (!handler) {
-    logger.error(`Unrecognise command: ${options.command}`);
+    logger.error(`Unrecognised command: ${options.command}`);
     process.exit(1);
   }
   try {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/cli",
-  "version": "0.0.27",
+  "version": "0.0.28",
   "description": "CLI devtools for the openfn toolchain.",
   "engines": {
     "node": ">=18",
@@ -36,15 +36,15 @@
     "typescript": "^4.7.4"
   },
   "dependencies": {
+    "@openfn/compiler": "0.0.24",
+    "@openfn/describe-package": "0.0.14",
+    "@openfn/logger": "0.0.9",
+    "@openfn/runtime": "0.0.18",
     "fast-safe-stringify": "^2.1.1",
     "figures": "^5.0.0",
     "rimraf": "^3.0.2",
     "treeify": "^1.1.0",
-    "yargs": "^17.5.1",
-    "@openfn/compiler": "0.0.23",
-    "@openfn/describe-package": "0.0.14",
-    "@openfn/logger": "0.0.8",
-    "@openfn/runtime": "0.0.17"
+    "yargs": "^17.5.1"
   },
   "files": [
     "dist",