@openfn/cli 0.0.17 → 0.0.20

package/dist/index.js

@@ -12,13 +12,15 @@ function spawn_default(basePath, opts2) {
   ];
   const dirname = path.dirname(url.fileURLToPath(import.meta.url));
   const child = fork(`${dirname}/process/runner.js`, [], { execArgv });
-  child.on("message", ({ done }) => {
+  child.on("message", ({ done, init }) => {
+    if (init) {
+      child.send({ init: true, basePath, opts: opts2 });
+    }
     if (done) {
       child.kill();
       process.exit(0);
     }
   });
-  child.send({ init: true, basePath, opts: opts2 });
 }
 
 // src/cli.ts
@@ -81,7 +83,7 @@ var list = {
 // src/execute/command.ts
 var executeCommand = {
   command: "execute [path]",
-  desc: "Run an openfn job",
+  desc: `Run an openfn job. Get more help by running openfn <command> help`,
   aliases: ["$0"],
   handler: (argv) => {
     argv.command = "execute";
@@ -103,6 +105,9 @@ var executeCommand = {
     }).option("no-compile", {
       boolean: true,
       description: "Skip compilation"
+    }).option("no-strict-output", {
+      boolean: true,
+      description: "Allow properties other than data to be returned in the output."
     }).example(
       "openfn foo/job.js",
       "Reads foo/job.js, looks for state and output in foo"
@@ -124,7 +129,7 @@ var applyExecuteOptions = (yargs2) => yargs2.positional("path", {
 }).option("output-stdout", {
   alias: "O",
   boolean: true,
-  description: "Print output to stdout (intead of a file)"
+  description: "Print output to stdout (instead of a file)"
 }).option("adaptors", {
   alias: ["a", "adaptor"],
   description: "A language adaptor to use for the job. Short-form names are allowed. Can include an explicit path to a local adaptor build.",
@@ -167,12 +172,38 @@ var command_default3 = {
   }).example("test", "run the test script").example("test -S 42", "run the test script with state 42")
 };
 
+// src/docgen/command.ts
+var docgenCommand = {
+  command: "docgen <specifier>",
+  desc: "Generate documentation into the repo. Specifier must include a version number.",
+  handler: (argv) => {
+    argv.command = "docgen";
+  },
+  builder: (yargs2) => {
+    return yargs2.example("docgen @openfn/language-common@1.7.5", "");
+  }
+};
+var command_default4 = docgenCommand;
+
+// src/docs/command.ts
+var command_default5 = {
+  command: "docs [adaptor] [operation]",
+  desc: "Print help for an adaptor function. You can use short-hand for adaptor names (ie, common instead of @openfn/language-common)",
+  handler: (argv) => {
+    argv.command = "docs";
+  },
+  builder: (yargs2) => yargs2.example("docs common fn", "Print help for the common fn operation")
+};
+
 // src/cli.ts
-var cmd = yargs(hideBin(process.argv)).command(command_default).command(command_default2).command(install).command(repo).command(command_default3).option("log", {
+var cmd = yargs(hideBin(process.argv)).command(command_default).command(command_default2).command(install).command(repo).command(command_default3).command(command_default5).command(command_default4).option("log", {
   alias: ["l"],
   description: "Set the default log level to none, default, info or debug",
   array: true
-}).alias("v", "version").help();
+}).example("openfn execute help", "Show documentation for the execute command").example(
+  "openfn docs @openfn/language-common each",
+  "Get more help on the common.each command"
+).alias("v", "version").help();
 
 // src/index.ts
 var opts = cmd.parse();

package/dist/process/runner.js

@@ -80,8 +80,11 @@ function ensureOpts(basePath = ".", opts) {
     noCompile: Boolean(opts.noCompile),
     expand: opts.expand !== false,
     outputStdout: Boolean(opts.outputStdout),
+    operation: opts.operation,
     packages: opts.packages,
     stateStdin: opts.stateStdin,
+    specifier: opts.specifier,
+    strictOutput: opts.strictOutput ?? true,
     immutable: opts.immutable || false
   };
   const set = (key, value) => {
@@ -105,9 +108,6 @@ function ensureOpts(basePath = ".", opts) {
   return newOpts;
 }
 
-// src/execute/handler.ts
-import { writeFile } from "node:fs/promises";
-
 // src/execute/load-state.ts
 import fs from "node:fs/promises";
 var load_state_default = async (opts, log) => {
@@ -158,11 +158,11 @@ var execute_default = (code, state, opts) => {
 function parseAdaptors(opts) {
   const adaptors = {};
   opts.adaptors.reduce((obj, exp) => {
-    const [module, path2] = exp.split("=");
+    const [module, path3] = exp.split("=");
     const { name, version } = getNameAndVersion(module);
     const info = {};
-    if (path2) {
-      info.path = path2;
+    if (path3) {
+      info.path = path3;
     }
     if (version) {
       info.version = version;
@@ -204,10 +204,10 @@ var stripVersionSpecifier = (specifier) => {
   return specifier;
 };
 var resolveSpecifierPath = async (pattern, repoDir, log) => {
-  const [specifier, path2] = pattern.split("=");
-  if (path2) {
-    log.debug(`Resolved ${specifier} to path: ${path2}`);
-    return path2;
+  const [specifier, path3] = pattern.split("=");
+  if (path3) {
+    log.debug(`Resolved ${specifier} to path: ${path3}`);
+    return path3;
   }
   const repoPath = await getModulePath(specifier, repoDir, log);
   if (repoPath) {
@@ -225,15 +225,15 @@ var loadTransformOptions = async (opts, log) => {
     const [pattern] = opts.adaptors;
     const [specifier] = pattern.split("=");
     log.debug(`Attempting to preload typedefs for ${specifier}`);
-    const path2 = await resolveSpecifierPath(pattern, opts.repoDir, log);
-    if (path2) {
+    const path3 = await resolveSpecifierPath(pattern, opts.repoDir, log);
+    if (path3) {
       try {
-        exports = await preloadAdaptorExports(path2);
+        exports = await preloadAdaptorExports(path3);
         if (exports) {
           log.info(`Loaded typedefs for ${specifier}`);
         }
       } catch (e) {
-        log.error(`Failed to load adaptor typedefs from path ${path2}`);
+        log.error(`Failed to load adaptor typedefs from path ${path3}`);
         log.error(e);
       }
     }
@@ -251,6 +251,38 @@ var loadTransformOptions = async (opts, log) => {
   return options;
 };
 
+// src/execute/serialize-output.ts
+import { writeFile } from "node:fs/promises";
+import stringify from "fast-safe-stringify";
+var serializeOutput = async (options, result, logger) => {
+  let output = result;
+  if (output && (output.configuration || output.data)) {
+    const { data, configuration, ...rest } = result;
+    if (options.strictOutput !== false) {
+      output = { data };
+    } else {
+      output = {
+        data,
+        ...rest
+      };
+    }
+  }
+  if (output === void 0) {
+    output = "";
+  } else {
+    output = stringify(output, void 0, 2);
+  }
+  if (options.outputStdout) {
+    logger.success(`Result: `);
+    logger.success(output);
+  } else if (options.outputPath) {
+    logger.success(`Writing output to ${options.outputPath}`);
+    await writeFile(options.outputPath, output);
+  }
+  return output;
+};
+var serialize_output_default = serializeOutput;
+
 // src/repo/handler.ts
 import { exec } from "node:child_process";
 import treeify from "treeify";
@@ -269,18 +301,18 @@ var expand_adaptors_default = (names, log = nullLogger) => names == null ? void
 
 // src/repo/handler.ts
 var install = async (opts, log = defaultLogger) => {
-  log.timer("install");
   let { packages, adaptor, repoDir } = opts;
-  log.success("Installing packages...");
   if (packages) {
+    log.timer("install");
+    log.success("Installing packages...");
     log.debug("repoDir is set to:", repoDir);
     if (adaptor) {
       packages = expand_adaptors_default(packages, log);
     }
     await rtInstall(packages, repoDir, log);
+    const duration = log.timer("install");
+    log.success(`Installation complete in ${duration}`);
   }
-  const duration = log.timer("install");
-  log.success(`Installation complete in ${duration}`);
 };
 var clean = async (options, logger) => {
   if (options.repoDir) {
@@ -335,23 +367,25 @@ var list = async (options, logger) => {
 };
 
 // src/execute/handler.ts
+var getAutoinstallTargets = (options) => {
+  var _a;
+  if (options.autoinstall && options.adaptors) {
+    return (_a = options.adaptors) == null ? void 0 : _a.filter((a) => !/=/.test(a));
+  }
+  return [];
+};
 var executeHandler = async (options, logger) => {
   const start = new Date().getTime();
-  if (options.autoinstall) {
+  const autoInstallTargets = getAutoinstallTargets(options);
+  if (autoInstallTargets.length) {
     const { repoDir } = options;
     logger.info("Auto-installing language adaptors");
-    await install({ packages: options.adaptors, repoDir }, logger);
+    await install({ packages: autoInstallTargets, repoDir }, logger);
   }
   const state = await load_state_default(options, logger);
   const code = await compile_default(options, logger);
   const result = await execute_default(code, state, options);
-  if (options.outputStdout) {
-    logger.success(`Result: `);
-    logger.success(result);
-  } else {
-    logger.success(`Writing output to ${options.outputPath}`);
-    await writeFile(options.outputPath, JSON.stringify(result, null, 4));
-  }
+  await serialize_output_default(options, result, logger);
   const duration = printDuration(new Date().getTime() - start);
   logger.success(`Done in ${duration}! \u2728`);
 };
@@ -392,7 +426,165 @@ var testHandler = async (options, logger) => {
 };
 var handler_default3 = testHandler;
 
+// src/docgen/handler.ts
+import { writeFile as writeFile3 } from "node:fs/promises";
+import { readFileSync, writeFileSync, mkdirSync, rmSync } from "node:fs";
+import path2 from "node:path";
+import { describePackage } from "@openfn/describe-package";
+import { getNameAndVersion as getNameAndVersion2 } from "@openfn/runtime";
+var RETRY_DURATION = 500;
+var RETRY_COUNT = 20;
+var TIMEOUT_MS = 1e3 * 60;
+var actualDocGen = (specifier) => describePackage(specifier, {});
+var ensurePath = (filePath) => mkdirSync(path2.dirname(filePath), { recursive: true });
+var generatePlaceholder = (path3) => {
+  writeFileSync(path3, `{ "loading": true, "timestamp": ${Date.now()}}`);
+};
+var finish = (logger, resultPath) => {
+  logger.success("Done! Docs can be found at:\n");
+  logger.print(`  ${path2.resolve(resultPath)}`);
+};
+var generateDocs = async (specifier, path3, docgen, logger) => {
+  const result = await docgen(specifier);
+  await writeFile3(path3, JSON.stringify(result, null, 2));
+  finish(logger, path3);
+  return path3;
+};
+var waitForDocs = async (docs, path3, logger, retryDuration = RETRY_DURATION) => {
+  try {
+    if (docs.hasOwnProperty("loading")) {
+      logger.info("Docs are being loaded by another process. Waiting.");
+      return new Promise((resolve, reject) => {
+        let count = 0;
+        let i = setInterval(() => {
+          logger.info("Waiting..");
+          if (count > RETRY_COUNT) {
+            clearInterval(i);
+            reject(new Error("Timed out waiting for docs to load"));
+          }
+          const updated = JSON.parse(readFileSync(path3, "utf8"));
+          if (!updated.hasOwnProperty("loading")) {
+            logger.info("Docs found!");
+            clearInterval(i);
+            resolve(path3);
+          }
+          count++;
+        }, retryDuration);
+      });
+    } else {
+      logger.info(`Docs already written to cache at ${path3}`);
+      finish(logger, path3);
+      return path3;
+    }
+  } catch (e) {
+    logger.error("Existing doc JSON corrupt. Aborting");
+    throw e;
+  }
+};
+var docgenHandler = (options, logger, docgen = actualDocGen, retryDuration = RETRY_DURATION) => {
+  const { specifier, repoDir } = options;
+  const { version } = getNameAndVersion2(specifier);
+  if (!version) {
+    logger.error("Error: No version number detected");
+    logger.error("eg, @openfn/language-common@1.7.5");
+    logger.error("Aborting");
+    process.exit(9);
+  }
+  logger.success(`Generating docs for ${specifier}`);
+  const path3 = `${repoDir}/docs/${specifier}.json`;
+  ensurePath(path3);
+  const handleError = () => {
+    logger.info("Removing placeholder");
+    rmSync(path3);
+  };
+  try {
+    const existing = readFileSync(path3, "utf8");
+    const json = JSON.parse(existing);
+    if (json && json.timeout && Date.now() - json.timeout >= TIMEOUT_MS) {
+      logger.info(`Expired placeholder found. Removing.`);
+      rmSync(path3);
+      throw new Error("TIMEOUT");
+    }
+    return waitForDocs(json, path3, logger, retryDuration);
+  } catch (e) {
+    if (e.message !== "TIMEOUT") {
+      logger.info(`Docs JSON not found at ${path3}`);
+    }
+    logger.debug("Generating placeholder");
+    generatePlaceholder(path3);
+    return generateDocs(specifier, path3, docgen, logger).catch((e2) => {
+      logger.error("Error generating documentation");
+      logger.error(e2);
+      handleError();
+    });
+  }
+};
+var handler_default4 = docgenHandler;
+
+// src/docs/handler.ts
+import { readFile } from "node:fs/promises";
+import { getNameAndVersion as getNameAndVersion3, getLatestVersion } from "@openfn/runtime";
+var describe = (adaptorName, fn) => `## ${fn.name}(${fn.parameters.map(({ name }) => name).join(",")})
+
+${fn.description}
+
+### Usage Examples
+
+${fn.examples.length ? fn.examples.map((eg) => eg).join("\n\n") : "None"}
+
+### API Reference
+
+https://docs.openfn.org/adaptors/packages/${adaptorName.replace(
+  "@openfn/language-",
+  ""
+)}-docs#${fn.name}
+`;
+var docsHandler = async (options, logger) => {
+  const { adaptor, operation, repoDir } = options;
+  const [adaptorName] = expand_adaptors_default([adaptor], logger);
+  let { name, version } = getNameAndVersion3(adaptorName);
+  if (!version) {
+    logger.info("No version number provided, looking for latest...");
+    version = await getLatestVersion(version);
+    logger.info("Found ", version);
+    logger.success(`Showing docs for ${adaptorName} v${version}`);
+  }
+  logger.info("Generating/loading documentation...");
+  const path3 = await handler_default4(
+    {
+      specifier: `${name}@${version}`,
+      repoDir
+    },
+    createNullLogger()
+  );
+  if (path3) {
+    const source = await readFile(path3, "utf8");
+    const data = JSON.parse(source);
+    const fn = data.functions.find(({ name: name2 }) => name2 === operation);
+    logger.debug("Operation schema:", fn);
+    logger.success(`Documentation for ${name}.${operation} v${version}:
+`);
+    const desc = describe(name, fn);
+    logger.print(desc);
+    logger.success("Done!");
+  } else {
+    logger.error("Not found");
+  }
+};
+var handler_default5 = docsHandler;
+
 // src/commands.ts
+var handlers = {
+  execute: handler_default,
+  compile: handler_default2,
+  test: handler_default3,
+  docgen: handler_default4,
+  docs: handler_default5,
+  ["repo-clean"]: clean,
+  ["repo-install"]: install,
+  ["repo-pwd"]: pwd,
+  ["repo-list"]: list
+};
 var parse = async (basePath, options, log) => {
   const opts = ensureOpts(basePath, options);
   const logger = log || logger_default(CLI, opts);
@@ -407,31 +599,13 @@ var parse = async (basePath, options, log) => {
       "You should set OPENFN_REPO_DIR or pass --repoDir=some/path in to the CLI"
     );
   }
-  let handler = () => null;
-  switch (options.command) {
-    case "repo-install":
-      handler = install;
-      break;
-    case "repo-clean":
-      handler = clean;
-      break;
-    case "repo-pwd":
-      handler = pwd;
-      break;
-    case "repo-list":
-      handler = list;
-      break;
-    case "compile":
-      assertPath(basePath);
-      handler = handler_default2;
-      break;
-    case "test":
-      handler = handler_default3;
-      break;
-    case "execute":
-    default:
-      assertPath(basePath);
-      handler = handler_default;
+  const handler = options.command ? handlers[options.command] : handler_default;
+  if (!opts.command || /^(compile|execute)$/.test(opts.command)) {
+    assertPath(basePath);
+  }
+  if (!handler) {
+    logger.error(`Unrecognise command: ${options.command}`);
+    process.exit(1);
   }
   return handler(opts, logger);
 };
@@ -455,3 +629,6 @@ process.on("message", ({ init, basePath, opts }) => {
     });
   }
 });
+process.send({
+  init: true
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/cli",
-  "version": "0.0.17",
+  "version": "0.0.20",
   "description": "CLI devtools for the openfn toolchain.",
   "engines": {
     "node": ">=16",
@@ -30,15 +30,17 @@
     "@types/yargs": "^17.0.12",
     "ava": "5.1.0",
     "mock-fs": "^5.1.4",
-    "ts-node": "^10.8.1",
+    "ts-node": "^10.9.1",
     "tslib": "^2.4.0",
     "tsup": "^6.2.3",
     "typescript": "^4.7.4"
   },
   "dependencies": {
-    "@openfn/compiler": "^0.0.16",
-    "@openfn/logger": "^0.0.6",
-    "@openfn/runtime": "^0.0.11",
+    "@openfn/compiler": "^0.0.19",
+    "@openfn/describe-package": "^0.0.11",
+    "@openfn/logger": "^0.0.8",
+    "@openfn/runtime": "^0.0.13",
+    "fast-safe-stringify": "^2.1.1",
     "rimraf": "^3.0.2",
     "treeify": "^1.1.0",
     "yargs": "^17.5.1"