@openfn/cli 0.0.31 → 0.0.34

package/README.md

@@ -36,6 +36,14 @@ Get help:
 openfn help
 ```
 
+## Migrating from devtools
+
+If you're coming to the CLI from the old openfn devtools, here are a couple of key points to be aware of:
+
+* The CLI has a shorter, sleeker syntax, so your command should be much shorter
+* The CLI will automatically install adaptors for you (with full version control)
+* By default, the CLI will only write state.data to output. This is to encourage better state management. Pass `--no-strict-output` to save the entire state object.
+
 ## Basic Usage
 
 You're probably here to run jobs (expressions), which the CLI makes easy:
@@ -46,13 +54,13 @@ openfn path/to/job.js -ia adaptor-name
 
 You MUST specify which adaptor to use. Pass the `-i` flag to auto-install that adaptor (it's safe to do this redundantly).
 
-If output.json is not passed, the CLI will create an `output.json` next to the job file. You can pass a path to state by adding `-s path/to/state.json`, and output by passing `-o path/to/output.json`. You can use `-S` and `-O` to pass state through stdin and return the output through stdout.
+When the job is finished, the CLI will write the `data` property of your state to disk. By default the CLI will create an `output.json` next to the job file. You can pass a path to output by passing `-o path/to/output.json` and state by adding `-s path/to/state.json`. You can use `-S` and `-O` to pass state through stdin and return the output through stdout. To write the entire state object (not just `data`), pass `--no-strict-output`.
 
-The CLI can auto-install language adaptors to its own privately maintained repo. Run `openfn repo list` to see where the repo is, and what's in it. Set the `OPENFN_REPO_DIR` env var to specify the repo folder. When autoinstalling, the CLI will check to see if a matching version is found in the repo.
+The CLI can auto-install language adaptors to its own privately maintained repo, just include the `-i` flag in the command and your adaptors will be forever fully managed. Run `openfn repo list` to see where the repo is, and what's in it. Set the `OPENFN_REPO_DIR` env var to specify the repo folder. When autoinstalling, the CLI will check to see if a matching version is found in the repo.
 
 You can specify adaptors with a shorthand (`http`) or use the full package name (`@openfn/language-http`). You can add a specific version like `http@2.0.0`. You can pass a path to a locally installed adaptor like `http=/repo/openfn/adaptors/my-http-build`.
 
-If you have the adaptors monorepo set up on your machine, you can also run from that. Pass the `-m` flag to load from the monorepo. Set the monorepo location by setting the OPENFN_ADAPTORS_REPO env var to a valid path. This runs from the built package, so remember to build an adaptor before running!
+If you have the adaptors monorepo set up on your machine, you can also run adaptors straight from source. Pass the `-m <path>` flag to load from the monorepo. You can also set the monorepo location by setting the `OPENFN_ADAPTORS_REPO` env var to a valid path. After that just include `-m` to load from the monorepo. Remember that adaptors will be loaded from the BUILT package in `dist`, so remember to build an adaptor before running!
 
 You can pass `--log info` to get more feedback about what's happening, or `--log debug` for more details than you could ever use.
 
@@ -119,9 +127,11 @@ The CLI uses openfn's own runtime to execute jobs in a safe environment.
 
 All jobs which work against `@openfn/core` will work in the new CLI and runtime environment (note: although this is a work in progress and we are actively looking for help to test this!).
 
+If you want to see how the compiler is changing your job, run `openfn compile path/to/job -a <adaptor>` to return the compiled code to stdout. Add `-o path/to/output.js` to save the result to disk.
+
 ## New Runtime notes
 
-The new openfunction runtime basically does one thing: load a Javascript Module, find the default export, and execute the functions it holds.
+The new OpenFn runtime will create a secure sandboxed environemtn which loads a Javascript Module, finds the default export, and execute the functions held within it.
 
 So long as your job has an array of functions as its default export, it will run in the new runtime.
 

package/dist/{chunk-Q2H7WYJB.js → chunk-XYZNU5CH.js}

@@ -1,7 +1,8 @@
 // src/util/expand-adaptors.ts
 var expand_adaptors_default = (names) => names?.map((name) => {
   if (typeof name === "string") {
-    if (name.startsWith("@openfn/language-")) {
+    const [left] = name.split("=");
+    if (left.match("/") || left.endsWith(".js")) {
       return name;
     }
     return `@openfn/language-${name}`;

package/dist/index.js

@@ -2,7 +2,7 @@
 import {
   DEFAULT_REPO_DIR,
   expand_adaptors_default
-} from "./chunk-Q2H7WYJB.js";
+} from "./chunk-XYZNU5CH.js";
 
 // src/process/spawn.ts
 import path from "node:path";
@@ -163,6 +163,15 @@ var expandAdaptors = {
     setDefaultValue(opts2, "expandAdaptors", true);
   }
 };
+var force = {
+  name: "force",
+  yargs: {
+    alias: ["f"],
+    boolean: true,
+    description: "Force metadata to be regenerated",
+    default: false
+  }
+};
 var immutable = {
   name: "immutable",
   yargs: {
@@ -171,6 +180,17 @@ var immutable = {
     default: false
   }
 };
+var ignoreImports = {
+  name: "ignore-imports",
+  yargs: {
+    description: "Don't auto-import references in compiled code. Can take a list of names to ignore."
+  },
+  ensure: (opts2) => {
+    if (typeof opts2.ignoreImports === "string") {
+      opts2.ignoreImports = opts2.ignoreImports.split(",").map((s) => s.trim());
+    }
+  }
+};
 var getBaseDir = (opts2) => {
   const basePath = opts2.path ?? ".";
   if (basePath.endsWith(".js")) {
@@ -304,6 +324,7 @@ var options = [
   autoinstall,
   compile,
   immutable,
+  ignoreImports,
   jobPath,
   logJson,
   outputPath,
@@ -318,7 +339,13 @@ var options = [
 ];
 var executeCommand = {
   command: "execute [path]",
-  desc: `Run an openfn job. Get more help by running openfn <command> help`,
+  desc: `Run an openfn job. Get more help by running openfn <command> help.
+  
+Execute will run a job/expression and write the output state to disk (to ./state.json unless otherwise specified)
+  
+By default only state.data will be written to the output. Include --no-strict-output to write the entire state object.
+  
+Remember to include the adaptor name with -a. Auto install adaptors with the -i flag.`,
   aliases: ["$0"],
   handler: ensure("execute", options),
   builder: (yargs2) => build(options, yargs2).positional("path", {
@@ -326,13 +353,16 @@ var executeCommand = {
     demandOption: true
   }).example(
     "openfn foo/job.js",
-    "Reads foo/job.js, looks for state and output in foo"
+    "Execute foo/job.js with no adaptor and write the final state to foo/job.json"
+  ).example(
+    "openfn job.js -ia common",
+    "Execute job.js using @openfn/language-commom , with autoinstall enabled)"
   ).example(
-    "openfn job.js -a common",
-    "Run job.js using @openfn/language-common"
+    "openfn job.js -a common --log info",
+    "Execute job.js with common adaptor and info-level logging"
   ).example(
-    "openfn install -a common",
-    "Install the latest version of language-common to the repo"
+    "openfn compile job.js -a http",
+    "Compile job.js with the http adaptor and print the code to stdout"
   )
 };
 var command_default = executeCommand;
@@ -341,12 +371,14 @@ var command_default = executeCommand;
 var options2 = [
   expandAdaptors,
   adaptors,
+  ignoreImports,
   jobPath,
   logJson,
   override(outputStdout, {
     default: true
   }),
   outputPath,
+  repoDir,
   useAdaptorsMonorepo
 ];
 var compileCommand = {
@@ -402,8 +434,30 @@ var command_default5 = {
   builder: (yargs2) => yargs2.example("docs common fn", "Print help for the common fn operation")
 };
 
+// src/metadata/command.ts
+var options3 = [
+  expandAdaptors,
+  adaptors,
+  force,
+  logJson,
+  repoDir,
+  statePath,
+  stateStdin,
+  useAdaptorsMonorepo
+];
+var command_default6 = {
+  command: "metadata",
+  desc: "Generate metadata for an adaptor config",
+  handler: ensure("metadata", options3),
+  builder: (yargs2) => build(options3, yargs2).example(
+    "metadata -a salesforce -s tmp/state.json",
+    "Generate salesforce metadata from config in state.json"
+  )
+};
+
 // src/cli.ts
-var cmd = yargs(hideBin(process.argv)).command(command_default).command(command_default2).command(install).command(repo).command(command_default3).command(command_default5).command(command_default4).option("log", {
+var y = yargs(hideBin(process.argv));
+var cmd = y.command(command_default).command(command_default2).command(install).command(repo).command(command_default3).command(command_default5).command(command_default6).command(command_default4).option("log", {
   alias: ["l"],
   description: "Set the default log level to none, default, info or debug",
   array: true
@@ -418,7 +472,7 @@ var cmd = yargs(hideBin(process.argv)).command(command_default).command(command_
   handler: (argv) => {
     argv.command = "version";
   }
-}).help();
+}).wrap(y.terminalWidth()).help();
 
 // src/index.ts
 var opts = cmd.parse();

package/dist/process/runner.js

@@ -10,7 +10,7 @@ import {
   expand_adaptors_default,
   logger_default,
   printDuration
-} from "../chunk-Q2H7WYJB.js";
+} from "../chunk-XYZNU5CH.js";
 
 // src/execute/handler.ts
 import { readFile } from "node:fs/promises";
@@ -71,11 +71,11 @@ var execute_default = (code, state, opts) => {
 function parseAdaptors(opts) {
   const adaptors = {};
   opts.adaptors.reduce((obj, exp) => {
-    const [module, path4] = exp.split("=");
+    const [module, path5] = exp.split("=");
     const { name, version } = getNameAndVersion(module);
     const info = {};
-    if (path4) {
-      info.path = path4;
+    if (path5) {
+      info.path = path5;
     }
     if (version) {
       info.version = version;
@@ -92,7 +92,6 @@ import { getModulePath } from "@openfn/runtime";
 var compile_default = async (opts, log) => {
   log.debug("Loading job...");
   const compilerOptions = await loadTransformOptions(opts, log);
-  compilerOptions.logger = logger_default(COMPILER, opts);
   const job = compile(opts.jobSource || opts.jobPath, compilerOptions);
   if (opts.jobPath) {
     log.success(`Compiled job from ${opts.jobPath}`);
@@ -109,10 +108,10 @@ var stripVersionSpecifier = (specifier) => {
   return specifier;
 };
 var resolveSpecifierPath = async (pattern, repoDir, log) => {
-  const [specifier, path4] = pattern.split("=");
-  if (path4) {
-    log.debug(`Resolved ${specifier} to path: ${path4}`);
-    return path4;
+  const [specifier, path5] = pattern.split("=");
+  if (path5) {
+    log.debug(`Resolved ${specifier} to path: ${path5}`);
+    return path5;
   }
   const repoPath = await getModulePath(specifier, repoDir, log);
   if (repoPath) {
@@ -122,22 +121,19 @@ var resolveSpecifierPath = async (pattern, repoDir, log) => {
 };
 var loadTransformOptions = async (opts, log) => {
   const options = {
-    logger: log
+    logger: log || logger_default(COMPILER, opts)
   };
-  if (opts.adaptors?.length) {
+  if (opts.adaptors?.length && opts.ignoreImports != true) {
     let exports;
     const [pattern] = opts.adaptors;
     const [specifier] = pattern.split("=");
-    log.debug(`Attempting to preload typedefs for ${specifier}`);
-    const path4 = await resolveSpecifierPath(pattern, opts.repoDir, log);
-    if (path4) {
+    log.debug(`Attempting to preload types for ${specifier}`);
+    const path5 = await resolveSpecifierPath(pattern, opts.repoDir, log);
+    if (path5) {
       try {
-        exports = await preloadAdaptorExports(path4);
-        if (exports) {
-          log.info(`Loaded typedefs for ${specifier}`);
-        }
+        exports = await preloadAdaptorExports(path5, log);
       } catch (e) {
-        log.error(`Failed to load adaptor typedefs from path ${path4}`);
+        log.error(`Failed to load adaptor typedefs from path ${path5}`);
         log.error(e);
       }
     }
@@ -145,6 +141,7 @@ var loadTransformOptions = async (opts, log) => {
       log.debug(`No module exports found for ${pattern}`);
     }
     options["add-imports"] = {
+      ignore: opts.ignoreImports,
       adaptor: {
         name: stripVersionSpecifier(specifier),
         exports,
@@ -366,20 +363,20 @@ var RETRY_COUNT = 20;
 var TIMEOUT_MS = 1e3 * 60;
 var actualDocGen = (specifier) => describePackage(specifier, {});
 var ensurePath = (filePath) => mkdirSync(path.dirname(filePath), { recursive: true });
-var generatePlaceholder = (path4) => {
-  writeFileSync(path4, `{ "loading": true, "timestamp": ${Date.now()}}`);
+var generatePlaceholder = (path5) => {
+  writeFileSync(path5, `{ "loading": true, "timestamp": ${Date.now()}}`);
 };
 var finish = (logger, resultPath) => {
   logger.success("Done! Docs can be found at:\n");
   logger.print(`  ${path.resolve(resultPath)}`);
 };
-var generateDocs = async (specifier, path4, docgen, logger) => {
+var generateDocs = async (specifier, path5, docgen, logger) => {
   const result = await docgen(specifier);
-  await writeFile3(path4, JSON.stringify(result, null, 2));
-  finish(logger, path4);
-  return path4;
+  await writeFile3(path5, JSON.stringify(result, null, 2));
+  finish(logger, path5);
+  return path5;
 };
-var waitForDocs = async (docs, path4, logger, retryDuration = RETRY_DURATION) => {
+var waitForDocs = async (docs, path5, logger, retryDuration = RETRY_DURATION) => {
   try {
     if (docs.hasOwnProperty("loading")) {
       logger.info("Docs are being loaded by another process. Waiting.");
@@ -391,19 +388,19 @@ var waitForDocs = async (docs, path4, logger, retryDuration = RETRY_DURATION) =>
             clearInterval(i);
             reject(new Error("Timed out waiting for docs to load"));
           }
-          const updated = JSON.parse(readFileSync(path4, "utf8"));
+          const updated = JSON.parse(readFileSync(path5, "utf8"));
           if (!updated.hasOwnProperty("loading")) {
             logger.info("Docs found!");
             clearInterval(i);
-            resolve(path4);
+            resolve(path5);
           }
           count++;
         }, retryDuration);
       });
     } else {
-      logger.info(`Docs already written to cache at ${path4}`);
-      finish(logger, path4);
-      return path4;
+      logger.info(`Docs already written to cache at ${path5}`);
+      finish(logger, path5);
+      return path5;
     }
   } catch (e) {
     logger.error("Existing doc JSON corrupt. Aborting");
@@ -420,28 +417,28 @@ var docgenHandler = (options, logger, docgen = actualDocGen, retryDuration = RET
     process.exit(9);
   }
   logger.success(`Generating docs for ${specifier}`);
-  const path4 = `${repoDir}/docs/${specifier}.json`;
-  ensurePath(path4);
+  const path5 = `${repoDir}/docs/${specifier}.json`;
+  ensurePath(path5);
   const handleError = () => {
     logger.info("Removing placeholder");
-    rmSync(path4);
+    rmSync(path5);
   };
   try {
-    const existing = readFileSync(path4, "utf8");
+    const existing = readFileSync(path5, "utf8");
     const json = JSON.parse(existing);
     if (json && json.timeout && Date.now() - json.timeout >= TIMEOUT_MS) {
       logger.info(`Expired placeholder found. Removing.`);
-      rmSync(path4);
+      rmSync(path5);
       throw new Error("TIMEOUT");
     }
-    return waitForDocs(json, path4, logger, retryDuration);
+    return waitForDocs(json, path5, logger, retryDuration);
   } catch (e) {
     if (e.message !== "TIMEOUT") {
-      logger.info(`Docs JSON not found at ${path4}`);
+      logger.info(`Docs JSON not found at ${path5}`);
     }
     logger.debug("Generating placeholder");
-    generatePlaceholder(path4);
-    return generateDocs(specifier, path4, docgen, logger).catch((e2) => {
+    generatePlaceholder(path5);
+    return generateDocs(specifier, path5, docgen, logger).catch((e2) => {
       logger.error("Error generating documentation");
       logger.error(e2);
       handleError();
@@ -489,15 +486,15 @@ var docsHandler = async (options, logger) => {
     logger.success(`Showing docs for ${adaptorName} v${version}`);
   }
   logger.info("Generating/loading documentation...");
-  const path4 = await handler_default4(
+  const path5 = await handler_default4(
     {
       specifier: `${name}@${version}`,
       repoDir
     },
     createNullLogger()
   );
-  if (path4) {
-    const source = await readFile2(path4, "utf8");
+  if (path5) {
+    const source = await readFile2(path5, "utf8");
     const data = JSON.parse(source);
     let desc;
     if (operation) {
@@ -518,9 +515,110 @@ var docsHandler = async (options, logger) => {
 };
 var handler_default5 = docsHandler;
 
+// src/metadata/cache.ts
+import { createHash } from "node:crypto";
+import { readFileSync as readFileSync2 } from "node:fs";
+import path2 from "node:path";
+import { writeFile as writeFile4, mkdir } from "node:fs/promises";
+var getPath = (repoDir, key) => `${repoDir}/meta/${key}.json`;
+var generateKey = (config) => createHash("sha256").update(JSON.stringify(config)).digest("hex");
+var get = (repoPath, key) => {
+  try {
+    const data = readFileSync2(getPath(repoPath, key));
+    const json = JSON.parse(data);
+    return json;
+  } catch (e) {
+    return null;
+  }
+};
+var set = async (repoPath, key, data) => {
+  const fullPath = getPath(repoPath, key);
+  await mkdir(path2.dirname(fullPath), { recursive: true });
+  await writeFile4(fullPath, JSON.stringify(data));
+};
+var cache_default = { get, set, generateKey, getPath };
+
+// src/metadata/handler.ts
+import { getModuleEntryPoint } from "@openfn/runtime";
+var decorateMetadata = (metadata) => {
+  metadata.created = new Date().toISOString();
+};
+var getAdaptorPath = async (adaptor, logger, repoDir) => {
+  let adaptorPath;
+  let adaptorSpecifier;
+  if (adaptor.match("=")) {
+    const parts = adaptor.split("=");
+    adaptorSpecifier = parts[0];
+    adaptorPath = parts[1];
+  } else {
+    if (adaptor.endsWith(".js")) {
+      return adaptor;
+    }
+    adaptorSpecifier = adaptor;
+    if (adaptor.startsWith("/")) {
+      adaptorPath = adaptor;
+    }
+  }
+  if (!adaptorPath || !adaptorPath.endsWith("js")) {
+    const entry = await getModuleEntryPoint(
+      adaptorSpecifier,
+      adaptorPath,
+      repoDir,
+      logger
+    );
+    adaptorPath = entry?.path;
+  }
+  logger.debug("loading adaptor from", adaptorPath);
+  return adaptorPath;
+};
+var metadataHandler = async (options, logger) => {
+  const { repoDir, adaptors } = options;
+  const adaptor = adaptors[0];
+  const state = await load_state_default(options, logger);
+  logger.success(`Generating metadata`);
+  const config = state.configuration;
+  logger.info("config:", config);
+  if (!config || Object.keys(config).length === 0) {
+    logger.error("ERROR: Invalid configuration passed");
+    process.exit(1);
+  }
+  const finish2 = () => {
+    logger.success("Done!");
+    logger.print(cache_default.getPath(repoDir, id));
+  };
+  const id = cache_default.generateKey(config);
+  if (!options.force) {
+    logger.debug("config hash: ", id);
+    const cached = await cache_default.get(repoDir, id);
+    if (cached) {
+      logger.success("Returning metadata from cache");
+      return finish2();
+    }
+  }
+  try {
+    const adaptorPath = await getAdaptorPath(adaptor, logger, options.repoDir);
+    const mod = await import(adaptorPath);
+    if (mod.metadata) {
+      logger.info("Metadata function found. Generating metadata...");
+      const result = await mod.metadata(config);
+      decorateMetadata(result);
+      await cache_default.set(repoDir, id, result);
+      finish2();
+    } else {
+      logger.error("No metadata helper found");
+      process.exit(1);
+    }
+  } catch (e) {
+    logger.error("Exception while generating metadata");
+    logger.error(e);
+    process.exit(1);
+  }
+};
+var handler_default6 = metadataHandler;
+
 // src/util/use-adaptors-repo.ts
 import { readFile as readFile3 } from "node:fs/promises";
-import path2 from "node:path";
+import path3 from "node:path";
 import assert from "node:assert";
 import { getNameAndVersion as getNameAndVersion4 } from "@openfn/runtime";
 var validateMonoRepo = async (repoPath, log) => {
@@ -544,7 +642,7 @@ var updatePath = (adaptor, repoPath, log) => {
     );
   }
   const shortName = name.replace("@openfn/language-", "");
-  const abspath = path2.resolve(repoPath, "packages", shortName);
+  const abspath = path3.resolve(repoPath, "packages", shortName);
   return `${name}=${abspath}`;
 };
 var useAdaptorsRepo = async (adaptors, repoPath, log) => {
@@ -560,8 +658,8 @@ var useAdaptorsRepo = async (adaptors, repoPath, log) => {
 var use_adaptors_repo_default = useAdaptorsRepo;
 
 // src/util/print-versions.ts
-import { readFileSync as readFileSync2 } from "node:fs";
-import path3 from "node:path";
+import { readFileSync as readFileSync3 } from "node:fs";
+import path4 from "node:path";
 import { getNameAndVersion as getNameAndVersion5 } from "@openfn/runtime";
 import { mainSymbols } from "figures";
 var NODE = "node.js";
@@ -571,7 +669,7 @@ var COMPILER2 = "compiler";
 var { triangleRightSmall: t } = mainSymbols;
 var loadVersionFromPath = (adaptorPath) => {
   try {
-    const pkg = JSON.parse(readFileSync2(path3.resolve(adaptorPath, "package.json"), "utf8"));
+    const pkg = JSON.parse(readFileSync3(path4.resolve(adaptorPath, "package.json"), "utf8"));
     return pkg.version;
   } catch (e) {
     return "unknown";
@@ -641,6 +739,7 @@ var handlers = {
   test: handler_default3,
   docgen: handler_default4,
   docs: handler_default5,
+  metadata: handler_default6,
   ["repo-clean"]: clean,
   ["repo-install"]: install,
   ["repo-pwd"]: pwd,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/cli",
-  "version": "0.0.31",
+  "version": "0.0.34",
   "description": "CLI devtools for the openfn toolchain.",
   "engines": {
     "node": ">=18",
@@ -41,10 +41,10 @@
     "rimraf": "^3.0.2",
     "treeify": "^1.1.0",
     "yargs": "^17.5.1",
-    "@openfn/compiler": "0.0.25",
-    "@openfn/describe-package": "0.0.14",
-    "@openfn/logger": "0.0.10",
-    "@openfn/runtime": "0.0.19"
+    "@openfn/compiler": "0.0.28",
+    "@openfn/describe-package": "0.0.15",
+    "@openfn/logger": "0.0.12",
+    "@openfn/runtime": "0.0.21"
   },
   "files": [
     "dist",