@openfn/cli 1.12.1 → 1.13.1

package/README.md

@@ -317,6 +317,7 @@ A workflow has a structure like this:
 {
   "workflow": {
     "name": "my-workflow", // human readable name used in logging
+    "globals": "./common-funcs.js", // code or path to functions that can be accessed in any step. globally scoped
     "steps": [
       {
         "name": "a", // human readable name used in logging

package/dist/index.js

@@ -291,6 +291,14 @@ var force = {
     default: false
   }
 };
+var keepUnsupported = {
+  name: "keep-unsupported",
+  yargs: {
+    boolean: true,
+    description: "Keep adaptors that do not support metadata (do not remove them)",
+    default: false
+  }
+};
 var immutable = {
   name: "immutable",
   yargs: {
@@ -916,6 +924,7 @@ var options6 = [
   expandAdaptors,
   adaptors,
   force,
+  keepUnsupported,
   log,
   logJson,
   repoDir,

package/dist/process/runner.js

@@ -320,7 +320,11 @@ var get_autoinstall_targets_default = getAutoinstallTargets;
 // src/repo/handler.ts
 import { exec } from "node:child_process";
 import treeify from "treeify";
-import { install as rtInstall, loadRepoPkg } from "@openfn/runtime";
+import {
+  install as rtInstall,
+  loadRepoPkg,
+  getNameAndVersion as getNameAndVersion2
+} from "@openfn/runtime";
 var install = async (opts, log = defaultLogger) => {
   let { packages, adaptors, repoDir } = opts;
   const targets = [].concat(packages ?? [], adaptors ?? []);
@@ -335,6 +339,29 @@ var install = async (opts, log = defaultLogger) => {
   }
   return [];
 };
+var removePackage = async (packageSpecifier, repoDir, logger) => {
+  const { name, version } = getNameAndVersion2(packageSpecifier);
+  if (!version) {
+    logger.warn(`Cannot remove ${packageSpecifier}: no version specified`);
+    return;
+  }
+  const aliasedName = `${name}_${version}`;
+  logger.info(`Removing package ${aliasedName} from repo...`);
+  try {
+    await new Promise((resolve, reject) => {
+      exec(`npm uninstall ${aliasedName}`, { cwd: repoDir }, (error) => {
+        if (error) {
+          reject(error);
+        } else {
+          resolve();
+        }
+      });
+    });
+    logger.success(`Successfully removed ${aliasedName}`);
+  } catch (error) {
+    logger.warn(`Failed to remove ${aliasedName}: ${error.message}`);
+  }
+};
 var clean = async (options, logger) => {
   if (options.repoDir) {
     const doIt = await logger.confirm(
@@ -390,7 +417,10 @@ var list = async (options, logger) => {
 };
 
 // src/compile/compile.ts
-import compile, { preloadAdaptorExports } from "@openfn/compiler";
+import compile, {
+  preloadAdaptorExports,
+  getExports
+} from "@openfn/compiler";
 import { getModulePath } from "@openfn/runtime";
 
 // src/util/abort.ts
@@ -459,11 +489,13 @@ var compileJob = async (job, opts, log, jobName) => {
   }
 };
 var compileWorkflow = async (plan, opts, log) => {
+  let globalsIgnoreList = getExports(plan.workflow.globals);
   for (const step of plan.workflow.steps) {
     const job = step;
     const jobOpts = {
       ...opts,
-      adaptors: job.adaptors ?? opts.adaptors
+      adaptors: job.adaptors ?? opts.adaptors,
+      ignoreImports: globalsIgnoreList
     };
     if (job.expression) {
       const { code, map } = await compileJob(
@@ -683,7 +715,7 @@ var expand_adaptors_default = (input) => {
 import { readFile as readFile2 } from "node:fs/promises";
 import path3 from "node:path";
 import assert from "node:assert";
-import { getNameAndVersion as getNameAndVersion2 } from "@openfn/runtime";
+import { getNameAndVersion as getNameAndVersion3 } from "@openfn/runtime";
 var validateMonoRepo = async (repoPath, log) => {
   try {
     const raw = await readFile2(`${repoPath}/package.json`, "utf8");
@@ -698,7 +730,7 @@ var updatePath = (adaptor, repoPath, log) => {
   if (adaptor.match("=")) {
     return adaptor;
   }
-  const { name, version } = getNameAndVersion2(adaptor);
+  const { name, version } = getNameAndVersion3(adaptor);
   if (version) {
     log.warn(
       `Warning: Ignoring version specifier on ${adaptor} as loading from the adaptors monorepo`
@@ -822,7 +854,8 @@ var loadExpression = async (options, logger) => {
     const plan = {
       workflow: {
         name,
-        steps: [step]
+        steps: [step],
+        globals: options.globals
       },
       options: wfOptions
     };
@@ -854,7 +887,8 @@ var loadOldWorkflow = async (workflow, options, logger, defaultName = "") => {
   logger.warn(final);
   return final;
 };
-var fetchFile = async (jobId, rootDir = "", filePath, log) => {
+var fetchFile = async (fileInfo, log) => {
+  const { rootDir = "", filePath, name } = fileInfo;
   try {
     const fullPath = filePath.startsWith("~") ? filePath : path4.resolve(rootDir, filePath);
     const result = await fs3.readFile(fullPath, "utf8");
@@ -863,7 +897,7 @@ var fetchFile = async (jobId, rootDir = "", filePath, log) => {
   } catch (e) {
     abort_default(
       log,
-      `File not found for job ${jobId}: ${filePath}`,
+      `File not found for ${name}: ${filePath}`,
       void 0,
       `This workflow references a file which cannot be found at ${filePath}
 
@@ -872,6 +906,19 @@ Paths inside the workflow are relative to the workflow.json`
     return ".";
   }
 };
+var importGlobals = async (plan, rootDir, log) => {
+  const fnStr = plan.workflow?.globals;
+  if (fnStr) {
+    if (isPath(fnStr)) {
+      plan.workflow.globals = await fetchFile(
+        { name: "globals", rootDir, filePath: fnStr },
+        log
+      );
+    } else {
+      plan.workflow.globals = fnStr;
+    }
+  }
+};
 var importExpressions = async (plan, rootDir, log) => {
   let idx = 0;
   for (const step of plan.workflow.steps) {
@@ -885,26 +932,32 @@ var importExpressions = async (plan, rootDir, log) => {
     const stateStr = typeof job.state === "string" && job.state?.trim();
     if (expressionStr && isPath(expressionStr)) {
       job.expression = await fetchFile(
-        job.id || `${idx}`,
-        rootDir,
-        expressionStr,
+        {
+          name: `job ${job.id || idx}`,
+          rootDir,
+          filePath: expressionStr
+        },
         log
       );
     }
     if (configurationStr && isPath(configurationStr)) {
       const configString = await fetchFile(
-        job.id || `${idx}`,
-        rootDir,
-        configurationStr,
+        {
+          name: `job configuration ${job.id || idx}`,
+          rootDir,
+          filePath: configurationStr
+        },
         log
       );
       job.configuration = JSON.parse(configString);
     }
     if (stateStr && isPath(stateStr)) {
       const stateString = await fetchFile(
-        job.id || `${idx}`,
-        rootDir,
-        stateStr,
+        {
+          name: `job state ${job.id || idx}`,
+          rootDir,
+          filePath: stateStr
+        },
         log
       );
       job.state = JSON.parse(stateString);
@@ -929,6 +982,9 @@ var loadXPlan = async (plan, options, logger, defaultName = "") => {
     plan.workflow.name = defaultName;
   }
   ensureAdaptors(plan);
+  if (options.globals)
+    plan.workflow.globals = options.globals;
+  await importGlobals(plan, options.baseDir, logger);
   await importExpressions(plan, options.baseDir, logger);
   if (options.expandAdaptors) {
     expand_adaptors_default(plan);
@@ -1620,7 +1676,7 @@ import { writeFile as writeFile5 } from "node:fs/promises";
 import { readFileSync, writeFileSync, mkdirSync, rmSync } from "node:fs";
 import path7 from "node:path";
 import { describePackage } from "@openfn/describe-package";
-import { getNameAndVersion as getNameAndVersion3 } from "@openfn/runtime";
+import { getNameAndVersion as getNameAndVersion4 } from "@openfn/runtime";
 var RETRY_DURATION = 500;
 var RETRY_COUNT = 20;
 var TIMEOUT_MS = 1e3 * 60;
@@ -1672,7 +1728,7 @@ var waitForDocs = async (docs, path12, logger, retryDuration = RETRY_DURATION) =
 };
 var docgenHandler = (options, logger, docgen = actualDocGen, retryDuration = RETRY_DURATION) => {
   const { specifier, repoDir } = options;
-  const { version } = getNameAndVersion3(specifier);
+  const { version } = getNameAndVersion4(specifier);
   if (!version) {
     logger.error("Error: No version number detected");
     logger.error("eg, @openfn/language-common@1.7.5");
@@ -1713,7 +1769,7 @@ var handler_default7 = docgenHandler;
 // src/docs/handler.ts
 import { readFile as readFile4 } from "node:fs/promises";
 import c from "chalk";
-import { getNameAndVersion as getNameAndVersion4, getLatestVersion } from "@openfn/runtime";
+import { getNameAndVersion as getNameAndVersion5, getLatestVersion } from "@openfn/runtime";
 var describeFn = (adaptorName, fn) => [
   c.green(
     `## ${fn.name}(${fn.parameters.map(({ name }) => name).join(",")})`
@@ -1744,7 +1800,7 @@ var docsHandler = async (options, logger) => {
   const { adaptor, operation, repoDir } = options;
   const adaptors = expand_adaptors_default([adaptor]);
   const [adaptorName] = adaptors;
-  let { name, version } = getNameAndVersion4(adaptorName);
+  let { name, version } = getNameAndVersion5(adaptorName);
   if (!version) {
     logger.info("No version number provided, looking for latest...");
     version = await getLatestVersion(name);
@@ -1798,11 +1854,28 @@ var docsHandler = async (options, logger) => {
 var handler_default8 = docsHandler;
 
 // src/metadata/cache.ts
+import { getNameAndVersion as getNameAndVersion6 } from "@openfn/runtime";
 import { createHash } from "node:crypto";
-import { readFileSync as readFileSync2 } from "node:fs";
+import { mkdir as mkdir3, readFile as readFile5, writeFile as writeFile6, readdir, rm } from "node:fs/promises";
 import path8 from "node:path";
-import { writeFile as writeFile6, mkdir as mkdir3 } from "node:fs/promises";
-var getPath = (repoDir, key) => `${repoDir}/meta/${key}.json`;
+var UNSUPPORTED_FILE_NAME = "unsupported.json";
+var getCachePath2 = (repoDir, key) => {
+  const base = path8.join(repoDir, "meta");
+  if (key) {
+    return path8.join(base, key.endsWith(".json") ? key : `${key}.json`);
+  }
+  return base;
+};
+var getCache = async (repoDir, key) => {
+  try {
+    const cachePath = getCachePath2(repoDir, key);
+    const content = await readFile5(cachePath, "utf8");
+    return JSON.parse(content);
+  } catch (e) {
+    return null;
+  }
+};
+var getUnsupportedCache = (repoDir) => getCache(repoDir, UNSUPPORTED_FILE_NAME);
 var sortKeys = (obj) => {
   const newObj = {};
   Object.keys(obj).sort().forEach((k) => {
@@ -1817,24 +1890,83 @@ var sortKeys = (obj) => {
 };
 var generateKey = (config, adaptor) => {
   const sorted = sortKeys(config);
-  const key = `${JSON.stringify(sorted)}-${adaptor}}`;
+  const key = `${JSON.stringify(sorted)}-${adaptor}`;
   return createHash("sha256").update(key).digest("hex");
 };
-var get2 = (repoPath, key) => {
+var get2 = async (repoPath, key) => {
+  const p = getCachePath2(repoPath, key);
   try {
-    const data = readFileSync2(getPath(repoPath, key), "utf8");
-    const json = JSON.parse(data);
-    return json;
+    const result = await readFile5(p, "utf8");
+    return JSON.parse(result);
   } catch (e) {
     return null;
   }
 };
-var set2 = async (repoPath, key, data) => {
-  const fullPath = getPath(repoPath, key);
-  await mkdir3(path8.dirname(fullPath), { recursive: true });
-  await writeFile6(fullPath, JSON.stringify(data));
+var set2 = async (repoPath, key, result) => {
+  const p = getCachePath2(repoPath, key);
+  await mkdir3(path8.dirname(p), { recursive: true });
+  await writeFile6(p, JSON.stringify(result));
+};
+var getUnsupportedCachePath = (repoDir) => {
+  return getCachePath2(repoDir, UNSUPPORTED_FILE_NAME);
+};
+var parseVersion = (version) => {
+  const parts = version.split(".").map(Number);
+  return {
+    major: parts[0] || 0,
+    minor: parts[1] || 0,
+    patch: parts[2] || 0,
+    majorMinor: `${parts[0] || 0}.${parts[1] || 0}`
+  };
+};
+var compareVersions = (version1, version2) => {
+  const v1 = parseVersion(version1);
+  const v2 = parseVersion(version2);
+  if (v1.major !== v2.major)
+    return v1.major - v2.major;
+  if (v1.minor !== v2.minor)
+    return v1.minor - v2.minor;
+  return v1.patch - v2.patch;
+};
+var isAdaptorUnsupported = async (adaptorSpecifier, repoDir) => {
+  const { name, version } = getNameAndVersion6(adaptorSpecifier);
+  if (!version)
+    return false;
+  const cache = await getUnsupportedCache(repoDir);
+  if (!cache || !cache[name]) {
+    return false;
+  }
+  const cached = cache[name];
+  const currentParsed = parseVersion(version);
+  const cachedParsed = parseVersion(cached.lastCheckedVersion);
+  if (currentParsed.major > cachedParsed.major || currentParsed.major === cachedParsed.major && currentParsed.minor > cachedParsed.minor) {
+    return false;
+  }
+  return true;
+};
+var markAdaptorAsUnsupported = async (adaptorSpecifier, repoDir) => {
+  const { name, version } = getNameAndVersion6(adaptorSpecifier);
+  if (!version)
+    return;
+  const cachePath = getUnsupportedCachePath(repoDir);
+  let cache = {};
+  try {
+    const cacheContent = await readFile5(cachePath, "utf8");
+    cache = JSON.parse(cacheContent);
+  } catch (error) {
+  }
+  const parsed = parseVersion(version);
+  const existing = cache[name];
+  if (!existing || compareVersions(version, existing.lastCheckedVersion) > 0) {
+    cache[name] = {
+      lastCheckedVersion: version,
+      majorMinor: parsed.majorMinor,
+      timestamp: Date.now()
+    };
+    await mkdir3(path8.dirname(cachePath), { recursive: true });
+    await writeFile6(cachePath, JSON.stringify(cache, null, 2));
+  }
 };
-var cache_default = { get: get2, set: set2, generateKey, getPath, sortKeys };
 
 // src/metadata/handler.ts
 import { getModuleEntryPoint } from "@openfn/runtime";
@@ -1871,8 +2003,15 @@ var getAdaptorPath = async (adaptor, logger, repoDir) => {
 };
 var shouldAutoinstall = (adaptor) => adaptor?.length > 0 && !adaptor.startsWith("/") && !adaptor.includes("=");
 var metadataHandler = async (options, logger) => {
-  const { repoDir, adaptors } = options;
-  const adaptor = adaptors[0];
+  const { repoDir, adaptors, keepUnsupported } = options;
+  let adaptor = adaptors[0];
+  if (await isAdaptorUnsupported(adaptor, repoDir)) {
+    logger.info(
+      `Adaptor ${adaptor} is known to not support metadata (cached) - skipping lookup`
+    );
+    logger.error("No metadata helper found");
+    process.exit(1);
+  }
   const state = await load_state_default({}, options, logger);
   logger.success(`Generating metadata`);
   logger.info("config:", state);
@@ -1883,31 +2022,58 @@ var metadataHandler = async (options, logger) => {
   }
   const finish2 = () => {
     logger.success("Done!");
-    logger.print(cache_default.getPath(repoDir, id));
+    logger.print(getCachePath2(repoDir, id));
   };
-  const id = cache_default.generateKey(config, adaptor);
+  const id = generateKey(config, adaptor);
   if (!options.force) {
     logger.debug("config hash: ", id);
-    const cached = await cache_default.get(repoDir, id);
+    const cached = await get2(repoDir, id);
     if (cached) {
       logger.success("Returning metadata from cache");
       return finish2();
     }
   }
+  let wasAutoInstalled = false;
   try {
     if (shouldAutoinstall(adaptor)) {
-      await install({ packages: [adaptor], repoDir }, logger);
+      const autoinstallResult = await install(
+        { packages: [adaptor], repoDir },
+        logger
+      );
+      wasAutoInstalled = true;
+      adaptor = autoinstallResult[0];
     }
     const adaptorPath = await getAdaptorPath(adaptor, logger, options.repoDir);
+    if (!adaptorPath) {
+      throw new Error(`Could not resolve adaptor path for ${adaptor}`);
+    }
     const mod = await import(adaptorPath);
-    if (mod.metadata) {
+    if (mod.metadata && typeof mod.metadata === "function") {
       logger.info("Metadata function found. Generating metadata...");
       const result = await mod.metadata(config);
       decorateMetadata(result);
-      await cache_default.set(repoDir, id, result);
+      await set2(
+        repoDir,
+        id,
+        result
+      );
       finish2();
     } else {
       logger.error("No metadata helper found");
+      if (wasAutoInstalled && !keepUnsupported) {
+        logger.info("Removing unsupported adaptor from disk...");
+        await removePackage(adaptor, repoDir, logger);
+        await markAdaptorAsUnsupported(adaptor, repoDir);
+        logger.info("Adaptor removed and marked as unsupported");
+      } else if (wasAutoInstalled && keepUnsupported) {
+        if (adaptor === "@openfn/language-openfn") {
+          logger.log({ wasAutoInstalled, keepUnsupported });
+        }
+        logger.info(
+          "Keeping unsupported adaptor as requested by --keep-unsupported flag"
+        );
+        await markAdaptorAsUnsupported(adaptor, repoDir);
+      }
       process.exit(1);
     }
   } catch (e) {
@@ -2099,10 +2265,10 @@ function pickFirst2(...args) {
 var handler_default10 = pullHandler;
 
 // src/util/print-versions.ts
-import { readFileSync as readFileSync3 } from "node:fs";
+import { readFileSync as readFileSync2 } from "node:fs";
 import path11 from "node:path";
 import url from "node:url";
-import { getNameAndVersion as getNameAndVersion5 } from "@openfn/runtime";
+import { getNameAndVersion as getNameAndVersion7 } from "@openfn/runtime";
 import { mainSymbols } from "figures";
 var NODE = "node.js";
 var CLI2 = "cli";
@@ -2112,7 +2278,7 @@ var { triangleRightSmall: t } = mainSymbols;
 var loadVersionFromPath = (adaptorPath) => {
   try {
     const pkg = JSON.parse(
-      readFileSync3(path11.resolve(adaptorPath, "package.json"), "utf8")
+      readFileSync2(path11.resolve(adaptorPath, "package.json"), "utf8")
     );
     return pkg.version;
   } catch (e) {
@@ -2129,12 +2295,12 @@ var printVersions = async (logger, options = {}, includeComponents = false) => {
     if (adaptor.match("=")) {
       const [namePart, pathPart] = adaptor.split("=");
       adaptorVersion = loadVersionFromPath(pathPart);
-      adaptorName = getNameAndVersion5(namePart).name;
+      adaptorName = getNameAndVersion7(namePart).name;
     } else if (options.monorepoPath) {
-      adaptorName = getNameAndVersion5(adaptor).name;
+      adaptorName = getNameAndVersion7(adaptor).name;
       adaptorVersion = "monorepo";
     } else {
-      const { name, version: version2 } = getNameAndVersion5(adaptor);
+      const { name, version: version2 } = getNameAndVersion7(adaptor);
       adaptorName = name;
       adaptorVersion = version2 || "latest";
     }
@@ -2148,7 +2314,7 @@ var printVersions = async (logger, options = {}, includeComponents = false) => {
   );
   const prefix = (str) => `         ${t} ${str.padEnd(longest + 4, " ")}`;
   const dirname3 = path11.dirname(url.fileURLToPath(import.meta.url));
-  const pkg = JSON.parse(readFileSync3(`${dirname3}/../../package.json`, "utf8"));
+  const pkg = JSON.parse(readFileSync2(`${dirname3}/../../package.json`, "utf8"));
   const { version, dependencies } = pkg;
   const compilerVersion = dependencies["@openfn/compiler"];
   const runtimeVersion = dependencies["@openfn/runtime"];

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@openfn/cli",
-  "version": "1.12.1",
-  "description": "CLI devtools for the openfn toolchain.",
+  "version": "1.13.1",
+  "description": "CLI devtools for the OpenFn toolchain",
   "engines": {
     "node": ">=18",
     "pnpm": ">=7"
@@ -48,12 +48,12 @@
     "ws": "^8.18.0",
     "yargs": "^17.7.2",
     "@openfn/deploy": "0.11.2",
-    "@openfn/compiler": "1.0.4",
+    "@openfn/compiler": "1.1.0",
     "@openfn/lexicon": "^1.2.2",
-    "@openfn/logger": "1.0.5",
+    "@openfn/describe-package": "0.1.4",
     "@openfn/project": "^0.1.0",
-    "@openfn/runtime": "1.6.4",
-    "@openfn/describe-package": "0.1.4"
+    "@openfn/logger": "1.0.5",
+    "@openfn/runtime": "1.7.1"
   },
   "files": [
     "dist",