@openfn/cli 1.8.11 → 1.9.0

package/dist/index.js

@@ -135,10 +135,10 @@ var ensureLogOpts = (opts2) => {
 var ensure_log_opts_default = ensureLogOpts;
 
 // src/options.ts
-var setDefaultValue = (opts2, key, value) => {
-  const v = opts2[key];
+var setDefaultValue = (opts2, key2, value2) => {
+  const v = opts2[key2];
   if (isNaN(v) && !v) {
-    opts2[key] = value;
+    opts2[key2] = value2;
   }
 };
 var adaptors = {
@@ -370,7 +370,18 @@ var repoDir = {
   yargs: () => ({
     description: "Provide a path to the repo root dir",
     default: process.env.OPENFN_REPO_DIR || DEFAULT_REPO_DIR
-  })
+  }),
+  ensure: (opts2) => {
+    if (opts2.repoDir === DEFAULT_REPO_DIR) {
+      console.warn(
+        "WARNING: no repo module dir found! Using the default (/tmp/repo)"
+      );
+      console.warn(
+        "You should set OPENFN_REPO_DIR or pass --repoDir=some/path in to the CLI"
+      );
+      console.log();
+    }
+  }
 };
 var start = {
   name: "start",
@@ -534,6 +545,200 @@ var command_default = {
   )
 };
 
+// src/collections/command.ts
+var desc2 = `Read and write from the OpenFn Collections API`;
+var command_default2 = {
+  command: "collections <subcommand>",
+  describe: desc2,
+  builder: (yargs2) => yargs2.command(get).command(set).command(remove).example(
+    "$0 collections get my-collection 2024* -o /tmp/output.json",
+    'Get all keys from my-collection starting with the string "2024" and output the results to file'
+  ).example(
+    "$0 collections set my-collection my-key path/to/value.json",
+    "Set a single key in my-collection to the contents of value.json"
+  ).example(
+    "$0 collections set my-collection --items path/to/items.json",
+    "Set multiple key/value pairs from items.json to my-collection"
+  ).example(
+    "$0 collections remove my-collection my-key",
+    "Remove a single key from my-collection"
+  )
+};
+var collectionName = {
+  name: "collection-name",
+  yargs: {
+    alias: ["name"],
+    description: "Name of the collection to fetch from",
+    demand: true
+  }
+};
+var key = {
+  name: "key",
+  yargs: {
+    description: "Key or key pattern to retrieve",
+    type: "string",
+    demand: true
+  },
+  ensure: (opts2) => {
+    if (opts2.key && typeof opts2.key !== "string") {
+      opts2.key = `${opts2.key}`;
+    }
+  }
+};
+var token = {
+  name: "pat",
+  yargs: {
+    alias: ["token"],
+    description: "Lightning Personal Access Token (PAT)"
+  }
+};
+var lightningUrl = {
+  name: "lightning",
+  yargs: {
+    description: "URL to OpenFn server. Defaults to OPENFN_ENDPOINT or https://app.openfn.org"
+  }
+};
+var pageSize = {
+  name: "page-size",
+  yargs: {
+    description: "Number of items to fetch per page",
+    type: "number"
+  }
+};
+var limit = {
+  name: "limit",
+  yargs: {
+    description: "Maximum number of items to download",
+    type: "number"
+  }
+};
+var pretty = {
+  name: "pretty",
+  yargs: {
+    description: "Prettify serialized output",
+    type: "boolean"
+  }
+};
+var createdBefore = {
+  name: "created-before",
+  yargs: {
+    description: "Matches keys created before the start of the created data"
+  }
+};
+var createdAfter = {
+  name: "created-after",
+  yargs: {
+    description: "Matches keys created after the end of the created data"
+  }
+};
+var updatedBefore = {
+  name: "updated-before",
+  yargs: {
+    description: "Matches keys updated before the start of the created data"
+  }
+};
+var updatedAfter = {
+  name: "updated-after",
+  yargs: {
+    description: "Matches keys updated after the end of the created data"
+  }
+};
+var getOptions = [
+  collectionName,
+  key,
+  token,
+  lightningUrl,
+  pageSize,
+  limit,
+  pretty,
+  createdBefore,
+  createdAfter,
+  updatedAfter,
+  updatedBefore,
+  override(log, {
+    default: "info"
+  }),
+  logJson,
+  {
+    ...outputPath,
+    // disable default output path behaviour
+    ensure: () => {
+    }
+  }
+];
+var get = {
+  command: "get <name> <key>",
+  describe: "Get values from a collection",
+  handler: ensure("collections-get", getOptions),
+  builder: (yargs2) => build(getOptions, yargs2)
+};
+var dryRun = {
+  name: "dry-run",
+  yargs: {
+    description: "[Alpha] Do not delete keys and instead return the keys that would be deleted",
+    type: "boolean"
+  }
+};
+var removeOptions = [
+  collectionName,
+  key,
+  token,
+  lightningUrl,
+  dryRun,
+  createdBefore,
+  createdAfter,
+  updatedAfter,
+  updatedBefore,
+  override(log, {
+    default: "info"
+  }),
+  logJson
+];
+var remove = {
+  command: "remove <name> <key>",
+  describe: "Remove values from a collection",
+  handler: ensure("collections-remove", removeOptions),
+  builder: (yargs2) => build(removeOptions, yargs2)
+};
+var value = {
+  name: "value",
+  yargs: {
+    description: "Path to the value to upsert"
+  }
+};
+var items = {
+  name: "items",
+  yargs: {
+    description: "Path to a batch of items to upsert. Must contain a JSON object where each key is an item key, and each value is an uploaded value"
+  }
+};
+var setOptions = [
+  collectionName,
+  override(key, {
+    demand: false
+  }),
+  token,
+  lightningUrl,
+  value,
+  items,
+  override(log, {
+    default: "info"
+  }),
+  logJson
+];
+var set = {
+  command: "set <name> [key] [value] [--items]",
+  describe: "Uploads values to a collection. Must set key & value OR --items.",
+  handler: ensure("collections-set", setOptions),
+  builder: (yargs2) => build(setOptions, yargs2).example(
+    "collections set my-collection cities-mapping ./citymap.json",
+    "Upload the data in ./citymap.json to the cities-mapping key"
+  ).example(
+    "collections set my-collection --items ./items.json",
+    "Upsert the object in ./items.json as a batch of items (key/value pairs)"
+  )
+};
+
 // src/compile/command.ts
 var options2 = [
   expandAdaptors,
@@ -564,7 +769,7 @@ var compileCommand = {
     "Compiles the workflow at foo/work.json and prints the result to -o foo/workflow-compiled.json"
   )
 };
-var command_default2 = compileCommand;
+var command_default3 = compileCommand;
 
 // src/deploy/command.ts
 var options3 = [
@@ -582,7 +787,7 @@ var deployCommand = {
   handler: ensure("deploy", options3),
   builder: (yargs2) => build(options3, yargs2)
 };
-var command_default3 = deployCommand;
+var command_default4 = deployCommand;
 
 // src/docgen/command.ts
 var docgenCommand = {
@@ -595,7 +800,7 @@ var docgenCommand = {
   },
   builder: (yargs2) => yargs2.example("docgen @openfn/language-common@1.7.5", "")
 };
-var command_default4 = docgenCommand;
+var command_default5 = docgenCommand;
 
 // src/docs/command.ts
 var options4 = [log, logJson, repoDir];
@@ -608,7 +813,7 @@ var docsCommand = {
     "Print help for the common fn operation"
   )
 };
-var command_default5 = docsCommand;
+var command_default6 = docsCommand;
 
 // src/execute/command.ts
 var options5 = [
@@ -661,7 +866,7 @@ Remember to include the adaptor name with -a. Auto install adaptors with the -i
     "Compile the expression at job.js with the http adaptor and print the code to stdout"
   )
 };
-var command_default6 = executeCommand;
+var command_default7 = executeCommand;
 
 // src/metadata/command.ts
 var options6 = [
@@ -675,7 +880,7 @@ var options6 = [
   stateStdin,
   useAdaptorsMonorepo
 ];
-var command_default7 = {
+var command_default8 = {
   command: "metadata",
   describe: "Generate metadata for an adaptor config",
   handler: ensure("metadata", options6),
@@ -706,7 +911,7 @@ var pullCommand = {
   ),
   handler: ensure("pull", options7)
 };
-var command_default8 = pullCommand;
+var command_default9 = pullCommand;
 
 // src/repo/command.ts
 var repo = {
@@ -768,7 +973,7 @@ var list = {
 
 // src/test/command.ts
 var options8 = [stateStdin, log, logJson];
-var command_default9 = {
+var command_default10 = {
   command: "test",
   desc: "Compiles and runs a test job, printing the result to stdout",
   handler: ensure("test", options8),
@@ -777,7 +982,7 @@ var command_default9 = {
 
 // src/cli.ts
 var y = yargs(hideBin(process.argv));
-var cmd = y.command(command_default6).command(command_default2).command(command_default3).command(install).command(repo).command(command_default9).command(command_default5).command(command_default).command(command_default7).command(command_default4).command(command_default8).command({
+var cmd = y.command(command_default7).command(command_default3).command(command_default2).command(command_default4).command(install).command(repo).command(command_default10).command(command_default6).command(command_default).command(command_default8).command(command_default5).command(command_default9).command({
   command: "version",
   describe: "Show the currently installed version of the CLI, compiler and runtime.",
   handler: (argv) => {

package/dist/process/runner.js

@@ -124,14 +124,14 @@ var callApollo = async (apolloBaseUrl, serviceName, payload, logger) => {
     });
   });
 };
-var loadPayload = async (logger, path9) => {
-  if (!path9) {
+var loadPayload = async (logger, path11) => {
+  if (!path11) {
     logger.warn("No JSON payload provided");
     logger.warn("Most apollo services require JSON to be uploaded");
     return {};
   }
-  if (path9.endsWith(".json")) {
-    const str = await readFile(path9, "utf8");
+  if (path11.endsWith(".json")) {
+    const str = await readFile(path11, "utf8");
     const json = JSON.parse(str);
     logger.debug("Loaded JSON payload");
     return json;
@@ -253,13 +253,13 @@ var execute_default = async (plan, input, opts, logger) => {
 };
 function parseAdaptors(plan) {
   const extractInfo = (specifier) => {
-    const [module, path9] = specifier.split("=");
+    const [module, path11] = specifier.split("=");
     const { name, version } = getNameAndVersion(module);
     const info = {
       name
     };
-    if (path9) {
-      info.path = path9;
+    if (path11) {
+      info.path = path11;
     }
     if (version) {
       info.version = version;
@@ -413,6 +413,19 @@ var abort_default = (logger, reason, error, help) => {
   process.exitCode = 1;
   throw e;
 };
+var DeferredAbort = class extends Error {
+  constructor(reason, help) {
+    super("DeferredAbortError");
+    this.reason = reason;
+    this.help = help ?? "";
+  }
+  abort = true;
+  reason = "";
+  help = "";
+};
+var throwAbortableError = (message, help) => {
+  throw new DeferredAbort(message, help);
+};
 
 // src/compile/compile.ts
 var compile_default = async (planOrPath, opts, log) => {
@@ -465,10 +478,10 @@ var stripVersionSpecifier = (specifier) => {
   return specifier;
 };
 var resolveSpecifierPath = async (pattern, repoDir, log) => {
-  const [specifier, path9] = pattern.split("=");
-  if (path9) {
-    log.debug(`Resolved ${specifier} to path: ${path9}`);
-    return path9;
+  const [specifier, path11] = pattern.split("=");
+  if (path11) {
+    log.debug(`Resolved ${specifier} to path: ${path11}`);
+    return path11;
   }
   const repoPath = await getModulePath(specifier, repoDir, log);
   if (repoPath) {
@@ -486,12 +499,12 @@ var loadTransformOptions = async (opts, log) => {
       let exports;
       const [specifier] = adaptorInput.split("=");
       log.debug(`Trying to preload types for ${specifier}`);
-      const path9 = await resolveSpecifierPath(adaptorInput, opts.repoDir, log);
-      if (path9) {
+      const path11 = await resolveSpecifierPath(adaptorInput, opts.repoDir, log);
+      if (path11) {
         try {
-          exports = await preloadAdaptorExports(path9, log);
+          exports = await preloadAdaptorExports(path11, log);
         } catch (e) {
-          log.error(`Failed to load adaptor typedefs from path ${path9}`);
+          log.error(`Failed to load adaptor typedefs from path ${path11}`);
           log.error(e);
         }
       }
@@ -894,8 +907,8 @@ var loadXPlan = async (plan, options, logger, defaultName = "") => {
 };
 
 // src/util/assert-path.ts
-var assert_path_default = (path9) => {
-  if (!path9) {
+var assert_path_default = (path11) => {
+  if (!path11) {
     console.error("ERROR: no path provided!");
     console.error("\nUsage:");
     console.error("  open path/to/job");
@@ -1114,6 +1127,285 @@ var compileHandler = async (options, logger) => {
 };
 var handler_default3 = compileHandler;
 
+// src/collections/handler.ts
+import path6 from "node:path";
+import { readFile as readFile3, writeFile as writeFile4 } from "node:fs/promises";
+
+// src/collections/request.ts
+import path5 from "node:path";
+import { request } from "undici";
+var DEFAULT_PAGE_SIZE = 1e3;
+var request_default = async (method, options, logger) => {
+  const base = options.lightning || process.env.OPENFN_ENDPOINT || "https://app.openfn.org";
+  const url2 = path5.join(base, "/collections", options.collectionName);
+  logger.debug("Calling Collections server at ", url2);
+  const headers = {
+    Authorization: `Bearer ${options.token}`
+  };
+  const query = Object.assign(
+    {
+      key: options.key,
+      limit: options.pageSize || DEFAULT_PAGE_SIZE
+    },
+    options.query
+  );
+  const args = {
+    headers,
+    method,
+    query
+  };
+  if (options.data) {
+    args.body = JSON.stringify(options.data);
+    headers["content-type"] = "application/json";
+  }
+  let result = {};
+  let cursor;
+  let count = 0;
+  let limit = Infinity;
+  do {
+    if (cursor) {
+      query.cursor = cursor;
+    }
+    if (options.limit) {
+      limit = options.limit;
+      query.limit = Math.min(
+        options.pageSize || DEFAULT_PAGE_SIZE,
+        options.limit - count
+      );
+    }
+    try {
+      const response = await request(url2, args);
+      if (response.statusCode >= 400) {
+        return handleError(logger, response);
+      }
+      const responseData = await response.body.json();
+      if (responseData.items) {
+        count += responseData.items.length;
+        logger.debug(
+          "Received",
+          response.statusCode,
+          `- ${responseData.items.length} values`
+        );
+        for (const item of responseData?.items) {
+          try {
+            result[item.key] = JSON.parse(item.value);
+          } catch (e) {
+            result[item.key] = item.value;
+          }
+        }
+        cursor = responseData.cursor;
+      } else {
+        logger.debug(
+          "Received",
+          response.statusCode,
+          `- ${JSON.stringify(responseData)}`
+        );
+        result = responseData;
+      }
+    } catch (e) {
+      logger.error(e);
+      throwAbortableError(
+        `CONNECTION_REFUSED: error connecting to server at ${base}`,
+        "Check you have passed the correct URL to --lightning or OPENFN_ENDPOINT"
+      );
+    }
+  } while (cursor && count < limit);
+  return result;
+};
+async function handleError(logger, response) {
+  logger.error("Error from server", response.statusCode);
+  let message;
+  let fix;
+  switch (response.statusCode) {
+    case 404:
+      message = `404: collection not found`;
+      fix = `Ensure the Collection has been created on the admin page`;
+      break;
+    default:
+      message = `Error from server: ${response.statusCode}`;
+  }
+  let contentType = response.headers?.["content-type"] ?? "";
+  if (contentType.startsWith("application/json")) {
+    try {
+      const body = await response.body.json();
+      logger.error(body);
+    } catch (e) {
+    }
+  } else {
+    try {
+      const text = await response.body.text();
+      logger.error(text);
+    } catch (e) {
+    }
+  }
+  throwAbortableError(message, fix);
+}
+
+// src/collections/handler.ts
+var ensureToken = (opts, logger) => {
+  if (!("token" in opts)) {
+    if (process.env.OPENFN_PAT) {
+      const token = process.env.OPENFN_PAT;
+      logger.info(
+        `Using access token ending in ${token?.substring(
+          token.length - 10
+        )} from env (OPENFN_PAT)`
+      );
+      opts.token = token;
+    } else {
+      logger.error("No access token detected!");
+      logger.error(
+        "Ensure you pass a Personal Access Token (PAT) with --token $MY_TOKEN or set the OPENFN_PAT env var"
+      );
+      logger.error(
+        "You can get a PAT from OpenFn, see https://docs.openfn.org/documentation/api-tokens"
+      );
+      throw new Error("NO_PAT");
+    }
+  }
+};
+var buildQuery = (options) => {
+  const map = {
+    createdBefore: "created_before",
+    createdAfter: "created_after",
+    updatedBefore: "updated_before",
+    updatedAfter: "updated_after"
+  };
+  const query = {};
+  Object.keys(map).forEach((key) => {
+    if (options[key]) {
+      query[map[key]] = options[key];
+    }
+  });
+  return query;
+};
+var get = async (options, logger) => {
+  ensureToken(options, logger);
+  const multiMode = options.key.includes("*");
+  if (multiMode) {
+    logger.info(
+      `Fetching multiple items from collection "${options.collectionName}" with pattern ${options.key}`
+    );
+  } else {
+    logger.info(
+      `Fetching "${options.key}" from collection "${options.collectionName}"`
+    );
+  }
+  let result = await request_default(
+    "GET",
+    {
+      lightning: options.lightning,
+      token: options.token,
+      pageSize: options.pageSize,
+      limit: options.limit,
+      key: options.key,
+      collectionName: options.collectionName,
+      query: buildQuery(options)
+    },
+    logger
+  );
+  if (multiMode) {
+    logger.success(`Fetched ${Object.keys(result).length} items!`);
+  } else {
+    result = Object.values(result)[0];
+    logger.success(`Fetched ${options.key}`);
+  }
+  if (options.outputPath) {
+    const content = JSON.stringify(
+      result,
+      null,
+      options.pretty ? 2 : void 0
+    );
+    await writeFile4(options.outputPath, content);
+    logger.always(`Wrote items to ${options.outputPath}`);
+  } else {
+    logger.print(result);
+  }
+};
+var set = async (options, logger) => {
+  if (options.key && options.items) {
+    throwAbortableError(
+      "ARGUMENT_ERROR: arguments for key and items were provided",
+      "If upserting multiple items with --items, do not pass a key"
+    );
+  }
+  ensureToken(options, logger);
+  logger.info(`Upserting items to collection "${options.collectionName}"`);
+  const items = [];
+  if (options.items) {
+    const resolvedPath = path6.resolve(options.items);
+    logger.debug("Loading items from ", resolvedPath);
+    const data = await readFile3(resolvedPath, "utf8");
+    const obj = JSON.parse(data);
+    Object.entries(obj).forEach(([key, value]) => {
+      items.push({ key, value: JSON.stringify(value) });
+    });
+    logger.info(`Upserting ${items.length} items`);
+  } else if (options.key && options.value) {
+    const resolvedPath = path6.resolve(options.value);
+    logger.debug("Loading value from ", resolvedPath);
+    const data = await readFile3(path6.resolve(options.value), "utf8");
+    const value = JSON.stringify(JSON.parse(data));
+    items.push({ key: options.key, value });
+    logger.info(`Upserting data to "${options.key}"`);
+  } else {
+    throw new Error("INVALID_ARGUMENTS");
+  }
+  const result = await request_default(
+    "POST",
+    {
+      lightning: options.lightning,
+      token: options.token,
+      key: options.key,
+      collectionName: options.collectionName,
+      data: { items }
+    },
+    logger
+  );
+  logger.success(`Upserted ${result.upserted} items!`);
+};
+var remove = async (options, logger) => {
+  ensureToken(options, logger);
+  logger.info(
+    `Removing "${options.key}" from collection "${options.collectionName}"`
+  );
+  if (options.dryRun) {
+    logger.info("--dry-run passed: fetching affected items");
+    let result = await request_default(
+      "GET",
+      {
+        lightning: options.lightning,
+        token: options.token,
+        key: options.key,
+        collectionName: options.collectionName
+      },
+      logger
+    );
+    const keys = Object.keys(result);
+    logger.info("Keys to be removed:");
+    logger.print(keys);
+    logger.always("Aborting request - keys have not been removed");
+  } else {
+    let result = await request_default(
+      "DELETE",
+      {
+        lightning: options.lightning,
+        token: options.token,
+        key: options.key,
+        collectionName: options.collectionName,
+        query: buildQuery(options)
+      },
+      logger
+    );
+    logger.success(`Removed ${result.deleted} items`);
+  }
+};
+var handler_default4 = {
+  get,
+  set,
+  remove
+};
+
 // src/test/handler.ts
 var testHandler = async (options, logger) => {
   logger.log("Running test workflow...");
@@ -1169,7 +1461,7 @@ var testHandler = async (options, logger) => {
   logger.success(`Result: ${result.data.answer}`);
   return result;
 };
-var handler_default4 = testHandler;
+var handler_default5 = testHandler;
 
 // src/deploy/handler.ts
 import {
@@ -1223,33 +1515,33 @@ function mergeOverrides(config, options) {
 function pickFirst(...args) {
   return args.find((arg) => arg !== void 0 && arg !== null);
 }
-var handler_default5 = deployHandler;
+var handler_default6 = deployHandler;
 
 // src/docgen/handler.ts
-import { writeFile as writeFile4 } from "node:fs/promises";
+import { writeFile as writeFile5 } from "node:fs/promises";
 import { readFileSync, writeFileSync, mkdirSync, rmSync } from "node:fs";
-import path5 from "node:path";
+import path7 from "node:path";
 import { describePackage } from "@openfn/describe-package";
 import { getNameAndVersion as getNameAndVersion3 } from "@openfn/runtime";
 var RETRY_DURATION = 500;
 var RETRY_COUNT = 20;
 var TIMEOUT_MS = 1e3 * 60;
 var actualDocGen = (specifier) => describePackage(specifier, {});
-var ensurePath = (filePath) => mkdirSync(path5.dirname(filePath), { recursive: true });
-var generatePlaceholder = (path9) => {
-  writeFileSync(path9, `{ "loading": true, "timestamp": ${Date.now()}}`);
+var ensurePath = (filePath) => mkdirSync(path7.dirname(filePath), { recursive: true });
+var generatePlaceholder = (path11) => {
+  writeFileSync(path11, `{ "loading": true, "timestamp": ${Date.now()}}`);
 };
 var finish = (logger, resultPath) => {
   logger.success("Done! Docs can be found at:\n");
-  logger.print(`  ${path5.resolve(resultPath)}`);
+  logger.print(`  ${path7.resolve(resultPath)}`);
 };
-var generateDocs = async (specifier, path9, docgen, logger) => {
+var generateDocs = async (specifier, path11, docgen, logger) => {
   const result = await docgen(specifier);
-  await writeFile4(path9, JSON.stringify(result, null, 2));
-  finish(logger, path9);
-  return path9;
+  await writeFile5(path11, JSON.stringify(result, null, 2));
+  finish(logger, path11);
+  return path11;
 };
-var waitForDocs = async (docs, path9, logger, retryDuration = RETRY_DURATION) => {
+var waitForDocs = async (docs, path11, logger, retryDuration = RETRY_DURATION) => {
   try {
     if (docs.hasOwnProperty("loading")) {
       logger.info("Docs are being loaded by another process. Waiting.");
@@ -1261,19 +1553,19 @@ var waitForDocs = async (docs, path9, logger, retryDuration = RETRY_DURATION) =>
             clearInterval(i);
             reject(new Error("Timed out waiting for docs to load"));
           }
-          const updated = JSON.parse(readFileSync(path9, "utf8"));
+          const updated = JSON.parse(readFileSync(path11, "utf8"));
           if (!updated.hasOwnProperty("loading")) {
             logger.info("Docs found!");
             clearInterval(i);
-            resolve(path9);
+            resolve(path11);
           }
           count++;
         }, retryDuration);
       });
     } else {
-      logger.info(`Docs already written to cache at ${path9}`);
-      finish(logger, path9);
-      return path9;
+      logger.info(`Docs already written to cache at ${path11}`);
+      finish(logger, path11);
+      return path11;
     }
   } catch (e) {
     logger.error("Existing doc JSON corrupt. Aborting");
@@ -1290,38 +1582,38 @@ var docgenHandler = (options, logger, docgen = actualDocGen, retryDuration = RET
     process.exit(9);
   }
   logger.success(`Generating docs for ${specifier}`);
-  const path9 = `${repoDir}/docs/${specifier}.json`;
-  ensurePath(path9);
-  const handleError = () => {
+  const path11 = `${repoDir}/docs/${specifier}.json`;
+  ensurePath(path11);
+  const handleError2 = () => {
     logger.info("Removing placeholder");
-    rmSync(path9);
+    rmSync(path11);
   };
   try {
-    const existing = readFileSync(path9, "utf8");
+    const existing = readFileSync(path11, "utf8");
     const json = JSON.parse(existing);
     if (json && json.timeout && Date.now() - json.timeout >= TIMEOUT_MS) {
       logger.info(`Expired placeholder found. Removing.`);
-      rmSync(path9);
+      rmSync(path11);
       throw new Error("TIMEOUT");
     }
-    return waitForDocs(json, path9, logger, retryDuration);
+    return waitForDocs(json, path11, logger, retryDuration);
   } catch (e) {
     if (e.message !== "TIMEOUT") {
-      logger.info(`Docs JSON not found at ${path9}`);
+      logger.info(`Docs JSON not found at ${path11}`);
     }
     logger.debug("Generating placeholder");
-    generatePlaceholder(path9);
-    return generateDocs(specifier, path9, docgen, logger).catch((e2) => {
+    generatePlaceholder(path11);
+    return generateDocs(specifier, path11, docgen, logger).catch((e2) => {
       logger.error("Error generating documentation");
       logger.error(e2);
-      handleError();
+      handleError2();
     });
   }
 };
-var handler_default6 = docgenHandler;
+var handler_default7 = docgenHandler;
 
 // src/docs/handler.ts
-import { readFile as readFile3 } from "node:fs/promises";
+import { readFile as readFile4 } from "node:fs/promises";
 import c from "chalk";
 import { getNameAndVersion as getNameAndVersion4, getLatestVersion } from "@openfn/runtime";
 var describeFn = (adaptorName, fn) => [
@@ -1362,7 +1654,7 @@ var docsHandler = async (options, logger) => {
     logger.success(`Showing docs for ${adaptorName} v${version}`);
   }
   logger.info("Generating/loading documentation...");
-  const path9 = await handler_default6(
+  const path11 = await handler_default7(
     {
       specifier: `${name}@${version}`,
       repoDir
@@ -1371,8 +1663,8 @@ var docsHandler = async (options, logger) => {
     createNullLogger()
   );
   let didError = false;
-  if (path9) {
-    const source = await readFile3(path9, "utf8");
+  if (path11) {
+    const source = await readFile4(path11, "utf8");
     const data = JSON.parse(source);
     let desc;
     if (operation) {
@@ -1405,13 +1697,13 @@ var docsHandler = async (options, logger) => {
     logger.error("Not found");
   }
 };
-var handler_default7 = docsHandler;
+var handler_default8 = docsHandler;
 
 // src/metadata/cache.ts
 import { createHash } from "node:crypto";
 import { readFileSync as readFileSync2 } from "node:fs";
-import path6 from "node:path";
-import { writeFile as writeFile5, mkdir as mkdir2 } from "node:fs/promises";
+import path8 from "node:path";
+import { writeFile as writeFile6, mkdir as mkdir2 } from "node:fs/promises";
 var getPath = (repoDir, key) => `${repoDir}/meta/${key}.json`;
 var sortKeys = (obj) => {
   const newObj = {};
@@ -1430,7 +1722,7 @@ var generateKey = (config, adaptor) => {
   const key = `${JSON.stringify(sorted)}-${adaptor}}`;
   return createHash("sha256").update(key).digest("hex");
 };
-var get = (repoPath, key) => {
+var get2 = (repoPath, key) => {
   try {
     const data = readFileSync2(getPath(repoPath, key), "utf8");
     const json = JSON.parse(data);
@@ -1439,12 +1731,12 @@ var get = (repoPath, key) => {
     return null;
   }
 };
-var set = async (repoPath, key, data) => {
+var set2 = async (repoPath, key, data) => {
   const fullPath = getPath(repoPath, key);
-  await mkdir2(path6.dirname(fullPath), { recursive: true });
-  await writeFile5(fullPath, JSON.stringify(data));
+  await mkdir2(path8.dirname(fullPath), { recursive: true });
+  await writeFile6(fullPath, JSON.stringify(data));
 };
-var cache_default = { get, set, generateKey, getPath, sortKeys };
+var cache_default = { get: get2, set: set2, generateKey, getPath, sortKeys };
 
 // src/metadata/handler.ts
 import { getModuleEntryPoint } from "@openfn/runtime";
@@ -1526,10 +1818,10 @@ var metadataHandler = async (options, logger) => {
     process.exit(1);
   }
 };
-var handler_default8 = metadataHandler;
+var handler_default9 = metadataHandler;
 
 // src/pull/handler.ts
-import path7 from "path";
+import path9 from "path";
 import fs4 from "node:fs/promises";
 import {
   getConfig as getConfig2,
@@ -1591,7 +1883,7 @@ async function pullHandler(options, logger) {
       process.exitCode = 1;
       process.exit(1);
     }
-    const resolvedPath = path7.resolve(config.specPath);
+    const resolvedPath = path9.resolve(config.specPath);
     logger.debug("reading spec from", resolvedPath);
     const updatedSpec = await syncRemoteSpec(
       await res.text(),
@@ -1600,7 +1892,7 @@ async function pullHandler(options, logger) {
       logger
     );
     await fs4.writeFile(
-      path7.resolve(config.statePath),
+      path9.resolve(config.statePath),
       JSON.stringify(state, null, 2)
     );
     await fs4.writeFile(resolvedPath, updatedSpec);
@@ -1630,11 +1922,11 @@ function mergeOverrides2(config, options) {
 function pickFirst2(...args) {
   return args.find((arg) => arg !== void 0 && arg !== null);
 }
-var handler_default9 = pullHandler;
+var handler_default10 = pullHandler;
 
 // src/util/print-versions.ts
 import { readFileSync as readFileSync3 } from "node:fs";
-import path8 from "node:path";
+import path10 from "node:path";
 import url from "node:url";
 import { getNameAndVersion as getNameAndVersion5 } from "@openfn/runtime";
 import { mainSymbols } from "figures";
@@ -1646,7 +1938,7 @@ var { triangleRightSmall: t } = mainSymbols;
 var loadVersionFromPath = (adaptorPath) => {
   try {
     const pkg = JSON.parse(
-      readFileSync3(path8.resolve(adaptorPath, "package.json"), "utf8")
+      readFileSync3(path10.resolve(adaptorPath, "package.json"), "utf8")
     );
     return pkg.version;
   } catch (e) {
@@ -1681,7 +1973,7 @@ var printVersions = async (logger, options = {}, includeComponents = false) => {
     ...[NODE, CLI2, RUNTIME2, COMPILER2, longestAdaptorName].map((s) => s.length)
   );
   const prefix = (str) => `         ${t} ${str.padEnd(longest + 4, " ")}`;
-  const dirname = path8.dirname(url.fileURLToPath(import.meta.url));
+  const dirname = path10.dirname(url.fileURLToPath(import.meta.url));
   const pkg = JSON.parse(readFileSync3(`${dirname}/../../package.json`, "utf8"));
   const { version, dependencies } = pkg;
   const compilerVersion = dependencies["@openfn/compiler"];
@@ -1728,12 +2020,15 @@ var handlers = {
   apollo: handler_default,
   execute: handler_default2,
   compile: handler_default3,
-  test: handler_default4,
-  deploy: handler_default5,
-  docgen: handler_default6,
-  docs: handler_default7,
-  metadata: handler_default8,
-  pull: handler_default9,
+  test: handler_default5,
+  deploy: handler_default6,
+  docgen: handler_default7,
+  docs: handler_default8,
+  metadata: handler_default9,
+  pull: handler_default10,
+  ["collections-get"]: handler_default4.get,
+  ["collections-set"]: handler_default4.set,
+  ["collections-remove"]: handler_default4.remove,
   ["repo-clean"]: clean,
   ["repo-install"]: install,
   ["repo-pwd"]: pwd,
@@ -1762,14 +2057,6 @@ var parse = async (options, log) => {
       logger
     );
   }
-  if (!/^(pull|deploy|test|version|apollo)$/.test(options.command) && !options.repoDir) {
-    logger.warn(
-      "WARNING: no repo module dir found! Using the default (/tmp/repo)"
-    );
-    logger.warn(
-      "You should set OPENFN_REPO_DIR or pass --repoDir=some/path in to the CLI"
-    );
-  }
   const handler = handlers[options.command];
   if (!handler) {
     logger.error(`Unrecognised command: ${options.command}`);
@@ -1782,6 +2069,11 @@ var parse = async (options, log) => {
       process.exitCode = e.exitCode || 1;
     }
     if (e.handled) {
+    } else if (e.abort) {
+      try {
+        abort_default(logger, e.reason, e.error, e.help);
+      } catch (e2) {
+      }
     } else {
       logger.break();
       logger.error("Command failed!");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/cli",
-  "version": "1.8.11",
+  "version": "1.9.0",
   "description": "CLI devtools for the openfn toolchain.",
   "engines": {
     "node": ">=18",
@@ -24,6 +24,7 @@
   "author": "Open Function Group <admin@openfn.org>",
   "license": "ISC",
   "devDependencies": {
+    "@openfn/language-collections": "^0.6.2",
     "@openfn/language-common": "2.0.0-rc3",
     "@types/mock-fs": "^4.13.1",
     "@types/node": "^18.15.13",
@@ -45,13 +46,14 @@
     "figures": "^5.0.0",
     "rimraf": "^3.0.2",
     "treeify": "^1.1.0",
+    "undici": "^7.1.0",
     "ws": "^8.18.0",
     "yargs": "^17.7.2",
-    "@openfn/compiler": "0.4.1",
-    "@openfn/logger": "1.0.2",
-    "@openfn/runtime": "1.5.3",
+    "@openfn/compiler": "0.4.2",
     "@openfn/deploy": "0.8.0",
-    "@openfn/describe-package": "0.1.3"
+    "@openfn/describe-package": "0.1.3",
+    "@openfn/runtime": "1.5.3",
+    "@openfn/logger": "1.0.2"
   },
   "files": [
     "dist",