@openfn/language-collections 1.0.0-next-13f27ea6

package/dist/index.js

@@ -0,0 +1,318 @@
+var __defProp = Object.defineProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+
+// src/collections.js
+var collections_exports = {};
+__export(collections_exports, {
+  createMockServer: () => createServer,
+  each: () => each,
+  expandQuery: () => expandQuery,
+  get: () => get,
+  remove: () => remove,
+  request: () => request,
+  set: () => set,
+  setMockClient: () => setMockClient,
+  streamResponse: () => streamResponse
+});
+import nodepath from "path";
+import undici from "undici";
+import { throwError, expandReferences } from "@openfn/language-common/util";
+import chain from "stream-chain";
+import parser from "stream-json";
+import Pick from "stream-json/filters/Pick";
+import streamArray from "stream-json/streamers/StreamArray";
+
+// src/mock.js
+import { MockAgent } from "undici";
+var COLLECTION_NOT_FOUND = "COLLECTION_NOT_FOUND";
+var INVALID_AUTH = "INVALID_AUTH";
+function API() {
+  let collections = {};
+  const createCollection = (name, values = {}) => {
+    collections[name] = values;
+  };
+  const reset = () => {
+    collections = api.collections = {};
+  };
+  const upsert = (name, key, value) => {
+    if (!(name in collections)) {
+      throw new Error(COLLECTION_NOT_FOUND);
+    }
+    collections[name][key] = value;
+  };
+  const fetch = (name, key, query) => {
+    if (!(name in collections)) {
+      throw new Error(COLLECTION_NOT_FOUND);
+    }
+    const col = collections[name];
+    const results = [];
+    const regex = new RegExp(key.replace("*", "(.*)"));
+    for (const key2 in col) {
+      if (regex.test(key2)) {
+        results.push({
+          key: key2,
+          value: col[key2]
+        });
+      }
+    }
+    return { results };
+  };
+  const byKey = (name, key) => {
+    return collections[name][key];
+  };
+  const remove2 = (name, key) => {
+    if (!(name in collections)) {
+      throw new Error(COLLECTION_NOT_FOUND);
+    }
+    const col = collections[name];
+    delete col[key];
+  };
+  const api = {
+    collections,
+    createCollection,
+    reset,
+    upsert,
+    fetch,
+    remove: remove2,
+    byKey
+  };
+  return api;
+}
+var parsePath = (path) => {
+  let [_collections, name, key] = path.split("/");
+  return { name, key };
+};
+var assertAuth = (req) => {
+  const auth = req.headers.Authorization;
+  if (!auth) {
+    throw new Error(INVALID_AUTH);
+  }
+};
+function createServer(url = "https://app.openfn.org") {
+  const agent = new MockAgent();
+  agent.disableNetConnect();
+  const mockPool = agent.get(url);
+  const api = new API();
+  const get2 = (req) => {
+    try {
+      assertAuth(req);
+    } catch (e) {
+      return { statusCode: 403 };
+    }
+    try {
+      let { name, key } = parsePath(req.path);
+      if (!key) {
+        key = "*";
+      }
+      const body = api.fetch(name, key);
+      return {
+        statusCode: 200,
+        data: JSON.stringify(body),
+        responseOptions: {
+          headers: { "Content-Type": "application/json" }
+        }
+      };
+    } catch (e) {
+      if (e.message === COLLECTION_NOT_FOUND) {
+        return { statusCode: 404 };
+      }
+    }
+    return { statusCode: 500 };
+  };
+  const post = (req) => {
+    try {
+      assertAuth(req);
+    } catch (e) {
+      return { statusCode: 403 };
+    }
+    try {
+      const { name, key } = parsePath(req.path);
+      const body = JSON.parse(req.body);
+      for (const { key: key2, value } of body) {
+        api.upsert(name, key2, value);
+      }
+      return { statusCode: 200 };
+    } catch (e) {
+      if (e.message === COLLECTION_NOT_FOUND) {
+        return { statusCode: 404 };
+      }
+    }
+  };
+  const remove2 = (req) => {
+    try {
+      assertAuth(req);
+    } catch (e) {
+      return { statusCode: 403 };
+    }
+    try {
+      const { name, key } = parsePath(req.path);
+      api.remove(name, key);
+      return { statusCode: 200 };
+    } catch (e) {
+      if (e.message === COLLECTION_NOT_FOUND) {
+        return { statusCode: 404 };
+      }
+    }
+  };
+  mockPool.intercept({ method: "get", path: /collections\/./ }).reply(get2).persist();
+  mockPool.intercept({ method: "post", path: /collections\/./ }).reply(post).persist();
+  mockPool.intercept({ method: "delete", path: /collections\/./ }).reply(remove2).persist();
+  return {
+    api,
+    request: ({ method, path, data, ...rest }) => {
+      const opts = {
+        method,
+        path,
+        origin: url,
+        headers: {
+          Authorization: `Bearer abc`
+        },
+        ...rest
+      };
+      if (data) {
+        Object.assign(opts.headers, {
+          "content-type": "application/json"
+        });
+        opts.body = JSON.stringify(data);
+      }
+      return mockPool.request(opts);
+    }
+  };
+}
+
+// src/collections.js
+var client;
+var getClient = (state) => {
+  var _a;
+  if (!client) {
+    const baseUrl = ((_a = state.configuration) == null ? void 0 : _a.collections_endpoint) ?? "https://app.openfn.org";
+    client = new undici.client(baseUrl);
+  }
+  return client;
+};
+var setMockClient = (mockClient) => {
+  client = mockClient;
+};
+function get(name, query = {}) {
+  return async (state) => {
+    const [resolvedName, resolvedQuery] = expandReferences(state, name, query);
+    const { key } = expandQuery(resolvedQuery);
+    const response = await request(
+      state,
+      getClient(state),
+      `${resolvedName}/${key}`
+    );
+    let data;
+    if (!key.match(/\*/) || Object.keys(resolvedQuery).length === 0) {
+      [data] = (await response.body.json()).results;
+      console.log(`Fetched "${key}" from collection "${name}"`);
+    } else {
+      data = [];
+      console.log(`Downloading data from collection "${name}"...`);
+      await streamResponse(response, (item) => {
+        data.push(item);
+      });
+      console.log(`Fetched "${data.length}" values from collection "${name}"`);
+    }
+    state.data = data;
+    return state;
+  };
+}
+function set(name, keyGen, values) {
+  return async (state) => {
+    const [resolvedName, resolvedValues] = expandReferences(
+      state,
+      name,
+      values
+    );
+    const dataArray = Array.isArray(resolvedValues) ? resolvedValues : [resolvedValues];
+    const keyGenFn = typeof keyGen === "string" ? () => keyGen : keyGen;
+    const pairs = dataArray.map((value) => ({ key: keyGenFn(value), value }));
+    console.log(`Setting ${pairs.length} values in collection "${name}"`);
+    const response = await request(state, getClient(state), resolvedName, {
+      method: "POST",
+      body: JSON.stringify(pairs),
+      heeaders: {
+        "content-type": "application/json"
+      }
+    });
+    return state;
+  };
+}
+function remove(name, query = {}, options = {}) {
+  return async (state) => {
+    const [resolvedName, resolvedQuery] = expandReferences(state, name, query);
+    const { key } = expandQuery(resolvedQuery);
+    const response = await request(
+      state,
+      getClient(state),
+      `${resolvedName}/${key}`,
+      {
+        method: "DELETE"
+      }
+    );
+    return state;
+  };
+}
+function each(name, query = {}, callback = {}) {
+  return async (state) => {
+    const [resolvedName, resolvedQuery] = expandReferences(state, name, query);
+    const { key } = expandQuery(resolvedQuery);
+    const response = await request(
+      state,
+      getClient(state),
+      `${resolvedName}/${key}`
+    );
+    await streamResponse(response, async ({ key: key2, value }) => {
+      await callback(state, value, key2);
+    });
+    return state;
+  };
+}
+var streamResponse = async (response, onValue) => {
+  const pipeline = chain([
+    response.body,
+    parser(),
+    new Pick({ filter: "results" }),
+    new streamArray()
+  ]);
+  for await (const { key, value } of pipeline) {
+    await onValue(value);
+  }
+};
+var expandQuery = (query) => {
+  let key;
+  if (typeof query === "string") {
+    key = query;
+    return {
+      key
+    };
+  }
+  return query;
+};
+var request = (state, client2, path, options = {}) => {
+  if (!state.configuration.collections_token) {
+    throwError("INVALID_AUTH", {
+      description: "No access key provided for collection request",
+      fix: 'Ensure the "collections_token" value is set on state.configuration',
+      path
+    });
+  }
+  const headers = {
+    Authorization: `Bearer ${state.configuration.collections_token}`
+  };
+  Object.assign(headers, options == null ? void 0 : options.headers);
+  const { headers: _, ...optionsWithoutHeaders } = options;
+  return client2.request({
+    path: nodepath.join("collections", path),
+    headers,
+    method: "GET",
+    ...optionsWithoutHeaders
+  });
+};
+export {
+  collections_exports as collections
+};

package/package.json

@@ -0,0 +1,50 @@
+{
+  "name": "@openfn/language-collections",
+  "version": "1.0.0-next-13f27ea6",
+  "description": "OpenFn collections adaptor",
+  "type": "module",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./types/index.d.ts",
+      "require": "./dist/index.cjs"
+    },
+    "./package.json": "./package.json"
+  },
+  "author": "Open Function Group",
+  "license": "LGPLv3",
+  "files": [
+    "dist/",
+    "types/",
+    "ast.json",
+    "configuration-schema.json"
+  ],
+  "dependencies": {
+    "stream-chain": "^3.3.2",
+    "stream-json": "^1.8.0",
+    "undici": "^5.22.1",
+    "@openfn/language-common": "2.1.0"
+  },
+  "devDependencies": {
+    "assertion-error": "2.0.0",
+    "chai": "4.3.6",
+    "deep-eql": "4.1.1",
+    "esno": "^0.16.3",
+    "rimraf": "3.0.2"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/openfn/adaptors.git"
+  },
+  "types": "types/index.d.ts",
+  "main": "dist/index.cjs",
+  "scripts": {
+    "tmp": "node test.js",
+    "build": "pnpm clean && build-adaptor collections",
+    "test": "mocha --experimental-specifier-resolution=node --no-warnings **/*.test.js",
+    "test:watch": "mocha -w --experimental-specifier-resolution=node --no-warnings *.test.js",
+    "clean": "rimraf dist types docs",
+    "pack": "pnpm pack --pack-destination ../../dist",
+    "lint": "eslint src"
+  }
+}

package/types/collections.d.ts

@@ -0,0 +1,93 @@
+/**
+ * Query options. All dates should be parseable as ISO 8601 strings, see https://simple.wikipedia.org/wiki/ISO_8601
+ * @typedef {Object} QueryOptions
+ * @public
+ * @property {string} key - key or key pattern to match against. Patterns support wildcards,  eg `2024-01*`
+ * @property {string} createdBefore - matches values that were created before the start of the provided date
+ * @property {string} createdAfter - matches values that were created after the end of the provided date
+ * @property {string} updatedBefore - matches values that were updated before the start of the provided date
+ * @property {string} updatedAfter - matches values that were updated after the end of the provided date*
+ */
+/**
+ * Fetch one or more values from a collection.
+ * For large datasets, we recommend using each(), which streams data.
+ * You can pass a specific key as a string to only fetch one item, or pass a query
+ * with a key-pattern or a date filter.
+ * @public
+ * @function
+ * @param {string} name - The name of the collection to fetch from
+ * @param {string|QueryOptions} query - A string key or key pattern (with wildcards '*') to fetch, or a query object
+ * @state data - the downloaded values as an array unless a specific string was specified
+ * @example <caption>Get a specific value from a collection</caption>
+ * collections.get('my-collection', '556e0a62')
+ * @example <caption>Get a range of values from a collection with a key pattern</caption>
+ * collections.get('my-collection', '2024*')
+ * @example <caption>Get all values created since the end of January 2024</caption>
+ * collections.get('my-collection', { createdAfter: '202401')
+ */
+export function get(name: string, query?: string | QueryOptions): (state: any) => Promise<any>;
+/**
+ * Adds one or more values to a collection. If a key already exists, its value will
+ * be replaced by the new value.
+ * You can pass a string key and a single value, or a key generator function and an array of values.
+ * The function will be called for each value, passing each value as the first argument, and should return
+ * a string key.
+ * @public
+ * @function
+ * @param keygen - a function which generates a key for each value. Pass a string to set a static key for a single item.
+ * @param values - an array of values to set, or a single value.
+ * @example <caption>Set a number of values using each value's id property as a key</caption>
+ * collections.set('my-collection', (item) => item.id, $.data)
+ * @example <caption>Set a number of values, generating an id from a string template</caption>
+ * collections.set('my-collection', (item) => `${item.category}-${Date.now()}`, $.data)
+ * @example <caption>Set a single value with a static key</caption>
+ * collections.set('my-collection', 'city-codes', { NY: 'New York', LDN: 'London' }})
+ */
+export function set(name: any, keyGen: any, values: any): (state: any) => Promise<any>;
+/**
+ * Remove one or more values from a collection.
+ * You can pass a specific key as a string to only fetch one item, or pass a query
+ * with a key-pattern or a date filter.
+ * @public
+ * @function
+ * @param {string} name - The name of the collection to remove from
+ * @param {string|QueryOptions} query - A string key or key pattern (with wildcards '*') to remove, or a query object
+ * @example <caption>Remove a specific value from a collection</caption>
+ * collections.remove('my-collection', '556e0a62')
+ * @example <caption>Remove a range of values from a collection with a key pattern</caption>
+ * collections.remove('my-collection', '2024*')
+ * @example <caption>Remove all values created since the end of January 2024</caption>
+ * collections.remove('my-collection', { createdAfter: '202401')
+ */
+export function remove(name: string, query?: string | QueryOptions, options?: {}): (state: any) => Promise<any>;
+/**
+ * Iterate over all values in a collection which match the provided query.
+ * each() maintains a low memory footprint by streaming items individually.
+ * You can pass a string key-pattern as a query, or pass a query object.
+ * The callback function will be invoked for each value with three parameters:
+ * `state`, `value` and `key`.
+ * @public
+ * @function
+ * @param {string} name - The name of the collection to remove from
+ * @param {string|QueryOptions} query - A string key or key pattern (with wildcards '*') to remove, or a query object
+ * @param {function} callback - A callback invoked for each item `(state,  value, key) => void`
+ * @example <caption>Iterate over a range of values with wildcards</caption>
+ * collections.each('my-collection', 'record-2024*-appointment-*', (state, value, key) => {
+ *   state.cumulativeCost += value.cost;
+ * })
+ * @example <caption>Iterate over a range of values with date filters</caption>
+ * collections.each('my-collection', { updatedBefore: new Date().toString() }, (state, value, key) => {
+ *   state.cumulativeCost += value.cost;
+ * })
+ */
+export function each(name: string, query?: string | QueryOptions, callback?: Function): (state: any) => Promise<any>;
+export { createServer as createMockServer };
+export function setMockClient(mockClient: any): void;
+export function streamResponse(response: any, onValue: any): Promise<void>;
+export function expandQuery(query: any): any;
+export function request(state: any, client: any, path: any, options?: {}): any;
+/**
+ * Query options. All dates should be parseable as ISO 8601 strings, see https://simple.wikipedia.org/wiki/ISO_8601
+ */
+export type QueryOptions = any;
+import { createServer } from "./mock";

package/types/index.d.ts

@@ -0,0 +1 @@
+export * as collections from "./collections";

package/types/mock.d.ts

@@ -0,0 +1,23 @@
+export function API(): {
+    collections: {};
+    createCollection: (name: any, values?: {}) => void;
+    reset: () => void;
+    upsert: (name: any, key: any, value: any) => void;
+    fetch: (name: any, key: any, query: any) => {
+        results: {
+            key: string;
+            value: any;
+        }[];
+    };
+    remove: (name: any, key: any) => void;
+    byKey: (name: any, key: any) => any;
+};
+export function createServer(url?: string): {
+    api: any;
+    request: ({ method, path, data, ...rest }: {
+        [x: string]: any;
+        method: any;
+        path: any;
+        data: any;
+    }) => Promise<import("undici/types/dispatcher").default.ResponseData>;
+};