@openfn/language-kobotoolbox 3.0.5 → 4.1.0

package/ast.json

@@ -43,18 +43,33 @@
         "options"
       ],
       "docs": {
-        "description": "Get submissions for a specific form. Calls `/api/v2/assets/<formId>/data/`.",
+        "description": "Get submissions for a specific form. Calls `/api/v2/assets/<formId>/data/`",
         "tags": [
           {
             "title": "example",
-            "description": "getSubmissions('aXecHjmbATuF6iGFmvBLBX');",
+            "description": "getSubmissions('aXecHjmbATuF6iGFmvBLBX');.",
+            "caption": "Get submissions for a specific form"
+          },
+          {
+            "title": "example",
+            "description": "getSubmissions('aXecHjmbATuF6iGFmvBLBX', { limit: Infinity });",
             "caption": "Get all submissions for a specific form"
           },
           {
             "title": "example",
-            "description": "getSubmissions('aXecHjmbATuF6iGFmvBLBX', { query: { _submission_time:{ $gte: \"2022-06-12T21:54:20\" } } });",
+            "description": "getSubmissions('aXecHjmbATuF6iGFmvBLBX', { query: { _submission_time:{ $gte: \"2025-03-12T21:54:20\" } } });",
             "caption": "Get form submissions with a query"
           },
+          {
+            "title": "example",
+            "description": "getSubmissions('aXecHjmbATuF6iGFmvBLBX', { sort: { _submission_time: -1 } });",
+            "caption": "Get form submissions with sorting"
+          },
+          {
+            "title": "example",
+            "description": "getSubmissions('aXecHjmbATuF6iGFmvBLBX', { start: 10 });",
+            "caption": "Get form submissions with specific start index"
+          },
           {
             "title": "function",
             "description": null,
@@ -76,7 +91,7 @@
           },
           {
             "title": "param",
-            "description": "Optional query params for the request",
+            "description": "Options to control the request",
             "type": {
               "type": "OptionalType",
               "expression": {
@@ -87,6 +102,69 @@
             "name": "options",
             "default": "{}"
           },
+          {
+            "title": "param",
+            "description": "Field and direction to sort submissions by.",
+            "type": {
+              "type": "OptionalType",
+              "expression": {
+                "type": "NameExpression",
+                "name": "object"
+              }
+            },
+            "name": "options.sort"
+          },
+          {
+            "title": "param",
+            "description": "Query options to filter the submissions. See query operators {@link http://docs.mongodb.org/manual/reference/operator/query/.}",
+            "type": {
+              "type": "OptionalType",
+              "expression": {
+                "type": "NameExpression",
+                "name": "object"
+              }
+            },
+            "name": "options.query"
+          },
+          {
+            "title": "param",
+            "description": "The index of the first submission to return.",
+            "type": {
+              "type": "OptionalType",
+              "expression": {
+                "type": "NameExpression",
+                "name": "number"
+              }
+            },
+            "name": "options.start",
+            "default": "0"
+          },
+          {
+            "title": "param",
+            "description": "Maximum number of submissions to fetch. Pass Infinity to disable the limit and download all submissions",
+            "type": {
+              "type": "OptionalType",
+              "expression": {
+                "type": "NameExpression",
+                "name": "number"
+              }
+            },
+            "name": "options.limit",
+            "default": "30000"
+          },
+          {
+            "title": "param",
+            "description": "Limits the size of each page of submissions. Maximum value is 30000.",
+            "type": {
+              "type": "OptionalType",
+              "expression": {
+                "type": "NameExpression",
+                "name": "number"
+              }
+            },
+            "name": "options.pageSize",
+            "default": "10000"
+          },
           {
             "title": "state",
             "description": "data - an array of submission objects"
@@ -101,7 +179,7 @@
           }
         ]
       },
-      "valid": true
+      "valid": false
     },
     {
       "name": "getDeploymentInfo",

package/dist/index.cjs

@@ -58,7 +58,6 @@ __export(Adaptor_exports, {
   getForms: () => getForms,
   getSubmissions: () => getSubmissions,
   group: () => import_language_common3.group,
-  http: () => import_language_common3.http,
   lastReferenceValue: () => import_language_common3.lastReferenceValue,
   merge: () => import_language_common3.merge,
   sourceValue: () => import_language_common3.sourceValue
@@ -66,36 +65,29 @@ __export(Adaptor_exports, {
 var import_language_common2 = require("@openfn/language-common");
 var import_util2 = require("@openfn/language-common/util");
 
-// src/Utils.js
+// src/util.js
 var import_language_common = require("@openfn/language-common");
 var import_util = require("@openfn/language-common/util");
-var prepareNextState = (state, response) => {
+var DEFAULT_LIMIT = 3e4;
+var DEFAULT_PAGE_SIZE = 1e4;
+function prepareNextState(state, response) {
   const { body, ...responseWithoutBody } = response;
   return {
-    ...(0, import_language_common.composeNextState)(state, response.body),
+    ...(0, import_language_common.composeNextState)(state, body),
     response: responseWithoutBody
   };
-};
-async function request(state, method, path, opts) {
-  const { baseURL, apiVersion, username, password } = state.configuration;
-  let baseUrl;
-  if (!state.configuration.baseUrl && baseURL) {
-    baseUrl = baseURL;
-    console.warn(
-      "No baseUrl found in state.configuration. baseURL will be used instead, but this will be deprecated in the future."
-    );
-  } else {
-    baseUrl = state.configuration.baseUrl;
-  }
+}
+async function request(state, method, path, opts = {}) {
+  const { baseUrl, apiVersion, username, password } = state.configuration;
   const {
     data = {},
     query = {},
     headers = {},
     parseAs = "json",
-    paginate = false
+    maxRedirections
   } = opts;
+  const requestPath = `/api/${apiVersion}/${path}/`;
   const authHeaders = (0, import_util.makeBasicAuthHeader)(username, password);
-  let start, limit;
   const options = {
     body: data,
     headers: {
@@ -107,34 +99,60 @@ async function request(state, method, path, opts) {
       format: "json",
       ...query
     },
+    maxRedirections,
     parseAs,
-    baseUrl: `${baseUrl}/api/${apiVersion}`
+    baseUrl
   };
-  if (paginate) {
-    const results = [];
-    do {
-      const response = await (0, import_util.request)(method, path, options).then(
-        import_util.logResponse
+  return (0, import_util.request)(method, requestPath, options).then(import_util.logResponse);
+}
+async function requestWithPagination(state, path, options = {}) {
+  var _a, _b, _c, _d;
+  const results = [];
+  let { pageSize = DEFAULT_PAGE_SIZE, start, limit, ...otherOptions } = options;
+  const maxResults = limit ?? DEFAULT_LIMIT;
+  let isFirstRequest = true;
+  let requestOptions = { query: { start, limit: pageSize }, ...otherOptions };
+  let shouldFetchMoreContent = false;
+  const didUserPassLimit = Boolean(limit);
+  do {
+    requestOptions.query ?? (requestOptions.query = {});
+    if (start) {
+      requestOptions.query.start = start;
+    }
+    if (didUserPassLimit || !isFirstRequest) {
+      requestOptions.query.limit = Math.min(
+        pageSize || maxResults,
+        maxResults - results.length
       );
+    } else if (!isNaN(pageSize)) {
+      requestOptions.query.limit = pageSize;
+    }
+    const response = await request(state, "GET", path, requestOptions);
+    if ((_a = response.body) == null ? void 0 : _a.results) {
       results.push(...response.body.results);
-      if (response.body.next) {
-        const nextUrl = new URL(response.body.next);
-        const startDigit = nextUrl.searchParams.get("start") !== null ? nextUrl.searchParams.get("start") : 0;
-        start = Number(startDigit);
-        limit = nextUrl.searchParams.get("limit");
-        options.query = {
-          ...options.query,
-          start,
-          limit
-        };
-      } else {
-        break;
+      if ((_b = response == null ? void 0 : response.body) == null ? void 0 : _b.next) {
+        const nextUrl = new URL((_c = response == null ? void 0 : response.body) == null ? void 0 : _c.next);
+        start = nextUrl.searchParams.get("start");
       }
-    } while (true);
-    return { results };
-  } else {
-    return (0, import_util.request)(method, path, options).then(import_util.logResponse);
-  }
+    } else {
+      results.push(response.body);
+    }
+    if (isFirstRequest && !pageSize) {
+      pageSize = results.length;
+    }
+    isFirstRequest = false;
+    const hasMoreContent = (_d = response.body.next) == null ? void 0 : _d.includes("start=");
+    if (hasMoreContent && didUserPassLimit && results.length === maxResults) {
+      console.warn(
+        `Warning: The default maximum number of items has been reached (${maxResults}), but more items are available on the server. To download all available items, make another request with start=${maxResults + 1} or set max to Infinity`
+      );
+    }
+    shouldFetchMoreContent = results.length < maxResults && hasMoreContent;
+  } while (shouldFetchMoreContent);
+  return results;
+}
+function maybeStringify(query) {
+  return typeof query === "string" ? query : JSON.stringify(query);
 }
 
 // src/Adaptor.js
@@ -153,34 +171,43 @@ function execute(...operations) {
 }
 function getForms() {
   return async (state) => {
-    const url = `/assets/?asset_type=survey`;
-    const response = await request(state, "GET", url, {});
-    console.log("\u2713", response.body.results.length, "forms fetched.");
+    var _a;
+    const response = await request(state, "GET", "assets", {
+      query: { asset_type: "survey" }
+    });
+    console.log("\u2713", (_a = response.body.results) == null ? void 0 : _a.length, "forms fetched.");
     return prepareNextState(state, response);
   };
 }
-function getSubmissions(formId, options = {}) {
+function getSubmissions(formId, options) {
   return async (state) => {
-    const [resolvedFormId, resolvedOptions] = (0, import_util2.expandReferences)(
+    const [resolvedFormId, resolvedOptions = {}] = (0, import_util2.expandReferences)(
       state,
       formId,
       options
     );
-    const url = `/assets/${resolvedFormId}/data/`;
-    const query = {};
-    if (resolvedOptions.query) {
-      if (typeof resolvedOptions.query == "string") {
-        query.query = resolvedOptions.query;
-      } else {
-        query.query = JSON.stringify(resolvedOptions.query);
-      }
+    const { query, limit, pageSize, sort, start } = resolvedOptions;
+    const path = `/assets/${resolvedFormId}/data/`;
+    const qs = {};
+    if (query) {
+      qs.query = maybeStringify(query);
     }
-    const { results } = await request(state, "GET", url, {
-      paginate: true,
-      query
-    });
-    console.log("\u2713", results.length, "submissions fetched.");
-    return (0, import_language_common2.composeNextState)(state, results);
+    if (sort) {
+      qs.sort = maybeStringify(sort);
+    }
+    const requestOptions = {
+      query: { ...qs },
+      start,
+      limit,
+      pageSize
+    };
+    const result = await requestWithPagination(
+      state,
+      path,
+      requestOptions
+    );
+    console.log("\u2713", result == null ? void 0 : result.length, "submissions fetched.");
+    return (0, import_language_common2.composeNextState)(state, result);
   };
 }
 function getDeploymentInfo(formId) {
@@ -198,12 +225,25 @@ var http_exports = {};
 __export(http_exports, {
   get: () => get,
   post: () => post,
-  put: () => put
+  put: () => put,
+  request: () => request2
 });
 var import_util3 = require("@openfn/language-common/util");
-function get(path, options = {}) {
+function request2(method, path, options = {}) {
+  return async (state) => {
+    const [resolvedMethod, resolvedPath, resolvedOptions = {}] = (0, import_util3.expandReferences)(state, method, path, options);
+    const response = await request(
+      state,
+      resolvedMethod,
+      resolvedPath,
+      resolvedOptions
+    );
+    return prepareNextState(state, response);
+  };
+}
+function get(path, options) {
   return async (state) => {
-    const [resolvedPath, resolvedOptions] = (0, import_util3.expandReferences)(
+    const [resolvedPath, resolvedOptions = {}] = (0, import_util3.expandReferences)(
       state,
       path,
       options
@@ -217,9 +257,9 @@ function get(path, options = {}) {
     return prepareNextState(state, response);
   };
 }
-function post(path, data, options = {}) {
+function post(path, data, options) {
   return async (state) => {
-    const [resolvedPath, resolvedData, resolvedOptions] = (0, import_util3.expandReferences)(
+    const [resolvedPath, resolvedData, resolvedOptions = {}] = (0, import_util3.expandReferences)(
       state,
       path,
       data,
@@ -238,9 +278,9 @@ function post(path, data, options = {}) {
     return prepareNextState(state, response);
   };
 }
-function put(path, data, options = {}) {
+function put(path, data, options) {
   return async (state) => {
-    const [resolvedPath, resolvedData, resolvedOptions] = (0, import_util3.expandReferences)(
+    const [resolvedPath, resolvedData, resolvedOptions = {}] = (0, import_util3.expandReferences)(
       state,
       path,
       data,

package/dist/index.js

@@ -21,7 +21,6 @@ __export(Adaptor_exports, {
   getForms: () => getForms,
   getSubmissions: () => getSubmissions,
   group: () => group,
-  http: () => http,
   lastReferenceValue: () => lastReferenceValue,
   merge: () => merge,
   sourceValue: () => sourceValue
@@ -32,40 +31,33 @@ import {
 } from "@openfn/language-common";
 import { expandReferences } from "@openfn/language-common/util";
 
-// src/Utils.js
+// src/util.js
 import { composeNextState } from "@openfn/language-common";
 import {
   request as commonRequest,
   makeBasicAuthHeader,
   logResponse
 } from "@openfn/language-common/util";
-var prepareNextState = (state, response) => {
+var DEFAULT_LIMIT = 3e4;
+var DEFAULT_PAGE_SIZE = 1e4;
+function prepareNextState(state, response) {
   const { body, ...responseWithoutBody } = response;
   return {
-    ...composeNextState(state, response.body),
+    ...composeNextState(state, body),
     response: responseWithoutBody
   };
-};
-async function request(state, method, path, opts) {
-  const { baseURL, apiVersion, username, password } = state.configuration;
-  let baseUrl;
-  if (!state.configuration.baseUrl && baseURL) {
-    baseUrl = baseURL;
-    console.warn(
-      "No baseUrl found in state.configuration. baseURL will be used instead, but this will be deprecated in the future."
-    );
-  } else {
-    baseUrl = state.configuration.baseUrl;
-  }
+}
+async function request(state, method, path, opts = {}) {
+  const { baseUrl, apiVersion, username, password } = state.configuration;
   const {
     data = {},
     query = {},
     headers = {},
     parseAs = "json",
-    paginate = false
+    maxRedirections
   } = opts;
+  const requestPath = `/api/${apiVersion}/${path}/`;
   const authHeaders = makeBasicAuthHeader(username, password);
-  let start, limit;
   const options = {
     body: data,
     headers: {
@@ -77,34 +69,60 @@ async function request(state, method, path, opts) {
       format: "json",
       ...query
     },
+    maxRedirections,
     parseAs,
-    baseUrl: `${baseUrl}/api/${apiVersion}`
+    baseUrl
   };
-  if (paginate) {
-    const results = [];
-    do {
-      const response = await commonRequest(method, path, options).then(
-        logResponse
+  return commonRequest(method, requestPath, options).then(logResponse);
+}
+async function requestWithPagination(state, path, options = {}) {
+  var _a, _b, _c, _d;
+  const results = [];
+  let { pageSize = DEFAULT_PAGE_SIZE, start, limit, ...otherOptions } = options;
+  const maxResults = limit ?? DEFAULT_LIMIT;
+  let isFirstRequest = true;
+  let requestOptions = { query: { start, limit: pageSize }, ...otherOptions };
+  let shouldFetchMoreContent = false;
+  const didUserPassLimit = Boolean(limit);
+  do {
+    requestOptions.query ?? (requestOptions.query = {});
+    if (start) {
+      requestOptions.query.start = start;
+    }
+    if (didUserPassLimit || !isFirstRequest) {
+      requestOptions.query.limit = Math.min(
+        pageSize || maxResults,
+        maxResults - results.length
       );
+    } else if (!isNaN(pageSize)) {
+      requestOptions.query.limit = pageSize;
+    }
+    const response = await request(state, "GET", path, requestOptions);
+    if ((_a = response.body) == null ? void 0 : _a.results) {
       results.push(...response.body.results);
-      if (response.body.next) {
-        const nextUrl = new URL(response.body.next);
-        const startDigit = nextUrl.searchParams.get("start") !== null ? nextUrl.searchParams.get("start") : 0;
-        start = Number(startDigit);
-        limit = nextUrl.searchParams.get("limit");
-        options.query = {
-          ...options.query,
-          start,
-          limit
-        };
-      } else {
-        break;
+      if ((_b = response == null ? void 0 : response.body) == null ? void 0 : _b.next) {
+        const nextUrl = new URL((_c = response == null ? void 0 : response.body) == null ? void 0 : _c.next);
+        start = nextUrl.searchParams.get("start");
       }
-    } while (true);
-    return { results };
-  } else {
-    return commonRequest(method, path, options).then(logResponse);
-  }
+    } else {
+      results.push(response.body);
+    }
+    if (isFirstRequest && !pageSize) {
+      pageSize = results.length;
+    }
+    isFirstRequest = false;
+    const hasMoreContent = (_d = response.body.next) == null ? void 0 : _d.includes("start=");
+    if (hasMoreContent && didUserPassLimit && results.length === maxResults) {
+      console.warn(
+        `Warning: The default maximum number of items has been reached (${maxResults}), but more items are available on the server. To download all available items, make another request with start=${maxResults + 1} or set max to Infinity`
+      );
+    }
+    shouldFetchMoreContent = results.length < maxResults && hasMoreContent;
+  } while (shouldFetchMoreContent);
+  return results;
+}
+function maybeStringify(query) {
+  return typeof query === "string" ? query : JSON.stringify(query);
 }
 
 // src/Adaptor.js
@@ -118,7 +136,6 @@ import {
   fields,
   fn,
   fnIf,
-  http,
   group,
   lastReferenceValue,
   merge,
@@ -138,34 +155,43 @@ function execute(...operations) {
 }
 function getForms() {
   return async (state) => {
-    const url = `/assets/?asset_type=survey`;
-    const response = await request(state, "GET", url, {});
-    console.log("\u2713", response.body.results.length, "forms fetched.");
+    var _a;
+    const response = await request(state, "GET", "assets", {
+      query: { asset_type: "survey" }
+    });
+    console.log("\u2713", (_a = response.body.results) == null ? void 0 : _a.length, "forms fetched.");
     return prepareNextState(state, response);
   };
 }
-function getSubmissions(formId, options = {}) {
+function getSubmissions(formId, options) {
   return async (state) => {
-    const [resolvedFormId, resolvedOptions] = expandReferences(
+    const [resolvedFormId, resolvedOptions = {}] = expandReferences(
       state,
       formId,
       options
     );
-    const url = `/assets/${resolvedFormId}/data/`;
-    const query = {};
-    if (resolvedOptions.query) {
-      if (typeof resolvedOptions.query == "string") {
-        query.query = resolvedOptions.query;
-      } else {
-        query.query = JSON.stringify(resolvedOptions.query);
-      }
+    const { query, limit, pageSize, sort, start } = resolvedOptions;
+    const path = `/assets/${resolvedFormId}/data/`;
+    const qs = {};
+    if (query) {
+      qs.query = maybeStringify(query);
     }
-    const { results } = await request(state, "GET", url, {
-      paginate: true,
-      query
-    });
-    console.log("\u2713", results.length, "submissions fetched.");
-    return composeNextState2(state, results);
+    if (sort) {
+      qs.sort = maybeStringify(sort);
+    }
+    const requestOptions = {
+      query: { ...qs },
+      start,
+      limit,
+      pageSize
+    };
+    const result = await requestWithPagination(
+      state,
+      path,
+      requestOptions
+    );
+    console.log("\u2713", result == null ? void 0 : result.length, "submissions fetched.");
+    return composeNextState2(state, result);
   };
 }
 function getDeploymentInfo(formId) {
@@ -183,12 +209,25 @@ var http_exports = {};
 __export(http_exports, {
   get: () => get,
   post: () => post,
-  put: () => put
+  put: () => put,
+  request: () => request2
 });
 import { expandReferences as expandReferences2 } from "@openfn/language-common/util";
-function get(path, options = {}) {
+function request2(method, path, options = {}) {
+  return async (state) => {
+    const [resolvedMethod, resolvedPath, resolvedOptions = {}] = expandReferences2(state, method, path, options);
+    const response = await request(
+      state,
+      resolvedMethod,
+      resolvedPath,
+      resolvedOptions
+    );
+    return prepareNextState(state, response);
+  };
+}
+function get(path, options) {
   return async (state) => {
-    const [resolvedPath, resolvedOptions] = expandReferences2(
+    const [resolvedPath, resolvedOptions = {}] = expandReferences2(
       state,
       path,
       options
@@ -202,9 +241,9 @@ function get(path, options = {}) {
     return prepareNextState(state, response);
   };
 }
-function post(path, data, options = {}) {
+function post(path, data, options) {
   return async (state) => {
-    const [resolvedPath, resolvedData, resolvedOptions] = expandReferences2(
+    const [resolvedPath, resolvedData, resolvedOptions = {}] = expandReferences2(
       state,
       path,
       data,
@@ -223,9 +262,9 @@ function post(path, data, options = {}) {
     return prepareNextState(state, response);
   };
 }
-function put(path, data, options = {}) {
+function put(path, data, options) {
   return async (state) => {
-    const [resolvedPath, resolvedData, resolvedOptions] = expandReferences2(
+    const [resolvedPath, resolvedData, resolvedOptions = {}] = expandReferences2(
       state,
       path,
       data,

package/package.json

@@ -1,7 +1,8 @@
 {
   "name": "@openfn/language-kobotoolbox",
-  "version": "3.0.5",
-  "description": "A Kobo Toolbox Language Pack for OpenFn",
+  "label": "KoboToolbox",
+  "version": "4.1.0",
+  "description": "A KoboToolbox Language Pack for OpenFn",
   "homepage": "https://docs.openfn.org",
   "repository": {
     "type": "git",
@@ -21,9 +22,8 @@
   },
   "devDependencies": {
     "assertion-error": "^1.0.1",
-    "chai": "^3.4.0",
+    "chai": "^5.2.0",
     "deep-eql": "^0.1.3",
-    "esno": "^0.16.3",
     "rimraf": "^3.0.2"
   },
   "type": "module",
@@ -38,8 +38,9 @@
   },
   "scripts": {
     "build": "pnpm clean && build-adaptor kobotoolbox",
-    "test": "mocha --experimental-specifier-resolution=node --no-warnings",
-    "test:watch": "mocha -w --experimental-specifier-resolution=node --no-warnings",
+    "test": "mocha --experimental-specifier-resolution=node --no-warnings --exclude test/integration.js --recursive",
+    "test:watch": "mocha -w --experimental-specifier-resolution=node --no-warnings --exclude test/integration.js --recursive",
+    "test:integration": "mocha --experimental-specifier-resolution=node --no-warnings test/integration.js",
     "clean": "rimraf dist types docs",
     "pack": "pnpm pack --pack-destination ../../dist",
     "lint": "eslint src"

package/types/Adaptor.d.ts

@@ -1,10 +1,3 @@
-/**
- * Options object
- * @typedef {Object} RequestOptions
- * @property {object} query - An object of query parameters to be encoded into the URL
- * @property {object} headers - An object of all request headers
- * @property {string} [parseAs='json'] - The response format to parse (e.g., 'json', 'text', or 'stream')
- */
 /**
  * Execute a sequence of operations.
  * Wraps `language-common/execute`, and prepends initial state for http.
@@ -29,19 +22,36 @@ export function execute(...operations: Operations): Operation;
  */
 export function getForms(): Operation;
 /**
- * Get submissions for a specific form. Calls `/api/v2/assets/<formId>/data/`.
+ * Get submissions for a specific form. Calls `/api/v2/assets/<formId>/data/`
+ * @example <caption>Get submissions for a specific form</caption>
+ * getSubmissions('aXecHjmbATuF6iGFmvBLBX');.
  * @example <caption>Get all submissions for a specific form</caption>
- * getSubmissions('aXecHjmbATuF6iGFmvBLBX');
+ * getSubmissions('aXecHjmbATuF6iGFmvBLBX', { limit: Infinity });
  * @example <caption>Get form submissions with a query</caption>
- * getSubmissions('aXecHjmbATuF6iGFmvBLBX', { query: { _submission_time:{ $gte: "2022-06-12T21:54:20" } } });
+ * getSubmissions('aXecHjmbATuF6iGFmvBLBX', { query: { _submission_time:{ $gte: "2025-03-12T21:54:20" } } });
+ * @example <caption>Get form submissions with sorting</caption>
+ * getSubmissions('aXecHjmbATuF6iGFmvBLBX', { sort: { _submission_time: -1 } });
+ * @example <caption>Get form submissions with specific start index</caption>
+ * getSubmissions('aXecHjmbATuF6iGFmvBLBX', { start: 10 });
  * @function
  * @public
  * @param {string} formId - Form Id to get the specific submissions
- * @param {object} [options={}] - Optional query params for the request
+ * @param {object} [options={}] - Options to control the request
+ * @param {object} [options.sort] - Field and direction to sort submissions by.
+ * @param {object} [options.query] - Query options to filter the submissions. See query operators {@link http://docs.mongodb.org/manual/reference/operator/query/.}
+ * @param {number} [options.start=0] - The index of the first submission to return.
+ * @param {number} [options.limit=30000] - Maximum number of submissions to fetch. Pass Infinity to disable the limit and download all submissions
+ * @param {number} [options.pageSize=10000] - Limits the size of each page of submissions. Maximum value is 30000.
  * @state data - an array of submission objects
  * @returns {Operation}
  */
-export function getSubmissions(formId: string, options?: object): Operation;
+export function getSubmissions(formId: string, options?: {
+    sort?: object;
+    query?: object;
+    start?: number;
+    limit?: number;
+    pageSize?: number;
+}): Operation;
 /**
  * Get deployment information for a specific form. Calls `/api/v2/assets/<id>/deployment/`.
  * @example
@@ -53,21 +63,4 @@ export function getSubmissions(formId: string, options?: object): Operation;
  * @returns {Operation}
  */
 export function getDeploymentInfo(formId: string): Operation;
-/**
- * Options object
- */
-export type RequestOptions = {
-    /**
-     * - An object of query parameters to be encoded into the URL
-     */
-    query: object;
-    /**
-     * - An object of all request headers
-     */
-    headers: object;
-    /**
-     * - The response format to parse (e.g., 'json', 'text', or 'stream')
-     */
-    parseAs?: string;
-};
-export { alterState, cursor, dataPath, dataValue, each, field, fields, fn, fnIf, http, group, lastReferenceValue, merge, sourceValue } from "@openfn/language-common";
+export { alterState, cursor, dataPath, dataValue, each, field, fields, fn, fnIf, group, lastReferenceValue, merge, sourceValue } from "@openfn/language-common";

package/types/http.d.ts

@@ -1,31 +1,52 @@
 /**
  * State object
- * @typedef {Object} KoboToolboxHttpState
+ * @typedef {Object} HttpState
+ * @private
  * @property data - The response body (as JSON)
- * @property response - The HTTP response from the KoboToolbox server (excluding the body). Responses will be returned in JSON format
- * @property references - An array of all previous data objects used in the Job
+ * @property response - The HTTP response from the KoboToolbox server (excluding the body)
+ * @property references - An array containing all previous data objects
  */
 /**
  * Options object
- * @typedef {Object} RequestOptions
+ * @typedef {Object} HTTPRequestOptions
  * @property {object} query - An object of query parameters to be encoded into the URL
  * @property {object} headers - An object of all request headers
+ * @property {object} body - The request body (as JSON)
+ * @property {number} maxRedirections - The maximum number of redirects to follow
  * @property {string} [parseAs='json'] - The response format to parse (e.g., 'json', 'text', or 'stream')
  */
+/**
+ * Make a HTTP request to any KoboToolbox endpoint
+ * @example <caption>Bulk updating of submissions</caption>
+ * http.request("PATCH", `assets/${$.form_uid}/data/bulk/`, {
+ *   body: {
+ *     submission_ids: [$.data.submission_id],
+ *     data: {
+ *       Transaction_status: "success",
+ *     },
+ *   },
+ * });
+ * @function
+ * @public
+ * @param {string} method - HTTP method to use
+ * @param {string} path - Path to resource
+ * @param {HTTPRequestOptions}  [options={}] - An object containing query, headers, and body for the request
+ * @state {HttpState}
+ * @returns {Operation}
+ */
+export function request(method: string, path: string, options?: HTTPRequestOptions): Operation;
 /**
  * Make a GET request to any KoboToolbox endpoint.
  * @public
  * @function
  * @example <caption>GET assets resource</caption>
- * http.get(
- *  "/assets/",
- *  )
+ * http.get('assets')
  * @param {string} path - path to resource
- * @param {RequestOptions} [options={}] - An object containing query params and headers for the request
- * @state {KoboToolboxHttpState}
+ * @param {HTTPRequestOptions} [options={}] - An object containing query params and headers for the request
+ * @state {HttpState}
  * @returns {operation}
  */
-export function get(path: string, options?: RequestOptions): operation;
+export function get(path: string, options?: HTTPRequestOptions): operation;
 /**
  * Make a POST request to a KoboToolbox endpoint
  * @public
@@ -40,11 +61,11 @@ export function get(path: string, options?: RequestOptions): operation;
  *  );
  * @param {string} path - path to resource
  * @param {any} data - the body data in JSON format
- * @param {RequestOptions} [options={}] - An object containing query params and headers for the request
- * @state {KoboToolboxHttpState}
+ * @param {HTTPRequestOptions} [options={}] - An object containing query params and headers for the request
+ * @state {HttpState}
  * @returns {operation}
  */
-export function post(path: string, data: any, options?: RequestOptions): operation;
+export function post(path: string, data: any, options?: HTTPRequestOptions): operation;
 /**
  * Make a PUT request to a KoboToolbox endpoint
  * @public
@@ -59,32 +80,19 @@ export function post(path: string, data: any, options?: RequestOptions): operati
  *  );
  * @param {string} path - path to resource
  * @param {any} data - the body data in JSON format
- * @param {RequestOptions} [options={}] - An object containing query params and headers for the request
- * @state {KoboToolboxHttpState}
+ * @param {HTTPRequestOptions} [options={}] - An object containing query params and headers for the request
+ * @state {HttpState}
  * @returns {operation}
  */
-export function put(path: string, data: any, options?: RequestOptions): operation;
+export function put(path: string, data: any, options?: HTTPRequestOptions): operation;
 /**
  * State object
  */
-export type KoboToolboxHttpState = {
-    /**
-     * - The response body (as JSON)
-     */
-    data: any;
-    /**
-     * - The HTTP response from the KoboToolbox server (excluding the body). Responses will be returned in JSON format
-     */
-    response: any;
-    /**
-     * - An array of all previous data objects used in the Job
-     */
-    references: any;
-};
+export type HttpState = any;
 /**
  * Options object
  */
-export type RequestOptions = {
+export type HTTPRequestOptions = {
     /**
      * - An object of query parameters to be encoded into the URL
      */
@@ -93,6 +101,14 @@ export type RequestOptions = {
      * - An object of all request headers
      */
     headers: object;
+    /**
+     * - The request body (as JSON)
+     */
+    body: object;
+    /**
+     * - The maximum number of redirects to follow
+     */
+    maxRedirections: number;
     /**
      * - The response format to parse (e.g., 'json', 'text', or 'stream')
      */

package/types/util.d.ts

@@ -0,0 +1,6 @@
+export function prepareNextState(state: any, response: any): any;
+export function request(state: any, method: any, path: any, opts?: {}): Promise<any>;
+export function requestWithPagination(state: any, path: any, options?: {}): Promise<any[]>;
+export function maybeStringify(query: any): string;
+export const DEFAULT_LIMIT: 30000;
+export const DEFAULT_PAGE_SIZE: 10000;

package/types/Utils.d.ts

@@ -1,2 +0,0 @@
-export function request(state: any, method: any, path: any, opts: any): Promise<any>;
-export function prepareNextState(state: any, response: any): any;