@orioro/util 0.6.0 → 0.8.0

package/dist/crawling/fetchAllPages.d.ts

@@ -0,0 +1,21 @@
+type PaginatedFetchParams = {
+    offset: number;
+    pageSize: number;
+};
+export declare function paginatedHttpFetch({ pageUrl, parseResponse, }: {
+    pageUrl: (params: PaginatedFetchParams) => string;
+    parseResponse?: (response: Response) => any[] | Promise<any[]>;
+}): (params: PaginatedFetchParams) => Promise<any[]>;
+export type DataStore = {
+    getSize: () => number | Promise<number>;
+    storePage: (data: any[], params: PaginatedFetchParams) => any | Promise<any>;
+    getResult: () => any;
+};
+export declare function inMemoryDataStore(): DataStore;
+export type FetchAllPagesOptions = {
+    fetchPage: (params: PaginatedFetchParams) => Promise<any[]>;
+    pageSize?: number;
+    dataStore?: () => DataStore;
+};
+export declare function fetchAllPages({ fetchPage, pageSize, dataStore, }: FetchAllPagesOptions): Promise<any>;
+export {};

package/dist/crawling/index.d.ts

@@ -0,0 +1 @@
+export * from './fetchAllPages';

package/dist/index.d.ts

@@ -7,3 +7,4 @@ export * from './promise';
 export * from './switchValue';
 export * from './typeOf';
 export * from './validate';
+export * from './crawling';

package/dist/index.mjs

@@ -523,7 +523,8 @@ function batchFn(itemFn, _a) {
     skip = _b.skip,
     _d = _b.retry,
     retry = _d === void 0 ? false : _d,
-    timeout = _b.timeout;
+    timeout = _b.timeout,
+    rollback = _b.rollback;
   //
   // Optionally wrap itemFn into timeout
   //
@@ -550,11 +551,11 @@ function batchFn(itemFn, _a) {
               });
               return [4 /*yield*/, Promise.all(batchItems.map(function (item) {
                 return __awaiter(_this, void 0, void 0, function () {
-                  var itemSkip, itemResult, _a, err_1;
+                  var itemSkip, itemResult, _a, err_1, rollbackErr_1;
                   return __generator(this, function (_b) {
                     switch (_b.label) {
                       case 0:
-                        _b.trys.push([0, 5,, 6]);
+                        _b.trys.push([0, 5,, 10]);
                         promise.emit('itemStart', {
                           batch: batch,
                           item: item
@@ -601,8 +602,21 @@ function batchFn(itemFn, _a) {
                           item: item,
                           result: err_1
                         });
-                        return [2 /*return*/, err_1];
+                        if (!(typeof rollback === 'function')) return [3 /*break*/, 9];
+                        _b.label = 6;
                       case 6:
+                        _b.trys.push([6, 8,, 9]);
+                        return [4 /*yield*/, rollback(item, err_1)];
+                      case 7:
+                        _b.sent();
+                        return [3 /*break*/, 9];
+                      case 8:
+                        rollbackErr_1 = _b.sent();
+                        console.error('Failure rolling back', rollbackErr_1);
+                        return [3 /*break*/, 9];
+                      case 9:
+                        return [2 /*return*/, err_1];
+                      case 10:
                         return [2 /*return*/];
                     }
                   });
@@ -1522,4 +1536,87 @@ validateAsync.or = or;
 validateAsync.not = not;
 validateAsync.assertValid = assertValidAsync;
 
-export { PromiseLikeEventEmitter, SKIPPED, TimeoutError, ValidationError, batchFn, debugFn, deepFreeze, interpolate, makeTypeOf, maybeFn, parseBatchedResults, pickPaths, promiseReduce, resolveNestedPromises, switchExec, switchValue, typeMap, typeOf, validate, validateAsync, wait, withTimeout };
+function defaultParseResponse(response) {
+  return response.json();
+}
+function paginatedHttpFetch(_a) {
+  var pageUrl = _a.pageUrl,
+    _b = _a.parseResponse,
+    parseResponse = _b === void 0 ? defaultParseResponse : _b;
+  return function (params) {
+    return __awaiter(this, void 0, void 0, function () {
+      var response, data;
+      return __generator(this, function (_a) {
+        switch (_a.label) {
+          case 0:
+            return [4 /*yield*/, fetch(pageUrl(params))];
+          case 1:
+            response = _a.sent();
+            return [4 /*yield*/, parseResponse(response)];
+          case 2:
+            data = _a.sent();
+            return [2 /*return*/, data];
+        }
+      });
+    });
+  };
+}
+function inMemoryDataStore() {
+  var data = [];
+  return {
+    getSize: function getSize() {
+      return data.length;
+    },
+    storePage: function storePage(pageData) {
+      data = __spreadArray(__spreadArray([], data, true), pageData, true);
+    },
+    getResult: function getResult() {
+      return data;
+    }
+  };
+}
+function fetchAllPages(_a) {
+  var fetchPage = _a.fetchPage,
+    _b = _a.pageSize,
+    pageSize = _b === void 0 ? 50 : _b,
+    _c = _a.dataStore,
+    dataStore = _c === void 0 ? inMemoryDataStore : _c;
+  return __awaiter(this, void 0, void 0, function () {
+    var store, currentPageData, currentOffset, result;
+    return __generator(this, function (_d) {
+      switch (_d.label) {
+        case 0:
+          store = dataStore();
+          currentPageData = null;
+          _d.label = 1;
+        case 1:
+          return [4 /*yield*/, store.getSize()];
+        case 2:
+          currentOffset = _d.sent() + 1;
+          return [4 /*yield*/, fetchPage({
+            offset: currentOffset,
+            pageSize: pageSize
+          })];
+        case 3:
+          currentPageData = _d.sent();
+          return [4 /*yield*/, store.storePage(currentPageData, {
+            offset: currentOffset,
+            pageSize: pageSize
+          })];
+        case 4:
+          _d.sent();
+          _d.label = 5;
+        case 5:
+          if (currentPageData.length === pageSize) return [3 /*break*/, 1];
+          _d.label = 6;
+        case 6:
+          return [4 /*yield*/, store.getResult()];
+        case 7:
+          result = _d.sent();
+          return [2 /*return*/, result];
+      }
+    });
+  });
+}
+
+export { PromiseLikeEventEmitter, SKIPPED, TimeoutError, ValidationError, batchFn, debugFn, deepFreeze, fetchAllPages, inMemoryDataStore, interpolate, makeTypeOf, maybeFn, paginatedHttpFetch, parseBatchedResults, pickPaths, promiseReduce, resolveNestedPromises, switchExec, switchValue, typeMap, typeOf, validate, validateAsync, wait, withTimeout };

package/dist/promise/batchFn.d.ts

@@ -5,6 +5,7 @@ type BatchFnOptions = {
     batchSize?: number;
     retry?: BackoffOptions | boolean;
     timeout?: number | false;
+    rollback?: (input: any, err: any) => Promise<any> | any;
 };
 type Batch = {
     index: number;
@@ -40,7 +41,7 @@ type EventTypes = {
     }) => void;
     error: (err: Error) => void;
 };
-export declare function batchFn(itemFn: (input: any) => Promise<any> | any, { batchSize, skip, retry, timeout }?: BatchFnOptions): (items: any[]) => PromiseLikeEventEmitter<any[], EventTypes>;
+export declare function batchFn(itemFn: (input: any) => Promise<any> | any, { batchSize, skip, retry, timeout, rollback, }?: BatchFnOptions): (items: any[]) => PromiseLikeEventEmitter<any[], EventTypes>;
 type ParsedBatchResults = {
     results: any[];
     errors: Error[];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@orioro/util",
-  "version": "0.6.0",
+  "version": "0.8.0",
   "packageManager": "yarn@4.0.2",
   "type": "module",
   "main": "dist/index.mjs",
@@ -24,7 +24,7 @@
     "prepublish": "yarn build"
   },
   "devDependencies": {
-    "@orioro/dev": "0.0.0",
+    "@orioro/dev": "0.0.1",
     "@types/clone-deep": "^4.0.4",
     "@types/jest": "^29.5.12",
     "babel-jest": "^29.7.0",