@orioro/util 0.7.0 → 0.8.0

package/dist/crawling/fetchAllPages.d.ts

@@ -0,0 +1,21 @@
+type PaginatedFetchParams = {
+    offset: number;
+    pageSize: number;
+};
+export declare function paginatedHttpFetch({ pageUrl, parseResponse, }: {
+    pageUrl: (params: PaginatedFetchParams) => string;
+    parseResponse?: (response: Response) => any[] | Promise<any[]>;
+}): (params: PaginatedFetchParams) => Promise<any[]>;
+export type DataStore = {
+    getSize: () => number | Promise<number>;
+    storePage: (data: any[], params: PaginatedFetchParams) => any | Promise<any>;
+    getResult: () => any;
+};
+export declare function inMemoryDataStore(): DataStore;
+export type FetchAllPagesOptions = {
+    fetchPage: (params: PaginatedFetchParams) => Promise<any[]>;
+    pageSize?: number;
+    dataStore?: () => DataStore;
+};
+export declare function fetchAllPages({ fetchPage, pageSize, dataStore, }: FetchAllPagesOptions): Promise<any>;
+export {};

package/dist/crawling/index.d.ts

@@ -0,0 +1 @@
+export * from './fetchAllPages';

package/dist/index.d.ts

@@ -7,3 +7,4 @@ export * from './promise';
 export * from './switchValue';
 export * from './typeOf';
 export * from './validate';
+export * from './crawling';

package/dist/index.mjs

@@ -1536,4 +1536,87 @@ validateAsync.or = or;
 validateAsync.not = not;
 validateAsync.assertValid = assertValidAsync;
 
-export { PromiseLikeEventEmitter, SKIPPED, TimeoutError, ValidationError, batchFn, debugFn, deepFreeze, interpolate, makeTypeOf, maybeFn, parseBatchedResults, pickPaths, promiseReduce, resolveNestedPromises, switchExec, switchValue, typeMap, typeOf, validate, validateAsync, wait, withTimeout };
+function defaultParseResponse(response) {
+  return response.json();
+}
+function paginatedHttpFetch(_a) {
+  var pageUrl = _a.pageUrl,
+    _b = _a.parseResponse,
+    parseResponse = _b === void 0 ? defaultParseResponse : _b;
+  return function (params) {
+    return __awaiter(this, void 0, void 0, function () {
+      var response, data;
+      return __generator(this, function (_a) {
+        switch (_a.label) {
+          case 0:
+            return [4 /*yield*/, fetch(pageUrl(params))];
+          case 1:
+            response = _a.sent();
+            return [4 /*yield*/, parseResponse(response)];
+          case 2:
+            data = _a.sent();
+            return [2 /*return*/, data];
+        }
+      });
+    });
+  };
+}
+function inMemoryDataStore() {
+  var data = [];
+  return {
+    getSize: function getSize() {
+      return data.length;
+    },
+    storePage: function storePage(pageData) {
+      data = __spreadArray(__spreadArray([], data, true), pageData, true);
+    },
+    getResult: function getResult() {
+      return data;
+    }
+  };
+}
+function fetchAllPages(_a) {
+  var fetchPage = _a.fetchPage,
+    _b = _a.pageSize,
+    pageSize = _b === void 0 ? 50 : _b,
+    _c = _a.dataStore,
+    dataStore = _c === void 0 ? inMemoryDataStore : _c;
+  return __awaiter(this, void 0, void 0, function () {
+    var store, currentPageData, currentOffset, result;
+    return __generator(this, function (_d) {
+      switch (_d.label) {
+        case 0:
+          store = dataStore();
+          currentPageData = null;
+          _d.label = 1;
+        case 1:
+          return [4 /*yield*/, store.getSize()];
+        case 2:
+          currentOffset = _d.sent() + 1;
+          return [4 /*yield*/, fetchPage({
+            offset: currentOffset,
+            pageSize: pageSize
+          })];
+        case 3:
+          currentPageData = _d.sent();
+          return [4 /*yield*/, store.storePage(currentPageData, {
+            offset: currentOffset,
+            pageSize: pageSize
+          })];
+        case 4:
+          _d.sent();
+          _d.label = 5;
+        case 5:
+          if (currentPageData.length === pageSize) return [3 /*break*/, 1];
+          _d.label = 6;
+        case 6:
+          return [4 /*yield*/, store.getResult()];
+        case 7:
+          result = _d.sent();
+          return [2 /*return*/, result];
+      }
+    });
+  });
+}
+
+export { PromiseLikeEventEmitter, SKIPPED, TimeoutError, ValidationError, batchFn, debugFn, deepFreeze, fetchAllPages, inMemoryDataStore, interpolate, makeTypeOf, maybeFn, paginatedHttpFetch, parseBatchedResults, pickPaths, promiseReduce, resolveNestedPromises, switchExec, switchValue, typeMap, typeOf, validate, validateAsync, wait, withTimeout };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@orioro/util",
-  "version": "0.7.0",
+  "version": "0.8.0",
   "packageManager": "yarn@4.0.2",
   "type": "module",
   "main": "dist/index.mjs",
@@ -24,7 +24,7 @@
     "prepublish": "yarn build"
   },
   "devDependencies": {
-    "@orioro/dev": "0.0.0",
+    "@orioro/dev": "0.0.1",
     "@types/clone-deep": "^4.0.4",
     "@types/jest": "^29.5.12",
     "babel-jest": "^29.7.0",