@polygonlabs/servercore 1.0.0-dev.31 → 1.0.0-dev.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -29,6 +29,9 @@ class RestAPIConsumer extends AbstractCronEventConsumer {
|
|
|
29
29
|
shouldFetchNextPage = await this.fetchPage();
|
|
30
30
|
}
|
|
31
31
|
this.config.startCount.value = this.highestValueSeen;
|
|
32
|
+
this.observer?.summary({
|
|
33
|
+
[`${this.config.startCount.key}`]: this.config.startCount.value
|
|
34
|
+
});
|
|
32
35
|
}
|
|
33
36
|
async fetchPage() {
|
|
34
37
|
try {
|
|
@@ -84,7 +87,7 @@ class RestAPIConsumer extends AbstractCronEventConsumer {
|
|
|
84
87
|
if (currentValue === void 0) {
|
|
85
88
|
return false;
|
|
86
89
|
}
|
|
87
|
-
return currentValue
|
|
90
|
+
return currentValue >= this.config.startCount.value;
|
|
88
91
|
} catch (error) {
|
|
89
92
|
this.onFatalError(error);
|
|
90
93
|
return false;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/consumers/rest_api_consumer.ts"],"sourcesContent":["import { errorCodes } from \"../constants\";\nimport { ConsumerError } from \"../errors\";\nimport type { IObserver, IRestAPIConsumerConfig } from \"../types\";\nimport { AbstractCronEventConsumer } from \"./abstract_cron_event_consumer\";\n\nexport class RestAPIConsumer<T, U> extends AbstractCronEventConsumer {\n protected observer: IObserver<T, ConsumerError, U> | null = null;\n private currentPage = 0;\n private highestValueSeen = 0;\n\n constructor(private config: IRestAPIConsumerConfig) {\n super();\n this.highestValueSeen = this.config.startCount.value;\n }\n\n public async start(\n observer: IObserver<T, ConsumerError, U>\n ): Promise<void> {\n this.observer = observer;\n this.currentPage = 1;\n if (this.config.cronExpr) {\n this.startCron(this.config.cronExpr);\n } else {\n // fallback: run once if no cron\n await this.onTick();\n }\n }\n\n public stop(): void {\n this.stopCron();\n }\n\n protected async onTick(): Promise<void> {\n // Reset page counter at the start of each cron run\n this.currentPage = 1;\n\n // Keep fetching pages until we've caught up\n let shouldFetchNextPage = true;\n\n while (shouldFetchNextPage) {\n shouldFetchNextPage = await this.fetchPage();\n }\n\n // Update config with the highest value we've seen\n // This prevents fetching the same data in the next cron run\n this.config.startCount.value = this.highestValueSeen;\n }\n\n private async fetchPage(): Promise<boolean> {\n try {\n // Use paginationParam from config or default to \"page\"\n const paginationParam = this.config.paginationParam || \"page\";\n const params = {\n ...(this.config.params || {}),\n [paginationParam]: this.currentPage.toString(),\n };\n\n let url = this.config.apiUrl.toString();\n if (Object.keys(params).length > 0) {\n const queryString = new URLSearchParams(params).toString();\n url += (url.includes(\"?\") ? \"&\" : \"?\") + queryString;\n }\n\n const response = await fetch(url, {\n method: this.config.method || \"GET\",\n headers: this.config.headers,\n body: this.config.body\n ? JSON.stringify(this.config.body)\n : undefined,\n });\n\n if (!response.ok) {\n throw new ConsumerError(\n `Failed to fetch from API: ${response.statusText}`,\n {\n code: errorCodes.external\n .UNKNOWN_EXTERNAL_DEPENDENCY_ERROR,\n isFatal: true,\n origin: \"rest_api_consumer\",\n }\n );\n }\n\n let data = await response.json();\n if (this.config.resultPath) {\n data = this.getValueByPath(data, this.config.resultPath);\n }\n\n // Extract the value we're tracking\n const countKey = this.config.startCount.key;\n let currentValue: number | undefined = undefined;\n\n if (Array.isArray(data)) {\n if (data.length === 0) {\n return false;\n }\n for (const elem of data) {\n currentValue = this.processDataForCurrentCountValue(\n elem,\n countKey\n );\n }\n } else {\n currentValue = this.processDataForCurrentCountValue(\n data,\n countKey\n );\n }\n\n // Send data to observer\n this.observer?.next(data);\n\n // Increment page for next potential fetch\n this.currentPage++;\n\n // Continue fetching if the current value is <= our threshold\n if (currentValue === undefined) {\n // If currentValue was never set, stop fetching\n return false;\n }\n return currentValue
|
|
1
|
+
{"version":3,"sources":["../../src/consumers/rest_api_consumer.ts"],"sourcesContent":["import { errorCodes } from \"../constants\";\nimport { ConsumerError } from \"../errors\";\nimport type { IObserver, IRestAPIConsumerConfig } from \"../types\";\nimport { AbstractCronEventConsumer } from \"./abstract_cron_event_consumer\";\n\nexport class RestAPIConsumer<T, U> extends AbstractCronEventConsumer {\n protected observer: IObserver<T, ConsumerError, U> | null = null;\n private currentPage = 0;\n private highestValueSeen = 0;\n\n constructor(private config: IRestAPIConsumerConfig) {\n super();\n this.highestValueSeen = this.config.startCount.value;\n }\n\n public async start(\n observer: IObserver<T, ConsumerError, U>\n ): Promise<void> {\n this.observer = observer;\n this.currentPage = 1;\n if (this.config.cronExpr) {\n this.startCron(this.config.cronExpr);\n } else {\n // fallback: run once if no cron\n await this.onTick();\n }\n }\n\n public stop(): void {\n this.stopCron();\n }\n\n protected async onTick(): Promise<void> {\n // Reset page counter at the start of each cron run\n this.currentPage = 1;\n\n // Keep fetching pages until we've caught up\n let shouldFetchNextPage = true;\n\n while (shouldFetchNextPage) {\n shouldFetchNextPage = await this.fetchPage();\n }\n\n // Update config with the highest value we've seen\n // This prevents fetching the same data in the next cron run\n this.config.startCount.value = this.highestValueSeen;\n this.observer?.summary({\n [`${this.config.startCount.key}`]: this.config.startCount.value,\n } as U);\n }\n\n private async fetchPage(): Promise<boolean> {\n try {\n // Use paginationParam from config or default to \"page\"\n const paginationParam = this.config.paginationParam || \"page\";\n const params = {\n ...(this.config.params || {}),\n [paginationParam]: this.currentPage.toString(),\n };\n\n let url = this.config.apiUrl.toString();\n if (Object.keys(params).length > 0) {\n const queryString = new URLSearchParams(params).toString();\n url += (url.includes(\"?\") ? \"&\" : \"?\") + queryString;\n }\n\n const response = await fetch(url, {\n method: this.config.method || \"GET\",\n headers: this.config.headers,\n body: this.config.body\n ? JSON.stringify(this.config.body)\n : undefined,\n });\n\n if (!response.ok) {\n throw new ConsumerError(\n `Failed to fetch from API: ${response.statusText}`,\n {\n code: errorCodes.external\n .UNKNOWN_EXTERNAL_DEPENDENCY_ERROR,\n isFatal: true,\n origin: \"rest_api_consumer\",\n }\n );\n }\n\n let data = await response.json();\n if (this.config.resultPath) {\n data = this.getValueByPath(data, this.config.resultPath);\n }\n\n // Extract the value we're tracking\n const countKey = this.config.startCount.key;\n let currentValue: number | undefined = undefined;\n\n if (Array.isArray(data)) {\n if (data.length === 0) {\n return false;\n }\n for (const elem of data) {\n currentValue = this.processDataForCurrentCountValue(\n elem,\n countKey\n );\n }\n } else {\n currentValue = this.processDataForCurrentCountValue(\n data,\n countKey\n );\n }\n\n // Send data to observer\n this.observer?.next(data);\n\n // Increment page for next potential fetch\n this.currentPage++;\n\n // Continue fetching if the current value is <= our threshold\n if (currentValue === undefined) {\n // If currentValue was never set, stop fetching\n return false;\n }\n return currentValue >= this.config.startCount.value;\n } catch (error) {\n this.onFatalError(error as ConsumerError);\n return false; // Stop fetching on error\n }\n }\n\n // Utility to get nested values using dot notation (e.g., \"response.data.count\")\n private getValueByPath(obj: any, path: string): any {\n return path.split(\".\").reduce((prev, curr) => {\n return prev && prev[curr] !== undefined ? prev[curr] : undefined;\n }, obj);\n }\n\n private processDataForCurrentCountValue(\n item: any,\n countKey: string\n ): number {\n const value = this.getValueByPath(item, countKey);\n if (value === undefined) {\n throw new ConsumerError(\n `Count key '${countKey}' not found in API response`,\n {\n code: errorCodes.external.UNKNOWN_EXTERNAL_DEPENDENCY_ERROR,\n isFatal: false,\n origin: \"rest_api_consumer\",\n }\n );\n }\n this.highestValueSeen = Math.max(this.highestValueSeen, value);\n return value;\n }\n\n /**\n * Updates the start count value to a specific value.\n * This allows reprocessing data from a specified point.\n *\n * @param value The value to set as the new start count\n */\n public setStartValue(value: number): void {\n this.config.startCount.value = value;\n this.highestValueSeen = value;\n }\n}\n"],"mappings":"AAAA,SAAS,kBAAkB;AAC3B,SAAS,qBAAqB;AAE9B,SAAS,iCAAiC;AAEnC,MAAM,wBAA8B,0BAA0B;AAAA,EAKjE,YAAoB,QAAgC;AAChD,UAAM;AADU;AAEhB,SAAK,mBAAmB,KAAK,OAAO,WAAW;AAAA,EACnD;AAAA,EAPU,WAAkD;AAAA,EACpD,cAAc;AAAA,EACd,mBAAmB;AAAA,EAO3B,MAAa,MACT,UACa;AACb,SAAK,WAAW;AAChB,SAAK,cAAc;AACnB,QAAI,KAAK,OAAO,UAAU;AACtB,WAAK,UAAU,KAAK,OAAO,QAAQ;AAAA,IACvC,OAAO;AAEH,YAAM,KAAK,OAAO;AAAA,IACtB;AAAA,EACJ;AAAA,EAEO,OAAa;AAChB,SAAK,SAAS;AAAA,EAClB;AAAA,EAEA,MAAgB,SAAwB;AAEpC,SAAK,cAAc;AAGnB,QAAI,sBAAsB;AAE1B,WAAO,qBAAqB;AACxB,4BAAsB,MAAM,KAAK,UAAU;AAAA,IAC/C;AAIA,SAAK,OAAO,WAAW,QAAQ,KAAK;AACpC,SAAK,UAAU,QAAQ;AAAA,MACnB,CAAC,GAAG,KAAK,OAAO,WAAW,GAAG,EAAE,GAAG,KAAK,OAAO,WAAW;AAAA,IAC9D,CAAM;AAAA,EACV;AAAA,EAEA,MAAc,YAA8B;AACxC,QAAI;AAEA,YAAM,kBAAkB,KAAK,OAAO,mBAAmB;AACvD,YAAM,SAAS;AAAA,QACX,GAAI,KAAK,OAAO,UAAU,CAAC;AAAA,QAC3B,CAAC,eAAe,GAAG,KAAK,YAAY,SAAS;AAAA,MACjD;AAEA,UAAI,MAAM,KAAK,OAAO,OAAO,SAAS;AACtC,UAAI,OAAO,KAAK,MAAM,EAAE,SAAS,GAAG;AAChC,cAAM,cAAc,IAAI,gBAAgB,MAAM,EAAE,SAAS;AACzD,gBAAQ,IAAI,SAAS,GAAG,IAAI,MAAM,OAAO;AAAA,MAC7C;AAEA,YAAM,WAAW,MAAM,MAAM,KAAK;AAAA,QAC9B,QAAQ,KAAK,OAAO,UAAU;AAAA,QAC9B,SAAS,KAAK,OAAO;AAAA,QACrB,MAAM,KAAK,OAAO,OACZ,KAAK,UAAU,KAAK,OAAO,IAAI,IAC/B;AAAA,MACV,CAAC;AAED,UAAI,CAAC,SAAS,IAAI;AACd,cAAM,IAAI;AAAA,UACN,6BAA6B,SAAS,UAAU;AAAA,UAChD;AAAA,YACI,MAAM,WAAW,SACZ;AAAA,YACL,SAAS;AAAA,YACT,QAAQ;AAAA,UACZ;AAAA,QACJ;AAAA,MACJ;AAEA,UAAI,OAAO,MAAM,SAAS,KAAK;AAC/B,UAAI,KAAK,OAAO,YAAY;AACxB,eAAO,KAAK,eAAe,MAAM,KAAK,OAAO,UAAU;AAAA,MAC3D;AAGA,YAAM,WAAW,KAAK,OAAO,WAAW;AACxC,UAAI,eAAmC;AAEvC,UAAI,MAAM,QAAQ,IAAI,GAAG;AACrB,YAAI,KAAK,WAAW,GAAG;AACnB,iBAAO;AAAA,QACX;AACA,mBAAW,QAAQ,MAAM;AACrB,yBAAe,KAAK;AAAA,YAChB;AAAA,YACA;AAAA,UACJ;AAAA,QACJ;AAAA,MACJ,OAAO;AACH,uBAAe,KAAK;AAAA,UAChB;AAAA,UACA;AAAA,QACJ;AAAA,MACJ;AAGA,WAAK,UAAU,KAAK,IAAI;AAGxB,WAAK;AAGL,UAAI,iBAAiB,QAAW;AAE5B,eAAO;AAAA,MACX;AACA,aAAO,gBAAgB,KAAK,OAAO,WAAW;AAAA,IAClD,SAAS,OAAO;AACZ,WAAK,aAAa,KAAsB;AACxC,aAAO;AAAA,IACX;AAAA,EACJ;AAAA;AAAA,EAGQ,eAAe,KAAU,MAAmB;AAChD,WAAO,KAAK,MAAM,GAAG,EAAE,OAAO,CAAC,MAAM,SAAS;AAC1C,aAAO,QAAQ,KAAK,IAAI,MAAM,SAAY,KAAK,IAAI,IAAI;AAAA,IAC3D,GAAG,GAAG;AAAA,EACV;AAAA,EAEQ,gCACJ,MACA,UACM;AACN,UAAM,QAAQ,KAAK,eAAe,MAAM,QAAQ;AAChD,QAAI,UAAU,QAAW;AACrB,YAAM,IAAI;AAAA,QACN,cAAc,QAAQ;AAAA,QACtB;AAAA,UACI,MAAM,WAAW,SAAS;AAAA,UAC1B,SAAS;AAAA,UACT,QAAQ;AAAA,QACZ;AAAA,MACJ;AAAA,IACJ;AACA,SAAK,mBAAmB,KAAK,IAAI,KAAK,kBAAkB,KAAK;AAC7D,WAAO;AAAA,EACX;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQO,cAAc,OAAqB;AACtC,SAAK,OAAO,WAAW,QAAQ;AAC/B,SAAK,mBAAmB;AAAA,EAC5B;AACJ;","names":[]}
|