@polygonlabs/servercore 1.0.0-dev.3 → 1.0.0-dev.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/health-check.d.ts +8 -0
- package/dist/api/health-check.js +79 -0
- package/dist/api/health-check.js.map +1 -0
- package/dist/api/index.d.ts +2 -0
- package/dist/api/index.js +1 -0
- package/dist/api/index.js.map +1 -1
- package/dist/api/response_handler.d.ts +1 -4
- package/dist/api/response_handler.js.map +1 -1
- package/dist/consumers/abstract_cron_event_consumer.d.ts +23 -0
- package/dist/consumers/abstract_cron_event_consumer.js +30 -0
- package/dist/consumers/abstract_cron_event_consumer.js.map +1 -0
- package/dist/{consumer → consumers}/abstract_event_consumer.d.ts +1 -0
- package/dist/{consumer → consumers}/abstract_event_consumer.js +3 -0
- package/dist/consumers/abstract_event_consumer.js.map +1 -0
- package/dist/{consumer → consumers}/event_consumer.d.ts +6 -4
- package/dist/consumers/event_consumer.js +141 -0
- package/dist/consumers/event_consumer.js.map +1 -0
- package/dist/consumers/index.d.ts +11 -0
- package/dist/consumers/index.js +3 -0
- package/dist/consumers/index.js.map +1 -0
- package/dist/consumers/rest_api_consumer.d.ts +30 -0
- package/dist/consumers/rest_api_consumer.js +128 -0
- package/dist/consumers/rest_api_consumer.js.map +1 -0
- package/dist/errors/database_errors.js +1 -1
- package/dist/errors/database_errors.js.map +1 -1
- package/dist/index.d.ts +12 -1
- package/dist/index.js +3 -1
- package/dist/index.js.map +1 -1
- package/dist/logger/logger.d.ts +2 -0
- package/dist/logger/logger.js +33 -30
- package/dist/logger/logger.js.map +1 -1
- package/dist/{database → storage}/db_interface.d.ts +4 -3
- package/dist/storage/index.d.ts +4 -0
- package/dist/storage/index.js +3 -0
- package/dist/storage/index.js.map +1 -0
- package/dist/storage/queue_interface.d.ts +11 -0
- package/dist/storage/queue_interface.js +1 -0
- package/dist/storage/queue_interface.js.map +1 -0
- package/dist/types/event_consumer_config.d.ts +5 -4
- package/dist/types/index.d.ts +3 -0
- package/dist/types/index.js +3 -0
- package/dist/types/index.js.map +1 -1
- package/dist/types/queue_job_opts.d.ts +5 -0
- package/dist/types/queue_job_opts.js +1 -0
- package/dist/types/queue_job_opts.js.map +1 -0
- package/dist/types/response_context.d.ts +6 -0
- package/dist/types/response_context.js +1 -0
- package/dist/types/response_context.js.map +1 -0
- package/dist/types/rest_api_consumer_config.d.ts +17 -0
- package/dist/types/rest_api_consumer_config.js +1 -0
- package/dist/types/rest_api_consumer_config.js.map +1 -0
- package/dist/utils/decoder.d.ts +6 -0
- package/dist/utils/decoder.js +8 -0
- package/dist/utils/decoder.js.map +1 -0
- package/dist/utils/index.d.ts +1 -0
- package/dist/utils/index.js +2 -0
- package/dist/utils/index.js.map +1 -0
- package/package.json +2 -1
- package/dist/consumer/abstract_event_consumer.js.map +0 -1
- package/dist/consumer/event_consumer.js +0 -78
- package/dist/consumer/event_consumer.js.map +0 -1
- package/dist/database/index.d.ts +0 -2
- package/dist/database/index.js +0 -2
- package/dist/database/index.js.map +0 -1
- /package/dist/{database → storage}/db_interface.js +0 -0
- /package/dist/{database → storage}/db_interface.js.map +0 -0
@@ -0,0 +1,79 @@
|
|
1
|
+
import { handleResponse, handleError } from "./response_handler";
|
2
|
+
import { ApiError } from "../errors";
|
3
|
+
class HealthCheck {
|
4
|
+
urls;
|
5
|
+
constructor(urls) {
|
6
|
+
this.urls = urls;
|
7
|
+
}
|
8
|
+
async checkHealth() {
|
9
|
+
try {
|
10
|
+
const results = await Promise.all(
|
11
|
+
this.urls.map(async (url) => {
|
12
|
+
const response = await fetch(`${url}`);
|
13
|
+
if (!response.ok) {
|
14
|
+
throw new Error(`Health check failed for ${url}`);
|
15
|
+
}
|
16
|
+
return response;
|
17
|
+
})
|
18
|
+
);
|
19
|
+
if (results.every((res) => res.ok)) {
|
20
|
+
return "ok";
|
21
|
+
}
|
22
|
+
throw new Error("One or more URLs failed the health check.");
|
23
|
+
} catch (error) {
|
24
|
+
const failedUrls = this.urls.filter(async (url) => {
|
25
|
+
const response = await fetch(`${url}`);
|
26
|
+
return !response.ok;
|
27
|
+
});
|
28
|
+
throw new ApiError(
|
29
|
+
`Health check failed for the following URLs: ${failedUrls.join(
|
30
|
+
", "
|
31
|
+
)}`
|
32
|
+
);
|
33
|
+
}
|
34
|
+
}
|
35
|
+
}
|
36
|
+
function createResponseContext(c) {
|
37
|
+
c.status = c.status || 200;
|
38
|
+
return {
|
39
|
+
status: (statusCode) => {
|
40
|
+
c.status = statusCode;
|
41
|
+
return createResponseContext(c);
|
42
|
+
},
|
43
|
+
json: (body) => {
|
44
|
+
return new Response(JSON.stringify(body), {
|
45
|
+
status: c.status,
|
46
|
+
headers: { "Content-Type": "application/json" }
|
47
|
+
});
|
48
|
+
}
|
49
|
+
};
|
50
|
+
}
|
51
|
+
function setupHealthCheckServer(urls, serverPort) {
|
52
|
+
const healthCheck = new HealthCheck(urls);
|
53
|
+
Bun.serve({
|
54
|
+
port: serverPort,
|
55
|
+
routes: {
|
56
|
+
"/health-check": {
|
57
|
+
GET: async (c) => {
|
58
|
+
try {
|
59
|
+
await healthCheck.checkHealth();
|
60
|
+
return handleResponse(createResponseContext(c), "ok");
|
61
|
+
} catch (error) {
|
62
|
+
return handleError(
|
63
|
+
createResponseContext(c),
|
64
|
+
error
|
65
|
+
);
|
66
|
+
}
|
67
|
+
}
|
68
|
+
}
|
69
|
+
}
|
70
|
+
});
|
71
|
+
console.log(
|
72
|
+
`Health check server running on http://localhost:${serverPort}/health-check`
|
73
|
+
);
|
74
|
+
}
|
75
|
+
export {
|
76
|
+
HealthCheck,
|
77
|
+
setupHealthCheckServer
|
78
|
+
};
|
79
|
+
//# sourceMappingURL=health-check.js.map
|
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"sources":["../../src/api/health-check.ts"],"sourcesContent":["import { handleResponse, handleError } from \"./response_handler\";\nimport type { ResponseContext } from \"../types\";\nimport { ApiError } from \"../errors\";\n\nexport class HealthCheck {\n private urls: string[];\n\n constructor(urls: string[]) {\n this.urls = urls;\n }\n\n async checkHealth(): Promise<string> {\n try {\n const results = await Promise.all(\n this.urls.map(async (url) => {\n const response = await fetch(`${url}`);\n if (!response.ok) {\n throw new Error(`Health check failed for ${url}`);\n }\n return response;\n })\n );\n\n if (results.every((res) => res.ok)) {\n return \"ok\";\n }\n throw new Error(\"One or more URLs failed the health check.\");\n } catch (error) {\n const failedUrls = this.urls.filter(async (url) => {\n const response = await fetch(`${url}`);\n return !response.ok;\n });\n throw new ApiError(\n `Health check failed for the following URLs: ${failedUrls.join(\n \", \"\n )}`\n );\n }\n }\n}\n\n// Utility to convert Bun context to ResponseContext\nfunction createResponseContext(c: any): ResponseContext {\n c.status = c.status || 200; // Default status to 200 if undefined\n return {\n status: (statusCode: number) => {\n c.status = statusCode;\n return createResponseContext(c); // Return the updated context\n },\n json: (body: any) => {\n return new Response(JSON.stringify(body), {\n status: c.status,\n headers: { \"Content-Type\": \"application/json\" },\n });\n },\n };\n}\n\n// Setup Bun server with health check route\nexport function setupHealthCheckServer(\n urls: string[],\n serverPort: number\n): void {\n const healthCheck = new HealthCheck(urls);\n\n Bun.serve({\n port: serverPort,\n routes: {\n \"/health-check\": {\n GET: async (c: any) => {\n try {\n await healthCheck.checkHealth();\n return handleResponse(createResponseContext(c), \"ok\");\n } catch (error) {\n return handleError(\n createResponseContext(c),\n error as ApiError\n );\n }\n },\n },\n },\n });\n\n console.log(\n `Health check server running on http://localhost:${serverPort}/health-check`\n );\n}\n"],"mappings":"AAAA,SAAS,gBAAgB,mBAAmB;AAE5C,SAAS,gBAAgB;AAElB,MAAM,YAAY;AAAA,EACb;AAAA,EAER,YAAY,MAAgB;AACxB,SAAK,OAAO;AAAA,EAChB;AAAA,EAEA,MAAM,cAA+B;AACjC,QAAI;AACA,YAAM,UAAU,MAAM,QAAQ;AAAA,QAC1B,KAAK,KAAK,IAAI,OAAO,QAAQ;AACzB,gBAAM,WAAW,MAAM,MAAM,GAAG,GAAG,EAAE;AACrC,cAAI,CAAC,SAAS,IAAI;AACd,kBAAM,IAAI,MAAM,2BAA2B,GAAG,EAAE;AAAA,UACpD;AACA,iBAAO;AAAA,QACX,CAAC;AAAA,MACL;AAEA,UAAI,QAAQ,MAAM,CAAC,QAAQ,IAAI,EAAE,GAAG;AAChC,eAAO;AAAA,MACX;AACA,YAAM,IAAI,MAAM,2CAA2C;AAAA,IAC/D,SAAS,OAAO;AACZ,YAAM,aAAa,KAAK,KAAK,OAAO,OAAO,QAAQ;AAC/C,cAAM,WAAW,MAAM,MAAM,GAAG,GAAG,EAAE;AACrC,eAAO,CAAC,SAAS;AAAA,MACrB,CAAC;AACD,YAAM,IAAI;AAAA,QACN,+CAA+C,WAAW;AAAA,UACtD;AAAA,QACJ,CAAC;AAAA,MACL;AAAA,IACJ;AAAA,EACJ;AACJ;AAGA,SAAS,sBAAsB,GAAyB;AACpD,IAAE,SAAS,EAAE,UAAU;AACvB,SAAO;AAAA,IACH,QAAQ,CAAC,eAAuB;AAC5B,QAAE,SAAS;AACX,aAAO,sBAAsB,CAAC;AAAA,IAClC;AAAA,IACA,MAAM,CAAC,SAAc;AACjB,aAAO,IAAI,SAAS,KAAK,UAAU,IAAI,GAAG;AAAA,QACtC,QAAQ,EAAE;AAAA,QACV,SAAS,EAAE,gBAAgB,mBAAmB;AAAA,MAClD,CAAC;AAAA,IACL;AAAA,EACJ;AACJ;AAGO,SAAS,uBACZ,MACA,YACI;AACJ,QAAM,cAAc,IAAI,YAAY,IAAI;AAExC,MAAI,MAAM;AAAA,IACN,MAAM;AAAA,IACN,QAAQ;AAAA,MACJ,iBAAiB;AAAA,QACb,KAAK,OAAO,MAAW;AACnB,cAAI;AACA,kBAAM,YAAY,YAAY;AAC9B,mBAAO,eAAe,sBAAsB,CAAC,GAAG,IAAI;AAAA,UACxD,SAAS,OAAO;AACZ,mBAAO;AAAA,cACH,sBAAsB,CAAC;AAAA,cACvB;AAAA,YACJ;AAAA,UACJ;AAAA,QACJ;AAAA,MACJ;AAAA,IACJ;AAAA,EACJ,CAAC;AAED,UAAQ;AAAA,IACJ,mDAAmD,UAAU;AAAA,EACjE;AACJ;","names":[]}
|
package/dist/api/index.d.ts
CHANGED
@@ -1,7 +1,9 @@
|
|
1
1
|
export { handleError, handleResponse } from './response_handler.js';
|
2
2
|
export { validateBody, validateParams, validateQuery } from './zod_utils.js';
|
3
|
+
export { HealthCheck, setupHealthCheckServer } from './health-check.js';
|
3
4
|
import '../errors/api_errors.js';
|
4
5
|
import '../errors/base_error.js';
|
5
6
|
import '../errors/database_errors.js';
|
6
7
|
import '../errors/external_dependency_error.js';
|
8
|
+
import '../types/response_context.js';
|
7
9
|
import 'zod';
|
package/dist/api/index.js
CHANGED
package/dist/api/index.js.map
CHANGED
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"sources":["../../src/api/index.ts"],"sourcesContent":["export * from \"./response_handler\";\nexport * from \"./zod_utils\";\n"],"mappings":"AAAA,cAAc;AACd,cAAc;","names":[]}
|
1
|
+
{"version":3,"sources":["../../src/api/index.ts"],"sourcesContent":["export * from \"./response_handler\";\nexport * from \"./zod_utils\";\nexport * from \"./health-check\";\n"],"mappings":"AAAA,cAAc;AACd,cAAc;AACd,cAAc;","names":[]}
|
@@ -1,12 +1,9 @@
|
|
1
1
|
import { ApiError } from '../errors/api_errors.js';
|
2
2
|
import { DatabaseError } from '../errors/database_errors.js';
|
3
3
|
import { ExternalDependencyError } from '../errors/external_dependency_error.js';
|
4
|
+
import { ResponseContext } from '../types/response_context.js';
|
4
5
|
import '../errors/base_error.js';
|
5
6
|
|
6
|
-
type ResponseContext = {
|
7
|
-
status: (statusCode: number) => ResponseContext;
|
8
|
-
json: (body: any) => any;
|
9
|
-
};
|
10
7
|
declare const handleResponse: (c: ResponseContext, data: any) => any;
|
11
8
|
declare const handleError: (c: ResponseContext, error: ApiError | ExternalDependencyError | DatabaseError) => any;
|
12
9
|
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"sources":["../../src/api/response_handler.ts"],"sourcesContent":["import { ApiError, DatabaseError, ExternalDependencyError } from \"../errors\"; // Define your error types\nimport { httpResposneCodes, errorCodes } from \"../constants\"; // Define your success codes\
|
1
|
+
{"version":3,"sources":["../../src/api/response_handler.ts"],"sourcesContent":["import { ApiError, DatabaseError, ExternalDependencyError } from \"../errors\"; // Define your error types\nimport { httpResposneCodes, errorCodes } from \"../constants\"; // Define your success codes\nimport type { ResponseContext } from \"../types\";\n\nexport const handleResponse = (c: ResponseContext, data: any) => {\n // This method returns a success response with the status and data.\n return c\n .status(httpResposneCodes.OK_RESPONSE) // Use success code from httpResposneCodes\n .json({\n status: \"success\",\n data: data,\n });\n};\n\nexport const handleError = (\n c: ResponseContext,\n error: ApiError | ExternalDependencyError | DatabaseError\n) => {\n // This method returns an error response with the error details.\n return c\n .status(error.code ?? errorCodes.api.INTERNAL_SERVER_ERROR) // Use error code or default to internal server error\n .json({\n status: \"error\",\n message: error.message,\n name: error.name,\n code: error.code,\n details: error.context,\n });\n};\n"],"mappings":"AAAA,SAAS,UAAU,eAAe,+BAA+B;AACjE,SAAS,mBAAmB,kBAAkB;AAGvC,MAAM,iBAAiB,CAAC,GAAoB,SAAc;AAE7D,SAAO,EACF,OAAO,kBAAkB,WAAW,EACpC,KAAK;AAAA,IACF,QAAQ;AAAA,IACR;AAAA,EACJ,CAAC;AACT;AAEO,MAAM,cAAc,CACvB,GACA,UACC;AAED,SAAO,EACF,OAAO,MAAM,QAAQ,WAAW,IAAI,qBAAqB,EACzD,KAAK;AAAA,IACF,QAAQ;AAAA,IACR,SAAS,MAAM;AAAA,IACf,MAAM,MAAM;AAAA,IACZ,MAAM,MAAM;AAAA,IACZ,SAAS,MAAM;AAAA,EACnB,CAAC;AACT;","names":[]}
|
@@ -0,0 +1,23 @@
|
|
1
|
+
import { AbstractEventConsumer } from './abstract_event_consumer.js';
|
2
|
+
import '../errors/consumer_errors.js';
|
3
|
+
import '../errors/base_error.js';
|
4
|
+
import 'events';
|
5
|
+
|
6
|
+
declare abstract class AbstractCronEventConsumer extends AbstractEventConsumer {
|
7
|
+
private cronJob;
|
8
|
+
/**
|
9
|
+
* Start the cron job with the given cron expression.
|
10
|
+
* @param cronExpr Cron expression string
|
11
|
+
*/
|
12
|
+
startCron(cronExpr: string): void;
|
13
|
+
/**
|
14
|
+
* Stop the cron job.
|
15
|
+
*/
|
16
|
+
stopCron(): void;
|
17
|
+
/**
|
18
|
+
* Implement this in subclasses to define what happens on each cron tick.
|
19
|
+
*/
|
20
|
+
protected abstract onTick(): Promise<void>;
|
21
|
+
}
|
22
|
+
|
23
|
+
export { AbstractCronEventConsumer };
|
@@ -0,0 +1,30 @@
|
|
1
|
+
import { Cron } from "croner";
|
2
|
+
import { AbstractEventConsumer } from "./abstract_event_consumer";
|
3
|
+
class AbstractCronEventConsumer extends AbstractEventConsumer {
|
4
|
+
cronJob = null;
|
5
|
+
/**
|
6
|
+
* Start the cron job with the given cron expression.
|
7
|
+
* @param cronExpr Cron expression string
|
8
|
+
*/
|
9
|
+
startCron(cronExpr) {
|
10
|
+
if (this.cronJob) {
|
11
|
+
this.cronJob.stop();
|
12
|
+
}
|
13
|
+
this.cronJob = new Cron(cronExpr, { protect: true }, async () => {
|
14
|
+
await this.onTick();
|
15
|
+
});
|
16
|
+
}
|
17
|
+
/**
|
18
|
+
* Stop the cron job.
|
19
|
+
*/
|
20
|
+
stopCron() {
|
21
|
+
if (this.cronJob) {
|
22
|
+
this.cronJob.stop();
|
23
|
+
this.cronJob = null;
|
24
|
+
}
|
25
|
+
}
|
26
|
+
}
|
27
|
+
export {
|
28
|
+
AbstractCronEventConsumer
|
29
|
+
};
|
30
|
+
//# sourceMappingURL=abstract_cron_event_consumer.js.map
|
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"sources":["../../src/consumers/abstract_cron_event_consumer.ts"],"sourcesContent":["import { Cron } from \"croner\";\nimport { AbstractEventConsumer } from \"./abstract_event_consumer\";\n\nexport abstract class AbstractCronEventConsumer extends AbstractEventConsumer {\n private cronJob: Cron | null = null;\n\n /**\n * Start the cron job with the given cron expression.\n * @param cronExpr Cron expression string\n */\n public startCron(cronExpr: string): void {\n if (this.cronJob) {\n this.cronJob.stop();\n }\n this.cronJob = new Cron(cronExpr, { protect: true }, async () => {\n await this.onTick();\n });\n }\n\n /**\n * Stop the cron job.\n */\n public stopCron(): void {\n if (this.cronJob) {\n this.cronJob.stop();\n this.cronJob = null;\n }\n }\n\n /**\n * Implement this in subclasses to define what happens on each cron tick.\n */\n protected abstract onTick(): Promise<void>;\n}\n"],"mappings":"AAAA,SAAS,YAAY;AACrB,SAAS,6BAA6B;AAE/B,MAAe,kCAAkC,sBAAsB;AAAA,EAClE,UAAuB;AAAA;AAAA;AAAA;AAAA;AAAA,EAMxB,UAAU,UAAwB;AACrC,QAAI,KAAK,SAAS;AACd,WAAK,QAAQ,KAAK;AAAA,IACtB;AACA,SAAK,UAAU,IAAI,KAAK,UAAU,EAAE,SAAS,KAAK,GAAG,YAAY;AAC7D,YAAM,KAAK,OAAO;AAAA,IACtB,CAAC;AAAA,EACL;AAAA;AAAA;AAAA;AAAA,EAKO,WAAiB;AACpB,QAAI,KAAK,SAAS;AACd,WAAK,QAAQ,KAAK;AAClB,WAAK,UAAU;AAAA,IACnB;AAAA,EACJ;AAMJ;","names":[]}
|
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"sources":["../../src/consumers/abstract_event_consumer.ts"],"sourcesContent":["import type { ConsumerError } from \"../errors\";\nimport { EventEmitter } from \"events\";\n\nexport class AbstractEventConsumer extends EventEmitter {\n constructor() {\n super();\n }\n\n /**\n * @public\n *\n * Method to register listener for events. The Abstract Event Consumer emits fatalError\n * event.\n *\n * @param {\"fatalError\"} eventName - Event name to register listener for.\n * @param listener - Listener to be called when emitting the event.\n *\n * @returns {this} - Returns an instance of the class.\n */\n on(\n eventName: \"fatalError\",\n listener: (error: Error | ConsumerError) => void\n ): this {\n return super.on(eventName, listener);\n }\n /**\n * @public\n *\n * Method to register listener for events that will be called only once. The Abstract Event Consumer emits fatalError\n * event.\n *\n * @param {\"fatalError\"} eventName - Event name to register listener for.\n * @param listener - Listener to be called when emitting the event.\n *\n * @returns {this} - Returns an instance of the class.\n */\n once(\n eventName: \"fatalError\",\n listener: (error: Error | ConsumerError) => void\n ): this {\n return super.on(eventName, listener);\n }\n\n protected onFatalError(error: Error | ConsumerError): void {\n this.emit(\"fatalError\", error);\n }\n}\n"],"mappings":"AACA,SAAS,oBAAoB;AAEtB,MAAM,8BAA8B,aAAa;AAAA,EACpD,cAAc;AACV,UAAM;AAAA,EACV;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAaA,GACI,WACA,UACI;AACJ,WAAO,MAAM,GAAG,WAAW,QAAQ;AAAA,EACvC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYA,KACI,WACA,UACI;AACJ,WAAO,MAAM,GAAG,WAAW,QAAQ;AAAA,EACvC;AAAA,EAEU,aAAa,OAAoC;AACvD,SAAK,KAAK,cAAc,KAAK;AAAA,EACjC;AACJ;","names":[]}
|
@@ -1,4 +1,4 @@
|
|
1
|
-
import { PublicClient } from 'viem';
|
1
|
+
import { PublicClient, WatchEventReturnType } from 'viem';
|
2
2
|
import { ConsumerError } from '../errors/consumer_errors.js';
|
3
3
|
import { IObserver } from '../types/observer.js';
|
4
4
|
import { IEventConsumerConfig } from '../types/event_consumer_config.js';
|
@@ -8,17 +8,19 @@ import 'events';
|
|
8
8
|
|
9
9
|
declare class EventConsumer<T, U> extends AbstractEventConsumer {
|
10
10
|
private config;
|
11
|
-
private consumerRunning;
|
12
11
|
protected client: PublicClient | null;
|
13
|
-
protected observer: IObserver<
|
12
|
+
protected observer: IObserver<T, ConsumerError, U> | null;
|
13
|
+
isBackfillingInProcess: boolean;
|
14
|
+
protected unWatchFunction: WatchEventReturnType | undefined;
|
14
15
|
constructor(config: IEventConsumerConfig);
|
16
|
+
private _backfillEvents;
|
15
17
|
start(observer: IObserver<T, ConsumerError, U>): Promise<void>;
|
16
18
|
/**
|
17
19
|
* Private method which updates the connection status of consumer to disconnected, and removes all listeners.
|
18
20
|
*
|
19
21
|
* @returns {void}
|
20
22
|
*/
|
21
|
-
|
23
|
+
onDisconnect(): void;
|
22
24
|
}
|
23
25
|
|
24
26
|
export { EventConsumer };
|
@@ -0,0 +1,141 @@
|
|
1
|
+
import { Logger } from "../logger";
|
2
|
+
import { createPublicClient, http } from "viem";
|
3
|
+
import { ConsumerError, ExternalDependencyError } from "../errors";
|
4
|
+
import { AbstractEventConsumer } from "./abstract_event_consumer";
|
5
|
+
import { errorCodes } from "../constants";
|
6
|
+
class EventConsumer extends AbstractEventConsumer {
|
7
|
+
constructor(config) {
|
8
|
+
super();
|
9
|
+
this.config = config;
|
10
|
+
this.client = createPublicClient({
|
11
|
+
chain: {
|
12
|
+
id: config.chainId,
|
13
|
+
name: "custom",
|
14
|
+
rpcUrls: { default: { http: [config.rpcUrl] } },
|
15
|
+
nativeCurrency: config.nativeCurrency
|
16
|
+
},
|
17
|
+
transport: http(config.rpcUrl)
|
18
|
+
});
|
19
|
+
}
|
20
|
+
client = null;
|
21
|
+
observer = null;
|
22
|
+
isBackfillingInProcess = false;
|
23
|
+
unWatchFunction;
|
24
|
+
async _backfillEvents(fromBlock, toBlock, step) {
|
25
|
+
Logger.info({
|
26
|
+
location: "event_consumer",
|
27
|
+
functon: "_backfillEvents",
|
28
|
+
status: `[Backfill] Starting backfill from block ${fromBlock.toString()} to ${toBlock.toString()}`
|
29
|
+
});
|
30
|
+
for (let start = fromBlock; start <= toBlock; start += step) {
|
31
|
+
const end = start + step - 1n > toBlock ? toBlock : start + step - 1n;
|
32
|
+
Logger.debug({
|
33
|
+
location: "event_consumer",
|
34
|
+
functon: "_backfillEvents",
|
35
|
+
status: `[Backfill] Fetching logs from block ${start.toString()} to ${end.toString()}`
|
36
|
+
});
|
37
|
+
const getLogsConfig = {
|
38
|
+
address: this.config.contractAddress,
|
39
|
+
fromBlock: start,
|
40
|
+
toBlock: end
|
41
|
+
};
|
42
|
+
if (this.config.events.length > 1) {
|
43
|
+
getLogsConfig.events = this.config.events;
|
44
|
+
} else {
|
45
|
+
getLogsConfig.event = this.config.events[0];
|
46
|
+
}
|
47
|
+
const logs = await this.client?.getLogs(getLogsConfig);
|
48
|
+
Logger.debug({
|
49
|
+
location: "event_consumer",
|
50
|
+
functon: "_backfillEvents",
|
51
|
+
status: `[Backfill] ${logs?.length} logs from block ${start.toString()} to ${end.toString()}`
|
52
|
+
});
|
53
|
+
await this.observer?.next(logs);
|
54
|
+
}
|
55
|
+
}
|
56
|
+
async start(observer) {
|
57
|
+
try {
|
58
|
+
this.isBackfillingInProcess = true;
|
59
|
+
this.observer = observer;
|
60
|
+
const latestBlock = await this.client?.getBlockNumber() ?? BigInt(0);
|
61
|
+
Logger.debug({
|
62
|
+
location: "event_consumer",
|
63
|
+
functon: "start",
|
64
|
+
data: {
|
65
|
+
latestBlock: latestBlock.toString(),
|
66
|
+
startBlock: this.config.startBlock.toString(),
|
67
|
+
pollBatchSize: this.config.pollBatchSize.toString()
|
68
|
+
}
|
69
|
+
});
|
70
|
+
if (latestBlock - this.config.startBlock > BigInt(0)) {
|
71
|
+
await this._backfillEvents(
|
72
|
+
this.config.startBlock,
|
73
|
+
latestBlock,
|
74
|
+
this.config.pollBatchSize
|
75
|
+
);
|
76
|
+
}
|
77
|
+
this.isBackfillingInProcess = false;
|
78
|
+
Logger.debug({
|
79
|
+
location: "event_consumer",
|
80
|
+
functon: "start",
|
81
|
+
status: `Starting event subscription`
|
82
|
+
});
|
83
|
+
this.unWatchFunction = this.client?.watchEvent({
|
84
|
+
address: this.config.contractAddress,
|
85
|
+
events: this.config.events,
|
86
|
+
fromBlock: latestBlock,
|
87
|
+
onLogs: (logs) => {
|
88
|
+
const transformedLogs = logs;
|
89
|
+
this.observer?.next(transformedLogs);
|
90
|
+
},
|
91
|
+
onError: (error) => {
|
92
|
+
this.onDisconnect();
|
93
|
+
this.onFatalError(
|
94
|
+
new ConsumerError(error.name, {
|
95
|
+
name: "CONSUMER_ERROR",
|
96
|
+
code: 1001,
|
97
|
+
isFatal: true,
|
98
|
+
origin: "EventConsumer",
|
99
|
+
context: { eventName: this.config.events }
|
100
|
+
})
|
101
|
+
);
|
102
|
+
}
|
103
|
+
});
|
104
|
+
Logger.debug({
|
105
|
+
location: "event_consumer",
|
106
|
+
functon: "start",
|
107
|
+
status: `Subscribed to event`
|
108
|
+
});
|
109
|
+
} catch (error) {
|
110
|
+
this.onDisconnect();
|
111
|
+
this.onFatalError(
|
112
|
+
new ExternalDependencyError(
|
113
|
+
this.config.rpcUrl,
|
114
|
+
"Failed to start the event consumer",
|
115
|
+
{
|
116
|
+
externalCode: errorCodes.external.UNKNOWN_EXTERNAL_DEPENDENCY_ERROR,
|
117
|
+
rawError: error,
|
118
|
+
origin: "EventConsumer",
|
119
|
+
context: { eventName: this.config.events }
|
120
|
+
}
|
121
|
+
)
|
122
|
+
);
|
123
|
+
}
|
124
|
+
}
|
125
|
+
/**
|
126
|
+
* Private method which updates the connection status of consumer to disconnected, and removes all listeners.
|
127
|
+
*
|
128
|
+
* @returns {void}
|
129
|
+
*/
|
130
|
+
onDisconnect() {
|
131
|
+
this.removeAllListeners();
|
132
|
+
if (this.unWatchFunction) {
|
133
|
+
this.unWatchFunction();
|
134
|
+
}
|
135
|
+
this.observer?.closed();
|
136
|
+
}
|
137
|
+
}
|
138
|
+
export {
|
139
|
+
EventConsumer
|
140
|
+
};
|
141
|
+
//# sourceMappingURL=event_consumer.js.map
|
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"sources":["../../src/consumers/event_consumer.ts"],"sourcesContent":["import { Logger } from \"../logger\";\nimport { createPublicClient, http, type PublicClient, type WatchEventReturnType } from \"viem\";\nimport { ConsumerError, ExternalDependencyError } from \"../errors\";\nimport type { IEventConsumerConfig, IObserver } from \"../types\";\nimport { AbstractEventConsumer } from \"./abstract_event_consumer\";\nimport { errorCodes } from \"../constants\";\n\nexport class EventConsumer<T, U> extends AbstractEventConsumer {\n protected client: PublicClient | null = null;\n protected observer: IObserver<T, ConsumerError, U> | null = null;\n public isBackfillingInProcess: boolean = false;\n protected unWatchFunction: WatchEventReturnType | undefined;\n\n constructor(private config: IEventConsumerConfig) {\n super();\n this.client = createPublicClient({\n chain: {\n id: config.chainId,\n name: \"custom\",\n rpcUrls: { default: { http: [config.rpcUrl] } },\n nativeCurrency: config.nativeCurrency,\n },\n transport: http(config.rpcUrl),\n });\n }\n\n private async _backfillEvents(\n fromBlock: bigint,\n toBlock: bigint,\n step: bigint\n ): Promise<void> {\n Logger.info({\n location: \"event_consumer\",\n functon: \"_backfillEvents\",\n status: `[Backfill] Starting backfill from block ${fromBlock.toString()} to ${toBlock.toString()}`,\n });\n\n for (let start = fromBlock; start <= toBlock; start += step) {\n const end =\n start + step - 1n > toBlock ? toBlock : start + step - 1n;\n Logger.debug({\n location: \"event_consumer\",\n functon: \"_backfillEvents\",\n status: `[Backfill] Fetching logs from block ${start.toString()} to ${end.toString()}`,\n });\n const getLogsConfig: any = {\n address: this.config.contractAddress,\n fromBlock: start,\n toBlock: end,\n };\n if (this.config.events.length > 1) {\n getLogsConfig.events = this.config.events;\n } else {\n getLogsConfig.event = this.config.events[0];\n }\n const logs = await this.client?.getLogs(getLogsConfig);\n Logger.debug({\n location: \"event_consumer\",\n functon: \"_backfillEvents\",\n status: `[Backfill] ${logs?.length} logs from block ${start.toString()} to ${end.toString()}`,\n });\n await this.observer?.next(logs as T);\n }\n }\n\n public async start(\n observer: IObserver<T, ConsumerError, U>\n ): Promise<void> {\n try {\n this.isBackfillingInProcess = true\n this.observer = observer;\n const latestBlock: bigint =\n (await this.client?.getBlockNumber()) ?? BigInt(0);\n Logger.debug({\n location: \"event_consumer\",\n functon: \"start\",\n data: {\n latestBlock: latestBlock.toString(),\n startBlock: this.config.startBlock.toString(),\n pollBatchSize: this.config.pollBatchSize.toString(),\n },\n });\n if (latestBlock - this.config.startBlock > BigInt(0)) {\n await this._backfillEvents(\n this.config.startBlock,\n latestBlock,\n this.config.pollBatchSize\n );\n }\n this.isBackfillingInProcess = false\n\n Logger.debug({\n location: \"event_consumer\",\n functon: \"start\",\n status: `Starting event subscription`,\n });\n\n this.unWatchFunction = this.client?.watchEvent({\n address: this.config.contractAddress,\n events: this.config.events,\n fromBlock: latestBlock,\n onLogs: (logs) => {\n const transformedLogs = logs as T;\n this.observer?.next(transformedLogs);\n },\n onError: (error) => {\n this.onDisconnect();\n this.onFatalError(\n new ConsumerError(error.name, {\n name: \"CONSUMER_ERROR\",\n code: 1001,\n isFatal: true,\n origin: \"EventConsumer\",\n context: { eventName: this.config.events },\n })\n );\n },\n });\n\n Logger.debug({\n location: \"event_consumer\",\n functon: \"start\",\n status: `Subscribed to event`,\n });\n } catch (error) {\n this.onDisconnect();\n this.onFatalError(\n new ExternalDependencyError(\n this.config.rpcUrl,\n \"Failed to start the event consumer\",\n {\n externalCode:\n errorCodes.external\n .UNKNOWN_EXTERNAL_DEPENDENCY_ERROR,\n rawError: error as Error,\n origin: \"EventConsumer\",\n context: { eventName: this.config.events },\n }\n )\n );\n }\n }\n\n /**\n * Private method which updates the connection status of consumer to disconnected, and removes all listeners.\n *\n * @returns {void}\n */\n public onDisconnect(): void {\n this.removeAllListeners();\n if (this.unWatchFunction) {\n this.unWatchFunction();\n }\n this.observer?.closed();\n }\n}\n"],"mappings":"AAAA,SAAS,cAAc;AACvB,SAAS,oBAAoB,YAA0D;AACvF,SAAS,eAAe,+BAA+B;AAEvD,SAAS,6BAA6B;AACtC,SAAS,kBAAkB;AAEpB,MAAM,sBAA4B,sBAAsB;AAAA,EAM3D,YAAoB,QAA8B;AAC9C,UAAM;AADU;AAEhB,SAAK,SAAS,mBAAmB;AAAA,MAC7B,OAAO;AAAA,QACH,IAAI,OAAO;AAAA,QACX,MAAM;AAAA,QACN,SAAS,EAAE,SAAS,EAAE,MAAM,CAAC,OAAO,MAAM,EAAE,EAAE;AAAA,QAC9C,gBAAgB,OAAO;AAAA,MAC3B;AAAA,MACA,WAAW,KAAK,OAAO,MAAM;AAAA,IACjC,CAAC;AAAA,EACL;AAAA,EAhBU,SAA8B;AAAA,EAC9B,WAAkD;AAAA,EACrD,yBAAkC;AAAA,EAC/B;AAAA,EAeV,MAAc,gBACV,WACA,SACA,MACa;AACb,WAAO,KAAK;AAAA,MACR,UAAU;AAAA,MACV,SAAS;AAAA,MACT,QAAQ,2CAA2C,UAAU,SAAS,CAAC,OAAO,QAAQ,SAAS,CAAC;AAAA,IACpG,CAAC;AAED,aAAS,QAAQ,WAAW,SAAS,SAAS,SAAS,MAAM;AACzD,YAAM,MACF,QAAQ,OAAO,KAAK,UAAU,UAAU,QAAQ,OAAO;AAC3D,aAAO,MAAM;AAAA,QACT,UAAU;AAAA,QACV,SAAS;AAAA,QACT,QAAQ,uCAAuC,MAAM,SAAS,CAAC,OAAO,IAAI,SAAS,CAAC;AAAA,MACxF,CAAC;AACD,YAAM,gBAAqB;AAAA,QACvB,SAAS,KAAK,OAAO;AAAA,QACrB,WAAW;AAAA,QACX,SAAS;AAAA,MACb;AACA,UAAI,KAAK,OAAO,OAAO,SAAS,GAAG;AAC/B,sBAAc,SAAS,KAAK,OAAO;AAAA,MACvC,OAAO;AACH,sBAAc,QAAQ,KAAK,OAAO,OAAO,CAAC;AAAA,MAC9C;AACA,YAAM,OAAO,MAAM,KAAK,QAAQ,QAAQ,aAAa;AACrD,aAAO,MAAM;AAAA,QACT,UAAU;AAAA,QACV,SAAS;AAAA,QACT,QAAQ,cAAc,MAAM,MAAM,oBAAoB,MAAM,SAAS,CAAC,OAAO,IAAI,SAAS,CAAC;AAAA,MAC/F,CAAC;AACD,YAAM,KAAK,UAAU,KAAK,IAAS;AAAA,IACvC;AAAA,EACJ;AAAA,EAEA,MAAa,MACT,UACa;AACb,QAAI;AACA,WAAK,yBAAyB;AAC9B,WAAK,WAAW;AAChB,YAAM,cACD,MAAM,KAAK,QAAQ,eAAe,KAAM,OAAO,CAAC;AACrD,aAAO,MAAM;AAAA,QACT,UAAU;AAAA,QACV,SAAS;AAAA,QACT,MAAM;AAAA,UACF,aAAa,YAAY,SAAS;AAAA,UAClC,YAAY,KAAK,OAAO,WAAW,SAAS;AAAA,UAC5C,eAAe,KAAK,OAAO,cAAc,SAAS;AAAA,QACtD;AAAA,MACJ,CAAC;AACD,UAAI,cAAc,KAAK,OAAO,aAAa,OAAO,CAAC,GAAG;AAClD,cAAM,KAAK;AAAA,UACP,KAAK,OAAO;AAAA,UACZ;AAAA,UACA,KAAK,OAAO;AAAA,QAChB;AAAA,MACJ;AACA,WAAK,yBAAyB;AAE9B,aAAO,MAAM;AAAA,QACT,UAAU;AAAA,QACV,SAAS;AAAA,QACT,QAAQ;AAAA,MACZ,CAAC;AAED,WAAK,kBAAkB,KAAK,QAAQ,WAAW;AAAA,QAC3C,SAAS,KAAK,OAAO;AAAA,QACrB,QAAQ,KAAK,OAAO;AAAA,QACpB,WAAW;AAAA,QACX,QAAQ,CAAC,SAAS;AACd,gBAAM,kBAAkB;AACxB,eAAK,UAAU,KAAK,eAAe;AAAA,QACvC;AAAA,QACA,SAAS,CAAC,UAAU;AAChB,eAAK,aAAa;AAClB,eAAK;AAAA,YACD,IAAI,cAAc,MAAM,MAAM;AAAA,cAC1B,MAAM;AAAA,cACN,MAAM;AAAA,cACN,SAAS;AAAA,cACT,QAAQ;AAAA,cACR,SAAS,EAAE,WAAW,KAAK,OAAO,OAAO;AAAA,YAC7C,CAAC;AAAA,UACL;AAAA,QACJ;AAAA,MACJ,CAAC;AAED,aAAO,MAAM;AAAA,QACT,UAAU;AAAA,QACV,SAAS;AAAA,QACT,QAAQ;AAAA,MACZ,CAAC;AAAA,IACL,SAAS,OAAO;AACZ,WAAK,aAAa;AAClB,WAAK;AAAA,QACD,IAAI;AAAA,UACA,KAAK,OAAO;AAAA,UACZ;AAAA,UACA;AAAA,YACI,cACI,WAAW,SACN;AAAA,YACT,UAAU;AAAA,YACV,QAAQ;AAAA,YACR,SAAS,EAAE,WAAW,KAAK,OAAO,OAAO;AAAA,UAC7C;AAAA,QACJ;AAAA,MACJ;AAAA,IACJ;AAAA,EACJ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOO,eAAqB;AACxB,SAAK,mBAAmB;AACxB,QAAI,KAAK,iBAAiB;AACtB,WAAK,gBAAgB;AAAA,IACzB;AACA,SAAK,UAAU,OAAO;AAAA,EAC1B;AACJ;","names":[]}
|
@@ -0,0 +1,11 @@
|
|
1
|
+
export { EventConsumer } from './event_consumer.js';
|
2
|
+
export { RestAPIConsumer } from './rest_api_consumer.js';
|
3
|
+
import 'viem';
|
4
|
+
import '../errors/consumer_errors.js';
|
5
|
+
import '../errors/base_error.js';
|
6
|
+
import '../types/observer.js';
|
7
|
+
import '../types/event_consumer_config.js';
|
8
|
+
import './abstract_event_consumer.js';
|
9
|
+
import 'events';
|
10
|
+
import '../types/rest_api_consumer_config.js';
|
11
|
+
import './abstract_cron_event_consumer.js';
|
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"sources":["../../src/consumers/index.ts"],"sourcesContent":["export * from \"./event_consumer\";\nexport * from \"./rest_api_consumer\";\n"],"mappings":"AAAA,cAAc;AACd,cAAc;","names":[]}
|
@@ -0,0 +1,30 @@
|
|
1
|
+
import { ConsumerError } from '../errors/consumer_errors.js';
|
2
|
+
import { IObserver } from '../types/observer.js';
|
3
|
+
import { IRestAPIConsumerConfig } from '../types/rest_api_consumer_config.js';
|
4
|
+
import { AbstractCronEventConsumer } from './abstract_cron_event_consumer.js';
|
5
|
+
import '../errors/base_error.js';
|
6
|
+
import './abstract_event_consumer.js';
|
7
|
+
import 'events';
|
8
|
+
|
9
|
+
declare class RestAPIConsumer<T, U> extends AbstractCronEventConsumer {
|
10
|
+
private config;
|
11
|
+
protected observer: IObserver<T, ConsumerError, U> | null;
|
12
|
+
private currentPage;
|
13
|
+
private highestValueSeen;
|
14
|
+
constructor(config: IRestAPIConsumerConfig);
|
15
|
+
start(observer: IObserver<T, ConsumerError, U>): Promise<void>;
|
16
|
+
stop(): void;
|
17
|
+
protected onTick(): Promise<void>;
|
18
|
+
private fetchPage;
|
19
|
+
private getValueByPath;
|
20
|
+
private processDataForCurrentCountValue;
|
21
|
+
/**
|
22
|
+
* Updates the start count value to a specific value.
|
23
|
+
* This allows reprocessing data from a specified point.
|
24
|
+
*
|
25
|
+
* @param value The value to set as the new start count
|
26
|
+
*/
|
27
|
+
setStartValue(value: number): void;
|
28
|
+
}
|
29
|
+
|
30
|
+
export { RestAPIConsumer };
|
@@ -0,0 +1,128 @@
|
|
1
|
+
import { errorCodes } from "../constants";
|
2
|
+
import { ConsumerError } from "../errors";
|
3
|
+
import { AbstractCronEventConsumer } from "./abstract_cron_event_consumer";
|
4
|
+
class RestAPIConsumer extends AbstractCronEventConsumer {
|
5
|
+
constructor(config) {
|
6
|
+
super();
|
7
|
+
this.config = config;
|
8
|
+
this.highestValueSeen = this.config.startCount.value;
|
9
|
+
}
|
10
|
+
observer = null;
|
11
|
+
currentPage = 0;
|
12
|
+
highestValueSeen = 0;
|
13
|
+
async start(observer) {
|
14
|
+
this.observer = observer;
|
15
|
+
this.currentPage = 1;
|
16
|
+
if (this.config.cronExpr) {
|
17
|
+
this.startCron(this.config.cronExpr);
|
18
|
+
} else {
|
19
|
+
await this.onTick();
|
20
|
+
}
|
21
|
+
}
|
22
|
+
stop() {
|
23
|
+
this.stopCron();
|
24
|
+
}
|
25
|
+
async onTick() {
|
26
|
+
this.currentPage = 1;
|
27
|
+
let shouldFetchNextPage = true;
|
28
|
+
while (shouldFetchNextPage) {
|
29
|
+
shouldFetchNextPage = await this.fetchPage();
|
30
|
+
}
|
31
|
+
this.config.startCount.value = this.highestValueSeen;
|
32
|
+
}
|
33
|
+
async fetchPage() {
|
34
|
+
try {
|
35
|
+
const paginationParam = this.config.paginationParam || "page";
|
36
|
+
const params = {
|
37
|
+
...this.config.params || {},
|
38
|
+
[paginationParam]: this.currentPage.toString()
|
39
|
+
};
|
40
|
+
let url = this.config.apiUrl.toString();
|
41
|
+
if (Object.keys(params).length > 0) {
|
42
|
+
const queryString = new URLSearchParams(params).toString();
|
43
|
+
url += (url.includes("?") ? "&" : "?") + queryString;
|
44
|
+
}
|
45
|
+
const response = await fetch(url, {
|
46
|
+
method: this.config.method || "GET",
|
47
|
+
headers: this.config.headers,
|
48
|
+
body: this.config.body ? JSON.stringify(this.config.body) : void 0
|
49
|
+
});
|
50
|
+
if (!response.ok) {
|
51
|
+
throw new ConsumerError(
|
52
|
+
`Failed to fetch from API: ${response.statusText}`,
|
53
|
+
{
|
54
|
+
code: errorCodes.external.UNKNOWN_EXTERNAL_DEPENDENCY_ERROR,
|
55
|
+
isFatal: true,
|
56
|
+
origin: "rest_api_consumer"
|
57
|
+
}
|
58
|
+
);
|
59
|
+
}
|
60
|
+
let data = await response.json();
|
61
|
+
if (this.config.resultPath) {
|
62
|
+
data = this.getValueByPath(data, this.config.resultPath);
|
63
|
+
}
|
64
|
+
const countKey = this.config.startCount.key;
|
65
|
+
let currentValue = void 0;
|
66
|
+
if (Array.isArray(data)) {
|
67
|
+
if (data.length === 0) {
|
68
|
+
return false;
|
69
|
+
}
|
70
|
+
for (const elem of data) {
|
71
|
+
currentValue = this.processDataForCurrentCountValue(
|
72
|
+
elem,
|
73
|
+
countKey
|
74
|
+
);
|
75
|
+
}
|
76
|
+
} else {
|
77
|
+
currentValue = this.processDataForCurrentCountValue(
|
78
|
+
data,
|
79
|
+
countKey
|
80
|
+
);
|
81
|
+
}
|
82
|
+
this.observer?.next(data);
|
83
|
+
this.currentPage++;
|
84
|
+
if (currentValue === void 0) {
|
85
|
+
return false;
|
86
|
+
}
|
87
|
+
return currentValue > this.config.startCount.value;
|
88
|
+
} catch (error) {
|
89
|
+
this.onFatalError(error);
|
90
|
+
return false;
|
91
|
+
}
|
92
|
+
}
|
93
|
+
// Utility to get nested values using dot notation (e.g., "response.data.count")
|
94
|
+
getValueByPath(obj, path) {
|
95
|
+
return path.split(".").reduce((prev, curr) => {
|
96
|
+
return prev && prev[curr] !== void 0 ? prev[curr] : void 0;
|
97
|
+
}, obj);
|
98
|
+
}
|
99
|
+
processDataForCurrentCountValue(item, countKey) {
|
100
|
+
const value = this.getValueByPath(item, countKey);
|
101
|
+
if (value === void 0) {
|
102
|
+
throw new ConsumerError(
|
103
|
+
`Count key '${countKey}' not found in API response`,
|
104
|
+
{
|
105
|
+
code: errorCodes.external.UNKNOWN_EXTERNAL_DEPENDENCY_ERROR,
|
106
|
+
isFatal: false,
|
107
|
+
origin: "rest_api_consumer"
|
108
|
+
}
|
109
|
+
);
|
110
|
+
}
|
111
|
+
this.highestValueSeen = Math.max(this.highestValueSeen, value);
|
112
|
+
return value;
|
113
|
+
}
|
114
|
+
/**
|
115
|
+
* Updates the start count value to a specific value.
|
116
|
+
* This allows reprocessing data from a specified point.
|
117
|
+
*
|
118
|
+
* @param value The value to set as the new start count
|
119
|
+
*/
|
120
|
+
setStartValue(value) {
|
121
|
+
this.config.startCount.value = value;
|
122
|
+
this.highestValueSeen = value;
|
123
|
+
}
|
124
|
+
}
|
125
|
+
export {
|
126
|
+
RestAPIConsumer
|
127
|
+
};
|
128
|
+
//# sourceMappingURL=rest_api_consumer.js.map
|
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"sources":["../../src/consumers/rest_api_consumer.ts"],"sourcesContent":["import { errorCodes } from \"../constants\";\nimport { ConsumerError } from \"../errors\";\nimport type { IObserver, IRestAPIConsumerConfig } from \"../types\";\nimport { AbstractCronEventConsumer } from \"./abstract_cron_event_consumer\";\n\nexport class RestAPIConsumer<T, U> extends AbstractCronEventConsumer {\n protected observer: IObserver<T, ConsumerError, U> | null = null;\n private currentPage = 0;\n private highestValueSeen = 0;\n\n constructor(private config: IRestAPIConsumerConfig) {\n super();\n this.highestValueSeen = this.config.startCount.value;\n }\n\n public async start(\n observer: IObserver<T, ConsumerError, U>\n ): Promise<void> {\n this.observer = observer;\n this.currentPage = 1;\n if (this.config.cronExpr) {\n this.startCron(this.config.cronExpr);\n } else {\n // fallback: run once if no cron\n await this.onTick();\n }\n }\n\n public stop(): void {\n this.stopCron();\n }\n\n protected async onTick(): Promise<void> {\n // Reset page counter at the start of each cron run\n this.currentPage = 1;\n\n // Keep fetching pages until we've caught up\n let shouldFetchNextPage = true;\n\n while (shouldFetchNextPage) {\n shouldFetchNextPage = await this.fetchPage();\n }\n\n // Update config with the highest value we've seen\n // This prevents fetching the same data in the next cron run\n this.config.startCount.value = this.highestValueSeen;\n }\n\n private async fetchPage(): Promise<boolean> {\n try {\n // Use paginationParam from config or default to \"page\"\n const paginationParam = this.config.paginationParam || \"page\";\n const params = {\n ...(this.config.params || {}),\n [paginationParam]: this.currentPage.toString(),\n };\n\n let url = this.config.apiUrl.toString();\n if (Object.keys(params).length > 0) {\n const queryString = new URLSearchParams(params).toString();\n url += (url.includes(\"?\") ? \"&\" : \"?\") + queryString;\n }\n\n const response = await fetch(url, {\n method: this.config.method || \"GET\",\n headers: this.config.headers,\n body: this.config.body\n ? JSON.stringify(this.config.body)\n : undefined,\n });\n\n if (!response.ok) {\n throw new ConsumerError(\n `Failed to fetch from API: ${response.statusText}`,\n {\n code: errorCodes.external\n .UNKNOWN_EXTERNAL_DEPENDENCY_ERROR,\n isFatal: true,\n origin: \"rest_api_consumer\",\n }\n );\n }\n\n let data = await response.json();\n if (this.config.resultPath) {\n data = this.getValueByPath(data, this.config.resultPath);\n }\n\n // Extract the value we're tracking\n const countKey = this.config.startCount.key;\n let currentValue: number | undefined = undefined;\n\n if (Array.isArray(data)) {\n if (data.length === 0) {\n return false;\n }\n for (const elem of data) {\n currentValue = this.processDataForCurrentCountValue(\n elem,\n countKey\n );\n }\n } else {\n currentValue = this.processDataForCurrentCountValue(\n data,\n countKey\n );\n }\n\n // Send data to observer\n this.observer?.next(data);\n\n // Increment page for next potential fetch\n this.currentPage++;\n\n // Continue fetching if the current value is <= our threshold\n if (currentValue === undefined) {\n // If currentValue was never set, stop fetching\n return false;\n }\n return currentValue > this.config.startCount.value;\n } catch (error) {\n this.onFatalError(error as ConsumerError);\n return false; // Stop fetching on error\n }\n }\n\n // Utility to get nested values using dot notation (e.g., \"response.data.count\")\n private getValueByPath(obj: any, path: string): any {\n return path.split(\".\").reduce((prev, curr) => {\n return prev && prev[curr] !== undefined ? prev[curr] : undefined;\n }, obj);\n }\n\n private processDataForCurrentCountValue(\n item: any,\n countKey: string\n ): number {\n const value = this.getValueByPath(item, countKey);\n if (value === undefined) {\n throw new ConsumerError(\n `Count key '${countKey}' not found in API response`,\n {\n code: errorCodes.external.UNKNOWN_EXTERNAL_DEPENDENCY_ERROR,\n isFatal: false,\n origin: \"rest_api_consumer\",\n }\n );\n }\n this.highestValueSeen = Math.max(this.highestValueSeen, value);\n return value;\n }\n\n /**\n * Updates the start count value to a specific value.\n * This allows reprocessing data from a specified point.\n *\n * @param value The value to set as the new start count\n */\n public setStartValue(value: number): void {\n this.config.startCount.value = value;\n this.highestValueSeen = value;\n }\n}\n"],"mappings":"AAAA,SAAS,kBAAkB;AAC3B,SAAS,qBAAqB;AAE9B,SAAS,iCAAiC;AAEnC,MAAM,wBAA8B,0BAA0B;AAAA,EAKjE,YAAoB,QAAgC;AAChD,UAAM;AADU;AAEhB,SAAK,mBAAmB,KAAK,OAAO,WAAW;AAAA,EACnD;AAAA,EAPU,WAAkD;AAAA,EACpD,cAAc;AAAA,EACd,mBAAmB;AAAA,EAO3B,MAAa,MACT,UACa;AACb,SAAK,WAAW;AAChB,SAAK,cAAc;AACnB,QAAI,KAAK,OAAO,UAAU;AACtB,WAAK,UAAU,KAAK,OAAO,QAAQ;AAAA,IACvC,OAAO;AAEH,YAAM,KAAK,OAAO;AAAA,IACtB;AAAA,EACJ;AAAA,EAEO,OAAa;AAChB,SAAK,SAAS;AAAA,EAClB;AAAA,EAEA,MAAgB,SAAwB;AAEpC,SAAK,cAAc;AAGnB,QAAI,sBAAsB;AAE1B,WAAO,qBAAqB;AACxB,4BAAsB,MAAM,KAAK,UAAU;AAAA,IAC/C;AAIA,SAAK,OAAO,WAAW,QAAQ,KAAK;AAAA,EACxC;AAAA,EAEA,MAAc,YAA8B;AACxC,QAAI;AAEA,YAAM,kBAAkB,KAAK,OAAO,mBAAmB;AACvD,YAAM,SAAS;AAAA,QACX,GAAI,KAAK,OAAO,UAAU,CAAC;AAAA,QAC3B,CAAC,eAAe,GAAG,KAAK,YAAY,SAAS;AAAA,MACjD;AAEA,UAAI,MAAM,KAAK,OAAO,OAAO,SAAS;AACtC,UAAI,OAAO,KAAK,MAAM,EAAE,SAAS,GAAG;AAChC,cAAM,cAAc,IAAI,gBAAgB,MAAM,EAAE,SAAS;AACzD,gBAAQ,IAAI,SAAS,GAAG,IAAI,MAAM,OAAO;AAAA,MAC7C;AAEA,YAAM,WAAW,MAAM,MAAM,KAAK;AAAA,QAC9B,QAAQ,KAAK,OAAO,UAAU;AAAA,QAC9B,SAAS,KAAK,OAAO;AAAA,QACrB,MAAM,KAAK,OAAO,OACZ,KAAK,UAAU,KAAK,OAAO,IAAI,IAC/B;AAAA,MACV,CAAC;AAED,UAAI,CAAC,SAAS,IAAI;AACd,cAAM,IAAI;AAAA,UACN,6BAA6B,SAAS,UAAU;AAAA,UAChD;AAAA,YACI,MAAM,WAAW,SACZ;AAAA,YACL,SAAS;AAAA,YACT,QAAQ;AAAA,UACZ;AAAA,QACJ;AAAA,MACJ;AAEA,UAAI,OAAO,MAAM,SAAS,KAAK;AAC/B,UAAI,KAAK,OAAO,YAAY;AACxB,eAAO,KAAK,eAAe,MAAM,KAAK,OAAO,UAAU;AAAA,MAC3D;AAGA,YAAM,WAAW,KAAK,OAAO,WAAW;AACxC,UAAI,eAAmC;AAEvC,UAAI,MAAM,QAAQ,IAAI,GAAG;AACrB,YAAI,KAAK,WAAW,GAAG;AACnB,iBAAO;AAAA,QACX;AACA,mBAAW,QAAQ,MAAM;AACrB,yBAAe,KAAK;AAAA,YAChB;AAAA,YACA;AAAA,UACJ;AAAA,QACJ;AAAA,MACJ,OAAO;AACH,uBAAe,KAAK;AAAA,UAChB;AAAA,UACA;AAAA,QACJ;AAAA,MACJ;AAGA,WAAK,UAAU,KAAK,IAAI;AAGxB,WAAK;AAGL,UAAI,iBAAiB,QAAW;AAE5B,eAAO;AAAA,MACX;AACA,aAAO,eAAe,KAAK,OAAO,WAAW;AAAA,IACjD,SAAS,OAAO;AACZ,WAAK,aAAa,KAAsB;AACxC,aAAO;AAAA,IACX;AAAA,EACJ;AAAA;AAAA,EAGQ,eAAe,KAAU,MAAmB;AAChD,WAAO,KAAK,MAAM,GAAG,EAAE,OAAO,CAAC,MAAM,SAAS;AAC1C,aAAO,QAAQ,KAAK,IAAI,MAAM,SAAY,KAAK,IAAI,IAAI;AAAA,IAC3D,GAAG,GAAG;AAAA,EACV;AAAA,EAEQ,gCACJ,MACA,UACM;AACN,UAAM,QAAQ,KAAK,eAAe,MAAM,QAAQ;AAChD,QAAI,UAAU,QAAW;AACrB,YAAM,IAAI;AAAA,QACN,cAAc,QAAQ;AAAA,QACtB;AAAA,UACI,MAAM,WAAW,SAAS;AAAA,UAC1B,SAAS;AAAA,UACT,QAAQ;AAAA,QACZ;AAAA,MACJ;AAAA,IACJ;AACA,SAAK,mBAAmB,KAAK,IAAI,KAAK,kBAAkB,KAAK;AAC7D,WAAO;AAAA,EACX;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQO,cAAc,OAAqB;AACtC,SAAK,OAAO,WAAW,QAAQ;AAC/B,SAAK,mBAAmB;AAAA,EAC5B;AACJ;","names":[]}
|
@@ -2,7 +2,7 @@ import { BaseError } from "./base_error";
|
|
2
2
|
import { errorCodes } from "../constants";
|
3
3
|
class DatabaseError extends BaseError {
|
4
4
|
constructor(message, originalError, {
|
5
|
-
name = "
|
5
|
+
name = "DATABASE_ERROR",
|
6
6
|
code = errorCodes.consumer.UNKNOWN_CONSUMER_ERR,
|
7
7
|
isFatal = true,
|
8
8
|
origin = "databse_errors",
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"sources":["../../src/errors/database_errors.ts"],"sourcesContent":["import { BaseError } from \"./base_error\";\nimport { errorCodes } from \"../constants\";\n\nexport class DatabaseError extends BaseError {\n constructor(\n message: string,\n originalError?: Error,\n {\n name = \"
|
1
|
+
{"version":3,"sources":["../../src/errors/database_errors.ts"],"sourcesContent":["import { BaseError } from \"./base_error\";\nimport { errorCodes } from \"../constants\";\n\nexport class DatabaseError extends BaseError {\n constructor(\n message: string,\n originalError?: Error,\n {\n name = \"DATABASE_ERROR\",\n code = errorCodes.consumer.UNKNOWN_CONSUMER_ERR,\n isFatal = true,\n origin = \"databse_errors\",\n context = {},\n }: {\n name?: string;\n code?: number;\n isFatal?: boolean;\n origin?: string;\n context?: Record<string, any>;\n } = {}\n ) {\n super(name, code, message, isFatal, origin, context);\n Error.captureStackTrace(this, this.constructor);\n }\n}\n"],"mappings":"AAAA,SAAS,iBAAiB;AAC1B,SAAS,kBAAkB;AAEpB,MAAM,sBAAsB,UAAU;AAAA,EACzC,YACI,SACA,eACA;AAAA,IACI,OAAO;AAAA,IACP,OAAO,WAAW,SAAS;AAAA,IAC3B,UAAU;AAAA,IACV,SAAS;AAAA,IACT,UAAU,CAAC;AAAA,EACf,IAMI,CAAC,GACP;AACE,UAAM,MAAM,MAAM,SAAS,SAAS,QAAQ,OAAO;AACnD,UAAM,kBAAkB,MAAM,KAAK,WAAW;AAAA,EAClD;AACJ;","names":[]}
|