@lucaapp/service-utils 5.6.1 → 5.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/lib/api/api.d.ts +1 -1
- package/dist/lib/lifecycle/index.d.ts +2 -2
- package/dist/lib/lifecycle/index.js +1 -1
- package/dist/lib/pgBoss/config.d.ts +7 -0
- package/dist/lib/pgBoss/config.js +16 -0
- package/dist/lib/pgBoss/controller/routes.d.ts +20 -0
- package/dist/lib/pgBoss/controller/routes.js +153 -0
- package/dist/lib/pgBoss/controller/routes.schema.d.ts +161 -0
- package/dist/lib/pgBoss/controller/routes.schema.js +61 -0
- package/dist/lib/pgBoss/helpers.d.ts +15 -0
- package/dist/lib/pgBoss/helpers.js +44 -0
- package/dist/lib/pgBoss/index.d.ts +29 -0
- package/dist/lib/pgBoss/index.js +74 -0
- package/dist/lib/pgBoss/metrics.d.ts +15 -0
- package/dist/lib/pgBoss/metrics.js +93 -0
- package/dist/lib/pgBoss/service/enqueue.d.ts +3 -0
- package/dist/lib/pgBoss/service/enqueue.js +9 -0
- package/dist/lib/pgBoss/service/getJob.d.ts +2 -0
- package/dist/lib/pgBoss/service/getJob.js +8 -0
- package/dist/lib/pgBoss/service/getQueueStats.d.ts +14 -0
- package/dist/lib/pgBoss/service/getQueueStats.js +22 -0
- package/dist/lib/pgBoss/service/getWarnings.d.ts +9 -0
- package/dist/lib/pgBoss/service/getWarnings.js +19 -0
- package/dist/lib/pgBoss/service/listJobs.d.ts +2 -0
- package/dist/lib/pgBoss/service/listJobs.js +8 -0
- package/dist/lib/pgBoss/service/manageJob.d.ts +3 -0
- package/dist/lib/pgBoss/service/manageJob.js +18 -0
- package/dist/lib/pgBoss/service/schedules.d.ts +6 -0
- package/dist/lib/pgBoss/service/schedules.js +24 -0
- package/dist/lib/pgBoss/service.d.ts +20 -0
- package/dist/lib/pgBoss/service.js +30 -0
- package/dist/lib/pgBoss/types.d.ts +26 -0
- package/dist/lib/pgBoss/types.js +2 -0
- package/dist/lib/serviceIdentity/environment.d.ts +1 -0
- package/dist/lib/serviceIdentity/environment.js +1 -0
- package/package.json +15 -8
package/dist/index.d.ts
CHANGED
package/dist/index.js
CHANGED
|
@@ -37,4 +37,5 @@ __exportStar(require("./lib/jobs"), exports);
|
|
|
37
37
|
__exportStar(require("./lib/validation"), exports);
|
|
38
38
|
__exportStar(require("./lib/s3"), exports);
|
|
39
39
|
__exportStar(require("./lib/iso3166"), exports);
|
|
40
|
+
__exportStar(require("./lib/pgBoss"), exports);
|
|
40
41
|
__exportStar(require("./types"), exports);
|
package/dist/lib/api/api.d.ts
CHANGED
|
@@ -37,7 +37,7 @@ export declare class Api {
|
|
|
37
37
|
patch<TResponseSchemas extends ReadonlyArray<EndpointResponseSchema>, TRequestBodySchema extends ZodObjectSchemaOrUndefined = undefined, TRequestParamsSchema extends ZodObjectSchemaOrUndefined = undefined, TRequestQuerySchema extends ZodObjectSchemaOrUndefined = undefined, TRequestHeadersSchema extends ZodObjectSchemaOrUndefined = undefined, TMiddlewares extends ReadonlyArray<Middleware<any, any, any, any, any, any>> | undefined = undefined>(path: string, summary: string, options: EndpointOptions<TResponseSchemas, TRequestBodySchema, TRequestParamsSchema, TRequestQuerySchema, TRequestHeadersSchema, TMiddlewares>, handler: EndpointHandler<TResponseSchemas, TRequestBodySchema, TRequestParamsSchema, TRequestQuerySchema, TRequestHeadersSchema, TMiddlewares>): void;
|
|
38
38
|
put<TResponseSchemas extends ReadonlyArray<EndpointResponseSchema>, TRequestBodySchema extends ZodObjectSchemaOrUndefined = undefined, TRequestParamsSchema extends ZodObjectSchemaOrUndefined = undefined, TRequestQuerySchema extends ZodObjectSchemaOrUndefined = undefined, TRequestHeadersSchema extends ZodObjectSchemaOrUndefined = undefined, TMiddlewares extends ReadonlyArray<Middleware<any, any, any, any, any, any>> | undefined = undefined>(path: string, summary: string, options: EndpointOptions<TResponseSchemas, TRequestBodySchema, TRequestParamsSchema, TRequestQuerySchema, TRequestHeadersSchema, TMiddlewares>, handler: EndpointHandler<TResponseSchemas, TRequestBodySchema, TRequestParamsSchema, TRequestQuerySchema, TRequestHeadersSchema, TMiddlewares>): void;
|
|
39
39
|
delete<TResponseSchemas extends ReadonlyArray<EndpointResponseSchema>, TRequestBodySchema extends ZodObjectSchemaOrUndefined = undefined, TRequestParamsSchema extends ZodObjectSchemaOrUndefined = undefined, TRequestQuerySchema extends ZodObjectSchemaOrUndefined = undefined, TRequestHeadersSchema extends ZodObjectSchemaOrUndefined = undefined, TMiddlewares extends ReadonlyArray<Middleware<any, any, any, any, any, any>> | undefined = undefined>(path: string, summary: string, options: EndpointOptions<TResponseSchemas, TRequestBodySchema, TRequestParamsSchema, TRequestQuerySchema, TRequestHeadersSchema, TMiddlewares>, handler: EndpointHandler<TResponseSchemas, TRequestBodySchema, TRequestParamsSchema, TRequestQuerySchema, TRequestHeadersSchema, TMiddlewares>): void;
|
|
40
|
-
generateOpenAPISpec(): import(
|
|
40
|
+
generateOpenAPISpec(): import('openapi3-ts/oas30').OpenAPIObject | import('openapi3-ts/oas31').OpenAPIObject;
|
|
41
41
|
mountSwaggerMiddlewares(): void;
|
|
42
42
|
}
|
|
43
43
|
export {};
|
|
@@ -6,12 +6,12 @@ declare class Lifecycle {
|
|
|
6
6
|
private readonly shutdownDelay;
|
|
7
7
|
private readonly logger;
|
|
8
8
|
constructor(shutdownDelay: number, logger: Logger);
|
|
9
|
-
shutdownHandlers: Array<() => void
|
|
9
|
+
shutdownHandlers: Array<() => void | Promise<void>>;
|
|
10
10
|
gracefulShutdown: (isImmediate?: boolean) => Promise<void>;
|
|
11
11
|
private sigtermHandler;
|
|
12
12
|
unhandledRejectionHandler: (error: unknown) => void;
|
|
13
13
|
uncaughtExceptionHandler: (error: unknown) => void;
|
|
14
|
-
registerShutdownHandler: (shutdownHandler: () => void) => void;
|
|
14
|
+
registerShutdownHandler: (shutdownHandler: () => void | Promise<void>) => void;
|
|
15
15
|
registerHooks: () => void;
|
|
16
16
|
}
|
|
17
17
|
export { Lifecycle };
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { PgBossConfig } from './types';
|
|
2
|
+
export declare const PG_BOSS_DEFAULTS: {
|
|
3
|
+
readonly maxConnectionPoolSize: 3;
|
|
4
|
+
readonly deleteAfterDays: 30;
|
|
5
|
+
readonly applicationName: "pgboss";
|
|
6
|
+
};
|
|
7
|
+
export declare const buildPgBossDefaults: (config: PgBossConfig) => Required<PgBossConfig>;
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.buildPgBossDefaults = exports.PG_BOSS_DEFAULTS = void 0;
|
|
4
|
+
exports.PG_BOSS_DEFAULTS = {
|
|
5
|
+
maxConnectionPoolSize: 3,
|
|
6
|
+
deleteAfterDays: 30,
|
|
7
|
+
applicationName: 'pgboss',
|
|
8
|
+
};
|
|
9
|
+
const buildPgBossDefaults = (config) => ({
|
|
10
|
+
maxConnectionPoolSize: exports.PG_BOSS_DEFAULTS.maxConnectionPoolSize,
|
|
11
|
+
deleteAfterDays: exports.PG_BOSS_DEFAULTS.deleteAfterDays,
|
|
12
|
+
applicationName: exports.PG_BOSS_DEFAULTS.applicationName,
|
|
13
|
+
mattermostWebhookUrl: '',
|
|
14
|
+
...config,
|
|
15
|
+
});
|
|
16
|
+
exports.buildPgBossDefaults = buildPgBossDefaults;
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { Api } from '../../api/api';
|
|
2
|
+
import { PgBossService } from '../service';
|
|
3
|
+
/**
|
|
4
|
+
* Mounts pg-boss dashboard API routes onto an Api instance.
|
|
5
|
+
*
|
|
6
|
+
* Endpoints:
|
|
7
|
+
* - GET /queues — list queues with stats
|
|
8
|
+
* - GET /queues/:name/jobs — list jobs in a queue
|
|
9
|
+
* - GET /queues/:queue/jobs/:id — job detail
|
|
10
|
+
* - POST /queues/:queue/jobs/:id/retry — retry a job
|
|
11
|
+
* - POST /queues/:queue/jobs/:id/cancel — cancel a job
|
|
12
|
+
* - DELETE /queues/:queue/jobs/:id — delete a job
|
|
13
|
+
* - GET /schedules — list cron schedules
|
|
14
|
+
* - POST /schedules — create a schedule
|
|
15
|
+
* - PUT /schedules/:name — update a schedule
|
|
16
|
+
* - DELETE /schedules/:name — delete a schedule
|
|
17
|
+
* - POST /queues/:name/jobs — enqueue a new job
|
|
18
|
+
* - GET /warnings — recent warnings
|
|
19
|
+
*/
|
|
20
|
+
export declare const mountPgBossApiRoutes: (api: Api, service: PgBossService) => void;
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.mountPgBossApiRoutes = void 0;
|
|
4
|
+
const send_1 = require("../../api/send");
|
|
5
|
+
const response_1 = require("../../api/response");
|
|
6
|
+
const routes_schema_1 = require("./routes.schema");
|
|
7
|
+
/**
|
|
8
|
+
* Mounts pg-boss dashboard API routes onto an Api instance.
|
|
9
|
+
*
|
|
10
|
+
* Endpoints:
|
|
11
|
+
* - GET /queues — list queues with stats
|
|
12
|
+
* - GET /queues/:name/jobs — list jobs in a queue
|
|
13
|
+
* - GET /queues/:queue/jobs/:id — job detail
|
|
14
|
+
* - POST /queues/:queue/jobs/:id/retry — retry a job
|
|
15
|
+
* - POST /queues/:queue/jobs/:id/cancel — cancel a job
|
|
16
|
+
* - DELETE /queues/:queue/jobs/:id — delete a job
|
|
17
|
+
* - GET /schedules — list cron schedules
|
|
18
|
+
* - POST /schedules — create a schedule
|
|
19
|
+
* - PUT /schedules/:name — update a schedule
|
|
20
|
+
* - DELETE /schedules/:name — delete a schedule
|
|
21
|
+
* - POST /queues/:name/jobs — enqueue a new job
|
|
22
|
+
* - GET /warnings — recent warnings
|
|
23
|
+
*/
|
|
24
|
+
const mountPgBossApiRoutes = (api, service) => {
|
|
25
|
+
const pgBossApi = api.child({ tags: ['PgBoss'] });
|
|
26
|
+
// GET /queues — list queues with stats
|
|
27
|
+
pgBossApi.get('/queues', 'List all queues with stats', {
|
|
28
|
+
responses: [(0, response_1.okResponse)(routes_schema_1.queueStatsArraySchema)],
|
|
29
|
+
}, async (_request, _context, respond) => {
|
|
30
|
+
const stats = await service.getQueueStats();
|
|
31
|
+
return respond((0, send_1.ok)(stats));
|
|
32
|
+
});
|
|
33
|
+
// GET /queues/:name/jobs — list jobs in a queue
|
|
34
|
+
pgBossApi.get('/queues/:name/jobs', 'List jobs in a queue', {
|
|
35
|
+
responses: [(0, response_1.okResponse)(routes_schema_1.jobArraySchema)],
|
|
36
|
+
schemas: {
|
|
37
|
+
params: routes_schema_1.queueNameParamsSchema,
|
|
38
|
+
},
|
|
39
|
+
}, async (request, _context, respond) => {
|
|
40
|
+
const jobs = await service.listJobs(request.params.name);
|
|
41
|
+
return respond((0, send_1.ok)(jobs));
|
|
42
|
+
});
|
|
43
|
+
// GET /queues/:queue/jobs/:id — job detail
|
|
44
|
+
pgBossApi.get('/queues/:queue/jobs/:id', 'Get job details', {
|
|
45
|
+
responses: [(0, response_1.okResponse)(routes_schema_1.jobDetailSchema)],
|
|
46
|
+
schemas: {
|
|
47
|
+
params: routes_schema_1.queueJobParamsSchema,
|
|
48
|
+
},
|
|
49
|
+
}, async (request, _context, respond) => {
|
|
50
|
+
const { queue, id } = request.params;
|
|
51
|
+
const job = await service.getJob(queue, id);
|
|
52
|
+
if (!job) {
|
|
53
|
+
throw new Error('Job not found');
|
|
54
|
+
}
|
|
55
|
+
return respond((0, send_1.ok)(job));
|
|
56
|
+
});
|
|
57
|
+
// POST /queues/:queue/jobs/:id/retry — retry a failed job
|
|
58
|
+
pgBossApi.post('/queues/:queue/jobs/:id/retry', 'Retry a failed job', {
|
|
59
|
+
responses: [(0, response_1.okResponse)(routes_schema_1.successResponseSchema)],
|
|
60
|
+
schemas: {
|
|
61
|
+
params: routes_schema_1.queueJobParamsSchema,
|
|
62
|
+
},
|
|
63
|
+
}, async (request, _context, respond) => {
|
|
64
|
+
const { queue, id } = request.params;
|
|
65
|
+
await service.retryJob(queue, id);
|
|
66
|
+
return respond((0, send_1.ok)({ success: true, jobId: id }));
|
|
67
|
+
});
|
|
68
|
+
// POST /queues/:queue/jobs/:id/cancel — cancel a job
|
|
69
|
+
pgBossApi.post('/queues/:queue/jobs/:id/cancel', 'Cancel a job', {
|
|
70
|
+
responses: [(0, response_1.okResponse)(routes_schema_1.successResponseSchema)],
|
|
71
|
+
schemas: {
|
|
72
|
+
params: routes_schema_1.queueJobParamsSchema,
|
|
73
|
+
},
|
|
74
|
+
}, async (request, _context, respond) => {
|
|
75
|
+
const { queue, id } = request.params;
|
|
76
|
+
await service.cancelJob(queue, id);
|
|
77
|
+
return respond((0, send_1.ok)({ success: true, jobId: id }));
|
|
78
|
+
});
|
|
79
|
+
// DELETE /queues/:queue/jobs/:id — delete a job
|
|
80
|
+
pgBossApi.delete('/queues/:queue/jobs/:id', 'Delete a job', {
|
|
81
|
+
responses: [(0, response_1.okResponse)(routes_schema_1.successResponseSchema)],
|
|
82
|
+
schemas: {
|
|
83
|
+
params: routes_schema_1.queueJobParamsSchema,
|
|
84
|
+
},
|
|
85
|
+
}, async (request, _context, respond) => {
|
|
86
|
+
const { queue, id } = request.params;
|
|
87
|
+
await service.deleteJob(queue, id);
|
|
88
|
+
return respond((0, send_1.ok)({ success: true, deleted: id }));
|
|
89
|
+
});
|
|
90
|
+
// GET /schedules — list cron schedules
|
|
91
|
+
pgBossApi.get('/schedules', 'List all cron schedules', {
|
|
92
|
+
responses: [(0, response_1.okResponse)(routes_schema_1.scheduleArraySchema)],
|
|
93
|
+
}, async (_request, _context, respond) => {
|
|
94
|
+
const schedules = await service.getSchedules();
|
|
95
|
+
return respond((0, send_1.ok)(schedules));
|
|
96
|
+
});
|
|
97
|
+
// POST /schedules — create a cron schedule
|
|
98
|
+
pgBossApi.post('/schedules', 'Create a cron schedule', {
|
|
99
|
+
responses: [(0, response_1.createdResponse)(routes_schema_1.successResponseSchema)],
|
|
100
|
+
schemas: {
|
|
101
|
+
body: routes_schema_1.scheduleBodySchema,
|
|
102
|
+
},
|
|
103
|
+
}, async (request, _context, respond) => {
|
|
104
|
+
const { name, cron, data, options } = request.body;
|
|
105
|
+
await service.createSchedule(name, cron, data, options);
|
|
106
|
+
return respond((0, send_1.created)({ success: true, name }));
|
|
107
|
+
});
|
|
108
|
+
// PUT /schedules/:name — update a cron schedule
|
|
109
|
+
pgBossApi.put('/schedules/:name', 'Update a cron schedule', {
|
|
110
|
+
responses: [(0, response_1.okResponse)(routes_schema_1.successResponseSchema)],
|
|
111
|
+
schemas: {
|
|
112
|
+
params: routes_schema_1.scheduleNameParamsSchema,
|
|
113
|
+
body: routes_schema_1.updateScheduleBodySchema,
|
|
114
|
+
},
|
|
115
|
+
}, async (request, _context, respond) => {
|
|
116
|
+
const { cron, data, options } = request.body;
|
|
117
|
+
await service.updateSchedule(request.params.name, cron, data, options);
|
|
118
|
+
return respond((0, send_1.ok)({ success: true, name: request.params.name }));
|
|
119
|
+
});
|
|
120
|
+
// DELETE /schedules/:name — delete a cron schedule
|
|
121
|
+
pgBossApi.delete('/schedules/:name', 'Delete a cron schedule', {
|
|
122
|
+
responses: [(0, response_1.okResponse)(routes_schema_1.successResponseSchema)],
|
|
123
|
+
schemas: {
|
|
124
|
+
params: routes_schema_1.scheduleNameParamsSchema,
|
|
125
|
+
},
|
|
126
|
+
}, async (request, _context, respond) => {
|
|
127
|
+
await service.deleteSchedule(request.params.name);
|
|
128
|
+
return respond((0, send_1.ok)({ success: true, deleted: request.params.name }));
|
|
129
|
+
});
|
|
130
|
+
// POST /queues/:name/jobs — enqueue a new job
|
|
131
|
+
pgBossApi.post('/queues/:name/jobs', 'Enqueue a new job', {
|
|
132
|
+
responses: [(0, response_1.createdResponse)(routes_schema_1.successResponseSchema)],
|
|
133
|
+
schemas: {
|
|
134
|
+
params: routes_schema_1.queueNameParamsSchema,
|
|
135
|
+
body: routes_schema_1.enqueueBodySchema,
|
|
136
|
+
},
|
|
137
|
+
}, async (request, _context, respond) => {
|
|
138
|
+
const { data, options } = request.body;
|
|
139
|
+
const jobId = await service.enqueueJob(request.params.name, data, options);
|
|
140
|
+
return respond((0, send_1.created)({ success: true, jobId: jobId ?? undefined }));
|
|
141
|
+
});
|
|
142
|
+
// GET /warnings — recent warnings
|
|
143
|
+
pgBossApi.get('/warnings', 'List recent warnings', {
|
|
144
|
+
responses: [(0, response_1.okResponse)(routes_schema_1.warningArraySchema)],
|
|
145
|
+
schemas: {
|
|
146
|
+
query: routes_schema_1.warningsQuerySchema,
|
|
147
|
+
},
|
|
148
|
+
}, async (request, _context, respond) => {
|
|
149
|
+
const warnings = await service.getWarnings(request.query.limit);
|
|
150
|
+
return respond((0, send_1.ok)(warnings));
|
|
151
|
+
});
|
|
152
|
+
};
|
|
153
|
+
exports.mountPgBossApiRoutes = mountPgBossApiRoutes;
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
export declare const queueNameParamsSchema: z.ZodObject<{
|
|
3
|
+
name: z.ZodString;
|
|
4
|
+
}, "strip", z.ZodTypeAny, {
|
|
5
|
+
name: string;
|
|
6
|
+
}, {
|
|
7
|
+
name: string;
|
|
8
|
+
}>;
|
|
9
|
+
export declare const queueJobParamsSchema: z.ZodObject<{
|
|
10
|
+
queue: z.ZodString;
|
|
11
|
+
id: z.ZodString;
|
|
12
|
+
}, "strip", z.ZodTypeAny, {
|
|
13
|
+
id: string;
|
|
14
|
+
queue: string;
|
|
15
|
+
}, {
|
|
16
|
+
id: string;
|
|
17
|
+
queue: string;
|
|
18
|
+
}>;
|
|
19
|
+
export declare const scheduleNameParamsSchema: z.ZodObject<{
|
|
20
|
+
name: z.ZodString;
|
|
21
|
+
}, "strip", z.ZodTypeAny, {
|
|
22
|
+
name: string;
|
|
23
|
+
}, {
|
|
24
|
+
name: string;
|
|
25
|
+
}>;
|
|
26
|
+
export declare const warningsQuerySchema: z.ZodObject<{
|
|
27
|
+
limit: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
|
|
28
|
+
}, "strip", z.ZodTypeAny, {
|
|
29
|
+
limit: number;
|
|
30
|
+
}, {
|
|
31
|
+
limit?: number | undefined;
|
|
32
|
+
}>;
|
|
33
|
+
export declare const enqueueBodySchema: z.ZodObject<{
|
|
34
|
+
data: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
35
|
+
options: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
36
|
+
}, "strip", z.ZodTypeAny, {
|
|
37
|
+
options?: Record<string, unknown> | undefined;
|
|
38
|
+
data?: Record<string, unknown> | undefined;
|
|
39
|
+
}, {
|
|
40
|
+
options?: Record<string, unknown> | undefined;
|
|
41
|
+
data?: Record<string, unknown> | undefined;
|
|
42
|
+
}>;
|
|
43
|
+
export declare const scheduleBodySchema: z.ZodObject<{
|
|
44
|
+
name: z.ZodString;
|
|
45
|
+
cron: z.ZodString;
|
|
46
|
+
data: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
47
|
+
options: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
48
|
+
}, "strip", z.ZodTypeAny, {
|
|
49
|
+
name: string;
|
|
50
|
+
cron: string;
|
|
51
|
+
options?: Record<string, unknown> | undefined;
|
|
52
|
+
data?: Record<string, unknown> | undefined;
|
|
53
|
+
}, {
|
|
54
|
+
name: string;
|
|
55
|
+
cron: string;
|
|
56
|
+
options?: Record<string, unknown> | undefined;
|
|
57
|
+
data?: Record<string, unknown> | undefined;
|
|
58
|
+
}>;
|
|
59
|
+
export declare const updateScheduleBodySchema: z.ZodObject<{
|
|
60
|
+
cron: z.ZodString;
|
|
61
|
+
data: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
62
|
+
options: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
63
|
+
}, "strip", z.ZodTypeAny, {
|
|
64
|
+
cron: string;
|
|
65
|
+
options?: Record<string, unknown> | undefined;
|
|
66
|
+
data?: Record<string, unknown> | undefined;
|
|
67
|
+
}, {
|
|
68
|
+
cron: string;
|
|
69
|
+
options?: Record<string, unknown> | undefined;
|
|
70
|
+
data?: Record<string, unknown> | undefined;
|
|
71
|
+
}>;
|
|
72
|
+
export declare const queueStatsSchema: z.ZodObject<{
|
|
73
|
+
name: z.ZodString;
|
|
74
|
+
policy: z.ZodOptional<z.ZodString>;
|
|
75
|
+
retryLimit: z.ZodOptional<z.ZodNumber>;
|
|
76
|
+
retryDelay: z.ZodOptional<z.ZodNumber>;
|
|
77
|
+
retryBackoff: z.ZodOptional<z.ZodBoolean>;
|
|
78
|
+
expireInSeconds: z.ZodOptional<z.ZodNumber>;
|
|
79
|
+
deadLetter: z.ZodOptional<z.ZodNullable<z.ZodString>>;
|
|
80
|
+
queuedCount: z.ZodNumber;
|
|
81
|
+
activeCount: z.ZodNumber;
|
|
82
|
+
totalCount: z.ZodNumber;
|
|
83
|
+
}, "strip", z.ZodTypeAny, {
|
|
84
|
+
name: string;
|
|
85
|
+
queuedCount: number;
|
|
86
|
+
activeCount: number;
|
|
87
|
+
totalCount: number;
|
|
88
|
+
retryDelay?: number | undefined;
|
|
89
|
+
policy?: string | undefined;
|
|
90
|
+
retryLimit?: number | undefined;
|
|
91
|
+
retryBackoff?: boolean | undefined;
|
|
92
|
+
expireInSeconds?: number | undefined;
|
|
93
|
+
deadLetter?: string | null | undefined;
|
|
94
|
+
}, {
|
|
95
|
+
name: string;
|
|
96
|
+
queuedCount: number;
|
|
97
|
+
activeCount: number;
|
|
98
|
+
totalCount: number;
|
|
99
|
+
retryDelay?: number | undefined;
|
|
100
|
+
policy?: string | undefined;
|
|
101
|
+
retryLimit?: number | undefined;
|
|
102
|
+
retryBackoff?: boolean | undefined;
|
|
103
|
+
expireInSeconds?: number | undefined;
|
|
104
|
+
deadLetter?: string | null | undefined;
|
|
105
|
+
}>;
|
|
106
|
+
export declare const queueStatsArraySchema: z.ZodArray<z.ZodObject<{
|
|
107
|
+
name: z.ZodString;
|
|
108
|
+
policy: z.ZodOptional<z.ZodString>;
|
|
109
|
+
retryLimit: z.ZodOptional<z.ZodNumber>;
|
|
110
|
+
retryDelay: z.ZodOptional<z.ZodNumber>;
|
|
111
|
+
retryBackoff: z.ZodOptional<z.ZodBoolean>;
|
|
112
|
+
expireInSeconds: z.ZodOptional<z.ZodNumber>;
|
|
113
|
+
deadLetter: z.ZodOptional<z.ZodNullable<z.ZodString>>;
|
|
114
|
+
queuedCount: z.ZodNumber;
|
|
115
|
+
activeCount: z.ZodNumber;
|
|
116
|
+
totalCount: z.ZodNumber;
|
|
117
|
+
}, "strip", z.ZodTypeAny, {
|
|
118
|
+
name: string;
|
|
119
|
+
queuedCount: number;
|
|
120
|
+
activeCount: number;
|
|
121
|
+
totalCount: number;
|
|
122
|
+
retryDelay?: number | undefined;
|
|
123
|
+
policy?: string | undefined;
|
|
124
|
+
retryLimit?: number | undefined;
|
|
125
|
+
retryBackoff?: boolean | undefined;
|
|
126
|
+
expireInSeconds?: number | undefined;
|
|
127
|
+
deadLetter?: string | null | undefined;
|
|
128
|
+
}, {
|
|
129
|
+
name: string;
|
|
130
|
+
queuedCount: number;
|
|
131
|
+
activeCount: number;
|
|
132
|
+
totalCount: number;
|
|
133
|
+
retryDelay?: number | undefined;
|
|
134
|
+
policy?: string | undefined;
|
|
135
|
+
retryLimit?: number | undefined;
|
|
136
|
+
retryBackoff?: boolean | undefined;
|
|
137
|
+
expireInSeconds?: number | undefined;
|
|
138
|
+
deadLetter?: string | null | undefined;
|
|
139
|
+
}>, "many">;
|
|
140
|
+
export declare const jobDetailSchema: z.ZodRecord<z.ZodString, z.ZodUnknown>;
|
|
141
|
+
export declare const jobArraySchema: z.ZodArray<z.ZodRecord<z.ZodString, z.ZodUnknown>, "many">;
|
|
142
|
+
export declare const scheduleSchema: z.ZodRecord<z.ZodString, z.ZodUnknown>;
|
|
143
|
+
export declare const scheduleArraySchema: z.ZodArray<z.ZodRecord<z.ZodString, z.ZodUnknown>, "many">;
|
|
144
|
+
export declare const warningSchema: z.ZodRecord<z.ZodString, z.ZodUnknown>;
|
|
145
|
+
export declare const warningArraySchema: z.ZodArray<z.ZodRecord<z.ZodString, z.ZodUnknown>, "many">;
|
|
146
|
+
export declare const successResponseSchema: z.ZodObject<{
|
|
147
|
+
success: z.ZodBoolean;
|
|
148
|
+
jobId: z.ZodOptional<z.ZodString>;
|
|
149
|
+
deleted: z.ZodOptional<z.ZodString>;
|
|
150
|
+
name: z.ZodOptional<z.ZodString>;
|
|
151
|
+
}, "strip", z.ZodTypeAny, {
|
|
152
|
+
success: boolean;
|
|
153
|
+
name?: string | undefined;
|
|
154
|
+
jobId?: string | undefined;
|
|
155
|
+
deleted?: string | undefined;
|
|
156
|
+
}, {
|
|
157
|
+
success: boolean;
|
|
158
|
+
name?: string | undefined;
|
|
159
|
+
jobId?: string | undefined;
|
|
160
|
+
deleted?: string | undefined;
|
|
161
|
+
}>;
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.successResponseSchema = exports.warningArraySchema = exports.warningSchema = exports.scheduleArraySchema = exports.scheduleSchema = exports.jobArraySchema = exports.jobDetailSchema = exports.queueStatsArraySchema = exports.queueStatsSchema = exports.updateScheduleBodySchema = exports.scheduleBodySchema = exports.enqueueBodySchema = exports.warningsQuerySchema = exports.scheduleNameParamsSchema = exports.queueJobParamsSchema = exports.queueNameParamsSchema = void 0;
|
|
4
|
+
const zod_1 = require("zod");
|
|
5
|
+
// ── Param schemas ────────────────────────────────────────────────────
|
|
6
|
+
exports.queueNameParamsSchema = zod_1.z.object({
|
|
7
|
+
name: zod_1.z.string(),
|
|
8
|
+
});
|
|
9
|
+
exports.queueJobParamsSchema = zod_1.z.object({
|
|
10
|
+
queue: zod_1.z.string(),
|
|
11
|
+
id: zod_1.z.string(),
|
|
12
|
+
});
|
|
13
|
+
exports.scheduleNameParamsSchema = zod_1.z.object({
|
|
14
|
+
name: zod_1.z.string(),
|
|
15
|
+
});
|
|
16
|
+
// ── Query schemas ────────────────────────────────────────────────────
|
|
17
|
+
exports.warningsQuerySchema = zod_1.z.object({
|
|
18
|
+
limit: zod_1.z.coerce.number().int().positive().optional().default(100),
|
|
19
|
+
});
|
|
20
|
+
// ── Body schemas ─────────────────────────────────────────────────────
|
|
21
|
+
exports.enqueueBodySchema = zod_1.z.object({
|
|
22
|
+
data: zod_1.z.record(zod_1.z.unknown()).optional(),
|
|
23
|
+
options: zod_1.z.record(zod_1.z.unknown()).optional(),
|
|
24
|
+
});
|
|
25
|
+
exports.scheduleBodySchema = zod_1.z.object({
|
|
26
|
+
name: zod_1.z.string(),
|
|
27
|
+
cron: zod_1.z.string(),
|
|
28
|
+
data: zod_1.z.record(zod_1.z.unknown()).optional(),
|
|
29
|
+
options: zod_1.z.record(zod_1.z.unknown()).optional(),
|
|
30
|
+
});
|
|
31
|
+
exports.updateScheduleBodySchema = zod_1.z.object({
|
|
32
|
+
cron: zod_1.z.string(),
|
|
33
|
+
data: zod_1.z.record(zod_1.z.unknown()).optional(),
|
|
34
|
+
options: zod_1.z.record(zod_1.z.unknown()).optional(),
|
|
35
|
+
});
|
|
36
|
+
// ── Response schemas ─────────────────────────────────────────────────
|
|
37
|
+
exports.queueStatsSchema = zod_1.z.object({
|
|
38
|
+
name: zod_1.z.string(),
|
|
39
|
+
policy: zod_1.z.string().optional(),
|
|
40
|
+
retryLimit: zod_1.z.number().optional(),
|
|
41
|
+
retryDelay: zod_1.z.number().optional(),
|
|
42
|
+
retryBackoff: zod_1.z.boolean().optional(),
|
|
43
|
+
expireInSeconds: zod_1.z.number().optional(),
|
|
44
|
+
deadLetter: zod_1.z.string().nullable().optional(),
|
|
45
|
+
queuedCount: zod_1.z.number(),
|
|
46
|
+
activeCount: zod_1.z.number(),
|
|
47
|
+
totalCount: zod_1.z.number(),
|
|
48
|
+
});
|
|
49
|
+
exports.queueStatsArraySchema = zod_1.z.array(exports.queueStatsSchema);
|
|
50
|
+
exports.jobDetailSchema = zod_1.z.record(zod_1.z.unknown());
|
|
51
|
+
exports.jobArraySchema = zod_1.z.array(exports.jobDetailSchema);
|
|
52
|
+
exports.scheduleSchema = zod_1.z.record(zod_1.z.unknown());
|
|
53
|
+
exports.scheduleArraySchema = zod_1.z.array(exports.scheduleSchema);
|
|
54
|
+
exports.warningSchema = zod_1.z.record(zod_1.z.unknown());
|
|
55
|
+
exports.warningArraySchema = zod_1.z.array(exports.warningSchema);
|
|
56
|
+
exports.successResponseSchema = zod_1.z.object({
|
|
57
|
+
success: zod_1.z.boolean(),
|
|
58
|
+
jobId: zod_1.z.string().optional(),
|
|
59
|
+
deleted: zod_1.z.string().optional(),
|
|
60
|
+
name: zod_1.z.string().optional(),
|
|
61
|
+
});
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { PgBoss } from 'pg-boss';
|
|
2
|
+
import type { SendOptions } from 'pg-boss';
|
|
3
|
+
import type { Logger } from 'pino';
|
|
4
|
+
import type { Sequelize, Transaction } from 'sequelize';
|
|
5
|
+
/**
|
|
6
|
+
* Enqueue a job via pg-boss with logging.
|
|
7
|
+
*/
|
|
8
|
+
export declare const enqueueJob: (boss: PgBoss, queueName: string, data: object, options: SendOptions | undefined, logger: Logger) => Promise<string | null>;
|
|
9
|
+
/**
|
|
10
|
+
* Enqueue a job inside an existing Sequelize transaction.
|
|
11
|
+
*
|
|
12
|
+
* Uses pg-boss's public `send()` API with a transactional database adapter
|
|
13
|
+
* instead of calling internal SQL functions directly.
|
|
14
|
+
*/
|
|
15
|
+
export declare const enqueueInTransaction: (boss: PgBoss, queueName: string, data: object, options: SendOptions, transaction: Transaction, sequelize: Sequelize) => Promise<string | null>;
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.enqueueInTransaction = exports.enqueueJob = void 0;
|
|
4
|
+
/**
|
|
5
|
+
* Enqueue a job via pg-boss with logging.
|
|
6
|
+
*/
|
|
7
|
+
const enqueueJob = async (boss, queueName, data, options = {}, logger) => {
|
|
8
|
+
const jobId = await boss.send(queueName, data, options);
|
|
9
|
+
logger.info({ queueName, jobId }, 'Job enqueued via pg-boss');
|
|
10
|
+
return jobId;
|
|
11
|
+
};
|
|
12
|
+
exports.enqueueJob = enqueueJob;
|
|
13
|
+
/**
|
|
14
|
+
* Create a pg-boss–compatible database adapter backed by a Sequelize transaction.
|
|
15
|
+
*
|
|
16
|
+
* Bridges Sequelize's transaction with pg-boss's `ConnectionOptions.db`
|
|
17
|
+
* so that `boss.send()` executes within the same database transaction
|
|
18
|
+
* as surrounding Sequelize operations.
|
|
19
|
+
*/
|
|
20
|
+
const createTransactionalDb = (sequelize, transaction) => ({
|
|
21
|
+
executeSql: async (text, values) => {
|
|
22
|
+
const [results] = await sequelize.query(text, {
|
|
23
|
+
bind: values,
|
|
24
|
+
transaction,
|
|
25
|
+
raw: true,
|
|
26
|
+
});
|
|
27
|
+
return {
|
|
28
|
+
rows: Array.isArray(results)
|
|
29
|
+
? results
|
|
30
|
+
: [],
|
|
31
|
+
};
|
|
32
|
+
},
|
|
33
|
+
});
|
|
34
|
+
/**
|
|
35
|
+
* Enqueue a job inside an existing Sequelize transaction.
|
|
36
|
+
*
|
|
37
|
+
* Uses pg-boss's public `send()` API with a transactional database adapter
|
|
38
|
+
* instead of calling internal SQL functions directly.
|
|
39
|
+
*/
|
|
40
|
+
const enqueueInTransaction = async (boss, queueName, data, options, transaction, sequelize) => {
|
|
41
|
+
const db = createTransactionalDb(sequelize, transaction);
|
|
42
|
+
return boss.send(queueName, data, { ...options, db });
|
|
43
|
+
};
|
|
44
|
+
exports.enqueueInTransaction = enqueueInTransaction;
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { PgBoss } from 'pg-boss';
|
|
2
|
+
import type { Logger } from 'pino';
|
|
3
|
+
import type { PgBossConfig } from './types';
|
|
4
|
+
export { PgBoss };
|
|
5
|
+
export type { PgBossConfig, QueueDefinition, WorkerRegistration, PgBossInstance, } from './types';
|
|
6
|
+
export { PG_BOSS_DEFAULTS, buildPgBossDefaults } from './config';
|
|
7
|
+
export { mountPgBossApiRoutes } from './controller/routes';
|
|
8
|
+
export { PgBossService } from './service';
|
|
9
|
+
export { registerPgBossMetrics } from './metrics';
|
|
10
|
+
export { enqueueJob, enqueueInTransaction } from './helpers';
|
|
11
|
+
export { sendDlqAlert } from './metrics';
|
|
12
|
+
export interface PgBossContext {
|
|
13
|
+
boss: PgBoss;
|
|
14
|
+
logger: Logger;
|
|
15
|
+
config: Required<PgBossConfig>;
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Creates a pg-boss instance with error recovery (auto-restart on connection loss).
|
|
19
|
+
* Issue #510 mitigation: on error, waits 5s then restarts.
|
|
20
|
+
*/
|
|
21
|
+
export declare const createPgBoss: (config: PgBossConfig, logger: Logger) => PgBossContext;
|
|
22
|
+
/**
|
|
23
|
+
* Starts the pg-boss instance. Call after database is connected.
|
|
24
|
+
*/
|
|
25
|
+
export declare const startBoss: (ctx: PgBossContext) => Promise<void>;
|
|
26
|
+
/**
|
|
27
|
+
* Gracefully stops the pg-boss instance. Suitable as a shutdown handler.
|
|
28
|
+
*/
|
|
29
|
+
export declare const stopBoss: (ctx: PgBossContext) => Promise<void>;
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.stopBoss = exports.startBoss = exports.createPgBoss = exports.sendDlqAlert = exports.enqueueInTransaction = exports.enqueueJob = exports.registerPgBossMetrics = exports.PgBossService = exports.mountPgBossApiRoutes = exports.buildPgBossDefaults = exports.PG_BOSS_DEFAULTS = exports.PgBoss = void 0;
|
|
4
|
+
const pg_boss_1 = require("pg-boss");
|
|
5
|
+
Object.defineProperty(exports, "PgBoss", { enumerable: true, get: function () { return pg_boss_1.PgBoss; } });
|
|
6
|
+
const config_1 = require("./config");
|
|
7
|
+
var config_2 = require("./config");
|
|
8
|
+
Object.defineProperty(exports, "PG_BOSS_DEFAULTS", { enumerable: true, get: function () { return config_2.PG_BOSS_DEFAULTS; } });
|
|
9
|
+
Object.defineProperty(exports, "buildPgBossDefaults", { enumerable: true, get: function () { return config_2.buildPgBossDefaults; } });
|
|
10
|
+
var routes_1 = require("./controller/routes");
|
|
11
|
+
Object.defineProperty(exports, "mountPgBossApiRoutes", { enumerable: true, get: function () { return routes_1.mountPgBossApiRoutes; } });
|
|
12
|
+
var service_1 = require("./service");
|
|
13
|
+
Object.defineProperty(exports, "PgBossService", { enumerable: true, get: function () { return service_1.PgBossService; } });
|
|
14
|
+
var metrics_1 = require("./metrics");
|
|
15
|
+
Object.defineProperty(exports, "registerPgBossMetrics", { enumerable: true, get: function () { return metrics_1.registerPgBossMetrics; } });
|
|
16
|
+
var helpers_1 = require("./helpers");
|
|
17
|
+
Object.defineProperty(exports, "enqueueJob", { enumerable: true, get: function () { return helpers_1.enqueueJob; } });
|
|
18
|
+
Object.defineProperty(exports, "enqueueInTransaction", { enumerable: true, get: function () { return helpers_1.enqueueInTransaction; } });
|
|
19
|
+
var metrics_2 = require("./metrics");
|
|
20
|
+
Object.defineProperty(exports, "sendDlqAlert", { enumerable: true, get: function () { return metrics_2.sendDlqAlert; } });
|
|
21
|
+
const RESTART_DELAY_MS = 5000;
|
|
22
|
+
/**
|
|
23
|
+
* Creates a pg-boss instance with error recovery (auto-restart on connection loss).
|
|
24
|
+
* Issue #510 mitigation: on error, waits 5s then restarts.
|
|
25
|
+
*/
|
|
26
|
+
const createPgBoss = (config, logger) => {
|
|
27
|
+
const resolvedConfig = (0, config_1.buildPgBossDefaults)(config);
|
|
28
|
+
const boss = new pg_boss_1.PgBoss({
|
|
29
|
+
connectionString: resolvedConfig.connectionString,
|
|
30
|
+
schema: resolvedConfig.schema,
|
|
31
|
+
application_name: resolvedConfig.applicationName,
|
|
32
|
+
migrate: true,
|
|
33
|
+
});
|
|
34
|
+
boss.on('error', (error) => {
|
|
35
|
+
logger.error({ error }, 'pg-boss error — will attempt restart');
|
|
36
|
+
scheduleRestart(boss, logger);
|
|
37
|
+
});
|
|
38
|
+
boss.on('warning', warning => {
|
|
39
|
+
logger.warn({ warning }, 'pg-boss warning');
|
|
40
|
+
});
|
|
41
|
+
return { boss, logger, config: resolvedConfig };
|
|
42
|
+
};
|
|
43
|
+
exports.createPgBoss = createPgBoss;
|
|
44
|
+
/**
|
|
45
|
+
* Starts the pg-boss instance. Call after database is connected.
|
|
46
|
+
*/
|
|
47
|
+
const startBoss = async (ctx) => {
|
|
48
|
+
ctx.logger.info({ schema: ctx.config.schema }, 'Starting pg-boss');
|
|
49
|
+
await ctx.boss.start();
|
|
50
|
+
ctx.logger.info({ schema: ctx.config.schema }, 'pg-boss started successfully');
|
|
51
|
+
};
|
|
52
|
+
exports.startBoss = startBoss;
|
|
53
|
+
/**
|
|
54
|
+
* Gracefully stops the pg-boss instance. Suitable as a shutdown handler.
|
|
55
|
+
*/
|
|
56
|
+
const stopBoss = async (ctx) => {
|
|
57
|
+
ctx.logger.info({ schema: ctx.config.schema }, 'Stopping pg-boss');
|
|
58
|
+
await ctx.boss.stop({ graceful: true, timeout: 30_000 });
|
|
59
|
+
ctx.logger.info({ schema: ctx.config.schema }, 'pg-boss stopped');
|
|
60
|
+
};
|
|
61
|
+
exports.stopBoss = stopBoss;
|
|
62
|
+
const scheduleRestart = (boss, logger) => {
|
|
63
|
+
setTimeout(async () => {
|
|
64
|
+
try {
|
|
65
|
+
logger.info('Attempting pg-boss restart after error');
|
|
66
|
+
await boss.start();
|
|
67
|
+
logger.info('pg-boss restarted successfully');
|
|
68
|
+
}
|
|
69
|
+
catch (restartError) {
|
|
70
|
+
logger.error({ error: restartError }, 'pg-boss restart failed — will retry');
|
|
71
|
+
scheduleRestart(boss, logger);
|
|
72
|
+
}
|
|
73
|
+
}, RESTART_DELAY_MS);
|
|
74
|
+
};
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { PgBoss } from 'pg-boss';
|
|
2
|
+
import type { Logger } from 'pino';
|
|
3
|
+
/**
|
|
4
|
+
* Register Prometheus metrics by polling pg-boss queue stats.
|
|
5
|
+
*
|
|
6
|
+
* pg-boss v12 removed the `monitor-states` event, so we poll
|
|
7
|
+
* `getQueues()` + `getQueueStats()` on a timer instead.
|
|
8
|
+
*
|
|
9
|
+
* Returns the interval handle so callers can clear it on shutdown.
|
|
10
|
+
*/
|
|
11
|
+
export declare const registerPgBossMetrics: (boss: PgBoss, serviceName: string, logger: Logger, mattermostWebhookUrl?: string) => NodeJS.Timeout;
|
|
12
|
+
/**
|
|
13
|
+
* Send a Mattermost alert for a dead-letter job.
|
|
14
|
+
*/
|
|
15
|
+
export declare const sendDlqAlert: (webhookUrl: string, queueName: string, jobId: string, error: string, logger: Logger) => Promise<void>;
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
+
if (mod && mod.__esModule) return mod;
|
|
20
|
+
var result = {};
|
|
21
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
+
__setModuleDefault(result, mod);
|
|
23
|
+
return result;
|
|
24
|
+
};
|
|
25
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
26
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
27
|
+
};
|
|
28
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
29
|
+
exports.sendDlqAlert = exports.registerPgBossMetrics = void 0;
|
|
30
|
+
const promClient = __importStar(require("prom-client"));
|
|
31
|
+
const axios_1 = __importDefault(require("axios"));
|
|
32
|
+
const queueDepth = new promClient.Gauge({
|
|
33
|
+
name: 'pgboss_queue_created_total',
|
|
34
|
+
help: 'Number of queued (pending) jobs per queue',
|
|
35
|
+
labelNames: ['queue', 'service'],
|
|
36
|
+
});
|
|
37
|
+
const queueActive = new promClient.Gauge({
|
|
38
|
+
name: 'pgboss_queue_active_total',
|
|
39
|
+
help: 'Number of active jobs per queue',
|
|
40
|
+
labelNames: ['queue', 'service'],
|
|
41
|
+
});
|
|
42
|
+
const queueTotal = new promClient.Gauge({
|
|
43
|
+
name: 'pgboss_queue_total',
|
|
44
|
+
help: 'Total number of jobs per queue (all states)',
|
|
45
|
+
labelNames: ['queue', 'service'],
|
|
46
|
+
});
|
|
47
|
+
const POLL_INTERVAL_MS = 30_000;
|
|
48
|
+
/**
|
|
49
|
+
* Register Prometheus metrics by polling pg-boss queue stats.
|
|
50
|
+
*
|
|
51
|
+
* pg-boss v12 removed the `monitor-states` event, so we poll
|
|
52
|
+
* `getQueues()` + `getQueueStats()` on a timer instead.
|
|
53
|
+
*
|
|
54
|
+
* Returns the interval handle so callers can clear it on shutdown.
|
|
55
|
+
*/
|
|
56
|
+
const registerPgBossMetrics = (boss, serviceName, logger, mattermostWebhookUrl) => {
|
|
57
|
+
const collectMetrics = async () => {
|
|
58
|
+
try {
|
|
59
|
+
const queues = await boss.getQueues();
|
|
60
|
+
for (const queue of queues) {
|
|
61
|
+
const stats = await boss.getQueueStats(queue.name);
|
|
62
|
+
const labels = { queue: queue.name, service: serviceName };
|
|
63
|
+
queueDepth.set(labels, stats.queuedCount ?? 0);
|
|
64
|
+
queueActive.set(labels, stats.activeCount ?? 0);
|
|
65
|
+
queueTotal.set(labels, stats.totalCount ?? 0);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
catch (error) {
|
|
69
|
+
logger.error({ error }, 'Failed to collect pg-boss metrics');
|
|
70
|
+
}
|
|
71
|
+
};
|
|
72
|
+
if (mattermostWebhookUrl) {
|
|
73
|
+
logger.info('Mattermost DLQ alerting enabled');
|
|
74
|
+
}
|
|
75
|
+
// Initial collection + interval
|
|
76
|
+
void collectMetrics();
|
|
77
|
+
return setInterval(collectMetrics, POLL_INTERVAL_MS);
|
|
78
|
+
};
|
|
79
|
+
exports.registerPgBossMetrics = registerPgBossMetrics;
|
|
80
|
+
/**
|
|
81
|
+
* Send a Mattermost alert for a dead-letter job.
|
|
82
|
+
*/
|
|
83
|
+
const sendDlqAlert = async (webhookUrl, queueName, jobId, error, logger) => {
|
|
84
|
+
try {
|
|
85
|
+
await axios_1.default.post(webhookUrl, {
|
|
86
|
+
text: `🚨 **pg-boss DLQ Alert**\n**Queue:** \`${queueName}\`\n**Job ID:** \`${jobId}\`\n**Error:** ${error}`,
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
catch (webhookError) {
|
|
90
|
+
logger.error({ error: webhookError }, 'Failed to send Mattermost DLQ alert');
|
|
91
|
+
}
|
|
92
|
+
};
|
|
93
|
+
exports.sendDlqAlert = sendDlqAlert;
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.enqueue = void 0;
|
|
4
|
+
const enqueue = async (boss, logger, queueName, data, options) => {
|
|
5
|
+
const jobId = await boss.send(queueName, data || {}, options || {});
|
|
6
|
+
logger.info({ queue: queueName, jobId }, 'Job enqueued');
|
|
7
|
+
return jobId;
|
|
8
|
+
};
|
|
9
|
+
exports.enqueue = enqueue;
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.getJob = void 0;
|
|
4
|
+
const getJob = async (boss, queueName, jobId) => {
|
|
5
|
+
const jobs = await boss.findJobs(queueName, { id: jobId });
|
|
6
|
+
return jobs.length > 0 ? { ...jobs[0] } : null;
|
|
7
|
+
};
|
|
8
|
+
exports.getJob = getJob;
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { PgBoss } from 'pg-boss';
|
|
2
|
+
export interface QueueStats {
|
|
3
|
+
name: string;
|
|
4
|
+
policy?: string;
|
|
5
|
+
retryLimit?: number;
|
|
6
|
+
retryDelay?: number;
|
|
7
|
+
retryBackoff?: boolean;
|
|
8
|
+
expireInSeconds?: number;
|
|
9
|
+
deadLetter?: string | null;
|
|
10
|
+
queuedCount: number;
|
|
11
|
+
activeCount: number;
|
|
12
|
+
totalCount: number;
|
|
13
|
+
}
|
|
14
|
+
export declare const getQueueStats: (boss: PgBoss) => Promise<QueueStats[]>;
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.getQueueStats = void 0;
|
|
4
|
+
const getQueueStats = async (boss) => {
|
|
5
|
+
const queues = await boss.getQueues();
|
|
6
|
+
return Promise.all(queues.map(async (q) => {
|
|
7
|
+
const stats = await boss.getQueueStats(q.name);
|
|
8
|
+
return {
|
|
9
|
+
name: q.name,
|
|
10
|
+
policy: q.policy,
|
|
11
|
+
retryLimit: q.retryLimit,
|
|
12
|
+
retryDelay: q.retryDelay,
|
|
13
|
+
retryBackoff: q.retryBackoff,
|
|
14
|
+
expireInSeconds: q.expireInSeconds,
|
|
15
|
+
deadLetter: q.deadLetter,
|
|
16
|
+
queuedCount: stats.queuedCount,
|
|
17
|
+
activeCount: stats.activeCount,
|
|
18
|
+
totalCount: stats.totalCount,
|
|
19
|
+
};
|
|
20
|
+
}));
|
|
21
|
+
};
|
|
22
|
+
exports.getQueueStats = getQueueStats;
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { PgBoss } from 'pg-boss';
|
|
2
|
+
/**
|
|
3
|
+
* Retrieve persisted warnings from the pg-boss warning table.
|
|
4
|
+
*
|
|
5
|
+
* pg-boss v12 persists warnings (slow queries, queue backlogs, clock skew)
|
|
6
|
+
* when `persistWarnings: true` is set. There is no public class method,
|
|
7
|
+
* so we query the warning table via `getDb()`.
|
|
8
|
+
*/
|
|
9
|
+
export declare const getWarnings: (boss: PgBoss, schema: string, limit?: number) => Promise<Record<string, unknown>[]>;
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.getWarnings = void 0;
|
|
4
|
+
/**
|
|
5
|
+
* Retrieve persisted warnings from the pg-boss warning table.
|
|
6
|
+
*
|
|
7
|
+
* pg-boss v12 persists warnings (slow queries, queue backlogs, clock skew)
|
|
8
|
+
* when `persistWarnings: true` is set. There is no public class method,
|
|
9
|
+
* so we query the warning table via `getDb()`.
|
|
10
|
+
*/
|
|
11
|
+
const getWarnings = async (boss, schema, limit = 100) => {
|
|
12
|
+
const db = boss.getDb();
|
|
13
|
+
const result = await db.executeSql(`SELECT id, type, message, data, created_on
|
|
14
|
+
FROM "${schema}".warning
|
|
15
|
+
ORDER BY created_on DESC
|
|
16
|
+
LIMIT $1`, [limit]);
|
|
17
|
+
return result.rows;
|
|
18
|
+
};
|
|
19
|
+
exports.getWarnings = getWarnings;
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.listJobs = void 0;
|
|
4
|
+
const listJobs = async (boss, queueName) => {
|
|
5
|
+
const jobs = await boss.findJobs(queueName);
|
|
6
|
+
return jobs.map(j => ({ ...j }));
|
|
7
|
+
};
|
|
8
|
+
exports.listJobs = listJobs;
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.manageJob = void 0;
|
|
4
|
+
const manageJob = async (boss, logger, action, queueName, jobId) => {
|
|
5
|
+
switch (action) {
|
|
6
|
+
case 'retry':
|
|
7
|
+
await boss.resume(queueName, jobId);
|
|
8
|
+
break;
|
|
9
|
+
case 'cancel':
|
|
10
|
+
await boss.cancel(queueName, jobId);
|
|
11
|
+
break;
|
|
12
|
+
case 'delete':
|
|
13
|
+
await boss.deleteJob(queueName, jobId);
|
|
14
|
+
break;
|
|
15
|
+
}
|
|
16
|
+
logger.info({ queue: queueName, jobId, action }, `Job ${action} completed`);
|
|
17
|
+
};
|
|
18
|
+
exports.manageJob = manageJob;
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import { PgBoss } from 'pg-boss';
|
|
2
|
+
import type { Logger } from 'pino';
|
|
3
|
+
export declare const getSchedules: (boss: PgBoss) => Promise<Record<string, unknown>[]>;
|
|
4
|
+
export declare const createSchedule: (boss: PgBoss, logger: Logger, name: string, cron: string, data?: Record<string, unknown>, options?: Record<string, unknown>) => Promise<void>;
|
|
5
|
+
export declare const updateSchedule: (boss: PgBoss, logger: Logger, name: string, cron: string, data?: Record<string, unknown>, options?: Record<string, unknown>) => Promise<void>;
|
|
6
|
+
export declare const deleteSchedule: (boss: PgBoss, logger: Logger, name: string) => Promise<void>;
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.deleteSchedule = exports.updateSchedule = exports.createSchedule = exports.getSchedules = void 0;
|
|
4
|
+
const getSchedules = async (boss) => {
|
|
5
|
+
const schedules = await boss.getSchedules();
|
|
6
|
+
return schedules.map(s => ({ ...s }));
|
|
7
|
+
};
|
|
8
|
+
exports.getSchedules = getSchedules;
|
|
9
|
+
const createSchedule = async (boss, logger, name, cron, data, options) => {
|
|
10
|
+
await boss.schedule(name, cron, data || {}, options || {});
|
|
11
|
+
logger.info({ name, cron }, 'Schedule created');
|
|
12
|
+
};
|
|
13
|
+
exports.createSchedule = createSchedule;
|
|
14
|
+
const updateSchedule = async (boss, logger, name, cron, data, options) => {
|
|
15
|
+
await boss.unschedule(name);
|
|
16
|
+
await boss.schedule(name, cron, data || {}, options || {});
|
|
17
|
+
logger.info({ name, cron }, 'Schedule updated');
|
|
18
|
+
};
|
|
19
|
+
exports.updateSchedule = updateSchedule;
|
|
20
|
+
const deleteSchedule = async (boss, logger, name) => {
|
|
21
|
+
await boss.unschedule(name);
|
|
22
|
+
logger.info({ name }, 'Schedule deleted');
|
|
23
|
+
};
|
|
24
|
+
exports.deleteSchedule = deleteSchedule;
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { PgBoss } from 'pg-boss';
|
|
2
|
+
import type { Logger } from 'pino';
|
|
3
|
+
export declare class PgBossService {
|
|
4
|
+
private readonly boss;
|
|
5
|
+
private readonly logger;
|
|
6
|
+
private readonly schema;
|
|
7
|
+
constructor(boss: PgBoss, logger: Logger, schema?: string);
|
|
8
|
+
getQueueStats: () => Promise<import("./service/getQueueStats").QueueStats[]>;
|
|
9
|
+
listJobs: (queueName: string) => Promise<Record<string, unknown>[]>;
|
|
10
|
+
getJob: (queueName: string, jobId: string) => Promise<Record<string, unknown> | null>;
|
|
11
|
+
retryJob: (queueName: string, jobId: string) => Promise<void>;
|
|
12
|
+
cancelJob: (queueName: string, jobId: string) => Promise<void>;
|
|
13
|
+
deleteJob: (queueName: string, jobId: string) => Promise<void>;
|
|
14
|
+
getSchedules: () => Promise<Record<string, unknown>[]>;
|
|
15
|
+
createSchedule: (name: string, cron: string, data?: Record<string, unknown>, options?: Record<string, unknown>) => Promise<void>;
|
|
16
|
+
updateSchedule: (name: string, cron: string, data?: Record<string, unknown>, options?: Record<string, unknown>) => Promise<void>;
|
|
17
|
+
deleteSchedule: (name: string) => Promise<void>;
|
|
18
|
+
enqueueJob: (queueName: string, data?: Record<string, unknown>, options?: Record<string, unknown>) => Promise<string | null>;
|
|
19
|
+
getWarnings: (limit?: number) => Promise<Record<string, unknown>[]>;
|
|
20
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.PgBossService = void 0;
|
|
4
|
+
const getQueueStats_1 = require("./service/getQueueStats");
|
|
5
|
+
const listJobs_1 = require("./service/listJobs");
|
|
6
|
+
const getJob_1 = require("./service/getJob");
|
|
7
|
+
const manageJob_1 = require("./service/manageJob");
|
|
8
|
+
const schedules_1 = require("./service/schedules");
|
|
9
|
+
const enqueue_1 = require("./service/enqueue");
|
|
10
|
+
const getWarnings_1 = require("./service/getWarnings");
|
|
11
|
+
class PgBossService {
|
|
12
|
+
constructor(boss, logger, schema = 'pgboss') {
|
|
13
|
+
this.boss = boss;
|
|
14
|
+
this.logger = logger;
|
|
15
|
+
this.schema = schema;
|
|
16
|
+
this.getQueueStats = () => (0, getQueueStats_1.getQueueStats)(this.boss);
|
|
17
|
+
this.listJobs = (queueName) => (0, listJobs_1.listJobs)(this.boss, queueName);
|
|
18
|
+
this.getJob = (queueName, jobId) => (0, getJob_1.getJob)(this.boss, queueName, jobId);
|
|
19
|
+
this.retryJob = (queueName, jobId) => (0, manageJob_1.manageJob)(this.boss, this.logger, 'retry', queueName, jobId);
|
|
20
|
+
this.cancelJob = (queueName, jobId) => (0, manageJob_1.manageJob)(this.boss, this.logger, 'cancel', queueName, jobId);
|
|
21
|
+
this.deleteJob = (queueName, jobId) => (0, manageJob_1.manageJob)(this.boss, this.logger, 'delete', queueName, jobId);
|
|
22
|
+
this.getSchedules = () => (0, schedules_1.getSchedules)(this.boss);
|
|
23
|
+
this.createSchedule = (name, cron, data, options) => (0, schedules_1.createSchedule)(this.boss, this.logger, name, cron, data, options);
|
|
24
|
+
this.updateSchedule = (name, cron, data, options) => (0, schedules_1.updateSchedule)(this.boss, this.logger, name, cron, data, options);
|
|
25
|
+
this.deleteSchedule = (name) => (0, schedules_1.deleteSchedule)(this.boss, this.logger, name);
|
|
26
|
+
this.enqueueJob = (queueName, data, options) => (0, enqueue_1.enqueue)(this.boss, this.logger, queueName, data, options);
|
|
27
|
+
this.getWarnings = (limit) => (0, getWarnings_1.getWarnings)(this.boss, this.schema, limit);
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
exports.PgBossService = PgBossService;
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import type { QueueOptions, Job, WorkOptions } from 'pg-boss';
|
|
2
|
+
import type { PgBoss } from 'pg-boss';
|
|
3
|
+
export type { SendOptions, QueueOptions, WorkOptions, Job } from 'pg-boss';
|
|
4
|
+
export interface PgBossConfig {
|
|
5
|
+
enabled: boolean;
|
|
6
|
+
connectionString: string;
|
|
7
|
+
schema: string;
|
|
8
|
+
/** Max connections in the pg-boss pool. Default: 3 */
|
|
9
|
+
maxConnectionPoolSize?: number;
|
|
10
|
+
/** Days to keep completed/failed jobs. Default: 30 (GDPR-aligned) */
|
|
11
|
+
deleteAfterDays?: number;
|
|
12
|
+
/** Application name for PG connections. Default: "pgboss" */
|
|
13
|
+
applicationName?: string;
|
|
14
|
+
/** Mattermost webhook URL for DLQ alerts (optional) */
|
|
15
|
+
mattermostWebhookUrl?: string;
|
|
16
|
+
}
|
|
17
|
+
export interface QueueDefinition {
|
|
18
|
+
name: string;
|
|
19
|
+
options?: QueueOptions;
|
|
20
|
+
}
|
|
21
|
+
export interface WorkerRegistration<T = object> {
|
|
22
|
+
queue: string;
|
|
23
|
+
handler: (job: Job<T>[]) => Promise<void>;
|
|
24
|
+
options?: WorkOptions;
|
|
25
|
+
}
|
|
26
|
+
export type PgBossInstance = PgBoss;
|
|
@@ -14,6 +14,7 @@ var Environment;
|
|
|
14
14
|
Environment["P4"] = "p4";
|
|
15
15
|
Environment["P5"] = "p5";
|
|
16
16
|
Environment["P6"] = "p6";
|
|
17
|
+
Environment["P7"] = "p7";
|
|
17
18
|
Environment["PREPROD"] = "preprod";
|
|
18
19
|
Environment["PROD"] = "prod";
|
|
19
20
|
})(Environment || (exports.Environment = Environment = {}));
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lucaapp/service-utils",
|
|
3
|
-
"version": "5.
|
|
3
|
+
"version": "5.7.0",
|
|
4
4
|
"main": "dist/index.js",
|
|
5
5
|
"types": "dist/index.d.ts",
|
|
6
6
|
"files": [
|
|
@@ -15,11 +15,11 @@
|
|
|
15
15
|
"test": "vitest run",
|
|
16
16
|
"test:coverage": "vitest run --coverage",
|
|
17
17
|
"test:ci": "VITEST_JUNIT_SUITE_NAME=service-utils CI=true vitest run --coverage",
|
|
18
|
-
"audit": "improved-yarn-audit --ignore-dev-deps"
|
|
18
|
+
"audit": "sh ../../scripts/cached-audit.sh improved-yarn-audit --ignore-dev-deps"
|
|
19
19
|
},
|
|
20
20
|
"dependencies": {
|
|
21
21
|
"@apidevtools/swagger-parser": "10.0.3",
|
|
22
|
-
"@asteasolutions/zod-to-openapi": "
|
|
22
|
+
"@asteasolutions/zod-to-openapi": "7.3.4",
|
|
23
23
|
"@aws-sdk/client-s3": "^3.993.0",
|
|
24
24
|
"@aws-sdk/lib-storage": "^3.993.0",
|
|
25
25
|
"@hapi/boom": "^10.0.1",
|
|
@@ -57,7 +57,8 @@
|
|
|
57
57
|
"url-value-parser": "^2.2.0",
|
|
58
58
|
"uuid": "^9.0.0",
|
|
59
59
|
"validator": "13.15.23",
|
|
60
|
-
"zod": "3.24.1"
|
|
60
|
+
"zod": "3.24.1",
|
|
61
|
+
"pg-boss": "12.14.0"
|
|
61
62
|
},
|
|
62
63
|
"devDependencies": {
|
|
63
64
|
"@types/busboy": "^1.5.4",
|
|
@@ -68,25 +69,31 @@
|
|
|
68
69
|
"@types/uuid": "^8.3.4",
|
|
69
70
|
"@typescript-eslint/eslint-plugin": "^7.15.0",
|
|
70
71
|
"@typescript-eslint/parser": "^7.15.0",
|
|
71
|
-
"@vitest/coverage-v8": "
|
|
72
|
+
"@vitest/coverage-v8": "4.1.1",
|
|
72
73
|
"conventional-changelog-conventionalcommits": "^8.0.0",
|
|
73
74
|
"eslint": "8.57.0",
|
|
74
75
|
"eslint-plugin-prettier": "4.2.1",
|
|
75
76
|
"eslint-plugin-vitest": "0.4.1",
|
|
76
77
|
"improved-yarn-audit": "^3.0.4",
|
|
77
78
|
"prettier": "2.7.1",
|
|
78
|
-
"semantic-release": "^
|
|
79
|
+
"semantic-release": "^25.0.3",
|
|
79
80
|
"semantic-release-monorepo": "8.0.2",
|
|
80
81
|
"supertest": "^6.3.3",
|
|
81
82
|
"typescript": "5.5.3",
|
|
82
83
|
"vite-tsconfig-paths": "4.3.2",
|
|
83
|
-
"vitest": "
|
|
84
|
+
"vitest": "4.1.1"
|
|
84
85
|
},
|
|
85
86
|
"resolutions": {
|
|
86
87
|
"tar": "^7.5.11",
|
|
87
88
|
"lodash": "^4.17.23",
|
|
88
89
|
"qs": "^6.14.2",
|
|
89
90
|
"dottie": "2.0.7",
|
|
90
|
-
"fast-xml-parser": "^5.5.6"
|
|
91
|
+
"fast-xml-parser": "^5.5.6",
|
|
92
|
+
"yaml": "^2.8.3",
|
|
93
|
+
"openapi3-ts": "4.5.0",
|
|
94
|
+
"**/micromatch/picomatch": "2.3.2",
|
|
95
|
+
"**/tinyglobby/picomatch": "4.0.4",
|
|
96
|
+
"picomatch": "2.3.2",
|
|
97
|
+
"node-forge": "1.4.0"
|
|
91
98
|
}
|
|
92
99
|
}
|