@balena/pinejs 16.2.0-build-joshbwlng-tasks-a83c83b4c78803915d0cb6297cc8cc1862622d08-1 → 17.0.0-build-wip-large-file-uploads-d6522dad962bc0bff6ee7c596df8f43f596b6aaa-1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (58) hide show
  1. package/.pinejs-cache.json +1 -1
  2. package/.versionbot/CHANGELOG.yml +7 -7
  3. package/CHANGELOG.md +3 -3
  4. package/VERSION +1 -1
  5. package/out/config-loader/env.d.ts +0 -4
  6. package/out/config-loader/env.js +1 -5
  7. package/out/config-loader/env.js.map +1 -1
  8. package/out/database-layer/db.d.ts +0 -3
  9. package/out/database-layer/db.js +0 -14
  10. package/out/database-layer/db.js.map +1 -1
  11. package/out/migrator/utils.js +2 -2
  12. package/out/migrator/utils.js.map +1 -1
  13. package/out/sbvr-api/sbvr-utils.d.ts +0 -1
  14. package/out/sbvr-api/sbvr-utils.js +1 -6
  15. package/out/sbvr-api/sbvr-utils.js.map +1 -1
  16. package/out/server-glue/module.d.ts +0 -1
  17. package/out/server-glue/module.js +1 -2
  18. package/out/server-glue/module.js.map +1 -1
  19. package/out/webresource-handler/handlers/NoopHandler.d.ts +3 -0
  20. package/out/webresource-handler/handlers/NoopHandler.js +6 -0
  21. package/out/webresource-handler/handlers/NoopHandler.js.map +1 -1
  22. package/out/webresource-handler/handlers/S3Handler.d.ts +7 -0
  23. package/out/webresource-handler/handlers/S3Handler.js +68 -2
  24. package/out/webresource-handler/handlers/S3Handler.js.map +1 -1
  25. package/out/webresource-handler/index.d.ts +5 -0
  26. package/out/webresource-handler/index.js +10 -5
  27. package/out/webresource-handler/index.js.map +1 -1
  28. package/out/webresource-handler/multipartUpload.d.ts +40 -0
  29. package/out/webresource-handler/multipartUpload.js +125 -0
  30. package/out/webresource-handler/multipartUpload.js.map +1 -0
  31. package/package.json +4 -7
  32. package/src/config-loader/env.ts +1 -6
  33. package/src/database-layer/db.ts +0 -24
  34. package/src/migrator/utils.ts +1 -1
  35. package/src/sbvr-api/sbvr-utils.ts +1 -5
  36. package/src/server-glue/module.ts +0 -1
  37. package/src/webresource-handler/handlers/NoopHandler.ts +21 -0
  38. package/src/webresource-handler/handlers/S3Handler.ts +130 -4
  39. package/src/webresource-handler/index.ts +24 -1
  40. package/src/webresource-handler/multipartUpload.ts +214 -0
  41. package/out/tasks/common.d.ts +0 -4
  42. package/out/tasks/common.js +0 -13
  43. package/out/tasks/common.js.map +0 -1
  44. package/out/tasks/index.d.ts +0 -10
  45. package/out/tasks/index.js +0 -139
  46. package/out/tasks/index.js.map +0 -1
  47. package/out/tasks/model.sbvr +0 -60
  48. package/out/tasks/types.d.ts +0 -38
  49. package/out/tasks/types.js +0 -10
  50. package/out/tasks/types.js.map +0 -1
  51. package/out/tasks/worker.d.ts +0 -16
  52. package/out/tasks/worker.js +0 -191
  53. package/out/tasks/worker.js.map +0 -1
  54. package/src/tasks/common.ts +0 -14
  55. package/src/tasks/index.ts +0 -158
  56. package/src/tasks/model.sbvr +0 -60
  57. package/src/tasks/types.ts +0 -58
  58. package/src/tasks/worker.ts +0 -246
@@ -1,139 +0,0 @@
1
- "use strict";
2
- var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
- if (k2 === undefined) k2 = k;
4
- var desc = Object.getOwnPropertyDescriptor(m, k);
5
- if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
- desc = { enumerable: true, get: function() { return m[k]; } };
7
- }
8
- Object.defineProperty(o, k2, desc);
9
- }) : (function(o, m, k, k2) {
10
- if (k2 === undefined) k2 = k;
11
- o[k2] = m[k];
12
- }));
13
- var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
- Object.defineProperty(o, "default", { enumerable: true, value: v });
15
- }) : function(o, v) {
16
- o["default"] = v;
17
- });
18
- var __importStar = (this && this.__importStar) || function (mod) {
19
- if (mod && mod.__esModule) return mod;
20
- var result = {};
21
- if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
- __setModuleDefault(result, mod);
23
- return result;
24
- };
25
- var __exportStar = (this && this.__exportStar) || function(m, exports) {
26
- for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
27
- };
28
- Object.defineProperty(exports, "__esModule", { value: true });
29
- exports.addTaskHandler = exports.setup = exports.config = void 0;
30
- const cronParser = __importStar(require("cron-parser"));
31
- const env_1 = require("../config-loader/env");
32
- const errors_1 = require("../sbvr-api/errors");
33
- const hooks_1 = require("../sbvr-api/hooks");
34
- const sbvr_utils_1 = require("../sbvr-api/sbvr-utils");
35
- const common_1 = require("./common");
36
- const worker_1 = require("./worker");
37
- __exportStar(require("./types"), exports);
38
- const modelText = require('./model.sbvr');
39
- exports.config = {
40
- models: [
41
- {
42
- modelName: common_1.apiRoot,
43
- apiRoot: common_1.apiRoot,
44
- modelText,
45
- customServerCode: exports,
46
- },
47
- ],
48
- };
49
- async function createTrigger(tx) {
50
- await tx.executeSql(`
51
- CREATE OR REPLACE FUNCTION notify_task_insert()
52
- RETURNS TRIGGER AS $$
53
- BEGIN
54
- PERFORM pg_notify('${common_1.channel}', NEW.id::text);
55
- RETURN NEW;
56
- END;
57
- $$ LANGUAGE plpgsql;
58
- `);
59
- await tx.executeSql(`
60
- CREATE OR REPLACE TRIGGER task_insert_trigger
61
- AFTER INSERT ON task
62
- FOR EACH ROW WHEN (NEW.status = 'pending' AND NEW."is scheduled to execute on-time" IS NULL)
63
- EXECUTE FUNCTION notify_task_insert();
64
- `);
65
- }
66
- let worker = null;
67
- async function setup(db, tx) {
68
- if (db.engine !== 'postgres') {
69
- return;
70
- }
71
- await createTrigger(tx);
72
- const client = new sbvr_utils_1.PinejsClient({
73
- apiPrefix: `/${common_1.apiRoot}/`,
74
- });
75
- worker = new worker_1.Worker(client);
76
- (0, hooks_1.addPureHook)('POST', common_1.apiRoot, 'task', {
77
- POSTPARSE: async ({ req, request }) => {
78
- request.values.is_created_by__actor =
79
- req.user?.actor ?? req.apiKey?.actor;
80
- if (request.values.is_created_by__actor == null) {
81
- throw new errors_1.BadRequestError('Creating tasks with missing actor on req is not allowed');
82
- }
83
- request.values.status = 'pending';
84
- request.values.attempt_count = 0;
85
- request.values.priority ??= 1;
86
- request.values.attempt_limit ??= 1;
87
- if (request.values.is_scheduled_with__cron_expression != null &&
88
- request.values.is_scheduled_to_execute_on__time == null) {
89
- try {
90
- request.values.is_scheduled_to_execute_on__time = cronParser
91
- .parseExpression(request.values.is_scheduled_with__cron_expression)
92
- .next()
93
- .toDate()
94
- .toISOString();
95
- }
96
- catch (_) {
97
- throw new errors_1.BadRequestError(`Invalid cron expression: ${request.values.is_scheduled_with__cron_expression}`);
98
- }
99
- }
100
- if (request.values.is_scheduled_to_execute_on__time != null) {
101
- const now = new Date(new Date().getTime() + env_1.tasks.queueIntervalMS);
102
- const startTime = new Date(request.values.is_scheduled_to_execute_on__time);
103
- if (startTime < now) {
104
- throw new errors_1.BadRequestError(`Task scheduled start time must be greater than ${env_1.tasks.queueIntervalMS} milliseconds in the future`);
105
- }
106
- }
107
- const handlerName = request.values.is_executed_by__handler;
108
- if (handlerName == null) {
109
- throw new errors_1.BadRequestError(`Must specify a task handler to execute`);
110
- }
111
- const handler = worker?.handlers[handlerName];
112
- if (handler == null) {
113
- throw new errors_1.BadRequestError(`No task handler with name '${handlerName}' registered`);
114
- }
115
- if (handler.validate != null) {
116
- if (!handler.validate(request.values.is_executed_with__parameter_set)) {
117
- throw new errors_1.BadRequestError(`Invalid parameter set: ${common_1.ajv.errorsText(handler.validate.errors)}`);
118
- }
119
- }
120
- },
121
- });
122
- worker.start();
123
- }
124
- exports.setup = setup;
125
- function addTaskHandler(name, fn, schema) {
126
- if (worker == null) {
127
- return;
128
- }
129
- if (worker.handlers[name] != null) {
130
- throw new Error(`Task handler with name '${name}' already registered`);
131
- }
132
- worker.handlers[name] = {
133
- name,
134
- fn,
135
- validate: schema != null ? common_1.ajv.compile(schema) : undefined,
136
- };
137
- }
138
- exports.addTaskHandler = addTaskHandler;
139
- //# sourceMappingURL=index.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/tasks/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AACA,wDAA0C;AAC1C,8CAAyD;AAEzD,+CAAqD;AACrD,6CAAgD;AAChD,uDAAsD;AAEtD,qCAAiD;AAEjD,qCAAkC;AAElC,0CAAwB;AAGxB,MAAM,SAAS,GAAW,OAAO,CAAC,cAAc,CAAC,CAAC;AAErC,QAAA,MAAM,GAAG;IACrB,MAAM,EAAE;QACP;YACC,SAAS,EAAE,gBAAO;YAClB,OAAO,EAAP,gBAAO;YACP,SAAS;YACT,gBAAgB,EAAE,OAAO;SACzB;KAC8B;CAChC,CAAC;AAIF,KAAK,UAAU,aAAa,CAAC,EAAS;IACrC,MAAM,EAAE,CAAC,UAAU,CAAC;;;;iCAIY,gBAAO;;;;KAInC,CAAC,CAAC;IAGN,MAAM,EAAE,CAAC,UAAU,CAAC;;;;;KAKhB,CAAC,CAAC;AACP,CAAC;AAED,IAAI,MAAM,GAAkB,IAAI,CAAC;AAC1B,KAAK,UAAU,KAAK,CAAC,EAAe,EAAE,EAAS;IAErD,IAAI,EAAE,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;QAC9B,OAAO;IACR,CAAC;IAGD,MAAM,aAAa,CAAC,EAAE,CAAC,CAAC;IAExB,MAAM,MAAM,GAAG,IAAI,yBAAY,CAAC;QAC/B,SAAS,EAAE,IAAI,gBAAO,GAAG;KACzB,CAAC,CAAC;IACH,MAAM,GAAG,IAAI,eAAM,CAAC,MAAM,CAAC,CAAC;IAG5B,IAAA,mBAAW,EAAC,MAAM,EAAE,gBAAO,EAAE,MAAM,EAAE;QACpC,SAAS,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE,EAAE;YAErC,OAAO,CAAC,MAAM,CAAC,oBAAoB;gBAClC,GAAG,CAAC,IAAI,EAAE,KAAK,IAAI,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC;YACtC,IAAI,OAAO,CAAC,MAAM,CAAC,oBAAoB,IAAI,IAAI,EAAE,CAAC;gBACjD,MAAM,IAAI,wBAAe,CACxB,yDAAyD,CACzD,CAAC;YACH,CAAC;YAGD,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,SAAS,CAAC;YAClC,OAAO,CAAC,MAAM,CAAC,aAAa,GAAG,CAAC,CAAC;YACjC,OAAO,CAAC,MAAM,CAAC,QAAQ,KAAK,CAAC,CAAC;YAC9B,OAAO,CAAC,MAAM,CAAC,aAAa,KAAK,CAAC,CAAC;YAGnC,IACC,OAAO,CAAC,MAAM,CAAC,kCAAkC,IAAI,IAAI;gBACzD,OAAO,CAAC,MAAM,CAAC,gCAAgC,IAAI,IAAI,EACtD,CAAC;gBACF,IAAI,CAAC;oBACJ,OAAO,CAAC,MAAM,CAAC,gCAAgC,GAAG,UAAU;yBAC1D,eAAe,CAAC,OAAO,CAAC,MAAM,CAAC,kCAAkC,CAAC;yBAClE,IAAI,EAAE;yBACN,MAAM,EAAE;yBACR,WAAW,EAAE,CAAC;gBACjB,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACZ,MAAM,IAAI,wBAAe,CACxB,4BAA4B,OAAO,CAAC,MAAM,CAAC,kCAAkC,EAAE,CAC/E,CAAC;gBACH,CAAC;YACF,CAAC;YAGD,IAAI,OAAO,CAAC,MAAM,CAAC,gCAAgC,IAAI,IAAI,EAAE,CAAC;gBAC7D,MAAM,GAAG,GAAG,IAAI,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,WAAQ,CAAC,eAAe,CAAC,CAAC;gBACtE,MAAM,SAAS,GAAG,IAAI,IAAI,CACzB,OAAO,CAAC,MAAM,CAAC,gCAAgC,CAC/C,CAAC;gBACF,IAAI,SAAS,GAAG,GAAG,EAAE,CAAC;oBACrB,MAAM,IAAI,wBAAe,CACxB,kDAAkD,WAAQ,CAAC,eAAe,6BAA6B,CACvG,CAAC;gBACH,CAAC;YACF,CAAC;YAGD,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,uBAAuB,CAAC;YAC3D,IAAI,WAAW,IAAI,IAAI,EAAE,CAAC;gBACzB,MAAM,IAAI,wBAAe,CAAC,wCAAwC,CAAC,CAAC;YACrE,CAAC;YACD,MAAM,OAAO,GAAG,MAAM,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;YAC9C,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;gBACrB,MAAM,IAAI,wBAAe,CACxB,8BAA8B,WAAW,cAAc,CACvD,CAAC;YACH,CAAC;YAGD,IAAI,OAAO,CAAC,QAAQ,IAAI,IAAI,EAAE,CAAC;gBAC9B,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,+BAA+B,CAAC,EAAE,CAAC;oBACvE,MAAM,IAAI,wBAAe,CACxB,0BAA0B,YAAG,CAAC,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CACnE,CAAC;gBACH,CAAC;YACF,CAAC;QACF,CAAC;KACD,CAAC,CAAC;IACH,MAAM,CAAC,KAAK,EAAE,CAAC;AAChB,CAAC;AAtFD,sBAsFC;AAGD,SAAgB,cAAc,CAC7B,IAAY,EACZ,EAAqB,EACrB,MAAe;IAEf,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;QACpB,OAAO;IACR,CAAC;IAED,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QACnC,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,sBAAsB,CAAC,CAAC;IACxE,CAAC;IACD,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG;QACvB,IAAI;QACJ,EAAE;QACF,QAAQ,EAAE,MAAM,IAAI,IAAI,CAAC,CAAC,CAAC,YAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS;KAC1D,CAAC;AACH,CAAC;AAjBD,wCAiBC"}
@@ -1,60 +0,0 @@
1
- Vocabulary: tasks
2
-
3
- Term: id
4
- Concept Type: Big Serial (Type)
5
- Term: actor
6
- Concept Type: Integer (Type)
7
- Term: attempt count
8
- Concept Type: Integer (Type)
9
- Term: attempt limit
10
- Concept Type: Integer (Type)
11
- Term: cron expression
12
- Concept Type: Short Text (Type)
13
- Term: error message
14
- Concept Type: Short Text (Type)
15
- Term: handler
16
- Concept Type: Short Text (Type)
17
- Term: key
18
- Concept Type: Short Text (Type)
19
- Term: parameter set
20
- Concept Type: JSON (Type)
21
- Term: priority
22
- Concept Type: Integer (Type)
23
- Term: status
24
- Concept Type: Short Text (Type)
25
- Term: time
26
- Concept Type: Date Time (Type)
27
-
28
- Term: task
29
- Fact type: task has id
30
- Necessity: each task has exactly one id
31
- Fact type: task has key
32
- Necessity: each task has at most one key
33
- Fact type: task is created by actor
34
- Necessity: each task is created by exactly one actor
35
- Fact type: task is executed by handler
36
- Necessity: each task is executed by exactly one handler
37
- Fact type: task is executed with parameter set
38
- Necessity: each task is executed with at most one parameter set
39
- Fact type: task has priority
40
- Necessity: each task has exactly one priority
41
- Necessity: each task has a priority that is greater than or equal to 0
42
- Fact type: task is scheduled with cron expression
43
- Necessity: each task is scheduled with at most one cron expression
44
- Fact type: task is scheduled to execute on time
45
- Necessity: each task is scheduled to execute on at most one time
46
- Fact type: task has status
47
- Necessity: each task has exactly one status
48
- Definition: "pending" or "cancelled" or "success" or "failed"
49
- Fact type: task started on time
50
- Necessity: each task started on at most one time
51
- Fact type: task ended on time
52
- Necessity: each task ended on at most one time
53
- Fact type: task has error message
54
- Necessity: each task has at most one error message
55
- Fact type: task has attempt count
56
- Necessity: each task has exactly one attempt count
57
- Fact type: task has attempt limit
58
- Necessity: each task has exactly one attempt limit
59
- Necessity: each task has an attempt limit that is greater than or equal to 1
60
-
@@ -1,38 +0,0 @@
1
- import type { ValidateFunction } from 'ajv';
2
- import type { AnyObject } from 'pinejs-client-core';
3
- import type * as Db from '../database-layer/db';
4
- import type { PinejsClient } from '../sbvr-api/sbvr-utils';
5
- export declare const taskStatuses: readonly ["pending", "cancelled", "success", "failed"];
6
- export type TaskStatus = (typeof taskStatuses)[number];
7
- export interface Task {
8
- id: number;
9
- created_at: Date;
10
- modified_at: Date;
11
- is_created_by__actor: number;
12
- is_executed_by__handler: string;
13
- is_executed_with__parameter_set: object | null;
14
- is_scheduled_with__cron_expression: string | null;
15
- is_scheduled_to_execute_on__time: Date | null;
16
- priority: number;
17
- status: TaskStatus;
18
- started_on__time: Date | null;
19
- ended_on__time: Date | null;
20
- error_message: string | null;
21
- attempt_count: number;
22
- attempt_limit: number;
23
- }
24
- export type PartialTask = Pick<Task, 'id' | 'is_created_by__actor' | 'is_executed_by__handler' | 'is_executed_with__parameter_set' | 'is_scheduled_with__cron_expression' | 'priority' | 'attempt_count' | 'attempt_limit'>;
25
- export interface TaskArgs {
26
- api: PinejsClient;
27
- params: AnyObject;
28
- tx: Db.Tx;
29
- }
30
- export type TaskResponse = Promise<{
31
- status: TaskStatus;
32
- error?: string;
33
- }>;
34
- export interface TaskHandler {
35
- name: string;
36
- fn: (options: TaskArgs) => TaskResponse;
37
- validate?: ValidateFunction;
38
- }
@@ -1,10 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.taskStatuses = void 0;
4
- exports.taskStatuses = [
5
- 'pending',
6
- 'cancelled',
7
- 'success',
8
- 'failed',
9
- ];
10
- //# sourceMappingURL=types.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/tasks/types.ts"],"names":[],"mappings":";;;AAKa,QAAA,YAAY,GAAG;IAC3B,SAAS;IACT,WAAW;IACX,SAAS;IACT,QAAQ;CACC,CAAC"}
@@ -1,16 +0,0 @@
1
- import { PinejsClient } from '../sbvr-api/sbvr-utils';
2
- import type { TaskHandler } from './types';
3
- export declare class Worker {
4
- handlers: Record<string, TaskHandler>;
5
- private readonly concurrency;
6
- private readonly interval;
7
- private client;
8
- private executing;
9
- constructor(client: PinejsClient);
10
- private canExecute;
11
- private execute;
12
- private finalize;
13
- private getNextAttemptTime;
14
- private poll;
15
- start(): void;
16
- }
@@ -1,191 +0,0 @@
1
- "use strict";
2
- var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
- if (k2 === undefined) k2 = k;
4
- var desc = Object.getOwnPropertyDescriptor(m, k);
5
- if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
- desc = { enumerable: true, get: function() { return m[k]; } };
7
- }
8
- Object.defineProperty(o, k2, desc);
9
- }) : (function(o, m, k, k2) {
10
- if (k2 === undefined) k2 = k;
11
- o[k2] = m[k];
12
- }));
13
- var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
- Object.defineProperty(o, "default", { enumerable: true, value: v });
15
- }) : function(o, v) {
16
- o["default"] = v;
17
- });
18
- var __importStar = (this && this.__importStar) || function (mod) {
19
- if (mod && mod.__esModule) return mod;
20
- var result = {};
21
- if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
- __setModuleDefault(result, mod);
23
- return result;
24
- };
25
- Object.defineProperty(exports, "__esModule", { value: true });
26
- exports.Worker = void 0;
27
- const env_1 = require("../config-loader/env");
28
- const permissions = __importStar(require("../sbvr-api/permissions"));
29
- const sbvr_utils_1 = require("../sbvr-api/sbvr-utils");
30
- const module_1 = require("../server-glue/module");
31
- const common_1 = require("./common");
32
- const selectColumns = Object.entries({
33
- id: 'id',
34
- 'is executed by-handler': 'is_executed_by__handler',
35
- 'is executed with-parameter set': 'is_executed_with__parameter_set',
36
- 'is scheduled with-cron expression': 'is_scheduled_with__cron_expression',
37
- 'attempt count': 'attempt_count',
38
- 'attempt limit': 'attempt_limit',
39
- priority: 'priority',
40
- 'is created by-actor': 'is_created_by__actor',
41
- })
42
- .map(([key, value]) => `t."${key}" AS "${value}"`)
43
- .join(', ');
44
- class Worker {
45
- constructor(client) {
46
- this.handlers = {};
47
- this.executing = 0;
48
- this.client = client;
49
- this.concurrency = env_1.tasks.queueConcurrency;
50
- this.interval = env_1.tasks.queueIntervalMS;
51
- }
52
- canExecute() {
53
- return (this.executing < this.concurrency && Object.keys(this.handlers).length > 0);
54
- }
55
- async execute(tx, task) {
56
- this.executing++;
57
- try {
58
- const handler = this.handlers[task.is_executed_by__handler];
59
- const startedOnTime = new Date();
60
- if (handler == null) {
61
- await this.finalize(tx, task, startedOnTime, 'failed', 'Matching task handler not found');
62
- return;
63
- }
64
- if (handler.validate != null &&
65
- !handler.validate(task.is_executed_with__parameter_set)) {
66
- await this.finalize(tx, task, startedOnTime, 'failed', `Invalid parameter set: ${common_1.ajv.errorsText(handler.validate.errors)}`);
67
- return;
68
- }
69
- const result = await handler.fn({
70
- api: new sbvr_utils_1.PinejsClient({
71
- passthrough: {
72
- tx,
73
- },
74
- }),
75
- params: task.is_executed_with__parameter_set ?? {},
76
- tx,
77
- });
78
- await this.finalize(tx, task, startedOnTime, result.status, result.error);
79
- }
80
- catch (err) {
81
- console.error('Task execution failed:', err);
82
- process.exit(1);
83
- }
84
- finally {
85
- this.executing--;
86
- }
87
- }
88
- async finalize(tx, task, startedOnTime, status, errorMessage) {
89
- const attemptCount = task.attempt_count + 1;
90
- const body = {
91
- started_on__time: startedOnTime,
92
- ended_on__time: new Date(),
93
- status,
94
- attempt_count: attemptCount,
95
- ...(errorMessage != null && { error_message: errorMessage }),
96
- };
97
- if (status === 'failed' && attemptCount < task.attempt_limit) {
98
- body.status = 'pending';
99
- body.is_scheduled_to_execute_on__time =
100
- this.getNextAttemptTime(attemptCount);
101
- }
102
- await this.client.patch({
103
- resource: 'task',
104
- passthrough: {
105
- tx,
106
- req: permissions.root,
107
- },
108
- id: task.id,
109
- body,
110
- });
111
- if (['failed', 'success'].includes(body.status) &&
112
- task.is_scheduled_with__cron_expression != null) {
113
- await this.client.post({
114
- resource: 'task',
115
- passthrough: {
116
- tx,
117
- req: permissions.root,
118
- },
119
- body: {
120
- attempt_limit: task.attempt_limit,
121
- is_created_by__actor: task.is_created_by__actor,
122
- is_executed_by__handler: task.is_executed_by__handler,
123
- is_executed_with__parameter_set: task.is_executed_with__parameter_set,
124
- is_scheduled_with__cron_expression: task.is_scheduled_with__cron_expression,
125
- priority: task.priority,
126
- },
127
- });
128
- }
129
- }
130
- getNextAttemptTime(attempt) {
131
- const delay = Math.ceil(Math.exp(Math.min(10, attempt))) * 1000;
132
- return new Date(Date.now() + delay);
133
- }
134
- poll() {
135
- let executed = false;
136
- const handlerNames = Object.keys(this.handlers);
137
- const binds = handlerNames.map((_, index) => `$${index + 1}`).join(', ');
138
- module_1.sbvrUtils.db
139
- .transaction(async (tx) => {
140
- if (!this.canExecute()) {
141
- return;
142
- }
143
- const result = await tx.executeSql(`SELECT ${selectColumns}
144
- FROM task AS t
145
- WHERE
146
- t."is executed by-handler" IN (${binds}) AND
147
- t."status" = 'pending' AND
148
- t."attempt count" <= t."attempt limit" AND
149
- (
150
- t."is scheduled to execute on-time" IS NULL OR
151
- t."is scheduled to execute on-time" <= CURRENT_TIMESTAMP + INTERVAL '${Math.ceil(this.interval / 1000)} second'
152
- )
153
- ORDER BY
154
- t."is scheduled to execute on-time" ASC,
155
- t."priority" DESC,
156
- t."id" ASC
157
- LIMIT ${Math.max(this.concurrency - this.executing, 0)}
158
- FOR UPDATE SKIP LOCKED`, handlerNames);
159
- if (result.rows.length === 0) {
160
- return;
161
- }
162
- await Promise.all(result.rows.map(async (row) => {
163
- await this.execute(tx, row);
164
- }));
165
- executed = true;
166
- })
167
- .catch((err) => {
168
- console.error('Failed polling for tasks:', err);
169
- })
170
- .finally(() => {
171
- setTimeout(() => this.poll(), executed ? 0 : this.interval);
172
- });
173
- }
174
- start() {
175
- module_1.sbvrUtils.db.on?.('notification', async (msg) => {
176
- if (this.canExecute()) {
177
- await module_1.sbvrUtils.db.transaction(async (tx) => {
178
- const result = await tx.executeSql(`SELECT ${selectColumns} FROM task AS t WHERE id = $1 FOR UPDATE SKIP LOCKED`, [msg.payload]);
179
- if (result.rows.length > 0) {
180
- await this.execute(tx, result.rows[0]);
181
- }
182
- });
183
- }
184
- }, {
185
- channel: common_1.channel,
186
- });
187
- this.poll();
188
- }
189
- }
190
- exports.Worker = Worker;
191
- //# sourceMappingURL=worker.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"worker.js","sourceRoot":"","sources":["../../src/tasks/worker.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AACA,8CAAyD;AAEzD,qEAAuD;AACvD,uDAAsD;AACtD,kDAAkD;AAClD,qCAAwC;AAIxC,MAAM,aAAa,GAAG,MAAM,CAAC,OAAO,CAAC;IACpC,EAAE,EAAE,IAAI;IACR,wBAAwB,EAAE,yBAAyB;IACnD,gCAAgC,EAAE,iCAAiC;IACnE,mCAAmC,EAAE,oCAAoC;IACzE,eAAe,EAAE,eAAe;IAChC,eAAe,EAAE,eAAe;IAChC,QAAQ,EAAE,UAAU;IACpB,qBAAqB,EAAE,sBAAsB;CAC7C,CAAC;KACA,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,MAAM,GAAG,SAAS,KAAK,GAAG,CAAC;KACjD,IAAI,CAAC,IAAI,CAAC,CAAC;AAKb,MAAa,MAAM;IAOlB,YAAY,MAAoB;QANzB,aAAQ,GAAgC,EAAE,CAAC;QAI1C,cAAS,GAAG,CAAC,CAAC;QAGrB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,WAAW,GAAG,WAAQ,CAAC,gBAAgB,CAAC;QAC7C,IAAI,CAAC,QAAQ,GAAG,WAAQ,CAAC,eAAe,CAAC;IAC1C,CAAC;IAGO,UAAU;QACjB,OAAO,CACN,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,WAAW,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,CAC1E,CAAC;IACH,CAAC;IAGO,KAAK,CAAC,OAAO,CAAC,EAAS,EAAE,IAAiB;QACjD,IAAI,CAAC,SAAS,EAAE,CAAC;QACjB,IAAI,CAAC;YAEJ,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;YAC5D,MAAM,aAAa,GAAG,IAAI,IAAI,EAAE,CAAC;YACjC,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;gBACrB,MAAM,IAAI,CAAC,QAAQ,CAClB,EAAE,EACF,IAAI,EACJ,aAAa,EACb,QAAQ,EACR,iCAAiC,CACjC,CAAC;gBACF,OAAO;YACR,CAAC;YAKD,IACC,OAAO,CAAC,QAAQ,IAAI,IAAI;gBACxB,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,+BAA+B,CAAC,EACtD,CAAC;gBACF,MAAM,IAAI,CAAC,QAAQ,CAClB,EAAE,EACF,IAAI,EACJ,aAAa,EACb,QAAQ,EACR,0BAA0B,YAAG,CAAC,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CACnE,CAAC;gBACF,OAAO;YACR,CAAC;YAGD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,EAAE,CAAC;gBAC/B,GAAG,EAAE,IAAI,yBAAY,CAAC;oBACrB,WAAW,EAAE;wBACZ,EAAE;qBACF;iBACD,CAAC;gBACF,MAAM,EAAE,IAAI,CAAC,+BAA+B,IAAI,EAAE;gBAClD,EAAE;aACF,CAAC,CAAC;YAGH,MAAM,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;QAC3E,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YAEd,OAAO,CAAC,KAAK,CAAC,wBAAwB,EAAE,GAAG,CAAC,CAAC;YAC7C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACjB,CAAC;gBAAS,CAAC;YACV,IAAI,CAAC,SAAS,EAAE,CAAC;QAClB,CAAC;IACF,CAAC;IAGO,KAAK,CAAC,QAAQ,CACrB,EAAS,EACT,IAAiB,EACjB,aAAmB,EACnB,MAAkB,EAClB,YAAqB;QAErB,MAAM,YAAY,GAAG,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC;QAC5C,MAAM,IAAI,GAAc;YACvB,gBAAgB,EAAE,aAAa;YAC/B,cAAc,EAAE,IAAI,IAAI,EAAE;YAC1B,MAAM;YACN,aAAa,EAAE,YAAY;YAC3B,GAAG,CAAC,YAAY,IAAI,IAAI,IAAI,EAAE,aAAa,EAAE,YAAY,EAAE,CAAC;SAC5D,CAAC;QAIF,IAAI,MAAM,KAAK,QAAQ,IAAI,YAAY,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;YAC9D,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC;YAGxB,IAAI,CAAC,gCAAgC;gBACpC,IAAI,CAAC,kBAAkB,CAAC,YAAY,CAAC,CAAC;QACxC,CAAC;QAGD,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;YACvB,QAAQ,EAAE,MAAM;YAChB,WAAW,EAAE;gBACZ,EAAE;gBACF,GAAG,EAAE,WAAW,CAAC,IAAI;aACrB;YACD,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,IAAI;SACJ,CAAC,CAAC;QAIH,IACC,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC;YAC3C,IAAI,CAAC,kCAAkC,IAAI,IAAI,EAC9C,CAAC;YACF,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;gBACtB,QAAQ,EAAE,MAAM;gBAChB,WAAW,EAAE;oBACZ,EAAE;oBACF,GAAG,EAAE,WAAW,CAAC,IAAI;iBACrB;gBACD,IAAI,EAAE;oBACL,aAAa,EAAE,IAAI,CAAC,aAAa;oBACjC,oBAAoB,EAAE,IAAI,CAAC,oBAAoB;oBAC/C,uBAAuB,EAAE,IAAI,CAAC,uBAAuB;oBACrD,+BAA+B,EAAE,IAAI,CAAC,+BAA+B;oBACrE,kCAAkC,EACjC,IAAI,CAAC,kCAAkC;oBACxC,QAAQ,EAAE,IAAI,CAAC,QAAQ;iBACvB;aACD,CAAC,CAAC;QACJ,CAAC;IACF,CAAC;IAGO,kBAAkB,CAAC,OAAe;QACzC,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;QAChE,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,CAAC;IACrC,CAAC;IAGO,IAAI;QACX,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAChD,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACzE,kBAAS,CAAC,EAAE;aACV,WAAW,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;YACzB,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,CAAC;gBACxB,OAAO;YACR,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,UAAU,CACjC,UAAU,aAAa;;;sCAGU,KAAK;;;;;6EAKkC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;;;;;;YAMhG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;2BAC/B,EACtB,YAAY,CACZ,CAAC;YACF,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC9B,OAAO;YACR,CAAC;YAGD,MAAM,OAAO,CAAC,GAAG,CAChB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;gBAC7B,MAAM,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,GAAkB,CAAC,CAAC;YAC5C,CAAC,CAAC,CACF,CAAC;YACF,QAAQ,GAAG,IAAI,CAAC;QACjB,CAAC,CAAC;aACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACd,OAAO,CAAC,KAAK,CAAC,2BAA2B,EAAE,GAAG,CAAC,CAAC;QACjD,CAAC,CAAC;aACD,OAAO,CAAC,GAAG,EAAE;YACb,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC7D,CAAC,CAAC,CAAC;IACL,CAAC;IAGM,KAAK;QACX,kBAAS,CAAC,EAAE,CAAC,EAAE,EAAE,CAChB,cAAc,EACd,KAAK,EAAE,GAAG,EAAE,EAAE;YACb,IAAI,IAAI,CAAC,UAAU,EAAE,EAAE,CAAC;gBACvB,MAAM,kBAAS,CAAC,EAAE,CAAC,WAAW,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;oBAC3C,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,UAAU,CACjC,UAAU,aAAa,sDAAsD,EAC7E,CAAC,GAAG,CAAC,OAAO,CAAC,CACb,CAAC;oBACF,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAC5B,MAAM,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAgB,CAAC,CAAC;oBACvD,CAAC;gBACF,CAAC,CAAC,CAAC;YACJ,CAAC;QACF,CAAC,EACD;YACC,OAAO,EAAP,gBAAO;SACP,CACD,CAAC;QACF,IAAI,CAAC,IAAI,EAAE,CAAC;IACb,CAAC;CACD;AA3ND,wBA2NC"}
@@ -1,14 +0,0 @@
1
- import Ajv from 'ajv';
2
-
3
- // Root path for the tasks API
4
- export const apiRoot = 'tasks';
5
-
6
- // Channel name for task insert notifications
7
- export const channel = 'task_insert';
8
-
9
- // Setting inlineRefs=false as without it we run into a
10
- // "Maximum call stack size exceeded" error apprarently caused
11
- // by String.prototype._uncountable_words being set in sbvr-parser?
12
- export const ajv = new Ajv({
13
- inlineRefs: false,
14
- });
@@ -1,158 +0,0 @@
1
- import type { Schema } from 'ajv';
2
- import * as cronParser from 'cron-parser';
3
- import { tasks as tasksEnv } from '../config-loader/env';
4
- import type * as Db from '../database-layer/db';
5
- import { BadRequestError } from '../sbvr-api/errors';
6
- import { addPureHook } from '../sbvr-api/hooks';
7
- import { PinejsClient } from '../sbvr-api/sbvr-utils';
8
- import type { sbvrUtils } from '../server-glue/module';
9
- import { ajv, apiRoot, channel } from './common';
10
- import type { TaskHandler } from './types';
11
- import { Worker } from './worker';
12
-
13
- export * from './types';
14
-
15
- // eslint-disable-next-line @typescript-eslint/no-var-requires
16
- const modelText: string = require('./model.sbvr');
17
-
18
- export const config = {
19
- models: [
20
- {
21
- modelName: apiRoot,
22
- apiRoot,
23
- modelText,
24
- customServerCode: exports,
25
- },
26
- ] as sbvrUtils.ExecutableModel[],
27
- };
28
-
29
- // Poll every second and also execute on new task insert
30
- // Only poll or execute on triggers if worker is not already at max concurrency
31
- async function createTrigger(tx: Db.Tx): Promise<void> {
32
- await tx.executeSql(`
33
- CREATE OR REPLACE FUNCTION notify_task_insert()
34
- RETURNS TRIGGER AS $$
35
- BEGIN
36
- PERFORM pg_notify('${channel}', NEW.id::text);
37
- RETURN NEW;
38
- END;
39
- $$ LANGUAGE plpgsql;
40
- `);
41
-
42
- // Only trigger if task is pending and not scheduled
43
- await tx.executeSql(`
44
- CREATE OR REPLACE TRIGGER task_insert_trigger
45
- AFTER INSERT ON task
46
- FOR EACH ROW WHEN (NEW.status = 'pending' AND NEW."is scheduled to execute on-time" IS NULL)
47
- EXECUTE FUNCTION notify_task_insert();
48
- `);
49
- }
50
-
51
- let worker: Worker | null = null;
52
- export async function setup(db: Db.Database, tx: Db.Tx): Promise<void> {
53
- // Async task functionality is only supported on Postgres
54
- if (db.engine !== 'postgres') {
55
- return;
56
- }
57
-
58
- // Create trigger function if it doesn't exist
59
- await createTrigger(tx);
60
-
61
- const client = new PinejsClient({
62
- apiPrefix: `/${apiRoot}/`,
63
- });
64
- worker = new Worker(client);
65
-
66
- // Add resource hooks
67
- addPureHook('POST', apiRoot, 'task', {
68
- POSTPARSE: async ({ req, request }) => {
69
- // Set the actor
70
- request.values.is_created_by__actor =
71
- req.user?.actor ?? req.apiKey?.actor;
72
- if (request.values.is_created_by__actor == null) {
73
- throw new BadRequestError(
74
- 'Creating tasks with missing actor on req is not allowed',
75
- );
76
- }
77
-
78
- // Set defaults
79
- request.values.status = 'pending';
80
- request.values.attempt_count = 0;
81
- request.values.priority ??= 1;
82
- request.values.attempt_limit ??= 1;
83
-
84
- // Set scheduled start time using cron expression if provided
85
- if (
86
- request.values.is_scheduled_with__cron_expression != null &&
87
- request.values.is_scheduled_to_execute_on__time == null
88
- ) {
89
- try {
90
- request.values.is_scheduled_to_execute_on__time = cronParser
91
- .parseExpression(request.values.is_scheduled_with__cron_expression)
92
- .next()
93
- .toDate()
94
- .toISOString();
95
- } catch (_) {
96
- throw new BadRequestError(
97
- `Invalid cron expression: ${request.values.is_scheduled_with__cron_expression}`,
98
- );
99
- }
100
- }
101
-
102
- // Assert that the provided start time is far enough in the future
103
- if (request.values.is_scheduled_to_execute_on__time != null) {
104
- const now = new Date(new Date().getTime() + tasksEnv.queueIntervalMS);
105
- const startTime = new Date(
106
- request.values.is_scheduled_to_execute_on__time,
107
- );
108
- if (startTime < now) {
109
- throw new BadRequestError(
110
- `Task scheduled start time must be greater than ${tasksEnv.queueIntervalMS} milliseconds in the future`,
111
- );
112
- }
113
- }
114
-
115
- // Assert that the requested handler exists
116
- const handlerName = request.values.is_executed_by__handler;
117
- if (handlerName == null) {
118
- throw new BadRequestError(`Must specify a task handler to execute`);
119
- }
120
- const handler = worker?.handlers[handlerName];
121
- if (handler == null) {
122
- throw new BadRequestError(
123
- `No task handler with name '${handlerName}' registered`,
124
- );
125
- }
126
-
127
- // Assert that the provided parameter set is valid
128
- if (handler.validate != null) {
129
- if (!handler.validate(request.values.is_executed_with__parameter_set)) {
130
- throw new BadRequestError(
131
- `Invalid parameter set: ${ajv.errorsText(handler.validate.errors)}`,
132
- );
133
- }
134
- }
135
- },
136
- });
137
- worker.start();
138
- }
139
-
140
- // Register a task handler
141
- export function addTaskHandler(
142
- name: string,
143
- fn: TaskHandler['fn'],
144
- schema?: Schema,
145
- ): void {
146
- if (worker == null) {
147
- return;
148
- }
149
-
150
- if (worker.handlers[name] != null) {
151
- throw new Error(`Task handler with name '${name}' already registered`);
152
- }
153
- worker.handlers[name] = {
154
- name,
155
- fn,
156
- validate: schema != null ? ajv.compile(schema) : undefined,
157
- };
158
- }