@hotmeshio/hotmesh 0.12.1 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -22
- package/build/modules/enums.d.ts +60 -5
- package/build/modules/enums.js +62 -7
- package/build/modules/errors.d.ts +15 -2
- package/build/modules/errors.js +17 -1
- package/build/modules/storage.d.ts +1 -0
- package/build/modules/storage.js +2 -1
- package/build/package.json +8 -2
- package/build/services/activities/activity/context.d.ts +22 -0
- package/build/services/activities/activity/context.js +76 -0
- package/build/services/activities/activity/index.d.ts +116 -0
- package/build/services/activities/activity/index.js +299 -0
- package/build/services/activities/activity/mapping.d.ts +12 -0
- package/build/services/activities/activity/mapping.js +63 -0
- package/build/services/activities/activity/process.d.ts +28 -0
- package/build/services/activities/activity/process.js +100 -0
- package/build/services/activities/activity/protocol.d.ts +39 -0
- package/build/services/activities/activity/protocol.js +151 -0
- package/build/services/activities/activity/state.d.ts +40 -0
- package/build/services/activities/activity/state.js +143 -0
- package/build/services/activities/activity/transition.d.ts +23 -0
- package/build/services/activities/activity/transition.js +71 -0
- package/build/services/activities/activity/verify.d.ts +22 -0
- package/build/services/activities/activity/verify.js +85 -0
- package/build/services/activities/await.d.ts +1 -4
- package/build/services/activities/await.js +2 -36
- package/build/services/activities/cycle.d.ts +1 -11
- package/build/services/activities/cycle.js +3 -46
- package/build/services/activities/hook.d.ts +2 -11
- package/build/services/activities/hook.js +30 -50
- package/build/services/activities/interrupt.d.ts +2 -4
- package/build/services/activities/interrupt.js +4 -38
- package/build/services/activities/signal.d.ts +1 -11
- package/build/services/activities/signal.js +3 -48
- package/build/services/activities/trigger.d.ts +1 -3
- package/build/services/activities/trigger.js +0 -3
- package/build/services/activities/worker.d.ts +3 -6
- package/build/services/activities/worker.js +4 -40
- package/build/services/connector/factory.d.ts +6 -0
- package/build/services/connector/factory.js +24 -0
- package/build/services/dba/index.d.ts +14 -4
- package/build/services/dba/index.js +57 -18
- package/build/services/durable/activity.d.ts +30 -0
- package/build/services/durable/activity.js +46 -0
- package/build/services/durable/client.d.ts +26 -31
- package/build/services/durable/client.js +26 -31
- package/build/services/durable/connection.d.ts +13 -7
- package/build/services/durable/connection.js +13 -7
- package/build/services/durable/exporter.d.ts +2 -2
- package/build/services/durable/exporter.js +27 -12
- package/build/services/durable/handle.d.ts +59 -41
- package/build/services/durable/handle.js +61 -41
- package/build/services/durable/index.d.ts +152 -283
- package/build/services/durable/index.js +161 -289
- package/build/services/durable/interceptor.d.ts +43 -33
- package/build/services/durable/interceptor.js +59 -39
- package/build/services/durable/schemas/factory.d.ts +2 -3
- package/build/services/durable/schemas/factory.js +180 -30
- package/build/services/durable/telemetry.d.ts +80 -0
- package/build/services/durable/telemetry.js +137 -0
- package/build/services/durable/worker.d.ts +100 -21
- package/build/services/durable/worker.js +314 -60
- package/build/services/durable/workflow/all.d.ts +1 -1
- package/build/services/durable/workflow/all.js +1 -1
- package/build/services/durable/workflow/cancellationScope.d.ts +104 -0
- package/build/services/durable/workflow/cancellationScope.js +139 -0
- package/build/services/durable/workflow/common.d.ts +5 -4
- package/build/services/durable/workflow/common.js +6 -1
- package/build/services/durable/workflow/{waitFor.d.ts → condition.d.ts} +9 -8
- package/build/services/durable/workflow/{waitFor.js → condition.js} +44 -11
- package/build/services/durable/workflow/continueAsNew.d.ts +65 -0
- package/build/services/durable/workflow/continueAsNew.js +92 -0
- package/build/services/durable/workflow/didRun.d.ts +2 -2
- package/build/services/durable/workflow/didRun.js +4 -4
- package/build/services/durable/workflow/enrich.d.ts +5 -0
- package/build/services/durable/workflow/enrich.js +5 -0
- package/build/services/durable/workflow/entityMethods.d.ts +7 -0
- package/build/services/durable/workflow/entityMethods.js +7 -0
- package/build/services/durable/workflow/execHook.js +3 -3
- package/build/services/durable/workflow/execHookBatch.js +2 -2
- package/build/services/durable/workflow/{execChild.d.ts → executeChild.d.ts} +4 -40
- package/build/services/durable/workflow/{execChild.js → executeChild.js} +36 -45
- package/build/services/durable/workflow/hook.d.ts +1 -1
- package/build/services/durable/workflow/hook.js +4 -3
- package/build/services/durable/workflow/index.d.ts +45 -50
- package/build/services/durable/workflow/index.js +46 -51
- package/build/services/durable/workflow/interruption.d.ts +7 -6
- package/build/services/durable/workflow/interruption.js +11 -7
- package/build/services/durable/workflow/patched.d.ts +72 -0
- package/build/services/durable/workflow/patched.js +110 -0
- package/build/services/durable/workflow/proxyActivities.d.ts +7 -7
- package/build/services/durable/workflow/proxyActivities.js +51 -15
- package/build/services/durable/workflow/searchMethods.d.ts +7 -0
- package/build/services/durable/workflow/searchMethods.js +7 -0
- package/build/services/durable/workflow/signal.d.ts +4 -4
- package/build/services/durable/workflow/signal.js +4 -4
- package/build/services/durable/workflow/{sleepFor.d.ts → sleep.d.ts} +7 -7
- package/build/services/durable/workflow/{sleepFor.js → sleep.js} +39 -10
- package/build/services/durable/workflow/terminate.d.ts +55 -0
- package/build/services/durable/workflow/{interrupt.js → terminate.js} +21 -21
- package/build/services/durable/workflow/trace.js +2 -2
- package/build/services/durable/workflow/uuid4.d.ts +14 -0
- package/build/services/durable/workflow/uuid4.js +39 -0
- package/build/services/durable/workflow/{context.d.ts → workflowInfo.d.ts} +5 -5
- package/build/services/durable/workflow/{context.js → workflowInfo.js} +7 -7
- package/build/services/engine/compiler.d.ts +19 -0
- package/build/services/engine/compiler.js +20 -0
- package/build/services/engine/completion.d.ts +46 -0
- package/build/services/engine/completion.js +145 -0
- package/build/services/engine/dispatch.d.ts +24 -0
- package/build/services/engine/dispatch.js +98 -0
- package/build/services/engine/index.d.ts +49 -81
- package/build/services/engine/index.js +175 -573
- package/build/services/engine/init.d.ts +42 -0
- package/build/services/engine/init.js +74 -0
- package/build/services/engine/pubsub.d.ts +50 -0
- package/build/services/engine/pubsub.js +118 -0
- package/build/services/engine/reporting.d.ts +20 -0
- package/build/services/engine/reporting.js +38 -0
- package/build/services/engine/schema.d.ts +23 -0
- package/build/services/engine/schema.js +62 -0
- package/build/services/engine/signal.d.ts +57 -0
- package/build/services/engine/signal.js +117 -0
- package/build/services/engine/state.d.ts +35 -0
- package/build/services/engine/state.js +61 -0
- package/build/services/engine/version.d.ts +31 -0
- package/build/services/engine/version.js +73 -0
- package/build/services/hotmesh/deployment.d.ts +21 -0
- package/build/services/hotmesh/deployment.js +25 -0
- package/build/services/hotmesh/index.d.ts +142 -533
- package/build/services/hotmesh/index.js +223 -674
- package/build/services/hotmesh/init.d.ts +42 -0
- package/build/services/hotmesh/init.js +93 -0
- package/build/services/hotmesh/jobs.d.ts +67 -0
- package/build/services/hotmesh/jobs.js +99 -0
- package/build/services/hotmesh/pubsub.d.ts +38 -0
- package/build/services/hotmesh/pubsub.js +54 -0
- package/build/services/hotmesh/quorum.d.ts +30 -0
- package/build/services/hotmesh/quorum.js +62 -0
- package/build/services/hotmesh/validation.d.ts +6 -0
- package/build/services/hotmesh/validation.js +28 -0
- package/build/services/quorum/index.js +1 -0
- package/build/services/router/consumption/index.d.ts +11 -5
- package/build/services/router/consumption/index.js +24 -17
- package/build/services/router/error-handling/index.d.ts +2 -2
- package/build/services/router/error-handling/index.js +14 -14
- package/build/services/router/index.d.ts +1 -1
- package/build/services/router/index.js +2 -2
- package/build/services/serializer/index.d.ts +22 -0
- package/build/services/serializer/index.js +39 -1
- package/build/services/store/index.d.ts +1 -0
- package/build/services/store/providers/postgres/exporter-sql.d.ts +2 -2
- package/build/services/store/providers/postgres/exporter-sql.js +4 -4
- package/build/services/store/providers/postgres/kvtables.js +7 -6
- package/build/services/store/providers/postgres/kvtypes/hash/basic.js +67 -52
- package/build/services/store/providers/postgres/kvtypes/hash/jsonb.js +87 -72
- package/build/services/store/providers/postgres/kvtypes/hash/udata.js +106 -79
- package/build/services/store/providers/postgres/kvtypes/hash/utils.d.ts +16 -0
- package/build/services/store/providers/postgres/kvtypes/hash/utils.js +29 -16
- package/build/services/store/providers/postgres/postgres.d.ts +1 -0
- package/build/services/store/providers/postgres/postgres.js +14 -4
- package/build/services/stream/factory.d.ts +3 -1
- package/build/services/stream/factory.js +2 -2
- package/build/services/stream/index.d.ts +1 -0
- package/build/services/stream/providers/nats/nats.d.ts +1 -0
- package/build/services/stream/providers/nats/nats.js +1 -0
- package/build/services/stream/providers/postgres/credentials.d.ts +56 -0
- package/build/services/stream/providers/postgres/credentials.js +129 -0
- package/build/services/stream/providers/postgres/kvtables.js +18 -0
- package/build/services/stream/providers/postgres/messages.js +7 -7
- package/build/services/stream/providers/postgres/notifications.js +16 -2
- package/build/services/stream/providers/postgres/postgres.d.ts +7 -0
- package/build/services/stream/providers/postgres/postgres.js +35 -4
- package/build/services/stream/providers/postgres/procedures.d.ts +21 -0
- package/build/services/stream/providers/postgres/procedures.js +213 -0
- package/build/services/stream/providers/postgres/secured.d.ts +34 -0
- package/build/services/stream/providers/postgres/secured.js +146 -0
- package/build/services/stream/providers/postgres/stats.d.ts +1 -0
- package/build/services/stream/providers/postgres/stats.js +1 -0
- package/build/services/stream/registry.d.ts +1 -1
- package/build/services/stream/registry.js +5 -2
- package/build/services/telemetry/index.d.ts +10 -1
- package/build/services/telemetry/index.js +40 -7
- package/build/services/worker/credentials.d.ts +51 -0
- package/build/services/worker/credentials.js +87 -0
- package/build/services/worker/index.d.ts +2 -2
- package/build/services/worker/index.js +7 -6
- package/build/types/codec.d.ts +84 -0
- package/build/types/codec.js +2 -0
- package/build/types/dba.d.ts +39 -3
- package/build/types/durable.d.ts +123 -25
- package/build/types/error.d.ts +10 -0
- package/build/types/exporter.d.ts +1 -1
- package/build/types/hotmesh.d.ts +67 -4
- package/build/types/index.d.ts +2 -1
- package/build/types/provider.d.ts +2 -2
- package/build/types/quorum.d.ts +35 -1
- package/build/types/stream.d.ts +12 -6
- package/package.json +8 -2
- package/build/services/activities/activity.d.ts +0 -192
- package/build/services/activities/activity.js +0 -786
- package/build/services/durable/workflow/interrupt.d.ts +0 -55
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.Worker = void 0;
|
|
4
|
-
const errors_1 = require("../../modules/errors");
|
|
5
4
|
const utils_1 = require("../../modules/utils");
|
|
6
5
|
const collator_1 = require("../collator");
|
|
7
6
|
const pipe_1 = require("../pipe");
|
|
@@ -74,7 +73,7 @@ const activity_1 = require("./activity");
|
|
|
74
73
|
* ## Retry Policy
|
|
75
74
|
*
|
|
76
75
|
* Retry behavior is configured at the **worker level** (not in YAML) via
|
|
77
|
-
* the `
|
|
76
|
+
* the `retry` option. Failed callbacks are retried with exponential
|
|
78
77
|
* backoff until `maximumAttempts` is exhausted. The `maximumInterval` caps
|
|
79
78
|
* the delay between retries.
|
|
80
79
|
*
|
|
@@ -85,7 +84,7 @@ const activity_1 = require("./activity");
|
|
|
85
84
|
* workers: [{
|
|
86
85
|
* topic: 'work.backoff',
|
|
87
86
|
* connection,
|
|
88
|
-
*
|
|
87
|
+
* retry: {
|
|
89
88
|
* maximumAttempts: 5, // retry up to 5 times
|
|
90
89
|
* backoffCoefficient: 2, // exponential: 2^0, 2^1, 2^2, ... seconds
|
|
91
90
|
* maximumInterval: '30s', // cap delay at 30 seconds
|
|
@@ -115,10 +114,6 @@ const activity_1 = require("./activity");
|
|
|
115
114
|
* @see {@link WorkerActivity} for the TypeScript interface
|
|
116
115
|
*/
|
|
117
116
|
class Worker extends activity_1.Activity {
|
|
118
|
-
constructor(config, data, metadata, hook, engine, context) {
|
|
119
|
-
super(config, data, metadata, hook, engine, context);
|
|
120
|
-
}
|
|
121
|
-
//******** INITIAL ENTRY POINT (A) ********//
|
|
122
117
|
async process() {
|
|
123
118
|
this.logger.debug('worker-process', {
|
|
124
119
|
jid: this.context.metadata.jid,
|
|
@@ -131,15 +126,12 @@ class Worker extends activity_1.Activity {
|
|
|
131
126
|
telemetry = new telemetry_1.TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
|
|
132
127
|
telemetry.startActivitySpan(this.leg);
|
|
133
128
|
this.mapInputData();
|
|
134
|
-
//save state and mark Leg1 complete
|
|
135
129
|
const transaction = this.store.transact();
|
|
136
|
-
//todo: await this.registerTimeout();
|
|
137
130
|
const messageId = await this.execActivity(transaction);
|
|
138
131
|
await collator_1.CollatorService.notarizeLeg1Completion(this, transaction);
|
|
139
132
|
await this.setState(transaction);
|
|
140
133
|
await this.setStatus(0, transaction);
|
|
141
134
|
const txResponse = (await transaction.exec());
|
|
142
|
-
//telemetry
|
|
143
135
|
telemetry.mapActivityAttributes();
|
|
144
136
|
const jobStatus = this.resolveStatus(txResponse);
|
|
145
137
|
telemetry.setActivityAttributes({
|
|
@@ -149,34 +141,8 @@ class Worker extends activity_1.Activity {
|
|
|
149
141
|
return this.context.metadata.aid;
|
|
150
142
|
}
|
|
151
143
|
catch (error) {
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
return;
|
|
155
|
-
}
|
|
156
|
-
else if (error instanceof errors_1.GenerationalError) {
|
|
157
|
-
this.logger.info('process-event-generational-job-error', { error });
|
|
158
|
-
return;
|
|
159
|
-
}
|
|
160
|
-
else if (error instanceof errors_1.GetStateError) {
|
|
161
|
-
this.logger.error('worker-get-state-error', { error });
|
|
162
|
-
return;
|
|
163
|
-
}
|
|
164
|
-
else if (error instanceof errors_1.CollationError) {
|
|
165
|
-
if (error.fault === 'duplicate') {
|
|
166
|
-
this.logger.info('worker-collation-overage', {
|
|
167
|
-
job_id: this.context.metadata.jid,
|
|
168
|
-
guid: this.context.metadata.guid,
|
|
169
|
-
});
|
|
170
|
-
return;
|
|
171
|
-
}
|
|
172
|
-
//unknown collation error
|
|
173
|
-
this.logger.error('worker-collation-error', { error });
|
|
174
|
-
}
|
|
175
|
-
else {
|
|
176
|
-
this.logger.error('worker-process-error', { error });
|
|
177
|
-
}
|
|
178
|
-
telemetry?.setActivityError(error.message);
|
|
179
|
-
throw error;
|
|
144
|
+
this.handleProcessError(error, telemetry, 'worker');
|
|
145
|
+
return;
|
|
180
146
|
}
|
|
181
147
|
finally {
|
|
182
148
|
telemetry?.endActivitySpan();
|
|
@@ -189,11 +155,9 @@ class Worker extends activity_1.Activity {
|
|
|
189
155
|
}
|
|
190
156
|
async execActivity(transaction) {
|
|
191
157
|
const topic = pipe_1.Pipe.resolve(this.config.subtype, this.context);
|
|
192
|
-
// Extract workflow name from job data (set by durable client) or derive from subscribes
|
|
193
158
|
const jobData = this.context.data;
|
|
194
159
|
let wfn = jobData?.workflowName || '';
|
|
195
160
|
if (!wfn && this.config.subscribes) {
|
|
196
|
-
// Fallback: derive from subscribes by removing topic prefix
|
|
197
161
|
wfn = this.config.subscribes.startsWith(`${topic}-`)
|
|
198
162
|
? this.config.subscribes.substring(topic.length + 1)
|
|
199
163
|
: this.config.subscribes;
|
|
@@ -11,6 +11,12 @@ export declare class ConnectorService {
|
|
|
11
11
|
static connectClient(ProviderConfig: ProviderConfig, taskQueue?: string): Promise<ProviderNativeClient>;
|
|
12
12
|
/**
|
|
13
13
|
* Initialize `store`, `stream`, and `subscription` clients for any provider.
|
|
14
|
+
*
|
|
15
|
+
* When the target has `workerCredentials`, the `user` and `password`
|
|
16
|
+
* in all connection options are overridden with the scoped Postgres
|
|
17
|
+
* role credentials, and `securedWorker` is set so downstream stream
|
|
18
|
+
* services use SECURITY DEFINER stored procedures.
|
|
19
|
+
*
|
|
14
20
|
* @private
|
|
15
21
|
*/
|
|
16
22
|
static initClients(target: HotMeshEngine | HotMeshWorker): Promise<void>;
|
|
@@ -22,6 +22,12 @@ class ConnectorService {
|
|
|
22
22
|
}
|
|
23
23
|
/**
|
|
24
24
|
* Initialize `store`, `stream`, and `subscription` clients for any provider.
|
|
25
|
+
*
|
|
26
|
+
* When the target has `workerCredentials`, the `user` and `password`
|
|
27
|
+
* in all connection options are overridden with the scoped Postgres
|
|
28
|
+
* role credentials, and `securedWorker` is set so downstream stream
|
|
29
|
+
* services use SECURITY DEFINER stored procedures.
|
|
30
|
+
*
|
|
25
31
|
* @private
|
|
26
32
|
*/
|
|
27
33
|
static async initClients(target) {
|
|
@@ -34,6 +40,24 @@ class ConnectorService {
|
|
|
34
40
|
sub: { ...connection },
|
|
35
41
|
};
|
|
36
42
|
}
|
|
43
|
+
// Apply scoped worker credentials to stream/sub only.
|
|
44
|
+
// The `store` connection retains full credentials because it is
|
|
45
|
+
// used by the engine for schema deployment, job-state reads, and
|
|
46
|
+
// other operations that require direct table access. Only the
|
|
47
|
+
// `stream` (dequeue/ack/respond) and `sub` (LISTEN/UNLISTEN)
|
|
48
|
+
// connections are scoped to the least-privilege worker role.
|
|
49
|
+
const workerCreds = target.workerCredentials;
|
|
50
|
+
if (workerCreds) {
|
|
51
|
+
for (const key of ['stream', 'sub']) {
|
|
52
|
+
if (connection[key]?.options) {
|
|
53
|
+
connection[key].options = {
|
|
54
|
+
...connection[key].options,
|
|
55
|
+
user: workerCreds.user,
|
|
56
|
+
password: workerCreds.password,
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
}
|
|
37
61
|
// Extract taskQueue from target for connection pooling
|
|
38
62
|
const taskQueue = target.taskQueue;
|
|
39
63
|
// Expanded form
|
|
@@ -29,7 +29,7 @@ import { PostgresClientType } from '../../types/postgres';
|
|
|
29
29
|
* from completed jobs while preserving:
|
|
30
30
|
* - `jdata` — workflow return data
|
|
31
31
|
* - `udata` — user-searchable data
|
|
32
|
-
* - `jmark` — timeline markers needed for
|
|
32
|
+
* - `jmark` — timeline markers needed for workflow execution export
|
|
33
33
|
*
|
|
34
34
|
* Set `keepHmark: true` to also preserve `hmark` (activity state markers).
|
|
35
35
|
*
|
|
@@ -63,11 +63,14 @@ import { PostgresClientType } from '../../types/postgres';
|
|
|
63
63
|
* jobs: false, streams: false, attributes: true,
|
|
64
64
|
* });
|
|
65
65
|
*
|
|
66
|
-
* // Cron 2 — Hourly:
|
|
66
|
+
* // Cron 2 — Hourly: aggressive engine stream cleanup, conservative worker streams
|
|
67
67
|
* await DBA.prune({
|
|
68
68
|
* appId: 'myapp', connection,
|
|
69
|
-
*
|
|
70
|
-
*
|
|
69
|
+
* jobs: false, attributes: false,
|
|
70
|
+
* engineStreams: true,
|
|
71
|
+
* engineStreamsExpire: '24 hours',
|
|
72
|
+
* workerStreams: true,
|
|
73
|
+
* workerStreamsExpire: '90 days', // preserve for export fidelity
|
|
71
74
|
* });
|
|
72
75
|
*
|
|
73
76
|
* // Cron 3 — Weekly: remove expired 'book' jobs older than 30 days
|
|
@@ -101,6 +104,13 @@ import { PostgresClientType } from '../../types/postgres';
|
|
|
101
104
|
*
|
|
102
105
|
* -- Prune everything older than 7 days and strip attributes
|
|
103
106
|
* SELECT * FROM myapp.prune('7 days', true, true, true);
|
|
107
|
+
*
|
|
108
|
+
* -- Independent stream retention: engine 24h, worker 90 days
|
|
109
|
+
* SELECT * FROM myapp.prune(
|
|
110
|
+
* '7 days', true, false, false, NULL, false, false,
|
|
111
|
+
* true, true, -- prune_engine_streams, prune_worker_streams
|
|
112
|
+
* INTERVAL '24 hours', INTERVAL '90 days' -- per-table retention
|
|
113
|
+
* );
|
|
104
114
|
* ```
|
|
105
115
|
*/
|
|
106
116
|
declare class DBA {
|
|
@@ -32,7 +32,7 @@ const postgres_1 = require("../connector/providers/postgres");
|
|
|
32
32
|
* from completed jobs while preserving:
|
|
33
33
|
* - `jdata` — workflow return data
|
|
34
34
|
* - `udata` — user-searchable data
|
|
35
|
-
* - `jmark` — timeline markers needed for
|
|
35
|
+
* - `jmark` — timeline markers needed for workflow execution export
|
|
36
36
|
*
|
|
37
37
|
* Set `keepHmark: true` to also preserve `hmark` (activity state markers).
|
|
38
38
|
*
|
|
@@ -66,11 +66,14 @@ const postgres_1 = require("../connector/providers/postgres");
|
|
|
66
66
|
* jobs: false, streams: false, attributes: true,
|
|
67
67
|
* });
|
|
68
68
|
*
|
|
69
|
-
* // Cron 2 — Hourly:
|
|
69
|
+
* // Cron 2 — Hourly: aggressive engine stream cleanup, conservative worker streams
|
|
70
70
|
* await DBA.prune({
|
|
71
71
|
* appId: 'myapp', connection,
|
|
72
|
-
*
|
|
73
|
-
*
|
|
72
|
+
* jobs: false, attributes: false,
|
|
73
|
+
* engineStreams: true,
|
|
74
|
+
* engineStreamsExpire: '24 hours',
|
|
75
|
+
* workerStreams: true,
|
|
76
|
+
* workerStreamsExpire: '90 days', // preserve for export fidelity
|
|
74
77
|
* });
|
|
75
78
|
*
|
|
76
79
|
* // Cron 3 — Weekly: remove expired 'book' jobs older than 30 days
|
|
@@ -104,6 +107,13 @@ const postgres_1 = require("../connector/providers/postgres");
|
|
|
104
107
|
*
|
|
105
108
|
* -- Prune everything older than 7 days and strip attributes
|
|
106
109
|
* SELECT * FROM myapp.prune('7 days', true, true, true);
|
|
110
|
+
*
|
|
111
|
+
* -- Independent stream retention: engine 24h, worker 90 days
|
|
112
|
+
* SELECT * FROM myapp.prune(
|
|
113
|
+
* '7 days', true, false, false, NULL, false, false,
|
|
114
|
+
* true, true, -- prune_engine_streams, prune_worker_streams
|
|
115
|
+
* INTERVAL '24 hours', INTERVAL '90 days' -- per-table retention
|
|
116
|
+
* );
|
|
107
117
|
* ```
|
|
108
118
|
*/
|
|
109
119
|
class DBA {
|
|
@@ -172,11 +182,17 @@ class DBA {
|
|
|
172
182
|
strip_attributes BOOLEAN DEFAULT FALSE,
|
|
173
183
|
entity_list TEXT[] DEFAULT NULL,
|
|
174
184
|
prune_transient BOOLEAN DEFAULT FALSE,
|
|
175
|
-
keep_hmark BOOLEAN DEFAULT FALSE
|
|
185
|
+
keep_hmark BOOLEAN DEFAULT FALSE,
|
|
186
|
+
prune_engine_streams BOOLEAN DEFAULT NULL,
|
|
187
|
+
prune_worker_streams BOOLEAN DEFAULT NULL,
|
|
188
|
+
engine_streams_retention INTERVAL DEFAULT NULL,
|
|
189
|
+
worker_streams_retention INTERVAL DEFAULT NULL
|
|
176
190
|
)
|
|
177
191
|
RETURNS TABLE(
|
|
178
192
|
deleted_jobs BIGINT,
|
|
179
193
|
deleted_streams BIGINT,
|
|
194
|
+
deleted_engine_streams BIGINT,
|
|
195
|
+
deleted_worker_streams BIGINT,
|
|
180
196
|
stripped_attributes BIGINT,
|
|
181
197
|
deleted_transient BIGINT,
|
|
182
198
|
marked_pruned BIGINT
|
|
@@ -185,12 +201,22 @@ class DBA {
|
|
|
185
201
|
AS $$
|
|
186
202
|
DECLARE
|
|
187
203
|
v_deleted_jobs BIGINT := 0;
|
|
188
|
-
|
|
204
|
+
v_deleted_engine_streams BIGINT := 0;
|
|
205
|
+
v_deleted_worker_streams BIGINT := 0;
|
|
189
206
|
v_stripped_attributes BIGINT := 0;
|
|
190
207
|
v_deleted_transient BIGINT := 0;
|
|
191
208
|
v_marked_pruned BIGINT := 0;
|
|
192
|
-
|
|
209
|
+
v_do_engine BOOLEAN;
|
|
210
|
+
v_do_worker BOOLEAN;
|
|
211
|
+
v_engine_retention INTERVAL;
|
|
212
|
+
v_worker_retention INTERVAL;
|
|
193
213
|
BEGIN
|
|
214
|
+
-- Resolve per-table overrides (fall back to legacy prune_streams + retention)
|
|
215
|
+
v_do_engine := COALESCE(prune_engine_streams, prune_streams);
|
|
216
|
+
v_do_worker := COALESCE(prune_worker_streams, prune_streams);
|
|
217
|
+
v_engine_retention := COALESCE(engine_streams_retention, retention);
|
|
218
|
+
v_worker_retention := COALESCE(worker_streams_retention, retention);
|
|
219
|
+
|
|
194
220
|
-- 1. Hard-delete expired jobs older than the retention window.
|
|
195
221
|
-- FK CASCADE on jobs_attributes handles attribute cleanup.
|
|
196
222
|
-- Optionally scoped to an entity allowlist.
|
|
@@ -211,22 +237,23 @@ class DBA {
|
|
|
211
237
|
GET DIAGNOSTICS v_deleted_transient = ROW_COUNT;
|
|
212
238
|
END IF;
|
|
213
239
|
|
|
214
|
-
-- 3. Hard-delete expired stream messages
|
|
215
|
-
|
|
216
|
-
IF prune_streams THEN
|
|
240
|
+
-- 3. Hard-delete expired engine stream messages.
|
|
241
|
+
IF v_do_engine THEN
|
|
217
242
|
DELETE FROM ${schema}.engine_streams
|
|
218
243
|
WHERE expired_at IS NOT NULL
|
|
219
|
-
AND expired_at < NOW() -
|
|
220
|
-
GET DIAGNOSTICS
|
|
244
|
+
AND expired_at < NOW() - v_engine_retention;
|
|
245
|
+
GET DIAGNOSTICS v_deleted_engine_streams = ROW_COUNT;
|
|
246
|
+
END IF;
|
|
221
247
|
|
|
248
|
+
-- 4. Hard-delete expired worker stream messages.
|
|
249
|
+
IF v_do_worker THEN
|
|
222
250
|
DELETE FROM ${schema}.worker_streams
|
|
223
251
|
WHERE expired_at IS NOT NULL
|
|
224
|
-
AND expired_at < NOW() -
|
|
225
|
-
GET DIAGNOSTICS
|
|
226
|
-
v_deleted_streams := v_deleted_streams + v_temp_count;
|
|
252
|
+
AND expired_at < NOW() - v_worker_retention;
|
|
253
|
+
GET DIAGNOSTICS v_deleted_worker_streams = ROW_COUNT;
|
|
227
254
|
END IF;
|
|
228
255
|
|
|
229
|
-
--
|
|
256
|
+
-- 5. Strip execution artifacts from completed, live, un-pruned jobs.
|
|
230
257
|
-- Always preserves: jdata, udata, jmark (timeline/export history).
|
|
231
258
|
-- Optionally preserves: hmark (when keep_hmark is true).
|
|
232
259
|
IF strip_attributes THEN
|
|
@@ -261,7 +288,9 @@ class DBA {
|
|
|
261
288
|
END IF;
|
|
262
289
|
|
|
263
290
|
deleted_jobs := v_deleted_jobs;
|
|
264
|
-
deleted_streams :=
|
|
291
|
+
deleted_streams := v_deleted_engine_streams + v_deleted_worker_streams;
|
|
292
|
+
deleted_engine_streams := v_deleted_engine_streams;
|
|
293
|
+
deleted_worker_streams := v_deleted_worker_streams;
|
|
265
294
|
stripped_attributes := v_stripped_attributes;
|
|
266
295
|
deleted_transient := v_deleted_transient;
|
|
267
296
|
marked_pruned := v_marked_pruned;
|
|
@@ -357,14 +386,24 @@ class DBA {
|
|
|
357
386
|
const entities = options.entities ?? null;
|
|
358
387
|
const pruneTransient = options.pruneTransient ?? false;
|
|
359
388
|
const keepHmark = options.keepHmark ?? false;
|
|
389
|
+
// Per-table overrides (NULL = fall back to streams/expire in SQL)
|
|
390
|
+
const engineStreams = options.engineStreams ?? null;
|
|
391
|
+
const workerStreams = options.workerStreams ?? null;
|
|
392
|
+
const engineStreamsExpire = options.engineStreamsExpire ?? null;
|
|
393
|
+
const workerStreamsExpire = options.workerStreamsExpire ?? null;
|
|
360
394
|
await DBA.deploy(options.connection, options.appId);
|
|
361
395
|
const { client, release } = await DBA.getClient(options.connection);
|
|
362
396
|
try {
|
|
363
|
-
const result = await client.query(`SELECT * FROM ${schema}.prune($1::interval, $2::boolean, $3::boolean, $4::boolean, $5::text[], $6::boolean, $7::boolean
|
|
397
|
+
const result = await client.query(`SELECT * FROM ${schema}.prune($1::interval, $2::boolean, $3::boolean, $4::boolean, $5::text[], $6::boolean, $7::boolean, $8::boolean, $9::boolean, $10::interval, $11::interval)`, [
|
|
398
|
+
expire, jobs, streams, attributes, entities, pruneTransient, keepHmark,
|
|
399
|
+
engineStreams, workerStreams, engineStreamsExpire, workerStreamsExpire,
|
|
400
|
+
]);
|
|
364
401
|
const row = result.rows[0];
|
|
365
402
|
return {
|
|
366
403
|
jobs: Number(row.deleted_jobs),
|
|
367
404
|
streams: Number(row.deleted_streams),
|
|
405
|
+
engineStreams: Number(row.deleted_engine_streams),
|
|
406
|
+
workerStreams: Number(row.deleted_worker_streams),
|
|
368
407
|
attributes: Number(row.stripped_attributes),
|
|
369
408
|
transient: Number(row.deleted_transient),
|
|
370
409
|
marked: Number(row.marked_pruned),
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { DurableActivityContext } from '../../types/durable';
|
|
2
|
+
/**
|
|
3
|
+
* The activity-internal API surface, exposed as `Durable.activity`. Methods
|
|
4
|
+
* on this class are designed to be called **inside** an activity function —
|
|
5
|
+
* they read from the activity's `AsyncLocalStorage` context that is populated
|
|
6
|
+
* by the activity worker before invoking the user's function.
|
|
7
|
+
*
|
|
8
|
+
* ## Usage
|
|
9
|
+
*
|
|
10
|
+
* ```typescript
|
|
11
|
+
* import { Durable } from '@hotmeshio/hotmesh';
|
|
12
|
+
*
|
|
13
|
+
* export async function processData(data: string): Promise<string> {
|
|
14
|
+
* const ctx = Durable.activity.getContext();
|
|
15
|
+
* console.log(`Activity ${ctx.activityName} for workflow ${ctx.workflowId}`);
|
|
16
|
+
* console.log(`Metadata:`, ctx.headers);
|
|
17
|
+
* return `Processed: ${data}`;
|
|
18
|
+
* }
|
|
19
|
+
* ```
|
|
20
|
+
*/
|
|
21
|
+
export declare class ActivityService {
|
|
22
|
+
/**
|
|
23
|
+
* Returns the current activity's execution context. Only available
|
|
24
|
+
* inside an activity function invoked by the durable activity worker.
|
|
25
|
+
*
|
|
26
|
+
* @returns The activity context including name, args, metadata, and parent workflow info.
|
|
27
|
+
* @throws If called outside of an activity execution context.
|
|
28
|
+
*/
|
|
29
|
+
static getContext(): DurableActivityContext;
|
|
30
|
+
}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.ActivityService = void 0;
|
|
4
|
+
const storage_1 = require("../../modules/storage");
|
|
5
|
+
/**
|
|
6
|
+
* The activity-internal API surface, exposed as `Durable.activity`. Methods
|
|
7
|
+
* on this class are designed to be called **inside** an activity function —
|
|
8
|
+
* they read from the activity's `AsyncLocalStorage` context that is populated
|
|
9
|
+
* by the activity worker before invoking the user's function.
|
|
10
|
+
*
|
|
11
|
+
* ## Usage
|
|
12
|
+
*
|
|
13
|
+
* ```typescript
|
|
14
|
+
* import { Durable } from '@hotmeshio/hotmesh';
|
|
15
|
+
*
|
|
16
|
+
* export async function processData(data: string): Promise<string> {
|
|
17
|
+
* const ctx = Durable.activity.getContext();
|
|
18
|
+
* console.log(`Activity ${ctx.activityName} for workflow ${ctx.workflowId}`);
|
|
19
|
+
* console.log(`Metadata:`, ctx.headers);
|
|
20
|
+
* return `Processed: ${data}`;
|
|
21
|
+
* }
|
|
22
|
+
* ```
|
|
23
|
+
*/
|
|
24
|
+
class ActivityService {
|
|
25
|
+
/**
|
|
26
|
+
* Returns the current activity's execution context. Only available
|
|
27
|
+
* inside an activity function invoked by the durable activity worker.
|
|
28
|
+
*
|
|
29
|
+
* @returns The activity context including name, args, metadata, and parent workflow info.
|
|
30
|
+
* @throws If called outside of an activity execution context.
|
|
31
|
+
*/
|
|
32
|
+
static getContext() {
|
|
33
|
+
const store = storage_1.activityAsyncLocalStorage.getStore();
|
|
34
|
+
if (!store) {
|
|
35
|
+
throw new Error('Durable.activity.getContext() called outside of an activity execution context');
|
|
36
|
+
}
|
|
37
|
+
return {
|
|
38
|
+
activityName: store.get('activityName'),
|
|
39
|
+
arguments: store.get('arguments'),
|
|
40
|
+
headers: store.get('headers') ?? {},
|
|
41
|
+
workflowId: store.get('workflowId'),
|
|
42
|
+
workflowTopic: store.get('workflowTopic'),
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
exports.ActivityService = ActivityService;
|
|
@@ -1,39 +1,34 @@
|
|
|
1
1
|
import { HotMesh } from '../hotmesh';
|
|
2
2
|
import { ClientConfig, ClientWorkflow, Connection, WorkflowOptions } from '../../types/durable';
|
|
3
3
|
/**
|
|
4
|
-
*
|
|
5
|
-
* equivalent to the Temporal `Client` service.
|
|
6
|
-
* Start a new workflow execution by calling
|
|
7
|
-
* `workflow.start`. Note the direct connection to
|
|
8
|
-
* Postgres.
|
|
4
|
+
* Workflow client. Starts workflows, sends signals, and reads results.
|
|
9
5
|
*
|
|
10
|
-
* NATS can be used as the message broker if advanced
|
|
11
|
-
* messaging is required (i.e, patterned subscriptions).
|
|
12
6
|
* @example
|
|
13
|
-
|
|
14
7
|
* ```typescript
|
|
15
|
-
*
|
|
16
|
-
* import { Client, HotMesh } from '@hotmeshio/hotmesh';
|
|
8
|
+
* import { Durable } from '@hotmeshio/hotmesh';
|
|
17
9
|
* import { Client as Postgres } from 'pg';
|
|
18
|
-
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
|
|
27
|
-
*
|
|
28
|
-
*
|
|
29
|
-
*
|
|
30
|
-
*
|
|
31
|
-
*
|
|
32
|
-
*
|
|
33
|
-
|
|
34
|
-
*
|
|
35
|
-
*
|
|
36
|
-
* }
|
|
10
|
+
*
|
|
11
|
+
* const client = new Durable.Client({
|
|
12
|
+
* connection: {
|
|
13
|
+
* class: Postgres,
|
|
14
|
+
* options: { connectionString: 'postgresql://usr:pwd@localhost:5432/db' },
|
|
15
|
+
* },
|
|
16
|
+
* });
|
|
17
|
+
*
|
|
18
|
+
* // Start a workflow and await its result
|
|
19
|
+
* const handle = await client.workflow.start({
|
|
20
|
+
* args: ['order-123'],
|
|
21
|
+
* taskQueue: 'orders',
|
|
22
|
+
* workflowName: 'orderWorkflow',
|
|
23
|
+
* workflowId: Durable.guid(),
|
|
24
|
+
* });
|
|
25
|
+
* const result = await handle.result();
|
|
26
|
+
*
|
|
27
|
+
* // Send a signal to a running workflow
|
|
28
|
+
* await handle.signal('approval', { approved: true });
|
|
29
|
+
*
|
|
30
|
+
* // Cancel a running workflow
|
|
31
|
+
* await handle.cancel();
|
|
37
32
|
* ```
|
|
38
33
|
*/
|
|
39
34
|
export declare class ClientService {
|
|
@@ -81,8 +76,8 @@ export declare class ClientService {
|
|
|
81
76
|
*/
|
|
82
77
|
search: (hotMeshClient: HotMesh, index: string, query: string[]) => Promise<string[]>;
|
|
83
78
|
/**
|
|
84
|
-
* The Durable `Client` service
|
|
85
|
-
*
|
|
79
|
+
* The Durable `Client` service provides methods for
|
|
80
|
+
* starting, signaling, and querying workflows.
|
|
86
81
|
* Starting a workflow is the primary use case and
|
|
87
82
|
* is accessed by calling workflow.start().
|
|
88
83
|
*/
|
|
@@ -11,39 +11,34 @@ const search_1 = require("./search");
|
|
|
11
11
|
const handle_1 = require("./handle");
|
|
12
12
|
const factory_1 = require("./schemas/factory");
|
|
13
13
|
/**
|
|
14
|
-
*
|
|
15
|
-
* equivalent to the Temporal `Client` service.
|
|
16
|
-
* Start a new workflow execution by calling
|
|
17
|
-
* `workflow.start`. Note the direct connection to
|
|
18
|
-
* Postgres.
|
|
14
|
+
* Workflow client. Starts workflows, sends signals, and reads results.
|
|
19
15
|
*
|
|
20
|
-
* NATS can be used as the message broker if advanced
|
|
21
|
-
* messaging is required (i.e, patterned subscriptions).
|
|
22
16
|
* @example
|
|
23
|
-
|
|
24
17
|
* ```typescript
|
|
25
|
-
*
|
|
26
|
-
* import { Client, HotMesh } from '@hotmeshio/hotmesh';
|
|
18
|
+
* import { Durable } from '@hotmeshio/hotmesh';
|
|
27
19
|
* import { Client as Postgres } from 'pg';
|
|
28
|
-
|
|
29
|
-
*
|
|
30
|
-
*
|
|
31
|
-
*
|
|
32
|
-
*
|
|
33
|
-
*
|
|
34
|
-
*
|
|
35
|
-
*
|
|
36
|
-
|
|
37
|
-
*
|
|
38
|
-
*
|
|
39
|
-
*
|
|
40
|
-
*
|
|
41
|
-
*
|
|
42
|
-
*
|
|
43
|
-
|
|
44
|
-
*
|
|
45
|
-
*
|
|
46
|
-
* }
|
|
20
|
+
*
|
|
21
|
+
* const client = new Durable.Client({
|
|
22
|
+
* connection: {
|
|
23
|
+
* class: Postgres,
|
|
24
|
+
* options: { connectionString: 'postgresql://usr:pwd@localhost:5432/db' },
|
|
25
|
+
* },
|
|
26
|
+
* });
|
|
27
|
+
*
|
|
28
|
+
* // Start a workflow and await its result
|
|
29
|
+
* const handle = await client.workflow.start({
|
|
30
|
+
* args: ['order-123'],
|
|
31
|
+
* taskQueue: 'orders',
|
|
32
|
+
* workflowName: 'orderWorkflow',
|
|
33
|
+
* workflowId: Durable.guid(),
|
|
34
|
+
* });
|
|
35
|
+
* const result = await handle.result();
|
|
36
|
+
*
|
|
37
|
+
* // Send a signal to a running workflow
|
|
38
|
+
* await handle.signal('approval', { approved: true });
|
|
39
|
+
*
|
|
40
|
+
* // Cancel a running workflow
|
|
41
|
+
* await handle.cancel();
|
|
47
42
|
* ```
|
|
48
43
|
*/
|
|
49
44
|
class ClientService {
|
|
@@ -114,8 +109,8 @@ class ClientService {
|
|
|
114
109
|
return await searchClient.sendIndexedQuery(index, query);
|
|
115
110
|
};
|
|
116
111
|
/**
|
|
117
|
-
* The Durable `Client` service
|
|
118
|
-
*
|
|
112
|
+
* The Durable `Client` service provides methods for
|
|
113
|
+
* starting, signaling, and querying workflows.
|
|
119
114
|
* Starting a workflow is the primary use case and
|
|
120
115
|
* is accessed by calling workflow.start().
|
|
121
116
|
*/
|
|
@@ -1,13 +1,12 @@
|
|
|
1
1
|
import { Connection } from '../../types/durable';
|
|
2
2
|
import { ProviderConfig, ProvidersConfig } from '../../types/provider';
|
|
3
3
|
/**
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
* is started by the Durable Client module (`(new Durable.Client())).start()`).
|
|
4
|
+
* Declares connection configuration (driver class + options) without
|
|
5
|
+
* opening a connection. The actual Postgres connection is established
|
|
6
|
+
* lazily when a Worker or Client is started.
|
|
8
7
|
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
8
|
+
* Supports both single-connection and split-connection layouts (separate
|
|
9
|
+
* `store`, `stream`, and `sub` connections for advanced deployments).
|
|
11
10
|
*/
|
|
12
11
|
export declare class ConnectionService {
|
|
13
12
|
/**
|
|
@@ -15,7 +14,14 @@ export declare class ConnectionService {
|
|
|
15
14
|
*/
|
|
16
15
|
constructor();
|
|
17
16
|
/**
|
|
18
|
-
*
|
|
17
|
+
* Create a connection configuration object.
|
|
18
|
+
*
|
|
19
|
+
* @param config - Single `{ class, options }` or split `{ store, stream, sub }`.
|
|
20
|
+
* @returns The normalized connection configuration.
|
|
19
21
|
*/
|
|
20
22
|
static connect(config: ProviderConfig | ProvidersConfig): Promise<Connection>;
|
|
23
|
+
/**
|
|
24
|
+
* Alias for {@link connect}.
|
|
25
|
+
*/
|
|
26
|
+
static create: typeof ConnectionService.connect;
|
|
21
27
|
}
|
|
@@ -2,13 +2,12 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.ConnectionService = void 0;
|
|
4
4
|
/**
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
* is started by the Durable Client module (`(new Durable.Client())).start()`).
|
|
5
|
+
* Declares connection configuration (driver class + options) without
|
|
6
|
+
* opening a connection. The actual Postgres connection is established
|
|
7
|
+
* lazily when a Worker or Client is started.
|
|
9
8
|
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
9
|
+
* Supports both single-connection and split-connection layouts (separate
|
|
10
|
+
* `store`, `stream`, and `sub` connections for advanced deployments).
|
|
12
11
|
*/
|
|
13
12
|
class ConnectionService {
|
|
14
13
|
/**
|
|
@@ -16,7 +15,10 @@ class ConnectionService {
|
|
|
16
15
|
*/
|
|
17
16
|
constructor() { }
|
|
18
17
|
/**
|
|
19
|
-
*
|
|
18
|
+
* Create a connection configuration object.
|
|
19
|
+
*
|
|
20
|
+
* @param config - Single `{ class, options }` or split `{ store, stream, sub }`.
|
|
21
|
+
* @returns The normalized connection configuration.
|
|
20
22
|
*/
|
|
21
23
|
static async connect(config) {
|
|
22
24
|
return 'store' in config
|
|
@@ -28,4 +30,8 @@ class ConnectionService {
|
|
|
28
30
|
};
|
|
29
31
|
}
|
|
30
32
|
}
|
|
33
|
+
/**
|
|
34
|
+
* Alias for {@link connect}.
|
|
35
|
+
*/
|
|
36
|
+
ConnectionService.create = ConnectionService.connect;
|
|
31
37
|
exports.ConnectionService = ConnectionService;
|