@hotmeshio/hotmesh 0.10.1 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/build/modules/errors.d.ts +2 -0
- package/build/modules/errors.js +2 -0
- package/build/modules/key.js +3 -2
- package/build/package.json +3 -1
- package/build/services/activities/worker.js +10 -0
- package/build/services/dba/index.d.ts +54 -19
- package/build/services/dba/index.js +129 -31
- package/build/services/durable/client.js +6 -1
- package/build/services/durable/exporter.d.ts +75 -3
- package/build/services/durable/exporter.js +768 -2
- package/build/services/durable/handle.d.ts +12 -1
- package/build/services/durable/handle.js +13 -0
- package/build/services/durable/schemas/factory.d.ts +1 -1
- package/build/services/durable/schemas/factory.js +27 -4
- package/build/services/durable/worker.d.ts +2 -2
- package/build/services/durable/worker.js +15 -9
- package/build/services/durable/workflow/context.js +2 -0
- package/build/services/durable/workflow/execChild.js +5 -2
- package/build/services/durable/workflow/hook.js +6 -0
- package/build/services/durable/workflow/proxyActivities.js +3 -4
- package/build/services/engine/index.js +5 -3
- package/build/services/store/index.d.ts +40 -0
- package/build/services/store/providers/postgres/exporter-sql.d.ts +23 -0
- package/build/services/store/providers/postgres/exporter-sql.js +52 -0
- package/build/services/store/providers/postgres/kvtables.js +12 -1
- package/build/services/store/providers/postgres/postgres.d.ts +34 -0
- package/build/services/store/providers/postgres/postgres.js +99 -0
- package/build/services/stream/providers/postgres/kvtables.d.ts +1 -1
- package/build/services/stream/providers/postgres/kvtables.js +175 -82
- package/build/services/stream/providers/postgres/lifecycle.d.ts +4 -3
- package/build/services/stream/providers/postgres/lifecycle.js +6 -5
- package/build/services/stream/providers/postgres/messages.d.ts +9 -6
- package/build/services/stream/providers/postgres/messages.js +121 -75
- package/build/services/stream/providers/postgres/notifications.d.ts +5 -2
- package/build/services/stream/providers/postgres/notifications.js +39 -35
- package/build/services/stream/providers/postgres/postgres.d.ts +20 -118
- package/build/services/stream/providers/postgres/postgres.js +83 -140
- package/build/services/stream/registry.d.ts +62 -0
- package/build/services/stream/registry.js +198 -0
- package/build/services/worker/index.js +20 -6
- package/build/types/dba.d.ts +31 -5
- package/build/types/durable.d.ts +6 -1
- package/build/types/error.d.ts +2 -0
- package/build/types/exporter.d.ts +166 -0
- package/build/types/hotmesh.d.ts +7 -1
- package/build/types/index.d.ts +1 -1
- package/build/types/stream.d.ts +2 -0
- package/package.json +3 -1
package/README.md
CHANGED
|
@@ -11,7 +11,7 @@ npm install @hotmeshio/hotmesh
|
|
|
11
11
|
## Use HotMesh for
|
|
12
12
|
|
|
13
13
|
- **Durable pipelines** — Orchestrate long-running, multi-step pipelines transactionally.
|
|
14
|
-
- **Temporal
|
|
14
|
+
- **Familiar Temporal syntax** — The `Durable` module provides a Temporal-compatible API (`Client`, `Worker`, `proxyActivities`, `sleepFor`, `startChild`, signals) that runs directly on Postgres. No app server required.
|
|
15
15
|
- **Distributed state machines** — Build stateful applications where every component can [fail and recover](https://github.com/hotmeshio/sdk-typescript/blob/main/services/collator/README.md).
|
|
16
16
|
- **AI and training pipelines** — Multi-step AI workloads where each stage is expensive and must not be repeated on failure. A crashed pipeline resumes from the last committed step, not from the beginning.
|
|
17
17
|
|
package/build/modules/errors.js
CHANGED
|
@@ -54,6 +54,8 @@ class DurableChildError extends Error {
|
|
|
54
54
|
this.arguments = params.arguments;
|
|
55
55
|
this.workflowId = params.workflowId;
|
|
56
56
|
this.workflowTopic = params.workflowTopic;
|
|
57
|
+
this.taskQueue = params.taskQueue;
|
|
58
|
+
this.workflowName = params.workflowName;
|
|
57
59
|
this.parentWorkflowId = params.parentWorkflowId;
|
|
58
60
|
this.expire = params.expire;
|
|
59
61
|
this.persistent = params.persistent;
|
package/build/modules/key.js
CHANGED
|
@@ -143,7 +143,7 @@ class KeyService {
|
|
|
143
143
|
case 'v':
|
|
144
144
|
return 'versions';
|
|
145
145
|
case 'x':
|
|
146
|
-
return id === '' ? '
|
|
146
|
+
return id === '' ? 'engine_streams' : 'worker_streams';
|
|
147
147
|
case 'hooks':
|
|
148
148
|
return 'signal_patterns';
|
|
149
149
|
case 'signals':
|
|
@@ -174,7 +174,8 @@ class KeyService {
|
|
|
174
174
|
return 's';
|
|
175
175
|
case 'versions':
|
|
176
176
|
return 'v';
|
|
177
|
-
case '
|
|
177
|
+
case 'engine_streams':
|
|
178
|
+
case 'worker_streams':
|
|
178
179
|
return 'x';
|
|
179
180
|
case 'signal_patterns':
|
|
180
181
|
return 'hooks';
|
package/build/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hotmeshio/hotmesh",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.11.0",
|
|
4
4
|
"description": "Permanent-Memory Workflows & AI Agents",
|
|
5
5
|
"main": "./build/index.js",
|
|
6
6
|
"types": "./build/index.d.ts",
|
|
@@ -46,6 +46,8 @@
|
|
|
46
46
|
"test:durable:sleep": "vitest run tests/durable/sleep/postgres.test.ts",
|
|
47
47
|
"test:durable:signal": "vitest run tests/durable/signal/postgres.test.ts",
|
|
48
48
|
"test:durable:unknown": "vitest run tests/durable/unknown/postgres.test.ts",
|
|
49
|
+
"test:durable:exporter": "HMSH_LOGLEVEL=info vitest run tests/durable/exporter",
|
|
50
|
+
"test:durable:exporter:debug": "EXPORT_DEBUG=1 HMSH_LOGLEVEL=error vitest run tests/durable/basic/postgres.test.ts",
|
|
49
51
|
"test:dba": "vitest run tests/dba",
|
|
50
52
|
"test:cycle": "vitest run tests/functional/cycle",
|
|
51
53
|
"test:functional": "vitest run tests/functional",
|
|
@@ -189,6 +189,15 @@ class Worker extends activity_1.Activity {
|
|
|
189
189
|
}
|
|
190
190
|
async execActivity(transaction) {
|
|
191
191
|
const topic = pipe_1.Pipe.resolve(this.config.subtype, this.context);
|
|
192
|
+
// Extract workflow name from job data (set by durable client) or derive from subscribes
|
|
193
|
+
const jobData = this.context.data;
|
|
194
|
+
let wfn = jobData?.workflowName || '';
|
|
195
|
+
if (!wfn && this.config.subscribes) {
|
|
196
|
+
// Fallback: derive from subscribes by removing topic prefix
|
|
197
|
+
wfn = this.config.subscribes.startsWith(`${topic}-`)
|
|
198
|
+
? this.config.subscribes.substring(topic.length + 1)
|
|
199
|
+
: this.config.subscribes;
|
|
200
|
+
}
|
|
192
201
|
const streamData = {
|
|
193
202
|
metadata: {
|
|
194
203
|
guid: (0, utils_1.guid)(),
|
|
@@ -197,6 +206,7 @@ class Worker extends activity_1.Activity {
|
|
|
197
206
|
dad: this.metadata.dad,
|
|
198
207
|
aid: this.metadata.aid,
|
|
199
208
|
topic,
|
|
209
|
+
wfn,
|
|
200
210
|
spn: this.context['$self'].output.metadata.l1s,
|
|
201
211
|
trc: this.context.metadata.trc,
|
|
202
212
|
},
|
|
@@ -11,8 +11,9 @@ import { PostgresClientType } from '../../types/postgres';
|
|
|
11
11
|
* | Table | What accumulates |
|
|
12
12
|
* |---|---|
|
|
13
13
|
* | `{appId}.jobs` | Completed/expired jobs with `expired_at` set |
|
|
14
|
-
* | `{appId}.jobs_attributes` | Execution artifacts (`adata`, `hmark`, `
|
|
15
|
-
* | `{appId}.
|
|
14
|
+
* | `{appId}.jobs_attributes` | Execution artifacts (`adata`, `hmark`, `status`, `other`) that are only needed during workflow execution |
|
|
15
|
+
* | `{appId}.engine_streams` | Processed engine stream messages with `expired_at` set |
|
|
16
|
+
* | `{appId}.worker_streams` | Processed worker stream messages with `expired_at` set |
|
|
16
17
|
*
|
|
17
18
|
* The `DBA` service addresses this with two methods:
|
|
18
19
|
*
|
|
@@ -22,10 +23,29 @@ import { PostgresClientType } from '../../types/postgres';
|
|
|
22
23
|
* - {@link DBA.deploy | deploy()} — Pre-deploys the Postgres function
|
|
23
24
|
* (e.g., during CI/CD migrations) without running a prune.
|
|
24
25
|
*
|
|
25
|
-
* ##
|
|
26
|
+
* ## Attribute stripping preserves export history
|
|
27
|
+
*
|
|
28
|
+
* Stripping removes `adata`, `hmark`, `status`, and `other` attributes
|
|
29
|
+
* from completed jobs while preserving:
|
|
30
|
+
* - `jdata` — workflow return data
|
|
31
|
+
* - `udata` — user-searchable data
|
|
32
|
+
* - `jmark` — timeline markers needed for Temporal-compatible export
|
|
33
|
+
*
|
|
34
|
+
* Set `keepHmark: true` to also preserve `hmark` (activity state markers).
|
|
35
|
+
*
|
|
36
|
+
* ## Entity-scoped pruning
|
|
26
37
|
*
|
|
27
|
-
*
|
|
28
|
-
*
|
|
38
|
+
* Use `entities` to restrict pruning/stripping to specific entity types
|
|
39
|
+
* (e.g., `['book', 'author']`). Use `pruneTransient` to delete expired
|
|
40
|
+
* jobs with no entity (`entity IS NULL`).
|
|
41
|
+
*
|
|
42
|
+
* ## Idempotent stripping with `pruned_at`
|
|
43
|
+
*
|
|
44
|
+
* After stripping, jobs are marked with `pruned_at = NOW()`. Subsequent
|
|
45
|
+
* prune calls skip already-pruned jobs, making the operation idempotent
|
|
46
|
+
* and efficient for repeated scheduling.
|
|
47
|
+
*
|
|
48
|
+
* ## Independent cron schedules (TypeScript)
|
|
29
49
|
*
|
|
30
50
|
* @example
|
|
31
51
|
* ```typescript
|
|
@@ -38,7 +58,6 @@ import { PostgresClientType } from '../../types/postgres';
|
|
|
38
58
|
* };
|
|
39
59
|
*
|
|
40
60
|
* // Cron 1 — Nightly: strip execution artifacts from completed jobs
|
|
41
|
-
* // Keeps all jobs and their jdata/udata; keeps all streams.
|
|
42
61
|
* await DBA.prune({
|
|
43
62
|
* appId: 'myapp', connection,
|
|
44
63
|
* jobs: false, streams: false, attributes: true,
|
|
@@ -51,29 +70,34 @@ import { PostgresClientType } from '../../types/postgres';
|
|
|
51
70
|
* jobs: false, streams: true,
|
|
52
71
|
* });
|
|
53
72
|
*
|
|
54
|
-
* // Cron 3 — Weekly: remove expired jobs older than 30 days
|
|
73
|
+
* // Cron 3 — Weekly: remove expired 'book' jobs older than 30 days
|
|
55
74
|
* await DBA.prune({
|
|
56
75
|
* appId: 'myapp', connection,
|
|
57
76
|
* expire: '30 days',
|
|
58
77
|
* jobs: true, streams: false,
|
|
78
|
+
* entities: ['book'],
|
|
79
|
+
* });
|
|
80
|
+
*
|
|
81
|
+
* // Cron 4 — Weekly: remove transient (no entity) expired jobs
|
|
82
|
+
* await DBA.prune({
|
|
83
|
+
* appId: 'myapp', connection,
|
|
84
|
+
* expire: '7 days',
|
|
85
|
+
* jobs: false, streams: false,
|
|
86
|
+
* pruneTransient: true,
|
|
59
87
|
* });
|
|
60
88
|
* ```
|
|
61
89
|
*
|
|
62
90
|
* ## Direct SQL (schedulable via pg_cron)
|
|
63
91
|
*
|
|
64
92
|
* The underlying Postgres function can be called directly, without
|
|
65
|
-
* the TypeScript SDK.
|
|
66
|
-
* SQL client:
|
|
93
|
+
* the TypeScript SDK. The first 4 parameters are backwards-compatible:
|
|
67
94
|
*
|
|
68
95
|
* ```sql
|
|
69
96
|
* -- Strip attributes only (keep all jobs and streams)
|
|
70
97
|
* SELECT * FROM myapp.prune('0 seconds', false, false, true);
|
|
71
98
|
*
|
|
72
|
-
* -- Prune
|
|
73
|
-
* SELECT * FROM myapp.prune('
|
|
74
|
-
*
|
|
75
|
-
* -- Prune expired jobs older than 30 days (keep streams)
|
|
76
|
-
* SELECT * FROM myapp.prune('30 days', true, false, false);
|
|
99
|
+
* -- Prune only 'book' entity jobs older than 30 days
|
|
100
|
+
* SELECT * FROM myapp.prune('30 days', true, false, false, ARRAY['book']);
|
|
77
101
|
*
|
|
78
102
|
* -- Prune everything older than 7 days and strip attributes
|
|
79
103
|
* SELECT * FROM myapp.prune('7 days', true, true, true);
|
|
@@ -98,6 +122,12 @@ declare class DBA {
|
|
|
98
122
|
client: PostgresClientType;
|
|
99
123
|
release: () => Promise<void>;
|
|
100
124
|
}>;
|
|
125
|
+
/**
|
|
126
|
+
* Returns migration SQL for the `pruned_at` column.
|
|
127
|
+
* Handles existing deployments that lack the column.
|
|
128
|
+
* @private
|
|
129
|
+
*/
|
|
130
|
+
static getMigrationSQL(schema: string): string;
|
|
101
131
|
/**
|
|
102
132
|
* Returns the SQL for the server-side `prune()` function.
|
|
103
133
|
* @private
|
|
@@ -105,7 +135,8 @@ declare class DBA {
|
|
|
105
135
|
static getPruneFunctionSQL(schema: string): string;
|
|
106
136
|
/**
|
|
107
137
|
* Deploys the `prune()` Postgres function into the target schema.
|
|
108
|
-
*
|
|
138
|
+
* Also runs schema migrations (e.g., adding `pruned_at` column).
|
|
139
|
+
* Idempotent — uses `CREATE OR REPLACE` and `IF NOT EXISTS`.
|
|
109
140
|
*
|
|
110
141
|
* The function is automatically deployed when {@link DBA.prune} is called,
|
|
111
142
|
* but this method is exposed for explicit control (e.g., CI/CD
|
|
@@ -138,12 +169,15 @@ declare class DBA {
|
|
|
138
169
|
*
|
|
139
170
|
* Operations (each enabled individually):
|
|
140
171
|
* 1. **jobs** — Hard-deletes expired jobs older than the retention
|
|
141
|
-
* window (FK CASCADE removes their attributes automatically)
|
|
172
|
+
* window (FK CASCADE removes their attributes automatically).
|
|
173
|
+
* Scoped by `entities` when set.
|
|
142
174
|
* 2. **streams** — Hard-deletes expired stream messages older than
|
|
143
175
|
* the retention window
|
|
144
176
|
* 3. **attributes** — Strips non-essential attributes (`adata`,
|
|
145
|
-
* `hmark`, `
|
|
146
|
-
*
|
|
177
|
+
* `hmark`, `status`, `other`) from completed, un-pruned jobs.
|
|
178
|
+
* Preserves `jdata`, `udata`, and `jmark`. Marks stripped
|
|
179
|
+
* jobs with `pruned_at` for idempotency.
|
|
180
|
+
* 4. **pruneTransient** — Deletes expired jobs with `entity IS NULL`
|
|
147
181
|
*
|
|
148
182
|
* @param options - Prune configuration
|
|
149
183
|
* @returns Counts of deleted/stripped rows
|
|
@@ -153,7 +187,7 @@ declare class DBA {
|
|
|
153
187
|
* import { Client as Postgres } from 'pg';
|
|
154
188
|
* import { DBA } from '@hotmeshio/hotmesh';
|
|
155
189
|
*
|
|
156
|
-
* // Strip attributes
|
|
190
|
+
* // Strip attributes from 'book' entities only
|
|
157
191
|
* await DBA.prune({
|
|
158
192
|
* appId: 'myapp',
|
|
159
193
|
* connection: {
|
|
@@ -163,6 +197,7 @@ declare class DBA {
|
|
|
163
197
|
* jobs: false,
|
|
164
198
|
* streams: false,
|
|
165
199
|
* attributes: true,
|
|
200
|
+
* entities: ['book'],
|
|
166
201
|
* });
|
|
167
202
|
* ```
|
|
168
203
|
*/
|
|
@@ -14,8 +14,9 @@ const postgres_1 = require("../connector/providers/postgres");
|
|
|
14
14
|
* | Table | What accumulates |
|
|
15
15
|
* |---|---|
|
|
16
16
|
* | `{appId}.jobs` | Completed/expired jobs with `expired_at` set |
|
|
17
|
-
* | `{appId}.jobs_attributes` | Execution artifacts (`adata`, `hmark`, `
|
|
18
|
-
* | `{appId}.
|
|
17
|
+
* | `{appId}.jobs_attributes` | Execution artifacts (`adata`, `hmark`, `status`, `other`) that are only needed during workflow execution |
|
|
18
|
+
* | `{appId}.engine_streams` | Processed engine stream messages with `expired_at` set |
|
|
19
|
+
* | `{appId}.worker_streams` | Processed worker stream messages with `expired_at` set |
|
|
19
20
|
*
|
|
20
21
|
* The `DBA` service addresses this with two methods:
|
|
21
22
|
*
|
|
@@ -25,10 +26,29 @@ const postgres_1 = require("../connector/providers/postgres");
|
|
|
25
26
|
* - {@link DBA.deploy | deploy()} — Pre-deploys the Postgres function
|
|
26
27
|
* (e.g., during CI/CD migrations) without running a prune.
|
|
27
28
|
*
|
|
28
|
-
* ##
|
|
29
|
+
* ## Attribute stripping preserves export history
|
|
30
|
+
*
|
|
31
|
+
* Stripping removes `adata`, `hmark`, `status`, and `other` attributes
|
|
32
|
+
* from completed jobs while preserving:
|
|
33
|
+
* - `jdata` — workflow return data
|
|
34
|
+
* - `udata` — user-searchable data
|
|
35
|
+
* - `jmark` — timeline markers needed for Temporal-compatible export
|
|
36
|
+
*
|
|
37
|
+
* Set `keepHmark: true` to also preserve `hmark` (activity state markers).
|
|
38
|
+
*
|
|
39
|
+
* ## Entity-scoped pruning
|
|
40
|
+
*
|
|
41
|
+
* Use `entities` to restrict pruning/stripping to specific entity types
|
|
42
|
+
* (e.g., `['book', 'author']`). Use `pruneTransient` to delete expired
|
|
43
|
+
* jobs with no entity (`entity IS NULL`).
|
|
29
44
|
*
|
|
30
|
-
*
|
|
31
|
-
*
|
|
45
|
+
* ## Idempotent stripping with `pruned_at`
|
|
46
|
+
*
|
|
47
|
+
* After stripping, jobs are marked with `pruned_at = NOW()`. Subsequent
|
|
48
|
+
* prune calls skip already-pruned jobs, making the operation idempotent
|
|
49
|
+
* and efficient for repeated scheduling.
|
|
50
|
+
*
|
|
51
|
+
* ## Independent cron schedules (TypeScript)
|
|
32
52
|
*
|
|
33
53
|
* @example
|
|
34
54
|
* ```typescript
|
|
@@ -41,7 +61,6 @@ const postgres_1 = require("../connector/providers/postgres");
|
|
|
41
61
|
* };
|
|
42
62
|
*
|
|
43
63
|
* // Cron 1 — Nightly: strip execution artifacts from completed jobs
|
|
44
|
-
* // Keeps all jobs and their jdata/udata; keeps all streams.
|
|
45
64
|
* await DBA.prune({
|
|
46
65
|
* appId: 'myapp', connection,
|
|
47
66
|
* jobs: false, streams: false, attributes: true,
|
|
@@ -54,29 +73,34 @@ const postgres_1 = require("../connector/providers/postgres");
|
|
|
54
73
|
* jobs: false, streams: true,
|
|
55
74
|
* });
|
|
56
75
|
*
|
|
57
|
-
* // Cron 3 — Weekly: remove expired jobs older than 30 days
|
|
76
|
+
* // Cron 3 — Weekly: remove expired 'book' jobs older than 30 days
|
|
58
77
|
* await DBA.prune({
|
|
59
78
|
* appId: 'myapp', connection,
|
|
60
79
|
* expire: '30 days',
|
|
61
80
|
* jobs: true, streams: false,
|
|
81
|
+
* entities: ['book'],
|
|
82
|
+
* });
|
|
83
|
+
*
|
|
84
|
+
* // Cron 4 — Weekly: remove transient (no entity) expired jobs
|
|
85
|
+
* await DBA.prune({
|
|
86
|
+
* appId: 'myapp', connection,
|
|
87
|
+
* expire: '7 days',
|
|
88
|
+
* jobs: false, streams: false,
|
|
89
|
+
* pruneTransient: true,
|
|
62
90
|
* });
|
|
63
91
|
* ```
|
|
64
92
|
*
|
|
65
93
|
* ## Direct SQL (schedulable via pg_cron)
|
|
66
94
|
*
|
|
67
95
|
* The underlying Postgres function can be called directly, without
|
|
68
|
-
* the TypeScript SDK.
|
|
69
|
-
* SQL client:
|
|
96
|
+
* the TypeScript SDK. The first 4 parameters are backwards-compatible:
|
|
70
97
|
*
|
|
71
98
|
* ```sql
|
|
72
99
|
* -- Strip attributes only (keep all jobs and streams)
|
|
73
100
|
* SELECT * FROM myapp.prune('0 seconds', false, false, true);
|
|
74
101
|
*
|
|
75
|
-
* -- Prune
|
|
76
|
-
* SELECT * FROM myapp.prune('
|
|
77
|
-
*
|
|
78
|
-
* -- Prune expired jobs older than 30 days (keep streams)
|
|
79
|
-
* SELECT * FROM myapp.prune('30 days', true, false, false);
|
|
102
|
+
* -- Prune only 'book' entity jobs older than 30 days
|
|
103
|
+
* SELECT * FROM myapp.prune('30 days', true, false, false, ARRAY['book']);
|
|
80
104
|
*
|
|
81
105
|
* -- Prune everything older than 7 days and strip attributes
|
|
82
106
|
* SELECT * FROM myapp.prune('7 days', true, true, true);
|
|
@@ -121,6 +145,20 @@ class DBA {
|
|
|
121
145
|
release: async () => { },
|
|
122
146
|
};
|
|
123
147
|
}
|
|
148
|
+
/**
|
|
149
|
+
* Returns migration SQL for the `pruned_at` column.
|
|
150
|
+
* Handles existing deployments that lack the column.
|
|
151
|
+
* @private
|
|
152
|
+
*/
|
|
153
|
+
static getMigrationSQL(schema) {
|
|
154
|
+
return `
|
|
155
|
+
ALTER TABLE ${schema}.jobs
|
|
156
|
+
ADD COLUMN IF NOT EXISTS pruned_at TIMESTAMP WITH TIME ZONE;
|
|
157
|
+
|
|
158
|
+
CREATE INDEX IF NOT EXISTS idx_jobs_pruned_at
|
|
159
|
+
ON ${schema}.jobs (pruned_at) WHERE pruned_at IS NULL;
|
|
160
|
+
`;
|
|
161
|
+
}
|
|
124
162
|
/**
|
|
125
163
|
* Returns the SQL for the server-side `prune()` function.
|
|
126
164
|
* @private
|
|
@@ -131,12 +169,17 @@ class DBA {
|
|
|
131
169
|
retention INTERVAL DEFAULT INTERVAL '7 days',
|
|
132
170
|
prune_jobs BOOLEAN DEFAULT TRUE,
|
|
133
171
|
prune_streams BOOLEAN DEFAULT TRUE,
|
|
134
|
-
strip_attributes BOOLEAN DEFAULT FALSE
|
|
172
|
+
strip_attributes BOOLEAN DEFAULT FALSE,
|
|
173
|
+
entity_list TEXT[] DEFAULT NULL,
|
|
174
|
+
prune_transient BOOLEAN DEFAULT FALSE,
|
|
175
|
+
keep_hmark BOOLEAN DEFAULT FALSE
|
|
135
176
|
)
|
|
136
177
|
RETURNS TABLE(
|
|
137
178
|
deleted_jobs BIGINT,
|
|
138
179
|
deleted_streams BIGINT,
|
|
139
|
-
stripped_attributes BIGINT
|
|
180
|
+
stripped_attributes BIGINT,
|
|
181
|
+
deleted_transient BIGINT,
|
|
182
|
+
marked_pruned BIGINT
|
|
140
183
|
)
|
|
141
184
|
LANGUAGE plpgsql
|
|
142
185
|
AS $$
|
|
@@ -144,40 +187,84 @@ class DBA {
|
|
|
144
187
|
v_deleted_jobs BIGINT := 0;
|
|
145
188
|
v_deleted_streams BIGINT := 0;
|
|
146
189
|
v_stripped_attributes BIGINT := 0;
|
|
190
|
+
v_deleted_transient BIGINT := 0;
|
|
191
|
+
v_marked_pruned BIGINT := 0;
|
|
192
|
+
v_temp_count BIGINT := 0;
|
|
147
193
|
BEGIN
|
|
148
194
|
-- 1. Hard-delete expired jobs older than the retention window.
|
|
149
195
|
-- FK CASCADE on jobs_attributes handles attribute cleanup.
|
|
196
|
+
-- Optionally scoped to an entity allowlist.
|
|
150
197
|
IF prune_jobs THEN
|
|
151
198
|
DELETE FROM ${schema}.jobs
|
|
152
199
|
WHERE expired_at IS NOT NULL
|
|
153
|
-
AND expired_at < NOW() - retention
|
|
200
|
+
AND expired_at < NOW() - retention
|
|
201
|
+
AND (entity_list IS NULL OR entity = ANY(entity_list));
|
|
154
202
|
GET DIAGNOSTICS v_deleted_jobs = ROW_COUNT;
|
|
155
203
|
END IF;
|
|
156
204
|
|
|
157
|
-
-- 2. Hard-delete
|
|
205
|
+
-- 2. Hard-delete transient (entity IS NULL) expired jobs.
|
|
206
|
+
IF prune_transient THEN
|
|
207
|
+
DELETE FROM ${schema}.jobs
|
|
208
|
+
WHERE entity IS NULL
|
|
209
|
+
AND expired_at IS NOT NULL
|
|
210
|
+
AND expired_at < NOW() - retention;
|
|
211
|
+
GET DIAGNOSTICS v_deleted_transient = ROW_COUNT;
|
|
212
|
+
END IF;
|
|
213
|
+
|
|
214
|
+
-- 3. Hard-delete expired stream messages older than the retention window.
|
|
215
|
+
-- Deletes from both engine_streams and worker_streams tables.
|
|
158
216
|
IF prune_streams THEN
|
|
159
|
-
DELETE FROM ${schema}.
|
|
217
|
+
DELETE FROM ${schema}.engine_streams
|
|
160
218
|
WHERE expired_at IS NOT NULL
|
|
161
219
|
AND expired_at < NOW() - retention;
|
|
162
220
|
GET DIAGNOSTICS v_deleted_streams = ROW_COUNT;
|
|
221
|
+
|
|
222
|
+
DELETE FROM ${schema}.worker_streams
|
|
223
|
+
WHERE expired_at IS NOT NULL
|
|
224
|
+
AND expired_at < NOW() - retention;
|
|
225
|
+
GET DIAGNOSTICS v_temp_count = ROW_COUNT;
|
|
226
|
+
v_deleted_streams := v_deleted_streams + v_temp_count;
|
|
163
227
|
END IF;
|
|
164
228
|
|
|
165
|
-
--
|
|
166
|
-
--
|
|
229
|
+
-- 4. Strip execution artifacts from completed, live, un-pruned jobs.
|
|
230
|
+
-- Always preserves: jdata, udata, jmark (timeline/export history).
|
|
231
|
+
-- Optionally preserves: hmark (when keep_hmark is true).
|
|
167
232
|
IF strip_attributes THEN
|
|
168
|
-
|
|
169
|
-
|
|
233
|
+
WITH target_jobs AS (
|
|
234
|
+
SELECT id FROM ${schema}.jobs
|
|
235
|
+
WHERE status = 0
|
|
236
|
+
AND is_live = TRUE
|
|
237
|
+
AND pruned_at IS NULL
|
|
238
|
+
AND (entity_list IS NULL OR entity = ANY(entity_list))
|
|
239
|
+
),
|
|
240
|
+
deleted AS (
|
|
241
|
+
DELETE FROM ${schema}.jobs_attributes
|
|
242
|
+
WHERE job_id IN (SELECT id FROM target_jobs)
|
|
243
|
+
AND type NOT IN ('jdata', 'udata', 'jmark')
|
|
244
|
+
AND (keep_hmark = FALSE OR type <> 'hmark')
|
|
245
|
+
RETURNING job_id
|
|
246
|
+
)
|
|
247
|
+
SELECT COUNT(*) INTO v_stripped_attributes FROM deleted;
|
|
248
|
+
|
|
249
|
+
-- Mark pruned jobs so they are skipped on future runs.
|
|
250
|
+
WITH target_jobs AS (
|
|
170
251
|
SELECT id FROM ${schema}.jobs
|
|
171
252
|
WHERE status = 0
|
|
172
253
|
AND is_live = TRUE
|
|
254
|
+
AND pruned_at IS NULL
|
|
255
|
+
AND (entity_list IS NULL OR entity = ANY(entity_list))
|
|
173
256
|
)
|
|
174
|
-
|
|
175
|
-
|
|
257
|
+
UPDATE ${schema}.jobs
|
|
258
|
+
SET pruned_at = NOW()
|
|
259
|
+
WHERE id IN (SELECT id FROM target_jobs);
|
|
260
|
+
GET DIAGNOSTICS v_marked_pruned = ROW_COUNT;
|
|
176
261
|
END IF;
|
|
177
262
|
|
|
178
263
|
deleted_jobs := v_deleted_jobs;
|
|
179
264
|
deleted_streams := v_deleted_streams;
|
|
180
265
|
stripped_attributes := v_stripped_attributes;
|
|
266
|
+
deleted_transient := v_deleted_transient;
|
|
267
|
+
marked_pruned := v_marked_pruned;
|
|
181
268
|
RETURN NEXT;
|
|
182
269
|
END;
|
|
183
270
|
$$;
|
|
@@ -185,7 +272,8 @@ class DBA {
|
|
|
185
272
|
}
|
|
186
273
|
/**
|
|
187
274
|
* Deploys the `prune()` Postgres function into the target schema.
|
|
188
|
-
*
|
|
275
|
+
* Also runs schema migrations (e.g., adding `pruned_at` column).
|
|
276
|
+
* Idempotent — uses `CREATE OR REPLACE` and `IF NOT EXISTS`.
|
|
189
277
|
*
|
|
190
278
|
* The function is automatically deployed when {@link DBA.prune} is called,
|
|
191
279
|
* but this method is exposed for explicit control (e.g., CI/CD
|
|
@@ -214,6 +302,7 @@ class DBA {
|
|
|
214
302
|
const schema = DBA.safeName(appId);
|
|
215
303
|
const { client, release } = await DBA.getClient(connection);
|
|
216
304
|
try {
|
|
305
|
+
await client.query(DBA.getMigrationSQL(schema));
|
|
217
306
|
await client.query(DBA.getPruneFunctionSQL(schema));
|
|
218
307
|
}
|
|
219
308
|
finally {
|
|
@@ -227,12 +316,15 @@ class DBA {
|
|
|
227
316
|
*
|
|
228
317
|
* Operations (each enabled individually):
|
|
229
318
|
* 1. **jobs** — Hard-deletes expired jobs older than the retention
|
|
230
|
-
* window (FK CASCADE removes their attributes automatically)
|
|
319
|
+
* window (FK CASCADE removes their attributes automatically).
|
|
320
|
+
* Scoped by `entities` when set.
|
|
231
321
|
* 2. **streams** — Hard-deletes expired stream messages older than
|
|
232
322
|
* the retention window
|
|
233
323
|
* 3. **attributes** — Strips non-essential attributes (`adata`,
|
|
234
|
-
* `hmark`, `
|
|
235
|
-
*
|
|
324
|
+
* `hmark`, `status`, `other`) from completed, un-pruned jobs.
|
|
325
|
+
* Preserves `jdata`, `udata`, and `jmark`. Marks stripped
|
|
326
|
+
* jobs with `pruned_at` for idempotency.
|
|
327
|
+
* 4. **pruneTransient** — Deletes expired jobs with `entity IS NULL`
|
|
236
328
|
*
|
|
237
329
|
* @param options - Prune configuration
|
|
238
330
|
* @returns Counts of deleted/stripped rows
|
|
@@ -242,7 +334,7 @@ class DBA {
|
|
|
242
334
|
* import { Client as Postgres } from 'pg';
|
|
243
335
|
* import { DBA } from '@hotmeshio/hotmesh';
|
|
244
336
|
*
|
|
245
|
-
* // Strip attributes
|
|
337
|
+
* // Strip attributes from 'book' entities only
|
|
246
338
|
* await DBA.prune({
|
|
247
339
|
* appId: 'myapp',
|
|
248
340
|
* connection: {
|
|
@@ -252,6 +344,7 @@ class DBA {
|
|
|
252
344
|
* jobs: false,
|
|
253
345
|
* streams: false,
|
|
254
346
|
* attributes: true,
|
|
347
|
+
* entities: ['book'],
|
|
255
348
|
* });
|
|
256
349
|
* ```
|
|
257
350
|
*/
|
|
@@ -261,15 +354,20 @@ class DBA {
|
|
|
261
354
|
const jobs = options.jobs ?? true;
|
|
262
355
|
const streams = options.streams ?? true;
|
|
263
356
|
const attributes = options.attributes ?? false;
|
|
357
|
+
const entities = options.entities ?? null;
|
|
358
|
+
const pruneTransient = options.pruneTransient ?? false;
|
|
359
|
+
const keepHmark = options.keepHmark ?? false;
|
|
264
360
|
await DBA.deploy(options.connection, options.appId);
|
|
265
361
|
const { client, release } = await DBA.getClient(options.connection);
|
|
266
362
|
try {
|
|
267
|
-
const result = await client.query(`SELECT * FROM ${schema}.prune($1::interval, $2::boolean, $3::boolean, $4::boolean)`, [expire, jobs, streams, attributes]);
|
|
363
|
+
const result = await client.query(`SELECT * FROM ${schema}.prune($1::interval, $2::boolean, $3::boolean, $4::boolean, $5::text[], $6::boolean, $7::boolean)`, [expire, jobs, streams, attributes, entities, pruneTransient, keepHmark]);
|
|
268
364
|
const row = result.rows[0];
|
|
269
365
|
return {
|
|
270
366
|
jobs: Number(row.deleted_jobs),
|
|
271
367
|
streams: Number(row.deleted_streams),
|
|
272
368
|
attributes: Number(row.stripped_attributes),
|
|
369
|
+
transient: Number(row.deleted_transient),
|
|
370
|
+
marked: Number(row.marked_pruned),
|
|
273
371
|
};
|
|
274
372
|
}
|
|
275
373
|
finally {
|
|
@@ -145,6 +145,8 @@ class ClientService {
|
|
|
145
145
|
parentWorkflowId: options.parentWorkflowId,
|
|
146
146
|
workflowId: options.workflowId || hotmesh_1.HotMesh.guid(),
|
|
147
147
|
workflowTopic: workflowTopic,
|
|
148
|
+
taskQueue: taskQueueName,
|
|
149
|
+
workflowName: workflowName,
|
|
148
150
|
backoffCoefficient: options.config?.backoffCoefficient || enums_1.HMSH_DURABLE_EXP_BACKOFF,
|
|
149
151
|
maximumAttempts: options.config?.maximumAttempts || enums_1.HMSH_DURABLE_MAX_ATTEMPTS,
|
|
150
152
|
maximumInterval: (0, utils_1.s)(options.config?.maximumInterval || enums_1.HMSH_DURABLE_MAX_INTERVAL),
|
|
@@ -186,11 +188,14 @@ class ClientService {
|
|
|
186
188
|
*/
|
|
187
189
|
hook: async (options) => {
|
|
188
190
|
const taskQueue = options.taskQueue ?? options.entity;
|
|
189
|
-
const
|
|
191
|
+
const hookWorkflowName = options.entity ?? options.workflowName;
|
|
192
|
+
const workflowTopic = `${taskQueue}-${hookWorkflowName}`;
|
|
190
193
|
const payload = {
|
|
191
194
|
arguments: [...options.args],
|
|
192
195
|
id: options.workflowId,
|
|
193
196
|
workflowTopic,
|
|
197
|
+
taskQueue,
|
|
198
|
+
workflowName: hookWorkflowName,
|
|
194
199
|
backoffCoefficient: options.config?.backoffCoefficient || enums_1.HMSH_DURABLE_EXP_BACKOFF,
|
|
195
200
|
maximumAttempts: options.config?.maximumAttempts || enums_1.HMSH_DURABLE_MAX_ATTEMPTS,
|
|
196
201
|
maximumInterval: (0, utils_1.s)(options.config?.maximumInterval || enums_1.HMSH_DURABLE_MAX_INTERVAL),
|