@hotmeshio/hotmesh 0.10.0 → 0.10.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/build/package.json +5 -4
- package/build/services/dba/index.d.ts +52 -18
- package/build/services/dba/index.js +118 -29
- package/build/services/durable/exporter.d.ts +60 -3
- package/build/services/durable/exporter.js +430 -2
- package/build/services/durable/handle.d.ts +12 -1
- package/build/services/durable/handle.js +13 -0
- package/build/services/durable/index.d.ts +31 -2
- package/build/services/durable/index.js +31 -2
- package/build/services/durable/interceptor.d.ts +73 -0
- package/build/services/durable/interceptor.js +73 -0
- package/build/services/durable/workflow/proxyActivities.js +33 -29
- package/build/services/store/providers/postgres/kvtables.js +6 -1
- package/build/types/dba.d.ts +31 -5
- package/build/types/durable.d.ts +30 -12
- package/build/types/exporter.d.ts +127 -0
- package/build/types/index.d.ts +1 -1
- package/package.json +5 -4
- /package/{vitest.config.ts → vitest.config.mts} +0 -0
package/README.md
CHANGED
|
@@ -11,7 +11,7 @@ npm install @hotmeshio/hotmesh
|
|
|
11
11
|
## Use HotMesh for
|
|
12
12
|
|
|
13
13
|
- **Durable pipelines** — Orchestrate long-running, multi-step pipelines transactionally.
|
|
14
|
-
- **Temporal
|
|
14
|
+
- **Temporal alternative** — The `Durable` module provides a Temporal-compatible API (`Client`, `Worker`, `proxyActivities`, `sleepFor`, `startChild`, signals) that runs directly on Postgres. No app server required.
|
|
15
15
|
- **Distributed state machines** — Build stateful applications where every component can [fail and recover](https://github.com/hotmeshio/sdk-typescript/blob/main/services/collator/README.md).
|
|
16
16
|
- **AI and training pipelines** — Multi-step AI workloads where each stage is expensive and must not be repeated on failure. A crashed pipeline resumes from the last committed step, not from the beginning.
|
|
17
17
|
|
|
@@ -337,7 +337,7 @@ Durable is designed as a drop-in-compatible alternative for common Temporal patt
|
|
|
337
337
|
|
|
338
338
|
**What's the same:** `Client`, `Worker`, `proxyActivities`, `sleepFor`, `startChild`/`execChild`, signals (`waitFor`/`signal`), retry policies, and the overall workflow-as-code programming model.
|
|
339
339
|
|
|
340
|
-
**What's different:**
|
|
340
|
+
**What's different:** Postgres is the only infrastructure dependency — it stores state and coordinates workers.
|
|
341
341
|
|
|
342
342
|
## Running tests
|
|
343
343
|
|
package/build/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hotmeshio/hotmesh",
|
|
3
|
-
"version": "0.10.
|
|
3
|
+
"version": "0.10.2",
|
|
4
4
|
"description": "Permanent-Memory Workflows & AI Agents",
|
|
5
5
|
"main": "./build/index.js",
|
|
6
6
|
"types": "./build/index.d.ts",
|
|
@@ -46,6 +46,8 @@
|
|
|
46
46
|
"test:durable:sleep": "vitest run tests/durable/sleep/postgres.test.ts",
|
|
47
47
|
"test:durable:signal": "vitest run tests/durable/signal/postgres.test.ts",
|
|
48
48
|
"test:durable:unknown": "vitest run tests/durable/unknown/postgres.test.ts",
|
|
49
|
+
"test:durable:exporter": "vitest run tests/durable/exporter/exporter.test.ts",
|
|
50
|
+
"test:durable:exporter:debug": "EXPORT_DEBUG=1 HMSH_LOGLEVEL=error vitest run tests/durable/basic/postgres.test.ts",
|
|
49
51
|
"test:dba": "vitest run tests/dba",
|
|
50
52
|
"test:cycle": "vitest run tests/functional/cycle",
|
|
51
53
|
"test:functional": "vitest run tests/functional",
|
|
@@ -110,13 +112,12 @@
|
|
|
110
112
|
"nats": "^2.28.0",
|
|
111
113
|
"openai": "^5.9.0",
|
|
112
114
|
"pg": "^8.10.0",
|
|
113
|
-
"rimraf": "^
|
|
115
|
+
"rimraf": "^6.1.3",
|
|
114
116
|
"terser": "^5.37.0",
|
|
115
117
|
"ts-node": "^10.9.1",
|
|
116
|
-
"ts-node-dev": "^2.0.0",
|
|
117
118
|
"typedoc": "^0.26.4",
|
|
118
119
|
"typescript": "^5.0.4",
|
|
119
|
-
"vitest": "^
|
|
120
|
+
"vitest": "^4.0.18"
|
|
120
121
|
},
|
|
121
122
|
"peerDependencies": {
|
|
122
123
|
"nats": "^2.0.0",
|
|
@@ -11,7 +11,7 @@ import { PostgresClientType } from '../../types/postgres';
|
|
|
11
11
|
* | Table | What accumulates |
|
|
12
12
|
* |---|---|
|
|
13
13
|
* | `{appId}.jobs` | Completed/expired jobs with `expired_at` set |
|
|
14
|
-
* | `{appId}.jobs_attributes` | Execution artifacts (`adata`, `hmark`, `
|
|
14
|
+
* | `{appId}.jobs_attributes` | Execution artifacts (`adata`, `hmark`, `status`, `other`) that are only needed during workflow execution |
|
|
15
15
|
* | `{appId}.streams` | Processed stream messages with `expired_at` set |
|
|
16
16
|
*
|
|
17
17
|
* The `DBA` service addresses this with two methods:
|
|
@@ -22,10 +22,29 @@ import { PostgresClientType } from '../../types/postgres';
|
|
|
22
22
|
* - {@link DBA.deploy | deploy()} — Pre-deploys the Postgres function
|
|
23
23
|
* (e.g., during CI/CD migrations) without running a prune.
|
|
24
24
|
*
|
|
25
|
-
* ##
|
|
25
|
+
* ## Attribute stripping preserves export history
|
|
26
|
+
*
|
|
27
|
+
* Stripping removes `adata`, `hmark`, `status`, and `other` attributes
|
|
28
|
+
* from completed jobs while preserving:
|
|
29
|
+
* - `jdata` — workflow return data
|
|
30
|
+
* - `udata` — user-searchable data
|
|
31
|
+
* - `jmark` — timeline markers needed for Temporal-compatible export
|
|
32
|
+
*
|
|
33
|
+
* Set `keepHmark: true` to also preserve `hmark` (activity state markers).
|
|
34
|
+
*
|
|
35
|
+
* ## Entity-scoped pruning
|
|
26
36
|
*
|
|
27
|
-
*
|
|
28
|
-
*
|
|
37
|
+
* Use `entities` to restrict pruning/stripping to specific entity types
|
|
38
|
+
* (e.g., `['book', 'author']`). Use `pruneTransient` to delete expired
|
|
39
|
+
* jobs with no entity (`entity IS NULL`).
|
|
40
|
+
*
|
|
41
|
+
* ## Idempotent stripping with `pruned_at`
|
|
42
|
+
*
|
|
43
|
+
* After stripping, jobs are marked with `pruned_at = NOW()`. Subsequent
|
|
44
|
+
* prune calls skip already-pruned jobs, making the operation idempotent
|
|
45
|
+
* and efficient for repeated scheduling.
|
|
46
|
+
*
|
|
47
|
+
* ## Independent cron schedules (TypeScript)
|
|
29
48
|
*
|
|
30
49
|
* @example
|
|
31
50
|
* ```typescript
|
|
@@ -38,7 +57,6 @@ import { PostgresClientType } from '../../types/postgres';
|
|
|
38
57
|
* };
|
|
39
58
|
*
|
|
40
59
|
* // Cron 1 — Nightly: strip execution artifacts from completed jobs
|
|
41
|
-
* // Keeps all jobs and their jdata/udata; keeps all streams.
|
|
42
60
|
* await DBA.prune({
|
|
43
61
|
* appId: 'myapp', connection,
|
|
44
62
|
* jobs: false, streams: false, attributes: true,
|
|
@@ -51,29 +69,34 @@ import { PostgresClientType } from '../../types/postgres';
|
|
|
51
69
|
* jobs: false, streams: true,
|
|
52
70
|
* });
|
|
53
71
|
*
|
|
54
|
-
* // Cron 3 — Weekly: remove expired jobs older than 30 days
|
|
72
|
+
* // Cron 3 — Weekly: remove expired 'book' jobs older than 30 days
|
|
55
73
|
* await DBA.prune({
|
|
56
74
|
* appId: 'myapp', connection,
|
|
57
75
|
* expire: '30 days',
|
|
58
76
|
* jobs: true, streams: false,
|
|
77
|
+
* entities: ['book'],
|
|
78
|
+
* });
|
|
79
|
+
*
|
|
80
|
+
* // Cron 4 — Weekly: remove transient (no entity) expired jobs
|
|
81
|
+
* await DBA.prune({
|
|
82
|
+
* appId: 'myapp', connection,
|
|
83
|
+
* expire: '7 days',
|
|
84
|
+
* jobs: false, streams: false,
|
|
85
|
+
* pruneTransient: true,
|
|
59
86
|
* });
|
|
60
87
|
* ```
|
|
61
88
|
*
|
|
62
89
|
* ## Direct SQL (schedulable via pg_cron)
|
|
63
90
|
*
|
|
64
91
|
* The underlying Postgres function can be called directly, without
|
|
65
|
-
* the TypeScript SDK.
|
|
66
|
-
* SQL client:
|
|
92
|
+
* the TypeScript SDK. The first 4 parameters are backwards-compatible:
|
|
67
93
|
*
|
|
68
94
|
* ```sql
|
|
69
95
|
* -- Strip attributes only (keep all jobs and streams)
|
|
70
96
|
* SELECT * FROM myapp.prune('0 seconds', false, false, true);
|
|
71
97
|
*
|
|
72
|
-
* -- Prune
|
|
73
|
-
* SELECT * FROM myapp.prune('
|
|
74
|
-
*
|
|
75
|
-
* -- Prune expired jobs older than 30 days (keep streams)
|
|
76
|
-
* SELECT * FROM myapp.prune('30 days', true, false, false);
|
|
98
|
+
* -- Prune only 'book' entity jobs older than 30 days
|
|
99
|
+
* SELECT * FROM myapp.prune('30 days', true, false, false, ARRAY['book']);
|
|
77
100
|
*
|
|
78
101
|
* -- Prune everything older than 7 days and strip attributes
|
|
79
102
|
* SELECT * FROM myapp.prune('7 days', true, true, true);
|
|
@@ -98,6 +121,12 @@ declare class DBA {
|
|
|
98
121
|
client: PostgresClientType;
|
|
99
122
|
release: () => Promise<void>;
|
|
100
123
|
}>;
|
|
124
|
+
/**
|
|
125
|
+
* Returns migration SQL for the `pruned_at` column.
|
|
126
|
+
* Handles existing deployments that lack the column.
|
|
127
|
+
* @private
|
|
128
|
+
*/
|
|
129
|
+
static getMigrationSQL(schema: string): string;
|
|
101
130
|
/**
|
|
102
131
|
* Returns the SQL for the server-side `prune()` function.
|
|
103
132
|
* @private
|
|
@@ -105,7 +134,8 @@ declare class DBA {
|
|
|
105
134
|
static getPruneFunctionSQL(schema: string): string;
|
|
106
135
|
/**
|
|
107
136
|
* Deploys the `prune()` Postgres function into the target schema.
|
|
108
|
-
*
|
|
137
|
+
* Also runs schema migrations (e.g., adding `pruned_at` column).
|
|
138
|
+
* Idempotent — uses `CREATE OR REPLACE` and `IF NOT EXISTS`.
|
|
109
139
|
*
|
|
110
140
|
* The function is automatically deployed when {@link DBA.prune} is called,
|
|
111
141
|
* but this method is exposed for explicit control (e.g., CI/CD
|
|
@@ -138,12 +168,15 @@ declare class DBA {
|
|
|
138
168
|
*
|
|
139
169
|
* Operations (each enabled individually):
|
|
140
170
|
* 1. **jobs** — Hard-deletes expired jobs older than the retention
|
|
141
|
-
* window (FK CASCADE removes their attributes automatically)
|
|
171
|
+
* window (FK CASCADE removes their attributes automatically).
|
|
172
|
+
* Scoped by `entities` when set.
|
|
142
173
|
* 2. **streams** — Hard-deletes expired stream messages older than
|
|
143
174
|
* the retention window
|
|
144
175
|
* 3. **attributes** — Strips non-essential attributes (`adata`,
|
|
145
|
-
* `hmark`, `
|
|
146
|
-
*
|
|
176
|
+
* `hmark`, `status`, `other`) from completed, un-pruned jobs.
|
|
177
|
+
* Preserves `jdata`, `udata`, and `jmark`. Marks stripped
|
|
178
|
+
* jobs with `pruned_at` for idempotency.
|
|
179
|
+
* 4. **pruneTransient** — Deletes expired jobs with `entity IS NULL`
|
|
147
180
|
*
|
|
148
181
|
* @param options - Prune configuration
|
|
149
182
|
* @returns Counts of deleted/stripped rows
|
|
@@ -153,7 +186,7 @@ declare class DBA {
|
|
|
153
186
|
* import { Client as Postgres } from 'pg';
|
|
154
187
|
* import { DBA } from '@hotmeshio/hotmesh';
|
|
155
188
|
*
|
|
156
|
-
* // Strip attributes
|
|
189
|
+
* // Strip attributes from 'book' entities only
|
|
157
190
|
* await DBA.prune({
|
|
158
191
|
* appId: 'myapp',
|
|
159
192
|
* connection: {
|
|
@@ -163,6 +196,7 @@ declare class DBA {
|
|
|
163
196
|
* jobs: false,
|
|
164
197
|
* streams: false,
|
|
165
198
|
* attributes: true,
|
|
199
|
+
* entities: ['book'],
|
|
166
200
|
* });
|
|
167
201
|
* ```
|
|
168
202
|
*/
|
|
@@ -14,7 +14,7 @@ const postgres_1 = require("../connector/providers/postgres");
|
|
|
14
14
|
* | Table | What accumulates |
|
|
15
15
|
* |---|---|
|
|
16
16
|
* | `{appId}.jobs` | Completed/expired jobs with `expired_at` set |
|
|
17
|
-
* | `{appId}.jobs_attributes` | Execution artifacts (`adata`, `hmark`, `
|
|
17
|
+
* | `{appId}.jobs_attributes` | Execution artifacts (`adata`, `hmark`, `status`, `other`) that are only needed during workflow execution |
|
|
18
18
|
* | `{appId}.streams` | Processed stream messages with `expired_at` set |
|
|
19
19
|
*
|
|
20
20
|
* The `DBA` service addresses this with two methods:
|
|
@@ -25,10 +25,29 @@ const postgres_1 = require("../connector/providers/postgres");
|
|
|
25
25
|
* - {@link DBA.deploy | deploy()} — Pre-deploys the Postgres function
|
|
26
26
|
* (e.g., during CI/CD migrations) without running a prune.
|
|
27
27
|
*
|
|
28
|
-
* ##
|
|
28
|
+
* ## Attribute stripping preserves export history
|
|
29
|
+
*
|
|
30
|
+
* Stripping removes `adata`, `hmark`, `status`, and `other` attributes
|
|
31
|
+
* from completed jobs while preserving:
|
|
32
|
+
* - `jdata` — workflow return data
|
|
33
|
+
* - `udata` — user-searchable data
|
|
34
|
+
* - `jmark` — timeline markers needed for Temporal-compatible export
|
|
35
|
+
*
|
|
36
|
+
* Set `keepHmark: true` to also preserve `hmark` (activity state markers).
|
|
37
|
+
*
|
|
38
|
+
* ## Entity-scoped pruning
|
|
29
39
|
*
|
|
30
|
-
*
|
|
31
|
-
*
|
|
40
|
+
* Use `entities` to restrict pruning/stripping to specific entity types
|
|
41
|
+
* (e.g., `['book', 'author']`). Use `pruneTransient` to delete expired
|
|
42
|
+
* jobs with no entity (`entity IS NULL`).
|
|
43
|
+
*
|
|
44
|
+
* ## Idempotent stripping with `pruned_at`
|
|
45
|
+
*
|
|
46
|
+
* After stripping, jobs are marked with `pruned_at = NOW()`. Subsequent
|
|
47
|
+
* prune calls skip already-pruned jobs, making the operation idempotent
|
|
48
|
+
* and efficient for repeated scheduling.
|
|
49
|
+
*
|
|
50
|
+
* ## Independent cron schedules (TypeScript)
|
|
32
51
|
*
|
|
33
52
|
* @example
|
|
34
53
|
* ```typescript
|
|
@@ -41,7 +60,6 @@ const postgres_1 = require("../connector/providers/postgres");
|
|
|
41
60
|
* };
|
|
42
61
|
*
|
|
43
62
|
* // Cron 1 — Nightly: strip execution artifacts from completed jobs
|
|
44
|
-
* // Keeps all jobs and their jdata/udata; keeps all streams.
|
|
45
63
|
* await DBA.prune({
|
|
46
64
|
* appId: 'myapp', connection,
|
|
47
65
|
* jobs: false, streams: false, attributes: true,
|
|
@@ -54,29 +72,34 @@ const postgres_1 = require("../connector/providers/postgres");
|
|
|
54
72
|
* jobs: false, streams: true,
|
|
55
73
|
* });
|
|
56
74
|
*
|
|
57
|
-
* // Cron 3 — Weekly: remove expired jobs older than 30 days
|
|
75
|
+
* // Cron 3 — Weekly: remove expired 'book' jobs older than 30 days
|
|
58
76
|
* await DBA.prune({
|
|
59
77
|
* appId: 'myapp', connection,
|
|
60
78
|
* expire: '30 days',
|
|
61
79
|
* jobs: true, streams: false,
|
|
80
|
+
* entities: ['book'],
|
|
81
|
+
* });
|
|
82
|
+
*
|
|
83
|
+
* // Cron 4 — Weekly: remove transient (no entity) expired jobs
|
|
84
|
+
* await DBA.prune({
|
|
85
|
+
* appId: 'myapp', connection,
|
|
86
|
+
* expire: '7 days',
|
|
87
|
+
* jobs: false, streams: false,
|
|
88
|
+
* pruneTransient: true,
|
|
62
89
|
* });
|
|
63
90
|
* ```
|
|
64
91
|
*
|
|
65
92
|
* ## Direct SQL (schedulable via pg_cron)
|
|
66
93
|
*
|
|
67
94
|
* The underlying Postgres function can be called directly, without
|
|
68
|
-
* the TypeScript SDK.
|
|
69
|
-
* SQL client:
|
|
95
|
+
* the TypeScript SDK. The first 4 parameters are backwards-compatible:
|
|
70
96
|
*
|
|
71
97
|
* ```sql
|
|
72
98
|
* -- Strip attributes only (keep all jobs and streams)
|
|
73
99
|
* SELECT * FROM myapp.prune('0 seconds', false, false, true);
|
|
74
100
|
*
|
|
75
|
-
* -- Prune
|
|
76
|
-
* SELECT * FROM myapp.prune('
|
|
77
|
-
*
|
|
78
|
-
* -- Prune expired jobs older than 30 days (keep streams)
|
|
79
|
-
* SELECT * FROM myapp.prune('30 days', true, false, false);
|
|
101
|
+
* -- Prune only 'book' entity jobs older than 30 days
|
|
102
|
+
* SELECT * FROM myapp.prune('30 days', true, false, false, ARRAY['book']);
|
|
80
103
|
*
|
|
81
104
|
* -- Prune everything older than 7 days and strip attributes
|
|
82
105
|
* SELECT * FROM myapp.prune('7 days', true, true, true);
|
|
@@ -121,6 +144,20 @@ class DBA {
|
|
|
121
144
|
release: async () => { },
|
|
122
145
|
};
|
|
123
146
|
}
|
|
147
|
+
/**
|
|
148
|
+
* Returns migration SQL for the `pruned_at` column.
|
|
149
|
+
* Handles existing deployments that lack the column.
|
|
150
|
+
* @private
|
|
151
|
+
*/
|
|
152
|
+
static getMigrationSQL(schema) {
|
|
153
|
+
return `
|
|
154
|
+
ALTER TABLE ${schema}.jobs
|
|
155
|
+
ADD COLUMN IF NOT EXISTS pruned_at TIMESTAMP WITH TIME ZONE;
|
|
156
|
+
|
|
157
|
+
CREATE INDEX IF NOT EXISTS idx_jobs_pruned_at
|
|
158
|
+
ON ${schema}.jobs (pruned_at) WHERE pruned_at IS NULL;
|
|
159
|
+
`;
|
|
160
|
+
}
|
|
124
161
|
/**
|
|
125
162
|
* Returns the SQL for the server-side `prune()` function.
|
|
126
163
|
* @private
|
|
@@ -131,12 +168,17 @@ class DBA {
|
|
|
131
168
|
retention INTERVAL DEFAULT INTERVAL '7 days',
|
|
132
169
|
prune_jobs BOOLEAN DEFAULT TRUE,
|
|
133
170
|
prune_streams BOOLEAN DEFAULT TRUE,
|
|
134
|
-
strip_attributes BOOLEAN DEFAULT FALSE
|
|
171
|
+
strip_attributes BOOLEAN DEFAULT FALSE,
|
|
172
|
+
entity_list TEXT[] DEFAULT NULL,
|
|
173
|
+
prune_transient BOOLEAN DEFAULT FALSE,
|
|
174
|
+
keep_hmark BOOLEAN DEFAULT FALSE
|
|
135
175
|
)
|
|
136
176
|
RETURNS TABLE(
|
|
137
177
|
deleted_jobs BIGINT,
|
|
138
178
|
deleted_streams BIGINT,
|
|
139
|
-
stripped_attributes BIGINT
|
|
179
|
+
stripped_attributes BIGINT,
|
|
180
|
+
deleted_transient BIGINT,
|
|
181
|
+
marked_pruned BIGINT
|
|
140
182
|
)
|
|
141
183
|
LANGUAGE plpgsql
|
|
142
184
|
AS $$
|
|
@@ -144,17 +186,30 @@ class DBA {
|
|
|
144
186
|
v_deleted_jobs BIGINT := 0;
|
|
145
187
|
v_deleted_streams BIGINT := 0;
|
|
146
188
|
v_stripped_attributes BIGINT := 0;
|
|
189
|
+
v_deleted_transient BIGINT := 0;
|
|
190
|
+
v_marked_pruned BIGINT := 0;
|
|
147
191
|
BEGIN
|
|
148
192
|
-- 1. Hard-delete expired jobs older than the retention window.
|
|
149
193
|
-- FK CASCADE on jobs_attributes handles attribute cleanup.
|
|
194
|
+
-- Optionally scoped to an entity allowlist.
|
|
150
195
|
IF prune_jobs THEN
|
|
151
196
|
DELETE FROM ${schema}.jobs
|
|
152
197
|
WHERE expired_at IS NOT NULL
|
|
153
|
-
AND expired_at < NOW() - retention
|
|
198
|
+
AND expired_at < NOW() - retention
|
|
199
|
+
AND (entity_list IS NULL OR entity = ANY(entity_list));
|
|
154
200
|
GET DIAGNOSTICS v_deleted_jobs = ROW_COUNT;
|
|
155
201
|
END IF;
|
|
156
202
|
|
|
157
|
-
-- 2. Hard-delete
|
|
203
|
+
-- 2. Hard-delete transient (entity IS NULL) expired jobs.
|
|
204
|
+
IF prune_transient THEN
|
|
205
|
+
DELETE FROM ${schema}.jobs
|
|
206
|
+
WHERE entity IS NULL
|
|
207
|
+
AND expired_at IS NOT NULL
|
|
208
|
+
AND expired_at < NOW() - retention;
|
|
209
|
+
GET DIAGNOSTICS v_deleted_transient = ROW_COUNT;
|
|
210
|
+
END IF;
|
|
211
|
+
|
|
212
|
+
-- 3. Hard-delete expired stream messages older than the retention window.
|
|
158
213
|
IF prune_streams THEN
|
|
159
214
|
DELETE FROM ${schema}.streams
|
|
160
215
|
WHERE expired_at IS NOT NULL
|
|
@@ -162,22 +217,45 @@ class DBA {
|
|
|
162
217
|
GET DIAGNOSTICS v_deleted_streams = ROW_COUNT;
|
|
163
218
|
END IF;
|
|
164
219
|
|
|
165
|
-
--
|
|
166
|
-
--
|
|
220
|
+
-- 4. Strip execution artifacts from completed, live, un-pruned jobs.
|
|
221
|
+
-- Always preserves: jdata, udata, jmark (timeline/export history).
|
|
222
|
+
-- Optionally preserves: hmark (when keep_hmark is true).
|
|
167
223
|
IF strip_attributes THEN
|
|
168
|
-
|
|
169
|
-
|
|
224
|
+
WITH target_jobs AS (
|
|
225
|
+
SELECT id FROM ${schema}.jobs
|
|
226
|
+
WHERE status = 0
|
|
227
|
+
AND is_live = TRUE
|
|
228
|
+
AND pruned_at IS NULL
|
|
229
|
+
AND (entity_list IS NULL OR entity = ANY(entity_list))
|
|
230
|
+
),
|
|
231
|
+
deleted AS (
|
|
232
|
+
DELETE FROM ${schema}.jobs_attributes
|
|
233
|
+
WHERE job_id IN (SELECT id FROM target_jobs)
|
|
234
|
+
AND type NOT IN ('jdata', 'udata', 'jmark')
|
|
235
|
+
AND (keep_hmark = FALSE OR type <> 'hmark')
|
|
236
|
+
RETURNING job_id
|
|
237
|
+
)
|
|
238
|
+
SELECT COUNT(*) INTO v_stripped_attributes FROM deleted;
|
|
239
|
+
|
|
240
|
+
-- Mark pruned jobs so they are skipped on future runs.
|
|
241
|
+
WITH target_jobs AS (
|
|
170
242
|
SELECT id FROM ${schema}.jobs
|
|
171
243
|
WHERE status = 0
|
|
172
244
|
AND is_live = TRUE
|
|
245
|
+
AND pruned_at IS NULL
|
|
246
|
+
AND (entity_list IS NULL OR entity = ANY(entity_list))
|
|
173
247
|
)
|
|
174
|
-
|
|
175
|
-
|
|
248
|
+
UPDATE ${schema}.jobs
|
|
249
|
+
SET pruned_at = NOW()
|
|
250
|
+
WHERE id IN (SELECT id FROM target_jobs);
|
|
251
|
+
GET DIAGNOSTICS v_marked_pruned = ROW_COUNT;
|
|
176
252
|
END IF;
|
|
177
253
|
|
|
178
254
|
deleted_jobs := v_deleted_jobs;
|
|
179
255
|
deleted_streams := v_deleted_streams;
|
|
180
256
|
stripped_attributes := v_stripped_attributes;
|
|
257
|
+
deleted_transient := v_deleted_transient;
|
|
258
|
+
marked_pruned := v_marked_pruned;
|
|
181
259
|
RETURN NEXT;
|
|
182
260
|
END;
|
|
183
261
|
$$;
|
|
@@ -185,7 +263,8 @@ class DBA {
|
|
|
185
263
|
}
|
|
186
264
|
/**
|
|
187
265
|
* Deploys the `prune()` Postgres function into the target schema.
|
|
188
|
-
*
|
|
266
|
+
* Also runs schema migrations (e.g., adding `pruned_at` column).
|
|
267
|
+
* Idempotent — uses `CREATE OR REPLACE` and `IF NOT EXISTS`.
|
|
189
268
|
*
|
|
190
269
|
* The function is automatically deployed when {@link DBA.prune} is called,
|
|
191
270
|
* but this method is exposed for explicit control (e.g., CI/CD
|
|
@@ -214,6 +293,7 @@ class DBA {
|
|
|
214
293
|
const schema = DBA.safeName(appId);
|
|
215
294
|
const { client, release } = await DBA.getClient(connection);
|
|
216
295
|
try {
|
|
296
|
+
await client.query(DBA.getMigrationSQL(schema));
|
|
217
297
|
await client.query(DBA.getPruneFunctionSQL(schema));
|
|
218
298
|
}
|
|
219
299
|
finally {
|
|
@@ -227,12 +307,15 @@ class DBA {
|
|
|
227
307
|
*
|
|
228
308
|
* Operations (each enabled individually):
|
|
229
309
|
* 1. **jobs** — Hard-deletes expired jobs older than the retention
|
|
230
|
-
* window (FK CASCADE removes their attributes automatically)
|
|
310
|
+
* window (FK CASCADE removes their attributes automatically).
|
|
311
|
+
* Scoped by `entities` when set.
|
|
231
312
|
* 2. **streams** — Hard-deletes expired stream messages older than
|
|
232
313
|
* the retention window
|
|
233
314
|
* 3. **attributes** — Strips non-essential attributes (`adata`,
|
|
234
|
-
* `hmark`, `
|
|
235
|
-
*
|
|
315
|
+
* `hmark`, `status`, `other`) from completed, un-pruned jobs.
|
|
316
|
+
* Preserves `jdata`, `udata`, and `jmark`. Marks stripped
|
|
317
|
+
* jobs with `pruned_at` for idempotency.
|
|
318
|
+
* 4. **pruneTransient** — Deletes expired jobs with `entity IS NULL`
|
|
236
319
|
*
|
|
237
320
|
* @param options - Prune configuration
|
|
238
321
|
* @returns Counts of deleted/stripped rows
|
|
@@ -242,7 +325,7 @@ class DBA {
|
|
|
242
325
|
* import { Client as Postgres } from 'pg';
|
|
243
326
|
* import { DBA } from '@hotmeshio/hotmesh';
|
|
244
327
|
*
|
|
245
|
-
* // Strip attributes
|
|
328
|
+
* // Strip attributes from 'book' entities only
|
|
246
329
|
* await DBA.prune({
|
|
247
330
|
* appId: 'myapp',
|
|
248
331
|
* connection: {
|
|
@@ -252,6 +335,7 @@ class DBA {
|
|
|
252
335
|
* jobs: false,
|
|
253
336
|
* streams: false,
|
|
254
337
|
* attributes: true,
|
|
338
|
+
* entities: ['book'],
|
|
255
339
|
* });
|
|
256
340
|
* ```
|
|
257
341
|
*/
|
|
@@ -261,15 +345,20 @@ class DBA {
|
|
|
261
345
|
const jobs = options.jobs ?? true;
|
|
262
346
|
const streams = options.streams ?? true;
|
|
263
347
|
const attributes = options.attributes ?? false;
|
|
348
|
+
const entities = options.entities ?? null;
|
|
349
|
+
const pruneTransient = options.pruneTransient ?? false;
|
|
350
|
+
const keepHmark = options.keepHmark ?? false;
|
|
264
351
|
await DBA.deploy(options.connection, options.appId);
|
|
265
352
|
const { client, release } = await DBA.getClient(options.connection);
|
|
266
353
|
try {
|
|
267
|
-
const result = await client.query(`SELECT * FROM ${schema}.prune($1::interval, $2::boolean, $3::boolean, $4::boolean)`, [expire, jobs, streams, attributes]);
|
|
354
|
+
const result = await client.query(`SELECT * FROM ${schema}.prune($1::interval, $2::boolean, $3::boolean, $4::boolean, $5::text[], $6::boolean, $7::boolean)`, [expire, jobs, streams, attributes, entities, pruneTransient, keepHmark]);
|
|
268
355
|
const row = result.rows[0];
|
|
269
356
|
return {
|
|
270
357
|
jobs: Number(row.deleted_jobs),
|
|
271
358
|
streams: Number(row.deleted_streams),
|
|
272
359
|
attributes: Number(row.stripped_attributes),
|
|
360
|
+
transient: Number(row.deleted_transient),
|
|
361
|
+
marked: Number(row.marked_pruned),
|
|
273
362
|
};
|
|
274
363
|
}
|
|
275
364
|
finally {
|
|
@@ -1,8 +1,46 @@
|
|
|
1
1
|
import { ILogger } from '../logger';
|
|
2
2
|
import { StoreService } from '../store';
|
|
3
|
-
import { ExportOptions, DurableJobExport, TimelineType, TransitionType, ExportFields } from '../../types/exporter';
|
|
3
|
+
import { ExportOptions, DurableJobExport, TimelineType, TransitionType, ExportFields, ExecutionExportOptions, WorkflowExecution, WorkflowExecutionStatus } from '../../types/exporter';
|
|
4
4
|
import { ProviderClient, ProviderTransaction } from '../../types/provider';
|
|
5
5
|
import { StringStringType, Symbols } from '../../types/serializer';
|
|
6
|
+
/**
|
|
7
|
+
* Parse a HotMesh compact timestamp (YYYYMMDDHHmmss.mmm) into ISO 8601.
|
|
8
|
+
* Also accepts ISO 8601 strings directly.
|
|
9
|
+
*/
|
|
10
|
+
declare function parseTimestamp(raw: string | undefined | null): string | null;
|
|
11
|
+
/**
|
|
12
|
+
* Compute duration in milliseconds between two HotMesh timestamps.
|
|
13
|
+
*/
|
|
14
|
+
declare function computeDuration(ac: string | undefined, au: string | undefined): number | null;
|
|
15
|
+
/**
|
|
16
|
+
* Extract the operation type (proxy, child, start, wait, sleep, hook)
|
|
17
|
+
* from a timeline key like `-proxy,0,0-1-`.
|
|
18
|
+
*/
|
|
19
|
+
declare function extractOperation(key: string): string;
|
|
20
|
+
/**
|
|
21
|
+
* Extract the activity name from a timeline entry value's job_id.
|
|
22
|
+
*
|
|
23
|
+
* Job ID format: `-{workflowId}-$${activityName}{dimension}-{execIndex}`
|
|
24
|
+
* Examples:
|
|
25
|
+
* `-wfId-$analyzeContent-5` → `'analyzeContent'`
|
|
26
|
+
* `-wfId-$processOrder,0,0-3` → `'processOrder'`
|
|
27
|
+
*/
|
|
28
|
+
declare function extractActivityName(value: Record<string, any> | null): string;
|
|
29
|
+
/**
|
|
30
|
+
* Check if an activity name is a system (interceptor) operation.
|
|
31
|
+
*/
|
|
32
|
+
declare function isSystemActivity(name: string): boolean;
|
|
33
|
+
/**
|
|
34
|
+
* Map HotMesh job state to a human-readable execution status.
|
|
35
|
+
*
|
|
36
|
+
* HotMesh semaphore: `0` = idle, `> 0` = pending activities,
|
|
37
|
+
* `< 0` = failed / interrupted.
|
|
38
|
+
*
|
|
39
|
+
* A workflow can be "done" (`state.data.done === true`) while the
|
|
40
|
+
* semaphore is still > 0 (cleanup activities pending). We check
|
|
41
|
+
* both the `done` flag and the semaphore to determine status.
|
|
42
|
+
*/
|
|
43
|
+
declare function mapStatus(rawStatus: number | undefined, isDone?: boolean, hasError?: boolean): WorkflowExecutionStatus;
|
|
6
44
|
declare class ExporterService {
|
|
7
45
|
appId: string;
|
|
8
46
|
logger: ILogger;
|
|
@@ -11,10 +49,29 @@ declare class ExporterService {
|
|
|
11
49
|
private static symbols;
|
|
12
50
|
constructor(appId: string, store: StoreService<ProviderClient, ProviderTransaction>, logger: ILogger);
|
|
13
51
|
/**
|
|
14
|
-
* Convert the job hash from its
|
|
52
|
+
* Convert the job hash from its compiled format into a DurableJobExport object with
|
|
15
53
|
* facets that describe the workflow in terms relevant to narrative storytelling.
|
|
16
54
|
*/
|
|
17
55
|
export(jobId: string, options?: ExportOptions): Promise<DurableJobExport>;
|
|
56
|
+
/**
|
|
57
|
+
* Export a workflow execution as a Temporal-compatible event history.
|
|
58
|
+
*
|
|
59
|
+
* **Sparse mode** (default): transforms the main workflow's timeline
|
|
60
|
+
* into a flat event list. No additional I/O beyond the initial export.
|
|
61
|
+
*
|
|
62
|
+
* **Verbose mode**: recursively fetches child workflow jobs and attaches
|
|
63
|
+
* their executions as nested `children`.
|
|
64
|
+
*/
|
|
65
|
+
exportExecution(jobId: string, workflowTopic: string, options?: ExecutionExportOptions): Promise<WorkflowExecution>;
|
|
66
|
+
/**
|
|
67
|
+
* Pure transformation: convert a raw DurableJobExport into a
|
|
68
|
+
* Temporal-compatible WorkflowExecution event history.
|
|
69
|
+
*/
|
|
70
|
+
transformToExecution(raw: DurableJobExport, workflowId: string, workflowTopic: string, options: ExecutionExportOptions): WorkflowExecution;
|
|
71
|
+
/**
|
|
72
|
+
* Recursively fetch child workflow executions for verbose mode.
|
|
73
|
+
*/
|
|
74
|
+
private fetchChildren;
|
|
18
75
|
/**
|
|
19
76
|
* Inflates the job data into a DurableJobExport object
|
|
20
77
|
* @param jobHash - the job data
|
|
@@ -48,4 +105,4 @@ declare class ExporterService {
|
|
|
48
105
|
*/
|
|
49
106
|
sortParts(parts: TimelineType[]): TimelineType[];
|
|
50
107
|
}
|
|
51
|
-
export { ExporterService };
|
|
108
|
+
export { ExporterService, parseTimestamp, computeDuration, extractOperation, extractActivityName, isSystemActivity, mapStatus, };
|