@hotmeshio/hotmesh 0.5.2 → 0.5.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +93 -175
- package/build/index.d.ts +1 -3
- package/build/index.js +1 -5
- package/build/modules/enums.d.ts +4 -0
- package/build/modules/enums.js +5 -1
- package/build/modules/utils.d.ts +1 -9
- package/build/modules/utils.js +0 -6
- package/build/package.json +3 -4
- package/build/services/connector/factory.d.ts +2 -2
- package/build/services/connector/factory.js +11 -8
- package/build/services/connector/providers/postgres.d.ts +47 -0
- package/build/services/connector/providers/postgres.js +107 -0
- package/build/services/hotmesh/index.d.ts +8 -0
- package/build/services/hotmesh/index.js +27 -0
- package/build/services/memflow/client.d.ts +1 -1
- package/build/services/memflow/client.js +8 -6
- package/build/services/memflow/worker.js +3 -0
- package/build/services/pipe/functions/cron.js +1 -1
- package/build/services/store/providers/postgres/kvtables.js +19 -6
- package/build/services/store/providers/postgres/postgres.js +13 -2
- package/build/services/stream/providers/postgres/postgres.d.ts +6 -3
- package/build/services/stream/providers/postgres/postgres.js +169 -59
- package/build/services/sub/providers/postgres/postgres.d.ts +9 -0
- package/build/services/sub/providers/postgres/postgres.js +109 -18
- package/build/services/worker/index.js +4 -0
- package/build/types/hotmesh.d.ts +19 -5
- package/build/types/index.d.ts +0 -2
- package/env.example +11 -0
- package/index.ts +0 -4
- package/package.json +3 -4
- package/build/services/meshdata/index.d.ts +0 -795
- package/build/services/meshdata/index.js +0 -1235
- package/build/services/meshos/index.d.ts +0 -293
- package/build/services/meshos/index.js +0 -547
- package/build/types/manifest.d.ts +0 -52
- package/build/types/manifest.js +0 -2
- package/build/types/meshdata.d.ts +0 -252
- package/build/types/meshdata.js +0 -2
package/README.md
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
# HotMesh
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
**Workflow That Remembers**
|
|
4
4
|
|
|
5
5
|
 
|
|
6
6
|
|
|
7
|
-
HotMesh brings a **memory model** to
|
|
7
|
+
HotMesh brings a **memory model** to durable functions. Built on PostgreSQL, it treats your database as the runtime hub for agents, pipelines, and long-lived processes.
|
|
8
8
|
|
|
9
9
|
Use HotMesh to:
|
|
10
10
|
|
|
@@ -106,9 +106,9 @@ This index improves performance for filtered queries while reducing index size.
|
|
|
106
106
|
|
|
107
107
|
## Durable AI Agents
|
|
108
108
|
|
|
109
|
-
Agents often require memory—context that persists between invocations, spans multiple perspectives, or outlives a single process.
|
|
109
|
+
Agents often require memory—context that persists between invocations, spans multiple perspectives, or outlives a single process.
|
|
110
110
|
|
|
111
|
-
The following example builds a "research agent" that
|
|
111
|
+
The following example builds a "research agent" that executes hooks with different perspectives and then synthesizes. The data-first approach sets up initial state and then uses temporary hook functions to augment over the lifecycle of the entity record.
|
|
112
112
|
|
|
113
113
|
### Research Agent Example
|
|
114
114
|
|
|
@@ -116,95 +116,58 @@ The following example builds a "research agent" that runs sub-flows and self-ref
|
|
|
116
116
|
|
|
117
117
|
```ts
|
|
118
118
|
export async function researchAgent(query: string): Promise<any> {
|
|
119
|
-
const
|
|
119
|
+
const agent = await MemFlow.workflow.entity();
|
|
120
120
|
|
|
121
121
|
// Set up shared memory for this agent session
|
|
122
|
-
|
|
122
|
+
const initialState = {
|
|
123
123
|
query,
|
|
124
124
|
findings: [],
|
|
125
125
|
perspectives: {},
|
|
126
126
|
confidence: 0,
|
|
127
|
-
|
|
128
|
-
|
|
127
|
+
verification: {},
|
|
128
|
+
status: 'researching',
|
|
129
|
+
startTime: new Date().toISOString(),
|
|
130
|
+
}
|
|
131
|
+
await agent.set<typeof initialState>(initialState);
|
|
129
132
|
|
|
130
|
-
// Launch perspective hooks
|
|
131
|
-
|
|
133
|
+
// Launch perspective hooks
|
|
134
|
+
await MemFlow.workflow.execHook({
|
|
132
135
|
taskQueue: 'agents',
|
|
133
136
|
workflowName: 'optimisticPerspective',
|
|
134
|
-
args: [query]
|
|
137
|
+
args: [query],
|
|
138
|
+
signalId: 'optimistic-complete'
|
|
135
139
|
});
|
|
136
140
|
|
|
137
|
-
|
|
141
|
+
await MemFlow.workflow.execHook({
|
|
138
142
|
taskQueue: 'agents',
|
|
139
143
|
workflowName: 'skepticalPerspective',
|
|
140
|
-
args: [query]
|
|
144
|
+
args: [query],
|
|
145
|
+
signalId: 'skeptical-complete'
|
|
141
146
|
});
|
|
142
147
|
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
workflowName: 'factCheckAgent',
|
|
147
|
-
workflowId: `fact-check-${Date.now()}`,
|
|
148
|
+
await MemFlow.workflow.execHook({
|
|
149
|
+
taskQueue: 'agents',
|
|
150
|
+
workflowName: 'verificationHook',
|
|
148
151
|
args: [query],
|
|
149
|
-
|
|
152
|
+
signalId: 'verification-complete'
|
|
150
153
|
});
|
|
151
154
|
|
|
152
|
-
// Wait for all views to complete before analyzing
|
|
153
|
-
await Promise.all([optimistic, skeptical, factChecker]);
|
|
154
|
-
|
|
155
|
-
// Final synthesis: aggregate and compare all perspectives
|
|
156
155
|
await MemFlow.workflow.execHook({
|
|
157
156
|
taskQueue: 'perspectives',
|
|
158
157
|
workflowName: 'synthesizePerspectives',
|
|
159
|
-
args: []
|
|
158
|
+
args: [],
|
|
159
|
+
signalId: 'synthesis-complete',
|
|
160
160
|
});
|
|
161
161
|
|
|
162
|
-
return
|
|
163
|
-
|
|
164
|
-
```
|
|
165
|
-
|
|
166
|
-
> 💡 **Developer Note**: A complete implementation of this Research Agent example with tests, OpenAI integration, and multi-perspective analysis can be found in the [test suite](./tests/memflow/agent).
|
|
167
|
-
|
|
168
|
-
#### Hooks: Perspectives
|
|
169
|
-
|
|
170
|
-
```ts
|
|
171
|
-
// Optimistic hook looks for affirming evidence
|
|
172
|
-
export async function optimisticPerspective(query: string, config: {signal: string}): Promise<void> {
|
|
173
|
-
const entity = await MemFlow.workflow.entity();
|
|
174
|
-
const findings = await searchForSupportingEvidence(query);
|
|
175
|
-
await entity.merge({ perspectives: { optimistic: { findings, confidence: 0.8 }}});
|
|
176
|
-
//signal the caller to notify all done
|
|
177
|
-
await MemFlow.workflow.signal(config.signal, {});
|
|
178
|
-
}
|
|
179
|
-
|
|
180
|
-
// Skeptical hook seeks out contradictions and counterpoints
|
|
181
|
-
export async function skepticalPerspective(query: string, config: {signal: string}): Promise<void> {
|
|
182
|
-
const entity = await MemFlow.workflow.entity();
|
|
183
|
-
const counterEvidence = await searchForCounterEvidence(query);
|
|
184
|
-
await entity.merge({ perspectives: { skeptical: { counterEvidence, confidence: 0.6 }}});
|
|
185
|
-
await MemFlow.workflow.signal(config.signal, {});
|
|
162
|
+
// return analysis, verification, and synthesis
|
|
163
|
+
return await agent.get();
|
|
186
164
|
}
|
|
187
165
|
```
|
|
188
166
|
|
|
189
|
-
#### Child Agent: Fact Checker
|
|
190
|
-
|
|
191
|
-
```ts
|
|
192
|
-
// A dedicated child agent with its own entity type and context
|
|
193
|
-
export async function factCheckAgent(query: string): Promise<any> {
|
|
194
|
-
const entity = await MemFlow.workflow.entity();
|
|
195
|
-
await entity.set({ query, sources: [], verifications: [] });
|
|
196
|
-
|
|
197
|
-
await MemFlow.workflow.execHook({
|
|
198
|
-
taskQueue: 'agents',
|
|
199
|
-
workflowName: 'verifySourceCredibility',
|
|
200
|
-
args: [query]
|
|
201
|
-
});
|
|
202
167
|
|
|
203
|
-
|
|
204
|
-
}
|
|
205
|
-
```
|
|
168
|
+
Let's look at one of these hooks in detail - the synthesis hook that combines all perspectives into a final assessment:
|
|
206
169
|
|
|
207
|
-
#### Synthesis
|
|
170
|
+
#### Synthesis Hook
|
|
208
171
|
|
|
209
172
|
```ts
|
|
210
173
|
// Synthesis hook aggregates different viewpoints
|
|
@@ -225,139 +188,94 @@ export async function synthesizePerspectives(config: {signal: string}): Promise<
|
|
|
225
188
|
});
|
|
226
189
|
await MemFlow.workflow.signal(config.signal, {});
|
|
227
190
|
}
|
|
191
|
+
|
|
192
|
+
//other hooks...
|
|
228
193
|
```
|
|
229
194
|
|
|
195
|
+
> 💡 A complete implementation of this Research Agent example with tests, OpenAI integration, and multi-perspective analysis can be found in the [agent test suite](https://github.com/hotmeshio/sdk-typescript/tree/main/tests/memflow/agent).
|
|
196
|
+
|
|
230
197
|
---
|
|
231
198
|
|
|
232
199
|
## Building Pipelines with State
|
|
233
200
|
|
|
234
|
-
HotMesh treats pipelines as long-lived records
|
|
201
|
+
HotMesh treats pipelines as long-lived records. Every pipeline run is stateful, resumable, and traceable. Hooks can be re-run at any time, and can be invoked by external callers. Sleep and run on a cadence to keep the pipeline up to date.
|
|
235
202
|
|
|
236
203
|
### Setup a Data Pipeline
|
|
237
204
|
|
|
238
205
|
```ts
|
|
239
|
-
export async function
|
|
240
|
-
const
|
|
241
|
-
|
|
242
|
-
//
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
}
|
|
255
|
-
}
|
|
256
|
-
```
|
|
257
|
-
|
|
258
|
-
### Orchestration Hook
|
|
259
|
-
|
|
260
|
-
```ts
|
|
261
|
-
export async function runPipeline(repeat = false): Promise<void> {
|
|
262
|
-
do {
|
|
263
|
-
// Perform transformation step
|
|
264
|
-
await MemFlow.workflow.execHook({
|
|
265
|
-
taskQueue: 'transform',
|
|
266
|
-
workflowName: 'cleanData',
|
|
267
|
-
args: []
|
|
268
|
-
});
|
|
269
|
-
|
|
270
|
-
if (repeat) {
|
|
271
|
-
// Schedule next execution
|
|
272
|
-
await MemFlow.workflow.execHook({
|
|
273
|
-
taskQueue: 'scheduler',
|
|
274
|
-
workflowName: 'scheduleRefresh',
|
|
275
|
-
args: []
|
|
276
|
-
});
|
|
277
|
-
}
|
|
278
|
-
} while (repeat)
|
|
279
|
-
}
|
|
280
|
-
|
|
281
|
-
/**
|
|
282
|
-
* Hook to clean and transform data
|
|
283
|
-
*/
|
|
284
|
-
export async function cleanData(signalInfo?: { signal: string }): Promise<void> {
|
|
285
|
-
const entity = await MemFlow.workflow.entity();
|
|
206
|
+
export async function documentProcessingPipeline(): Promise<any> {
|
|
207
|
+
const pipeline = await MemFlow.workflow.entity();
|
|
208
|
+
|
|
209
|
+
// Initialize pipeline state with empty arrays
|
|
210
|
+
const initialState = {
|
|
211
|
+
documentId: `doc-${Date.now()}`,
|
|
212
|
+
status: 'started',
|
|
213
|
+
startTime: new Date().toISOString(),
|
|
214
|
+
imageRefs: [],
|
|
215
|
+
extractedInfo: [],
|
|
216
|
+
validationResults: [],
|
|
217
|
+
finalResult: null,
|
|
218
|
+
processingSteps: [],
|
|
219
|
+
errors: [],
|
|
220
|
+
pageSignals: {}
|
|
221
|
+
};
|
|
286
222
|
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
223
|
+
await pipeline.set<typeof initialState>(initialState);
|
|
224
|
+
|
|
225
|
+
// Step 1: Get list of image file references
|
|
226
|
+
await pipeline.merge({status: 'loading-images'});
|
|
227
|
+
await pipeline.append('processingSteps', 'image-load-started');
|
|
228
|
+
const imageRefs = await activities.loadImagePages();
|
|
229
|
+
if (!imageRefs || imageRefs.length === 0) {
|
|
230
|
+
throw new Error('No image references found');
|
|
231
|
+
}
|
|
232
|
+
await pipeline.merge({imageRefs});
|
|
233
|
+
await pipeline.append('processingSteps', 'image-load-completed');
|
|
292
234
|
|
|
293
|
-
//
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
timestamp: new Date().toISOString()
|
|
297
|
-
});
|
|
235
|
+
// Step 2: Launch processing hooks for each page
|
|
236
|
+
for (const [index, imageRef] of imageRefs.entries()) {
|
|
237
|
+
const pageNumber = index + 1;
|
|
298
238
|
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
239
|
+
await MemFlow.workflow.execHook({
|
|
240
|
+
taskQueue: 'pipeline',
|
|
241
|
+
workflowName: 'pageProcessingHook',
|
|
242
|
+
args: [imageRef, pageNumber, initialState.documentId],
|
|
243
|
+
signalId: `page-${pageNumber}-complete`
|
|
304
244
|
});
|
|
305
|
-
}
|
|
306
|
-
}
|
|
245
|
+
};
|
|
307
246
|
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
// Get refresh interval from policy
|
|
315
|
-
const currentEntity = await entity.get();
|
|
316
|
-
const refreshInterval = currentEntity.pipeline.policy.refreshInterval;
|
|
317
|
-
|
|
318
|
-
// Sleep for the configured interval
|
|
319
|
-
await MemFlow.workflow.sleepFor(refreshInterval);
|
|
320
|
-
|
|
321
|
-
// Update status after sleep
|
|
322
|
-
await entity.merge({
|
|
323
|
-
status: 'ready_for_refresh',
|
|
324
|
-
nextRefreshAt: new Date().toISOString()
|
|
247
|
+
// Step 3: Launch validation hook
|
|
248
|
+
await MemFlow.workflow.execHook({
|
|
249
|
+
taskQueue: 'pipeline',
|
|
250
|
+
workflowName: 'validationHook',
|
|
251
|
+
args: [initialState.documentId],
|
|
252
|
+
signalId: 'validation-complete'
|
|
325
253
|
});
|
|
326
254
|
|
|
327
|
-
//
|
|
328
|
-
await
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
255
|
+
// Step 4: Launch approval hook
|
|
256
|
+
await MemFlow.workflow.execHook({
|
|
257
|
+
taskQueue: 'pipeline',
|
|
258
|
+
workflowName: 'approvalHook',
|
|
259
|
+
args: [initialState.documentId],
|
|
260
|
+
signalId: 'approval-complete',
|
|
332
261
|
});
|
|
333
262
|
|
|
334
|
-
//
|
|
335
|
-
|
|
336
|
-
await MemFlow.workflow.signal(signalInfo.signal, {
|
|
337
|
-
status: 'scheduled',
|
|
338
|
-
nextRefresh: new Date().toISOString()
|
|
339
|
-
});
|
|
340
|
-
}
|
|
341
|
-
}
|
|
342
|
-
```
|
|
343
|
-
|
|
344
|
-
### Trigger from Outside
|
|
345
|
-
|
|
346
|
-
```ts
|
|
347
|
-
// External systems can trigger a single pipeline run
|
|
348
|
-
export async function triggerRefresh() {
|
|
349
|
-
const client = new MemFlow.Client({/*...*/});
|
|
350
|
-
|
|
351
|
-
await client.workflow.hook({
|
|
352
|
-
workflowId: 'pipeline-123',
|
|
263
|
+
// Step 5: Launch notification hook
|
|
264
|
+
await MemFlow.workflow.execHook({
|
|
353
265
|
taskQueue: 'pipeline',
|
|
354
|
-
workflowName: '
|
|
355
|
-
args: []
|
|
266
|
+
workflowName: 'notificationHook',
|
|
267
|
+
args: [initialState.documentId],
|
|
268
|
+
signalId: 'processing-complete',
|
|
356
269
|
});
|
|
270
|
+
|
|
271
|
+
// Step 6: Return final state
|
|
272
|
+
await pipeline.merge({status: 'completed', completedAt: new Date().toISOString()});
|
|
273
|
+
await pipeline.append('processingSteps', 'pipeline-completed');
|
|
274
|
+
return await pipeline.get();
|
|
357
275
|
}
|
|
358
276
|
```
|
|
359
277
|
|
|
360
|
-
> 💡
|
|
278
|
+
> 💡 A complete implementation of this Pipeline example with OpenAI Vision integration, processing hooks, and document workflow automation can be found in the [pipeline test suite](https://github.com/hotmeshio/sdk-typescript/tree/main/tests/memflow/pipeline).
|
|
361
279
|
|
|
362
280
|
---
|
|
363
281
|
|
|
@@ -370,4 +288,4 @@ export async function triggerRefresh() {
|
|
|
370
288
|
|
|
371
289
|
## License
|
|
372
290
|
|
|
373
|
-
Apache 2.0 – See `LICENSE` for details.
|
|
291
|
+
Apache 2.0 with commercial restrictions – See `LICENSE` for details.
|
package/build/index.d.ts
CHANGED
|
@@ -10,8 +10,6 @@ import { WorkerService as Worker } from './services/memflow/worker';
|
|
|
10
10
|
import { WorkflowService as workflow } from './services/memflow/workflow';
|
|
11
11
|
import { WorkflowHandleService as WorkflowHandle } from './services/memflow/handle';
|
|
12
12
|
import { proxyActivities } from './services/memflow/workflow/proxyActivities';
|
|
13
|
-
import { MeshData } from './services/meshdata';
|
|
14
|
-
import { MeshOS } from './services/meshos';
|
|
15
13
|
import * as Errors from './modules/errors';
|
|
16
14
|
import * as Utils from './modules/utils';
|
|
17
15
|
import * as Enums from './modules/enums';
|
|
@@ -22,5 +20,5 @@ import { RedisConnection as ConnectorIORedis } from './services/connector/provid
|
|
|
22
20
|
import { RedisConnection as ConnectorRedis } from './services/connector/providers/redis';
|
|
23
21
|
import { NatsConnection as ConnectorNATS } from './services/connector/providers/nats';
|
|
24
22
|
export { Connector, //factory
|
|
25
|
-
ConnectorIORedis, ConnectorNATS, ConnectorPostgres, ConnectorRedis, HotMesh, HotMeshConfig, MeshCall,
|
|
23
|
+
ConnectorIORedis, ConnectorNATS, ConnectorPostgres, ConnectorRedis, HotMesh, HotMeshConfig, MeshCall, MemFlow, Client, Connection, proxyActivities, Search, Entity, Worker, workflow, WorkflowHandle, Enums, Errors, Utils, KeyStore, };
|
|
26
24
|
export * as Types from './types';
|
package/build/index.js
CHANGED
|
@@ -23,7 +23,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
|
|
|
23
23
|
return result;
|
|
24
24
|
};
|
|
25
25
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
|
-
exports.Types = exports.KeyStore = exports.Utils = exports.Errors = exports.Enums = exports.WorkflowHandle = exports.workflow = exports.Worker = exports.Entity = exports.Search = exports.proxyActivities = exports.Connection = exports.Client = exports.
|
|
26
|
+
exports.Types = exports.KeyStore = exports.Utils = exports.Errors = exports.Enums = exports.WorkflowHandle = exports.workflow = exports.Worker = exports.Entity = exports.Search = exports.proxyActivities = exports.Connection = exports.Client = exports.MemFlow = exports.MeshCall = exports.HotMesh = exports.ConnectorRedis = exports.ConnectorPostgres = exports.ConnectorNATS = exports.ConnectorIORedis = exports.Connector = void 0;
|
|
27
27
|
const hotmesh_1 = require("./services/hotmesh");
|
|
28
28
|
Object.defineProperty(exports, "HotMesh", { enumerable: true, get: function () { return hotmesh_1.HotMesh; } });
|
|
29
29
|
const meshcall_1 = require("./services/meshcall");
|
|
@@ -46,10 +46,6 @@ const handle_1 = require("./services/memflow/handle");
|
|
|
46
46
|
Object.defineProperty(exports, "WorkflowHandle", { enumerable: true, get: function () { return handle_1.WorkflowHandleService; } });
|
|
47
47
|
const proxyActivities_1 = require("./services/memflow/workflow/proxyActivities");
|
|
48
48
|
Object.defineProperty(exports, "proxyActivities", { enumerable: true, get: function () { return proxyActivities_1.proxyActivities; } });
|
|
49
|
-
const meshdata_1 = require("./services/meshdata");
|
|
50
|
-
Object.defineProperty(exports, "MeshData", { enumerable: true, get: function () { return meshdata_1.MeshData; } });
|
|
51
|
-
const meshos_1 = require("./services/meshos");
|
|
52
|
-
Object.defineProperty(exports, "MeshOS", { enumerable: true, get: function () { return meshos_1.MeshOS; } });
|
|
53
49
|
const Errors = __importStar(require("./modules/errors"));
|
|
54
50
|
exports.Errors = Errors;
|
|
55
51
|
const Utils = __importStar(require("./modules/utils"));
|
package/build/modules/enums.d.ts
CHANGED
|
@@ -108,3 +108,7 @@ export declare const HMSH_EXPIRE_DURATION: number;
|
|
|
108
108
|
export declare const HMSH_FIDELITY_SECONDS: number;
|
|
109
109
|
export declare const HMSH_SCOUT_INTERVAL_SECONDS: number;
|
|
110
110
|
export declare const HMSH_GUID_SIZE: number;
|
|
111
|
+
/**
|
|
112
|
+
* Default task queue name used when no task queue is specified
|
|
113
|
+
*/
|
|
114
|
+
export declare const DEFAULT_TASK_QUEUE = "default";
|
package/build/modules/enums.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.HMSH_GUID_SIZE = exports.HMSH_SCOUT_INTERVAL_SECONDS = exports.HMSH_FIDELITY_SECONDS = exports.HMSH_EXPIRE_DURATION = exports.HMSH_XPENDING_COUNT = exports.HMSH_XCLAIM_COUNT = exports.HMSH_XCLAIM_DELAY_MS = exports.HMSH_BLOCK_TIME_MS = exports.HMSH_MEMFLOW_EXP_BACKOFF = exports.HMSH_MEMFLOW_MAX_INTERVAL = exports.HMSH_MEMFLOW_MAX_ATTEMPTS = exports.HMSH_GRADUATED_INTERVAL_MS = exports.HMSH_MAX_TIMEOUT_MS = exports.HMSH_MAX_RETRIES = exports.MAX_DELAY = exports.MAX_STREAM_RETRIES = exports.INITIAL_STREAM_BACKOFF = exports.MAX_STREAM_BACKOFF = exports.HMSH_EXPIRE_JOB_SECONDS = exports.HMSH_OTT_WAIT_TIME = exports.HMSH_DEPLOYMENT_PAUSE = exports.HMSH_DEPLOYMENT_DELAY = exports.HMSH_ACTIVATION_MAX_RETRY = exports.HMSH_QUORUM_DELAY_MS = exports.HMSH_QUORUM_ROLLCALL_CYCLES = exports.HMSH_STATUS_UNKNOWN = exports.HMSH_CODE_MEMFLOW_RETRYABLE = exports.HMSH_CODE_MEMFLOW_FATAL = exports.HMSH_CODE_MEMFLOW_MAXED = exports.HMSH_CODE_MEMFLOW_TIMEOUT = exports.HMSH_CODE_MEMFLOW_WAIT = exports.HMSH_CODE_MEMFLOW_PROXY = exports.HMSH_CODE_MEMFLOW_CHILD = exports.HMSH_CODE_MEMFLOW_ALL = exports.HMSH_CODE_MEMFLOW_SLEEP = exports.HMSH_CODE_UNACKED = exports.HMSH_CODE_TIMEOUT = exports.HMSH_CODE_UNKNOWN = exports.HMSH_CODE_INTERRUPT = exports.HMSH_CODE_NOTFOUND = exports.HMSH_CODE_PENDING = exports.HMSH_CODE_SUCCESS = exports.HMSH_SIGNAL_EXPIRE = exports.HMSH_IS_CLUSTER = exports.HMSH_TELEMETRY = exports.HMSH_LOGLEVEL = void 0;
|
|
3
|
+
exports.DEFAULT_TASK_QUEUE = exports.HMSH_GUID_SIZE = exports.HMSH_SCOUT_INTERVAL_SECONDS = exports.HMSH_FIDELITY_SECONDS = exports.HMSH_EXPIRE_DURATION = exports.HMSH_XPENDING_COUNT = exports.HMSH_XCLAIM_COUNT = exports.HMSH_XCLAIM_DELAY_MS = exports.HMSH_BLOCK_TIME_MS = exports.HMSH_MEMFLOW_EXP_BACKOFF = exports.HMSH_MEMFLOW_MAX_INTERVAL = exports.HMSH_MEMFLOW_MAX_ATTEMPTS = exports.HMSH_GRADUATED_INTERVAL_MS = exports.HMSH_MAX_TIMEOUT_MS = exports.HMSH_MAX_RETRIES = exports.MAX_DELAY = exports.MAX_STREAM_RETRIES = exports.INITIAL_STREAM_BACKOFF = exports.MAX_STREAM_BACKOFF = exports.HMSH_EXPIRE_JOB_SECONDS = exports.HMSH_OTT_WAIT_TIME = exports.HMSH_DEPLOYMENT_PAUSE = exports.HMSH_DEPLOYMENT_DELAY = exports.HMSH_ACTIVATION_MAX_RETRY = exports.HMSH_QUORUM_DELAY_MS = exports.HMSH_QUORUM_ROLLCALL_CYCLES = exports.HMSH_STATUS_UNKNOWN = exports.HMSH_CODE_MEMFLOW_RETRYABLE = exports.HMSH_CODE_MEMFLOW_FATAL = exports.HMSH_CODE_MEMFLOW_MAXED = exports.HMSH_CODE_MEMFLOW_TIMEOUT = exports.HMSH_CODE_MEMFLOW_WAIT = exports.HMSH_CODE_MEMFLOW_PROXY = exports.HMSH_CODE_MEMFLOW_CHILD = exports.HMSH_CODE_MEMFLOW_ALL = exports.HMSH_CODE_MEMFLOW_SLEEP = exports.HMSH_CODE_UNACKED = exports.HMSH_CODE_TIMEOUT = exports.HMSH_CODE_UNKNOWN = exports.HMSH_CODE_INTERRUPT = exports.HMSH_CODE_NOTFOUND = exports.HMSH_CODE_PENDING = exports.HMSH_CODE_SUCCESS = exports.HMSH_SIGNAL_EXPIRE = exports.HMSH_IS_CLUSTER = exports.HMSH_TELEMETRY = exports.HMSH_LOGLEVEL = void 0;
|
|
4
4
|
/**
|
|
5
5
|
* Determines the log level for the application. The default is 'info'.
|
|
6
6
|
*/
|
|
@@ -132,3 +132,7 @@ exports.HMSH_FIDELITY_SECONDS = process.env.HMSH_FIDELITY_SECONDS
|
|
|
132
132
|
exports.HMSH_SCOUT_INTERVAL_SECONDS = parseInt(process.env.HMSH_SCOUT_INTERVAL_SECONDS, 10) || 60;
|
|
133
133
|
// UTILS
|
|
134
134
|
exports.HMSH_GUID_SIZE = Math.min(parseInt(process.env.HMSH_GUID_SIZE, 10) || 22, 32);
|
|
135
|
+
/**
|
|
136
|
+
* Default task queue name used when no task queue is specified
|
|
137
|
+
*/
|
|
138
|
+
exports.DEFAULT_TASK_QUEUE = 'default';
|
package/build/modules/utils.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
2
|
import { StoreService } from '../services/store';
|
|
3
3
|
import { AppSubscriptions, AppTransitions, AppVID } from '../types/app';
|
|
4
|
-
import { ProviderClient,
|
|
4
|
+
import { ProviderClient, ProviderTransaction, Providers } from '../types/provider';
|
|
5
5
|
import { StringAnyType } from '../types/serializer';
|
|
6
6
|
import { StreamCode, StreamData, StreamStatus } from '../types/stream';
|
|
7
7
|
import { SystemHealth } from '../types/quorum';
|
|
@@ -34,14 +34,6 @@ export declare const polyfill: {
|
|
|
34
34
|
* `redis` is deprecated; `connection` is the generic replacement
|
|
35
35
|
*/
|
|
36
36
|
providerConfig(obj: any): any;
|
|
37
|
-
/**
|
|
38
|
-
* NOTE: `redisClass and redisOptions` input parameters are deprecated; use `connection` for all configuration inputs
|
|
39
|
-
*/
|
|
40
|
-
meshDataConfig(obj: {
|
|
41
|
-
connection?: Partial<ProviderConfig | ProvidersConfig>;
|
|
42
|
-
redisClass?: any;
|
|
43
|
-
redisOptions?: StringAnyType;
|
|
44
|
-
}): Partial<ProviderConfig> | Partial<ProvidersConfig>;
|
|
45
37
|
};
|
|
46
38
|
/**
|
|
47
39
|
* @private
|
package/build/modules/utils.js
CHANGED
|
@@ -127,12 +127,6 @@ exports.polyfill = {
|
|
|
127
127
|
providerConfig(obj) {
|
|
128
128
|
return obj?.connection ?? obj?.redis ?? obj?.connections;
|
|
129
129
|
},
|
|
130
|
-
/**
|
|
131
|
-
* NOTE: `redisClass and redisOptions` input parameters are deprecated; use `connection` for all configuration inputs
|
|
132
|
-
*/
|
|
133
|
-
meshDataConfig(obj) {
|
|
134
|
-
return { ...obj.connection };
|
|
135
|
-
},
|
|
136
130
|
};
|
|
137
131
|
/**
|
|
138
132
|
* @private
|
package/build/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hotmeshio/hotmesh",
|
|
3
|
-
"version": "0.5.
|
|
3
|
+
"version": "0.5.4",
|
|
4
4
|
"description": "Permanent-Memory Workflows & AI Agents",
|
|
5
5
|
"main": "./build/index.js",
|
|
6
6
|
"types": "./build/index.d.ts",
|
|
@@ -20,7 +20,7 @@
|
|
|
20
20
|
"lint:fix": "eslint . --fix --ext .ts",
|
|
21
21
|
"start": "ts-node src/index.ts",
|
|
22
22
|
"test": "NODE_ENV=test jest --detectOpenHandles --forceExit --verbose",
|
|
23
|
-
"test:await": "NODE_ENV=test jest ./tests/functional/awaiter
|
|
23
|
+
"test:await": "NODE_ENV=test jest ./tests/functional/awaiter/postgres.test.ts --detectOpenHandles --forceExit --verbose",
|
|
24
24
|
"test:compile": "NODE_ENV=test jest ./tests/functional/compile/index.test.ts --detectOpenHandles --forceExit --verbose",
|
|
25
25
|
"test:connect": "NODE_ENV=test jest ./tests/unit/services/connector/* --detectOpenHandles --forceExit --verbose",
|
|
26
26
|
"test:connect:ioredis": "NODE_ENV=test jest ./tests/unit/services/connector/providers/ioredis.test.ts --detectOpenHandles --forceExit --verbose",
|
|
@@ -59,7 +59,7 @@
|
|
|
59
59
|
"test:reporter": "NODE_ENV=test jest ./tests/unit/services/reporter/index.test.ts --detectOpenHandles --forceExit --verbose",
|
|
60
60
|
"test:reentrant": "NODE_ENV=test jest ./tests/functional/reentrant/*.test.ts --detectOpenHandles --forceExit --verbose",
|
|
61
61
|
"test:retry": "NODE_ENV=test jest ./tests/functional/retry/*.test.ts --detectOpenHandles --forceExit --verbose",
|
|
62
|
-
"test:sequence": "NODE_ENV=test HMSH_LOGLEVEL=
|
|
62
|
+
"test:sequence": "NODE_ENV=test HMSH_LOGLEVEL=debug jest ./tests/functional/sequence/postgres.test.ts --detectOpenHandles --forceExit --verbose",
|
|
63
63
|
"test:signal": "NODE_ENV=test jest ./tests/functional/signal/*.test.ts --detectOpenHandles --forceExit --verbose",
|
|
64
64
|
"test:status": "NODE_ENV=test jest ./tests/functional/status/index.test.ts --detectOpenHandles --forceExit --verbose",
|
|
65
65
|
"test:providers": "NODE_ENV=test jest ./tests/functional/*/providers/*/*.test.ts --detectOpenHandles --forceExit --verbose",
|
|
@@ -75,7 +75,6 @@
|
|
|
75
75
|
"test:sub:postgres": "NODE_ENV=test jest ./tests/functional/sub/providers/postgres/postgres.test.ts --detectOpenHandles --forceExit --verbose",
|
|
76
76
|
"test:sub:nats": "NODE_ENV=test jest ./tests/functional/sub/providers/nats/nats.test.ts --detectOpenHandles --forceExit --verbose",
|
|
77
77
|
"test:trigger": "NODE_ENV=test jest ./tests/unit/services/activities/trigger.test.ts --detectOpenHandles --forceExit --verbose",
|
|
78
|
-
"test:meshos": "HMSH_LOGLEVEL=info NODE_ENV=test HMSH_IS_CLUSTER=true jest ./tests/meshos/*.test.ts --forceExit --verbose --detectOpenHandles",
|
|
79
78
|
"test:meshcall": "NODE_ENV=test jest ./tests/meshcall/*.test.ts --forceExit --verbose --detectOpenHandles",
|
|
80
79
|
"test:unit": "NODE_ENV=test jest ./tests/unit/*/*/index.test.ts --detectOpenHandles --forceExit --verbose"
|
|
81
80
|
},
|
|
@@ -8,7 +8,7 @@ export declare class ConnectorService {
|
|
|
8
8
|
* initialization, but the factory method provided here is useful
|
|
9
9
|
* for testing provider configurations.
|
|
10
10
|
*/
|
|
11
|
-
static connectClient(ProviderConfig: ProviderConfig): Promise<ProviderNativeClient>;
|
|
11
|
+
static connectClient(ProviderConfig: ProviderConfig, taskQueue?: string): Promise<ProviderNativeClient>;
|
|
12
12
|
/**
|
|
13
13
|
* Initialize `store`, `stream`, and `subscription` clients for any provider.
|
|
14
14
|
* @private
|
|
@@ -18,5 +18,5 @@ export declare class ConnectorService {
|
|
|
18
18
|
* Binds a provider client native instance to the target object.
|
|
19
19
|
* @private
|
|
20
20
|
*/
|
|
21
|
-
static initClient(ProviderConfig: ProviderConfig, target: HotMeshEngine | HotMeshWorker, field: string): Promise<void>;
|
|
21
|
+
static initClient(ProviderConfig: ProviderConfig, target: HotMeshEngine | HotMeshWorker, field: string, taskQueue?: string): Promise<void>;
|
|
22
22
|
}
|
|
@@ -19,9 +19,9 @@ class ConnectorService {
|
|
|
19
19
|
* initialization, but the factory method provided here is useful
|
|
20
20
|
* for testing provider configurations.
|
|
21
21
|
*/
|
|
22
|
-
static async connectClient(ProviderConfig) {
|
|
22
|
+
static async connectClient(ProviderConfig, taskQueue) {
|
|
23
23
|
const target = {};
|
|
24
|
-
await ConnectorService.initClient(ProviderConfig, target, 'client');
|
|
24
|
+
await ConnectorService.initClient(ProviderConfig, target, 'client', taskQueue);
|
|
25
25
|
return target.client;
|
|
26
26
|
}
|
|
27
27
|
/**
|
|
@@ -38,15 +38,17 @@ class ConnectorService {
|
|
|
38
38
|
sub: { ...connection },
|
|
39
39
|
};
|
|
40
40
|
}
|
|
41
|
+
// Extract taskQueue from target for connection pooling
|
|
42
|
+
const taskQueue = target.taskQueue;
|
|
41
43
|
// Expanded form
|
|
42
44
|
if (connection.store) {
|
|
43
|
-
await ConnectorService.initClient(connection.store, target, 'store');
|
|
45
|
+
await ConnectorService.initClient(connection.store, target, 'store', taskQueue);
|
|
44
46
|
}
|
|
45
47
|
if (connection.stream) {
|
|
46
|
-
await ConnectorService.initClient(connection.stream, target, 'stream');
|
|
48
|
+
await ConnectorService.initClient(connection.stream, target, 'stream', taskQueue);
|
|
47
49
|
}
|
|
48
50
|
if (connection.sub) {
|
|
49
|
-
await ConnectorService.initClient(connection.sub, target, 'sub');
|
|
51
|
+
await ConnectorService.initClient(connection.sub, target, 'sub', taskQueue);
|
|
50
52
|
// use store for publishing events if same as subscription
|
|
51
53
|
if (connection.sub.class !== connection.store.class) {
|
|
52
54
|
//initialize a separate client for publishing events, using
|
|
@@ -56,7 +58,7 @@ class ConnectorService {
|
|
|
56
58
|
options: { ...connection.sub.options },
|
|
57
59
|
provider: connection.sub.provider,
|
|
58
60
|
};
|
|
59
|
-
await ConnectorService.initClient(connection.pub, target, 'pub');
|
|
61
|
+
await ConnectorService.initClient(connection.pub, target, 'pub', taskQueue);
|
|
60
62
|
}
|
|
61
63
|
}
|
|
62
64
|
}
|
|
@@ -64,7 +66,7 @@ class ConnectorService {
|
|
|
64
66
|
* Binds a provider client native instance to the target object.
|
|
65
67
|
* @private
|
|
66
68
|
*/
|
|
67
|
-
static async initClient(ProviderConfig, target, field) {
|
|
69
|
+
static async initClient(ProviderConfig, target, field, taskQueue) {
|
|
68
70
|
if (target[field]) {
|
|
69
71
|
return;
|
|
70
72
|
}
|
|
@@ -87,7 +89,8 @@ class ConnectorService {
|
|
|
87
89
|
case 'postgres':
|
|
88
90
|
//if connecting as a poolClient for subscription, auto connect the client
|
|
89
91
|
const bAutoConnect = field === 'sub';
|
|
90
|
-
|
|
92
|
+
// Use taskQueue-based connection pooling for PostgreSQL
|
|
93
|
+
clientInstance = await postgres_1.PostgresConnection.getOrCreateTaskQueueConnection(id, taskQueue, providerClass, options, { connect: bAutoConnect, provider: providerName });
|
|
91
94
|
break;
|
|
92
95
|
default:
|
|
93
96
|
throw new Error(`Unknown provider type: ${providerType}`);
|
|
@@ -4,17 +4,64 @@ declare class PostgresConnection extends AbstractConnection<PostgresClassType, P
|
|
|
4
4
|
defaultOptions: PostgresClientOptions;
|
|
5
5
|
protected static poolClientInstances: Set<PostgresPoolClientType>;
|
|
6
6
|
protected static connectionInstances: Set<PostgresClientType>;
|
|
7
|
+
protected static taskQueueConnections: Map<string, PostgresConnection>;
|
|
8
|
+
/**
|
|
9
|
+
* Get comprehensive connection statistics for monitoring taskQueue pooling effectiveness
|
|
10
|
+
*/
|
|
11
|
+
static getConnectionStats(): {
|
|
12
|
+
totalPoolClients: number;
|
|
13
|
+
totalConnections: number;
|
|
14
|
+
taskQueueConnections: number;
|
|
15
|
+
taskQueueDetails: Array<{
|
|
16
|
+
key: string;
|
|
17
|
+
connectionId: string;
|
|
18
|
+
reusedCount: number;
|
|
19
|
+
}>;
|
|
20
|
+
};
|
|
21
|
+
/**
|
|
22
|
+
* Log current connection statistics - useful for debugging connection pooling
|
|
23
|
+
*/
|
|
24
|
+
static logConnectionStats(logger?: any): void;
|
|
25
|
+
/**
|
|
26
|
+
* Check taskQueue pooling effectiveness - returns metrics about connection reuse
|
|
27
|
+
*/
|
|
28
|
+
static getPoolingEffectiveness(): {
|
|
29
|
+
totalConnections: number;
|
|
30
|
+
taskQueuePools: number;
|
|
31
|
+
totalReuses: number;
|
|
32
|
+
averageReusesPerPool: number;
|
|
33
|
+
poolingEfficiency: number;
|
|
34
|
+
};
|
|
7
35
|
poolClientInstance: PostgresPoolClientType;
|
|
8
36
|
createConnection(clientConstructor: any, options: PostgresClientOptions, config?: {
|
|
9
37
|
connect?: boolean;
|
|
10
38
|
provider?: string;
|
|
11
39
|
}): Promise<PostgresClientType>;
|
|
12
40
|
getClient(): PostgresClientType;
|
|
41
|
+
/**
|
|
42
|
+
* Get the connection ID for monitoring purposes
|
|
43
|
+
*/
|
|
44
|
+
getConnectionId(): string | null;
|
|
13
45
|
static disconnectAll(): Promise<void>;
|
|
14
46
|
static disconnectPoolClients(): Promise<void>;
|
|
15
47
|
static disconnectConnections(): Promise<void>;
|
|
16
48
|
closeConnection(connection: PostgresClientType): Promise<void>;
|
|
17
49
|
static isPoolClient(client: any): client is PostgresPoolClientType;
|
|
50
|
+
/**
|
|
51
|
+
* Creates a taskQueue-based connection key for connection pooling.
|
|
52
|
+
* This allows multiple providers (store, sub, stream) to reuse the same connection
|
|
53
|
+
* when they share the same taskQueue and database configuration.
|
|
54
|
+
*/
|
|
55
|
+
private static createTaskQueueConnectionKey;
|
|
56
|
+
/**
|
|
57
|
+
* Gets or creates a PostgreSQL connection based on taskQueue and database configuration.
|
|
58
|
+
* If a connection already exists for the same taskQueue + config, it will be reused.
|
|
59
|
+
* This optimization reduces connection overhead for PostgreSQL providers.
|
|
60
|
+
*/
|
|
61
|
+
static getOrCreateTaskQueueConnection(id: string, taskQueue: string | undefined, clientConstructor: PostgresClassType, options: PostgresClientOptions, config?: {
|
|
62
|
+
connect?: boolean;
|
|
63
|
+
provider?: string;
|
|
64
|
+
}): Promise<PostgresConnection>;
|
|
18
65
|
static getTransactionClient(transactionClient: any): Promise<['client' | 'poolclient', PostgresClientType]>;
|
|
19
66
|
}
|
|
20
67
|
export { PostgresConnection };
|