@hotmeshio/hotmesh 0.5.3 → 0.5.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +67 -134
- package/build/index.d.ts +1 -3
- package/build/index.js +1 -5
- package/build/modules/enums.d.ts +4 -0
- package/build/modules/enums.js +5 -1
- package/build/modules/utils.d.ts +1 -9
- package/build/modules/utils.js +0 -6
- package/build/package.json +3 -4
- package/build/services/connector/factory.d.ts +2 -2
- package/build/services/connector/factory.js +11 -8
- package/build/services/connector/providers/postgres.d.ts +47 -0
- package/build/services/connector/providers/postgres.js +107 -0
- package/build/services/hotmesh/index.d.ts +8 -0
- package/build/services/hotmesh/index.js +27 -0
- package/build/services/memflow/client.d.ts +1 -1
- package/build/services/memflow/client.js +8 -6
- package/build/services/memflow/worker.js +3 -0
- package/build/services/pipe/functions/cron.js +1 -1
- package/build/services/store/providers/postgres/kvtables.js +19 -6
- package/build/services/store/providers/postgres/postgres.js +13 -2
- package/build/services/stream/providers/postgres/postgres.d.ts +6 -3
- package/build/services/stream/providers/postgres/postgres.js +169 -59
- package/build/services/sub/providers/postgres/postgres.d.ts +9 -0
- package/build/services/sub/providers/postgres/postgres.js +109 -18
- package/build/services/worker/index.js +4 -0
- package/build/types/hotmesh.d.ts +19 -5
- package/build/types/index.d.ts +0 -2
- package/env.example +11 -0
- package/index.ts +0 -4
- package/package.json +3 -4
- package/build/services/meshdata/index.d.ts +0 -795
- package/build/services/meshdata/index.js +0 -1235
- package/build/services/meshos/index.d.ts +0 -293
- package/build/services/meshos/index.js +0 -547
- package/build/types/manifest.d.ts +0 -52
- package/build/types/manifest.js +0 -2
- package/build/types/meshdata.d.ts +0 -252
- package/build/types/meshdata.js +0 -2
package/README.md
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
# HotMesh
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
**Workflow That Remembers**
|
|
4
4
|
|
|
5
5
|
 
|
|
6
6
|
|
|
7
|
-
HotMesh brings a **memory model** to
|
|
7
|
+
HotMesh brings a **memory model** to durable functions. Built on PostgreSQL, it treats your database as the runtime hub for agents, pipelines, and long-lived processes.
|
|
8
8
|
|
|
9
9
|
Use HotMesh to:
|
|
10
10
|
|
|
@@ -130,7 +130,7 @@ export async function researchAgent(query: string): Promise<any> {
|
|
|
130
130
|
}
|
|
131
131
|
await agent.set<typeof initialState>(initialState);
|
|
132
132
|
|
|
133
|
-
// Launch perspective hooks
|
|
133
|
+
// Launch perspective hooks
|
|
134
134
|
await MemFlow.workflow.execHook({
|
|
135
135
|
taskQueue: 'agents',
|
|
136
136
|
workflowName: 'optimisticPerspective',
|
|
@@ -164,30 +164,8 @@ export async function researchAgent(query: string): Promise<any> {
|
|
|
164
164
|
}
|
|
165
165
|
```
|
|
166
166
|
|
|
167
|
-
> 💡 A complete implementation of this Research Agent example with tests, OpenAI integration, and multi-perspective analysis can be found in the [agent test suite](https://github.com/hotmeshio/sdk-typescript/tree/main/tests/memflow/agent).
|
|
168
|
-
|
|
169
|
-
#### Perspective Hooks
|
|
170
|
-
|
|
171
|
-
```ts
|
|
172
|
-
// Optimistic hook looks for affirming evidence
|
|
173
|
-
export async function optimisticPerspective(query: string, config: {signal: string}): Promise<void> {
|
|
174
|
-
const entity = await MemFlow.workflow.entity();
|
|
175
|
-
const findings = await searchForSupportingEvidence(query);
|
|
176
|
-
await entity.merge({ perspectives: { optimistic: { findings, confidence: 0.8 }}});
|
|
177
|
-
//signal the caller to notify all done
|
|
178
|
-
await MemFlow.workflow.signal(config.signal, {});
|
|
179
|
-
}
|
|
180
|
-
|
|
181
|
-
// Skeptical hook seeks out contradictions and counterpoints
|
|
182
|
-
export async function skepticalPerspective(query: string, config: {signal: string}): Promise<void> {
|
|
183
|
-
const entity = await MemFlow.workflow.entity();
|
|
184
|
-
const counterEvidence = await searchForCounterEvidence(query);
|
|
185
|
-
await entity.merge({ perspectives: { skeptical: { counterEvidence, confidence: 0.6 }}});
|
|
186
|
-
await MemFlow.workflow.signal(config.signal, {});
|
|
187
|
-
}
|
|
188
167
|
|
|
189
|
-
|
|
190
|
-
```
|
|
168
|
+
Let's look at one of these hooks in detail - the synthesis hook that combines all perspectives into a final assessment:
|
|
191
169
|
|
|
192
170
|
#### Synthesis Hook
|
|
193
171
|
|
|
@@ -210,135 +188,90 @@ export async function synthesizePerspectives(config: {signal: string}): Promise<
|
|
|
210
188
|
});
|
|
211
189
|
await MemFlow.workflow.signal(config.signal, {});
|
|
212
190
|
}
|
|
191
|
+
|
|
192
|
+
//other hooks...
|
|
213
193
|
```
|
|
214
194
|
|
|
195
|
+
> 💡 A complete implementation of this Research Agent example with tests, OpenAI integration, and multi-perspective analysis can be found in the [agent test suite](https://github.com/hotmeshio/sdk-typescript/tree/main/tests/memflow/agent).
|
|
196
|
+
|
|
215
197
|
---
|
|
216
198
|
|
|
217
199
|
## Building Pipelines with State
|
|
218
200
|
|
|
219
|
-
HotMesh treats pipelines as long-lived records. Every pipeline run is stateful, resumable, and traceable.
|
|
201
|
+
HotMesh treats pipelines as long-lived records. Every pipeline run is stateful, resumable, and traceable. Hooks can be re-run at any time, and can be invoked by external callers. Sleep and run on a cadence to keep the pipeline up to date.
|
|
220
202
|
|
|
221
203
|
### Setup a Data Pipeline
|
|
222
204
|
|
|
223
205
|
```ts
|
|
224
|
-
export async function
|
|
225
|
-
const
|
|
206
|
+
export async function documentProcessingPipeline(): Promise<any> {
|
|
207
|
+
const pipeline = await MemFlow.workflow.entity();
|
|
226
208
|
|
|
227
|
-
//
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
}
|
|
240
|
-
}
|
|
241
|
-
```
|
|
242
|
-
|
|
243
|
-
### Orchestration Hook
|
|
244
|
-
|
|
245
|
-
```ts
|
|
246
|
-
export async function runPipeline(repeat = false): Promise<void> {
|
|
247
|
-
do {
|
|
248
|
-
// Perform transformation step
|
|
249
|
-
await MemFlow.workflow.execHook({
|
|
250
|
-
taskQueue: 'transform',
|
|
251
|
-
workflowName: 'cleanData',
|
|
252
|
-
args: []
|
|
253
|
-
});
|
|
254
|
-
|
|
255
|
-
if (repeat) {
|
|
256
|
-
// Schedule next execution
|
|
257
|
-
await MemFlow.workflow.execHook({
|
|
258
|
-
taskQueue: 'scheduler',
|
|
259
|
-
workflowName: 'scheduleRefresh',
|
|
260
|
-
args: []
|
|
261
|
-
});
|
|
262
|
-
}
|
|
263
|
-
} while (repeat)
|
|
264
|
-
}
|
|
265
|
-
|
|
266
|
-
/**
|
|
267
|
-
* Hook to clean and transform data
|
|
268
|
-
*/
|
|
269
|
-
export async function cleanData(signalInfo?: { signal: string }): Promise<void> {
|
|
270
|
-
const entity = await MemFlow.workflow.entity();
|
|
209
|
+
// Initialize pipeline state with empty arrays
|
|
210
|
+
const initialState = {
|
|
211
|
+
documentId: `doc-${Date.now()}`,
|
|
212
|
+
status: 'started',
|
|
213
|
+
startTime: new Date().toISOString(),
|
|
214
|
+
imageRefs: [],
|
|
215
|
+
extractedInfo: [],
|
|
216
|
+
validationResults: [],
|
|
217
|
+
finalResult: null,
|
|
218
|
+
processingSteps: [],
|
|
219
|
+
errors: [],
|
|
220
|
+
pageSignals: {}
|
|
221
|
+
};
|
|
271
222
|
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
223
|
+
await pipeline.set<typeof initialState>(initialState);
|
|
224
|
+
|
|
225
|
+
// Step 1: Get list of image file references
|
|
226
|
+
await pipeline.merge({status: 'loading-images'});
|
|
227
|
+
await pipeline.append('processingSteps', 'image-load-started');
|
|
228
|
+
const imageRefs = await activities.loadImagePages();
|
|
229
|
+
if (!imageRefs || imageRefs.length === 0) {
|
|
230
|
+
throw new Error('No image references found');
|
|
231
|
+
}
|
|
232
|
+
await pipeline.merge({imageRefs});
|
|
233
|
+
await pipeline.append('processingSteps', 'image-load-completed');
|
|
277
234
|
|
|
278
|
-
//
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
timestamp: new Date().toISOString()
|
|
282
|
-
});
|
|
235
|
+
// Step 2: Launch processing hooks for each page
|
|
236
|
+
for (const [index, imageRef] of imageRefs.entries()) {
|
|
237
|
+
const pageNumber = index + 1;
|
|
283
238
|
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
239
|
+
await MemFlow.workflow.execHook({
|
|
240
|
+
taskQueue: 'pipeline',
|
|
241
|
+
workflowName: 'pageProcessingHook',
|
|
242
|
+
args: [imageRef, pageNumber, initialState.documentId],
|
|
243
|
+
signalId: `page-${pageNumber}-complete`
|
|
289
244
|
});
|
|
290
|
-
}
|
|
291
|
-
}
|
|
245
|
+
};
|
|
292
246
|
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
// Get refresh interval from policy
|
|
300
|
-
const currentEntity = await entity.get();
|
|
301
|
-
const refreshInterval = currentEntity.pipeline.policy.refreshInterval;
|
|
302
|
-
|
|
303
|
-
// Sleep for the configured interval
|
|
304
|
-
await MemFlow.workflow.sleepFor(refreshInterval);
|
|
305
|
-
|
|
306
|
-
// Update status after sleep
|
|
307
|
-
await entity.merge({
|
|
308
|
-
status: 'ready_for_refresh',
|
|
309
|
-
nextRefreshAt: new Date().toISOString()
|
|
247
|
+
// Step 3: Launch validation hook
|
|
248
|
+
await MemFlow.workflow.execHook({
|
|
249
|
+
taskQueue: 'pipeline',
|
|
250
|
+
workflowName: 'validationHook',
|
|
251
|
+
args: [initialState.documentId],
|
|
252
|
+
signalId: 'validation-complete'
|
|
310
253
|
});
|
|
311
254
|
|
|
312
|
-
//
|
|
313
|
-
await
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
255
|
+
// Step 4: Launch approval hook
|
|
256
|
+
await MemFlow.workflow.execHook({
|
|
257
|
+
taskQueue: 'pipeline',
|
|
258
|
+
workflowName: 'approvalHook',
|
|
259
|
+
args: [initialState.documentId],
|
|
260
|
+
signalId: 'approval-complete',
|
|
317
261
|
});
|
|
318
262
|
|
|
319
|
-
//
|
|
320
|
-
|
|
321
|
-
await MemFlow.workflow.signal(signalInfo.signal, {
|
|
322
|
-
status: 'scheduled',
|
|
323
|
-
nextRefresh: new Date().toISOString()
|
|
324
|
-
});
|
|
325
|
-
}
|
|
326
|
-
}
|
|
327
|
-
```
|
|
328
|
-
|
|
329
|
-
### Trigger from Outside
|
|
330
|
-
|
|
331
|
-
```ts
|
|
332
|
-
// External systems can trigger a single pipeline run
|
|
333
|
-
export async function triggerRefresh() {
|
|
334
|
-
const client = new MemFlow.Client({/*...*/});
|
|
335
|
-
|
|
336
|
-
await client.workflow.hook({
|
|
337
|
-
workflowId: 'pipeline-123',
|
|
263
|
+
// Step 5: Launch notification hook
|
|
264
|
+
await MemFlow.workflow.execHook({
|
|
338
265
|
taskQueue: 'pipeline',
|
|
339
|
-
workflowName: '
|
|
340
|
-
args: []
|
|
266
|
+
workflowName: 'notificationHook',
|
|
267
|
+
args: [initialState.documentId],
|
|
268
|
+
signalId: 'processing-complete',
|
|
341
269
|
});
|
|
270
|
+
|
|
271
|
+
// Step 6: Return final state
|
|
272
|
+
await pipeline.merge({status: 'completed', completedAt: new Date().toISOString()});
|
|
273
|
+
await pipeline.append('processingSteps', 'pipeline-completed');
|
|
274
|
+
return await pipeline.get();
|
|
342
275
|
}
|
|
343
276
|
```
|
|
344
277
|
|
|
@@ -355,4 +288,4 @@ export async function triggerRefresh() {
|
|
|
355
288
|
|
|
356
289
|
## License
|
|
357
290
|
|
|
358
|
-
Apache 2.0 – See `LICENSE` for details.
|
|
291
|
+
Apache 2.0 with commercial restrictions – See `LICENSE` for details.
|
package/build/index.d.ts
CHANGED
|
@@ -10,8 +10,6 @@ import { WorkerService as Worker } from './services/memflow/worker';
|
|
|
10
10
|
import { WorkflowService as workflow } from './services/memflow/workflow';
|
|
11
11
|
import { WorkflowHandleService as WorkflowHandle } from './services/memflow/handle';
|
|
12
12
|
import { proxyActivities } from './services/memflow/workflow/proxyActivities';
|
|
13
|
-
import { MeshData } from './services/meshdata';
|
|
14
|
-
import { MeshOS } from './services/meshos';
|
|
15
13
|
import * as Errors from './modules/errors';
|
|
16
14
|
import * as Utils from './modules/utils';
|
|
17
15
|
import * as Enums from './modules/enums';
|
|
@@ -22,5 +20,5 @@ import { RedisConnection as ConnectorIORedis } from './services/connector/provid
|
|
|
22
20
|
import { RedisConnection as ConnectorRedis } from './services/connector/providers/redis';
|
|
23
21
|
import { NatsConnection as ConnectorNATS } from './services/connector/providers/nats';
|
|
24
22
|
export { Connector, //factory
|
|
25
|
-
ConnectorIORedis, ConnectorNATS, ConnectorPostgres, ConnectorRedis, HotMesh, HotMeshConfig, MeshCall,
|
|
23
|
+
ConnectorIORedis, ConnectorNATS, ConnectorPostgres, ConnectorRedis, HotMesh, HotMeshConfig, MeshCall, MemFlow, Client, Connection, proxyActivities, Search, Entity, Worker, workflow, WorkflowHandle, Enums, Errors, Utils, KeyStore, };
|
|
26
24
|
export * as Types from './types';
|
package/build/index.js
CHANGED
|
@@ -23,7 +23,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
|
|
|
23
23
|
return result;
|
|
24
24
|
};
|
|
25
25
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
|
-
exports.Types = exports.KeyStore = exports.Utils = exports.Errors = exports.Enums = exports.WorkflowHandle = exports.workflow = exports.Worker = exports.Entity = exports.Search = exports.proxyActivities = exports.Connection = exports.Client = exports.
|
|
26
|
+
exports.Types = exports.KeyStore = exports.Utils = exports.Errors = exports.Enums = exports.WorkflowHandle = exports.workflow = exports.Worker = exports.Entity = exports.Search = exports.proxyActivities = exports.Connection = exports.Client = exports.MemFlow = exports.MeshCall = exports.HotMesh = exports.ConnectorRedis = exports.ConnectorPostgres = exports.ConnectorNATS = exports.ConnectorIORedis = exports.Connector = void 0;
|
|
27
27
|
const hotmesh_1 = require("./services/hotmesh");
|
|
28
28
|
Object.defineProperty(exports, "HotMesh", { enumerable: true, get: function () { return hotmesh_1.HotMesh; } });
|
|
29
29
|
const meshcall_1 = require("./services/meshcall");
|
|
@@ -46,10 +46,6 @@ const handle_1 = require("./services/memflow/handle");
|
|
|
46
46
|
Object.defineProperty(exports, "WorkflowHandle", { enumerable: true, get: function () { return handle_1.WorkflowHandleService; } });
|
|
47
47
|
const proxyActivities_1 = require("./services/memflow/workflow/proxyActivities");
|
|
48
48
|
Object.defineProperty(exports, "proxyActivities", { enumerable: true, get: function () { return proxyActivities_1.proxyActivities; } });
|
|
49
|
-
const meshdata_1 = require("./services/meshdata");
|
|
50
|
-
Object.defineProperty(exports, "MeshData", { enumerable: true, get: function () { return meshdata_1.MeshData; } });
|
|
51
|
-
const meshos_1 = require("./services/meshos");
|
|
52
|
-
Object.defineProperty(exports, "MeshOS", { enumerable: true, get: function () { return meshos_1.MeshOS; } });
|
|
53
49
|
const Errors = __importStar(require("./modules/errors"));
|
|
54
50
|
exports.Errors = Errors;
|
|
55
51
|
const Utils = __importStar(require("./modules/utils"));
|
package/build/modules/enums.d.ts
CHANGED
|
@@ -108,3 +108,7 @@ export declare const HMSH_EXPIRE_DURATION: number;
|
|
|
108
108
|
export declare const HMSH_FIDELITY_SECONDS: number;
|
|
109
109
|
export declare const HMSH_SCOUT_INTERVAL_SECONDS: number;
|
|
110
110
|
export declare const HMSH_GUID_SIZE: number;
|
|
111
|
+
/**
|
|
112
|
+
* Default task queue name used when no task queue is specified
|
|
113
|
+
*/
|
|
114
|
+
export declare const DEFAULT_TASK_QUEUE = "default";
|
package/build/modules/enums.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.HMSH_GUID_SIZE = exports.HMSH_SCOUT_INTERVAL_SECONDS = exports.HMSH_FIDELITY_SECONDS = exports.HMSH_EXPIRE_DURATION = exports.HMSH_XPENDING_COUNT = exports.HMSH_XCLAIM_COUNT = exports.HMSH_XCLAIM_DELAY_MS = exports.HMSH_BLOCK_TIME_MS = exports.HMSH_MEMFLOW_EXP_BACKOFF = exports.HMSH_MEMFLOW_MAX_INTERVAL = exports.HMSH_MEMFLOW_MAX_ATTEMPTS = exports.HMSH_GRADUATED_INTERVAL_MS = exports.HMSH_MAX_TIMEOUT_MS = exports.HMSH_MAX_RETRIES = exports.MAX_DELAY = exports.MAX_STREAM_RETRIES = exports.INITIAL_STREAM_BACKOFF = exports.MAX_STREAM_BACKOFF = exports.HMSH_EXPIRE_JOB_SECONDS = exports.HMSH_OTT_WAIT_TIME = exports.HMSH_DEPLOYMENT_PAUSE = exports.HMSH_DEPLOYMENT_DELAY = exports.HMSH_ACTIVATION_MAX_RETRY = exports.HMSH_QUORUM_DELAY_MS = exports.HMSH_QUORUM_ROLLCALL_CYCLES = exports.HMSH_STATUS_UNKNOWN = exports.HMSH_CODE_MEMFLOW_RETRYABLE = exports.HMSH_CODE_MEMFLOW_FATAL = exports.HMSH_CODE_MEMFLOW_MAXED = exports.HMSH_CODE_MEMFLOW_TIMEOUT = exports.HMSH_CODE_MEMFLOW_WAIT = exports.HMSH_CODE_MEMFLOW_PROXY = exports.HMSH_CODE_MEMFLOW_CHILD = exports.HMSH_CODE_MEMFLOW_ALL = exports.HMSH_CODE_MEMFLOW_SLEEP = exports.HMSH_CODE_UNACKED = exports.HMSH_CODE_TIMEOUT = exports.HMSH_CODE_UNKNOWN = exports.HMSH_CODE_INTERRUPT = exports.HMSH_CODE_NOTFOUND = exports.HMSH_CODE_PENDING = exports.HMSH_CODE_SUCCESS = exports.HMSH_SIGNAL_EXPIRE = exports.HMSH_IS_CLUSTER = exports.HMSH_TELEMETRY = exports.HMSH_LOGLEVEL = void 0;
|
|
3
|
+
exports.DEFAULT_TASK_QUEUE = exports.HMSH_GUID_SIZE = exports.HMSH_SCOUT_INTERVAL_SECONDS = exports.HMSH_FIDELITY_SECONDS = exports.HMSH_EXPIRE_DURATION = exports.HMSH_XPENDING_COUNT = exports.HMSH_XCLAIM_COUNT = exports.HMSH_XCLAIM_DELAY_MS = exports.HMSH_BLOCK_TIME_MS = exports.HMSH_MEMFLOW_EXP_BACKOFF = exports.HMSH_MEMFLOW_MAX_INTERVAL = exports.HMSH_MEMFLOW_MAX_ATTEMPTS = exports.HMSH_GRADUATED_INTERVAL_MS = exports.HMSH_MAX_TIMEOUT_MS = exports.HMSH_MAX_RETRIES = exports.MAX_DELAY = exports.MAX_STREAM_RETRIES = exports.INITIAL_STREAM_BACKOFF = exports.MAX_STREAM_BACKOFF = exports.HMSH_EXPIRE_JOB_SECONDS = exports.HMSH_OTT_WAIT_TIME = exports.HMSH_DEPLOYMENT_PAUSE = exports.HMSH_DEPLOYMENT_DELAY = exports.HMSH_ACTIVATION_MAX_RETRY = exports.HMSH_QUORUM_DELAY_MS = exports.HMSH_QUORUM_ROLLCALL_CYCLES = exports.HMSH_STATUS_UNKNOWN = exports.HMSH_CODE_MEMFLOW_RETRYABLE = exports.HMSH_CODE_MEMFLOW_FATAL = exports.HMSH_CODE_MEMFLOW_MAXED = exports.HMSH_CODE_MEMFLOW_TIMEOUT = exports.HMSH_CODE_MEMFLOW_WAIT = exports.HMSH_CODE_MEMFLOW_PROXY = exports.HMSH_CODE_MEMFLOW_CHILD = exports.HMSH_CODE_MEMFLOW_ALL = exports.HMSH_CODE_MEMFLOW_SLEEP = exports.HMSH_CODE_UNACKED = exports.HMSH_CODE_TIMEOUT = exports.HMSH_CODE_UNKNOWN = exports.HMSH_CODE_INTERRUPT = exports.HMSH_CODE_NOTFOUND = exports.HMSH_CODE_PENDING = exports.HMSH_CODE_SUCCESS = exports.HMSH_SIGNAL_EXPIRE = exports.HMSH_IS_CLUSTER = exports.HMSH_TELEMETRY = exports.HMSH_LOGLEVEL = void 0;
|
|
4
4
|
/**
|
|
5
5
|
* Determines the log level for the application. The default is 'info'.
|
|
6
6
|
*/
|
|
@@ -132,3 +132,7 @@ exports.HMSH_FIDELITY_SECONDS = process.env.HMSH_FIDELITY_SECONDS
|
|
|
132
132
|
exports.HMSH_SCOUT_INTERVAL_SECONDS = parseInt(process.env.HMSH_SCOUT_INTERVAL_SECONDS, 10) || 60;
|
|
133
133
|
// UTILS
|
|
134
134
|
exports.HMSH_GUID_SIZE = Math.min(parseInt(process.env.HMSH_GUID_SIZE, 10) || 22, 32);
|
|
135
|
+
/**
|
|
136
|
+
* Default task queue name used when no task queue is specified
|
|
137
|
+
*/
|
|
138
|
+
exports.DEFAULT_TASK_QUEUE = 'default';
|
package/build/modules/utils.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
2
|
import { StoreService } from '../services/store';
|
|
3
3
|
import { AppSubscriptions, AppTransitions, AppVID } from '../types/app';
|
|
4
|
-
import { ProviderClient,
|
|
4
|
+
import { ProviderClient, ProviderTransaction, Providers } from '../types/provider';
|
|
5
5
|
import { StringAnyType } from '../types/serializer';
|
|
6
6
|
import { StreamCode, StreamData, StreamStatus } from '../types/stream';
|
|
7
7
|
import { SystemHealth } from '../types/quorum';
|
|
@@ -34,14 +34,6 @@ export declare const polyfill: {
|
|
|
34
34
|
* `redis` is deprecated; `connection` is the generic replacement
|
|
35
35
|
*/
|
|
36
36
|
providerConfig(obj: any): any;
|
|
37
|
-
/**
|
|
38
|
-
* NOTE: `redisClass and redisOptions` input parameters are deprecated; use `connection` for all configuration inputs
|
|
39
|
-
*/
|
|
40
|
-
meshDataConfig(obj: {
|
|
41
|
-
connection?: Partial<ProviderConfig | ProvidersConfig>;
|
|
42
|
-
redisClass?: any;
|
|
43
|
-
redisOptions?: StringAnyType;
|
|
44
|
-
}): Partial<ProviderConfig> | Partial<ProvidersConfig>;
|
|
45
37
|
};
|
|
46
38
|
/**
|
|
47
39
|
* @private
|
package/build/modules/utils.js
CHANGED
|
@@ -127,12 +127,6 @@ exports.polyfill = {
|
|
|
127
127
|
providerConfig(obj) {
|
|
128
128
|
return obj?.connection ?? obj?.redis ?? obj?.connections;
|
|
129
129
|
},
|
|
130
|
-
/**
|
|
131
|
-
* NOTE: `redisClass and redisOptions` input parameters are deprecated; use `connection` for all configuration inputs
|
|
132
|
-
*/
|
|
133
|
-
meshDataConfig(obj) {
|
|
134
|
-
return { ...obj.connection };
|
|
135
|
-
},
|
|
136
130
|
};
|
|
137
131
|
/**
|
|
138
132
|
* @private
|
package/build/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hotmeshio/hotmesh",
|
|
3
|
-
"version": "0.5.
|
|
3
|
+
"version": "0.5.4",
|
|
4
4
|
"description": "Permanent-Memory Workflows & AI Agents",
|
|
5
5
|
"main": "./build/index.js",
|
|
6
6
|
"types": "./build/index.d.ts",
|
|
@@ -20,7 +20,7 @@
|
|
|
20
20
|
"lint:fix": "eslint . --fix --ext .ts",
|
|
21
21
|
"start": "ts-node src/index.ts",
|
|
22
22
|
"test": "NODE_ENV=test jest --detectOpenHandles --forceExit --verbose",
|
|
23
|
-
"test:await": "NODE_ENV=test jest ./tests/functional/awaiter
|
|
23
|
+
"test:await": "NODE_ENV=test jest ./tests/functional/awaiter/postgres.test.ts --detectOpenHandles --forceExit --verbose",
|
|
24
24
|
"test:compile": "NODE_ENV=test jest ./tests/functional/compile/index.test.ts --detectOpenHandles --forceExit --verbose",
|
|
25
25
|
"test:connect": "NODE_ENV=test jest ./tests/unit/services/connector/* --detectOpenHandles --forceExit --verbose",
|
|
26
26
|
"test:connect:ioredis": "NODE_ENV=test jest ./tests/unit/services/connector/providers/ioredis.test.ts --detectOpenHandles --forceExit --verbose",
|
|
@@ -59,7 +59,7 @@
|
|
|
59
59
|
"test:reporter": "NODE_ENV=test jest ./tests/unit/services/reporter/index.test.ts --detectOpenHandles --forceExit --verbose",
|
|
60
60
|
"test:reentrant": "NODE_ENV=test jest ./tests/functional/reentrant/*.test.ts --detectOpenHandles --forceExit --verbose",
|
|
61
61
|
"test:retry": "NODE_ENV=test jest ./tests/functional/retry/*.test.ts --detectOpenHandles --forceExit --verbose",
|
|
62
|
-
"test:sequence": "NODE_ENV=test HMSH_LOGLEVEL=
|
|
62
|
+
"test:sequence": "NODE_ENV=test HMSH_LOGLEVEL=debug jest ./tests/functional/sequence/postgres.test.ts --detectOpenHandles --forceExit --verbose",
|
|
63
63
|
"test:signal": "NODE_ENV=test jest ./tests/functional/signal/*.test.ts --detectOpenHandles --forceExit --verbose",
|
|
64
64
|
"test:status": "NODE_ENV=test jest ./tests/functional/status/index.test.ts --detectOpenHandles --forceExit --verbose",
|
|
65
65
|
"test:providers": "NODE_ENV=test jest ./tests/functional/*/providers/*/*.test.ts --detectOpenHandles --forceExit --verbose",
|
|
@@ -75,7 +75,6 @@
|
|
|
75
75
|
"test:sub:postgres": "NODE_ENV=test jest ./tests/functional/sub/providers/postgres/postgres.test.ts --detectOpenHandles --forceExit --verbose",
|
|
76
76
|
"test:sub:nats": "NODE_ENV=test jest ./tests/functional/sub/providers/nats/nats.test.ts --detectOpenHandles --forceExit --verbose",
|
|
77
77
|
"test:trigger": "NODE_ENV=test jest ./tests/unit/services/activities/trigger.test.ts --detectOpenHandles --forceExit --verbose",
|
|
78
|
-
"test:meshos": "HMSH_LOGLEVEL=info NODE_ENV=test HMSH_IS_CLUSTER=true jest ./tests/meshos/*.test.ts --forceExit --verbose --detectOpenHandles",
|
|
79
78
|
"test:meshcall": "NODE_ENV=test jest ./tests/meshcall/*.test.ts --forceExit --verbose --detectOpenHandles",
|
|
80
79
|
"test:unit": "NODE_ENV=test jest ./tests/unit/*/*/index.test.ts --detectOpenHandles --forceExit --verbose"
|
|
81
80
|
},
|
|
@@ -8,7 +8,7 @@ export declare class ConnectorService {
|
|
|
8
8
|
* initialization, but the factory method provided here is useful
|
|
9
9
|
* for testing provider configurations.
|
|
10
10
|
*/
|
|
11
|
-
static connectClient(ProviderConfig: ProviderConfig): Promise<ProviderNativeClient>;
|
|
11
|
+
static connectClient(ProviderConfig: ProviderConfig, taskQueue?: string): Promise<ProviderNativeClient>;
|
|
12
12
|
/**
|
|
13
13
|
* Initialize `store`, `stream`, and `subscription` clients for any provider.
|
|
14
14
|
* @private
|
|
@@ -18,5 +18,5 @@ export declare class ConnectorService {
|
|
|
18
18
|
* Binds a provider client native instance to the target object.
|
|
19
19
|
* @private
|
|
20
20
|
*/
|
|
21
|
-
static initClient(ProviderConfig: ProviderConfig, target: HotMeshEngine | HotMeshWorker, field: string): Promise<void>;
|
|
21
|
+
static initClient(ProviderConfig: ProviderConfig, target: HotMeshEngine | HotMeshWorker, field: string, taskQueue?: string): Promise<void>;
|
|
22
22
|
}
|
|
@@ -19,9 +19,9 @@ class ConnectorService {
|
|
|
19
19
|
* initialization, but the factory method provided here is useful
|
|
20
20
|
* for testing provider configurations.
|
|
21
21
|
*/
|
|
22
|
-
static async connectClient(ProviderConfig) {
|
|
22
|
+
static async connectClient(ProviderConfig, taskQueue) {
|
|
23
23
|
const target = {};
|
|
24
|
-
await ConnectorService.initClient(ProviderConfig, target, 'client');
|
|
24
|
+
await ConnectorService.initClient(ProviderConfig, target, 'client', taskQueue);
|
|
25
25
|
return target.client;
|
|
26
26
|
}
|
|
27
27
|
/**
|
|
@@ -38,15 +38,17 @@ class ConnectorService {
|
|
|
38
38
|
sub: { ...connection },
|
|
39
39
|
};
|
|
40
40
|
}
|
|
41
|
+
// Extract taskQueue from target for connection pooling
|
|
42
|
+
const taskQueue = target.taskQueue;
|
|
41
43
|
// Expanded form
|
|
42
44
|
if (connection.store) {
|
|
43
|
-
await ConnectorService.initClient(connection.store, target, 'store');
|
|
45
|
+
await ConnectorService.initClient(connection.store, target, 'store', taskQueue);
|
|
44
46
|
}
|
|
45
47
|
if (connection.stream) {
|
|
46
|
-
await ConnectorService.initClient(connection.stream, target, 'stream');
|
|
48
|
+
await ConnectorService.initClient(connection.stream, target, 'stream', taskQueue);
|
|
47
49
|
}
|
|
48
50
|
if (connection.sub) {
|
|
49
|
-
await ConnectorService.initClient(connection.sub, target, 'sub');
|
|
51
|
+
await ConnectorService.initClient(connection.sub, target, 'sub', taskQueue);
|
|
50
52
|
// use store for publishing events if same as subscription
|
|
51
53
|
if (connection.sub.class !== connection.store.class) {
|
|
52
54
|
//initialize a separate client for publishing events, using
|
|
@@ -56,7 +58,7 @@ class ConnectorService {
|
|
|
56
58
|
options: { ...connection.sub.options },
|
|
57
59
|
provider: connection.sub.provider,
|
|
58
60
|
};
|
|
59
|
-
await ConnectorService.initClient(connection.pub, target, 'pub');
|
|
61
|
+
await ConnectorService.initClient(connection.pub, target, 'pub', taskQueue);
|
|
60
62
|
}
|
|
61
63
|
}
|
|
62
64
|
}
|
|
@@ -64,7 +66,7 @@ class ConnectorService {
|
|
|
64
66
|
* Binds a provider client native instance to the target object.
|
|
65
67
|
* @private
|
|
66
68
|
*/
|
|
67
|
-
static async initClient(ProviderConfig, target, field) {
|
|
69
|
+
static async initClient(ProviderConfig, target, field, taskQueue) {
|
|
68
70
|
if (target[field]) {
|
|
69
71
|
return;
|
|
70
72
|
}
|
|
@@ -87,7 +89,8 @@ class ConnectorService {
|
|
|
87
89
|
case 'postgres':
|
|
88
90
|
//if connecting as a poolClient for subscription, auto connect the client
|
|
89
91
|
const bAutoConnect = field === 'sub';
|
|
90
|
-
|
|
92
|
+
// Use taskQueue-based connection pooling for PostgreSQL
|
|
93
|
+
clientInstance = await postgres_1.PostgresConnection.getOrCreateTaskQueueConnection(id, taskQueue, providerClass, options, { connect: bAutoConnect, provider: providerName });
|
|
91
94
|
break;
|
|
92
95
|
default:
|
|
93
96
|
throw new Error(`Unknown provider type: ${providerType}`);
|
|
@@ -4,17 +4,64 @@ declare class PostgresConnection extends AbstractConnection<PostgresClassType, P
|
|
|
4
4
|
defaultOptions: PostgresClientOptions;
|
|
5
5
|
protected static poolClientInstances: Set<PostgresPoolClientType>;
|
|
6
6
|
protected static connectionInstances: Set<PostgresClientType>;
|
|
7
|
+
protected static taskQueueConnections: Map<string, PostgresConnection>;
|
|
8
|
+
/**
|
|
9
|
+
* Get comprehensive connection statistics for monitoring taskQueue pooling effectiveness
|
|
10
|
+
*/
|
|
11
|
+
static getConnectionStats(): {
|
|
12
|
+
totalPoolClients: number;
|
|
13
|
+
totalConnections: number;
|
|
14
|
+
taskQueueConnections: number;
|
|
15
|
+
taskQueueDetails: Array<{
|
|
16
|
+
key: string;
|
|
17
|
+
connectionId: string;
|
|
18
|
+
reusedCount: number;
|
|
19
|
+
}>;
|
|
20
|
+
};
|
|
21
|
+
/**
|
|
22
|
+
* Log current connection statistics - useful for debugging connection pooling
|
|
23
|
+
*/
|
|
24
|
+
static logConnectionStats(logger?: any): void;
|
|
25
|
+
/**
|
|
26
|
+
* Check taskQueue pooling effectiveness - returns metrics about connection reuse
|
|
27
|
+
*/
|
|
28
|
+
static getPoolingEffectiveness(): {
|
|
29
|
+
totalConnections: number;
|
|
30
|
+
taskQueuePools: number;
|
|
31
|
+
totalReuses: number;
|
|
32
|
+
averageReusesPerPool: number;
|
|
33
|
+
poolingEfficiency: number;
|
|
34
|
+
};
|
|
7
35
|
poolClientInstance: PostgresPoolClientType;
|
|
8
36
|
createConnection(clientConstructor: any, options: PostgresClientOptions, config?: {
|
|
9
37
|
connect?: boolean;
|
|
10
38
|
provider?: string;
|
|
11
39
|
}): Promise<PostgresClientType>;
|
|
12
40
|
getClient(): PostgresClientType;
|
|
41
|
+
/**
|
|
42
|
+
* Get the connection ID for monitoring purposes
|
|
43
|
+
*/
|
|
44
|
+
getConnectionId(): string | null;
|
|
13
45
|
static disconnectAll(): Promise<void>;
|
|
14
46
|
static disconnectPoolClients(): Promise<void>;
|
|
15
47
|
static disconnectConnections(): Promise<void>;
|
|
16
48
|
closeConnection(connection: PostgresClientType): Promise<void>;
|
|
17
49
|
static isPoolClient(client: any): client is PostgresPoolClientType;
|
|
50
|
+
/**
|
|
51
|
+
* Creates a taskQueue-based connection key for connection pooling.
|
|
52
|
+
* This allows multiple providers (store, sub, stream) to reuse the same connection
|
|
53
|
+
* when they share the same taskQueue and database configuration.
|
|
54
|
+
*/
|
|
55
|
+
private static createTaskQueueConnectionKey;
|
|
56
|
+
/**
|
|
57
|
+
* Gets or creates a PostgreSQL connection based on taskQueue and database configuration.
|
|
58
|
+
* If a connection already exists for the same taskQueue + config, it will be reused.
|
|
59
|
+
* This optimization reduces connection overhead for PostgreSQL providers.
|
|
60
|
+
*/
|
|
61
|
+
static getOrCreateTaskQueueConnection(id: string, taskQueue: string | undefined, clientConstructor: PostgresClassType, options: PostgresClientOptions, config?: {
|
|
62
|
+
connect?: boolean;
|
|
63
|
+
provider?: string;
|
|
64
|
+
}): Promise<PostgresConnection>;
|
|
18
65
|
static getTransactionClient(transactionClient: any): Promise<['client' | 'poolclient', PostgresClientType]>;
|
|
19
66
|
}
|
|
20
67
|
export { PostgresConnection };
|