@hotmeshio/hotmesh 0.14.2 → 0.14.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/package.json +5 -3
- package/build/services/durable/worker.js +4 -0
- package/build/services/engine/init.js +1 -1
- package/build/services/engine/schema.js +5 -1
- package/build/services/mapper/index.d.ts +57 -2
- package/build/services/mapper/index.js +57 -2
- package/build/services/pipe/index.d.ts +444 -10
- package/build/services/pipe/index.js +444 -10
- package/build/services/quorum/index.js +1 -1
- package/build/services/router/consumption/index.js +20 -2
- package/build/services/router/error-handling/index.js +1 -1
- package/build/services/store/factory.d.ts +1 -1
- package/build/services/store/factory.js +2 -2
- package/build/services/store/index.d.ts +1 -1
- package/build/services/store/providers/postgres/kvsql.d.ts +11 -1
- package/build/services/store/providers/postgres/kvsql.js +22 -12
- package/build/services/store/providers/postgres/kvtables.js +39 -6
- package/build/services/store/providers/postgres/kvtypes/hash/basic.js +6 -6
- package/build/services/store/providers/postgres/kvtypes/hash/scan.js +2 -1
- package/build/services/store/providers/postgres/kvtypes/list.js +7 -6
- package/build/services/store/providers/postgres/kvtypes/string.js +3 -3
- package/build/services/store/providers/postgres/kvtypes/zset.js +7 -7
- package/build/services/store/providers/postgres/postgres.d.ts +3 -2
- package/build/services/store/providers/postgres/postgres.js +55 -55
- package/build/services/store/providers/postgres/time-notify.js +18 -25
- package/build/services/store/providers/store-initializable.d.ts +1 -1
- package/build/services/stream/registry.d.ts +1 -0
- package/build/services/stream/registry.js +12 -8
- package/build/services/worker/index.js +3 -1
- package/build/types/hotmesh.d.ts +8 -0
- package/package.json +5 -3
- package/vitest.config.mts +1 -1
|
@@ -5,7 +5,358 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.Pipe = void 0;
|
|
7
7
|
const functions_1 = __importDefault(require("./functions"));
|
|
8
|
+
/**
|
|
9
|
+
* A functional data-transformation pipeline that resolves expressions
|
|
10
|
+
* row-by-row against live job data.
|
|
11
|
+
*
|
|
12
|
+
* @remarks
|
|
13
|
+
* ## Overview
|
|
14
|
+
*
|
|
15
|
+
* Pipe is the engine behind HotMesh's `@pipe` syntax — a uniform,
|
|
16
|
+
* functional approach to data mapping and transformation. Every
|
|
17
|
+
* ECMAScript operation is expressed as a **function with inputs**,
|
|
18
|
+
* eliminating the syntactic variability of the language (ternaries,
|
|
19
|
+
* property access, instance methods) in favor of a single,
|
|
20
|
+
* composable pattern.
|
|
21
|
+
*
|
|
22
|
+
* ## Postfix notation (Reverse Polish Notation)
|
|
23
|
+
*
|
|
24
|
+
* If you've used an RPN calculator, `@pipe` will feel familiar.
|
|
25
|
+
* Operands come first, then the operator consumes them:
|
|
26
|
+
*
|
|
27
|
+
* ```
|
|
28
|
+
* RPN: 4 4 + → 8
|
|
29
|
+
*
|
|
30
|
+
* @pipe:
|
|
31
|
+
* - [4, 4] ← operands
|
|
32
|
+
* - ["{@math.add}"] ← operator → 8
|
|
33
|
+
* ```
|
|
34
|
+
*
|
|
35
|
+
* That's the entire model. Operands are pushed, an operator pops
|
|
36
|
+
* them, and the result becomes an operand for the next row.
|
|
37
|
+
*
|
|
38
|
+
* ## How it works — row by row
|
|
39
|
+
*
|
|
40
|
+
* A pipe is an ordered array of **rows**. Each row is an array of
|
|
41
|
+
* **cells**. The key rule: **every resolved value on a row flows
|
|
42
|
+
* into cell 0 of the next row** (the operator). The operator
|
|
43
|
+
* consumes all upstream operands, resolves to a new value, and
|
|
44
|
+
* that result *becomes an operand*. Any remaining cells on the
|
|
45
|
+
* same row are resolved independently and appended as additional
|
|
46
|
+
* operands. Then the cycle repeats.
|
|
47
|
+
*
|
|
48
|
+
* ```
|
|
49
|
+
* @pipe:
|
|
50
|
+
* ┌──────────────────────────────────────┐
|
|
51
|
+
* │ Row 0: [operand, operand, operand] │ ← resolve all cells
|
|
52
|
+
* │ │ │ │ │
|
|
53
|
+
* │ └────────┼────────┘ │
|
|
54
|
+
* │ │ │
|
|
55
|
+
* ├─────────────────────┼────────────────┤
|
|
56
|
+
* │ ▼ │
|
|
57
|
+
* │ Row 1: [ OPERATOR , operand] │ ← ALL operands from Row 0
|
|
58
|
+
* │ ▲ receives all ▲ │ flow into cell 0;
|
|
59
|
+
* │ │ from above │ │ result becomes operand;
|
|
60
|
+
* │ └──────────────┘ │ remaining cells resolve
|
|
61
|
+
* │ │ │ │
|
|
62
|
+
* │ └────────────┘ │
|
|
63
|
+
* │ │ │
|
|
64
|
+
* ├─────────────────────┼────────────────┤
|
|
65
|
+
* │ ▼ │
|
|
66
|
+
* │ Row 2: [ OPERATOR , operand] │ ← same pattern repeats
|
|
67
|
+
* └──────────────────────────────────────┘
|
|
68
|
+
* ▼
|
|
69
|
+
* final value = cell 0 of last row
|
|
70
|
+
* ```
|
|
71
|
+
*
|
|
72
|
+
* Step by step:
|
|
73
|
+
*
|
|
74
|
+
* 1. **Row 0** — Every cell is resolved independently (static
|
|
75
|
+
* literals, `{data.*}` references, or nullary functions like
|
|
76
|
+
* `{@date.now}`). The resolved values are all **operands**.
|
|
77
|
+
* 2. **Row 1** — Cell 0 is the **operator** (`{@domain.method}`).
|
|
78
|
+
* It receives *all* operands from Row 0 as its arguments and
|
|
79
|
+
* produces a result. That result replaces cell 0 — it is now
|
|
80
|
+
* an operand. Any remaining cells (cell 1, 2, ...) on this row
|
|
81
|
+
* are resolved independently and become additional operands.
|
|
82
|
+
* 3. **Row 2** — Cell 0 is the next operator. It receives *all*
|
|
83
|
+
* operands from Row 1 (the prior result + any extra cells).
|
|
84
|
+
* The cycle repeats.
|
|
85
|
+
* 4. **Return** — The first cell of the final row is the result.
|
|
86
|
+
*
|
|
87
|
+
* When an additional cell on an operator row needs computed (not
|
|
88
|
+
* just a static value or simple reference), use a **nested
|
|
89
|
+
* `@pipe`** (sub-pipe) to resolve it. See Example 4 below.
|
|
90
|
+
*
|
|
91
|
+
* ## Three types of cell values
|
|
92
|
+
*
|
|
93
|
+
* | Type | Syntax | Example |
|
|
94
|
+
* |------|--------|---------|
|
|
95
|
+
* | **Static** | literal | `42`, `"hello"`, `true` |
|
|
96
|
+
* | **Dynamic** | `{path}` | `{a.output.data.name}` |
|
|
97
|
+
* | **Function** | `{@domain.method}` | `{@string.concat}`, `{@math.add}` |
|
|
98
|
+
*
|
|
99
|
+
* ## Available function domains
|
|
100
|
+
*
|
|
101
|
+
* - **array** — `get`, `length`, `join`, `concat`, `push`, `indexOf`, ...
|
|
102
|
+
* - **bitwise** — `and`, `or`, `xor`, `not`, ...
|
|
103
|
+
* - **conditional** — `ternary`, `equality`, `nullish`, ...
|
|
104
|
+
* - **cron** — `nextDelay`
|
|
105
|
+
* - **date** — `now`, `toLocaleString`, `yyyymmdd`, ...
|
|
106
|
+
* - **json** — `parse`, `stringify`
|
|
107
|
+
* - **logical** — `and`, `or`, `not`
|
|
108
|
+
* - **math** — `add`, `multiply`, `pow`, `max`, `min`, `abs`, ...
|
|
109
|
+
* - **number** — `isEven`, `isOdd`, `gte`, `lte`, ...
|
|
110
|
+
* - **object** — `create`, `get`, `set`, `keys`, ...
|
|
111
|
+
* - **string** — `concat`, `split`, `charAt`, `toLowerCase`, `includes`, ...
|
|
112
|
+
* - **symbol**
|
|
113
|
+
* - **unary**
|
|
114
|
+
*
|
|
115
|
+
* ---
|
|
116
|
+
*
|
|
117
|
+
* ## Example 1 — Simple field mapping (YAML)
|
|
118
|
+
*
|
|
119
|
+
* Most fields need only a one-to-one reference. Curly braces pull
|
|
120
|
+
* values from upstream activity outputs:
|
|
121
|
+
*
|
|
122
|
+
* ```yaml
|
|
123
|
+
* # Map fields from activities a and b into a new shape
|
|
124
|
+
* maps:
|
|
125
|
+
* first: "{a.output.data.first_name}"
|
|
126
|
+
* last: "{a.output.data.last_name}"
|
|
127
|
+
* email: "{b.output.data.email}"
|
|
128
|
+
* age: "{b.output.data.age}"
|
|
129
|
+
* company: "ACME Corp" # static string
|
|
130
|
+
* bonus: 500 # static number
|
|
131
|
+
* ```
|
|
132
|
+
*
|
|
133
|
+
* The equivalent JavaScript object passed to the mapper:
|
|
134
|
+
*
|
|
135
|
+
* ```typescript
|
|
136
|
+
* const rules = {
|
|
137
|
+
* first: '{a.output.data.first_name}',
|
|
138
|
+
* last: '{a.output.data.last_name}',
|
|
139
|
+
* email: '{b.output.data.email}',
|
|
140
|
+
* age: '{b.output.data.age}',
|
|
141
|
+
* company: 'ACME Corp',
|
|
142
|
+
* bonus: 500,
|
|
143
|
+
* };
|
|
144
|
+
* ```
|
|
145
|
+
*
|
|
146
|
+
* ## Example 2 — String transformation (YAML → JS)
|
|
147
|
+
*
|
|
148
|
+
* Build a `user_name` like `jdoe` from "John Doe". Follow the
|
|
149
|
+
* RPN flow — operands feed into the operator on the next row,
|
|
150
|
+
* the result becomes an operand, extra cells append more operands:
|
|
151
|
+
*
|
|
152
|
+
* ```yaml
|
|
153
|
+
* user_name:
|
|
154
|
+
* "@pipe":
|
|
155
|
+
* - ["{a.output.data.first_name}", 0]
|
|
156
|
+
* - ["{@string.charAt}", "{a.output.data.last_name}"]
|
|
157
|
+
* - ["{@string.concat}"]
|
|
158
|
+
* - ["{@string.toLowerCase}"]
|
|
159
|
+
* ```
|
|
160
|
+
*
|
|
161
|
+
* ```typescript
|
|
162
|
+
* // Identical logic in JavaScript
|
|
163
|
+
* const rules = {
|
|
164
|
+
* user_name: {
|
|
165
|
+
* '@pipe': [
|
|
166
|
+
* ['{a.output.data.first_name}', 0],
|
|
167
|
+
* ['{@string.charAt}', '{a.output.data.last_name}'],
|
|
168
|
+
* ['{@string.concat}'],
|
|
169
|
+
* ['{@string.toLowerCase}'],
|
|
170
|
+
* ],
|
|
171
|
+
* },
|
|
172
|
+
* };
|
|
173
|
+
* ```
|
|
174
|
+
*
|
|
175
|
+
* RPN trace (given first_name="John", last_name="Doe"):
|
|
176
|
+
*
|
|
177
|
+
* ```
|
|
178
|
+
* Row 0: ["John", 0] ← two operands
|
|
179
|
+
* Row 1: charAt("John", 0)="J" ← operator consumes both → result "J"
|
|
180
|
+
* then resolve "Doe" ← extra cell → operands are ["J", "Doe"]
|
|
181
|
+
* Row 2: concat("J", "Doe") ← operator → result "JDoe"
|
|
182
|
+
* operands are ["JDoe"]
|
|
183
|
+
* Row 3: toLowerCase("JDoe") ← operator → result "jdoe"
|
|
184
|
+
* ```
|
|
185
|
+
*
|
|
186
|
+
* ## Example 3 — Conditional logic (YAML → JS)
|
|
187
|
+
*
|
|
188
|
+
* Classify an employee as "Senior" or "Junior" based on age:
|
|
189
|
+
*
|
|
190
|
+
* ```yaml
|
|
191
|
+
* status:
|
|
192
|
+
* "@pipe":
|
|
193
|
+
* - ["{b.output.data.age}", 40]
|
|
194
|
+
* - ["{@number.gte}", "Senior", "Junior"]
|
|
195
|
+
* - ["{@conditional.ternary}"]
|
|
196
|
+
* ```
|
|
197
|
+
*
|
|
198
|
+
* ```typescript
|
|
199
|
+
* const rules = {
|
|
200
|
+
* status: {
|
|
201
|
+
* '@pipe': [
|
|
202
|
+
* ['{b.output.data.age}', 40],
|
|
203
|
+
* ['{@number.gte}', 'Senior', 'Junior'],
|
|
204
|
+
* ['{@conditional.ternary}'],
|
|
205
|
+
* ],
|
|
206
|
+
* },
|
|
207
|
+
* };
|
|
208
|
+
* ```
|
|
209
|
+
*
|
|
210
|
+
* RPN trace (given age=30):
|
|
211
|
+
*
|
|
212
|
+
* ```
|
|
213
|
+
* Row 0: [30, 40] ← two operands
|
|
214
|
+
* Row 1: gte(30, 40)=false ← operator consumes both → false
|
|
215
|
+
* resolve "Senior", "Junior" ← extra cells → [false, "Senior", "Junior"]
|
|
216
|
+
* Row 2: ternary(false, "Senior", "Junior") ← operator → "Junior"
|
|
217
|
+
* ```
|
|
218
|
+
*
|
|
219
|
+
* ## Example 4 — Nested pipes (fan-out / fan-in)
|
|
220
|
+
*
|
|
221
|
+
* Extract initials from a full name. Each nested `@pipe` runs
|
|
222
|
+
* independently (fan-out), then the first standard row after them
|
|
223
|
+
* receives all sub-pipe results as inputs (fan-in):
|
|
224
|
+
*
|
|
225
|
+
* ```yaml
|
|
226
|
+
* initials:
|
|
227
|
+
* "@pipe":
|
|
228
|
+
* - ["{a.output.data.full_name}", " "]
|
|
229
|
+
* - "@pipe": # fan-out: first initial
|
|
230
|
+
* - ["{@string.split}", 0]
|
|
231
|
+
* - ["{@array.get}", 0]
|
|
232
|
+
* - ["{@string.charAt}"]
|
|
233
|
+
* - "@pipe": # fan-out: last initial
|
|
234
|
+
* - ["{@string.split}", 1]
|
|
235
|
+
* - ["{@array.get}", 0]
|
|
236
|
+
* - ["{@string.charAt}"]
|
|
237
|
+
* - ["{@string.concat}"] # fan-in: combine both
|
|
238
|
+
* ```
|
|
239
|
+
*
|
|
240
|
+
* ```typescript
|
|
241
|
+
* const rules = {
|
|
242
|
+
* initials: {
|
|
243
|
+
* '@pipe': [
|
|
244
|
+
* ['{a.output.data.full_name}', ' '],
|
|
245
|
+
* { '@pipe': [['{@string.split}', 0], ['{@array.get}', 0], ['{@string.charAt}']] },
|
|
246
|
+
* { '@pipe': [['{@string.split}', 1], ['{@array.get}', 0], ['{@string.charAt}']] },
|
|
247
|
+
* ['{@string.concat}'],
|
|
248
|
+
* ],
|
|
249
|
+
* },
|
|
250
|
+
* };
|
|
251
|
+
* // "Luke Birdeau" → split → ["Luke","Birdeau"]
|
|
252
|
+
* // sub-pipe 1: get(0) → "Luke" → charAt(0) → "L"
|
|
253
|
+
* // sub-pipe 2: get(1) → "Birdeau" → charAt(0) → "B"
|
|
254
|
+
* // fan-in: concat("L", "B") → "LB"
|
|
255
|
+
* ```
|
|
256
|
+
*
|
|
257
|
+
* ## Example 5 — Reduce (iterate and accumulate)
|
|
258
|
+
*
|
|
259
|
+
* Transform an array of `{ full_name }` objects into
|
|
260
|
+
* `{ first, last }` pairs using `@reduce`. Context variables
|
|
261
|
+
* `{$item}`, `{$key}`, `{$index}`, `{$input}`, and `{$output}`
|
|
262
|
+
* are available inside the reducer body:
|
|
263
|
+
*
|
|
264
|
+
* ```typescript
|
|
265
|
+
* const jobData = {
|
|
266
|
+
* a: { output: { data: [
|
|
267
|
+
* { full_name: 'Luke Birdeau' },
|
|
268
|
+
* { full_name: 'John Doe' },
|
|
269
|
+
* ]}}
|
|
270
|
+
* };
|
|
271
|
+
*
|
|
272
|
+
* const rules = [
|
|
273
|
+
* ['{a.output.data}', []], // input array + initial accumulator
|
|
274
|
+
* { '@reduce': [
|
|
275
|
+
* { '@pipe': [['{$output}']] }, // carry forward accumulator
|
|
276
|
+
* { '@pipe': [
|
|
277
|
+
* { '@pipe': [['first']] },
|
|
278
|
+
* { '@pipe': [
|
|
279
|
+
* ['{$item.full_name}', ' '],
|
|
280
|
+
* ['{@string.split}', 0],
|
|
281
|
+
* ['{@array.get}'],
|
|
282
|
+
* ]},
|
|
283
|
+
* { '@pipe': [['last']] },
|
|
284
|
+
* { '@pipe': [
|
|
285
|
+
* ['{$item.full_name}', ' '],
|
|
286
|
+
* ['{@string.split}', 1],
|
|
287
|
+
* ['{@array.get}'],
|
|
288
|
+
* ]},
|
|
289
|
+
* ['{@object.create}'],
|
|
290
|
+
* ]},
|
|
291
|
+
* ['{@array.push}'],
|
|
292
|
+
* ]},
|
|
293
|
+
* ];
|
|
294
|
+
* // => [{ first: 'Luke', last: 'Birdeau' }, { first: 'John', last: 'Doe' }]
|
|
295
|
+
* ```
|
|
296
|
+
*
|
|
297
|
+
* ## Example 6 — Transition conditions
|
|
298
|
+
*
|
|
299
|
+
* Pipes power conditional transitions between workflow activities.
|
|
300
|
+
* In YAML, a transition fires only when the `@pipe` resolves to the
|
|
301
|
+
* `expected` value:
|
|
302
|
+
*
|
|
303
|
+
* ```yaml
|
|
304
|
+
* transitions:
|
|
305
|
+
* t1:
|
|
306
|
+
* - to: a1
|
|
307
|
+
* conditions:
|
|
308
|
+
* match:
|
|
309
|
+
* - expected: false
|
|
310
|
+
* actual:
|
|
311
|
+
* "@pipe":
|
|
312
|
+
* - ["{t1.output.data.a}", "goodbye"]
|
|
313
|
+
* - ["{@conditional.equality}"]
|
|
314
|
+
* ```
|
|
315
|
+
*
|
|
316
|
+
* ## Example 7 — Inline YAML with HotMesh.deploy
|
|
317
|
+
*
|
|
318
|
+
* HotMesh ships with an inline YAML parser, so a complete
|
|
319
|
+
* workflow with data mapping can be deployed in a single call:
|
|
320
|
+
*
|
|
321
|
+
* ```typescript
|
|
322
|
+
* await hotMesh.deploy(`
|
|
323
|
+
* app:
|
|
324
|
+
* id: myapp
|
|
325
|
+
* version: '1'
|
|
326
|
+
* graphs:
|
|
327
|
+
* - subscribes: order.process
|
|
328
|
+
* activities:
|
|
329
|
+
* t1:
|
|
330
|
+
* type: trigger
|
|
331
|
+
* a1:
|
|
332
|
+
* type: worker
|
|
333
|
+
* topic: inventory.check
|
|
334
|
+
* input:
|
|
335
|
+
* maps:
|
|
336
|
+
* itemId: "{t1.output.data.itemId}"
|
|
337
|
+
* output:
|
|
338
|
+
* schema:
|
|
339
|
+
* type: object
|
|
340
|
+
* job:
|
|
341
|
+
* maps:
|
|
342
|
+
* result: "{$self.output.data.available}"
|
|
343
|
+
* transitions:
|
|
344
|
+
* t1:
|
|
345
|
+
* - to: a1
|
|
346
|
+
* `);
|
|
347
|
+
*
|
|
348
|
+
* await hotMesh.activate('1');
|
|
349
|
+
* const response = await hotMesh.pubsub('order.process', { itemId: 'sku-42' });
|
|
350
|
+
* ```
|
|
351
|
+
*/
|
|
8
352
|
class Pipe {
|
|
353
|
+
/**
|
|
354
|
+
* @param rules - The ordered row array defining the pipeline.
|
|
355
|
+
* @param jobData - The current job data used to resolve `{data.*}`
|
|
356
|
+
* and `{activity.*}` references.
|
|
357
|
+
* @param context - Optional iteration context (`$item`, `$key`,
|
|
358
|
+
* `$output`, etc.) supplied during `@reduce` execution.
|
|
359
|
+
*/
|
|
9
360
|
constructor(rules, jobData, context) {
|
|
10
361
|
this.rules = rules;
|
|
11
362
|
this.jobData = jobData;
|
|
@@ -17,12 +368,32 @@ class Pipe {
|
|
|
17
368
|
isreduceType(currentRow) {
|
|
18
369
|
return !Array.isArray(currentRow) && '@reduce' in currentRow;
|
|
19
370
|
}
|
|
371
|
+
/**
|
|
372
|
+
* Returns `true` if the value is a `@pipe` object (i.e. `{ '@pipe': [...] }`).
|
|
373
|
+
*
|
|
374
|
+
* @param obj - The value to test.
|
|
375
|
+
*/
|
|
20
376
|
static isPipeObject(obj) {
|
|
21
377
|
return (typeof obj === 'object' &&
|
|
22
378
|
obj !== null &&
|
|
23
379
|
!Array.isArray(obj) &&
|
|
24
380
|
'@pipe' in obj);
|
|
25
381
|
}
|
|
382
|
+
/**
|
|
383
|
+
* One-shot convenience method that resolves a single value or
|
|
384
|
+
* `@pipe` expression against the given context.
|
|
385
|
+
*
|
|
386
|
+
* @param unresolved - A literal value, a `{data.*}` reference string,
|
|
387
|
+
* or a `@pipe` object.
|
|
388
|
+
* @param context - Partial {@link JobState} used for resolution.
|
|
389
|
+
* @returns The fully resolved value.
|
|
390
|
+
*
|
|
391
|
+
* @example
|
|
392
|
+
* ```typescript
|
|
393
|
+
* Pipe.resolve('{data.user.email}', jobState);
|
|
394
|
+
* Pipe.resolve({ '@pipe': [['{data.a}', '{data.b}'], ['{@math.multiply}']] }, jobState);
|
|
395
|
+
* ```
|
|
396
|
+
*/
|
|
26
397
|
static resolve(unresolved, context) {
|
|
27
398
|
let pipe;
|
|
28
399
|
if (Pipe.isPipeObject(unresolved)) {
|
|
@@ -34,8 +405,29 @@ class Pipe {
|
|
|
34
405
|
return pipe.process();
|
|
35
406
|
}
|
|
36
407
|
/**
|
|
37
|
-
*
|
|
38
|
-
*
|
|
408
|
+
* Executes the pipeline row-by-row, resolving and transforming
|
|
409
|
+
* until the final value is produced.
|
|
410
|
+
*
|
|
411
|
+
* @remarks
|
|
412
|
+
* Row 0 is resolved independently (values only). Each subsequent
|
|
413
|
+
* row feeds the prior row's resolved output as arguments to the
|
|
414
|
+
* function named in cell 0. Nested `@pipe` rows are queued
|
|
415
|
+
* (fan-out) and collected into the next standard row (fan-in).
|
|
416
|
+
* `@reduce` rows iterate over the prior row's array output.
|
|
417
|
+
*
|
|
418
|
+
* @param resolved - Optional pre-resolved seed values (used
|
|
419
|
+
* internally by `reduce` to inject the accumulator).
|
|
420
|
+
* @returns The first cell of the final resolved row.
|
|
421
|
+
*
|
|
422
|
+
* @example
|
|
423
|
+
* ```typescript
|
|
424
|
+
* // Split a full name and grab the first element
|
|
425
|
+
* const pipe = new Pipe(
|
|
426
|
+
* [['{a.output.data.full_name}', ' '], ['{@string.split}', 0], ['{@array.get}']],
|
|
427
|
+
* { a: { output: { data: { full_name: 'Luke Birdeau' } } } },
|
|
428
|
+
* );
|
|
429
|
+
* pipe.process(); // => 'Luke'
|
|
430
|
+
* ```
|
|
39
431
|
*/
|
|
40
432
|
process(resolved = null) {
|
|
41
433
|
let index = 0;
|
|
@@ -74,14 +466,26 @@ class Pipe {
|
|
|
74
466
|
return clonedObj;
|
|
75
467
|
}
|
|
76
468
|
/**
|
|
77
|
-
*
|
|
78
|
-
*
|
|
79
|
-
*
|
|
80
|
-
*
|
|
81
|
-
*
|
|
82
|
-
*
|
|
83
|
-
*
|
|
84
|
-
*
|
|
469
|
+
* Iterates over an array or object and accumulates a result,
|
|
470
|
+
* similar to `Array.prototype.reduce`.
|
|
471
|
+
*
|
|
472
|
+
* @remarks
|
|
473
|
+
* Inside the reducer body the following context variables are
|
|
474
|
+
* available as cell references:
|
|
475
|
+
*
|
|
476
|
+
* | Variable | Description |
|
|
477
|
+
* |----------|-------------|
|
|
478
|
+
* | `{$input}` | The full input collection |
|
|
479
|
+
* | `{$output}` | The current accumulator |
|
|
480
|
+
* | `{$item}` | The current element |
|
|
481
|
+
* | `{$key}` | The current key (string for objects, index string for arrays) |
|
|
482
|
+
* | `{$index}` | The current numeric index |
|
|
483
|
+
*
|
|
484
|
+
* The preceding row supplies `[inputCollection, initialAccumulator]`.
|
|
485
|
+
* If no initial accumulator is provided, it defaults to `null`.
|
|
486
|
+
*
|
|
487
|
+
* @param input - A two-element array: `[collection, initialAccumulator]`.
|
|
488
|
+
* @returns The final accumulated value, wrapped in an array.
|
|
85
489
|
* @private
|
|
86
490
|
*/
|
|
87
491
|
reduce(input) {
|
|
@@ -152,6 +556,14 @@ class Pipe {
|
|
|
152
556
|
}
|
|
153
557
|
}
|
|
154
558
|
}
|
|
559
|
+
/**
|
|
560
|
+
* Looks up a domain function by its `{@domain.method}` name string
|
|
561
|
+
* and returns the callable.
|
|
562
|
+
*
|
|
563
|
+
* @param functionName - A string like `{@math.add}` or `{@string.concat}`.
|
|
564
|
+
* @returns The resolved function reference.
|
|
565
|
+
* @throws If the domain or method is not registered.
|
|
566
|
+
*/
|
|
155
567
|
static resolveFunction(functionName) {
|
|
156
568
|
let [prefix, suffix] = functionName.split('.');
|
|
157
569
|
prefix = prefix.substring(2);
|
|
@@ -165,6 +577,14 @@ class Pipe {
|
|
|
165
577
|
}
|
|
166
578
|
return domain[suffix];
|
|
167
579
|
}
|
|
580
|
+
/**
|
|
581
|
+
* Resolves every cell in a single row independently — each cell
|
|
582
|
+
* is evaluated for function calls, context variables, or mappable
|
|
583
|
+
* references.
|
|
584
|
+
*
|
|
585
|
+
* @param cells - The array of {@link PipeItems} to resolve.
|
|
586
|
+
* @returns An array of resolved values, one per input cell.
|
|
587
|
+
*/
|
|
168
588
|
processCells(cells) {
|
|
169
589
|
const resolved = [];
|
|
170
590
|
if (Array.isArray(cells)) {
|
|
@@ -193,6 +613,14 @@ class Pipe {
|
|
|
193
613
|
currentCell.startsWith('{') &&
|
|
194
614
|
currentCell.endsWith('}'));
|
|
195
615
|
}
|
|
616
|
+
/**
|
|
617
|
+
* Resolves a single cell value by detecting its type — function
|
|
618
|
+
* call (`{@domain.fn}`), context variable (`{$item}`, `{$key}`,
|
|
619
|
+
* etc.), mappable reference (`{data.*}`), or literal.
|
|
620
|
+
*
|
|
621
|
+
* @param currentCell - The cell to resolve.
|
|
622
|
+
* @returns The resolved runtime value.
|
|
623
|
+
*/
|
|
196
624
|
resolveCellValue(currentCell) {
|
|
197
625
|
if (this.isFunction(currentCell)) {
|
|
198
626
|
const fn = Pipe.resolveFunction(currentCell);
|
|
@@ -221,6 +649,12 @@ class Pipe {
|
|
|
221
649
|
}
|
|
222
650
|
return current;
|
|
223
651
|
}
|
|
652
|
+
/**
|
|
653
|
+
* Resolves a `{data.*}` or `{activity.*}` reference by walking
|
|
654
|
+
* the dot-delimited path into {@link jobData}.
|
|
655
|
+
*
|
|
656
|
+
* @param currentCell - The mappable reference string (e.g. `{data.user.name}`).
|
|
657
|
+
*/
|
|
224
658
|
resolveMappableValue(currentCell) {
|
|
225
659
|
const term = this.resolveMapTerm(currentCell);
|
|
226
660
|
return this.getNestedProperty(this.jobData, term);
|
|
@@ -54,7 +54,7 @@ class QuorumService {
|
|
|
54
54
|
* @private
|
|
55
55
|
*/
|
|
56
56
|
async initStoreChannel(store) {
|
|
57
|
-
this.store = await factory_2.StoreServiceFactory.init(store, this.namespace, this.appId, this.logger);
|
|
57
|
+
this.store = await factory_2.StoreServiceFactory.init(store, this.namespace, this.appId, this.logger, this.guid, 'quorum');
|
|
58
58
|
}
|
|
59
59
|
/**
|
|
60
60
|
* @private
|
|
@@ -439,8 +439,26 @@ class ConsumptionManager {
|
|
|
439
439
|
output = this.errorHandler.structureUnhandledError(input, err instanceof Error ? err : new Error(String(err)));
|
|
440
440
|
}
|
|
441
441
|
try {
|
|
442
|
-
|
|
443
|
-
|
|
442
|
+
// When the ENGINE itself fails to process a message (e.g., schema not
|
|
443
|
+
// found, missing subscription), do NOT republish the error back to the
|
|
444
|
+
// engine stream — that creates an infinite poison loop. The engine
|
|
445
|
+
// When the ENGINE encounters an infrastructure error (schema not found,
|
|
446
|
+
// subscription missing — code 598), the message is permanently unprocessable.
|
|
447
|
+
// Do NOT republish it — that creates an infinite poison loop. Only suppress
|
|
448
|
+
// these specific infrastructure errors; application-level errors (retries,
|
|
449
|
+
// duplicates, workflow failures) must still flow through normally.
|
|
450
|
+
if (group === 'ENGINE' && output?.code === 598) {
|
|
451
|
+
this.logger.error(`stream-engine-dispatch-fatal`, {
|
|
452
|
+
stream, id, group,
|
|
453
|
+
aid: input.metadata?.aid,
|
|
454
|
+
jid: input.metadata?.jid,
|
|
455
|
+
message: output.data?.message,
|
|
456
|
+
});
|
|
457
|
+
}
|
|
458
|
+
else {
|
|
459
|
+
const messageId = await this.publishResponse(input, output);
|
|
460
|
+
telemetry.setStreamAttributes({ 'app.worker.mid': messageId });
|
|
461
|
+
}
|
|
444
462
|
}
|
|
445
463
|
catch (publishErr) {
|
|
446
464
|
// If publishResponse fails, still ack the message to prevent
|
|
@@ -3,6 +3,6 @@ import { ProviderClient, ProviderTransaction } from '../../types/provider';
|
|
|
3
3
|
import { StoreInitializable } from './providers/store-initializable';
|
|
4
4
|
import { StoreService } from './index';
|
|
5
5
|
declare class StoreServiceFactory {
|
|
6
|
-
static init(providerClient: ProviderClient, namespace: string, appId: string, logger: ILogger): Promise<StoreService<ProviderClient, ProviderTransaction> & StoreInitializable>;
|
|
6
|
+
static init(providerClient: ProviderClient, namespace: string, appId: string, logger: ILogger, guid?: string, role?: string): Promise<StoreService<ProviderClient, ProviderTransaction> & StoreInitializable>;
|
|
7
7
|
}
|
|
8
8
|
export { StoreServiceFactory };
|
|
@@ -4,12 +4,12 @@ exports.StoreServiceFactory = void 0;
|
|
|
4
4
|
const utils_1 = require("../../modules/utils");
|
|
5
5
|
const postgres_1 = require("./providers/postgres/postgres");
|
|
6
6
|
class StoreServiceFactory {
|
|
7
|
-
static async init(providerClient, namespace, appId, logger) {
|
|
7
|
+
static async init(providerClient, namespace, appId, logger, guid, role) {
|
|
8
8
|
let service;
|
|
9
9
|
if ((0, utils_1.identifyProvider)(providerClient) === 'postgres') {
|
|
10
10
|
service = new postgres_1.PostgresStoreService(providerClient);
|
|
11
11
|
} //etc
|
|
12
|
-
await service.init(namespace, appId, logger);
|
|
12
|
+
await service.init(namespace, appId, logger, guid, role);
|
|
13
13
|
return service;
|
|
14
14
|
}
|
|
15
15
|
}
|
|
@@ -21,7 +21,7 @@ declare abstract class StoreService<Provider extends ProviderClient, Transaction
|
|
|
21
21
|
serializer: Serializer;
|
|
22
22
|
constructor(client: Provider);
|
|
23
23
|
abstract transact(): TransactionProvider;
|
|
24
|
-
abstract init(namespace: string, appId: string, logger: ILogger): Promise<HotMeshApps>;
|
|
24
|
+
abstract init(namespace: string, appId: string, logger: ILogger, guid?: string, role?: string): Promise<HotMeshApps>;
|
|
25
25
|
abstract mintKey(type: KeyType, params: KeyStoreParams): string;
|
|
26
26
|
abstract getSettings(bCreate?: boolean): Promise<HotMeshSettings>;
|
|
27
27
|
abstract setSettings(manifest: HotMeshSettings): Promise<any>;
|
|
@@ -26,9 +26,19 @@ export declare class KVSQL {
|
|
|
26
26
|
exec(...args: any[]): Promise<Array<any>>;
|
|
27
27
|
mintKey(type: KeyType, params: KeyStoreParams): string;
|
|
28
28
|
/**
|
|
29
|
-
* Resolves the table name when provided a key
|
|
29
|
+
* Resolves the table name when provided a key.
|
|
30
|
+
* Public tables (applications, connections) are no longer routed through
|
|
31
|
+
* the KV layer — they use direct SQL in postgres.ts.
|
|
30
32
|
*/
|
|
31
33
|
tableForKey(key: string, stats_type?: 'hash' | 'sorted_set' | 'list'): string;
|
|
34
|
+
/**
|
|
35
|
+
* Strips the `hmsh:<appId>:<entity>:` prefix from a full Redis-style key,
|
|
36
|
+
* keeping only the meaningful suffix for SQL storage. Applied only to the
|
|
37
|
+
* `key` column in SQL params — never to member values, field names, or values.
|
|
38
|
+
*
|
|
39
|
+
* Excluded tables (jobs, streams, job attributes) retain the full key.
|
|
40
|
+
*/
|
|
41
|
+
storageKey(fullKey: string): string;
|
|
32
42
|
safeName(input: string, prefix?: string): string;
|
|
33
43
|
set: (key: string, value: string, options?: import("./kvtypes/hash/index").SetOptions, multi?: ProviderTransaction) => Promise<boolean>;
|
|
34
44
|
_set: (key: string, value: string, options?: import("./kvtypes/hash/index").SetOptions) => {
|