@514labs/moose-lib 0.6.295-ci-18-g185e40dc → 0.6.295-ci-16-gad4ec11a
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{browserCompatible-DhVPp9qX.d.ts → browserCompatible-CMEunMFq.d.ts} +1 -1
- package/dist/{browserCompatible-DCCiBirg.d.mts → browserCompatible-FzU17dxm.d.mts} +1 -1
- package/dist/browserCompatible.d.mts +2 -2
- package/dist/browserCompatible.d.ts +2 -2
- package/dist/browserCompatible.js +2445 -2186
- package/dist/browserCompatible.js.map +1 -1
- package/dist/browserCompatible.mjs +2392 -2131
- package/dist/browserCompatible.mjs.map +1 -1
- package/dist/dmv2/index.d.mts +1 -1
- package/dist/dmv2/index.d.ts +1 -1
- package/dist/dmv2/index.js +2341 -2082
- package/dist/dmv2/index.js.map +1 -1
- package/dist/dmv2/index.mjs +2302 -2041
- package/dist/dmv2/index.mjs.map +1 -1
- package/dist/{index-CcZRaA0b.d.mts → index-CcHF2cVT.d.mts} +56 -87
- package/dist/{index-CcZRaA0b.d.ts → index-CcHF2cVT.d.ts} +56 -87
- package/dist/index.d.mts +79 -6
- package/dist/index.d.ts +79 -6
- package/dist/index.js +3070 -2758
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +2968 -2653
- package/dist/index.mjs.map +1 -1
- package/dist/moose-runner.js +1712 -1159
- package/dist/moose-runner.js.map +1 -1
- package/dist/moose-runner.mjs +1705 -1154
- package/dist/moose-runner.mjs.map +1 -1
- package/package.json +1 -1
package/dist/moose-runner.js
CHANGED
|
@@ -9,6 +9,10 @@ var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
|
9
9
|
var __esm = (fn, res) => function __init() {
|
|
10
10
|
return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
|
|
11
11
|
};
|
|
12
|
+
var __export = (target, all) => {
|
|
13
|
+
for (var name in all)
|
|
14
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
15
|
+
};
|
|
12
16
|
var __copyProps = (to, from, except, desc) => {
|
|
13
17
|
if (from && typeof from === "object" || typeof from === "function") {
|
|
14
18
|
for (let key of __getOwnPropNames(from))
|
|
@@ -26,6 +30,344 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
|
|
|
26
30
|
mod
|
|
27
31
|
));
|
|
28
32
|
|
|
33
|
+
// src/dmv2/utils/stackTrace.ts
|
|
34
|
+
var init_stackTrace = __esm({
|
|
35
|
+
"src/dmv2/utils/stackTrace.ts"() {
|
|
36
|
+
"use strict";
|
|
37
|
+
}
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
// src/dmv2/typedBase.ts
|
|
41
|
+
var init_typedBase = __esm({
|
|
42
|
+
"src/dmv2/typedBase.ts"() {
|
|
43
|
+
"use strict";
|
|
44
|
+
init_stackTrace();
|
|
45
|
+
}
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
// src/dataModels/dataModelTypes.ts
|
|
49
|
+
var init_dataModelTypes = __esm({
|
|
50
|
+
"src/dataModels/dataModelTypes.ts"() {
|
|
51
|
+
"use strict";
|
|
52
|
+
}
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
// src/sqlHelpers.ts
|
|
56
|
+
function sql(strings, ...values) {
|
|
57
|
+
return new Sql(strings, values);
|
|
58
|
+
}
|
|
59
|
+
function createClickhouseParameter(parameterIndex, value) {
|
|
60
|
+
return `{p${parameterIndex}:${mapToClickHouseType(value)}}`;
|
|
61
|
+
}
|
|
62
|
+
function emptyIfUndefined(value) {
|
|
63
|
+
return value === void 0 ? "" : value;
|
|
64
|
+
}
|
|
65
|
+
var isTable, isColumn, instanceofSql, Sql, toQuery, toQueryPreview, getValueFromParameter, mapToClickHouseType;
|
|
66
|
+
var init_sqlHelpers = __esm({
|
|
67
|
+
"src/sqlHelpers.ts"() {
|
|
68
|
+
"use strict";
|
|
69
|
+
isTable = (value) => typeof value === "object" && value !== null && "kind" in value && value.kind === "OlapTable";
|
|
70
|
+
isColumn = (value) => typeof value === "object" && "name" in value && "annotations" in value;
|
|
71
|
+
instanceofSql = (value) => typeof value === "object" && "values" in value && "strings" in value;
|
|
72
|
+
Sql = class {
|
|
73
|
+
values;
|
|
74
|
+
strings;
|
|
75
|
+
constructor(rawStrings, rawValues) {
|
|
76
|
+
if (rawStrings.length - 1 !== rawValues.length) {
|
|
77
|
+
if (rawStrings.length === 0) {
|
|
78
|
+
throw new TypeError("Expected at least 1 string");
|
|
79
|
+
}
|
|
80
|
+
throw new TypeError(
|
|
81
|
+
`Expected ${rawStrings.length} strings to have ${rawStrings.length - 1} values`
|
|
82
|
+
);
|
|
83
|
+
}
|
|
84
|
+
const valuesLength = rawValues.reduce(
|
|
85
|
+
(len, value) => len + (instanceofSql(value) ? value.values.length : isColumn(value) || isTable(value) ? 0 : 1),
|
|
86
|
+
0
|
|
87
|
+
);
|
|
88
|
+
this.values = new Array(valuesLength);
|
|
89
|
+
this.strings = new Array(valuesLength + 1);
|
|
90
|
+
this.strings[0] = rawStrings[0];
|
|
91
|
+
let i = 0, pos = 0;
|
|
92
|
+
while (i < rawValues.length) {
|
|
93
|
+
const child = rawValues[i++];
|
|
94
|
+
const rawString = rawStrings[i];
|
|
95
|
+
if (instanceofSql(child)) {
|
|
96
|
+
this.strings[pos] += child.strings[0];
|
|
97
|
+
let childIndex = 0;
|
|
98
|
+
while (childIndex < child.values.length) {
|
|
99
|
+
this.values[pos++] = child.values[childIndex++];
|
|
100
|
+
this.strings[pos] = child.strings[childIndex];
|
|
101
|
+
}
|
|
102
|
+
this.strings[pos] += rawString;
|
|
103
|
+
} else if (isColumn(child)) {
|
|
104
|
+
const aggregationFunction = child.annotations.find(
|
|
105
|
+
([k, _]) => k === "aggregationFunction"
|
|
106
|
+
);
|
|
107
|
+
if (aggregationFunction !== void 0) {
|
|
108
|
+
this.strings[pos] += `${aggregationFunction[1].functionName}Merge(\`${child.name}\`)`;
|
|
109
|
+
} else {
|
|
110
|
+
this.strings[pos] += `\`${child.name}\``;
|
|
111
|
+
}
|
|
112
|
+
this.strings[pos] += rawString;
|
|
113
|
+
} else if (isTable(child)) {
|
|
114
|
+
if (child.config.database) {
|
|
115
|
+
this.strings[pos] += `\`${child.config.database}\`.\`${child.name}\``;
|
|
116
|
+
} else {
|
|
117
|
+
this.strings[pos] += `\`${child.name}\``;
|
|
118
|
+
}
|
|
119
|
+
this.strings[pos] += rawString;
|
|
120
|
+
} else {
|
|
121
|
+
this.values[pos++] = child;
|
|
122
|
+
this.strings[pos] = rawString;
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
};
|
|
127
|
+
toQuery = (sql3) => {
|
|
128
|
+
const parameterizedStubs = sql3.values.map(
|
|
129
|
+
(v, i) => createClickhouseParameter(i, v)
|
|
130
|
+
);
|
|
131
|
+
const query = sql3.strings.map(
|
|
132
|
+
(s, i) => s != "" ? `${s}${emptyIfUndefined(parameterizedStubs[i])}` : ""
|
|
133
|
+
).join("");
|
|
134
|
+
const query_params = sql3.values.reduce(
|
|
135
|
+
(acc, v, i) => ({
|
|
136
|
+
...acc,
|
|
137
|
+
[`p${i}`]: getValueFromParameter(v)
|
|
138
|
+
}),
|
|
139
|
+
{}
|
|
140
|
+
);
|
|
141
|
+
return [query, query_params];
|
|
142
|
+
};
|
|
143
|
+
toQueryPreview = (sql3) => {
|
|
144
|
+
try {
|
|
145
|
+
const formatValue = (v) => {
|
|
146
|
+
if (Array.isArray(v)) {
|
|
147
|
+
const [type, val] = v;
|
|
148
|
+
if (type === "Identifier") {
|
|
149
|
+
return `\`${String(val)}\``;
|
|
150
|
+
}
|
|
151
|
+
return `[${v.map((x) => formatValue(x)).join(", ")}]`;
|
|
152
|
+
}
|
|
153
|
+
if (v === null || v === void 0) return "NULL";
|
|
154
|
+
if (typeof v === "string") return `'${v.replace(/'/g, "''")}'`;
|
|
155
|
+
if (typeof v === "number") return String(v);
|
|
156
|
+
if (typeof v === "boolean") return v ? "true" : "false";
|
|
157
|
+
if (v instanceof Date)
|
|
158
|
+
return `'${v.toISOString().replace("T", " ").slice(0, 19)}'`;
|
|
159
|
+
try {
|
|
160
|
+
return JSON.stringify(v);
|
|
161
|
+
} catch {
|
|
162
|
+
return String(v);
|
|
163
|
+
}
|
|
164
|
+
};
|
|
165
|
+
let out = sql3.strings[0] ?? "";
|
|
166
|
+
for (let i = 0; i < sql3.values.length; i++) {
|
|
167
|
+
const val = getValueFromParameter(sql3.values[i]);
|
|
168
|
+
out += formatValue(val);
|
|
169
|
+
out += sql3.strings[i + 1] ?? "";
|
|
170
|
+
}
|
|
171
|
+
return out.replace(/\s+/g, " ").trim();
|
|
172
|
+
} catch (error) {
|
|
173
|
+
console.log(`toQueryPreview error: ${error}`);
|
|
174
|
+
return "/* query preview unavailable */";
|
|
175
|
+
}
|
|
176
|
+
};
|
|
177
|
+
getValueFromParameter = (value) => {
|
|
178
|
+
if (Array.isArray(value)) {
|
|
179
|
+
const [type, val] = value;
|
|
180
|
+
if (type === "Identifier") return val;
|
|
181
|
+
}
|
|
182
|
+
return value;
|
|
183
|
+
};
|
|
184
|
+
mapToClickHouseType = (value) => {
|
|
185
|
+
if (typeof value === "number") {
|
|
186
|
+
return Number.isInteger(value) ? "Int" : "Float";
|
|
187
|
+
}
|
|
188
|
+
if (typeof value === "boolean") return "Bool";
|
|
189
|
+
if (value instanceof Date) return "DateTime";
|
|
190
|
+
if (Array.isArray(value)) {
|
|
191
|
+
const [type, _] = value;
|
|
192
|
+
return type;
|
|
193
|
+
}
|
|
194
|
+
return "String";
|
|
195
|
+
};
|
|
196
|
+
}
|
|
197
|
+
});
|
|
198
|
+
|
|
199
|
+
// src/blocks/helpers.ts
|
|
200
|
+
var init_helpers = __esm({
|
|
201
|
+
"src/blocks/helpers.ts"() {
|
|
202
|
+
"use strict";
|
|
203
|
+
init_sqlHelpers();
|
|
204
|
+
}
|
|
205
|
+
});
|
|
206
|
+
|
|
207
|
+
// src/dmv2/sdk/olapTable.ts
|
|
208
|
+
var import_node_stream, import_node_crypto;
|
|
209
|
+
var init_olapTable = __esm({
|
|
210
|
+
"src/dmv2/sdk/olapTable.ts"() {
|
|
211
|
+
"use strict";
|
|
212
|
+
init_typedBase();
|
|
213
|
+
init_dataModelTypes();
|
|
214
|
+
init_helpers();
|
|
215
|
+
init_internal();
|
|
216
|
+
import_node_stream = require("stream");
|
|
217
|
+
import_node_crypto = require("crypto");
|
|
218
|
+
init_sqlHelpers();
|
|
219
|
+
}
|
|
220
|
+
});
|
|
221
|
+
|
|
222
|
+
// src/dmv2/sdk/stream.ts
|
|
223
|
+
var import_node_crypto2;
|
|
224
|
+
var init_stream = __esm({
|
|
225
|
+
"src/dmv2/sdk/stream.ts"() {
|
|
226
|
+
"use strict";
|
|
227
|
+
init_typedBase();
|
|
228
|
+
init_internal();
|
|
229
|
+
import_node_crypto2 = require("crypto");
|
|
230
|
+
init_stackTrace();
|
|
231
|
+
}
|
|
232
|
+
});
|
|
233
|
+
|
|
234
|
+
// src/dmv2/sdk/workflow.ts
|
|
235
|
+
var init_workflow = __esm({
|
|
236
|
+
"src/dmv2/sdk/workflow.ts"() {
|
|
237
|
+
"use strict";
|
|
238
|
+
init_internal();
|
|
239
|
+
}
|
|
240
|
+
});
|
|
241
|
+
|
|
242
|
+
// src/dmv2/sdk/ingestApi.ts
|
|
243
|
+
var init_ingestApi = __esm({
|
|
244
|
+
"src/dmv2/sdk/ingestApi.ts"() {
|
|
245
|
+
"use strict";
|
|
246
|
+
init_typedBase();
|
|
247
|
+
init_internal();
|
|
248
|
+
}
|
|
249
|
+
});
|
|
250
|
+
|
|
251
|
+
// src/dmv2/sdk/consumptionApi.ts
|
|
252
|
+
var init_consumptionApi = __esm({
|
|
253
|
+
"src/dmv2/sdk/consumptionApi.ts"() {
|
|
254
|
+
"use strict";
|
|
255
|
+
init_typedBase();
|
|
256
|
+
init_internal();
|
|
257
|
+
}
|
|
258
|
+
});
|
|
259
|
+
|
|
260
|
+
// src/dmv2/sdk/ingestPipeline.ts
|
|
261
|
+
var init_ingestPipeline = __esm({
|
|
262
|
+
"src/dmv2/sdk/ingestPipeline.ts"() {
|
|
263
|
+
"use strict";
|
|
264
|
+
init_typedBase();
|
|
265
|
+
init_stream();
|
|
266
|
+
init_olapTable();
|
|
267
|
+
init_ingestApi();
|
|
268
|
+
init_helpers();
|
|
269
|
+
}
|
|
270
|
+
});
|
|
271
|
+
|
|
272
|
+
// src/dmv2/sdk/etlPipeline.ts
|
|
273
|
+
var init_etlPipeline = __esm({
|
|
274
|
+
"src/dmv2/sdk/etlPipeline.ts"() {
|
|
275
|
+
"use strict";
|
|
276
|
+
init_workflow();
|
|
277
|
+
}
|
|
278
|
+
});
|
|
279
|
+
|
|
280
|
+
// src/dmv2/sdk/sqlResource.ts
|
|
281
|
+
var init_sqlResource = __esm({
|
|
282
|
+
"src/dmv2/sdk/sqlResource.ts"() {
|
|
283
|
+
"use strict";
|
|
284
|
+
init_internal();
|
|
285
|
+
init_sqlHelpers();
|
|
286
|
+
init_stackTrace();
|
|
287
|
+
}
|
|
288
|
+
});
|
|
289
|
+
|
|
290
|
+
// src/dmv2/sdk/materializedView.ts
|
|
291
|
+
var init_materializedView = __esm({
|
|
292
|
+
"src/dmv2/sdk/materializedView.ts"() {
|
|
293
|
+
"use strict";
|
|
294
|
+
init_helpers();
|
|
295
|
+
init_sqlHelpers();
|
|
296
|
+
init_olapTable();
|
|
297
|
+
init_sqlResource();
|
|
298
|
+
}
|
|
299
|
+
});
|
|
300
|
+
|
|
301
|
+
// src/dmv2/sdk/view.ts
|
|
302
|
+
var init_view = __esm({
|
|
303
|
+
"src/dmv2/sdk/view.ts"() {
|
|
304
|
+
"use strict";
|
|
305
|
+
init_helpers();
|
|
306
|
+
init_sqlHelpers();
|
|
307
|
+
init_sqlResource();
|
|
308
|
+
}
|
|
309
|
+
});
|
|
310
|
+
|
|
311
|
+
// src/dmv2/sdk/lifeCycle.ts
|
|
312
|
+
var init_lifeCycle = __esm({
|
|
313
|
+
"src/dmv2/sdk/lifeCycle.ts"() {
|
|
314
|
+
"use strict";
|
|
315
|
+
}
|
|
316
|
+
});
|
|
317
|
+
|
|
318
|
+
// src/dmv2/sdk/webApp.ts
|
|
319
|
+
var init_webApp = __esm({
|
|
320
|
+
"src/dmv2/sdk/webApp.ts"() {
|
|
321
|
+
"use strict";
|
|
322
|
+
init_internal();
|
|
323
|
+
}
|
|
324
|
+
});
|
|
325
|
+
|
|
326
|
+
// src/dmv2/registry.ts
|
|
327
|
+
var init_registry = __esm({
|
|
328
|
+
"src/dmv2/registry.ts"() {
|
|
329
|
+
"use strict";
|
|
330
|
+
init_internal();
|
|
331
|
+
}
|
|
332
|
+
});
|
|
333
|
+
|
|
334
|
+
// src/dmv2/index.ts
|
|
335
|
+
var init_dmv2 = __esm({
|
|
336
|
+
"src/dmv2/index.ts"() {
|
|
337
|
+
"use strict";
|
|
338
|
+
init_olapTable();
|
|
339
|
+
init_stream();
|
|
340
|
+
init_workflow();
|
|
341
|
+
init_ingestApi();
|
|
342
|
+
init_consumptionApi();
|
|
343
|
+
init_ingestPipeline();
|
|
344
|
+
init_etlPipeline();
|
|
345
|
+
init_materializedView();
|
|
346
|
+
init_sqlResource();
|
|
347
|
+
init_view();
|
|
348
|
+
init_lifeCycle();
|
|
349
|
+
init_webApp();
|
|
350
|
+
init_registry();
|
|
351
|
+
}
|
|
352
|
+
});
|
|
353
|
+
|
|
354
|
+
// src/dataModels/types.ts
|
|
355
|
+
var init_types = __esm({
|
|
356
|
+
"src/dataModels/types.ts"() {
|
|
357
|
+
"use strict";
|
|
358
|
+
}
|
|
359
|
+
});
|
|
360
|
+
|
|
361
|
+
// src/browserCompatible.ts
|
|
362
|
+
var init_browserCompatible = __esm({
|
|
363
|
+
"src/browserCompatible.ts"() {
|
|
364
|
+
"use strict";
|
|
365
|
+
init_dmv2();
|
|
366
|
+
init_types();
|
|
367
|
+
init_sqlHelpers();
|
|
368
|
+
}
|
|
369
|
+
});
|
|
370
|
+
|
|
29
371
|
// src/commons.ts
|
|
30
372
|
function isTruthy(value) {
|
|
31
373
|
if (!value) return false;
|
|
@@ -155,165 +497,14 @@ var init_commons = __esm({
|
|
|
155
497
|
}
|
|
156
498
|
});
|
|
157
499
|
|
|
158
|
-
// src/
|
|
159
|
-
var
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
var import_process = __toESM(require("process"));
|
|
163
|
-
|
|
164
|
-
// src/sqlHelpers.ts
|
|
165
|
-
var isTable = (value) => typeof value === "object" && value !== null && "kind" in value && value.kind === "OlapTable";
|
|
166
|
-
var isColumn = (value) => typeof value === "object" && "name" in value && "annotations" in value;
|
|
167
|
-
function sql(strings, ...values) {
|
|
168
|
-
return new Sql(strings, values);
|
|
169
|
-
}
|
|
170
|
-
var instanceofSql = (value) => typeof value === "object" && "values" in value && "strings" in value;
|
|
171
|
-
var Sql = class {
|
|
172
|
-
values;
|
|
173
|
-
strings;
|
|
174
|
-
constructor(rawStrings, rawValues) {
|
|
175
|
-
if (rawStrings.length - 1 !== rawValues.length) {
|
|
176
|
-
if (rawStrings.length === 0) {
|
|
177
|
-
throw new TypeError("Expected at least 1 string");
|
|
178
|
-
}
|
|
179
|
-
throw new TypeError(
|
|
180
|
-
`Expected ${rawStrings.length} strings to have ${rawStrings.length - 1} values`
|
|
181
|
-
);
|
|
182
|
-
}
|
|
183
|
-
const valuesLength = rawValues.reduce(
|
|
184
|
-
(len, value) => len + (instanceofSql(value) ? value.values.length : isColumn(value) || isTable(value) ? 0 : 1),
|
|
185
|
-
0
|
|
186
|
-
);
|
|
187
|
-
this.values = new Array(valuesLength);
|
|
188
|
-
this.strings = new Array(valuesLength + 1);
|
|
189
|
-
this.strings[0] = rawStrings[0];
|
|
190
|
-
let i = 0, pos = 0;
|
|
191
|
-
while (i < rawValues.length) {
|
|
192
|
-
const child = rawValues[i++];
|
|
193
|
-
const rawString = rawStrings[i];
|
|
194
|
-
if (instanceofSql(child)) {
|
|
195
|
-
this.strings[pos] += child.strings[0];
|
|
196
|
-
let childIndex = 0;
|
|
197
|
-
while (childIndex < child.values.length) {
|
|
198
|
-
this.values[pos++] = child.values[childIndex++];
|
|
199
|
-
this.strings[pos] = child.strings[childIndex];
|
|
200
|
-
}
|
|
201
|
-
this.strings[pos] += rawString;
|
|
202
|
-
} else if (isColumn(child)) {
|
|
203
|
-
const aggregationFunction = child.annotations.find(
|
|
204
|
-
([k, _]) => k === "aggregationFunction"
|
|
205
|
-
);
|
|
206
|
-
if (aggregationFunction !== void 0) {
|
|
207
|
-
this.strings[pos] += `${aggregationFunction[1].functionName}Merge(\`${child.name}\`)`;
|
|
208
|
-
} else {
|
|
209
|
-
this.strings[pos] += `\`${child.name}\``;
|
|
210
|
-
}
|
|
211
|
-
this.strings[pos] += rawString;
|
|
212
|
-
} else if (isTable(child)) {
|
|
213
|
-
if (child.config.database) {
|
|
214
|
-
this.strings[pos] += `\`${child.config.database}\`.\`${child.name}\``;
|
|
215
|
-
} else {
|
|
216
|
-
this.strings[pos] += `\`${child.name}\``;
|
|
217
|
-
}
|
|
218
|
-
this.strings[pos] += rawString;
|
|
219
|
-
} else {
|
|
220
|
-
this.values[pos++] = child;
|
|
221
|
-
this.strings[pos] = rawString;
|
|
222
|
-
}
|
|
223
|
-
}
|
|
224
|
-
}
|
|
225
|
-
};
|
|
226
|
-
var toQuery = (sql3) => {
|
|
227
|
-
const parameterizedStubs = sql3.values.map(
|
|
228
|
-
(v, i) => createClickhouseParameter(i, v)
|
|
229
|
-
);
|
|
230
|
-
const query = sql3.strings.map(
|
|
231
|
-
(s, i) => s != "" ? `${s}${emptyIfUndefined(parameterizedStubs[i])}` : ""
|
|
232
|
-
).join("");
|
|
233
|
-
const query_params = sql3.values.reduce(
|
|
234
|
-
(acc, v, i) => ({
|
|
235
|
-
...acc,
|
|
236
|
-
[`p${i}`]: getValueFromParameter(v)
|
|
237
|
-
}),
|
|
238
|
-
{}
|
|
239
|
-
);
|
|
240
|
-
return [query, query_params];
|
|
241
|
-
};
|
|
242
|
-
var toQueryPreview = (sql3) => {
|
|
243
|
-
try {
|
|
244
|
-
const formatValue = (v) => {
|
|
245
|
-
if (Array.isArray(v)) {
|
|
246
|
-
const [type, val] = v;
|
|
247
|
-
if (type === "Identifier") {
|
|
248
|
-
return `\`${String(val)}\``;
|
|
249
|
-
}
|
|
250
|
-
return `[${v.map((x) => formatValue(x)).join(", ")}]`;
|
|
251
|
-
}
|
|
252
|
-
if (v === null || v === void 0) return "NULL";
|
|
253
|
-
if (typeof v === "string") return `'${v.replace(/'/g, "''")}'`;
|
|
254
|
-
if (typeof v === "number") return String(v);
|
|
255
|
-
if (typeof v === "boolean") return v ? "true" : "false";
|
|
256
|
-
if (v instanceof Date)
|
|
257
|
-
return `'${v.toISOString().replace("T", " ").slice(0, 19)}'`;
|
|
258
|
-
try {
|
|
259
|
-
return JSON.stringify(v);
|
|
260
|
-
} catch {
|
|
261
|
-
return String(v);
|
|
262
|
-
}
|
|
263
|
-
};
|
|
264
|
-
let out = sql3.strings[0] ?? "";
|
|
265
|
-
for (let i = 0; i < sql3.values.length; i++) {
|
|
266
|
-
const val = getValueFromParameter(sql3.values[i]);
|
|
267
|
-
out += formatValue(val);
|
|
268
|
-
out += sql3.strings[i + 1] ?? "";
|
|
269
|
-
}
|
|
270
|
-
return out.replace(/\s+/g, " ").trim();
|
|
271
|
-
} catch (error) {
|
|
272
|
-
console.log(`toQueryPreview error: ${error}`);
|
|
273
|
-
return "/* query preview unavailable */";
|
|
274
|
-
}
|
|
275
|
-
};
|
|
276
|
-
var getValueFromParameter = (value) => {
|
|
277
|
-
if (Array.isArray(value)) {
|
|
278
|
-
const [type, val] = value;
|
|
279
|
-
if (type === "Identifier") return val;
|
|
280
|
-
}
|
|
281
|
-
return value;
|
|
282
|
-
};
|
|
283
|
-
function createClickhouseParameter(parameterIndex, value) {
|
|
284
|
-
return `{p${parameterIndex}:${mapToClickHouseType(value)}}`;
|
|
285
|
-
}
|
|
286
|
-
var mapToClickHouseType = (value) => {
|
|
287
|
-
if (typeof value === "number") {
|
|
288
|
-
return Number.isInteger(value) ? "Int" : "Float";
|
|
289
|
-
}
|
|
290
|
-
if (typeof value === "boolean") return "Bool";
|
|
291
|
-
if (value instanceof Date) return "DateTime";
|
|
292
|
-
if (Array.isArray(value)) {
|
|
293
|
-
const [type, _] = value;
|
|
294
|
-
return type;
|
|
500
|
+
// src/secrets.ts
|
|
501
|
+
var init_secrets = __esm({
|
|
502
|
+
"src/secrets.ts"() {
|
|
503
|
+
"use strict";
|
|
295
504
|
}
|
|
296
|
-
|
|
297
|
-
};
|
|
298
|
-
function emptyIfUndefined(value) {
|
|
299
|
-
return value === void 0 ? "" : value;
|
|
300
|
-
}
|
|
301
|
-
|
|
302
|
-
// src/dmv2/sdk/olapTable.ts
|
|
303
|
-
var import_node_stream = require("stream");
|
|
304
|
-
var import_node_crypto = require("crypto");
|
|
305
|
-
|
|
306
|
-
// src/dmv2/sdk/stream.ts
|
|
307
|
-
var import_node_crypto2 = require("crypto");
|
|
308
|
-
|
|
309
|
-
// src/index.ts
|
|
310
|
-
init_commons();
|
|
505
|
+
});
|
|
311
506
|
|
|
312
507
|
// src/consumption-apis/helpers.ts
|
|
313
|
-
var import_client2 = require("@temporalio/client");
|
|
314
|
-
var import_node_crypto3 = require("crypto");
|
|
315
|
-
var import_perf_hooks = require("perf_hooks");
|
|
316
|
-
var fs = __toESM(require("fs"));
|
|
317
508
|
function formatElapsedTime(ms) {
|
|
318
509
|
if (ms < 1e3) {
|
|
319
510
|
return `${Math.round(ms)} ms`;
|
|
@@ -326,142 +517,6 @@ function formatElapsedTime(ms) {
|
|
|
326
517
|
const remainingSeconds = seconds % 60;
|
|
327
518
|
return `${minutes} minutes and ${remainingSeconds.toFixed(2)} seconds`;
|
|
328
519
|
}
|
|
329
|
-
var MooseClient = class {
|
|
330
|
-
query;
|
|
331
|
-
workflow;
|
|
332
|
-
constructor(queryClient, temporalClient) {
|
|
333
|
-
this.query = queryClient;
|
|
334
|
-
this.workflow = new WorkflowClient(temporalClient);
|
|
335
|
-
}
|
|
336
|
-
};
|
|
337
|
-
var QueryClient = class {
|
|
338
|
-
client;
|
|
339
|
-
query_id_prefix;
|
|
340
|
-
constructor(client, query_id_prefix) {
|
|
341
|
-
this.client = client;
|
|
342
|
-
this.query_id_prefix = query_id_prefix;
|
|
343
|
-
}
|
|
344
|
-
async execute(sql3) {
|
|
345
|
-
const [query, query_params] = toQuery(sql3);
|
|
346
|
-
console.log(`[QueryClient] | Query: ${toQueryPreview(sql3)}`);
|
|
347
|
-
const start = import_perf_hooks.performance.now();
|
|
348
|
-
const result = await this.client.query({
|
|
349
|
-
query,
|
|
350
|
-
query_params,
|
|
351
|
-
format: "JSONEachRow",
|
|
352
|
-
query_id: this.query_id_prefix + (0, import_node_crypto3.randomUUID)()
|
|
353
|
-
// Note: wait_end_of_query deliberately NOT set here as this is used for SELECT queries
|
|
354
|
-
// where response buffering would harm streaming performance and concurrency
|
|
355
|
-
});
|
|
356
|
-
const elapsedMs = import_perf_hooks.performance.now() - start;
|
|
357
|
-
console.log(
|
|
358
|
-
`[QueryClient] | Query completed: ${formatElapsedTime(elapsedMs)}`
|
|
359
|
-
);
|
|
360
|
-
return result;
|
|
361
|
-
}
|
|
362
|
-
async command(sql3) {
|
|
363
|
-
const [query, query_params] = toQuery(sql3);
|
|
364
|
-
console.log(`[QueryClient] | Command: ${toQueryPreview(sql3)}`);
|
|
365
|
-
const start = import_perf_hooks.performance.now();
|
|
366
|
-
const result = await this.client.command({
|
|
367
|
-
query,
|
|
368
|
-
query_params,
|
|
369
|
-
query_id: this.query_id_prefix + (0, import_node_crypto3.randomUUID)()
|
|
370
|
-
});
|
|
371
|
-
const elapsedMs = import_perf_hooks.performance.now() - start;
|
|
372
|
-
console.log(
|
|
373
|
-
`[QueryClient] | Command completed: ${formatElapsedTime(elapsedMs)}`
|
|
374
|
-
);
|
|
375
|
-
return result;
|
|
376
|
-
}
|
|
377
|
-
};
|
|
378
|
-
var WorkflowClient = class {
|
|
379
|
-
client;
|
|
380
|
-
constructor(temporalClient) {
|
|
381
|
-
this.client = temporalClient;
|
|
382
|
-
}
|
|
383
|
-
async execute(name, input_data) {
|
|
384
|
-
try {
|
|
385
|
-
if (!this.client) {
|
|
386
|
-
return {
|
|
387
|
-
status: 404,
|
|
388
|
-
body: `Temporal client not found. Is the feature flag enabled?`
|
|
389
|
-
};
|
|
390
|
-
}
|
|
391
|
-
const config = await this.getWorkflowConfig(name);
|
|
392
|
-
const [processedInput, workflowId] = this.processInputData(
|
|
393
|
-
name,
|
|
394
|
-
input_data
|
|
395
|
-
);
|
|
396
|
-
console.log(
|
|
397
|
-
`WorkflowClient - starting workflow: ${name} with config ${JSON.stringify(config)} and input_data ${JSON.stringify(processedInput)}`
|
|
398
|
-
);
|
|
399
|
-
const handle = await this.client.workflow.start("ScriptWorkflow", {
|
|
400
|
-
args: [
|
|
401
|
-
{ workflow_name: name, execution_mode: "start" },
|
|
402
|
-
processedInput
|
|
403
|
-
],
|
|
404
|
-
taskQueue: "typescript-script-queue",
|
|
405
|
-
workflowId,
|
|
406
|
-
workflowIdConflictPolicy: "FAIL",
|
|
407
|
-
workflowIdReusePolicy: "ALLOW_DUPLICATE",
|
|
408
|
-
retry: {
|
|
409
|
-
maximumAttempts: config.retries
|
|
410
|
-
},
|
|
411
|
-
workflowRunTimeout: config.timeout
|
|
412
|
-
});
|
|
413
|
-
return {
|
|
414
|
-
status: 200,
|
|
415
|
-
body: `Workflow started: ${name}. View it in the Temporal dashboard: http://localhost:8080/namespaces/default/workflows/${workflowId}/${handle.firstExecutionRunId}/history`
|
|
416
|
-
};
|
|
417
|
-
} catch (error) {
|
|
418
|
-
return {
|
|
419
|
-
status: 400,
|
|
420
|
-
body: `Error starting workflow: ${error}`
|
|
421
|
-
};
|
|
422
|
-
}
|
|
423
|
-
}
|
|
424
|
-
async terminate(workflowId) {
|
|
425
|
-
try {
|
|
426
|
-
if (!this.client) {
|
|
427
|
-
return {
|
|
428
|
-
status: 404,
|
|
429
|
-
body: `Temporal client not found. Is the feature flag enabled?`
|
|
430
|
-
};
|
|
431
|
-
}
|
|
432
|
-
const handle = this.client.workflow.getHandle(workflowId);
|
|
433
|
-
await handle.terminate();
|
|
434
|
-
return {
|
|
435
|
-
status: 200,
|
|
436
|
-
body: `Workflow terminated: ${workflowId}`
|
|
437
|
-
};
|
|
438
|
-
} catch (error) {
|
|
439
|
-
return {
|
|
440
|
-
status: 400,
|
|
441
|
-
body: `Error terminating workflow: ${error}`
|
|
442
|
-
};
|
|
443
|
-
}
|
|
444
|
-
}
|
|
445
|
-
async getWorkflowConfig(name) {
|
|
446
|
-
const workflows = await getWorkflows2();
|
|
447
|
-
const dmv2Workflow = workflows.get(name);
|
|
448
|
-
if (dmv2Workflow) {
|
|
449
|
-
return {
|
|
450
|
-
retries: dmv2Workflow.config.retries || 3,
|
|
451
|
-
timeout: dmv2Workflow.config.timeout || "1h"
|
|
452
|
-
};
|
|
453
|
-
}
|
|
454
|
-
throw new Error(`Workflow config not found for ${name}`);
|
|
455
|
-
}
|
|
456
|
-
processInputData(name, input_data) {
|
|
457
|
-
let workflowId = name;
|
|
458
|
-
if (input_data) {
|
|
459
|
-
const hash = (0, import_node_crypto3.createHash)("sha256").update(JSON.stringify(input_data)).digest("hex").slice(0, 16);
|
|
460
|
-
workflowId = `${name}-${hash}`;
|
|
461
|
-
}
|
|
462
|
-
return [input_data, workflowId];
|
|
463
|
-
}
|
|
464
|
-
};
|
|
465
520
|
async function getTemporalClient(temporalUrl, namespace, clientCert, clientKey, apiKey) {
|
|
466
521
|
try {
|
|
467
522
|
console.info(
|
|
@@ -498,538 +553,989 @@ async function getTemporalClient(temporalUrl, namespace, clientCert, clientKey,
|
|
|
498
553
|
return void 0;
|
|
499
554
|
}
|
|
500
555
|
}
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
this.workerStart = options.workerStart;
|
|
542
|
-
this.workerStop = options.workerStop;
|
|
543
|
-
if (options.maxCpuUsageRatio && (options.maxCpuUsageRatio > 1 || options.maxCpuUsageRatio < 0)) {
|
|
544
|
-
throw new Error("maxCpuUsageRatio must be between 0 and 1");
|
|
545
|
-
}
|
|
546
|
-
this.maxCpuUsageRatio = options.maxCpuUsageRatio || DEFAULT_MAX_CPU_USAGE_RATIO;
|
|
547
|
-
this.usedCpuCount = this.computeCPUUsageCount(
|
|
548
|
-
this.maxCpuUsageRatio,
|
|
549
|
-
options.maxWorkerCount
|
|
550
|
-
);
|
|
551
|
-
}
|
|
552
|
-
/**
|
|
553
|
-
* Calculates the number of CPU cores to utilize based on available parallelism and constraints.
|
|
554
|
-
*
|
|
555
|
-
* @param cpuUsageRatio - Ratio of CPU cores to use (0-1)
|
|
556
|
-
* @param maxWorkerCount - Optional maximum number of workers
|
|
557
|
-
* @returns The number of CPU cores to utilize
|
|
558
|
-
*/
|
|
559
|
-
computeCPUUsageCount(cpuUsageRatio, maxWorkerCount) {
|
|
560
|
-
const cpuCount = (0, import_node_os.availableParallelism)();
|
|
561
|
-
const maxWorkers = maxWorkerCount || cpuCount;
|
|
562
|
-
return Math.min(
|
|
563
|
-
maxWorkers,
|
|
564
|
-
Math.max(1, Math.floor(cpuCount * cpuUsageRatio))
|
|
565
|
-
);
|
|
566
|
-
}
|
|
567
|
-
/**
|
|
568
|
-
* Initializes the cluster by spawning worker processes and setting up signal handlers.
|
|
569
|
-
* For the primary process, spawns workers and monitors parent process.
|
|
570
|
-
* For worker processes, executes the worker startup function.
|
|
571
|
-
*
|
|
572
|
-
* @throws {Error} If worker is undefined in worker process
|
|
573
|
-
*/
|
|
574
|
-
async start() {
|
|
575
|
-
process.on(SIGTERM, this.gracefulClusterShutdown(SIGTERM));
|
|
576
|
-
process.on(SIGINT, this.gracefulClusterShutdown(SIGINT));
|
|
577
|
-
if (import_node_cluster.default.isPrimary) {
|
|
578
|
-
const parentPid = process.ppid;
|
|
579
|
-
setInterval(() => {
|
|
580
|
-
try {
|
|
581
|
-
process.kill(parentPid, 0);
|
|
582
|
-
} catch (e) {
|
|
583
|
-
console.log("Parent process has exited.");
|
|
584
|
-
this.gracefulClusterShutdown(SIGTERM)();
|
|
585
|
-
}
|
|
586
|
-
}, 1e3);
|
|
587
|
-
await this.bootWorkers(this.usedCpuCount);
|
|
588
|
-
} else {
|
|
589
|
-
if (!import_node_cluster.default.worker) {
|
|
590
|
-
throw new Error(
|
|
591
|
-
"Worker is not defined, it should be defined in worker process"
|
|
556
|
+
var import_client2, import_node_crypto3, import_perf_hooks, fs, MooseClient, QueryClient, WorkflowClient;
|
|
557
|
+
var init_helpers2 = __esm({
|
|
558
|
+
"src/consumption-apis/helpers.ts"() {
|
|
559
|
+
"use strict";
|
|
560
|
+
import_client2 = require("@temporalio/client");
|
|
561
|
+
import_node_crypto3 = require("crypto");
|
|
562
|
+
import_perf_hooks = require("perf_hooks");
|
|
563
|
+
fs = __toESM(require("fs"));
|
|
564
|
+
init_internal();
|
|
565
|
+
init_sqlHelpers();
|
|
566
|
+
MooseClient = class {
|
|
567
|
+
query;
|
|
568
|
+
workflow;
|
|
569
|
+
constructor(queryClient, temporalClient) {
|
|
570
|
+
this.query = queryClient;
|
|
571
|
+
this.workflow = new WorkflowClient(temporalClient);
|
|
572
|
+
}
|
|
573
|
+
};
|
|
574
|
+
QueryClient = class {
|
|
575
|
+
client;
|
|
576
|
+
query_id_prefix;
|
|
577
|
+
constructor(client, query_id_prefix) {
|
|
578
|
+
this.client = client;
|
|
579
|
+
this.query_id_prefix = query_id_prefix;
|
|
580
|
+
}
|
|
581
|
+
async execute(sql3) {
|
|
582
|
+
const [query, query_params] = toQuery(sql3);
|
|
583
|
+
console.log(`[QueryClient] | Query: ${toQueryPreview(sql3)}`);
|
|
584
|
+
const start = import_perf_hooks.performance.now();
|
|
585
|
+
const result = await this.client.query({
|
|
586
|
+
query,
|
|
587
|
+
query_params,
|
|
588
|
+
format: "JSONEachRow",
|
|
589
|
+
query_id: this.query_id_prefix + (0, import_node_crypto3.randomUUID)()
|
|
590
|
+
// Note: wait_end_of_query deliberately NOT set here as this is used for SELECT queries
|
|
591
|
+
// where response buffering would harm streaming performance and concurrency
|
|
592
|
+
});
|
|
593
|
+
const elapsedMs = import_perf_hooks.performance.now() - start;
|
|
594
|
+
console.log(
|
|
595
|
+
`[QueryClient] | Query completed: ${formatElapsedTime(elapsedMs)}`
|
|
592
596
|
);
|
|
597
|
+
return result;
|
|
593
598
|
}
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
console.info(`Setting ${numWorkers} workers...`);
|
|
609
|
-
for (let i = 0; i < numWorkers; i++) {
|
|
610
|
-
import_node_cluster.default.fork();
|
|
611
|
-
}
|
|
612
|
-
import_node_cluster.default.on("online", (worker) => {
|
|
613
|
-
console.info(`worker process ${worker.process.pid} is online`);
|
|
614
|
-
});
|
|
615
|
-
import_node_cluster.default.on("exit", (worker, code, signal) => {
|
|
616
|
-
console.info(
|
|
617
|
-
`worker ${worker.process.pid} exited with code ${code} and signal ${signal}`
|
|
618
|
-
);
|
|
619
|
-
if (!this.shutdownInProgress) {
|
|
620
|
-
setTimeout(() => import_node_cluster.default.fork(), RESTART_TIME_MS);
|
|
599
|
+
async command(sql3) {
|
|
600
|
+
const [query, query_params] = toQuery(sql3);
|
|
601
|
+
console.log(`[QueryClient] | Command: ${toQueryPreview(sql3)}`);
|
|
602
|
+
const start = import_perf_hooks.performance.now();
|
|
603
|
+
const result = await this.client.command({
|
|
604
|
+
query,
|
|
605
|
+
query_params,
|
|
606
|
+
query_id: this.query_id_prefix + (0, import_node_crypto3.randomUUID)()
|
|
607
|
+
});
|
|
608
|
+
const elapsedMs = import_perf_hooks.performance.now() - start;
|
|
609
|
+
console.log(
|
|
610
|
+
`[QueryClient] | Command completed: ${formatElapsedTime(elapsedMs)}`
|
|
611
|
+
);
|
|
612
|
+
return result;
|
|
621
613
|
}
|
|
622
|
-
|
|
623
|
-
|
|
614
|
+
};
|
|
615
|
+
WorkflowClient = class {
|
|
616
|
+
client;
|
|
617
|
+
constructor(temporalClient) {
|
|
618
|
+
this.client = temporalClient;
|
|
624
619
|
}
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
*/
|
|
638
|
-
gracefulClusterShutdown = (signal) => async () => {
|
|
639
|
-
if (this.shutdownInProgress) {
|
|
640
|
-
return;
|
|
641
|
-
}
|
|
642
|
-
this.shutdownInProgress = true;
|
|
643
|
-
this.hasCleanWorkerExit = true;
|
|
644
|
-
console.info(
|
|
645
|
-
`Got ${signal} on ${this.processStr}. Graceful shutdown start at ${(/* @__PURE__ */ new Date()).toISOString()}`
|
|
646
|
-
);
|
|
647
|
-
try {
|
|
648
|
-
if (import_node_cluster.default.isPrimary) {
|
|
649
|
-
await this.shutdownWorkers(signal);
|
|
650
|
-
console.info(`${this.processStr} - worker shutdown successful`);
|
|
651
|
-
(0, import_node_process.exit)(0);
|
|
652
|
-
} else {
|
|
653
|
-
if (this.startOutput) {
|
|
654
|
-
await this.workerStop(this.startOutput);
|
|
655
|
-
} else {
|
|
656
|
-
console.info(
|
|
657
|
-
`${this.processStr} - shutdown before worker fully started`
|
|
620
|
+
async execute(name, input_data) {
|
|
621
|
+
try {
|
|
622
|
+
if (!this.client) {
|
|
623
|
+
return {
|
|
624
|
+
status: 404,
|
|
625
|
+
body: `Temporal client not found. Is the feature flag enabled?`
|
|
626
|
+
};
|
|
627
|
+
}
|
|
628
|
+
const config = await this.getWorkflowConfig(name);
|
|
629
|
+
const [processedInput, workflowId] = this.processInputData(
|
|
630
|
+
name,
|
|
631
|
+
input_data
|
|
658
632
|
);
|
|
633
|
+
console.log(
|
|
634
|
+
`WorkflowClient - starting workflow: ${name} with config ${JSON.stringify(config)} and input_data ${JSON.stringify(processedInput)}`
|
|
635
|
+
);
|
|
636
|
+
const handle = await this.client.workflow.start("ScriptWorkflow", {
|
|
637
|
+
args: [
|
|
638
|
+
{ workflow_name: name, execution_mode: "start" },
|
|
639
|
+
processedInput
|
|
640
|
+
],
|
|
641
|
+
taskQueue: "typescript-script-queue",
|
|
642
|
+
workflowId,
|
|
643
|
+
workflowIdConflictPolicy: "FAIL",
|
|
644
|
+
workflowIdReusePolicy: "ALLOW_DUPLICATE",
|
|
645
|
+
retry: {
|
|
646
|
+
maximumAttempts: config.retries
|
|
647
|
+
},
|
|
648
|
+
workflowRunTimeout: config.timeout
|
|
649
|
+
});
|
|
650
|
+
return {
|
|
651
|
+
status: 200,
|
|
652
|
+
body: `Workflow started: ${name}. View it in the Temporal dashboard: http://localhost:8080/namespaces/default/workflows/${workflowId}/${handle.firstExecutionRunId}/history`
|
|
653
|
+
};
|
|
654
|
+
} catch (error) {
|
|
655
|
+
return {
|
|
656
|
+
status: 400,
|
|
657
|
+
body: `Error starting workflow: ${error}`
|
|
658
|
+
};
|
|
659
659
|
}
|
|
660
|
-
console.info(`${this.processStr} shutdown successful`);
|
|
661
|
-
this.hasCleanWorkerExit ? (0, import_node_process.exit)(0) : (0, import_node_process.exit)(1);
|
|
662
|
-
}
|
|
663
|
-
} catch (e) {
|
|
664
|
-
console.error(`${this.processStr} - shutdown failed`, e);
|
|
665
|
-
(0, import_node_process.exit)(1);
|
|
666
|
-
}
|
|
667
|
-
};
|
|
668
|
-
/**
|
|
669
|
-
* Gracefully terminates all worker processes.
|
|
670
|
-
* Monitors workers until they all exit or timeout occurs.
|
|
671
|
-
* Only relevant for the primary process.
|
|
672
|
-
*
|
|
673
|
-
* @param signal - The signal to send to worker processes
|
|
674
|
-
* @returns A promise that resolves when all workers have terminated
|
|
675
|
-
*/
|
|
676
|
-
shutdownWorkers = (signal) => {
|
|
677
|
-
return new Promise((resolve2, reject) => {
|
|
678
|
-
if (!import_node_cluster.default.isPrimary) {
|
|
679
|
-
return resolve2();
|
|
680
660
|
}
|
|
681
|
-
|
|
682
|
-
|
|
661
|
+
async terminate(workflowId) {
|
|
662
|
+
try {
|
|
663
|
+
if (!this.client) {
|
|
664
|
+
return {
|
|
665
|
+
status: 404,
|
|
666
|
+
body: `Temporal client not found. Is the feature flag enabled?`
|
|
667
|
+
};
|
|
668
|
+
}
|
|
669
|
+
const handle = this.client.workflow.getHandle(workflowId);
|
|
670
|
+
await handle.terminate();
|
|
671
|
+
return {
|
|
672
|
+
status: 200,
|
|
673
|
+
body: `Workflow terminated: ${workflowId}`
|
|
674
|
+
};
|
|
675
|
+
} catch (error) {
|
|
676
|
+
return {
|
|
677
|
+
status: 400,
|
|
678
|
+
body: `Error terminating workflow: ${error}`
|
|
679
|
+
};
|
|
680
|
+
}
|
|
683
681
|
}
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
682
|
+
async getWorkflowConfig(name) {
|
|
683
|
+
const workflows = await getWorkflows2();
|
|
684
|
+
const dmv2Workflow = workflows.get(name);
|
|
685
|
+
if (dmv2Workflow) {
|
|
686
|
+
return {
|
|
687
|
+
retries: dmv2Workflow.config.retries || 3,
|
|
688
|
+
timeout: dmv2Workflow.config.timeout || "1h"
|
|
689
|
+
};
|
|
690
|
+
}
|
|
691
|
+
throw new Error(`Workflow config not found for ${name}`);
|
|
687
692
|
}
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
Object.values(import_node_cluster.default.workers || {}).filter((worker) => !!worker).forEach((worker) => {
|
|
694
|
-
if (worker && !worker.isDead()) {
|
|
695
|
-
++workersAlive;
|
|
696
|
-
if (funcRun == 1) {
|
|
697
|
-
worker.kill(signal);
|
|
698
|
-
}
|
|
699
|
-
}
|
|
700
|
-
});
|
|
701
|
-
console.info(workersAlive + " workers alive");
|
|
702
|
-
if (workersAlive == 0) {
|
|
703
|
-
clearInterval(interval);
|
|
704
|
-
return resolve2();
|
|
693
|
+
processInputData(name, input_data) {
|
|
694
|
+
let workflowId = name;
|
|
695
|
+
if (input_data) {
|
|
696
|
+
const hash = (0, import_node_crypto3.createHash)("sha256").update(JSON.stringify(input_data)).digest("hex").slice(0, 16);
|
|
697
|
+
workflowId = `${name}-${hash}`;
|
|
705
698
|
}
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
}
|
|
709
|
-
}
|
|
710
|
-
};
|
|
699
|
+
return [input_data, workflowId];
|
|
700
|
+
}
|
|
701
|
+
};
|
|
702
|
+
}
|
|
703
|
+
});
|
|
711
704
|
|
|
712
|
-
// src/consumption-apis/
|
|
713
|
-
var
|
|
714
|
-
|
|
715
|
-
|
|
705
|
+
// src/consumption-apis/webAppHelpers.ts
|
|
706
|
+
var init_webAppHelpers = __esm({
|
|
707
|
+
"src/consumption-apis/webAppHelpers.ts"() {
|
|
708
|
+
"use strict";
|
|
709
|
+
}
|
|
716
710
|
});
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
}
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
711
|
+
|
|
712
|
+
// src/scripts/task.ts
|
|
713
|
+
var init_task = __esm({
|
|
714
|
+
"src/scripts/task.ts"() {
|
|
715
|
+
"use strict";
|
|
716
|
+
}
|
|
717
|
+
});
|
|
718
|
+
|
|
719
|
+
// src/cluster-utils.ts
|
|
720
|
+
var import_node_cluster, import_node_os, import_node_process, DEFAULT_MAX_CPU_USAGE_RATIO, RESTART_TIME_MS, SIGTERM, SIGINT, SHUTDOWN_WORKERS_INTERVAL, Cluster;
|
|
721
|
+
var init_cluster_utils = __esm({
|
|
722
|
+
"src/cluster-utils.ts"() {
|
|
723
|
+
"use strict";
|
|
724
|
+
import_node_cluster = __toESM(require("cluster"));
|
|
725
|
+
import_node_os = require("os");
|
|
726
|
+
import_node_process = require("process");
|
|
727
|
+
DEFAULT_MAX_CPU_USAGE_RATIO = 0.7;
|
|
728
|
+
RESTART_TIME_MS = 1e4;
|
|
729
|
+
SIGTERM = "SIGTERM";
|
|
730
|
+
SIGINT = "SIGINT";
|
|
731
|
+
SHUTDOWN_WORKERS_INTERVAL = 500;
|
|
732
|
+
Cluster = class {
|
|
733
|
+
// Tracks if shutdown is currently in progress
|
|
734
|
+
shutdownInProgress = false;
|
|
735
|
+
// Tracks if workers exited cleanly during shutdown
|
|
736
|
+
hasCleanWorkerExit = true;
|
|
737
|
+
// String identifying if this is primary or worker process
|
|
738
|
+
processStr = `${import_node_cluster.default.isPrimary ? "primary" : "worker"} process ${process.pid}`;
|
|
739
|
+
// Functions for starting and stopping workers
|
|
740
|
+
workerStart;
|
|
741
|
+
workerStop;
|
|
742
|
+
// Result from starting worker, needed for cleanup
|
|
743
|
+
startOutput;
|
|
744
|
+
maxCpuUsageRatio;
|
|
745
|
+
usedCpuCount;
|
|
746
|
+
/**
|
|
747
|
+
* Creates a new cluster manager instance.
|
|
748
|
+
*
|
|
749
|
+
* @param options - Configuration options for the cluster
|
|
750
|
+
* @param options.workerStart - Async function to execute when starting a worker
|
|
751
|
+
* @param options.workerStop - Async function to execute when stopping a worker
|
|
752
|
+
* @param options.maxCpuUsageRatio - Maximum ratio of CPU cores to utilize (0-1)
|
|
753
|
+
* @param options.maxWorkerCount - Maximum number of workers to spawn
|
|
754
|
+
* @throws {Error} If maxCpuUsageRatio is not between 0 and 1
|
|
755
|
+
*/
|
|
756
|
+
constructor(options) {
|
|
757
|
+
this.workerStart = options.workerStart;
|
|
758
|
+
this.workerStop = options.workerStop;
|
|
759
|
+
if (options.maxCpuUsageRatio && (options.maxCpuUsageRatio > 1 || options.maxCpuUsageRatio < 0)) {
|
|
760
|
+
throw new Error("maxCpuUsageRatio must be between 0 and 1");
|
|
761
|
+
}
|
|
762
|
+
this.maxCpuUsageRatio = options.maxCpuUsageRatio || DEFAULT_MAX_CPU_USAGE_RATIO;
|
|
763
|
+
this.usedCpuCount = this.computeCPUUsageCount(
|
|
764
|
+
this.maxCpuUsageRatio,
|
|
765
|
+
options.maxWorkerCount
|
|
766
|
+
);
|
|
767
|
+
}
|
|
768
|
+
/**
|
|
769
|
+
* Calculates the number of CPU cores to utilize based on available parallelism and constraints.
|
|
770
|
+
*
|
|
771
|
+
* @param cpuUsageRatio - Ratio of CPU cores to use (0-1)
|
|
772
|
+
* @param maxWorkerCount - Optional maximum number of workers
|
|
773
|
+
* @returns The number of CPU cores to utilize
|
|
774
|
+
*/
|
|
775
|
+
computeCPUUsageCount(cpuUsageRatio, maxWorkerCount) {
|
|
776
|
+
const cpuCount = (0, import_node_os.availableParallelism)();
|
|
777
|
+
const maxWorkers = maxWorkerCount || cpuCount;
|
|
778
|
+
return Math.min(
|
|
779
|
+
maxWorkers,
|
|
780
|
+
Math.max(1, Math.floor(cpuCount * cpuUsageRatio))
|
|
781
|
+
);
|
|
782
|
+
}
|
|
783
|
+
/**
|
|
784
|
+
* Initializes the cluster by spawning worker processes and setting up signal handlers.
|
|
785
|
+
* For the primary process, spawns workers and monitors parent process.
|
|
786
|
+
* For worker processes, executes the worker startup function.
|
|
787
|
+
*
|
|
788
|
+
* @throws {Error} If worker is undefined in worker process
|
|
789
|
+
*/
|
|
790
|
+
async start() {
|
|
791
|
+
process.on(SIGTERM, this.gracefulClusterShutdown(SIGTERM));
|
|
792
|
+
process.on(SIGINT, this.gracefulClusterShutdown(SIGINT));
|
|
793
|
+
if (import_node_cluster.default.isPrimary) {
|
|
794
|
+
const parentPid = process.ppid;
|
|
795
|
+
setInterval(() => {
|
|
796
|
+
try {
|
|
797
|
+
process.kill(parentPid, 0);
|
|
798
|
+
} catch (e) {
|
|
799
|
+
console.log("Parent process has exited.");
|
|
800
|
+
this.gracefulClusterShutdown(SIGTERM)();
|
|
748
801
|
}
|
|
802
|
+
}, 1e3);
|
|
803
|
+
await this.bootWorkers(this.usedCpuCount);
|
|
804
|
+
} else {
|
|
805
|
+
if (!import_node_cluster.default.worker) {
|
|
806
|
+
throw new Error(
|
|
807
|
+
"Worker is not defined, it should be defined in worker process"
|
|
808
|
+
);
|
|
749
809
|
}
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
return;
|
|
810
|
+
this.startOutput = await this.workerStart(
|
|
811
|
+
import_node_cluster.default.worker,
|
|
812
|
+
this.usedCpuCount
|
|
813
|
+
);
|
|
755
814
|
}
|
|
756
|
-
} else if (enforceAuth) {
|
|
757
|
-
res.writeHead(401, { "Content-Type": "application/json" });
|
|
758
|
-
res.end(JSON.stringify({ error: "Unauthorized" }));
|
|
759
|
-
httpLogger(req, res, start);
|
|
760
|
-
return;
|
|
761
815
|
}
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
816
|
+
/**
|
|
817
|
+
* Spawns worker processes and configures their lifecycle event handlers.
|
|
818
|
+
* Handles worker online, exit and disconnect events.
|
|
819
|
+
* Automatically restarts failed workers during normal operation.
|
|
820
|
+
*
|
|
821
|
+
* @param numWorkers - Number of worker processes to spawn
|
|
822
|
+
*/
|
|
823
|
+
bootWorkers = async (numWorkers) => {
|
|
824
|
+
console.info(`Setting ${numWorkers} workers...`);
|
|
825
|
+
for (let i = 0; i < numWorkers; i++) {
|
|
826
|
+
import_node_cluster.default.fork();
|
|
827
|
+
}
|
|
828
|
+
import_node_cluster.default.on("online", (worker) => {
|
|
829
|
+
console.info(`worker process ${worker.process.pid} is online`);
|
|
830
|
+
});
|
|
831
|
+
import_node_cluster.default.on("exit", (worker, code, signal) => {
|
|
832
|
+
console.info(
|
|
833
|
+
`worker ${worker.process.pid} exited with code ${code} and signal ${signal}`
|
|
834
|
+
);
|
|
835
|
+
if (!this.shutdownInProgress) {
|
|
836
|
+
setTimeout(() => import_node_cluster.default.fork(), RESTART_TIME_MS);
|
|
837
|
+
}
|
|
838
|
+
if (this.shutdownInProgress && code != 0) {
|
|
839
|
+
this.hasCleanWorkerExit = false;
|
|
840
|
+
}
|
|
841
|
+
});
|
|
842
|
+
import_node_cluster.default.on("disconnect", (worker) => {
|
|
843
|
+
console.info(`worker process ${worker.process.pid} has disconnected`);
|
|
844
|
+
});
|
|
845
|
+
};
|
|
846
|
+
/**
|
|
847
|
+
* Creates a handler function for graceful shutdown on receipt of a signal.
|
|
848
|
+
* Ensures only one shutdown can occur at a time.
|
|
849
|
+
* Handles shutdown differently for primary and worker processes.
|
|
850
|
+
*
|
|
851
|
+
* @param signal - The signal triggering the shutdown (e.g. SIGTERM)
|
|
852
|
+
* @returns An async function that performs the shutdown
|
|
853
|
+
*/
|
|
854
|
+
gracefulClusterShutdown = (signal) => async () => {
|
|
855
|
+
if (this.shutdownInProgress) {
|
|
856
|
+
return;
|
|
857
|
+
}
|
|
858
|
+
this.shutdownInProgress = true;
|
|
859
|
+
this.hasCleanWorkerExit = true;
|
|
860
|
+
console.info(
|
|
861
|
+
`Got ${signal} on ${this.processStr}. Graceful shutdown start at ${(/* @__PURE__ */ new Date()).toISOString()}`
|
|
862
|
+
);
|
|
863
|
+
try {
|
|
864
|
+
if (import_node_cluster.default.isPrimary) {
|
|
865
|
+
await this.shutdownWorkers(signal);
|
|
866
|
+
console.info(`${this.processStr} - worker shutdown successful`);
|
|
867
|
+
(0, import_node_process.exit)(0);
|
|
868
|
+
} else {
|
|
869
|
+
if (this.startOutput) {
|
|
870
|
+
await this.workerStop(this.startOutput);
|
|
769
871
|
} else {
|
|
770
|
-
|
|
872
|
+
console.info(
|
|
873
|
+
`${this.processStr} - shutdown before worker fully started`
|
|
874
|
+
);
|
|
771
875
|
}
|
|
772
|
-
|
|
773
|
-
|
|
876
|
+
console.info(`${this.processStr} shutdown successful`);
|
|
877
|
+
this.hasCleanWorkerExit ? (0, import_node_process.exit)(0) : (0, import_node_process.exit)(1);
|
|
774
878
|
}
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
879
|
+
} catch (e) {
|
|
880
|
+
console.error(`${this.processStr} - shutdown failed`, e);
|
|
881
|
+
(0, import_node_process.exit)(1);
|
|
882
|
+
}
|
|
883
|
+
};
|
|
884
|
+
/**
|
|
885
|
+
* Gracefully terminates all worker processes.
|
|
886
|
+
* Monitors workers until they all exit or timeout occurs.
|
|
887
|
+
* Only relevant for the primary process.
|
|
888
|
+
*
|
|
889
|
+
* @param signal - The signal to send to worker processes
|
|
890
|
+
* @returns A promise that resolves when all workers have terminated
|
|
891
|
+
*/
|
|
892
|
+
shutdownWorkers = (signal) => {
|
|
893
|
+
return new Promise((resolve2, reject) => {
|
|
894
|
+
if (!import_node_cluster.default.isPrimary) {
|
|
895
|
+
return resolve2();
|
|
896
|
+
}
|
|
897
|
+
if (!import_node_cluster.default.workers) {
|
|
898
|
+
return resolve2();
|
|
899
|
+
}
|
|
900
|
+
const workerIds = Object.keys(import_node_cluster.default.workers);
|
|
901
|
+
if (workerIds.length == 0) {
|
|
902
|
+
return resolve2();
|
|
903
|
+
}
|
|
904
|
+
let workersAlive = 0;
|
|
905
|
+
let funcRun = 0;
|
|
906
|
+
const cleanWorkers = () => {
|
|
907
|
+
++funcRun;
|
|
908
|
+
workersAlive = 0;
|
|
909
|
+
Object.values(import_node_cluster.default.workers || {}).filter((worker) => !!worker).forEach((worker) => {
|
|
910
|
+
if (worker && !worker.isDead()) {
|
|
911
|
+
++workersAlive;
|
|
912
|
+
if (funcRun == 1) {
|
|
913
|
+
worker.kill(signal);
|
|
794
914
|
}
|
|
795
915
|
}
|
|
916
|
+
});
|
|
917
|
+
console.info(workersAlive + " workers alive");
|
|
918
|
+
if (workersAlive == 0) {
|
|
919
|
+
clearInterval(interval);
|
|
920
|
+
return resolve2();
|
|
796
921
|
}
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
922
|
+
};
|
|
923
|
+
const interval = setInterval(cleanWorkers, SHUTDOWN_WORKERS_INTERVAL);
|
|
924
|
+
});
|
|
925
|
+
};
|
|
926
|
+
};
|
|
927
|
+
}
|
|
928
|
+
});
|
|
929
|
+
|
|
930
|
+
// src/config/configFile.ts
|
|
931
|
+
async function findConfigFile(startDir = process.cwd()) {
|
|
932
|
+
const fs4 = await import("fs");
|
|
933
|
+
let currentDir = import_node_path.default.resolve(startDir);
|
|
934
|
+
while (true) {
|
|
935
|
+
const configPath = import_node_path.default.join(currentDir, "moose.config.toml");
|
|
936
|
+
if (fs4.existsSync(configPath)) {
|
|
937
|
+
return configPath;
|
|
938
|
+
}
|
|
939
|
+
const parentDir = import_node_path.default.dirname(currentDir);
|
|
940
|
+
if (parentDir === currentDir) {
|
|
941
|
+
break;
|
|
942
|
+
}
|
|
943
|
+
currentDir = parentDir;
|
|
944
|
+
}
|
|
945
|
+
return null;
|
|
946
|
+
}
|
|
947
|
+
async function readProjectConfig() {
|
|
948
|
+
const fs4 = await import("fs");
|
|
949
|
+
const configPath = await findConfigFile();
|
|
950
|
+
if (!configPath) {
|
|
951
|
+
throw new ConfigError(
|
|
952
|
+
"moose.config.toml not found in current directory or any parent directory"
|
|
953
|
+
);
|
|
954
|
+
}
|
|
955
|
+
try {
|
|
956
|
+
const configContent = fs4.readFileSync(configPath, "utf-8");
|
|
957
|
+
const config = toml.parse(configContent);
|
|
958
|
+
return config;
|
|
959
|
+
} catch (error) {
|
|
960
|
+
throw new ConfigError(`Failed to parse moose.config.toml: ${error}`);
|
|
961
|
+
}
|
|
962
|
+
}
|
|
963
|
+
var import_node_path, toml, ConfigError;
|
|
964
|
+
var init_configFile = __esm({
|
|
965
|
+
"src/config/configFile.ts"() {
|
|
966
|
+
"use strict";
|
|
967
|
+
import_node_path = __toESM(require("path"));
|
|
968
|
+
toml = __toESM(require("toml"));
|
|
969
|
+
ConfigError = class extends Error {
|
|
970
|
+
constructor(message) {
|
|
971
|
+
super(message);
|
|
972
|
+
this.name = "ConfigError";
|
|
973
|
+
}
|
|
974
|
+
};
|
|
975
|
+
}
|
|
976
|
+
});
|
|
977
|
+
|
|
978
|
+
// src/config/runtime.ts
|
|
979
|
+
var runtime_exports = {};
|
|
980
|
+
var ConfigurationRegistry;
|
|
981
|
+
var init_runtime = __esm({
|
|
982
|
+
"src/config/runtime.ts"() {
|
|
983
|
+
"use strict";
|
|
984
|
+
init_configFile();
|
|
985
|
+
ConfigurationRegistry = class _ConfigurationRegistry {
|
|
986
|
+
static instance;
|
|
987
|
+
clickhouseConfig;
|
|
988
|
+
kafkaConfig;
|
|
989
|
+
static getInstance() {
|
|
990
|
+
if (!_ConfigurationRegistry.instance) {
|
|
991
|
+
_ConfigurationRegistry.instance = new _ConfigurationRegistry();
|
|
818
992
|
}
|
|
993
|
+
return _ConfigurationRegistry.instance;
|
|
819
994
|
}
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
}
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
995
|
+
setClickHouseConfig(config) {
|
|
996
|
+
this.clickhouseConfig = config;
|
|
997
|
+
}
|
|
998
|
+
setKafkaConfig(config) {
|
|
999
|
+
this.kafkaConfig = config;
|
|
1000
|
+
}
|
|
1001
|
+
_env(name) {
|
|
1002
|
+
const value = process.env[name];
|
|
1003
|
+
if (value === void 0) return void 0;
|
|
1004
|
+
const trimmed = value.trim();
|
|
1005
|
+
return trimmed.length > 0 ? trimmed : void 0;
|
|
1006
|
+
}
|
|
1007
|
+
_parseBool(value) {
|
|
1008
|
+
if (value === void 0) return void 0;
|
|
1009
|
+
switch (value.trim().toLowerCase()) {
|
|
1010
|
+
case "1":
|
|
1011
|
+
case "true":
|
|
1012
|
+
case "yes":
|
|
1013
|
+
case "on":
|
|
1014
|
+
return true;
|
|
1015
|
+
case "0":
|
|
1016
|
+
case "false":
|
|
1017
|
+
case "no":
|
|
1018
|
+
case "off":
|
|
1019
|
+
return false;
|
|
1020
|
+
default:
|
|
1021
|
+
return void 0;
|
|
840
1022
|
}
|
|
841
1023
|
}
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
1024
|
+
async getClickHouseConfig() {
|
|
1025
|
+
if (this.clickhouseConfig) {
|
|
1026
|
+
return this.clickhouseConfig;
|
|
1027
|
+
}
|
|
1028
|
+
const projectConfig = await readProjectConfig();
|
|
1029
|
+
const envHost = this._env("MOOSE_CLICKHOUSE_CONFIG__HOST");
|
|
1030
|
+
const envPort = this._env("MOOSE_CLICKHOUSE_CONFIG__HOST_PORT");
|
|
1031
|
+
const envUser = this._env("MOOSE_CLICKHOUSE_CONFIG__USER");
|
|
1032
|
+
const envPassword = this._env("MOOSE_CLICKHOUSE_CONFIG__PASSWORD");
|
|
1033
|
+
const envDb = this._env("MOOSE_CLICKHOUSE_CONFIG__DB_NAME");
|
|
1034
|
+
const envUseSSL = this._parseBool(
|
|
1035
|
+
this._env("MOOSE_CLICKHOUSE_CONFIG__USE_SSL")
|
|
1036
|
+
);
|
|
1037
|
+
return {
|
|
1038
|
+
host: envHost ?? projectConfig.clickhouse_config.host,
|
|
1039
|
+
port: envPort ?? projectConfig.clickhouse_config.host_port.toString(),
|
|
1040
|
+
username: envUser ?? projectConfig.clickhouse_config.user,
|
|
1041
|
+
password: envPassword ?? projectConfig.clickhouse_config.password,
|
|
1042
|
+
database: envDb ?? projectConfig.clickhouse_config.db_name,
|
|
1043
|
+
useSSL: envUseSSL !== void 0 ? envUseSSL : projectConfig.clickhouse_config.use_ssl || false
|
|
1044
|
+
};
|
|
848
1045
|
}
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
1046
|
+
async getStandaloneClickhouseConfig(overrides) {
|
|
1047
|
+
if (this.clickhouseConfig) {
|
|
1048
|
+
return { ...this.clickhouseConfig, ...overrides };
|
|
1049
|
+
}
|
|
1050
|
+
const envHost = this._env("MOOSE_CLICKHOUSE_CONFIG__HOST");
|
|
1051
|
+
const envPort = this._env("MOOSE_CLICKHOUSE_CONFIG__HOST_PORT");
|
|
1052
|
+
const envUser = this._env("MOOSE_CLICKHOUSE_CONFIG__USER");
|
|
1053
|
+
const envPassword = this._env("MOOSE_CLICKHOUSE_CONFIG__PASSWORD");
|
|
1054
|
+
const envDb = this._env("MOOSE_CLICKHOUSE_CONFIG__DB_NAME");
|
|
1055
|
+
const envUseSSL = this._parseBool(
|
|
1056
|
+
this._env("MOOSE_CLICKHOUSE_CONFIG__USE_SSL")
|
|
1057
|
+
);
|
|
1058
|
+
let projectConfig;
|
|
1059
|
+
try {
|
|
1060
|
+
projectConfig = await readProjectConfig();
|
|
1061
|
+
} catch (error) {
|
|
1062
|
+
projectConfig = null;
|
|
1063
|
+
}
|
|
1064
|
+
const defaults = {
|
|
1065
|
+
host: "localhost",
|
|
1066
|
+
port: "18123",
|
|
1067
|
+
username: "default",
|
|
1068
|
+
password: "",
|
|
1069
|
+
database: "local",
|
|
1070
|
+
useSSL: false
|
|
1071
|
+
};
|
|
1072
|
+
return {
|
|
1073
|
+
host: overrides?.host ?? envHost ?? projectConfig?.clickhouse_config.host ?? defaults.host,
|
|
1074
|
+
port: overrides?.port ?? envPort ?? projectConfig?.clickhouse_config.host_port.toString() ?? defaults.port,
|
|
1075
|
+
username: overrides?.username ?? envUser ?? projectConfig?.clickhouse_config.user ?? defaults.username,
|
|
1076
|
+
password: overrides?.password ?? envPassword ?? projectConfig?.clickhouse_config.password ?? defaults.password,
|
|
1077
|
+
database: overrides?.database ?? envDb ?? projectConfig?.clickhouse_config.db_name ?? defaults.database,
|
|
1078
|
+
useSSL: overrides?.useSSL ?? envUseSSL ?? projectConfig?.clickhouse_config.use_ssl ?? defaults.useSSL
|
|
1079
|
+
};
|
|
856
1080
|
}
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
1081
|
+
async getKafkaConfig() {
|
|
1082
|
+
if (this.kafkaConfig) {
|
|
1083
|
+
return this.kafkaConfig;
|
|
1084
|
+
}
|
|
1085
|
+
const projectConfig = await readProjectConfig();
|
|
1086
|
+
const envBroker = this._env("MOOSE_REDPANDA_CONFIG__BROKER") ?? this._env("MOOSE_KAFKA_CONFIG__BROKER");
|
|
1087
|
+
const envMsgTimeout = this._env("MOOSE_REDPANDA_CONFIG__MESSAGE_TIMEOUT_MS") ?? this._env("MOOSE_KAFKA_CONFIG__MESSAGE_TIMEOUT_MS");
|
|
1088
|
+
const envSaslUsername = this._env("MOOSE_REDPANDA_CONFIG__SASL_USERNAME") ?? this._env("MOOSE_KAFKA_CONFIG__SASL_USERNAME");
|
|
1089
|
+
const envSaslPassword = this._env("MOOSE_REDPANDA_CONFIG__SASL_PASSWORD") ?? this._env("MOOSE_KAFKA_CONFIG__SASL_PASSWORD");
|
|
1090
|
+
const envSaslMechanism = this._env("MOOSE_REDPANDA_CONFIG__SASL_MECHANISM") ?? this._env("MOOSE_KAFKA_CONFIG__SASL_MECHANISM");
|
|
1091
|
+
const envSecurityProtocol = this._env("MOOSE_REDPANDA_CONFIG__SECURITY_PROTOCOL") ?? this._env("MOOSE_KAFKA_CONFIG__SECURITY_PROTOCOL");
|
|
1092
|
+
const envNamespace = this._env("MOOSE_REDPANDA_CONFIG__NAMESPACE") ?? this._env("MOOSE_KAFKA_CONFIG__NAMESPACE");
|
|
1093
|
+
const envSchemaRegistryUrl = this._env("MOOSE_REDPANDA_CONFIG__SCHEMA_REGISTRY_URL") ?? this._env("MOOSE_KAFKA_CONFIG__SCHEMA_REGISTRY_URL");
|
|
1094
|
+
const fileKafka = projectConfig.kafka_config ?? projectConfig.redpanda_config;
|
|
1095
|
+
return {
|
|
1096
|
+
broker: envBroker ?? fileKafka?.broker ?? "localhost:19092",
|
|
1097
|
+
messageTimeoutMs: envMsgTimeout ? parseInt(envMsgTimeout, 10) : fileKafka?.message_timeout_ms ?? 1e3,
|
|
1098
|
+
saslUsername: envSaslUsername ?? fileKafka?.sasl_username,
|
|
1099
|
+
saslPassword: envSaslPassword ?? fileKafka?.sasl_password,
|
|
1100
|
+
saslMechanism: envSaslMechanism ?? fileKafka?.sasl_mechanism,
|
|
1101
|
+
securityProtocol: envSecurityProtocol ?? fileKafka?.security_protocol,
|
|
1102
|
+
namespace: envNamespace ?? fileKafka?.namespace,
|
|
1103
|
+
schemaRegistryUrl: envSchemaRegistryUrl ?? fileKafka?.schema_registry_url
|
|
1104
|
+
};
|
|
1105
|
+
}
|
|
1106
|
+
hasRuntimeConfig() {
|
|
1107
|
+
return !!this.clickhouseConfig || !!this.kafkaConfig;
|
|
865
1108
|
}
|
|
1109
|
+
};
|
|
1110
|
+
globalThis._mooseConfigRegistry = ConfigurationRegistry.getInstance();
|
|
1111
|
+
}
|
|
1112
|
+
});
|
|
1113
|
+
|
|
1114
|
+
// src/consumption-apis/standalone.ts
|
|
1115
|
+
var standalone_exports = {};
|
|
1116
|
+
__export(standalone_exports, {
|
|
1117
|
+
getMooseClients: () => getMooseClients,
|
|
1118
|
+
getMooseUtils: () => getMooseUtils
|
|
1119
|
+
});
|
|
1120
|
+
async function getMooseUtils(req) {
|
|
1121
|
+
if (req !== void 0) {
|
|
1122
|
+
console.warn(
|
|
1123
|
+
"[DEPRECATED] getMooseUtils(req) no longer requires a request parameter. Use getMooseUtils() instead."
|
|
1124
|
+
);
|
|
1125
|
+
}
|
|
1126
|
+
const runtimeContext = globalThis._mooseRuntimeContext;
|
|
1127
|
+
if (runtimeContext) {
|
|
1128
|
+
return {
|
|
1129
|
+
client: runtimeContext.client,
|
|
1130
|
+
sql,
|
|
1131
|
+
jwt: runtimeContext.jwt
|
|
1132
|
+
};
|
|
1133
|
+
}
|
|
1134
|
+
if (standaloneUtils) {
|
|
1135
|
+
return standaloneUtils;
|
|
1136
|
+
}
|
|
1137
|
+
if (initPromise) {
|
|
1138
|
+
return initPromise;
|
|
1139
|
+
}
|
|
1140
|
+
initPromise = (async () => {
|
|
1141
|
+
await Promise.resolve().then(() => (init_runtime(), runtime_exports));
|
|
1142
|
+
const configRegistry = globalThis._mooseConfigRegistry;
|
|
1143
|
+
if (!configRegistry) {
|
|
1144
|
+
throw new Error(
|
|
1145
|
+
"Moose not initialized. Ensure you're running within a Moose app or have proper configuration set up."
|
|
1146
|
+
);
|
|
866
1147
|
}
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
1148
|
+
const clickhouseConfig = await configRegistry.getStandaloneClickhouseConfig();
|
|
1149
|
+
const clickhouseClient = getClickhouseClient(
|
|
1150
|
+
toClientConfig(clickhouseConfig)
|
|
1151
|
+
);
|
|
1152
|
+
const queryClient = new QueryClient(clickhouseClient, "standalone");
|
|
1153
|
+
const mooseClient = new MooseClient(queryClient);
|
|
1154
|
+
standaloneUtils = {
|
|
1155
|
+
client: mooseClient,
|
|
1156
|
+
sql,
|
|
1157
|
+
jwt: void 0
|
|
1158
|
+
};
|
|
1159
|
+
return standaloneUtils;
|
|
1160
|
+
})();
|
|
1161
|
+
try {
|
|
1162
|
+
return await initPromise;
|
|
1163
|
+
} finally {
|
|
1164
|
+
initPromise = null;
|
|
1165
|
+
}
|
|
1166
|
+
}
|
|
1167
|
+
async function getMooseClients(config) {
|
|
1168
|
+
console.warn(
|
|
1169
|
+
"[DEPRECATED] getMooseClients() is deprecated. Use getMooseUtils() instead."
|
|
878
1170
|
);
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
const
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
return async (req, res) => {
|
|
886
|
-
const start = Date.now();
|
|
887
|
-
const url = new URL(req.url || "", "http://localhost");
|
|
888
|
-
const pathname = url.pathname;
|
|
889
|
-
if (pathname === "/_moose_internal/health") {
|
|
890
|
-
res.writeHead(200, { "Content-Type": "application/json" });
|
|
891
|
-
res.end(
|
|
892
|
-
JSON.stringify({
|
|
893
|
-
status: "healthy",
|
|
894
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
895
|
-
})
|
|
1171
|
+
if (config && Object.keys(config).length > 0) {
|
|
1172
|
+
await Promise.resolve().then(() => (init_runtime(), runtime_exports));
|
|
1173
|
+
const configRegistry = globalThis._mooseConfigRegistry;
|
|
1174
|
+
if (!configRegistry) {
|
|
1175
|
+
throw new Error(
|
|
1176
|
+
"Configuration registry not initialized. Ensure the Moose framework is properly set up."
|
|
896
1177
|
);
|
|
897
|
-
return;
|
|
898
1178
|
}
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
1179
|
+
const clickhouseConfig = await configRegistry.getStandaloneClickhouseConfig(config);
|
|
1180
|
+
const clickhouseClient = getClickhouseClient(
|
|
1181
|
+
toClientConfig(clickhouseConfig)
|
|
1182
|
+
);
|
|
1183
|
+
const queryClient = new QueryClient(clickhouseClient, "standalone");
|
|
1184
|
+
const mooseClient = new MooseClient(queryClient);
|
|
1185
|
+
return { client: mooseClient };
|
|
1186
|
+
}
|
|
1187
|
+
const utils = await getMooseUtils();
|
|
1188
|
+
return { client: utils.client };
|
|
1189
|
+
}
|
|
1190
|
+
var standaloneUtils, initPromise, toClientConfig;
|
|
1191
|
+
var init_standalone = __esm({
|
|
1192
|
+
"src/consumption-apis/standalone.ts"() {
|
|
1193
|
+
"use strict";
|
|
1194
|
+
init_helpers2();
|
|
1195
|
+
init_commons();
|
|
1196
|
+
init_sqlHelpers();
|
|
1197
|
+
standaloneUtils = null;
|
|
1198
|
+
initPromise = null;
|
|
1199
|
+
toClientConfig = (config) => ({
|
|
1200
|
+
...config,
|
|
1201
|
+
useSSL: config.useSSL ? "true" : "false"
|
|
1202
|
+
});
|
|
1203
|
+
}
|
|
1204
|
+
});
|
|
1205
|
+
|
|
1206
|
+
// src/consumption-apis/runner.ts
|
|
1207
|
+
var import_http2, jose, toClientConfig2, createPath, httpLogger, modulesCache, apiHandler, createMainRouter, runApis;
|
|
1208
|
+
var init_runner = __esm({
|
|
1209
|
+
"src/consumption-apis/runner.ts"() {
|
|
1210
|
+
"use strict";
|
|
1211
|
+
import_http2 = __toESM(require("http"));
|
|
1212
|
+
init_commons();
|
|
1213
|
+
init_helpers2();
|
|
1214
|
+
jose = __toESM(require("jose"));
|
|
1215
|
+
init_cluster_utils();
|
|
1216
|
+
init_sqlHelpers();
|
|
1217
|
+
init_internal();
|
|
1218
|
+
toClientConfig2 = (config) => ({
|
|
1219
|
+
...config,
|
|
1220
|
+
useSSL: config.useSSL ? "true" : "false"
|
|
1221
|
+
});
|
|
1222
|
+
createPath = (apisDir, path4) => `${apisDir}${path4}.ts`;
|
|
1223
|
+
httpLogger = (req, res, startMs) => {
|
|
1224
|
+
console.log(
|
|
1225
|
+
`${req.method} ${req.url} ${res.statusCode} ${Date.now() - startMs}ms`
|
|
1226
|
+
);
|
|
1227
|
+
};
|
|
1228
|
+
modulesCache = /* @__PURE__ */ new Map();
|
|
1229
|
+
apiHandler = async (publicKey, clickhouseClient, temporalClient, apisDir, enforceAuth, isDmv2, jwtConfig) => {
|
|
1230
|
+
const apis = isDmv2 ? await getApis2() : /* @__PURE__ */ new Map();
|
|
1231
|
+
return async (req, res) => {
|
|
1232
|
+
const start = Date.now();
|
|
903
1233
|
try {
|
|
904
|
-
const
|
|
905
|
-
|
|
906
|
-
|
|
1234
|
+
const url = new URL(req.url || "", "http://localhost");
|
|
1235
|
+
const fileName = url.pathname;
|
|
1236
|
+
let jwtPayload;
|
|
1237
|
+
if (publicKey && jwtConfig) {
|
|
1238
|
+
const jwt = req.headers.authorization?.split(" ")[1];
|
|
1239
|
+
if (jwt) {
|
|
1240
|
+
try {
|
|
1241
|
+
const { payload } = await jose.jwtVerify(jwt, publicKey, {
|
|
1242
|
+
issuer: jwtConfig.issuer,
|
|
1243
|
+
audience: jwtConfig.audience
|
|
1244
|
+
});
|
|
1245
|
+
jwtPayload = payload;
|
|
1246
|
+
} catch (error) {
|
|
1247
|
+
console.log("JWT verification failed");
|
|
1248
|
+
if (enforceAuth) {
|
|
1249
|
+
res.writeHead(401, { "Content-Type": "application/json" });
|
|
1250
|
+
res.end(JSON.stringify({ error: "Unauthorized" }));
|
|
1251
|
+
httpLogger(req, res, start);
|
|
1252
|
+
return;
|
|
1253
|
+
}
|
|
1254
|
+
}
|
|
1255
|
+
} else if (enforceAuth) {
|
|
1256
|
+
res.writeHead(401, { "Content-Type": "application/json" });
|
|
1257
|
+
res.end(JSON.stringify({ error: "Unauthorized" }));
|
|
1258
|
+
httpLogger(req, res, start);
|
|
1259
|
+
return;
|
|
1260
|
+
}
|
|
1261
|
+
} else if (enforceAuth) {
|
|
1262
|
+
res.writeHead(401, { "Content-Type": "application/json" });
|
|
1263
|
+
res.end(JSON.stringify({ error: "Unauthorized" }));
|
|
1264
|
+
httpLogger(req, res, start);
|
|
1265
|
+
return;
|
|
1266
|
+
}
|
|
1267
|
+
const pathName = createPath(apisDir, fileName);
|
|
1268
|
+
const paramsObject = Array.from(url.searchParams.entries()).reduce(
|
|
1269
|
+
(obj, [key, value]) => {
|
|
1270
|
+
const existingValue = obj[key];
|
|
1271
|
+
if (existingValue) {
|
|
1272
|
+
if (Array.isArray(existingValue)) {
|
|
1273
|
+
existingValue.push(value);
|
|
1274
|
+
} else {
|
|
1275
|
+
obj[key] = [existingValue, value];
|
|
1276
|
+
}
|
|
1277
|
+
} else {
|
|
1278
|
+
obj[key] = value;
|
|
1279
|
+
}
|
|
1280
|
+
return obj;
|
|
1281
|
+
},
|
|
1282
|
+
{}
|
|
1283
|
+
);
|
|
1284
|
+
let userFuncModule = modulesCache.get(pathName);
|
|
1285
|
+
if (userFuncModule === void 0) {
|
|
1286
|
+
if (isDmv2) {
|
|
1287
|
+
let apiName = fileName.replace(/^\/+|\/+$/g, "");
|
|
1288
|
+
let version = null;
|
|
1289
|
+
userFuncModule = apis.get(apiName);
|
|
1290
|
+
if (!userFuncModule) {
|
|
1291
|
+
version = url.searchParams.get("version");
|
|
1292
|
+
if (!version && apiName.includes("/")) {
|
|
1293
|
+
const pathParts = apiName.split("/");
|
|
1294
|
+
if (pathParts.length >= 2) {
|
|
1295
|
+
userFuncModule = apis.get(apiName);
|
|
1296
|
+
if (!userFuncModule) {
|
|
1297
|
+
apiName = pathParts[0];
|
|
1298
|
+
version = pathParts.slice(1).join("/");
|
|
1299
|
+
}
|
|
1300
|
+
}
|
|
1301
|
+
}
|
|
1302
|
+
if (!userFuncModule) {
|
|
1303
|
+
if (version) {
|
|
1304
|
+
const versionedKey = `${apiName}:${version}`;
|
|
1305
|
+
userFuncModule = apis.get(versionedKey);
|
|
1306
|
+
} else {
|
|
1307
|
+
userFuncModule = apis.get(apiName);
|
|
1308
|
+
}
|
|
1309
|
+
}
|
|
1310
|
+
}
|
|
1311
|
+
if (!userFuncModule) {
|
|
1312
|
+
const availableApis = Array.from(apis.keys()).map(
|
|
1313
|
+
(key) => key.replace(":", "/")
|
|
1314
|
+
);
|
|
1315
|
+
const errorMessage = version ? `API ${apiName} with version ${version} not found. Available APIs: ${availableApis.join(", ")}` : `API ${apiName} not found. Available APIs: ${availableApis.join(", ")}`;
|
|
1316
|
+
throw new Error(errorMessage);
|
|
1317
|
+
}
|
|
1318
|
+
modulesCache.set(pathName, userFuncModule);
|
|
1319
|
+
console.log(`[API] | Executing API: ${apiName}`);
|
|
1320
|
+
} else {
|
|
1321
|
+
userFuncModule = require(pathName);
|
|
1322
|
+
modulesCache.set(pathName, userFuncModule);
|
|
1323
|
+
}
|
|
1324
|
+
}
|
|
1325
|
+
const queryClient = new QueryClient(clickhouseClient, fileName);
|
|
1326
|
+
let result = isDmv2 ? await userFuncModule(paramsObject, {
|
|
1327
|
+
client: new MooseClient(queryClient, temporalClient),
|
|
1328
|
+
sql,
|
|
1329
|
+
jwt: jwtPayload
|
|
1330
|
+
}) : await userFuncModule.default(paramsObject, {
|
|
1331
|
+
client: new MooseClient(queryClient, temporalClient),
|
|
1332
|
+
sql,
|
|
1333
|
+
jwt: jwtPayload
|
|
907
1334
|
});
|
|
908
|
-
|
|
1335
|
+
let body;
|
|
1336
|
+
let status;
|
|
1337
|
+
if (Object.getPrototypeOf(result).constructor.name === "ResultSet") {
|
|
1338
|
+
body = JSON.stringify(await result.json());
|
|
1339
|
+
} else {
|
|
1340
|
+
if ("body" in result && "status" in result) {
|
|
1341
|
+
body = JSON.stringify(result.body);
|
|
1342
|
+
status = result.status;
|
|
1343
|
+
} else {
|
|
1344
|
+
body = JSON.stringify(result);
|
|
1345
|
+
}
|
|
1346
|
+
}
|
|
1347
|
+
if (status) {
|
|
1348
|
+
res.writeHead(status, { "Content-Type": "application/json" });
|
|
1349
|
+
httpLogger(req, res, start);
|
|
1350
|
+
} else {
|
|
1351
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
1352
|
+
httpLogger(req, res, start);
|
|
1353
|
+
}
|
|
1354
|
+
res.end(body);
|
|
909
1355
|
} catch (error) {
|
|
910
|
-
console.log("
|
|
1356
|
+
console.log("error in path ", req.url, error);
|
|
1357
|
+
if (Object.getPrototypeOf(error).constructor.name === "TypeGuardError") {
|
|
1358
|
+
res.writeHead(400, { "Content-Type": "application/json" });
|
|
1359
|
+
res.end(JSON.stringify({ error: error.message }));
|
|
1360
|
+
httpLogger(req, res, start);
|
|
1361
|
+
}
|
|
1362
|
+
if (error instanceof Error) {
|
|
1363
|
+
res.writeHead(500, { "Content-Type": "application/json" });
|
|
1364
|
+
res.end(JSON.stringify({ error: error.message }));
|
|
1365
|
+
httpLogger(req, res, start);
|
|
1366
|
+
} else {
|
|
1367
|
+
res.writeHead(500, { "Content-Type": "application/json" });
|
|
1368
|
+
res.end();
|
|
1369
|
+
httpLogger(req, res, start);
|
|
1370
|
+
}
|
|
911
1371
|
}
|
|
912
|
-
}
|
|
913
|
-
}
|
|
914
|
-
|
|
915
|
-
const
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
1372
|
+
};
|
|
1373
|
+
};
|
|
1374
|
+
createMainRouter = async (publicKey, clickhouseClient, temporalClient, apisDir, enforceAuth, isDmv2, jwtConfig) => {
|
|
1375
|
+
const apiRequestHandler = await apiHandler(
|
|
1376
|
+
publicKey,
|
|
1377
|
+
clickhouseClient,
|
|
1378
|
+
temporalClient,
|
|
1379
|
+
apisDir,
|
|
1380
|
+
enforceAuth,
|
|
1381
|
+
isDmv2,
|
|
1382
|
+
jwtConfig
|
|
1383
|
+
);
|
|
1384
|
+
const webApps = isDmv2 ? await getWebApps2() : /* @__PURE__ */ new Map();
|
|
1385
|
+
const sortedWebApps = Array.from(webApps.values()).sort((a, b) => {
|
|
1386
|
+
const pathA = a.config.mountPath || "/";
|
|
1387
|
+
const pathB = b.config.mountPath || "/";
|
|
1388
|
+
return pathB.length - pathA.length;
|
|
1389
|
+
});
|
|
1390
|
+
return async (req, res) => {
|
|
1391
|
+
const start = Date.now();
|
|
1392
|
+
const url = new URL(req.url || "", "http://localhost");
|
|
1393
|
+
const pathname = url.pathname;
|
|
1394
|
+
if (pathname === "/_moose_internal/health") {
|
|
1395
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
1396
|
+
res.end(
|
|
1397
|
+
JSON.stringify({
|
|
1398
|
+
status: "healthy",
|
|
1399
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
1400
|
+
})
|
|
1401
|
+
);
|
|
1402
|
+
return;
|
|
1403
|
+
}
|
|
1404
|
+
let jwtPayload;
|
|
1405
|
+
if (publicKey && jwtConfig) {
|
|
1406
|
+
const jwt = req.headers.authorization?.split(" ")[1];
|
|
1407
|
+
if (jwt) {
|
|
1408
|
+
try {
|
|
1409
|
+
const { payload } = await jose.jwtVerify(jwt, publicKey, {
|
|
1410
|
+
issuer: jwtConfig.issuer,
|
|
1411
|
+
audience: jwtConfig.audience
|
|
1412
|
+
});
|
|
1413
|
+
jwtPayload = payload;
|
|
1414
|
+
} catch (error) {
|
|
1415
|
+
console.log("JWT verification failed for WebApp route");
|
|
1416
|
+
}
|
|
1417
|
+
}
|
|
1418
|
+
}
|
|
1419
|
+
for (const webApp of sortedWebApps) {
|
|
1420
|
+
const mountPath = webApp.config.mountPath || "/";
|
|
1421
|
+
const normalizedMount = mountPath.endsWith("/") && mountPath !== "/" ? mountPath.slice(0, -1) : mountPath;
|
|
1422
|
+
const matches = pathname === normalizedMount || pathname.startsWith(normalizedMount + "/");
|
|
1423
|
+
if (matches) {
|
|
1424
|
+
if (webApp.config.injectMooseUtils !== false) {
|
|
1425
|
+
const { getMooseUtils: getMooseUtils2 } = await Promise.resolve().then(() => (init_standalone(), standalone_exports));
|
|
1426
|
+
req.moose = await getMooseUtils2();
|
|
1427
|
+
}
|
|
1428
|
+
let proxiedUrl = req.url;
|
|
1429
|
+
if (normalizedMount !== "/") {
|
|
1430
|
+
const pathWithoutMount = pathname.substring(normalizedMount.length) || "/";
|
|
1431
|
+
proxiedUrl = pathWithoutMount + url.search;
|
|
1432
|
+
}
|
|
1433
|
+
try {
|
|
1434
|
+
const modifiedReq = Object.assign(
|
|
1435
|
+
Object.create(Object.getPrototypeOf(req)),
|
|
1436
|
+
req,
|
|
1437
|
+
{
|
|
1438
|
+
url: proxiedUrl
|
|
1439
|
+
}
|
|
1440
|
+
);
|
|
1441
|
+
await webApp.handler(modifiedReq, res);
|
|
1442
|
+
return;
|
|
1443
|
+
} catch (error) {
|
|
1444
|
+
console.error(`Error in WebApp ${webApp.name}:`, error);
|
|
1445
|
+
if (!res.headersSent) {
|
|
1446
|
+
res.writeHead(500, { "Content-Type": "application/json" });
|
|
1447
|
+
res.end(JSON.stringify({ error: "Internal Server Error" }));
|
|
1448
|
+
}
|
|
1449
|
+
return;
|
|
1450
|
+
}
|
|
1451
|
+
}
|
|
926
1452
|
}
|
|
927
|
-
let
|
|
928
|
-
if (
|
|
929
|
-
|
|
930
|
-
|
|
1453
|
+
let apiPath = pathname;
|
|
1454
|
+
if (pathname.startsWith("/api/")) {
|
|
1455
|
+
apiPath = pathname.substring(4);
|
|
1456
|
+
} else if (pathname.startsWith("/consumption/")) {
|
|
1457
|
+
apiPath = pathname.substring(13);
|
|
931
1458
|
}
|
|
932
|
-
|
|
1459
|
+
if (apiPath !== pathname) {
|
|
933
1460
|
const modifiedReq = Object.assign(
|
|
934
1461
|
Object.create(Object.getPrototypeOf(req)),
|
|
935
1462
|
req,
|
|
936
1463
|
{
|
|
937
|
-
url:
|
|
1464
|
+
url: apiPath + url.search
|
|
938
1465
|
}
|
|
939
1466
|
);
|
|
940
|
-
await
|
|
941
|
-
return;
|
|
942
|
-
} catch (error) {
|
|
943
|
-
console.error(`Error in WebApp ${webApp.name}:`, error);
|
|
944
|
-
if (!res.headersSent) {
|
|
945
|
-
res.writeHead(500, { "Content-Type": "application/json" });
|
|
946
|
-
res.end(JSON.stringify({ error: "Internal Server Error" }));
|
|
947
|
-
}
|
|
1467
|
+
await apiRequestHandler(modifiedReq, res);
|
|
948
1468
|
return;
|
|
949
1469
|
}
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
1470
|
+
res.writeHead(404, { "Content-Type": "application/json" });
|
|
1471
|
+
res.end(JSON.stringify({ error: "Not Found" }));
|
|
1472
|
+
httpLogger(req, res, start);
|
|
1473
|
+
};
|
|
1474
|
+
};
|
|
1475
|
+
runApis = async (config) => {
|
|
1476
|
+
const apisCluster = new Cluster({
|
|
1477
|
+
maxWorkerCount: (config.workerCount ?? 0) > 0 ? config.workerCount : void 0,
|
|
1478
|
+
workerStart: async () => {
|
|
1479
|
+
let temporalClient;
|
|
1480
|
+
if (config.temporalConfig) {
|
|
1481
|
+
temporalClient = await getTemporalClient(
|
|
1482
|
+
config.temporalConfig.url,
|
|
1483
|
+
config.temporalConfig.namespace,
|
|
1484
|
+
config.temporalConfig.clientCert,
|
|
1485
|
+
config.temporalConfig.clientKey,
|
|
1486
|
+
config.temporalConfig.apiKey
|
|
1487
|
+
);
|
|
1488
|
+
}
|
|
1489
|
+
const clickhouseClient = getClickhouseClient(
|
|
1490
|
+
toClientConfig2(config.clickhouseConfig)
|
|
1491
|
+
);
|
|
1492
|
+
let publicKey;
|
|
1493
|
+
if (config.jwtConfig?.secret) {
|
|
1494
|
+
console.log("Importing JWT public key...");
|
|
1495
|
+
publicKey = await jose.importSPKI(config.jwtConfig.secret, "RS256");
|
|
1496
|
+
}
|
|
1497
|
+
const runtimeQueryClient = new QueryClient(clickhouseClient, "runtime");
|
|
1498
|
+
globalThis._mooseRuntimeContext = {
|
|
1499
|
+
client: new MooseClient(runtimeQueryClient, temporalClient)
|
|
1500
|
+
};
|
|
1501
|
+
const server = import_http2.default.createServer(
|
|
1502
|
+
await createMainRouter(
|
|
1503
|
+
publicKey,
|
|
1504
|
+
clickhouseClient,
|
|
1505
|
+
temporalClient,
|
|
1506
|
+
config.apisDir,
|
|
1507
|
+
config.enforceAuth,
|
|
1508
|
+
config.isDmv2,
|
|
1509
|
+
config.jwtConfig
|
|
1510
|
+
)
|
|
1511
|
+
);
|
|
1512
|
+
const port = config.proxyPort !== void 0 ? config.proxyPort : 4001;
|
|
1513
|
+
server.listen(port, "localhost", () => {
|
|
1514
|
+
console.log(`Server running on port ${port}`);
|
|
1515
|
+
});
|
|
1516
|
+
return server;
|
|
1517
|
+
},
|
|
1518
|
+
workerStop: async (server) => {
|
|
1519
|
+
return new Promise((resolve2) => {
|
|
1520
|
+
server.close(() => resolve2());
|
|
1521
|
+
});
|
|
964
1522
|
}
|
|
965
|
-
);
|
|
966
|
-
await apiRequestHandler(modifiedReq, res);
|
|
967
|
-
return;
|
|
968
|
-
}
|
|
969
|
-
res.writeHead(404, { "Content-Type": "application/json" });
|
|
970
|
-
res.end(JSON.stringify({ error: "Not Found" }));
|
|
971
|
-
httpLogger(req, res, start);
|
|
972
|
-
};
|
|
973
|
-
};
|
|
974
|
-
var runApis = async (config) => {
|
|
975
|
-
const apisCluster = new Cluster({
|
|
976
|
-
maxWorkerCount: (config.workerCount ?? 0) > 0 ? config.workerCount : void 0,
|
|
977
|
-
workerStart: async () => {
|
|
978
|
-
let temporalClient;
|
|
979
|
-
if (config.temporalConfig) {
|
|
980
|
-
temporalClient = await getTemporalClient(
|
|
981
|
-
config.temporalConfig.url,
|
|
982
|
-
config.temporalConfig.namespace,
|
|
983
|
-
config.temporalConfig.clientCert,
|
|
984
|
-
config.temporalConfig.clientKey,
|
|
985
|
-
config.temporalConfig.apiKey
|
|
986
|
-
);
|
|
987
|
-
}
|
|
988
|
-
const clickhouseClient = getClickhouseClient(
|
|
989
|
-
toClientConfig(config.clickhouseConfig)
|
|
990
|
-
);
|
|
991
|
-
let publicKey;
|
|
992
|
-
if (config.jwtConfig?.secret) {
|
|
993
|
-
console.log("Importing JWT public key...");
|
|
994
|
-
publicKey = await jose.importSPKI(config.jwtConfig.secret, "RS256");
|
|
995
|
-
}
|
|
996
|
-
const server = import_http2.default.createServer(
|
|
997
|
-
await createMainRouter(
|
|
998
|
-
publicKey,
|
|
999
|
-
clickhouseClient,
|
|
1000
|
-
temporalClient,
|
|
1001
|
-
config.apisDir,
|
|
1002
|
-
config.enforceAuth,
|
|
1003
|
-
config.isDmv2,
|
|
1004
|
-
config.jwtConfig
|
|
1005
|
-
)
|
|
1006
|
-
);
|
|
1007
|
-
const port = config.proxyPort !== void 0 ? config.proxyPort : 4001;
|
|
1008
|
-
server.listen(port, "localhost", () => {
|
|
1009
|
-
console.log(`Server running on port ${port}`);
|
|
1010
|
-
});
|
|
1011
|
-
return server;
|
|
1012
|
-
},
|
|
1013
|
-
workerStop: async (server) => {
|
|
1014
|
-
return new Promise((resolve2) => {
|
|
1015
|
-
server.close(() => resolve2());
|
|
1016
1523
|
});
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
};
|
|
1524
|
+
apisCluster.start();
|
|
1525
|
+
};
|
|
1526
|
+
}
|
|
1527
|
+
});
|
|
1021
1528
|
|
|
1022
1529
|
// src/clients/redisClient.ts
|
|
1023
|
-
var import_redis
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1530
|
+
var import_redis;
|
|
1531
|
+
var init_redisClient = __esm({
|
|
1532
|
+
"src/clients/redisClient.ts"() {
|
|
1533
|
+
"use strict";
|
|
1534
|
+
import_redis = require("redis");
|
|
1535
|
+
}
|
|
1536
|
+
});
|
|
1030
1537
|
|
|
1031
1538
|
// src/utilities/json.ts
|
|
1032
|
-
var STRING_DATE_ANNOTATION = "stringDate";
|
|
1033
1539
|
function isNullableType(dt) {
|
|
1034
1540
|
return typeof dt === "object" && dt !== null && "nullable" in dt && typeof dt.nullable !== "undefined";
|
|
1035
1541
|
}
|
|
@@ -1148,38 +1654,77 @@ function mutateParsedJson(data, fieldMutations) {
|
|
|
1148
1654
|
}
|
|
1149
1655
|
applyFieldMutations(data, fieldMutations);
|
|
1150
1656
|
}
|
|
1657
|
+
var STRING_DATE_ANNOTATION;
|
|
1658
|
+
var init_json = __esm({
|
|
1659
|
+
"src/utilities/json.ts"() {
|
|
1660
|
+
"use strict";
|
|
1661
|
+
STRING_DATE_ANNOTATION = "stringDate";
|
|
1662
|
+
}
|
|
1663
|
+
});
|
|
1151
1664
|
|
|
1152
1665
|
// src/utilities/dataParser.ts
|
|
1153
|
-
var CSV_DELIMITERS
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
};
|
|
1666
|
+
var import_csv_parse, CSV_DELIMITERS, DEFAULT_CSV_CONFIG;
|
|
1667
|
+
var init_dataParser = __esm({
|
|
1668
|
+
"src/utilities/dataParser.ts"() {
|
|
1669
|
+
"use strict";
|
|
1670
|
+
import_csv_parse = require("csv-parse");
|
|
1671
|
+
init_json();
|
|
1672
|
+
CSV_DELIMITERS = {
|
|
1673
|
+
COMMA: ",",
|
|
1674
|
+
TAB: " ",
|
|
1675
|
+
SEMICOLON: ";",
|
|
1676
|
+
PIPE: "|"
|
|
1677
|
+
};
|
|
1678
|
+
DEFAULT_CSV_CONFIG = {
|
|
1679
|
+
delimiter: CSV_DELIMITERS.COMMA,
|
|
1680
|
+
columns: true,
|
|
1681
|
+
skipEmptyLines: true,
|
|
1682
|
+
trim: true
|
|
1683
|
+
};
|
|
1684
|
+
}
|
|
1685
|
+
});
|
|
1686
|
+
|
|
1687
|
+
// src/utilities/index.ts
|
|
1688
|
+
var init_utilities = __esm({
|
|
1689
|
+
"src/utilities/index.ts"() {
|
|
1690
|
+
"use strict";
|
|
1691
|
+
init_dataParser();
|
|
1692
|
+
}
|
|
1693
|
+
});
|
|
1694
|
+
|
|
1695
|
+
// src/connectors/dataSource.ts
|
|
1696
|
+
var init_dataSource = __esm({
|
|
1697
|
+
"src/connectors/dataSource.ts"() {
|
|
1698
|
+
"use strict";
|
|
1699
|
+
}
|
|
1700
|
+
});
|
|
1701
|
+
|
|
1702
|
+
// src/index.ts
|
|
1703
|
+
var init_index = __esm({
|
|
1704
|
+
"src/index.ts"() {
|
|
1705
|
+
"use strict";
|
|
1706
|
+
init_browserCompatible();
|
|
1707
|
+
init_helpers();
|
|
1708
|
+
init_commons();
|
|
1709
|
+
init_secrets();
|
|
1710
|
+
init_helpers2();
|
|
1711
|
+
init_webAppHelpers();
|
|
1712
|
+
init_task();
|
|
1713
|
+
init_runner();
|
|
1714
|
+
init_redisClient();
|
|
1715
|
+
init_helpers2();
|
|
1716
|
+
init_standalone();
|
|
1717
|
+
init_sqlHelpers();
|
|
1718
|
+
init_utilities();
|
|
1719
|
+
init_dataSource();
|
|
1720
|
+
init_types();
|
|
1721
|
+
}
|
|
1722
|
+
});
|
|
1165
1723
|
|
|
1166
1724
|
// src/dmv2/internal.ts
|
|
1167
|
-
init_commons();
|
|
1168
1725
|
function getSourceDir() {
|
|
1169
1726
|
return import_process.default.env.MOOSE_SOURCE_DIR || "app";
|
|
1170
1727
|
}
|
|
1171
|
-
var moose_internal = {
|
|
1172
|
-
tables: /* @__PURE__ */ new Map(),
|
|
1173
|
-
streams: /* @__PURE__ */ new Map(),
|
|
1174
|
-
ingestApis: /* @__PURE__ */ new Map(),
|
|
1175
|
-
apis: /* @__PURE__ */ new Map(),
|
|
1176
|
-
sqlResources: /* @__PURE__ */ new Map(),
|
|
1177
|
-
workflows: /* @__PURE__ */ new Map(),
|
|
1178
|
-
webApps: /* @__PURE__ */ new Map(),
|
|
1179
|
-
materializedViews: /* @__PURE__ */ new Map(),
|
|
1180
|
-
customViews: /* @__PURE__ */ new Map()
|
|
1181
|
-
};
|
|
1182
|
-
var defaultRetentionPeriod = 60 * 60 * 24 * 7;
|
|
1183
1728
|
function isS3QueueConfig(config) {
|
|
1184
1729
|
return "engine" in config && config.engine === "S3Queue" /* S3Queue */;
|
|
1185
1730
|
}
|
|
@@ -1416,388 +1961,387 @@ function convertTableConfigToEngineConfig(config) {
|
|
|
1416
1961
|
}
|
|
1417
1962
|
return void 0;
|
|
1418
1963
|
}
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
|
|
1428
|
-
const customViews = {};
|
|
1429
|
-
registry.tables.forEach((table) => {
|
|
1430
|
-
const id = table.config.version ? `${table.name}_${table.config.version}` : table.name;
|
|
1431
|
-
let metadata = table.metadata;
|
|
1432
|
-
if (!metadata && table.config && table.pipelineParent) {
|
|
1433
|
-
metadata = table.pipelineParent.metadata;
|
|
1434
|
-
}
|
|
1435
|
-
const engineConfig = convertTableConfigToEngineConfig(table.config);
|
|
1436
|
-
let tableSettings = void 0;
|
|
1437
|
-
if (table.config.settings) {
|
|
1438
|
-
tableSettings = Object.entries(table.config.settings).reduce(
|
|
1439
|
-
(acc, [key, value]) => {
|
|
1440
|
-
if (value !== void 0) {
|
|
1441
|
-
acc[key] = String(value);
|
|
1442
|
-
}
|
|
1443
|
-
return acc;
|
|
1444
|
-
},
|
|
1445
|
-
{}
|
|
1446
|
-
);
|
|
1447
|
-
}
|
|
1448
|
-
if (engineConfig?.engine === "S3Queue") {
|
|
1449
|
-
if (!tableSettings) {
|
|
1450
|
-
tableSettings = {};
|
|
1451
|
-
}
|
|
1452
|
-
if (!tableSettings.mode) {
|
|
1453
|
-
tableSettings.mode = "unordered";
|
|
1964
|
+
function findTaskInTree(task, targetName) {
|
|
1965
|
+
if (task.name === targetName) {
|
|
1966
|
+
return task;
|
|
1967
|
+
}
|
|
1968
|
+
if (task.config.onComplete?.length) {
|
|
1969
|
+
for (const childTask of task.config.onComplete) {
|
|
1970
|
+
const found = findTaskInTree(childTask, targetName);
|
|
1971
|
+
if (found) {
|
|
1972
|
+
return found;
|
|
1454
1973
|
}
|
|
1455
1974
|
}
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
lifeCycle: table.config.lifeCycle,
|
|
1475
|
-
// Map 'settings' to 'tableSettings' for internal use
|
|
1476
|
-
tableSettings: tableSettings && Object.keys(tableSettings).length > 0 ? tableSettings : void 0,
|
|
1477
|
-
indexes: table.config.indexes?.map((i) => ({
|
|
1478
|
-
...i,
|
|
1479
|
-
granularity: i.granularity === void 0 ? 1 : i.granularity,
|
|
1480
|
-
arguments: i.arguments === void 0 ? [] : i.arguments
|
|
1481
|
-
})) || [],
|
|
1482
|
-
ttl: table.config.ttl,
|
|
1483
|
-
database: table.config.database,
|
|
1484
|
-
cluster: table.config.cluster
|
|
1975
|
+
}
|
|
1976
|
+
return void 0;
|
|
1977
|
+
}
|
|
1978
|
+
var import_process, moose_internal, defaultRetentionPeriod, toInfraMap, getMooseInternal, dumpMooseInternal, loadIndex, getStreamingFunctions, getApis2, getWorkflows2, getTaskForWorkflow, getWebApps2;
|
|
1979
|
+
var init_internal = __esm({
|
|
1980
|
+
"src/dmv2/internal.ts"() {
|
|
1981
|
+
"use strict";
|
|
1982
|
+
import_process = __toESM(require("process"));
|
|
1983
|
+
init_index();
|
|
1984
|
+
init_commons();
|
|
1985
|
+
moose_internal = {
|
|
1986
|
+
tables: /* @__PURE__ */ new Map(),
|
|
1987
|
+
streams: /* @__PURE__ */ new Map(),
|
|
1988
|
+
ingestApis: /* @__PURE__ */ new Map(),
|
|
1989
|
+
apis: /* @__PURE__ */ new Map(),
|
|
1990
|
+
sqlResources: /* @__PURE__ */ new Map(),
|
|
1991
|
+
workflows: /* @__PURE__ */ new Map(),
|
|
1992
|
+
webApps: /* @__PURE__ */ new Map()
|
|
1485
1993
|
};
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1994
|
+
defaultRetentionPeriod = 60 * 60 * 24 * 7;
|
|
1995
|
+
toInfraMap = (registry) => {
|
|
1996
|
+
const tables = {};
|
|
1997
|
+
const topics = {};
|
|
1998
|
+
const ingestApis = {};
|
|
1999
|
+
const apis = {};
|
|
2000
|
+
const sqlResources = {};
|
|
2001
|
+
const workflows = {};
|
|
2002
|
+
const webApps = {};
|
|
2003
|
+
registry.tables.forEach((table) => {
|
|
2004
|
+
const id = table.config.version ? `${table.name}_${table.config.version}` : table.name;
|
|
2005
|
+
let metadata = table.metadata;
|
|
2006
|
+
if (!metadata && table.config && table.pipelineParent) {
|
|
2007
|
+
metadata = table.pipelineParent.metadata;
|
|
2008
|
+
}
|
|
2009
|
+
const engineConfig = convertTableConfigToEngineConfig(table.config);
|
|
2010
|
+
let tableSettings = void 0;
|
|
2011
|
+
if (table.config.settings) {
|
|
2012
|
+
tableSettings = Object.entries(table.config.settings).reduce(
|
|
2013
|
+
(acc, [key, value]) => {
|
|
2014
|
+
if (value !== void 0) {
|
|
2015
|
+
acc[key] = String(value);
|
|
2016
|
+
}
|
|
2017
|
+
return acc;
|
|
2018
|
+
},
|
|
2019
|
+
{}
|
|
2020
|
+
);
|
|
2021
|
+
}
|
|
2022
|
+
if (engineConfig?.engine === "S3Queue") {
|
|
2023
|
+
if (!tableSettings) {
|
|
2024
|
+
tableSettings = {};
|
|
2025
|
+
}
|
|
2026
|
+
if (!tableSettings.mode) {
|
|
2027
|
+
tableSettings.mode = "unordered";
|
|
2028
|
+
}
|
|
2029
|
+
}
|
|
2030
|
+
const hasOrderByFields = "orderByFields" in table.config && Array.isArray(table.config.orderByFields) && table.config.orderByFields.length > 0;
|
|
2031
|
+
const hasOrderByExpression = "orderByExpression" in table.config && typeof table.config.orderByExpression === "string" && table.config.orderByExpression.length > 0;
|
|
2032
|
+
if (hasOrderByFields && hasOrderByExpression) {
|
|
2033
|
+
throw new Error(
|
|
2034
|
+
`Table ${table.name}: Provide either orderByFields or orderByExpression, not both.`
|
|
2035
|
+
);
|
|
2036
|
+
}
|
|
2037
|
+
const orderBy = hasOrderByExpression && "orderByExpression" in table.config ? table.config.orderByExpression ?? "" : "orderByFields" in table.config ? table.config.orderByFields ?? [] : [];
|
|
2038
|
+
tables[id] = {
|
|
2039
|
+
name: table.name,
|
|
2040
|
+
columns: table.columnArray,
|
|
2041
|
+
orderBy,
|
|
2042
|
+
partitionBy: "partitionBy" in table.config ? table.config.partitionBy : void 0,
|
|
2043
|
+
sampleByExpression: "sampleByExpression" in table.config ? table.config.sampleByExpression : void 0,
|
|
2044
|
+
primaryKeyExpression: "primaryKeyExpression" in table.config ? table.config.primaryKeyExpression : void 0,
|
|
2045
|
+
engineConfig,
|
|
2046
|
+
version: table.config.version,
|
|
2047
|
+
metadata,
|
|
2048
|
+
lifeCycle: table.config.lifeCycle,
|
|
2049
|
+
// Map 'settings' to 'tableSettings' for internal use
|
|
2050
|
+
tableSettings: tableSettings && Object.keys(tableSettings).length > 0 ? tableSettings : void 0,
|
|
2051
|
+
indexes: table.config.indexes?.map((i) => ({
|
|
2052
|
+
...i,
|
|
2053
|
+
granularity: i.granularity === void 0 ? 1 : i.granularity,
|
|
2054
|
+
arguments: i.arguments === void 0 ? [] : i.arguments
|
|
2055
|
+
})) || [],
|
|
2056
|
+
ttl: table.config.ttl,
|
|
2057
|
+
database: table.config.database,
|
|
2058
|
+
cluster: table.config.cluster
|
|
2059
|
+
};
|
|
2060
|
+
});
|
|
2061
|
+
registry.streams.forEach((stream) => {
|
|
2062
|
+
let metadata = stream.metadata;
|
|
2063
|
+
if (!metadata && stream.config && stream.pipelineParent) {
|
|
2064
|
+
metadata = stream.pipelineParent.metadata;
|
|
2065
|
+
}
|
|
2066
|
+
const transformationTargets = [];
|
|
2067
|
+
const consumers = [];
|
|
2068
|
+
stream._transformations.forEach((transforms, destinationName) => {
|
|
2069
|
+
transforms.forEach(([destination, _, config]) => {
|
|
2070
|
+
transformationTargets.push({
|
|
2071
|
+
kind: "stream",
|
|
2072
|
+
name: destinationName,
|
|
2073
|
+
version: config.version,
|
|
2074
|
+
metadata: config.metadata,
|
|
2075
|
+
sourceFile: config.sourceFile
|
|
2076
|
+
});
|
|
2077
|
+
});
|
|
2078
|
+
});
|
|
2079
|
+
stream._consumers.forEach((consumer) => {
|
|
2080
|
+
consumers.push({
|
|
2081
|
+
version: consumer.config.version,
|
|
2082
|
+
sourceFile: consumer.config.sourceFile
|
|
2083
|
+
});
|
|
1502
2084
|
});
|
|
2085
|
+
topics[stream.name] = {
|
|
2086
|
+
name: stream.name,
|
|
2087
|
+
columns: stream.columnArray,
|
|
2088
|
+
targetTable: stream.config.destination?.name,
|
|
2089
|
+
targetTableVersion: stream.config.destination?.config.version,
|
|
2090
|
+
retentionPeriod: stream.config.retentionPeriod ?? defaultRetentionPeriod,
|
|
2091
|
+
partitionCount: stream.config.parallelism ?? 1,
|
|
2092
|
+
version: stream.config.version,
|
|
2093
|
+
transformationTargets,
|
|
2094
|
+
hasMultiTransform: stream._multipleTransformations === void 0,
|
|
2095
|
+
consumers,
|
|
2096
|
+
metadata,
|
|
2097
|
+
lifeCycle: stream.config.lifeCycle,
|
|
2098
|
+
schemaConfig: stream.config.schemaConfig
|
|
2099
|
+
};
|
|
1503
2100
|
});
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
|
|
2101
|
+
registry.ingestApis.forEach((api) => {
|
|
2102
|
+
let metadata = api.metadata;
|
|
2103
|
+
if (!metadata && api.config && api.pipelineParent) {
|
|
2104
|
+
metadata = api.pipelineParent.metadata;
|
|
2105
|
+
}
|
|
2106
|
+
ingestApis[api.name] = {
|
|
2107
|
+
name: api.name,
|
|
2108
|
+
columns: api.columnArray,
|
|
2109
|
+
version: api.config.version,
|
|
2110
|
+
path: api.config.path,
|
|
2111
|
+
writeTo: {
|
|
2112
|
+
kind: "stream",
|
|
2113
|
+
name: api.config.destination.name
|
|
2114
|
+
},
|
|
2115
|
+
deadLetterQueue: api.config.deadLetterQueue?.name,
|
|
2116
|
+
metadata,
|
|
2117
|
+
schema: api.schema,
|
|
2118
|
+
allowExtraFields: api.allowExtraFields
|
|
2119
|
+
};
|
|
1509
2120
|
});
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
2121
|
+
registry.apis.forEach((api, key) => {
|
|
2122
|
+
const rustKey = api.config.version ? `${api.name}:${api.config.version}` : api.name;
|
|
2123
|
+
apis[rustKey] = {
|
|
2124
|
+
name: api.name,
|
|
2125
|
+
queryParams: api.columnArray,
|
|
2126
|
+
responseSchema: api.responseSchema,
|
|
2127
|
+
version: api.config.version,
|
|
2128
|
+
path: api.config.path,
|
|
2129
|
+
metadata: api.metadata
|
|
2130
|
+
};
|
|
2131
|
+
});
|
|
2132
|
+
registry.sqlResources.forEach((sqlResource) => {
|
|
2133
|
+
sqlResources[sqlResource.name] = {
|
|
2134
|
+
name: sqlResource.name,
|
|
2135
|
+
setup: sqlResource.setup,
|
|
2136
|
+
teardown: sqlResource.teardown,
|
|
2137
|
+
sourceFile: sqlResource.sourceFile,
|
|
2138
|
+
sourceLine: sqlResource.sourceLine,
|
|
2139
|
+
sourceColumn: sqlResource.sourceColumn,
|
|
2140
|
+
pullsDataFrom: sqlResource.pullsDataFrom.map((r) => {
|
|
2141
|
+
if (r.kind === "OlapTable") {
|
|
2142
|
+
const table = r;
|
|
2143
|
+
const id = table.config.version ? `${table.name}_${table.config.version}` : table.name;
|
|
2144
|
+
return {
|
|
2145
|
+
id,
|
|
2146
|
+
kind: "Table"
|
|
2147
|
+
};
|
|
2148
|
+
} else if (r.kind === "SqlResource") {
|
|
2149
|
+
const resource = r;
|
|
2150
|
+
return {
|
|
2151
|
+
id: resource.name,
|
|
2152
|
+
kind: "SqlResource"
|
|
2153
|
+
};
|
|
2154
|
+
} else {
|
|
2155
|
+
throw new Error(`Unknown sql resource dependency type: ${r}`);
|
|
2156
|
+
}
|
|
2157
|
+
}),
|
|
2158
|
+
pushesDataTo: sqlResource.pushesDataTo.map((r) => {
|
|
2159
|
+
if (r.kind === "OlapTable") {
|
|
2160
|
+
const table = r;
|
|
2161
|
+
const id = table.config.version ? `${table.name}_${table.config.version}` : table.name;
|
|
2162
|
+
return {
|
|
2163
|
+
id,
|
|
2164
|
+
kind: "Table"
|
|
2165
|
+
};
|
|
2166
|
+
} else if (r.kind === "SqlResource") {
|
|
2167
|
+
const resource = r;
|
|
2168
|
+
return {
|
|
2169
|
+
id: resource.name,
|
|
2170
|
+
kind: "SqlResource"
|
|
2171
|
+
};
|
|
2172
|
+
} else {
|
|
2173
|
+
throw new Error(`Unknown sql resource dependency type: ${r}`);
|
|
2174
|
+
}
|
|
2175
|
+
})
|
|
2176
|
+
};
|
|
2177
|
+
});
|
|
2178
|
+
registry.workflows.forEach((workflow) => {
|
|
2179
|
+
workflows[workflow.name] = {
|
|
2180
|
+
name: workflow.name,
|
|
2181
|
+
retries: workflow.config.retries,
|
|
2182
|
+
timeout: workflow.config.timeout,
|
|
2183
|
+
schedule: workflow.config.schedule
|
|
2184
|
+
};
|
|
2185
|
+
});
|
|
2186
|
+
registry.webApps.forEach((webApp) => {
|
|
2187
|
+
webApps[webApp.name] = {
|
|
2188
|
+
name: webApp.name,
|
|
2189
|
+
mountPath: webApp.config.mountPath || "/",
|
|
2190
|
+
metadata: webApp.config.metadata
|
|
2191
|
+
};
|
|
2192
|
+
});
|
|
2193
|
+
return {
|
|
2194
|
+
topics,
|
|
2195
|
+
tables,
|
|
2196
|
+
ingestApis,
|
|
2197
|
+
apis,
|
|
2198
|
+
sqlResources,
|
|
2199
|
+
workflows,
|
|
2200
|
+
webApps
|
|
2201
|
+
};
|
|
1525
2202
|
};
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
if (!metadata && api.config && api.pipelineParent) {
|
|
1530
|
-
metadata = api.pipelineParent.metadata;
|
|
2203
|
+
getMooseInternal = () => globalThis.moose_internal;
|
|
2204
|
+
if (getMooseInternal() === void 0) {
|
|
2205
|
+
globalThis.moose_internal = moose_internal;
|
|
1531
2206
|
}
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
name: api.config.destination.name
|
|
1540
|
-
},
|
|
1541
|
-
deadLetterQueue: api.config.deadLetterQueue?.name,
|
|
1542
|
-
metadata,
|
|
1543
|
-
schema: api.schema,
|
|
1544
|
-
allowExtraFields: api.allowExtraFields
|
|
1545
|
-
};
|
|
1546
|
-
});
|
|
1547
|
-
registry.apis.forEach((api, key) => {
|
|
1548
|
-
const rustKey = api.config.version ? `${api.name}:${api.config.version}` : api.name;
|
|
1549
|
-
apis[rustKey] = {
|
|
1550
|
-
name: api.name,
|
|
1551
|
-
queryParams: api.columnArray,
|
|
1552
|
-
responseSchema: api.responseSchema,
|
|
1553
|
-
version: api.config.version,
|
|
1554
|
-
path: api.config.path,
|
|
1555
|
-
metadata: api.metadata
|
|
2207
|
+
dumpMooseInternal = async () => {
|
|
2208
|
+
loadIndex();
|
|
2209
|
+
console.log(
|
|
2210
|
+
"___MOOSE_STUFF___start",
|
|
2211
|
+
JSON.stringify(toInfraMap(getMooseInternal())),
|
|
2212
|
+
"end___MOOSE_STUFF___"
|
|
2213
|
+
);
|
|
1556
2214
|
};
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
|
|
1562
|
-
|
|
1563
|
-
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
return {
|
|
1571
|
-
id,
|
|
1572
|
-
kind: "Table"
|
|
1573
|
-
};
|
|
1574
|
-
} else if (r.kind === "SqlResource") {
|
|
1575
|
-
const resource = r;
|
|
1576
|
-
return {
|
|
1577
|
-
id: resource.name,
|
|
1578
|
-
kind: "SqlResource"
|
|
1579
|
-
};
|
|
1580
|
-
} else {
|
|
1581
|
-
throw new Error(`Unknown sql resource dependency type: ${r}`);
|
|
2215
|
+
loadIndex = () => {
|
|
2216
|
+
const registry = getMooseInternal();
|
|
2217
|
+
registry.tables.clear();
|
|
2218
|
+
registry.streams.clear();
|
|
2219
|
+
registry.ingestApis.clear();
|
|
2220
|
+
registry.apis.clear();
|
|
2221
|
+
registry.sqlResources.clear();
|
|
2222
|
+
registry.workflows.clear();
|
|
2223
|
+
registry.webApps.clear();
|
|
2224
|
+
const appDir = `${import_process.default.cwd()}/${getSourceDir()}`;
|
|
2225
|
+
Object.keys(require.cache).forEach((key) => {
|
|
2226
|
+
if (key.startsWith(appDir)) {
|
|
2227
|
+
delete require.cache[key];
|
|
1582
2228
|
}
|
|
1583
|
-
})
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
};
|
|
1592
|
-
} else if (r.kind === "SqlResource") {
|
|
1593
|
-
const resource = r;
|
|
1594
|
-
return {
|
|
1595
|
-
id: resource.name,
|
|
1596
|
-
kind: "SqlResource"
|
|
1597
|
-
};
|
|
1598
|
-
} else {
|
|
1599
|
-
throw new Error(`Unknown sql resource dependency type: ${r}`);
|
|
2229
|
+
});
|
|
2230
|
+
try {
|
|
2231
|
+
require(`${import_process.default.cwd()}/${getSourceDir()}/index.ts`);
|
|
2232
|
+
} catch (error) {
|
|
2233
|
+
let hint;
|
|
2234
|
+
const details = error instanceof Error ? error.message : String(error);
|
|
2235
|
+
if (details.includes("ERR_REQUIRE_ESM") || details.includes("ES Module")) {
|
|
2236
|
+
hint = "The file or its dependencies are ESM-only. Switch to packages that dual-support CJS & ESM, or upgrade to Node 22.12+. If you must use Node 20, you may try Node 20.19\n\n";
|
|
1600
2237
|
}
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
1605
|
-
workflows[workflow.name] = {
|
|
1606
|
-
name: workflow.name,
|
|
1607
|
-
retries: workflow.config.retries,
|
|
1608
|
-
timeout: workflow.config.timeout,
|
|
1609
|
-
schedule: workflow.config.schedule
|
|
2238
|
+
const errorMsg = `${hint ?? ""}${details}`;
|
|
2239
|
+
const cause = error instanceof Error ? error : void 0;
|
|
2240
|
+
throw new Error(errorMsg, { cause });
|
|
2241
|
+
}
|
|
1610
2242
|
};
|
|
1611
|
-
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
2243
|
+
getStreamingFunctions = async () => {
|
|
2244
|
+
loadIndex();
|
|
2245
|
+
const registry = getMooseInternal();
|
|
2246
|
+
const transformFunctions = /* @__PURE__ */ new Map();
|
|
2247
|
+
registry.streams.forEach((stream) => {
|
|
2248
|
+
stream._transformations.forEach((transforms, destinationName) => {
|
|
2249
|
+
transforms.forEach(([_, transform, config]) => {
|
|
2250
|
+
const transformFunctionKey = `${stream.name}_${destinationName}${config.version ? `_${config.version}` : ""}`;
|
|
2251
|
+
compilerLog(`getStreamingFunctions: ${transformFunctionKey}`);
|
|
2252
|
+
transformFunctions.set(transformFunctionKey, [
|
|
2253
|
+
transform,
|
|
2254
|
+
config,
|
|
2255
|
+
stream.columnArray
|
|
2256
|
+
]);
|
|
2257
|
+
});
|
|
2258
|
+
});
|
|
2259
|
+
stream._consumers.forEach((consumer) => {
|
|
2260
|
+
const consumerFunctionKey = `${stream.name}_<no-target>${consumer.config.version ? `_${consumer.config.version}` : ""}`;
|
|
2261
|
+
transformFunctions.set(consumerFunctionKey, [
|
|
2262
|
+
consumer.consumer,
|
|
2263
|
+
consumer.config,
|
|
2264
|
+
stream.columnArray
|
|
2265
|
+
]);
|
|
2266
|
+
});
|
|
2267
|
+
});
|
|
2268
|
+
return transformFunctions;
|
|
1617
2269
|
};
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
2270
|
+
getApis2 = async () => {
|
|
2271
|
+
loadIndex();
|
|
2272
|
+
const apiFunctions = /* @__PURE__ */ new Map();
|
|
2273
|
+
const registry = getMooseInternal();
|
|
2274
|
+
const versionCountByName = /* @__PURE__ */ new Map();
|
|
2275
|
+
const nameToSoleVersionHandler = /* @__PURE__ */ new Map();
|
|
2276
|
+
registry.apis.forEach((api, key) => {
|
|
2277
|
+
const handler = api.getHandler();
|
|
2278
|
+
apiFunctions.set(key, handler);
|
|
2279
|
+
if (!api.config.version) {
|
|
2280
|
+
if (!apiFunctions.has(api.name)) {
|
|
2281
|
+
apiFunctions.set(api.name, handler);
|
|
2282
|
+
}
|
|
2283
|
+
nameToSoleVersionHandler.delete(api.name);
|
|
2284
|
+
versionCountByName.delete(api.name);
|
|
2285
|
+
} else if (!apiFunctions.has(api.name)) {
|
|
2286
|
+
const count = (versionCountByName.get(api.name) ?? 0) + 1;
|
|
2287
|
+
versionCountByName.set(api.name, count);
|
|
2288
|
+
if (count === 1) {
|
|
2289
|
+
nameToSoleVersionHandler.set(api.name, handler);
|
|
2290
|
+
} else {
|
|
2291
|
+
nameToSoleVersionHandler.delete(api.name);
|
|
2292
|
+
}
|
|
2293
|
+
}
|
|
2294
|
+
});
|
|
2295
|
+
nameToSoleVersionHandler.forEach((handler, name) => {
|
|
2296
|
+
if (!apiFunctions.has(name)) {
|
|
2297
|
+
apiFunctions.set(name, handler);
|
|
2298
|
+
}
|
|
2299
|
+
});
|
|
2300
|
+
return apiFunctions;
|
|
1627
2301
|
};
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
selectSql: view.selectSql,
|
|
1633
|
-
sourceTables: view.sourceTables,
|
|
1634
|
-
sourceFile: view.sourceFile
|
|
2302
|
+
getWorkflows2 = async () => {
|
|
2303
|
+
loadIndex();
|
|
2304
|
+
const registry = getMooseInternal();
|
|
2305
|
+
return registry.workflows;
|
|
1635
2306
|
};
|
|
1636
|
-
|
|
1637
|
-
|
|
1638
|
-
|
|
1639
|
-
|
|
1640
|
-
|
|
1641
|
-
apis,
|
|
1642
|
-
sqlResources,
|
|
1643
|
-
workflows,
|
|
1644
|
-
webApps,
|
|
1645
|
-
materializedViews,
|
|
1646
|
-
customViews
|
|
1647
|
-
};
|
|
1648
|
-
};
|
|
1649
|
-
var getMooseInternal = () => globalThis.moose_internal;
|
|
1650
|
-
if (getMooseInternal() === void 0) {
|
|
1651
|
-
globalThis.moose_internal = moose_internal;
|
|
1652
|
-
}
|
|
1653
|
-
var dumpMooseInternal = async () => {
|
|
1654
|
-
loadIndex();
|
|
1655
|
-
console.log(
|
|
1656
|
-
"___MOOSE_STUFF___start",
|
|
1657
|
-
JSON.stringify(toInfraMap(getMooseInternal())),
|
|
1658
|
-
"end___MOOSE_STUFF___"
|
|
1659
|
-
);
|
|
1660
|
-
};
|
|
1661
|
-
var loadIndex = () => {
|
|
1662
|
-
const registry = getMooseInternal();
|
|
1663
|
-
registry.tables.clear();
|
|
1664
|
-
registry.streams.clear();
|
|
1665
|
-
registry.ingestApis.clear();
|
|
1666
|
-
registry.apis.clear();
|
|
1667
|
-
registry.sqlResources.clear();
|
|
1668
|
-
registry.workflows.clear();
|
|
1669
|
-
registry.webApps.clear();
|
|
1670
|
-
registry.materializedViews.clear();
|
|
1671
|
-
registry.customViews.clear();
|
|
1672
|
-
const appDir = `${import_process.default.cwd()}/${getSourceDir()}`;
|
|
1673
|
-
Object.keys(require.cache).forEach((key) => {
|
|
1674
|
-
if (key.startsWith(appDir)) {
|
|
1675
|
-
delete require.cache[key];
|
|
1676
|
-
}
|
|
1677
|
-
});
|
|
1678
|
-
try {
|
|
1679
|
-
require(`${import_process.default.cwd()}/${getSourceDir()}/index.ts`);
|
|
1680
|
-
} catch (error) {
|
|
1681
|
-
let hint;
|
|
1682
|
-
const details = error instanceof Error ? error.message : String(error);
|
|
1683
|
-
if (details.includes("ERR_REQUIRE_ESM") || details.includes("ES Module")) {
|
|
1684
|
-
hint = "The file or its dependencies are ESM-only. Switch to packages that dual-support CJS & ESM, or upgrade to Node 22.12+. If you must use Node 20, you may try Node 20.19\n\n";
|
|
1685
|
-
}
|
|
1686
|
-
const errorMsg = `${hint ?? ""}${details}`;
|
|
1687
|
-
const cause = error instanceof Error ? error : void 0;
|
|
1688
|
-
throw new Error(errorMsg, { cause });
|
|
1689
|
-
}
|
|
1690
|
-
};
|
|
1691
|
-
var getStreamingFunctions = async () => {
|
|
1692
|
-
loadIndex();
|
|
1693
|
-
const registry = getMooseInternal();
|
|
1694
|
-
const transformFunctions = /* @__PURE__ */ new Map();
|
|
1695
|
-
registry.streams.forEach((stream) => {
|
|
1696
|
-
stream._transformations.forEach((transforms, destinationName) => {
|
|
1697
|
-
transforms.forEach(([_, transform, config]) => {
|
|
1698
|
-
const transformFunctionKey = `${stream.name}_${destinationName}${config.version ? `_${config.version}` : ""}`;
|
|
1699
|
-
compilerLog(`getStreamingFunctions: ${transformFunctionKey}`);
|
|
1700
|
-
transformFunctions.set(transformFunctionKey, [
|
|
1701
|
-
transform,
|
|
1702
|
-
config,
|
|
1703
|
-
stream.columnArray
|
|
1704
|
-
]);
|
|
1705
|
-
});
|
|
1706
|
-
});
|
|
1707
|
-
stream._consumers.forEach((consumer) => {
|
|
1708
|
-
const consumerFunctionKey = `${stream.name}_<no-target>${consumer.config.version ? `_${consumer.config.version}` : ""}`;
|
|
1709
|
-
transformFunctions.set(consumerFunctionKey, [
|
|
1710
|
-
consumer.consumer,
|
|
1711
|
-
consumer.config,
|
|
1712
|
-
stream.columnArray
|
|
1713
|
-
]);
|
|
1714
|
-
});
|
|
1715
|
-
});
|
|
1716
|
-
return transformFunctions;
|
|
1717
|
-
};
|
|
1718
|
-
var getApis2 = async () => {
|
|
1719
|
-
loadIndex();
|
|
1720
|
-
const apiFunctions = /* @__PURE__ */ new Map();
|
|
1721
|
-
const registry = getMooseInternal();
|
|
1722
|
-
const versionCountByName = /* @__PURE__ */ new Map();
|
|
1723
|
-
const nameToSoleVersionHandler = /* @__PURE__ */ new Map();
|
|
1724
|
-
registry.apis.forEach((api, key) => {
|
|
1725
|
-
const handler = api.getHandler();
|
|
1726
|
-
apiFunctions.set(key, handler);
|
|
1727
|
-
if (!api.config.version) {
|
|
1728
|
-
if (!apiFunctions.has(api.name)) {
|
|
1729
|
-
apiFunctions.set(api.name, handler);
|
|
1730
|
-
}
|
|
1731
|
-
nameToSoleVersionHandler.delete(api.name);
|
|
1732
|
-
versionCountByName.delete(api.name);
|
|
1733
|
-
} else if (!apiFunctions.has(api.name)) {
|
|
1734
|
-
const count = (versionCountByName.get(api.name) ?? 0) + 1;
|
|
1735
|
-
versionCountByName.set(api.name, count);
|
|
1736
|
-
if (count === 1) {
|
|
1737
|
-
nameToSoleVersionHandler.set(api.name, handler);
|
|
1738
|
-
} else {
|
|
1739
|
-
nameToSoleVersionHandler.delete(api.name);
|
|
2307
|
+
getTaskForWorkflow = async (workflowName, taskName) => {
|
|
2308
|
+
const workflows = await getWorkflows2();
|
|
2309
|
+
const workflow = workflows.get(workflowName);
|
|
2310
|
+
if (!workflow) {
|
|
2311
|
+
throw new Error(`Workflow ${workflowName} not found`);
|
|
1740
2312
|
}
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
});
|
|
1748
|
-
return apiFunctions;
|
|
1749
|
-
};
|
|
1750
|
-
var getWorkflows2 = async () => {
|
|
1751
|
-
loadIndex();
|
|
1752
|
-
const registry = getMooseInternal();
|
|
1753
|
-
return registry.workflows;
|
|
1754
|
-
};
|
|
1755
|
-
function findTaskInTree(task, targetName) {
|
|
1756
|
-
if (task.name === targetName) {
|
|
1757
|
-
return task;
|
|
1758
|
-
}
|
|
1759
|
-
if (task.config.onComplete?.length) {
|
|
1760
|
-
for (const childTask of task.config.onComplete) {
|
|
1761
|
-
const found = findTaskInTree(childTask, targetName);
|
|
1762
|
-
if (found) {
|
|
1763
|
-
return found;
|
|
2313
|
+
const task = findTaskInTree(
|
|
2314
|
+
workflow.config.startingTask,
|
|
2315
|
+
taskName
|
|
2316
|
+
);
|
|
2317
|
+
if (!task) {
|
|
2318
|
+
throw new Error(`Task ${taskName} not found in workflow ${workflowName}`);
|
|
1764
2319
|
}
|
|
1765
|
-
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
|
|
1769
|
-
|
|
1770
|
-
|
|
1771
|
-
const workflow = workflows.get(workflowName);
|
|
1772
|
-
if (!workflow) {
|
|
1773
|
-
throw new Error(`Workflow ${workflowName} not found`);
|
|
1774
|
-
}
|
|
1775
|
-
const task = findTaskInTree(
|
|
1776
|
-
workflow.config.startingTask,
|
|
1777
|
-
taskName
|
|
1778
|
-
);
|
|
1779
|
-
if (!task) {
|
|
1780
|
-
throw new Error(`Task ${taskName} not found in workflow ${workflowName}`);
|
|
2320
|
+
return task;
|
|
2321
|
+
};
|
|
2322
|
+
getWebApps2 = async () => {
|
|
2323
|
+
loadIndex();
|
|
2324
|
+
return getMooseInternal().webApps;
|
|
2325
|
+
};
|
|
1781
2326
|
}
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
|
|
1785
|
-
|
|
1786
|
-
|
|
1787
|
-
};
|
|
2327
|
+
});
|
|
2328
|
+
|
|
2329
|
+
// src/moose-runner.ts
|
|
2330
|
+
var import_ts_node = require("ts-node");
|
|
2331
|
+
init_internal();
|
|
1788
2332
|
|
|
1789
2333
|
// src/blocks/runner.ts
|
|
1790
2334
|
var import_fastq = __toESM(require("fastq"));
|
|
1791
2335
|
init_commons();
|
|
1792
2336
|
var import_node_fs = __toESM(require("fs"));
|
|
1793
|
-
var
|
|
2337
|
+
var import_node_path2 = __toESM(require("path"));
|
|
1794
2338
|
var walkDir = (dir, fileExtension, fileList) => {
|
|
1795
2339
|
const files = import_node_fs.default.readdirSync(dir);
|
|
1796
2340
|
files.forEach((file) => {
|
|
1797
|
-
if (import_node_fs.default.statSync(
|
|
1798
|
-
fileList = walkDir(
|
|
2341
|
+
if (import_node_fs.default.statSync(import_node_path2.default.join(dir, file)).isDirectory()) {
|
|
2342
|
+
fileList = walkDir(import_node_path2.default.join(dir, file), fileExtension, fileList);
|
|
1799
2343
|
} else if (file.endsWith(fileExtension)) {
|
|
1800
|
-
fileList.push(
|
|
2344
|
+
fileList.push(import_node_path2.default.join(dir, file));
|
|
1801
2345
|
}
|
|
1802
2346
|
});
|
|
1803
2347
|
return fileList;
|
|
@@ -1808,7 +2352,7 @@ var DependencyError = class extends Error {
|
|
|
1808
2352
|
this.name = "DependencyError";
|
|
1809
2353
|
}
|
|
1810
2354
|
};
|
|
1811
|
-
var
|
|
2355
|
+
var toClientConfig3 = (config) => ({
|
|
1812
2356
|
...config,
|
|
1813
2357
|
useSSL: config.useSSL ? "true" : "false"
|
|
1814
2358
|
});
|
|
@@ -1860,7 +2404,7 @@ var asyncWorker = async (task) => {
|
|
|
1860
2404
|
await createBlocks(task.chClient, task.blocks);
|
|
1861
2405
|
};
|
|
1862
2406
|
var runBlocks = async (config) => {
|
|
1863
|
-
const chClient = getClickhouseClient(
|
|
2407
|
+
const chClient = getClickhouseClient(toClientConfig3(config.clickhouseConfig));
|
|
1864
2408
|
console.log(`Connected`);
|
|
1865
2409
|
const blocksFiles = walkDir(config.blocksDir, ".ts", []);
|
|
1866
2410
|
const numOfBlockFiles = blocksFiles.length;
|
|
@@ -1873,10 +2417,10 @@ var runBlocks = async (config) => {
|
|
|
1873
2417
|
}
|
|
1874
2418
|
}
|
|
1875
2419
|
});
|
|
1876
|
-
for (const
|
|
1877
|
-
console.log(`Adding to queue: ${
|
|
2420
|
+
for (const path4 of blocksFiles) {
|
|
2421
|
+
console.log(`Adding to queue: ${path4}`);
|
|
1878
2422
|
try {
|
|
1879
|
-
const blocks = require(
|
|
2423
|
+
const blocks = require(path4).default;
|
|
1880
2424
|
queue.push({
|
|
1881
2425
|
chClient,
|
|
1882
2426
|
blocks,
|
|
@@ -1885,7 +2429,7 @@ var runBlocks = async (config) => {
|
|
|
1885
2429
|
} catch (err) {
|
|
1886
2430
|
cliLog({
|
|
1887
2431
|
action: "Blocks",
|
|
1888
|
-
message: `Failed to import blocks from ${
|
|
2432
|
+
message: `Failed to import blocks from ${path4}: ${err}`,
|
|
1889
2433
|
message_type: "Error"
|
|
1890
2434
|
});
|
|
1891
2435
|
}
|
|
@@ -1895,6 +2439,9 @@ var runBlocks = async (config) => {
|
|
|
1895
2439
|
}
|
|
1896
2440
|
};
|
|
1897
2441
|
|
|
2442
|
+
// src/moose-runner.ts
|
|
2443
|
+
init_runner();
|
|
2444
|
+
|
|
1898
2445
|
// src/streaming-functions/runner.ts
|
|
1899
2446
|
var import_node_stream2 = require("stream");
|
|
1900
2447
|
var import_kafka_javascript2 = require("@514labs/kafka-javascript");
|
|
@@ -1902,6 +2449,9 @@ var import_node_buffer = require("buffer");
|
|
|
1902
2449
|
var process3 = __toESM(require("process"));
|
|
1903
2450
|
var http3 = __toESM(require("http"));
|
|
1904
2451
|
init_commons();
|
|
2452
|
+
init_cluster_utils();
|
|
2453
|
+
init_internal();
|
|
2454
|
+
init_json();
|
|
1905
2455
|
var { Kafka: Kafka2 } = import_kafka_javascript2.KafkaJS;
|
|
1906
2456
|
var HOSTNAME = process3.env.HOSTNAME;
|
|
1907
2457
|
var AUTO_COMMIT_INTERVAL_MS = 5e3;
|
|
@@ -2460,12 +3010,15 @@ async function runApiTypeSerializer(targetModel) {
|
|
|
2460
3010
|
|
|
2461
3011
|
// src/scripts/runner.ts
|
|
2462
3012
|
var import_worker2 = require("@temporalio/worker");
|
|
2463
|
-
var
|
|
3013
|
+
var path3 = __toESM(require("path"));
|
|
2464
3014
|
var fs3 = __toESM(require("fs"));
|
|
3015
|
+
init_internal();
|
|
2465
3016
|
|
|
2466
3017
|
// src/scripts/activity.ts
|
|
2467
3018
|
var import_activity = require("@temporalio/activity");
|
|
2468
3019
|
var import_workflow3 = require("@temporalio/workflow");
|
|
3020
|
+
init_internal();
|
|
3021
|
+
init_json();
|
|
2469
3022
|
var activities = {
|
|
2470
3023
|
async hasDmv2Workflow(name) {
|
|
2471
3024
|
try {
|
|
@@ -2750,7 +3303,7 @@ async function registerWorkflows(logger2, config) {
|
|
|
2750
3303
|
}
|
|
2751
3304
|
};
|
|
2752
3305
|
const workflowBundle = await (0, import_worker2.bundleWorkflowCode)({
|
|
2753
|
-
workflowsPath:
|
|
3306
|
+
workflowsPath: path3.resolve(__dirname, "scripts/workflow.js"),
|
|
2754
3307
|
logger: silentLogger
|
|
2755
3308
|
});
|
|
2756
3309
|
const worker = await import_worker2.Worker.create({
|