@514labs/moose-lib 0.6.296 → 0.6.297-ci-28-g84f3192e
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{browserCompatible-B8CAYjv5.d.ts → browserCompatible-CMEunMFq.d.ts} +1 -1
- package/dist/{browserCompatible-ChWHzgtb.d.mts → browserCompatible-FzU17dxm.d.mts} +1 -1
- package/dist/browserCompatible.d.mts +2 -2
- package/dist/browserCompatible.d.ts +2 -2
- package/dist/browserCompatible.js +2443 -2160
- package/dist/browserCompatible.js.map +1 -1
- package/dist/browserCompatible.mjs +2440 -2159
- package/dist/browserCompatible.mjs.map +1 -1
- package/dist/dmv2/index.d.mts +1 -1
- package/dist/dmv2/index.d.ts +1 -1
- package/dist/dmv2/index.js +2334 -2051
- package/dist/dmv2/index.js.map +1 -1
- package/dist/dmv2/index.mjs +2299 -2018
- package/dist/dmv2/index.mjs.map +1 -1
- package/dist/{index-rQOQo9sv.d.mts → index-CcHF2cVT.d.mts} +16 -5
- package/dist/{index-rQOQo9sv.d.ts → index-CcHF2cVT.d.ts} +16 -5
- package/dist/index.d.mts +76 -6
- package/dist/index.d.ts +76 -6
- package/dist/index.js +3063 -2719
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +2961 -2618
- package/dist/index.mjs.map +1 -1
- package/dist/moose-runner.js +1715 -1136
- package/dist/moose-runner.js.map +1 -1
- package/dist/moose-runner.mjs +1706 -1129
- package/dist/moose-runner.mjs.map +1 -1
- package/package.json +1 -1
package/dist/moose-runner.mjs
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
+
var __defProp = Object.defineProperty;
|
|
2
3
|
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
3
4
|
var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require : typeof Proxy !== "undefined" ? new Proxy(x, {
|
|
4
5
|
get: (a, b) => (typeof require !== "undefined" ? require : a)[b]
|
|
@@ -9,6 +10,346 @@ var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require
|
|
|
9
10
|
var __esm = (fn, res) => function __init() {
|
|
10
11
|
return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
|
|
11
12
|
};
|
|
13
|
+
var __export = (target, all) => {
|
|
14
|
+
for (var name in all)
|
|
15
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
16
|
+
};
|
|
17
|
+
|
|
18
|
+
// src/dmv2/utils/stackTrace.ts
|
|
19
|
+
var init_stackTrace = __esm({
|
|
20
|
+
"src/dmv2/utils/stackTrace.ts"() {
|
|
21
|
+
"use strict";
|
|
22
|
+
}
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
// src/dmv2/typedBase.ts
|
|
26
|
+
var init_typedBase = __esm({
|
|
27
|
+
"src/dmv2/typedBase.ts"() {
|
|
28
|
+
"use strict";
|
|
29
|
+
init_stackTrace();
|
|
30
|
+
}
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
// src/dataModels/dataModelTypes.ts
|
|
34
|
+
var init_dataModelTypes = __esm({
|
|
35
|
+
"src/dataModels/dataModelTypes.ts"() {
|
|
36
|
+
"use strict";
|
|
37
|
+
}
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
// src/sqlHelpers.ts
|
|
41
|
+
function sql(strings, ...values) {
|
|
42
|
+
return new Sql(strings, values);
|
|
43
|
+
}
|
|
44
|
+
function createClickhouseParameter(parameterIndex, value) {
|
|
45
|
+
return `{p${parameterIndex}:${mapToClickHouseType(value)}}`;
|
|
46
|
+
}
|
|
47
|
+
function emptyIfUndefined(value) {
|
|
48
|
+
return value === void 0 ? "" : value;
|
|
49
|
+
}
|
|
50
|
+
var isTable, isColumn, instanceofSql, Sql, toQuery, toQueryPreview, getValueFromParameter, mapToClickHouseType;
|
|
51
|
+
var init_sqlHelpers = __esm({
|
|
52
|
+
"src/sqlHelpers.ts"() {
|
|
53
|
+
"use strict";
|
|
54
|
+
isTable = (value) => typeof value === "object" && value !== null && "kind" in value && value.kind === "OlapTable";
|
|
55
|
+
isColumn = (value) => typeof value === "object" && "name" in value && "annotations" in value;
|
|
56
|
+
instanceofSql = (value) => typeof value === "object" && "values" in value && "strings" in value;
|
|
57
|
+
Sql = class {
|
|
58
|
+
values;
|
|
59
|
+
strings;
|
|
60
|
+
constructor(rawStrings, rawValues) {
|
|
61
|
+
if (rawStrings.length - 1 !== rawValues.length) {
|
|
62
|
+
if (rawStrings.length === 0) {
|
|
63
|
+
throw new TypeError("Expected at least 1 string");
|
|
64
|
+
}
|
|
65
|
+
throw new TypeError(
|
|
66
|
+
`Expected ${rawStrings.length} strings to have ${rawStrings.length - 1} values`
|
|
67
|
+
);
|
|
68
|
+
}
|
|
69
|
+
const valuesLength = rawValues.reduce(
|
|
70
|
+
(len, value) => len + (instanceofSql(value) ? value.values.length : isColumn(value) || isTable(value) ? 0 : 1),
|
|
71
|
+
0
|
|
72
|
+
);
|
|
73
|
+
this.values = new Array(valuesLength);
|
|
74
|
+
this.strings = new Array(valuesLength + 1);
|
|
75
|
+
this.strings[0] = rawStrings[0];
|
|
76
|
+
let i = 0, pos = 0;
|
|
77
|
+
while (i < rawValues.length) {
|
|
78
|
+
const child = rawValues[i++];
|
|
79
|
+
const rawString = rawStrings[i];
|
|
80
|
+
if (instanceofSql(child)) {
|
|
81
|
+
this.strings[pos] += child.strings[0];
|
|
82
|
+
let childIndex = 0;
|
|
83
|
+
while (childIndex < child.values.length) {
|
|
84
|
+
this.values[pos++] = child.values[childIndex++];
|
|
85
|
+
this.strings[pos] = child.strings[childIndex];
|
|
86
|
+
}
|
|
87
|
+
this.strings[pos] += rawString;
|
|
88
|
+
} else if (isColumn(child)) {
|
|
89
|
+
const aggregationFunction = child.annotations.find(
|
|
90
|
+
([k, _]) => k === "aggregationFunction"
|
|
91
|
+
);
|
|
92
|
+
if (aggregationFunction !== void 0) {
|
|
93
|
+
this.strings[pos] += `${aggregationFunction[1].functionName}Merge(\`${child.name}\`)`;
|
|
94
|
+
} else {
|
|
95
|
+
this.strings[pos] += `\`${child.name}\``;
|
|
96
|
+
}
|
|
97
|
+
this.strings[pos] += rawString;
|
|
98
|
+
} else if (isTable(child)) {
|
|
99
|
+
if (child.config.database) {
|
|
100
|
+
this.strings[pos] += `\`${child.config.database}\`.\`${child.name}\``;
|
|
101
|
+
} else {
|
|
102
|
+
this.strings[pos] += `\`${child.name}\``;
|
|
103
|
+
}
|
|
104
|
+
this.strings[pos] += rawString;
|
|
105
|
+
} else {
|
|
106
|
+
this.values[pos++] = child;
|
|
107
|
+
this.strings[pos] = rawString;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
};
|
|
112
|
+
toQuery = (sql3) => {
|
|
113
|
+
const parameterizedStubs = sql3.values.map(
|
|
114
|
+
(v, i) => createClickhouseParameter(i, v)
|
|
115
|
+
);
|
|
116
|
+
const query = sql3.strings.map(
|
|
117
|
+
(s, i) => s != "" ? `${s}${emptyIfUndefined(parameterizedStubs[i])}` : ""
|
|
118
|
+
).join("");
|
|
119
|
+
const query_params = sql3.values.reduce(
|
|
120
|
+
(acc, v, i) => ({
|
|
121
|
+
...acc,
|
|
122
|
+
[`p${i}`]: getValueFromParameter(v)
|
|
123
|
+
}),
|
|
124
|
+
{}
|
|
125
|
+
);
|
|
126
|
+
return [query, query_params];
|
|
127
|
+
};
|
|
128
|
+
toQueryPreview = (sql3) => {
|
|
129
|
+
try {
|
|
130
|
+
const formatValue = (v) => {
|
|
131
|
+
if (Array.isArray(v)) {
|
|
132
|
+
const [type, val] = v;
|
|
133
|
+
if (type === "Identifier") {
|
|
134
|
+
return `\`${String(val)}\``;
|
|
135
|
+
}
|
|
136
|
+
return `[${v.map((x) => formatValue(x)).join(", ")}]`;
|
|
137
|
+
}
|
|
138
|
+
if (v === null || v === void 0) return "NULL";
|
|
139
|
+
if (typeof v === "string") return `'${v.replace(/'/g, "''")}'`;
|
|
140
|
+
if (typeof v === "number") return String(v);
|
|
141
|
+
if (typeof v === "boolean") return v ? "true" : "false";
|
|
142
|
+
if (v instanceof Date)
|
|
143
|
+
return `'${v.toISOString().replace("T", " ").slice(0, 19)}'`;
|
|
144
|
+
try {
|
|
145
|
+
return JSON.stringify(v);
|
|
146
|
+
} catch {
|
|
147
|
+
return String(v);
|
|
148
|
+
}
|
|
149
|
+
};
|
|
150
|
+
let out = sql3.strings[0] ?? "";
|
|
151
|
+
for (let i = 0; i < sql3.values.length; i++) {
|
|
152
|
+
const val = getValueFromParameter(sql3.values[i]);
|
|
153
|
+
out += formatValue(val);
|
|
154
|
+
out += sql3.strings[i + 1] ?? "";
|
|
155
|
+
}
|
|
156
|
+
return out.replace(/\s+/g, " ").trim();
|
|
157
|
+
} catch (error) {
|
|
158
|
+
console.log(`toQueryPreview error: ${error}`);
|
|
159
|
+
return "/* query preview unavailable */";
|
|
160
|
+
}
|
|
161
|
+
};
|
|
162
|
+
getValueFromParameter = (value) => {
|
|
163
|
+
if (Array.isArray(value)) {
|
|
164
|
+
const [type, val] = value;
|
|
165
|
+
if (type === "Identifier") return val;
|
|
166
|
+
}
|
|
167
|
+
return value;
|
|
168
|
+
};
|
|
169
|
+
mapToClickHouseType = (value) => {
|
|
170
|
+
if (typeof value === "number") {
|
|
171
|
+
return Number.isInteger(value) ? "Int" : "Float";
|
|
172
|
+
}
|
|
173
|
+
if (typeof value === "boolean") return "Bool";
|
|
174
|
+
if (value instanceof Date) return "DateTime";
|
|
175
|
+
if (Array.isArray(value)) {
|
|
176
|
+
const [type, _] = value;
|
|
177
|
+
return type;
|
|
178
|
+
}
|
|
179
|
+
return "String";
|
|
180
|
+
};
|
|
181
|
+
}
|
|
182
|
+
});
|
|
183
|
+
|
|
184
|
+
// src/blocks/helpers.ts
|
|
185
|
+
var init_helpers = __esm({
|
|
186
|
+
"src/blocks/helpers.ts"() {
|
|
187
|
+
"use strict";
|
|
188
|
+
init_sqlHelpers();
|
|
189
|
+
}
|
|
190
|
+
});
|
|
191
|
+
|
|
192
|
+
// src/dmv2/sdk/olapTable.ts
|
|
193
|
+
import { Readable } from "stream";
|
|
194
|
+
import { createHash } from "crypto";
|
|
195
|
+
var init_olapTable = __esm({
|
|
196
|
+
"src/dmv2/sdk/olapTable.ts"() {
|
|
197
|
+
"use strict";
|
|
198
|
+
init_typedBase();
|
|
199
|
+
init_dataModelTypes();
|
|
200
|
+
init_helpers();
|
|
201
|
+
init_internal();
|
|
202
|
+
init_sqlHelpers();
|
|
203
|
+
}
|
|
204
|
+
});
|
|
205
|
+
|
|
206
|
+
// src/dmv2/sdk/stream.ts
|
|
207
|
+
import { createHash as createHash2 } from "crypto";
|
|
208
|
+
var init_stream = __esm({
|
|
209
|
+
"src/dmv2/sdk/stream.ts"() {
|
|
210
|
+
"use strict";
|
|
211
|
+
init_typedBase();
|
|
212
|
+
init_internal();
|
|
213
|
+
init_stackTrace();
|
|
214
|
+
}
|
|
215
|
+
});
|
|
216
|
+
|
|
217
|
+
// src/dmv2/sdk/workflow.ts
|
|
218
|
+
var init_workflow = __esm({
|
|
219
|
+
"src/dmv2/sdk/workflow.ts"() {
|
|
220
|
+
"use strict";
|
|
221
|
+
init_internal();
|
|
222
|
+
}
|
|
223
|
+
});
|
|
224
|
+
|
|
225
|
+
// src/dmv2/sdk/ingestApi.ts
|
|
226
|
+
var init_ingestApi = __esm({
|
|
227
|
+
"src/dmv2/sdk/ingestApi.ts"() {
|
|
228
|
+
"use strict";
|
|
229
|
+
init_typedBase();
|
|
230
|
+
init_internal();
|
|
231
|
+
}
|
|
232
|
+
});
|
|
233
|
+
|
|
234
|
+
// src/dmv2/sdk/consumptionApi.ts
|
|
235
|
+
var init_consumptionApi = __esm({
|
|
236
|
+
"src/dmv2/sdk/consumptionApi.ts"() {
|
|
237
|
+
"use strict";
|
|
238
|
+
init_typedBase();
|
|
239
|
+
init_internal();
|
|
240
|
+
}
|
|
241
|
+
});
|
|
242
|
+
|
|
243
|
+
// src/dmv2/sdk/ingestPipeline.ts
|
|
244
|
+
var init_ingestPipeline = __esm({
|
|
245
|
+
"src/dmv2/sdk/ingestPipeline.ts"() {
|
|
246
|
+
"use strict";
|
|
247
|
+
init_typedBase();
|
|
248
|
+
init_stream();
|
|
249
|
+
init_olapTable();
|
|
250
|
+
init_ingestApi();
|
|
251
|
+
init_helpers();
|
|
252
|
+
}
|
|
253
|
+
});
|
|
254
|
+
|
|
255
|
+
// src/dmv2/sdk/etlPipeline.ts
|
|
256
|
+
var init_etlPipeline = __esm({
|
|
257
|
+
"src/dmv2/sdk/etlPipeline.ts"() {
|
|
258
|
+
"use strict";
|
|
259
|
+
init_workflow();
|
|
260
|
+
}
|
|
261
|
+
});
|
|
262
|
+
|
|
263
|
+
// src/dmv2/sdk/sqlResource.ts
|
|
264
|
+
var init_sqlResource = __esm({
|
|
265
|
+
"src/dmv2/sdk/sqlResource.ts"() {
|
|
266
|
+
"use strict";
|
|
267
|
+
init_internal();
|
|
268
|
+
init_sqlHelpers();
|
|
269
|
+
init_stackTrace();
|
|
270
|
+
}
|
|
271
|
+
});
|
|
272
|
+
|
|
273
|
+
// src/dmv2/sdk/materializedView.ts
|
|
274
|
+
var init_materializedView = __esm({
|
|
275
|
+
"src/dmv2/sdk/materializedView.ts"() {
|
|
276
|
+
"use strict";
|
|
277
|
+
init_helpers();
|
|
278
|
+
init_sqlHelpers();
|
|
279
|
+
init_olapTable();
|
|
280
|
+
init_sqlResource();
|
|
281
|
+
}
|
|
282
|
+
});
|
|
283
|
+
|
|
284
|
+
// src/dmv2/sdk/view.ts
|
|
285
|
+
var init_view = __esm({
|
|
286
|
+
"src/dmv2/sdk/view.ts"() {
|
|
287
|
+
"use strict";
|
|
288
|
+
init_helpers();
|
|
289
|
+
init_sqlHelpers();
|
|
290
|
+
init_sqlResource();
|
|
291
|
+
}
|
|
292
|
+
});
|
|
293
|
+
|
|
294
|
+
// src/dmv2/sdk/lifeCycle.ts
|
|
295
|
+
var init_lifeCycle = __esm({
|
|
296
|
+
"src/dmv2/sdk/lifeCycle.ts"() {
|
|
297
|
+
"use strict";
|
|
298
|
+
}
|
|
299
|
+
});
|
|
300
|
+
|
|
301
|
+
// src/dmv2/sdk/webApp.ts
|
|
302
|
+
var init_webApp = __esm({
|
|
303
|
+
"src/dmv2/sdk/webApp.ts"() {
|
|
304
|
+
"use strict";
|
|
305
|
+
init_internal();
|
|
306
|
+
}
|
|
307
|
+
});
|
|
308
|
+
|
|
309
|
+
// src/dmv2/registry.ts
|
|
310
|
+
var init_registry = __esm({
|
|
311
|
+
"src/dmv2/registry.ts"() {
|
|
312
|
+
"use strict";
|
|
313
|
+
init_internal();
|
|
314
|
+
}
|
|
315
|
+
});
|
|
316
|
+
|
|
317
|
+
// src/dmv2/index.ts
|
|
318
|
+
var init_dmv2 = __esm({
|
|
319
|
+
"src/dmv2/index.ts"() {
|
|
320
|
+
"use strict";
|
|
321
|
+
init_olapTable();
|
|
322
|
+
init_stream();
|
|
323
|
+
init_workflow();
|
|
324
|
+
init_ingestApi();
|
|
325
|
+
init_consumptionApi();
|
|
326
|
+
init_ingestPipeline();
|
|
327
|
+
init_etlPipeline();
|
|
328
|
+
init_materializedView();
|
|
329
|
+
init_sqlResource();
|
|
330
|
+
init_view();
|
|
331
|
+
init_lifeCycle();
|
|
332
|
+
init_webApp();
|
|
333
|
+
init_registry();
|
|
334
|
+
}
|
|
335
|
+
});
|
|
336
|
+
|
|
337
|
+
// src/dataModels/types.ts
|
|
338
|
+
var init_types = __esm({
|
|
339
|
+
"src/dataModels/types.ts"() {
|
|
340
|
+
"use strict";
|
|
341
|
+
}
|
|
342
|
+
});
|
|
343
|
+
|
|
344
|
+
// src/browserCompatible.ts
|
|
345
|
+
var init_browserCompatible = __esm({
|
|
346
|
+
"src/browserCompatible.ts"() {
|
|
347
|
+
"use strict";
|
|
348
|
+
init_dmv2();
|
|
349
|
+
init_types();
|
|
350
|
+
init_sqlHelpers();
|
|
351
|
+
}
|
|
352
|
+
});
|
|
12
353
|
|
|
13
354
|
// src/commons.ts
|
|
14
355
|
import http from "http";
|
|
@@ -139,159 +480,12 @@ var init_commons = __esm({
|
|
|
139
480
|
}
|
|
140
481
|
});
|
|
141
482
|
|
|
142
|
-
// src/
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
import process2 from "process";
|
|
147
|
-
|
|
148
|
-
// src/sqlHelpers.ts
|
|
149
|
-
var isTable = (value) => typeof value === "object" && value !== null && "kind" in value && value.kind === "OlapTable";
|
|
150
|
-
var isColumn = (value) => typeof value === "object" && "name" in value && "annotations" in value;
|
|
151
|
-
function sql(strings, ...values) {
|
|
152
|
-
return new Sql(strings, values);
|
|
153
|
-
}
|
|
154
|
-
var instanceofSql = (value) => typeof value === "object" && "values" in value && "strings" in value;
|
|
155
|
-
var Sql = class {
|
|
156
|
-
values;
|
|
157
|
-
strings;
|
|
158
|
-
constructor(rawStrings, rawValues) {
|
|
159
|
-
if (rawStrings.length - 1 !== rawValues.length) {
|
|
160
|
-
if (rawStrings.length === 0) {
|
|
161
|
-
throw new TypeError("Expected at least 1 string");
|
|
162
|
-
}
|
|
163
|
-
throw new TypeError(
|
|
164
|
-
`Expected ${rawStrings.length} strings to have ${rawStrings.length - 1} values`
|
|
165
|
-
);
|
|
166
|
-
}
|
|
167
|
-
const valuesLength = rawValues.reduce(
|
|
168
|
-
(len, value) => len + (instanceofSql(value) ? value.values.length : isColumn(value) || isTable(value) ? 0 : 1),
|
|
169
|
-
0
|
|
170
|
-
);
|
|
171
|
-
this.values = new Array(valuesLength);
|
|
172
|
-
this.strings = new Array(valuesLength + 1);
|
|
173
|
-
this.strings[0] = rawStrings[0];
|
|
174
|
-
let i = 0, pos = 0;
|
|
175
|
-
while (i < rawValues.length) {
|
|
176
|
-
const child = rawValues[i++];
|
|
177
|
-
const rawString = rawStrings[i];
|
|
178
|
-
if (instanceofSql(child)) {
|
|
179
|
-
this.strings[pos] += child.strings[0];
|
|
180
|
-
let childIndex = 0;
|
|
181
|
-
while (childIndex < child.values.length) {
|
|
182
|
-
this.values[pos++] = child.values[childIndex++];
|
|
183
|
-
this.strings[pos] = child.strings[childIndex];
|
|
184
|
-
}
|
|
185
|
-
this.strings[pos] += rawString;
|
|
186
|
-
} else if (isColumn(child)) {
|
|
187
|
-
const aggregationFunction = child.annotations.find(
|
|
188
|
-
([k, _]) => k === "aggregationFunction"
|
|
189
|
-
);
|
|
190
|
-
if (aggregationFunction !== void 0) {
|
|
191
|
-
this.strings[pos] += `${aggregationFunction[1].functionName}Merge(\`${child.name}\`)`;
|
|
192
|
-
} else {
|
|
193
|
-
this.strings[pos] += `\`${child.name}\``;
|
|
194
|
-
}
|
|
195
|
-
this.strings[pos] += rawString;
|
|
196
|
-
} else if (isTable(child)) {
|
|
197
|
-
if (child.config.database) {
|
|
198
|
-
this.strings[pos] += `\`${child.config.database}\`.\`${child.name}\``;
|
|
199
|
-
} else {
|
|
200
|
-
this.strings[pos] += `\`${child.name}\``;
|
|
201
|
-
}
|
|
202
|
-
this.strings[pos] += rawString;
|
|
203
|
-
} else {
|
|
204
|
-
this.values[pos++] = child;
|
|
205
|
-
this.strings[pos] = rawString;
|
|
206
|
-
}
|
|
207
|
-
}
|
|
208
|
-
}
|
|
209
|
-
};
|
|
210
|
-
var toQuery = (sql3) => {
|
|
211
|
-
const parameterizedStubs = sql3.values.map(
|
|
212
|
-
(v, i) => createClickhouseParameter(i, v)
|
|
213
|
-
);
|
|
214
|
-
const query = sql3.strings.map(
|
|
215
|
-
(s, i) => s != "" ? `${s}${emptyIfUndefined(parameterizedStubs[i])}` : ""
|
|
216
|
-
).join("");
|
|
217
|
-
const query_params = sql3.values.reduce(
|
|
218
|
-
(acc, v, i) => ({
|
|
219
|
-
...acc,
|
|
220
|
-
[`p${i}`]: getValueFromParameter(v)
|
|
221
|
-
}),
|
|
222
|
-
{}
|
|
223
|
-
);
|
|
224
|
-
return [query, query_params];
|
|
225
|
-
};
|
|
226
|
-
var toQueryPreview = (sql3) => {
|
|
227
|
-
try {
|
|
228
|
-
const formatValue = (v) => {
|
|
229
|
-
if (Array.isArray(v)) {
|
|
230
|
-
const [type, val] = v;
|
|
231
|
-
if (type === "Identifier") {
|
|
232
|
-
return `\`${String(val)}\``;
|
|
233
|
-
}
|
|
234
|
-
return `[${v.map((x) => formatValue(x)).join(", ")}]`;
|
|
235
|
-
}
|
|
236
|
-
if (v === null || v === void 0) return "NULL";
|
|
237
|
-
if (typeof v === "string") return `'${v.replace(/'/g, "''")}'`;
|
|
238
|
-
if (typeof v === "number") return String(v);
|
|
239
|
-
if (typeof v === "boolean") return v ? "true" : "false";
|
|
240
|
-
if (v instanceof Date)
|
|
241
|
-
return `'${v.toISOString().replace("T", " ").slice(0, 19)}'`;
|
|
242
|
-
try {
|
|
243
|
-
return JSON.stringify(v);
|
|
244
|
-
} catch {
|
|
245
|
-
return String(v);
|
|
246
|
-
}
|
|
247
|
-
};
|
|
248
|
-
let out = sql3.strings[0] ?? "";
|
|
249
|
-
for (let i = 0; i < sql3.values.length; i++) {
|
|
250
|
-
const val = getValueFromParameter(sql3.values[i]);
|
|
251
|
-
out += formatValue(val);
|
|
252
|
-
out += sql3.strings[i + 1] ?? "";
|
|
253
|
-
}
|
|
254
|
-
return out.replace(/\s+/g, " ").trim();
|
|
255
|
-
} catch (error) {
|
|
256
|
-
console.log(`toQueryPreview error: ${error}`);
|
|
257
|
-
return "/* query preview unavailable */";
|
|
258
|
-
}
|
|
259
|
-
};
|
|
260
|
-
var getValueFromParameter = (value) => {
|
|
261
|
-
if (Array.isArray(value)) {
|
|
262
|
-
const [type, val] = value;
|
|
263
|
-
if (type === "Identifier") return val;
|
|
264
|
-
}
|
|
265
|
-
return value;
|
|
266
|
-
};
|
|
267
|
-
function createClickhouseParameter(parameterIndex, value) {
|
|
268
|
-
return `{p${parameterIndex}:${mapToClickHouseType(value)}}`;
|
|
269
|
-
}
|
|
270
|
-
var mapToClickHouseType = (value) => {
|
|
271
|
-
if (typeof value === "number") {
|
|
272
|
-
return Number.isInteger(value) ? "Int" : "Float";
|
|
273
|
-
}
|
|
274
|
-
if (typeof value === "boolean") return "Bool";
|
|
275
|
-
if (value instanceof Date) return "DateTime";
|
|
276
|
-
if (Array.isArray(value)) {
|
|
277
|
-
const [type, _] = value;
|
|
278
|
-
return type;
|
|
483
|
+
// src/secrets.ts
|
|
484
|
+
var init_secrets = __esm({
|
|
485
|
+
"src/secrets.ts"() {
|
|
486
|
+
"use strict";
|
|
279
487
|
}
|
|
280
|
-
|
|
281
|
-
};
|
|
282
|
-
function emptyIfUndefined(value) {
|
|
283
|
-
return value === void 0 ? "" : value;
|
|
284
|
-
}
|
|
285
|
-
|
|
286
|
-
// src/dmv2/sdk/olapTable.ts
|
|
287
|
-
import { Readable } from "stream";
|
|
288
|
-
import { createHash } from "crypto";
|
|
289
|
-
|
|
290
|
-
// src/dmv2/sdk/stream.ts
|
|
291
|
-
import { createHash as createHash2 } from "crypto";
|
|
292
|
-
|
|
293
|
-
// src/index.ts
|
|
294
|
-
init_commons();
|
|
488
|
+
});
|
|
295
489
|
|
|
296
490
|
// src/consumption-apis/helpers.ts
|
|
297
491
|
import {
|
|
@@ -313,142 +507,6 @@ function formatElapsedTime(ms) {
|
|
|
313
507
|
const remainingSeconds = seconds % 60;
|
|
314
508
|
return `${minutes} minutes and ${remainingSeconds.toFixed(2)} seconds`;
|
|
315
509
|
}
|
|
316
|
-
var MooseClient = class {
|
|
317
|
-
query;
|
|
318
|
-
workflow;
|
|
319
|
-
constructor(queryClient, temporalClient) {
|
|
320
|
-
this.query = queryClient;
|
|
321
|
-
this.workflow = new WorkflowClient(temporalClient);
|
|
322
|
-
}
|
|
323
|
-
};
|
|
324
|
-
var QueryClient = class {
|
|
325
|
-
client;
|
|
326
|
-
query_id_prefix;
|
|
327
|
-
constructor(client, query_id_prefix) {
|
|
328
|
-
this.client = client;
|
|
329
|
-
this.query_id_prefix = query_id_prefix;
|
|
330
|
-
}
|
|
331
|
-
async execute(sql3) {
|
|
332
|
-
const [query, query_params] = toQuery(sql3);
|
|
333
|
-
console.log(`[QueryClient] | Query: ${toQueryPreview(sql3)}`);
|
|
334
|
-
const start = performance.now();
|
|
335
|
-
const result = await this.client.query({
|
|
336
|
-
query,
|
|
337
|
-
query_params,
|
|
338
|
-
format: "JSONEachRow",
|
|
339
|
-
query_id: this.query_id_prefix + randomUUID()
|
|
340
|
-
// Note: wait_end_of_query deliberately NOT set here as this is used for SELECT queries
|
|
341
|
-
// where response buffering would harm streaming performance and concurrency
|
|
342
|
-
});
|
|
343
|
-
const elapsedMs = performance.now() - start;
|
|
344
|
-
console.log(
|
|
345
|
-
`[QueryClient] | Query completed: ${formatElapsedTime(elapsedMs)}`
|
|
346
|
-
);
|
|
347
|
-
return result;
|
|
348
|
-
}
|
|
349
|
-
async command(sql3) {
|
|
350
|
-
const [query, query_params] = toQuery(sql3);
|
|
351
|
-
console.log(`[QueryClient] | Command: ${toQueryPreview(sql3)}`);
|
|
352
|
-
const start = performance.now();
|
|
353
|
-
const result = await this.client.command({
|
|
354
|
-
query,
|
|
355
|
-
query_params,
|
|
356
|
-
query_id: this.query_id_prefix + randomUUID()
|
|
357
|
-
});
|
|
358
|
-
const elapsedMs = performance.now() - start;
|
|
359
|
-
console.log(
|
|
360
|
-
`[QueryClient] | Command completed: ${formatElapsedTime(elapsedMs)}`
|
|
361
|
-
);
|
|
362
|
-
return result;
|
|
363
|
-
}
|
|
364
|
-
};
|
|
365
|
-
var WorkflowClient = class {
|
|
366
|
-
client;
|
|
367
|
-
constructor(temporalClient) {
|
|
368
|
-
this.client = temporalClient;
|
|
369
|
-
}
|
|
370
|
-
async execute(name, input_data) {
|
|
371
|
-
try {
|
|
372
|
-
if (!this.client) {
|
|
373
|
-
return {
|
|
374
|
-
status: 404,
|
|
375
|
-
body: `Temporal client not found. Is the feature flag enabled?`
|
|
376
|
-
};
|
|
377
|
-
}
|
|
378
|
-
const config = await this.getWorkflowConfig(name);
|
|
379
|
-
const [processedInput, workflowId] = this.processInputData(
|
|
380
|
-
name,
|
|
381
|
-
input_data
|
|
382
|
-
);
|
|
383
|
-
console.log(
|
|
384
|
-
`WorkflowClient - starting workflow: ${name} with config ${JSON.stringify(config)} and input_data ${JSON.stringify(processedInput)}`
|
|
385
|
-
);
|
|
386
|
-
const handle = await this.client.workflow.start("ScriptWorkflow", {
|
|
387
|
-
args: [
|
|
388
|
-
{ workflow_name: name, execution_mode: "start" },
|
|
389
|
-
processedInput
|
|
390
|
-
],
|
|
391
|
-
taskQueue: "typescript-script-queue",
|
|
392
|
-
workflowId,
|
|
393
|
-
workflowIdConflictPolicy: "FAIL",
|
|
394
|
-
workflowIdReusePolicy: "ALLOW_DUPLICATE",
|
|
395
|
-
retry: {
|
|
396
|
-
maximumAttempts: config.retries
|
|
397
|
-
},
|
|
398
|
-
workflowRunTimeout: config.timeout
|
|
399
|
-
});
|
|
400
|
-
return {
|
|
401
|
-
status: 200,
|
|
402
|
-
body: `Workflow started: ${name}. View it in the Temporal dashboard: http://localhost:8080/namespaces/default/workflows/${workflowId}/${handle.firstExecutionRunId}/history`
|
|
403
|
-
};
|
|
404
|
-
} catch (error) {
|
|
405
|
-
return {
|
|
406
|
-
status: 400,
|
|
407
|
-
body: `Error starting workflow: ${error}`
|
|
408
|
-
};
|
|
409
|
-
}
|
|
410
|
-
}
|
|
411
|
-
async terminate(workflowId) {
|
|
412
|
-
try {
|
|
413
|
-
if (!this.client) {
|
|
414
|
-
return {
|
|
415
|
-
status: 404,
|
|
416
|
-
body: `Temporal client not found. Is the feature flag enabled?`
|
|
417
|
-
};
|
|
418
|
-
}
|
|
419
|
-
const handle = this.client.workflow.getHandle(workflowId);
|
|
420
|
-
await handle.terminate();
|
|
421
|
-
return {
|
|
422
|
-
status: 200,
|
|
423
|
-
body: `Workflow terminated: ${workflowId}`
|
|
424
|
-
};
|
|
425
|
-
} catch (error) {
|
|
426
|
-
return {
|
|
427
|
-
status: 400,
|
|
428
|
-
body: `Error terminating workflow: ${error}`
|
|
429
|
-
};
|
|
430
|
-
}
|
|
431
|
-
}
|
|
432
|
-
async getWorkflowConfig(name) {
|
|
433
|
-
const workflows = await getWorkflows2();
|
|
434
|
-
const dmv2Workflow = workflows.get(name);
|
|
435
|
-
if (dmv2Workflow) {
|
|
436
|
-
return {
|
|
437
|
-
retries: dmv2Workflow.config.retries || 3,
|
|
438
|
-
timeout: dmv2Workflow.config.timeout || "1h"
|
|
439
|
-
};
|
|
440
|
-
}
|
|
441
|
-
throw new Error(`Workflow config not found for ${name}`);
|
|
442
|
-
}
|
|
443
|
-
processInputData(name, input_data) {
|
|
444
|
-
let workflowId = name;
|
|
445
|
-
if (input_data) {
|
|
446
|
-
const hash = createHash3("sha256").update(JSON.stringify(input_data)).digest("hex").slice(0, 16);
|
|
447
|
-
workflowId = `${name}-${hash}`;
|
|
448
|
-
}
|
|
449
|
-
return [input_data, workflowId];
|
|
450
|
-
}
|
|
451
|
-
};
|
|
452
510
|
async function getTemporalClient(temporalUrl, namespace, clientCert, clientKey, apiKey) {
|
|
453
511
|
try {
|
|
454
512
|
console.info(
|
|
@@ -485,538 +543,984 @@ async function getTemporalClient(temporalUrl, namespace, clientCert, clientKey,
|
|
|
485
543
|
return void 0;
|
|
486
544
|
}
|
|
487
545
|
}
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
* @param options.maxWorkerCount - Maximum number of workers to spawn
|
|
525
|
-
* @throws {Error} If maxCpuUsageRatio is not between 0 and 1
|
|
526
|
-
*/
|
|
527
|
-
constructor(options) {
|
|
528
|
-
this.workerStart = options.workerStart;
|
|
529
|
-
this.workerStop = options.workerStop;
|
|
530
|
-
if (options.maxCpuUsageRatio && (options.maxCpuUsageRatio > 1 || options.maxCpuUsageRatio < 0)) {
|
|
531
|
-
throw new Error("maxCpuUsageRatio must be between 0 and 1");
|
|
532
|
-
}
|
|
533
|
-
this.maxCpuUsageRatio = options.maxCpuUsageRatio || DEFAULT_MAX_CPU_USAGE_RATIO;
|
|
534
|
-
this.usedCpuCount = this.computeCPUUsageCount(
|
|
535
|
-
this.maxCpuUsageRatio,
|
|
536
|
-
options.maxWorkerCount
|
|
537
|
-
);
|
|
538
|
-
}
|
|
539
|
-
/**
|
|
540
|
-
* Calculates the number of CPU cores to utilize based on available parallelism and constraints.
|
|
541
|
-
*
|
|
542
|
-
* @param cpuUsageRatio - Ratio of CPU cores to use (0-1)
|
|
543
|
-
* @param maxWorkerCount - Optional maximum number of workers
|
|
544
|
-
* @returns The number of CPU cores to utilize
|
|
545
|
-
*/
|
|
546
|
-
computeCPUUsageCount(cpuUsageRatio, maxWorkerCount) {
|
|
547
|
-
const cpuCount = availableParallelism();
|
|
548
|
-
const maxWorkers = maxWorkerCount || cpuCount;
|
|
549
|
-
return Math.min(
|
|
550
|
-
maxWorkers,
|
|
551
|
-
Math.max(1, Math.floor(cpuCount * cpuUsageRatio))
|
|
552
|
-
);
|
|
553
|
-
}
|
|
554
|
-
/**
|
|
555
|
-
* Initializes the cluster by spawning worker processes and setting up signal handlers.
|
|
556
|
-
* For the primary process, spawns workers and monitors parent process.
|
|
557
|
-
* For worker processes, executes the worker startup function.
|
|
558
|
-
*
|
|
559
|
-
* @throws {Error} If worker is undefined in worker process
|
|
560
|
-
*/
|
|
561
|
-
async start() {
|
|
562
|
-
process.on(SIGTERM, this.gracefulClusterShutdown(SIGTERM));
|
|
563
|
-
process.on(SIGINT, this.gracefulClusterShutdown(SIGINT));
|
|
564
|
-
if (cluster.isPrimary) {
|
|
565
|
-
const parentPid = process.ppid;
|
|
566
|
-
setInterval(() => {
|
|
567
|
-
try {
|
|
568
|
-
process.kill(parentPid, 0);
|
|
569
|
-
} catch (e) {
|
|
570
|
-
console.log("Parent process has exited.");
|
|
571
|
-
this.gracefulClusterShutdown(SIGTERM)();
|
|
572
|
-
}
|
|
573
|
-
}, 1e3);
|
|
574
|
-
await this.bootWorkers(this.usedCpuCount);
|
|
575
|
-
} else {
|
|
576
|
-
if (!cluster.worker) {
|
|
577
|
-
throw new Error(
|
|
578
|
-
"Worker is not defined, it should be defined in worker process"
|
|
546
|
+
var MooseClient, QueryClient, WorkflowClient;
|
|
547
|
+
var init_helpers2 = __esm({
|
|
548
|
+
"src/consumption-apis/helpers.ts"() {
|
|
549
|
+
"use strict";
|
|
550
|
+
init_internal();
|
|
551
|
+
init_sqlHelpers();
|
|
552
|
+
MooseClient = class {
|
|
553
|
+
query;
|
|
554
|
+
workflow;
|
|
555
|
+
constructor(queryClient, temporalClient) {
|
|
556
|
+
this.query = queryClient;
|
|
557
|
+
this.workflow = new WorkflowClient(temporalClient);
|
|
558
|
+
}
|
|
559
|
+
};
|
|
560
|
+
QueryClient = class {
|
|
561
|
+
client;
|
|
562
|
+
query_id_prefix;
|
|
563
|
+
constructor(client, query_id_prefix) {
|
|
564
|
+
this.client = client;
|
|
565
|
+
this.query_id_prefix = query_id_prefix;
|
|
566
|
+
}
|
|
567
|
+
async execute(sql3) {
|
|
568
|
+
const [query, query_params] = toQuery(sql3);
|
|
569
|
+
console.log(`[QueryClient] | Query: ${toQueryPreview(sql3)}`);
|
|
570
|
+
const start = performance.now();
|
|
571
|
+
const result = await this.client.query({
|
|
572
|
+
query,
|
|
573
|
+
query_params,
|
|
574
|
+
format: "JSONEachRow",
|
|
575
|
+
query_id: this.query_id_prefix + randomUUID()
|
|
576
|
+
// Note: wait_end_of_query deliberately NOT set here as this is used for SELECT queries
|
|
577
|
+
// where response buffering would harm streaming performance and concurrency
|
|
578
|
+
});
|
|
579
|
+
const elapsedMs = performance.now() - start;
|
|
580
|
+
console.log(
|
|
581
|
+
`[QueryClient] | Query completed: ${formatElapsedTime(elapsedMs)}`
|
|
579
582
|
);
|
|
583
|
+
return result;
|
|
580
584
|
}
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
console.info(`Setting ${numWorkers} workers...`);
|
|
596
|
-
for (let i = 0; i < numWorkers; i++) {
|
|
597
|
-
cluster.fork();
|
|
598
|
-
}
|
|
599
|
-
cluster.on("online", (worker) => {
|
|
600
|
-
console.info(`worker process ${worker.process.pid} is online`);
|
|
601
|
-
});
|
|
602
|
-
cluster.on("exit", (worker, code, signal) => {
|
|
603
|
-
console.info(
|
|
604
|
-
`worker ${worker.process.pid} exited with code ${code} and signal ${signal}`
|
|
605
|
-
);
|
|
606
|
-
if (!this.shutdownInProgress) {
|
|
607
|
-
setTimeout(() => cluster.fork(), RESTART_TIME_MS);
|
|
585
|
+
async command(sql3) {
|
|
586
|
+
const [query, query_params] = toQuery(sql3);
|
|
587
|
+
console.log(`[QueryClient] | Command: ${toQueryPreview(sql3)}`);
|
|
588
|
+
const start = performance.now();
|
|
589
|
+
const result = await this.client.command({
|
|
590
|
+
query,
|
|
591
|
+
query_params,
|
|
592
|
+
query_id: this.query_id_prefix + randomUUID()
|
|
593
|
+
});
|
|
594
|
+
const elapsedMs = performance.now() - start;
|
|
595
|
+
console.log(
|
|
596
|
+
`[QueryClient] | Command completed: ${formatElapsedTime(elapsedMs)}`
|
|
597
|
+
);
|
|
598
|
+
return result;
|
|
608
599
|
}
|
|
609
|
-
|
|
610
|
-
|
|
600
|
+
};
|
|
601
|
+
WorkflowClient = class {
|
|
602
|
+
client;
|
|
603
|
+
constructor(temporalClient) {
|
|
604
|
+
this.client = temporalClient;
|
|
611
605
|
}
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
return;
|
|
628
|
-
}
|
|
629
|
-
this.shutdownInProgress = true;
|
|
630
|
-
this.hasCleanWorkerExit = true;
|
|
631
|
-
console.info(
|
|
632
|
-
`Got ${signal} on ${this.processStr}. Graceful shutdown start at ${(/* @__PURE__ */ new Date()).toISOString()}`
|
|
633
|
-
);
|
|
634
|
-
try {
|
|
635
|
-
if (cluster.isPrimary) {
|
|
636
|
-
await this.shutdownWorkers(signal);
|
|
637
|
-
console.info(`${this.processStr} - worker shutdown successful`);
|
|
638
|
-
exit(0);
|
|
639
|
-
} else {
|
|
640
|
-
if (this.startOutput) {
|
|
641
|
-
await this.workerStop(this.startOutput);
|
|
642
|
-
} else {
|
|
643
|
-
console.info(
|
|
644
|
-
`${this.processStr} - shutdown before worker fully started`
|
|
606
|
+
async execute(name, input_data) {
|
|
607
|
+
try {
|
|
608
|
+
if (!this.client) {
|
|
609
|
+
return {
|
|
610
|
+
status: 404,
|
|
611
|
+
body: `Temporal client not found. Is the feature flag enabled?`
|
|
612
|
+
};
|
|
613
|
+
}
|
|
614
|
+
const config = await this.getWorkflowConfig(name);
|
|
615
|
+
const [processedInput, workflowId] = this.processInputData(
|
|
616
|
+
name,
|
|
617
|
+
input_data
|
|
618
|
+
);
|
|
619
|
+
console.log(
|
|
620
|
+
`WorkflowClient - starting workflow: ${name} with config ${JSON.stringify(config)} and input_data ${JSON.stringify(processedInput)}`
|
|
645
621
|
);
|
|
622
|
+
const handle = await this.client.workflow.start("ScriptWorkflow", {
|
|
623
|
+
args: [
|
|
624
|
+
{ workflow_name: name, execution_mode: "start" },
|
|
625
|
+
processedInput
|
|
626
|
+
],
|
|
627
|
+
taskQueue: "typescript-script-queue",
|
|
628
|
+
workflowId,
|
|
629
|
+
workflowIdConflictPolicy: "FAIL",
|
|
630
|
+
workflowIdReusePolicy: "ALLOW_DUPLICATE",
|
|
631
|
+
retry: {
|
|
632
|
+
maximumAttempts: config.retries
|
|
633
|
+
},
|
|
634
|
+
workflowRunTimeout: config.timeout
|
|
635
|
+
});
|
|
636
|
+
return {
|
|
637
|
+
status: 200,
|
|
638
|
+
body: `Workflow started: ${name}. View it in the Temporal dashboard: http://localhost:8080/namespaces/default/workflows/${workflowId}/${handle.firstExecutionRunId}/history`
|
|
639
|
+
};
|
|
640
|
+
} catch (error) {
|
|
641
|
+
return {
|
|
642
|
+
status: 400,
|
|
643
|
+
body: `Error starting workflow: ${error}`
|
|
644
|
+
};
|
|
646
645
|
}
|
|
647
|
-
console.info(`${this.processStr} shutdown successful`);
|
|
648
|
-
this.hasCleanWorkerExit ? exit(0) : exit(1);
|
|
649
646
|
}
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
647
|
+
async terminate(workflowId) {
|
|
648
|
+
try {
|
|
649
|
+
if (!this.client) {
|
|
650
|
+
return {
|
|
651
|
+
status: 404,
|
|
652
|
+
body: `Temporal client not found. Is the feature flag enabled?`
|
|
653
|
+
};
|
|
654
|
+
}
|
|
655
|
+
const handle = this.client.workflow.getHandle(workflowId);
|
|
656
|
+
await handle.terminate();
|
|
657
|
+
return {
|
|
658
|
+
status: 200,
|
|
659
|
+
body: `Workflow terminated: ${workflowId}`
|
|
660
|
+
};
|
|
661
|
+
} catch (error) {
|
|
662
|
+
return {
|
|
663
|
+
status: 400,
|
|
664
|
+
body: `Error terminating workflow: ${error}`
|
|
665
|
+
};
|
|
666
|
+
}
|
|
670
667
|
}
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
668
|
+
async getWorkflowConfig(name) {
|
|
669
|
+
const workflows = await getWorkflows2();
|
|
670
|
+
const dmv2Workflow = workflows.get(name);
|
|
671
|
+
if (dmv2Workflow) {
|
|
672
|
+
return {
|
|
673
|
+
retries: dmv2Workflow.config.retries || 3,
|
|
674
|
+
timeout: dmv2Workflow.config.timeout || "1h"
|
|
675
|
+
};
|
|
676
|
+
}
|
|
677
|
+
throw new Error(`Workflow config not found for ${name}`);
|
|
674
678
|
}
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
Object.values(cluster.workers || {}).filter((worker) => !!worker).forEach((worker) => {
|
|
681
|
-
if (worker && !worker.isDead()) {
|
|
682
|
-
++workersAlive;
|
|
683
|
-
if (funcRun == 1) {
|
|
684
|
-
worker.kill(signal);
|
|
685
|
-
}
|
|
686
|
-
}
|
|
687
|
-
});
|
|
688
|
-
console.info(workersAlive + " workers alive");
|
|
689
|
-
if (workersAlive == 0) {
|
|
690
|
-
clearInterval(interval);
|
|
691
|
-
return resolve2();
|
|
679
|
+
processInputData(name, input_data) {
|
|
680
|
+
let workflowId = name;
|
|
681
|
+
if (input_data) {
|
|
682
|
+
const hash = createHash3("sha256").update(JSON.stringify(input_data)).digest("hex").slice(0, 16);
|
|
683
|
+
workflowId = `${name}-${hash}`;
|
|
692
684
|
}
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
}
|
|
696
|
-
}
|
|
697
|
-
};
|
|
685
|
+
return [input_data, workflowId];
|
|
686
|
+
}
|
|
687
|
+
};
|
|
688
|
+
}
|
|
689
|
+
});
|
|
698
690
|
|
|
699
|
-
// src/consumption-apis/
|
|
700
|
-
var
|
|
701
|
-
|
|
702
|
-
|
|
691
|
+
// src/consumption-apis/webAppHelpers.ts
|
|
692
|
+
var init_webAppHelpers = __esm({
|
|
693
|
+
"src/consumption-apis/webAppHelpers.ts"() {
|
|
694
|
+
"use strict";
|
|
695
|
+
}
|
|
703
696
|
});
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
}
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
697
|
+
|
|
698
|
+
// src/scripts/task.ts
|
|
699
|
+
var init_task = __esm({
|
|
700
|
+
"src/scripts/task.ts"() {
|
|
701
|
+
"use strict";
|
|
702
|
+
}
|
|
703
|
+
});
|
|
704
|
+
|
|
705
|
+
// src/cluster-utils.ts
|
|
706
|
+
import cluster from "cluster";
|
|
707
|
+
import { availableParallelism } from "os";
|
|
708
|
+
import { exit } from "process";
|
|
709
|
+
var DEFAULT_MAX_CPU_USAGE_RATIO, RESTART_TIME_MS, SIGTERM, SIGINT, SHUTDOWN_WORKERS_INTERVAL, Cluster;
|
|
710
|
+
var init_cluster_utils = __esm({
|
|
711
|
+
"src/cluster-utils.ts"() {
|
|
712
|
+
"use strict";
|
|
713
|
+
DEFAULT_MAX_CPU_USAGE_RATIO = 0.7;
|
|
714
|
+
RESTART_TIME_MS = 1e4;
|
|
715
|
+
SIGTERM = "SIGTERM";
|
|
716
|
+
SIGINT = "SIGINT";
|
|
717
|
+
SHUTDOWN_WORKERS_INTERVAL = 500;
|
|
718
|
+
Cluster = class {
|
|
719
|
+
// Tracks if shutdown is currently in progress
|
|
720
|
+
shutdownInProgress = false;
|
|
721
|
+
// Tracks if workers exited cleanly during shutdown
|
|
722
|
+
hasCleanWorkerExit = true;
|
|
723
|
+
// String identifying if this is primary or worker process
|
|
724
|
+
processStr = `${cluster.isPrimary ? "primary" : "worker"} process ${process.pid}`;
|
|
725
|
+
// Functions for starting and stopping workers
|
|
726
|
+
workerStart;
|
|
727
|
+
workerStop;
|
|
728
|
+
// Result from starting worker, needed for cleanup
|
|
729
|
+
startOutput;
|
|
730
|
+
maxCpuUsageRatio;
|
|
731
|
+
usedCpuCount;
|
|
732
|
+
/**
|
|
733
|
+
* Creates a new cluster manager instance.
|
|
734
|
+
*
|
|
735
|
+
* @param options - Configuration options for the cluster
|
|
736
|
+
* @param options.workerStart - Async function to execute when starting a worker
|
|
737
|
+
* @param options.workerStop - Async function to execute when stopping a worker
|
|
738
|
+
* @param options.maxCpuUsageRatio - Maximum ratio of CPU cores to utilize (0-1)
|
|
739
|
+
* @param options.maxWorkerCount - Maximum number of workers to spawn
|
|
740
|
+
* @throws {Error} If maxCpuUsageRatio is not between 0 and 1
|
|
741
|
+
*/
|
|
742
|
+
constructor(options) {
|
|
743
|
+
this.workerStart = options.workerStart;
|
|
744
|
+
this.workerStop = options.workerStop;
|
|
745
|
+
if (options.maxCpuUsageRatio && (options.maxCpuUsageRatio > 1 || options.maxCpuUsageRatio < 0)) {
|
|
746
|
+
throw new Error("maxCpuUsageRatio must be between 0 and 1");
|
|
747
|
+
}
|
|
748
|
+
this.maxCpuUsageRatio = options.maxCpuUsageRatio || DEFAULT_MAX_CPU_USAGE_RATIO;
|
|
749
|
+
this.usedCpuCount = this.computeCPUUsageCount(
|
|
750
|
+
this.maxCpuUsageRatio,
|
|
751
|
+
options.maxWorkerCount
|
|
752
|
+
);
|
|
753
|
+
}
|
|
754
|
+
/**
|
|
755
|
+
* Calculates the number of CPU cores to utilize based on available parallelism and constraints.
|
|
756
|
+
*
|
|
757
|
+
* @param cpuUsageRatio - Ratio of CPU cores to use (0-1)
|
|
758
|
+
* @param maxWorkerCount - Optional maximum number of workers
|
|
759
|
+
* @returns The number of CPU cores to utilize
|
|
760
|
+
*/
|
|
761
|
+
computeCPUUsageCount(cpuUsageRatio, maxWorkerCount) {
|
|
762
|
+
const cpuCount = availableParallelism();
|
|
763
|
+
const maxWorkers = maxWorkerCount || cpuCount;
|
|
764
|
+
return Math.min(
|
|
765
|
+
maxWorkers,
|
|
766
|
+
Math.max(1, Math.floor(cpuCount * cpuUsageRatio))
|
|
767
|
+
);
|
|
768
|
+
}
|
|
769
|
+
/**
|
|
770
|
+
* Initializes the cluster by spawning worker processes and setting up signal handlers.
|
|
771
|
+
* For the primary process, spawns workers and monitors parent process.
|
|
772
|
+
* For worker processes, executes the worker startup function.
|
|
773
|
+
*
|
|
774
|
+
* @throws {Error} If worker is undefined in worker process
|
|
775
|
+
*/
|
|
776
|
+
async start() {
|
|
777
|
+
process.on(SIGTERM, this.gracefulClusterShutdown(SIGTERM));
|
|
778
|
+
process.on(SIGINT, this.gracefulClusterShutdown(SIGINT));
|
|
779
|
+
if (cluster.isPrimary) {
|
|
780
|
+
const parentPid = process.ppid;
|
|
781
|
+
setInterval(() => {
|
|
782
|
+
try {
|
|
783
|
+
process.kill(parentPid, 0);
|
|
784
|
+
} catch (e) {
|
|
785
|
+
console.log("Parent process has exited.");
|
|
786
|
+
this.gracefulClusterShutdown(SIGTERM)();
|
|
735
787
|
}
|
|
788
|
+
}, 1e3);
|
|
789
|
+
await this.bootWorkers(this.usedCpuCount);
|
|
790
|
+
} else {
|
|
791
|
+
if (!cluster.worker) {
|
|
792
|
+
throw new Error(
|
|
793
|
+
"Worker is not defined, it should be defined in worker process"
|
|
794
|
+
);
|
|
736
795
|
}
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
return;
|
|
796
|
+
this.startOutput = await this.workerStart(
|
|
797
|
+
cluster.worker,
|
|
798
|
+
this.usedCpuCount
|
|
799
|
+
);
|
|
742
800
|
}
|
|
743
|
-
} else if (enforceAuth) {
|
|
744
|
-
res.writeHead(401, { "Content-Type": "application/json" });
|
|
745
|
-
res.end(JSON.stringify({ error: "Unauthorized" }));
|
|
746
|
-
httpLogger(req, res, start);
|
|
747
|
-
return;
|
|
748
801
|
}
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
802
|
+
/**
|
|
803
|
+
* Spawns worker processes and configures their lifecycle event handlers.
|
|
804
|
+
* Handles worker online, exit and disconnect events.
|
|
805
|
+
* Automatically restarts failed workers during normal operation.
|
|
806
|
+
*
|
|
807
|
+
* @param numWorkers - Number of worker processes to spawn
|
|
808
|
+
*/
|
|
809
|
+
bootWorkers = async (numWorkers) => {
|
|
810
|
+
console.info(`Setting ${numWorkers} workers...`);
|
|
811
|
+
for (let i = 0; i < numWorkers; i++) {
|
|
812
|
+
cluster.fork();
|
|
813
|
+
}
|
|
814
|
+
cluster.on("online", (worker) => {
|
|
815
|
+
console.info(`worker process ${worker.process.pid} is online`);
|
|
816
|
+
});
|
|
817
|
+
cluster.on("exit", (worker, code, signal) => {
|
|
818
|
+
console.info(
|
|
819
|
+
`worker ${worker.process.pid} exited with code ${code} and signal ${signal}`
|
|
820
|
+
);
|
|
821
|
+
if (!this.shutdownInProgress) {
|
|
822
|
+
setTimeout(() => cluster.fork(), RESTART_TIME_MS);
|
|
823
|
+
}
|
|
824
|
+
if (this.shutdownInProgress && code != 0) {
|
|
825
|
+
this.hasCleanWorkerExit = false;
|
|
826
|
+
}
|
|
827
|
+
});
|
|
828
|
+
cluster.on("disconnect", (worker) => {
|
|
829
|
+
console.info(`worker process ${worker.process.pid} has disconnected`);
|
|
830
|
+
});
|
|
831
|
+
};
|
|
832
|
+
/**
|
|
833
|
+
* Creates a handler function for graceful shutdown on receipt of a signal.
|
|
834
|
+
* Ensures only one shutdown can occur at a time.
|
|
835
|
+
* Handles shutdown differently for primary and worker processes.
|
|
836
|
+
*
|
|
837
|
+
* @param signal - The signal triggering the shutdown (e.g. SIGTERM)
|
|
838
|
+
* @returns An async function that performs the shutdown
|
|
839
|
+
*/
|
|
840
|
+
gracefulClusterShutdown = (signal) => async () => {
|
|
841
|
+
if (this.shutdownInProgress) {
|
|
842
|
+
return;
|
|
843
|
+
}
|
|
844
|
+
this.shutdownInProgress = true;
|
|
845
|
+
this.hasCleanWorkerExit = true;
|
|
846
|
+
console.info(
|
|
847
|
+
`Got ${signal} on ${this.processStr}. Graceful shutdown start at ${(/* @__PURE__ */ new Date()).toISOString()}`
|
|
848
|
+
);
|
|
849
|
+
try {
|
|
850
|
+
if (cluster.isPrimary) {
|
|
851
|
+
await this.shutdownWorkers(signal);
|
|
852
|
+
console.info(`${this.processStr} - worker shutdown successful`);
|
|
853
|
+
exit(0);
|
|
854
|
+
} else {
|
|
855
|
+
if (this.startOutput) {
|
|
856
|
+
await this.workerStop(this.startOutput);
|
|
756
857
|
} else {
|
|
757
|
-
|
|
858
|
+
console.info(
|
|
859
|
+
`${this.processStr} - shutdown before worker fully started`
|
|
860
|
+
);
|
|
758
861
|
}
|
|
759
|
-
|
|
760
|
-
|
|
862
|
+
console.info(`${this.processStr} shutdown successful`);
|
|
863
|
+
this.hasCleanWorkerExit ? exit(0) : exit(1);
|
|
761
864
|
}
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
865
|
+
} catch (e) {
|
|
866
|
+
console.error(`${this.processStr} - shutdown failed`, e);
|
|
867
|
+
exit(1);
|
|
868
|
+
}
|
|
869
|
+
};
|
|
870
|
+
/**
|
|
871
|
+
* Gracefully terminates all worker processes.
|
|
872
|
+
* Monitors workers until they all exit or timeout occurs.
|
|
873
|
+
* Only relevant for the primary process.
|
|
874
|
+
*
|
|
875
|
+
* @param signal - The signal to send to worker processes
|
|
876
|
+
* @returns A promise that resolves when all workers have terminated
|
|
877
|
+
*/
|
|
878
|
+
shutdownWorkers = (signal) => {
|
|
879
|
+
return new Promise((resolve2, reject) => {
|
|
880
|
+
if (!cluster.isPrimary) {
|
|
881
|
+
return resolve2();
|
|
882
|
+
}
|
|
883
|
+
if (!cluster.workers) {
|
|
884
|
+
return resolve2();
|
|
885
|
+
}
|
|
886
|
+
const workerIds = Object.keys(cluster.workers);
|
|
887
|
+
if (workerIds.length == 0) {
|
|
888
|
+
return resolve2();
|
|
889
|
+
}
|
|
890
|
+
let workersAlive = 0;
|
|
891
|
+
let funcRun = 0;
|
|
892
|
+
const cleanWorkers = () => {
|
|
893
|
+
++funcRun;
|
|
894
|
+
workersAlive = 0;
|
|
895
|
+
Object.values(cluster.workers || {}).filter((worker) => !!worker).forEach((worker) => {
|
|
896
|
+
if (worker && !worker.isDead()) {
|
|
897
|
+
++workersAlive;
|
|
898
|
+
if (funcRun == 1) {
|
|
899
|
+
worker.kill(signal);
|
|
781
900
|
}
|
|
782
901
|
}
|
|
902
|
+
});
|
|
903
|
+
console.info(workersAlive + " workers alive");
|
|
904
|
+
if (workersAlive == 0) {
|
|
905
|
+
clearInterval(interval);
|
|
906
|
+
return resolve2();
|
|
783
907
|
}
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
908
|
+
};
|
|
909
|
+
const interval = setInterval(cleanWorkers, SHUTDOWN_WORKERS_INTERVAL);
|
|
910
|
+
});
|
|
911
|
+
};
|
|
912
|
+
};
|
|
913
|
+
}
|
|
914
|
+
});
|
|
915
|
+
|
|
916
|
+
// src/config/configFile.ts
|
|
917
|
+
import path from "path";
|
|
918
|
+
import * as toml from "toml";
|
|
919
|
+
async function findConfigFile(startDir = process.cwd()) {
|
|
920
|
+
const fs4 = await import("fs");
|
|
921
|
+
let currentDir = path.resolve(startDir);
|
|
922
|
+
while (true) {
|
|
923
|
+
const configPath = path.join(currentDir, "moose.config.toml");
|
|
924
|
+
if (fs4.existsSync(configPath)) {
|
|
925
|
+
return configPath;
|
|
926
|
+
}
|
|
927
|
+
const parentDir = path.dirname(currentDir);
|
|
928
|
+
if (parentDir === currentDir) {
|
|
929
|
+
break;
|
|
930
|
+
}
|
|
931
|
+
currentDir = parentDir;
|
|
932
|
+
}
|
|
933
|
+
return null;
|
|
934
|
+
}
|
|
935
|
+
async function readProjectConfig() {
|
|
936
|
+
const fs4 = await import("fs");
|
|
937
|
+
const configPath = await findConfigFile();
|
|
938
|
+
if (!configPath) {
|
|
939
|
+
throw new ConfigError(
|
|
940
|
+
"moose.config.toml not found in current directory or any parent directory"
|
|
941
|
+
);
|
|
942
|
+
}
|
|
943
|
+
try {
|
|
944
|
+
const configContent = fs4.readFileSync(configPath, "utf-8");
|
|
945
|
+
const config = toml.parse(configContent);
|
|
946
|
+
return config;
|
|
947
|
+
} catch (error) {
|
|
948
|
+
throw new ConfigError(`Failed to parse moose.config.toml: ${error}`);
|
|
949
|
+
}
|
|
950
|
+
}
|
|
951
|
+
var ConfigError;
|
|
952
|
+
var init_configFile = __esm({
|
|
953
|
+
"src/config/configFile.ts"() {
|
|
954
|
+
"use strict";
|
|
955
|
+
ConfigError = class extends Error {
|
|
956
|
+
constructor(message) {
|
|
957
|
+
super(message);
|
|
958
|
+
this.name = "ConfigError";
|
|
959
|
+
}
|
|
960
|
+
};
|
|
961
|
+
}
|
|
962
|
+
});
|
|
963
|
+
|
|
964
|
+
// src/config/runtime.ts
|
|
965
|
+
var runtime_exports = {};
|
|
966
|
+
var ConfigurationRegistry;
|
|
967
|
+
var init_runtime = __esm({
|
|
968
|
+
"src/config/runtime.ts"() {
|
|
969
|
+
"use strict";
|
|
970
|
+
init_configFile();
|
|
971
|
+
ConfigurationRegistry = class _ConfigurationRegistry {
|
|
972
|
+
static instance;
|
|
973
|
+
clickhouseConfig;
|
|
974
|
+
kafkaConfig;
|
|
975
|
+
static getInstance() {
|
|
976
|
+
if (!_ConfigurationRegistry.instance) {
|
|
977
|
+
_ConfigurationRegistry.instance = new _ConfigurationRegistry();
|
|
805
978
|
}
|
|
979
|
+
return _ConfigurationRegistry.instance;
|
|
806
980
|
}
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
}
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
981
|
+
setClickHouseConfig(config) {
|
|
982
|
+
this.clickhouseConfig = config;
|
|
983
|
+
}
|
|
984
|
+
setKafkaConfig(config) {
|
|
985
|
+
this.kafkaConfig = config;
|
|
986
|
+
}
|
|
987
|
+
_env(name) {
|
|
988
|
+
const value = process.env[name];
|
|
989
|
+
if (value === void 0) return void 0;
|
|
990
|
+
const trimmed = value.trim();
|
|
991
|
+
return trimmed.length > 0 ? trimmed : void 0;
|
|
992
|
+
}
|
|
993
|
+
_parseBool(value) {
|
|
994
|
+
if (value === void 0) return void 0;
|
|
995
|
+
switch (value.trim().toLowerCase()) {
|
|
996
|
+
case "1":
|
|
997
|
+
case "true":
|
|
998
|
+
case "yes":
|
|
999
|
+
case "on":
|
|
1000
|
+
return true;
|
|
1001
|
+
case "0":
|
|
1002
|
+
case "false":
|
|
1003
|
+
case "no":
|
|
1004
|
+
case "off":
|
|
1005
|
+
return false;
|
|
1006
|
+
default:
|
|
1007
|
+
return void 0;
|
|
827
1008
|
}
|
|
828
1009
|
}
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
1010
|
+
async getClickHouseConfig() {
|
|
1011
|
+
if (this.clickhouseConfig) {
|
|
1012
|
+
return this.clickhouseConfig;
|
|
1013
|
+
}
|
|
1014
|
+
const projectConfig = await readProjectConfig();
|
|
1015
|
+
const envHost = this._env("MOOSE_CLICKHOUSE_CONFIG__HOST");
|
|
1016
|
+
const envPort = this._env("MOOSE_CLICKHOUSE_CONFIG__HOST_PORT");
|
|
1017
|
+
const envUser = this._env("MOOSE_CLICKHOUSE_CONFIG__USER");
|
|
1018
|
+
const envPassword = this._env("MOOSE_CLICKHOUSE_CONFIG__PASSWORD");
|
|
1019
|
+
const envDb = this._env("MOOSE_CLICKHOUSE_CONFIG__DB_NAME");
|
|
1020
|
+
const envUseSSL = this._parseBool(
|
|
1021
|
+
this._env("MOOSE_CLICKHOUSE_CONFIG__USE_SSL")
|
|
1022
|
+
);
|
|
1023
|
+
return {
|
|
1024
|
+
host: envHost ?? projectConfig.clickhouse_config.host,
|
|
1025
|
+
port: envPort ?? projectConfig.clickhouse_config.host_port.toString(),
|
|
1026
|
+
username: envUser ?? projectConfig.clickhouse_config.user,
|
|
1027
|
+
password: envPassword ?? projectConfig.clickhouse_config.password,
|
|
1028
|
+
database: envDb ?? projectConfig.clickhouse_config.db_name,
|
|
1029
|
+
useSSL: envUseSSL !== void 0 ? envUseSSL : projectConfig.clickhouse_config.use_ssl || false
|
|
1030
|
+
};
|
|
835
1031
|
}
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
1032
|
+
async getStandaloneClickhouseConfig(overrides) {
|
|
1033
|
+
if (this.clickhouseConfig) {
|
|
1034
|
+
return { ...this.clickhouseConfig, ...overrides };
|
|
1035
|
+
}
|
|
1036
|
+
const envHost = this._env("MOOSE_CLICKHOUSE_CONFIG__HOST");
|
|
1037
|
+
const envPort = this._env("MOOSE_CLICKHOUSE_CONFIG__HOST_PORT");
|
|
1038
|
+
const envUser = this._env("MOOSE_CLICKHOUSE_CONFIG__USER");
|
|
1039
|
+
const envPassword = this._env("MOOSE_CLICKHOUSE_CONFIG__PASSWORD");
|
|
1040
|
+
const envDb = this._env("MOOSE_CLICKHOUSE_CONFIG__DB_NAME");
|
|
1041
|
+
const envUseSSL = this._parseBool(
|
|
1042
|
+
this._env("MOOSE_CLICKHOUSE_CONFIG__USE_SSL")
|
|
1043
|
+
);
|
|
1044
|
+
let projectConfig;
|
|
1045
|
+
try {
|
|
1046
|
+
projectConfig = await readProjectConfig();
|
|
1047
|
+
} catch (error) {
|
|
1048
|
+
projectConfig = null;
|
|
1049
|
+
}
|
|
1050
|
+
const defaults = {
|
|
1051
|
+
host: "localhost",
|
|
1052
|
+
port: "18123",
|
|
1053
|
+
username: "default",
|
|
1054
|
+
password: "",
|
|
1055
|
+
database: "local",
|
|
1056
|
+
useSSL: false
|
|
1057
|
+
};
|
|
1058
|
+
return {
|
|
1059
|
+
host: overrides?.host ?? envHost ?? projectConfig?.clickhouse_config.host ?? defaults.host,
|
|
1060
|
+
port: overrides?.port ?? envPort ?? projectConfig?.clickhouse_config.host_port.toString() ?? defaults.port,
|
|
1061
|
+
username: overrides?.username ?? envUser ?? projectConfig?.clickhouse_config.user ?? defaults.username,
|
|
1062
|
+
password: overrides?.password ?? envPassword ?? projectConfig?.clickhouse_config.password ?? defaults.password,
|
|
1063
|
+
database: overrides?.database ?? envDb ?? projectConfig?.clickhouse_config.db_name ?? defaults.database,
|
|
1064
|
+
useSSL: overrides?.useSSL ?? envUseSSL ?? projectConfig?.clickhouse_config.use_ssl ?? defaults.useSSL
|
|
1065
|
+
};
|
|
843
1066
|
}
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
1067
|
+
async getKafkaConfig() {
|
|
1068
|
+
if (this.kafkaConfig) {
|
|
1069
|
+
return this.kafkaConfig;
|
|
1070
|
+
}
|
|
1071
|
+
const projectConfig = await readProjectConfig();
|
|
1072
|
+
const envBroker = this._env("MOOSE_REDPANDA_CONFIG__BROKER") ?? this._env("MOOSE_KAFKA_CONFIG__BROKER");
|
|
1073
|
+
const envMsgTimeout = this._env("MOOSE_REDPANDA_CONFIG__MESSAGE_TIMEOUT_MS") ?? this._env("MOOSE_KAFKA_CONFIG__MESSAGE_TIMEOUT_MS");
|
|
1074
|
+
const envSaslUsername = this._env("MOOSE_REDPANDA_CONFIG__SASL_USERNAME") ?? this._env("MOOSE_KAFKA_CONFIG__SASL_USERNAME");
|
|
1075
|
+
const envSaslPassword = this._env("MOOSE_REDPANDA_CONFIG__SASL_PASSWORD") ?? this._env("MOOSE_KAFKA_CONFIG__SASL_PASSWORD");
|
|
1076
|
+
const envSaslMechanism = this._env("MOOSE_REDPANDA_CONFIG__SASL_MECHANISM") ?? this._env("MOOSE_KAFKA_CONFIG__SASL_MECHANISM");
|
|
1077
|
+
const envSecurityProtocol = this._env("MOOSE_REDPANDA_CONFIG__SECURITY_PROTOCOL") ?? this._env("MOOSE_KAFKA_CONFIG__SECURITY_PROTOCOL");
|
|
1078
|
+
const envNamespace = this._env("MOOSE_REDPANDA_CONFIG__NAMESPACE") ?? this._env("MOOSE_KAFKA_CONFIG__NAMESPACE");
|
|
1079
|
+
const envSchemaRegistryUrl = this._env("MOOSE_REDPANDA_CONFIG__SCHEMA_REGISTRY_URL") ?? this._env("MOOSE_KAFKA_CONFIG__SCHEMA_REGISTRY_URL");
|
|
1080
|
+
const fileKafka = projectConfig.kafka_config ?? projectConfig.redpanda_config;
|
|
1081
|
+
return {
|
|
1082
|
+
broker: envBroker ?? fileKafka?.broker ?? "localhost:19092",
|
|
1083
|
+
messageTimeoutMs: envMsgTimeout ? parseInt(envMsgTimeout, 10) : fileKafka?.message_timeout_ms ?? 1e3,
|
|
1084
|
+
saslUsername: envSaslUsername ?? fileKafka?.sasl_username,
|
|
1085
|
+
saslPassword: envSaslPassword ?? fileKafka?.sasl_password,
|
|
1086
|
+
saslMechanism: envSaslMechanism ?? fileKafka?.sasl_mechanism,
|
|
1087
|
+
securityProtocol: envSecurityProtocol ?? fileKafka?.security_protocol,
|
|
1088
|
+
namespace: envNamespace ?? fileKafka?.namespace,
|
|
1089
|
+
schemaRegistryUrl: envSchemaRegistryUrl ?? fileKafka?.schema_registry_url
|
|
1090
|
+
};
|
|
1091
|
+
}
|
|
1092
|
+
hasRuntimeConfig() {
|
|
1093
|
+
return !!this.clickhouseConfig || !!this.kafkaConfig;
|
|
852
1094
|
}
|
|
1095
|
+
};
|
|
1096
|
+
globalThis._mooseConfigRegistry = ConfigurationRegistry.getInstance();
|
|
1097
|
+
}
|
|
1098
|
+
});
|
|
1099
|
+
|
|
1100
|
+
// src/consumption-apis/standalone.ts
|
|
1101
|
+
var standalone_exports = {};
|
|
1102
|
+
__export(standalone_exports, {
|
|
1103
|
+
getMooseClients: () => getMooseClients,
|
|
1104
|
+
getMooseUtils: () => getMooseUtils
|
|
1105
|
+
});
|
|
1106
|
+
async function getMooseUtils(req) {
|
|
1107
|
+
if (req !== void 0) {
|
|
1108
|
+
console.warn(
|
|
1109
|
+
"[DEPRECATED] getMooseUtils(req) no longer requires a request parameter. Use getMooseUtils() instead."
|
|
1110
|
+
);
|
|
1111
|
+
}
|
|
1112
|
+
const runtimeContext = globalThis._mooseRuntimeContext;
|
|
1113
|
+
if (runtimeContext) {
|
|
1114
|
+
return {
|
|
1115
|
+
client: runtimeContext.client,
|
|
1116
|
+
sql,
|
|
1117
|
+
jwt: runtimeContext.jwt
|
|
1118
|
+
};
|
|
1119
|
+
}
|
|
1120
|
+
if (standaloneUtils) {
|
|
1121
|
+
return standaloneUtils;
|
|
1122
|
+
}
|
|
1123
|
+
if (initPromise) {
|
|
1124
|
+
return initPromise;
|
|
1125
|
+
}
|
|
1126
|
+
initPromise = (async () => {
|
|
1127
|
+
await Promise.resolve().then(() => (init_runtime(), runtime_exports));
|
|
1128
|
+
const configRegistry = globalThis._mooseConfigRegistry;
|
|
1129
|
+
if (!configRegistry) {
|
|
1130
|
+
throw new Error(
|
|
1131
|
+
"Moose not initialized. Ensure you're running within a Moose app or have proper configuration set up."
|
|
1132
|
+
);
|
|
853
1133
|
}
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
1134
|
+
const clickhouseConfig = await configRegistry.getStandaloneClickhouseConfig();
|
|
1135
|
+
const clickhouseClient = getClickhouseClient(
|
|
1136
|
+
toClientConfig(clickhouseConfig)
|
|
1137
|
+
);
|
|
1138
|
+
const queryClient = new QueryClient(clickhouseClient, "standalone");
|
|
1139
|
+
const mooseClient = new MooseClient(queryClient);
|
|
1140
|
+
standaloneUtils = {
|
|
1141
|
+
client: mooseClient,
|
|
1142
|
+
sql,
|
|
1143
|
+
jwt: void 0
|
|
1144
|
+
};
|
|
1145
|
+
return standaloneUtils;
|
|
1146
|
+
})();
|
|
1147
|
+
try {
|
|
1148
|
+
return await initPromise;
|
|
1149
|
+
} finally {
|
|
1150
|
+
initPromise = null;
|
|
1151
|
+
}
|
|
1152
|
+
}
|
|
1153
|
+
async function getMooseClients(config) {
|
|
1154
|
+
console.warn(
|
|
1155
|
+
"[DEPRECATED] getMooseClients() is deprecated. Use getMooseUtils() instead."
|
|
865
1156
|
);
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
const
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
return async (req, res) => {
|
|
873
|
-
const start = Date.now();
|
|
874
|
-
const url = new URL(req.url || "", "http://localhost");
|
|
875
|
-
const pathname = url.pathname;
|
|
876
|
-
if (pathname === "/_moose_internal/health") {
|
|
877
|
-
res.writeHead(200, { "Content-Type": "application/json" });
|
|
878
|
-
res.end(
|
|
879
|
-
JSON.stringify({
|
|
880
|
-
status: "healthy",
|
|
881
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
882
|
-
})
|
|
1157
|
+
if (config && Object.keys(config).length > 0) {
|
|
1158
|
+
await Promise.resolve().then(() => (init_runtime(), runtime_exports));
|
|
1159
|
+
const configRegistry = globalThis._mooseConfigRegistry;
|
|
1160
|
+
if (!configRegistry) {
|
|
1161
|
+
throw new Error(
|
|
1162
|
+
"Configuration registry not initialized. Ensure the Moose framework is properly set up."
|
|
883
1163
|
);
|
|
884
|
-
return;
|
|
885
1164
|
}
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
1165
|
+
const clickhouseConfig = await configRegistry.getStandaloneClickhouseConfig(config);
|
|
1166
|
+
const clickhouseClient = getClickhouseClient(
|
|
1167
|
+
toClientConfig(clickhouseConfig)
|
|
1168
|
+
);
|
|
1169
|
+
const queryClient = new QueryClient(clickhouseClient, "standalone");
|
|
1170
|
+
const mooseClient = new MooseClient(queryClient);
|
|
1171
|
+
return { client: mooseClient };
|
|
1172
|
+
}
|
|
1173
|
+
const utils = await getMooseUtils();
|
|
1174
|
+
return { client: utils.client };
|
|
1175
|
+
}
|
|
1176
|
+
var standaloneUtils, initPromise, toClientConfig;
|
|
1177
|
+
var init_standalone = __esm({
|
|
1178
|
+
"src/consumption-apis/standalone.ts"() {
|
|
1179
|
+
"use strict";
|
|
1180
|
+
init_helpers2();
|
|
1181
|
+
init_commons();
|
|
1182
|
+
init_sqlHelpers();
|
|
1183
|
+
standaloneUtils = null;
|
|
1184
|
+
initPromise = null;
|
|
1185
|
+
toClientConfig = (config) => ({
|
|
1186
|
+
...config,
|
|
1187
|
+
useSSL: config.useSSL ? "true" : "false"
|
|
1188
|
+
});
|
|
1189
|
+
}
|
|
1190
|
+
});
|
|
1191
|
+
|
|
1192
|
+
// src/consumption-apis/runner.ts
|
|
1193
|
+
import http2 from "http";
|
|
1194
|
+
import * as jose from "jose";
|
|
1195
|
+
var toClientConfig2, createPath, httpLogger, modulesCache, apiHandler, createMainRouter, runApis;
|
|
1196
|
+
var init_runner = __esm({
|
|
1197
|
+
"src/consumption-apis/runner.ts"() {
|
|
1198
|
+
"use strict";
|
|
1199
|
+
init_commons();
|
|
1200
|
+
init_helpers2();
|
|
1201
|
+
init_cluster_utils();
|
|
1202
|
+
init_sqlHelpers();
|
|
1203
|
+
init_internal();
|
|
1204
|
+
toClientConfig2 = (config) => ({
|
|
1205
|
+
...config,
|
|
1206
|
+
useSSL: config.useSSL ? "true" : "false"
|
|
1207
|
+
});
|
|
1208
|
+
createPath = (apisDir, path4) => `${apisDir}${path4}.ts`;
|
|
1209
|
+
httpLogger = (req, res, startMs) => {
|
|
1210
|
+
console.log(
|
|
1211
|
+
`${req.method} ${req.url} ${res.statusCode} ${Date.now() - startMs}ms`
|
|
1212
|
+
);
|
|
1213
|
+
};
|
|
1214
|
+
modulesCache = /* @__PURE__ */ new Map();
|
|
1215
|
+
apiHandler = async (publicKey, clickhouseClient, temporalClient, apisDir, enforceAuth, isDmv2, jwtConfig) => {
|
|
1216
|
+
const apis = isDmv2 ? await getApis2() : /* @__PURE__ */ new Map();
|
|
1217
|
+
return async (req, res) => {
|
|
1218
|
+
const start = Date.now();
|
|
890
1219
|
try {
|
|
891
|
-
const
|
|
892
|
-
|
|
893
|
-
|
|
1220
|
+
const url = new URL(req.url || "", "http://localhost");
|
|
1221
|
+
const fileName = url.pathname;
|
|
1222
|
+
let jwtPayload;
|
|
1223
|
+
if (publicKey && jwtConfig) {
|
|
1224
|
+
const jwt = req.headers.authorization?.split(" ")[1];
|
|
1225
|
+
if (jwt) {
|
|
1226
|
+
try {
|
|
1227
|
+
const { payload } = await jose.jwtVerify(jwt, publicKey, {
|
|
1228
|
+
issuer: jwtConfig.issuer,
|
|
1229
|
+
audience: jwtConfig.audience
|
|
1230
|
+
});
|
|
1231
|
+
jwtPayload = payload;
|
|
1232
|
+
} catch (error) {
|
|
1233
|
+
console.log("JWT verification failed");
|
|
1234
|
+
if (enforceAuth) {
|
|
1235
|
+
res.writeHead(401, { "Content-Type": "application/json" });
|
|
1236
|
+
res.end(JSON.stringify({ error: "Unauthorized" }));
|
|
1237
|
+
httpLogger(req, res, start);
|
|
1238
|
+
return;
|
|
1239
|
+
}
|
|
1240
|
+
}
|
|
1241
|
+
} else if (enforceAuth) {
|
|
1242
|
+
res.writeHead(401, { "Content-Type": "application/json" });
|
|
1243
|
+
res.end(JSON.stringify({ error: "Unauthorized" }));
|
|
1244
|
+
httpLogger(req, res, start);
|
|
1245
|
+
return;
|
|
1246
|
+
}
|
|
1247
|
+
} else if (enforceAuth) {
|
|
1248
|
+
res.writeHead(401, { "Content-Type": "application/json" });
|
|
1249
|
+
res.end(JSON.stringify({ error: "Unauthorized" }));
|
|
1250
|
+
httpLogger(req, res, start);
|
|
1251
|
+
return;
|
|
1252
|
+
}
|
|
1253
|
+
const pathName = createPath(apisDir, fileName);
|
|
1254
|
+
const paramsObject = Array.from(url.searchParams.entries()).reduce(
|
|
1255
|
+
(obj, [key, value]) => {
|
|
1256
|
+
const existingValue = obj[key];
|
|
1257
|
+
if (existingValue) {
|
|
1258
|
+
if (Array.isArray(existingValue)) {
|
|
1259
|
+
existingValue.push(value);
|
|
1260
|
+
} else {
|
|
1261
|
+
obj[key] = [existingValue, value];
|
|
1262
|
+
}
|
|
1263
|
+
} else {
|
|
1264
|
+
obj[key] = value;
|
|
1265
|
+
}
|
|
1266
|
+
return obj;
|
|
1267
|
+
},
|
|
1268
|
+
{}
|
|
1269
|
+
);
|
|
1270
|
+
let userFuncModule = modulesCache.get(pathName);
|
|
1271
|
+
if (userFuncModule === void 0) {
|
|
1272
|
+
if (isDmv2) {
|
|
1273
|
+
let apiName = fileName.replace(/^\/+|\/+$/g, "");
|
|
1274
|
+
let version = null;
|
|
1275
|
+
userFuncModule = apis.get(apiName);
|
|
1276
|
+
if (!userFuncModule) {
|
|
1277
|
+
version = url.searchParams.get("version");
|
|
1278
|
+
if (!version && apiName.includes("/")) {
|
|
1279
|
+
const pathParts = apiName.split("/");
|
|
1280
|
+
if (pathParts.length >= 2) {
|
|
1281
|
+
userFuncModule = apis.get(apiName);
|
|
1282
|
+
if (!userFuncModule) {
|
|
1283
|
+
apiName = pathParts[0];
|
|
1284
|
+
version = pathParts.slice(1).join("/");
|
|
1285
|
+
}
|
|
1286
|
+
}
|
|
1287
|
+
}
|
|
1288
|
+
if (!userFuncModule) {
|
|
1289
|
+
if (version) {
|
|
1290
|
+
const versionedKey = `${apiName}:${version}`;
|
|
1291
|
+
userFuncModule = apis.get(versionedKey);
|
|
1292
|
+
} else {
|
|
1293
|
+
userFuncModule = apis.get(apiName);
|
|
1294
|
+
}
|
|
1295
|
+
}
|
|
1296
|
+
}
|
|
1297
|
+
if (!userFuncModule) {
|
|
1298
|
+
const availableApis = Array.from(apis.keys()).map(
|
|
1299
|
+
(key) => key.replace(":", "/")
|
|
1300
|
+
);
|
|
1301
|
+
const errorMessage = version ? `API ${apiName} with version ${version} not found. Available APIs: ${availableApis.join(", ")}` : `API ${apiName} not found. Available APIs: ${availableApis.join(", ")}`;
|
|
1302
|
+
throw new Error(errorMessage);
|
|
1303
|
+
}
|
|
1304
|
+
modulesCache.set(pathName, userFuncModule);
|
|
1305
|
+
console.log(`[API] | Executing API: ${apiName}`);
|
|
1306
|
+
} else {
|
|
1307
|
+
userFuncModule = __require(pathName);
|
|
1308
|
+
modulesCache.set(pathName, userFuncModule);
|
|
1309
|
+
}
|
|
1310
|
+
}
|
|
1311
|
+
const queryClient = new QueryClient(clickhouseClient, fileName);
|
|
1312
|
+
let result = isDmv2 ? await userFuncModule(paramsObject, {
|
|
1313
|
+
client: new MooseClient(queryClient, temporalClient),
|
|
1314
|
+
sql,
|
|
1315
|
+
jwt: jwtPayload
|
|
1316
|
+
}) : await userFuncModule.default(paramsObject, {
|
|
1317
|
+
client: new MooseClient(queryClient, temporalClient),
|
|
1318
|
+
sql,
|
|
1319
|
+
jwt: jwtPayload
|
|
894
1320
|
});
|
|
895
|
-
|
|
1321
|
+
let body;
|
|
1322
|
+
let status;
|
|
1323
|
+
if (Object.getPrototypeOf(result).constructor.name === "ResultSet") {
|
|
1324
|
+
body = JSON.stringify(await result.json());
|
|
1325
|
+
} else {
|
|
1326
|
+
if ("body" in result && "status" in result) {
|
|
1327
|
+
body = JSON.stringify(result.body);
|
|
1328
|
+
status = result.status;
|
|
1329
|
+
} else {
|
|
1330
|
+
body = JSON.stringify(result);
|
|
1331
|
+
}
|
|
1332
|
+
}
|
|
1333
|
+
if (status) {
|
|
1334
|
+
res.writeHead(status, { "Content-Type": "application/json" });
|
|
1335
|
+
httpLogger(req, res, start);
|
|
1336
|
+
} else {
|
|
1337
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
1338
|
+
httpLogger(req, res, start);
|
|
1339
|
+
}
|
|
1340
|
+
res.end(body);
|
|
896
1341
|
} catch (error) {
|
|
897
|
-
console.log("
|
|
1342
|
+
console.log("error in path ", req.url, error);
|
|
1343
|
+
if (Object.getPrototypeOf(error).constructor.name === "TypeGuardError") {
|
|
1344
|
+
res.writeHead(400, { "Content-Type": "application/json" });
|
|
1345
|
+
res.end(JSON.stringify({ error: error.message }));
|
|
1346
|
+
httpLogger(req, res, start);
|
|
1347
|
+
}
|
|
1348
|
+
if (error instanceof Error) {
|
|
1349
|
+
res.writeHead(500, { "Content-Type": "application/json" });
|
|
1350
|
+
res.end(JSON.stringify({ error: error.message }));
|
|
1351
|
+
httpLogger(req, res, start);
|
|
1352
|
+
} else {
|
|
1353
|
+
res.writeHead(500, { "Content-Type": "application/json" });
|
|
1354
|
+
res.end();
|
|
1355
|
+
httpLogger(req, res, start);
|
|
1356
|
+
}
|
|
898
1357
|
}
|
|
899
|
-
}
|
|
900
|
-
}
|
|
901
|
-
|
|
902
|
-
const
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
1358
|
+
};
|
|
1359
|
+
};
|
|
1360
|
+
createMainRouter = async (publicKey, clickhouseClient, temporalClient, apisDir, enforceAuth, isDmv2, jwtConfig) => {
|
|
1361
|
+
const apiRequestHandler = await apiHandler(
|
|
1362
|
+
publicKey,
|
|
1363
|
+
clickhouseClient,
|
|
1364
|
+
temporalClient,
|
|
1365
|
+
apisDir,
|
|
1366
|
+
enforceAuth,
|
|
1367
|
+
isDmv2,
|
|
1368
|
+
jwtConfig
|
|
1369
|
+
);
|
|
1370
|
+
const webApps = isDmv2 ? await getWebApps2() : /* @__PURE__ */ new Map();
|
|
1371
|
+
const sortedWebApps = Array.from(webApps.values()).sort((a, b) => {
|
|
1372
|
+
const pathA = a.config.mountPath || "/";
|
|
1373
|
+
const pathB = b.config.mountPath || "/";
|
|
1374
|
+
return pathB.length - pathA.length;
|
|
1375
|
+
});
|
|
1376
|
+
return async (req, res) => {
|
|
1377
|
+
const start = Date.now();
|
|
1378
|
+
const url = new URL(req.url || "", "http://localhost");
|
|
1379
|
+
const pathname = url.pathname;
|
|
1380
|
+
if (pathname === "/_moose_internal/health") {
|
|
1381
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
1382
|
+
res.end(
|
|
1383
|
+
JSON.stringify({
|
|
1384
|
+
status: "healthy",
|
|
1385
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
1386
|
+
})
|
|
1387
|
+
);
|
|
1388
|
+
return;
|
|
1389
|
+
}
|
|
1390
|
+
let jwtPayload;
|
|
1391
|
+
if (publicKey && jwtConfig) {
|
|
1392
|
+
const jwt = req.headers.authorization?.split(" ")[1];
|
|
1393
|
+
if (jwt) {
|
|
1394
|
+
try {
|
|
1395
|
+
const { payload } = await jose.jwtVerify(jwt, publicKey, {
|
|
1396
|
+
issuer: jwtConfig.issuer,
|
|
1397
|
+
audience: jwtConfig.audience
|
|
1398
|
+
});
|
|
1399
|
+
jwtPayload = payload;
|
|
1400
|
+
} catch (error) {
|
|
1401
|
+
console.log("JWT verification failed for WebApp route");
|
|
1402
|
+
}
|
|
1403
|
+
}
|
|
1404
|
+
}
|
|
1405
|
+
for (const webApp of sortedWebApps) {
|
|
1406
|
+
const mountPath = webApp.config.mountPath || "/";
|
|
1407
|
+
const normalizedMount = mountPath.endsWith("/") && mountPath !== "/" ? mountPath.slice(0, -1) : mountPath;
|
|
1408
|
+
const matches = pathname === normalizedMount || pathname.startsWith(normalizedMount + "/");
|
|
1409
|
+
if (matches) {
|
|
1410
|
+
if (webApp.config.injectMooseUtils !== false) {
|
|
1411
|
+
const { getMooseUtils: getMooseUtils2 } = await Promise.resolve().then(() => (init_standalone(), standalone_exports));
|
|
1412
|
+
req.moose = await getMooseUtils2();
|
|
1413
|
+
}
|
|
1414
|
+
let proxiedUrl = req.url;
|
|
1415
|
+
if (normalizedMount !== "/") {
|
|
1416
|
+
const pathWithoutMount = pathname.substring(normalizedMount.length) || "/";
|
|
1417
|
+
proxiedUrl = pathWithoutMount + url.search;
|
|
1418
|
+
}
|
|
1419
|
+
try {
|
|
1420
|
+
const modifiedReq = Object.assign(
|
|
1421
|
+
Object.create(Object.getPrototypeOf(req)),
|
|
1422
|
+
req,
|
|
1423
|
+
{
|
|
1424
|
+
url: proxiedUrl
|
|
1425
|
+
}
|
|
1426
|
+
);
|
|
1427
|
+
await webApp.handler(modifiedReq, res);
|
|
1428
|
+
return;
|
|
1429
|
+
} catch (error) {
|
|
1430
|
+
console.error(`Error in WebApp ${webApp.name}:`, error);
|
|
1431
|
+
if (!res.headersSent) {
|
|
1432
|
+
res.writeHead(500, { "Content-Type": "application/json" });
|
|
1433
|
+
res.end(JSON.stringify({ error: "Internal Server Error" }));
|
|
1434
|
+
}
|
|
1435
|
+
return;
|
|
1436
|
+
}
|
|
1437
|
+
}
|
|
913
1438
|
}
|
|
914
|
-
let
|
|
915
|
-
if (
|
|
916
|
-
|
|
917
|
-
|
|
1439
|
+
let apiPath = pathname;
|
|
1440
|
+
if (pathname.startsWith("/api/")) {
|
|
1441
|
+
apiPath = pathname.substring(4);
|
|
1442
|
+
} else if (pathname.startsWith("/consumption/")) {
|
|
1443
|
+
apiPath = pathname.substring(13);
|
|
918
1444
|
}
|
|
919
|
-
|
|
1445
|
+
if (apiPath !== pathname) {
|
|
920
1446
|
const modifiedReq = Object.assign(
|
|
921
1447
|
Object.create(Object.getPrototypeOf(req)),
|
|
922
1448
|
req,
|
|
923
1449
|
{
|
|
924
|
-
url:
|
|
1450
|
+
url: apiPath + url.search
|
|
925
1451
|
}
|
|
926
1452
|
);
|
|
927
|
-
await
|
|
928
|
-
return;
|
|
929
|
-
} catch (error) {
|
|
930
|
-
console.error(`Error in WebApp ${webApp.name}:`, error);
|
|
931
|
-
if (!res.headersSent) {
|
|
932
|
-
res.writeHead(500, { "Content-Type": "application/json" });
|
|
933
|
-
res.end(JSON.stringify({ error: "Internal Server Error" }));
|
|
934
|
-
}
|
|
1453
|
+
await apiRequestHandler(modifiedReq, res);
|
|
935
1454
|
return;
|
|
936
1455
|
}
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
1456
|
+
res.writeHead(404, { "Content-Type": "application/json" });
|
|
1457
|
+
res.end(JSON.stringify({ error: "Not Found" }));
|
|
1458
|
+
httpLogger(req, res, start);
|
|
1459
|
+
};
|
|
1460
|
+
};
|
|
1461
|
+
runApis = async (config) => {
|
|
1462
|
+
const apisCluster = new Cluster({
|
|
1463
|
+
maxWorkerCount: (config.workerCount ?? 0) > 0 ? config.workerCount : void 0,
|
|
1464
|
+
workerStart: async () => {
|
|
1465
|
+
let temporalClient;
|
|
1466
|
+
if (config.temporalConfig) {
|
|
1467
|
+
temporalClient = await getTemporalClient(
|
|
1468
|
+
config.temporalConfig.url,
|
|
1469
|
+
config.temporalConfig.namespace,
|
|
1470
|
+
config.temporalConfig.clientCert,
|
|
1471
|
+
config.temporalConfig.clientKey,
|
|
1472
|
+
config.temporalConfig.apiKey
|
|
1473
|
+
);
|
|
1474
|
+
}
|
|
1475
|
+
const clickhouseClient = getClickhouseClient(
|
|
1476
|
+
toClientConfig2(config.clickhouseConfig)
|
|
1477
|
+
);
|
|
1478
|
+
let publicKey;
|
|
1479
|
+
if (config.jwtConfig?.secret) {
|
|
1480
|
+
console.log("Importing JWT public key...");
|
|
1481
|
+
publicKey = await jose.importSPKI(config.jwtConfig.secret, "RS256");
|
|
1482
|
+
}
|
|
1483
|
+
const runtimeQueryClient = new QueryClient(clickhouseClient, "runtime");
|
|
1484
|
+
globalThis._mooseRuntimeContext = {
|
|
1485
|
+
client: new MooseClient(runtimeQueryClient, temporalClient)
|
|
1486
|
+
};
|
|
1487
|
+
const server = http2.createServer(
|
|
1488
|
+
await createMainRouter(
|
|
1489
|
+
publicKey,
|
|
1490
|
+
clickhouseClient,
|
|
1491
|
+
temporalClient,
|
|
1492
|
+
config.apisDir,
|
|
1493
|
+
config.enforceAuth,
|
|
1494
|
+
config.isDmv2,
|
|
1495
|
+
config.jwtConfig
|
|
1496
|
+
)
|
|
1497
|
+
);
|
|
1498
|
+
const port = config.proxyPort !== void 0 ? config.proxyPort : 4001;
|
|
1499
|
+
server.listen(port, "localhost", () => {
|
|
1500
|
+
console.log(`Server running on port ${port}`);
|
|
1501
|
+
});
|
|
1502
|
+
return server;
|
|
1503
|
+
},
|
|
1504
|
+
workerStop: async (server) => {
|
|
1505
|
+
return new Promise((resolve2) => {
|
|
1506
|
+
server.close(() => resolve2());
|
|
1507
|
+
});
|
|
951
1508
|
}
|
|
952
|
-
);
|
|
953
|
-
await apiRequestHandler(modifiedReq, res);
|
|
954
|
-
return;
|
|
955
|
-
}
|
|
956
|
-
res.writeHead(404, { "Content-Type": "application/json" });
|
|
957
|
-
res.end(JSON.stringify({ error: "Not Found" }));
|
|
958
|
-
httpLogger(req, res, start);
|
|
959
|
-
};
|
|
960
|
-
};
|
|
961
|
-
var runApis = async (config) => {
|
|
962
|
-
const apisCluster = new Cluster({
|
|
963
|
-
maxWorkerCount: (config.workerCount ?? 0) > 0 ? config.workerCount : void 0,
|
|
964
|
-
workerStart: async () => {
|
|
965
|
-
let temporalClient;
|
|
966
|
-
if (config.temporalConfig) {
|
|
967
|
-
temporalClient = await getTemporalClient(
|
|
968
|
-
config.temporalConfig.url,
|
|
969
|
-
config.temporalConfig.namespace,
|
|
970
|
-
config.temporalConfig.clientCert,
|
|
971
|
-
config.temporalConfig.clientKey,
|
|
972
|
-
config.temporalConfig.apiKey
|
|
973
|
-
);
|
|
974
|
-
}
|
|
975
|
-
const clickhouseClient = getClickhouseClient(
|
|
976
|
-
toClientConfig(config.clickhouseConfig)
|
|
977
|
-
);
|
|
978
|
-
let publicKey;
|
|
979
|
-
if (config.jwtConfig?.secret) {
|
|
980
|
-
console.log("Importing JWT public key...");
|
|
981
|
-
publicKey = await jose.importSPKI(config.jwtConfig.secret, "RS256");
|
|
982
|
-
}
|
|
983
|
-
const server = http2.createServer(
|
|
984
|
-
await createMainRouter(
|
|
985
|
-
publicKey,
|
|
986
|
-
clickhouseClient,
|
|
987
|
-
temporalClient,
|
|
988
|
-
config.apisDir,
|
|
989
|
-
config.enforceAuth,
|
|
990
|
-
config.isDmv2,
|
|
991
|
-
config.jwtConfig
|
|
992
|
-
)
|
|
993
|
-
);
|
|
994
|
-
const port = config.proxyPort !== void 0 ? config.proxyPort : 4001;
|
|
995
|
-
server.listen(port, "localhost", () => {
|
|
996
|
-
console.log(`Server running on port ${port}`);
|
|
997
|
-
});
|
|
998
|
-
return server;
|
|
999
|
-
},
|
|
1000
|
-
workerStop: async (server) => {
|
|
1001
|
-
return new Promise((resolve2) => {
|
|
1002
|
-
server.close(() => resolve2());
|
|
1003
1509
|
});
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
};
|
|
1510
|
+
apisCluster.start();
|
|
1511
|
+
};
|
|
1512
|
+
}
|
|
1513
|
+
});
|
|
1008
1514
|
|
|
1009
1515
|
// src/clients/redisClient.ts
|
|
1010
1516
|
import { createClient as createClient2 } from "redis";
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
import { parse } from "csv-parse";
|
|
1517
|
+
var init_redisClient = __esm({
|
|
1518
|
+
"src/clients/redisClient.ts"() {
|
|
1519
|
+
"use strict";
|
|
1520
|
+
}
|
|
1521
|
+
});
|
|
1017
1522
|
|
|
1018
1523
|
// src/utilities/json.ts
|
|
1019
|
-
var STRING_DATE_ANNOTATION = "stringDate";
|
|
1020
1524
|
function isNullableType(dt) {
|
|
1021
1525
|
return typeof dt === "object" && dt !== null && "nullable" in dt && typeof dt.nullable !== "undefined";
|
|
1022
1526
|
}
|
|
@@ -1135,36 +1639,78 @@ function mutateParsedJson(data, fieldMutations) {
|
|
|
1135
1639
|
}
|
|
1136
1640
|
applyFieldMutations(data, fieldMutations);
|
|
1137
1641
|
}
|
|
1642
|
+
var STRING_DATE_ANNOTATION;
|
|
1643
|
+
var init_json = __esm({
|
|
1644
|
+
"src/utilities/json.ts"() {
|
|
1645
|
+
"use strict";
|
|
1646
|
+
STRING_DATE_ANNOTATION = "stringDate";
|
|
1647
|
+
}
|
|
1648
|
+
});
|
|
1138
1649
|
|
|
1139
1650
|
// src/utilities/dataParser.ts
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
};
|
|
1651
|
+
import { parse as parse2 } from "csv-parse";
|
|
1652
|
+
var CSV_DELIMITERS, DEFAULT_CSV_CONFIG;
|
|
1653
|
+
var init_dataParser = __esm({
|
|
1654
|
+
"src/utilities/dataParser.ts"() {
|
|
1655
|
+
"use strict";
|
|
1656
|
+
init_json();
|
|
1657
|
+
CSV_DELIMITERS = {
|
|
1658
|
+
COMMA: ",",
|
|
1659
|
+
TAB: " ",
|
|
1660
|
+
SEMICOLON: ";",
|
|
1661
|
+
PIPE: "|"
|
|
1662
|
+
};
|
|
1663
|
+
DEFAULT_CSV_CONFIG = {
|
|
1664
|
+
delimiter: CSV_DELIMITERS.COMMA,
|
|
1665
|
+
columns: true,
|
|
1666
|
+
skipEmptyLines: true,
|
|
1667
|
+
trim: true
|
|
1668
|
+
};
|
|
1669
|
+
}
|
|
1670
|
+
});
|
|
1671
|
+
|
|
1672
|
+
// src/utilities/index.ts
|
|
1673
|
+
var init_utilities = __esm({
|
|
1674
|
+
"src/utilities/index.ts"() {
|
|
1675
|
+
"use strict";
|
|
1676
|
+
init_dataParser();
|
|
1677
|
+
}
|
|
1678
|
+
});
|
|
1679
|
+
|
|
1680
|
+
// src/connectors/dataSource.ts
|
|
1681
|
+
var init_dataSource = __esm({
|
|
1682
|
+
"src/connectors/dataSource.ts"() {
|
|
1683
|
+
"use strict";
|
|
1684
|
+
}
|
|
1685
|
+
});
|
|
1686
|
+
|
|
1687
|
+
// src/index.ts
|
|
1688
|
+
var init_index = __esm({
|
|
1689
|
+
"src/index.ts"() {
|
|
1690
|
+
"use strict";
|
|
1691
|
+
init_browserCompatible();
|
|
1692
|
+
init_helpers();
|
|
1693
|
+
init_commons();
|
|
1694
|
+
init_secrets();
|
|
1695
|
+
init_helpers2();
|
|
1696
|
+
init_webAppHelpers();
|
|
1697
|
+
init_task();
|
|
1698
|
+
init_runner();
|
|
1699
|
+
init_redisClient();
|
|
1700
|
+
init_helpers2();
|
|
1701
|
+
init_standalone();
|
|
1702
|
+
init_sqlHelpers();
|
|
1703
|
+
init_utilities();
|
|
1704
|
+
init_dataSource();
|
|
1705
|
+
init_types();
|
|
1706
|
+
}
|
|
1707
|
+
});
|
|
1152
1708
|
|
|
1153
1709
|
// src/dmv2/internal.ts
|
|
1154
|
-
|
|
1710
|
+
import process2 from "process";
|
|
1155
1711
|
function getSourceDir() {
|
|
1156
1712
|
return process2.env.MOOSE_SOURCE_DIR || "app";
|
|
1157
1713
|
}
|
|
1158
|
-
var moose_internal = {
|
|
1159
|
-
tables: /* @__PURE__ */ new Map(),
|
|
1160
|
-
streams: /* @__PURE__ */ new Map(),
|
|
1161
|
-
ingestApis: /* @__PURE__ */ new Map(),
|
|
1162
|
-
apis: /* @__PURE__ */ new Map(),
|
|
1163
|
-
sqlResources: /* @__PURE__ */ new Map(),
|
|
1164
|
-
workflows: /* @__PURE__ */ new Map(),
|
|
1165
|
-
webApps: /* @__PURE__ */ new Map()
|
|
1166
|
-
};
|
|
1167
|
-
var defaultRetentionPeriod = 60 * 60 * 24 * 7;
|
|
1168
1714
|
function isS3QueueConfig(config) {
|
|
1169
1715
|
return "engine" in config && config.engine === "S3Queue" /* S3Queue */;
|
|
1170
1716
|
}
|
|
@@ -1401,364 +1947,386 @@ function convertTableConfigToEngineConfig(config) {
|
|
|
1401
1947
|
}
|
|
1402
1948
|
return void 0;
|
|
1403
1949
|
}
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
const id = table.config.version ? `${table.name}_${table.config.version}` : table.name;
|
|
1414
|
-
let metadata = table.metadata;
|
|
1415
|
-
if (!metadata && table.config && table.pipelineParent) {
|
|
1416
|
-
metadata = table.pipelineParent.metadata;
|
|
1417
|
-
}
|
|
1418
|
-
const engineConfig = convertTableConfigToEngineConfig(table.config);
|
|
1419
|
-
let tableSettings = void 0;
|
|
1420
|
-
if (table.config.settings) {
|
|
1421
|
-
tableSettings = Object.entries(table.config.settings).reduce(
|
|
1422
|
-
(acc, [key, value]) => {
|
|
1423
|
-
if (value !== void 0) {
|
|
1424
|
-
acc[key] = String(value);
|
|
1425
|
-
}
|
|
1426
|
-
return acc;
|
|
1427
|
-
},
|
|
1428
|
-
{}
|
|
1429
|
-
);
|
|
1430
|
-
}
|
|
1431
|
-
if (engineConfig?.engine === "S3Queue") {
|
|
1432
|
-
if (!tableSettings) {
|
|
1433
|
-
tableSettings = {};
|
|
1434
|
-
}
|
|
1435
|
-
if (!tableSettings.mode) {
|
|
1436
|
-
tableSettings.mode = "unordered";
|
|
1950
|
+
function findTaskInTree(task, targetName) {
|
|
1951
|
+
if (task.name === targetName) {
|
|
1952
|
+
return task;
|
|
1953
|
+
}
|
|
1954
|
+
if (task.config.onComplete?.length) {
|
|
1955
|
+
for (const childTask of task.config.onComplete) {
|
|
1956
|
+
const found = findTaskInTree(childTask, targetName);
|
|
1957
|
+
if (found) {
|
|
1958
|
+
return found;
|
|
1437
1959
|
}
|
|
1438
1960
|
}
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
metadata,
|
|
1457
|
-
lifeCycle: table.config.lifeCycle,
|
|
1458
|
-
// Map 'settings' to 'tableSettings' for internal use
|
|
1459
|
-
tableSettings: tableSettings && Object.keys(tableSettings).length > 0 ? tableSettings : void 0,
|
|
1460
|
-
indexes: table.config.indexes?.map((i) => ({
|
|
1461
|
-
...i,
|
|
1462
|
-
granularity: i.granularity === void 0 ? 1 : i.granularity,
|
|
1463
|
-
arguments: i.arguments === void 0 ? [] : i.arguments
|
|
1464
|
-
})) || [],
|
|
1465
|
-
ttl: table.config.ttl,
|
|
1466
|
-
database: table.config.database,
|
|
1467
|
-
cluster: table.config.cluster
|
|
1961
|
+
}
|
|
1962
|
+
return void 0;
|
|
1963
|
+
}
|
|
1964
|
+
var moose_internal, defaultRetentionPeriod, toInfraMap, getMooseInternal, dumpMooseInternal, loadIndex, getStreamingFunctions, getApis2, getWorkflows2, getTaskForWorkflow, getWebApps2;
|
|
1965
|
+
var init_internal = __esm({
|
|
1966
|
+
"src/dmv2/internal.ts"() {
|
|
1967
|
+
"use strict";
|
|
1968
|
+
init_index();
|
|
1969
|
+
init_commons();
|
|
1970
|
+
moose_internal = {
|
|
1971
|
+
tables: /* @__PURE__ */ new Map(),
|
|
1972
|
+
streams: /* @__PURE__ */ new Map(),
|
|
1973
|
+
ingestApis: /* @__PURE__ */ new Map(),
|
|
1974
|
+
apis: /* @__PURE__ */ new Map(),
|
|
1975
|
+
sqlResources: /* @__PURE__ */ new Map(),
|
|
1976
|
+
workflows: /* @__PURE__ */ new Map(),
|
|
1977
|
+
webApps: /* @__PURE__ */ new Map()
|
|
1468
1978
|
};
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1979
|
+
defaultRetentionPeriod = 60 * 60 * 24 * 7;
|
|
1980
|
+
toInfraMap = (registry) => {
|
|
1981
|
+
const tables = {};
|
|
1982
|
+
const topics = {};
|
|
1983
|
+
const ingestApis = {};
|
|
1984
|
+
const apis = {};
|
|
1985
|
+
const sqlResources = {};
|
|
1986
|
+
const workflows = {};
|
|
1987
|
+
const webApps = {};
|
|
1988
|
+
registry.tables.forEach((table) => {
|
|
1989
|
+
const id = table.config.version ? `${table.name}_${table.config.version}` : table.name;
|
|
1990
|
+
let metadata = table.metadata;
|
|
1991
|
+
if (!metadata && table.config && table.pipelineParent) {
|
|
1992
|
+
metadata = table.pipelineParent.metadata;
|
|
1993
|
+
}
|
|
1994
|
+
const engineConfig = convertTableConfigToEngineConfig(table.config);
|
|
1995
|
+
let tableSettings = void 0;
|
|
1996
|
+
if (table.config.settings) {
|
|
1997
|
+
tableSettings = Object.entries(table.config.settings).reduce(
|
|
1998
|
+
(acc, [key, value]) => {
|
|
1999
|
+
if (value !== void 0) {
|
|
2000
|
+
acc[key] = String(value);
|
|
2001
|
+
}
|
|
2002
|
+
return acc;
|
|
2003
|
+
},
|
|
2004
|
+
{}
|
|
2005
|
+
);
|
|
2006
|
+
}
|
|
2007
|
+
if (engineConfig?.engine === "S3Queue") {
|
|
2008
|
+
if (!tableSettings) {
|
|
2009
|
+
tableSettings = {};
|
|
2010
|
+
}
|
|
2011
|
+
if (!tableSettings.mode) {
|
|
2012
|
+
tableSettings.mode = "unordered";
|
|
2013
|
+
}
|
|
2014
|
+
}
|
|
2015
|
+
const hasOrderByFields = "orderByFields" in table.config && Array.isArray(table.config.orderByFields) && table.config.orderByFields.length > 0;
|
|
2016
|
+
const hasOrderByExpression = "orderByExpression" in table.config && typeof table.config.orderByExpression === "string" && table.config.orderByExpression.length > 0;
|
|
2017
|
+
if (hasOrderByFields && hasOrderByExpression) {
|
|
2018
|
+
throw new Error(
|
|
2019
|
+
`Table ${table.name}: Provide either orderByFields or orderByExpression, not both.`
|
|
2020
|
+
);
|
|
2021
|
+
}
|
|
2022
|
+
const orderBy = hasOrderByExpression && "orderByExpression" in table.config ? table.config.orderByExpression ?? "" : "orderByFields" in table.config ? table.config.orderByFields ?? [] : [];
|
|
2023
|
+
tables[id] = {
|
|
2024
|
+
name: table.name,
|
|
2025
|
+
columns: table.columnArray,
|
|
2026
|
+
orderBy,
|
|
2027
|
+
partitionBy: "partitionBy" in table.config ? table.config.partitionBy : void 0,
|
|
2028
|
+
sampleByExpression: "sampleByExpression" in table.config ? table.config.sampleByExpression : void 0,
|
|
2029
|
+
primaryKeyExpression: "primaryKeyExpression" in table.config ? table.config.primaryKeyExpression : void 0,
|
|
2030
|
+
engineConfig,
|
|
2031
|
+
version: table.config.version,
|
|
2032
|
+
metadata,
|
|
2033
|
+
lifeCycle: table.config.lifeCycle,
|
|
2034
|
+
// Map 'settings' to 'tableSettings' for internal use
|
|
2035
|
+
tableSettings: tableSettings && Object.keys(tableSettings).length > 0 ? tableSettings : void 0,
|
|
2036
|
+
indexes: table.config.indexes?.map((i) => ({
|
|
2037
|
+
...i,
|
|
2038
|
+
granularity: i.granularity === void 0 ? 1 : i.granularity,
|
|
2039
|
+
arguments: i.arguments === void 0 ? [] : i.arguments
|
|
2040
|
+
})) || [],
|
|
2041
|
+
ttl: table.config.ttl,
|
|
2042
|
+
database: table.config.database,
|
|
2043
|
+
cluster: table.config.cluster
|
|
2044
|
+
};
|
|
2045
|
+
});
|
|
2046
|
+
registry.streams.forEach((stream) => {
|
|
2047
|
+
let metadata = stream.metadata;
|
|
2048
|
+
if (!metadata && stream.config && stream.pipelineParent) {
|
|
2049
|
+
metadata = stream.pipelineParent.metadata;
|
|
2050
|
+
}
|
|
2051
|
+
const transformationTargets = [];
|
|
2052
|
+
const consumers = [];
|
|
2053
|
+
stream._transformations.forEach((transforms, destinationName) => {
|
|
2054
|
+
transforms.forEach(([destination, _, config]) => {
|
|
2055
|
+
transformationTargets.push({
|
|
2056
|
+
kind: "stream",
|
|
2057
|
+
name: destinationName,
|
|
2058
|
+
version: config.version,
|
|
2059
|
+
metadata: config.metadata,
|
|
2060
|
+
sourceFile: config.sourceFile
|
|
2061
|
+
});
|
|
2062
|
+
});
|
|
1485
2063
|
});
|
|
2064
|
+
stream._consumers.forEach((consumer) => {
|
|
2065
|
+
consumers.push({
|
|
2066
|
+
version: consumer.config.version,
|
|
2067
|
+
sourceFile: consumer.config.sourceFile
|
|
2068
|
+
});
|
|
2069
|
+
});
|
|
2070
|
+
topics[stream.name] = {
|
|
2071
|
+
name: stream.name,
|
|
2072
|
+
columns: stream.columnArray,
|
|
2073
|
+
targetTable: stream.config.destination?.name,
|
|
2074
|
+
targetTableVersion: stream.config.destination?.config.version,
|
|
2075
|
+
retentionPeriod: stream.config.retentionPeriod ?? defaultRetentionPeriod,
|
|
2076
|
+
partitionCount: stream.config.parallelism ?? 1,
|
|
2077
|
+
version: stream.config.version,
|
|
2078
|
+
transformationTargets,
|
|
2079
|
+
hasMultiTransform: stream._multipleTransformations === void 0,
|
|
2080
|
+
consumers,
|
|
2081
|
+
metadata,
|
|
2082
|
+
lifeCycle: stream.config.lifeCycle,
|
|
2083
|
+
schemaConfig: stream.config.schemaConfig
|
|
2084
|
+
};
|
|
1486
2085
|
});
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
2086
|
+
registry.ingestApis.forEach((api) => {
|
|
2087
|
+
let metadata = api.metadata;
|
|
2088
|
+
if (!metadata && api.config && api.pipelineParent) {
|
|
2089
|
+
metadata = api.pipelineParent.metadata;
|
|
2090
|
+
}
|
|
2091
|
+
ingestApis[api.name] = {
|
|
2092
|
+
name: api.name,
|
|
2093
|
+
columns: api.columnArray,
|
|
2094
|
+
version: api.config.version,
|
|
2095
|
+
path: api.config.path,
|
|
2096
|
+
writeTo: {
|
|
2097
|
+
kind: "stream",
|
|
2098
|
+
name: api.config.destination.name
|
|
2099
|
+
},
|
|
2100
|
+
deadLetterQueue: api.config.deadLetterQueue?.name,
|
|
2101
|
+
metadata,
|
|
2102
|
+
schema: api.schema,
|
|
2103
|
+
allowExtraFields: api.allowExtraFields
|
|
2104
|
+
};
|
|
1492
2105
|
});
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
2106
|
+
registry.apis.forEach((api, key) => {
|
|
2107
|
+
const rustKey = api.config.version ? `${api.name}:${api.config.version}` : api.name;
|
|
2108
|
+
apis[rustKey] = {
|
|
2109
|
+
name: api.name,
|
|
2110
|
+
queryParams: api.columnArray,
|
|
2111
|
+
responseSchema: api.responseSchema,
|
|
2112
|
+
version: api.config.version,
|
|
2113
|
+
path: api.config.path,
|
|
2114
|
+
metadata: api.metadata
|
|
2115
|
+
};
|
|
2116
|
+
});
|
|
2117
|
+
registry.sqlResources.forEach((sqlResource) => {
|
|
2118
|
+
sqlResources[sqlResource.name] = {
|
|
2119
|
+
name: sqlResource.name,
|
|
2120
|
+
setup: sqlResource.setup,
|
|
2121
|
+
teardown: sqlResource.teardown,
|
|
2122
|
+
sourceFile: sqlResource.sourceFile,
|
|
2123
|
+
sourceLine: sqlResource.sourceLine,
|
|
2124
|
+
sourceColumn: sqlResource.sourceColumn,
|
|
2125
|
+
pullsDataFrom: sqlResource.pullsDataFrom.map((r) => {
|
|
2126
|
+
if (r.kind === "OlapTable") {
|
|
2127
|
+
const table = r;
|
|
2128
|
+
const id = table.config.version ? `${table.name}_${table.config.version}` : table.name;
|
|
2129
|
+
return {
|
|
2130
|
+
id,
|
|
2131
|
+
kind: "Table"
|
|
2132
|
+
};
|
|
2133
|
+
} else if (r.kind === "SqlResource") {
|
|
2134
|
+
const resource = r;
|
|
2135
|
+
return {
|
|
2136
|
+
id: resource.name,
|
|
2137
|
+
kind: "SqlResource"
|
|
2138
|
+
};
|
|
2139
|
+
} else {
|
|
2140
|
+
throw new Error(`Unknown sql resource dependency type: ${r}`);
|
|
2141
|
+
}
|
|
2142
|
+
}),
|
|
2143
|
+
pushesDataTo: sqlResource.pushesDataTo.map((r) => {
|
|
2144
|
+
if (r.kind === "OlapTable") {
|
|
2145
|
+
const table = r;
|
|
2146
|
+
const id = table.config.version ? `${table.name}_${table.config.version}` : table.name;
|
|
2147
|
+
return {
|
|
2148
|
+
id,
|
|
2149
|
+
kind: "Table"
|
|
2150
|
+
};
|
|
2151
|
+
} else if (r.kind === "SqlResource") {
|
|
2152
|
+
const resource = r;
|
|
2153
|
+
return {
|
|
2154
|
+
id: resource.name,
|
|
2155
|
+
kind: "SqlResource"
|
|
2156
|
+
};
|
|
2157
|
+
} else {
|
|
2158
|
+
throw new Error(`Unknown sql resource dependency type: ${r}`);
|
|
2159
|
+
}
|
|
2160
|
+
})
|
|
2161
|
+
};
|
|
2162
|
+
});
|
|
2163
|
+
registry.workflows.forEach((workflow) => {
|
|
2164
|
+
workflows[workflow.name] = {
|
|
2165
|
+
name: workflow.name,
|
|
2166
|
+
retries: workflow.config.retries,
|
|
2167
|
+
timeout: workflow.config.timeout,
|
|
2168
|
+
schedule: workflow.config.schedule
|
|
2169
|
+
};
|
|
2170
|
+
});
|
|
2171
|
+
registry.webApps.forEach((webApp) => {
|
|
2172
|
+
webApps[webApp.name] = {
|
|
2173
|
+
name: webApp.name,
|
|
2174
|
+
mountPath: webApp.config.mountPath || "/",
|
|
2175
|
+
metadata: webApp.config.metadata
|
|
2176
|
+
};
|
|
2177
|
+
});
|
|
2178
|
+
return {
|
|
2179
|
+
topics,
|
|
2180
|
+
tables,
|
|
2181
|
+
ingestApis,
|
|
2182
|
+
apis,
|
|
2183
|
+
sqlResources,
|
|
2184
|
+
workflows,
|
|
2185
|
+
webApps
|
|
2186
|
+
};
|
|
1508
2187
|
};
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
if (!metadata && api.config && api.pipelineParent) {
|
|
1513
|
-
metadata = api.pipelineParent.metadata;
|
|
2188
|
+
getMooseInternal = () => globalThis.moose_internal;
|
|
2189
|
+
if (getMooseInternal() === void 0) {
|
|
2190
|
+
globalThis.moose_internal = moose_internal;
|
|
1514
2191
|
}
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
name: api.config.destination.name
|
|
1523
|
-
},
|
|
1524
|
-
deadLetterQueue: api.config.deadLetterQueue?.name,
|
|
1525
|
-
metadata,
|
|
1526
|
-
schema: api.schema,
|
|
1527
|
-
allowExtraFields: api.allowExtraFields
|
|
1528
|
-
};
|
|
1529
|
-
});
|
|
1530
|
-
registry.apis.forEach((api, key) => {
|
|
1531
|
-
const rustKey = api.config.version ? `${api.name}:${api.config.version}` : api.name;
|
|
1532
|
-
apis[rustKey] = {
|
|
1533
|
-
name: api.name,
|
|
1534
|
-
queryParams: api.columnArray,
|
|
1535
|
-
responseSchema: api.responseSchema,
|
|
1536
|
-
version: api.config.version,
|
|
1537
|
-
path: api.config.path,
|
|
1538
|
-
metadata: api.metadata
|
|
2192
|
+
dumpMooseInternal = async () => {
|
|
2193
|
+
loadIndex();
|
|
2194
|
+
console.log(
|
|
2195
|
+
"___MOOSE_STUFF___start",
|
|
2196
|
+
JSON.stringify(toInfraMap(getMooseInternal())),
|
|
2197
|
+
"end___MOOSE_STUFF___"
|
|
2198
|
+
);
|
|
1539
2199
|
};
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
return {
|
|
1554
|
-
id,
|
|
1555
|
-
kind: "Table"
|
|
1556
|
-
};
|
|
1557
|
-
} else if (r.kind === "SqlResource") {
|
|
1558
|
-
const resource = r;
|
|
1559
|
-
return {
|
|
1560
|
-
id: resource.name,
|
|
1561
|
-
kind: "SqlResource"
|
|
1562
|
-
};
|
|
1563
|
-
} else {
|
|
1564
|
-
throw new Error(`Unknown sql resource dependency type: ${r}`);
|
|
2200
|
+
loadIndex = () => {
|
|
2201
|
+
const registry = getMooseInternal();
|
|
2202
|
+
registry.tables.clear();
|
|
2203
|
+
registry.streams.clear();
|
|
2204
|
+
registry.ingestApis.clear();
|
|
2205
|
+
registry.apis.clear();
|
|
2206
|
+
registry.sqlResources.clear();
|
|
2207
|
+
registry.workflows.clear();
|
|
2208
|
+
registry.webApps.clear();
|
|
2209
|
+
const appDir = `${process2.cwd()}/${getSourceDir()}`;
|
|
2210
|
+
Object.keys(__require.cache).forEach((key) => {
|
|
2211
|
+
if (key.startsWith(appDir)) {
|
|
2212
|
+
delete __require.cache[key];
|
|
1565
2213
|
}
|
|
1566
|
-
})
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
|
-
};
|
|
1575
|
-
} else if (r.kind === "SqlResource") {
|
|
1576
|
-
const resource = r;
|
|
1577
|
-
return {
|
|
1578
|
-
id: resource.name,
|
|
1579
|
-
kind: "SqlResource"
|
|
1580
|
-
};
|
|
1581
|
-
} else {
|
|
1582
|
-
throw new Error(`Unknown sql resource dependency type: ${r}`);
|
|
2214
|
+
});
|
|
2215
|
+
try {
|
|
2216
|
+
__require(`${process2.cwd()}/${getSourceDir()}/index.ts`);
|
|
2217
|
+
} catch (error) {
|
|
2218
|
+
let hint;
|
|
2219
|
+
const details = error instanceof Error ? error.message : String(error);
|
|
2220
|
+
if (details.includes("ERR_REQUIRE_ESM") || details.includes("ES Module")) {
|
|
2221
|
+
hint = "The file or its dependencies are ESM-only. Switch to packages that dual-support CJS & ESM, or upgrade to Node 22.12+. If you must use Node 20, you may try Node 20.19\n\n";
|
|
1583
2222
|
}
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
workflows[workflow.name] = {
|
|
1589
|
-
name: workflow.name,
|
|
1590
|
-
retries: workflow.config.retries,
|
|
1591
|
-
timeout: workflow.config.timeout,
|
|
1592
|
-
schedule: workflow.config.schedule
|
|
2223
|
+
const errorMsg = `${hint ?? ""}${details}`;
|
|
2224
|
+
const cause = error instanceof Error ? error : void 0;
|
|
2225
|
+
throw new Error(errorMsg, { cause });
|
|
2226
|
+
}
|
|
1593
2227
|
};
|
|
1594
|
-
|
|
1595
|
-
|
|
1596
|
-
|
|
1597
|
-
|
|
1598
|
-
|
|
1599
|
-
|
|
2228
|
+
getStreamingFunctions = async () => {
|
|
2229
|
+
loadIndex();
|
|
2230
|
+
const registry = getMooseInternal();
|
|
2231
|
+
const transformFunctions = /* @__PURE__ */ new Map();
|
|
2232
|
+
registry.streams.forEach((stream) => {
|
|
2233
|
+
stream._transformations.forEach((transforms, destinationName) => {
|
|
2234
|
+
transforms.forEach(([_, transform, config]) => {
|
|
2235
|
+
const transformFunctionKey = `${stream.name}_${destinationName}${config.version ? `_${config.version}` : ""}`;
|
|
2236
|
+
compilerLog(`getStreamingFunctions: ${transformFunctionKey}`);
|
|
2237
|
+
transformFunctions.set(transformFunctionKey, [
|
|
2238
|
+
transform,
|
|
2239
|
+
config,
|
|
2240
|
+
stream.columnArray
|
|
2241
|
+
]);
|
|
2242
|
+
});
|
|
2243
|
+
});
|
|
2244
|
+
stream._consumers.forEach((consumer) => {
|
|
2245
|
+
const consumerFunctionKey = `${stream.name}_<no-target>${consumer.config.version ? `_${consumer.config.version}` : ""}`;
|
|
2246
|
+
transformFunctions.set(consumerFunctionKey, [
|
|
2247
|
+
consumer.consumer,
|
|
2248
|
+
consumer.config,
|
|
2249
|
+
stream.columnArray
|
|
2250
|
+
]);
|
|
2251
|
+
});
|
|
2252
|
+
});
|
|
2253
|
+
return transformFunctions;
|
|
1600
2254
|
};
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
1605
|
-
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
|
|
1611
|
-
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
}
|
|
1624
|
-
|
|
1625
|
-
const registry = getMooseInternal();
|
|
1626
|
-
registry.tables.clear();
|
|
1627
|
-
registry.streams.clear();
|
|
1628
|
-
registry.ingestApis.clear();
|
|
1629
|
-
registry.apis.clear();
|
|
1630
|
-
registry.sqlResources.clear();
|
|
1631
|
-
registry.workflows.clear();
|
|
1632
|
-
registry.webApps.clear();
|
|
1633
|
-
const appDir = `${process2.cwd()}/${getSourceDir()}`;
|
|
1634
|
-
Object.keys(__require.cache).forEach((key) => {
|
|
1635
|
-
if (key.startsWith(appDir)) {
|
|
1636
|
-
delete __require.cache[key];
|
|
1637
|
-
}
|
|
1638
|
-
});
|
|
1639
|
-
try {
|
|
1640
|
-
__require(`${process2.cwd()}/${getSourceDir()}/index.ts`);
|
|
1641
|
-
} catch (error) {
|
|
1642
|
-
let hint;
|
|
1643
|
-
const details = error instanceof Error ? error.message : String(error);
|
|
1644
|
-
if (details.includes("ERR_REQUIRE_ESM") || details.includes("ES Module")) {
|
|
1645
|
-
hint = "The file or its dependencies are ESM-only. Switch to packages that dual-support CJS & ESM, or upgrade to Node 22.12+. If you must use Node 20, you may try Node 20.19\n\n";
|
|
1646
|
-
}
|
|
1647
|
-
const errorMsg = `${hint ?? ""}${details}`;
|
|
1648
|
-
const cause = error instanceof Error ? error : void 0;
|
|
1649
|
-
throw new Error(errorMsg, { cause });
|
|
1650
|
-
}
|
|
1651
|
-
};
|
|
1652
|
-
var getStreamingFunctions = async () => {
|
|
1653
|
-
loadIndex();
|
|
1654
|
-
const registry = getMooseInternal();
|
|
1655
|
-
const transformFunctions = /* @__PURE__ */ new Map();
|
|
1656
|
-
registry.streams.forEach((stream) => {
|
|
1657
|
-
stream._transformations.forEach((transforms, destinationName) => {
|
|
1658
|
-
transforms.forEach(([_, transform, config]) => {
|
|
1659
|
-
const transformFunctionKey = `${stream.name}_${destinationName}${config.version ? `_${config.version}` : ""}`;
|
|
1660
|
-
compilerLog(`getStreamingFunctions: ${transformFunctionKey}`);
|
|
1661
|
-
transformFunctions.set(transformFunctionKey, [
|
|
1662
|
-
transform,
|
|
1663
|
-
config,
|
|
1664
|
-
stream.columnArray
|
|
1665
|
-
]);
|
|
2255
|
+
getApis2 = async () => {
|
|
2256
|
+
loadIndex();
|
|
2257
|
+
const apiFunctions = /* @__PURE__ */ new Map();
|
|
2258
|
+
const registry = getMooseInternal();
|
|
2259
|
+
const versionCountByName = /* @__PURE__ */ new Map();
|
|
2260
|
+
const nameToSoleVersionHandler = /* @__PURE__ */ new Map();
|
|
2261
|
+
registry.apis.forEach((api, key) => {
|
|
2262
|
+
const handler = api.getHandler();
|
|
2263
|
+
apiFunctions.set(key, handler);
|
|
2264
|
+
if (!api.config.version) {
|
|
2265
|
+
if (!apiFunctions.has(api.name)) {
|
|
2266
|
+
apiFunctions.set(api.name, handler);
|
|
2267
|
+
}
|
|
2268
|
+
nameToSoleVersionHandler.delete(api.name);
|
|
2269
|
+
versionCountByName.delete(api.name);
|
|
2270
|
+
} else if (!apiFunctions.has(api.name)) {
|
|
2271
|
+
const count = (versionCountByName.get(api.name) ?? 0) + 1;
|
|
2272
|
+
versionCountByName.set(api.name, count);
|
|
2273
|
+
if (count === 1) {
|
|
2274
|
+
nameToSoleVersionHandler.set(api.name, handler);
|
|
2275
|
+
} else {
|
|
2276
|
+
nameToSoleVersionHandler.delete(api.name);
|
|
2277
|
+
}
|
|
2278
|
+
}
|
|
1666
2279
|
});
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
};
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
const nameToSoleVersionHandler = /* @__PURE__ */ new Map();
|
|
1685
|
-
registry.apis.forEach((api, key) => {
|
|
1686
|
-
const handler = api.getHandler();
|
|
1687
|
-
apiFunctions.set(key, handler);
|
|
1688
|
-
if (!api.config.version) {
|
|
1689
|
-
if (!apiFunctions.has(api.name)) {
|
|
1690
|
-
apiFunctions.set(api.name, handler);
|
|
1691
|
-
}
|
|
1692
|
-
nameToSoleVersionHandler.delete(api.name);
|
|
1693
|
-
versionCountByName.delete(api.name);
|
|
1694
|
-
} else if (!apiFunctions.has(api.name)) {
|
|
1695
|
-
const count = (versionCountByName.get(api.name) ?? 0) + 1;
|
|
1696
|
-
versionCountByName.set(api.name, count);
|
|
1697
|
-
if (count === 1) {
|
|
1698
|
-
nameToSoleVersionHandler.set(api.name, handler);
|
|
1699
|
-
} else {
|
|
1700
|
-
nameToSoleVersionHandler.delete(api.name);
|
|
2280
|
+
nameToSoleVersionHandler.forEach((handler, name) => {
|
|
2281
|
+
if (!apiFunctions.has(name)) {
|
|
2282
|
+
apiFunctions.set(name, handler);
|
|
2283
|
+
}
|
|
2284
|
+
});
|
|
2285
|
+
return apiFunctions;
|
|
2286
|
+
};
|
|
2287
|
+
getWorkflows2 = async () => {
|
|
2288
|
+
loadIndex();
|
|
2289
|
+
const registry = getMooseInternal();
|
|
2290
|
+
return registry.workflows;
|
|
2291
|
+
};
|
|
2292
|
+
getTaskForWorkflow = async (workflowName, taskName) => {
|
|
2293
|
+
const workflows = await getWorkflows2();
|
|
2294
|
+
const workflow = workflows.get(workflowName);
|
|
2295
|
+
if (!workflow) {
|
|
2296
|
+
throw new Error(`Workflow ${workflowName} not found`);
|
|
1701
2297
|
}
|
|
1702
|
-
|
|
1703
|
-
|
|
1704
|
-
|
|
1705
|
-
|
|
1706
|
-
|
|
1707
|
-
|
|
1708
|
-
});
|
|
1709
|
-
return apiFunctions;
|
|
1710
|
-
};
|
|
1711
|
-
var getWorkflows2 = async () => {
|
|
1712
|
-
loadIndex();
|
|
1713
|
-
const registry = getMooseInternal();
|
|
1714
|
-
return registry.workflows;
|
|
1715
|
-
};
|
|
1716
|
-
function findTaskInTree(task, targetName) {
|
|
1717
|
-
if (task.name === targetName) {
|
|
1718
|
-
return task;
|
|
1719
|
-
}
|
|
1720
|
-
if (task.config.onComplete?.length) {
|
|
1721
|
-
for (const childTask of task.config.onComplete) {
|
|
1722
|
-
const found = findTaskInTree(childTask, targetName);
|
|
1723
|
-
if (found) {
|
|
1724
|
-
return found;
|
|
2298
|
+
const task = findTaskInTree(
|
|
2299
|
+
workflow.config.startingTask,
|
|
2300
|
+
taskName
|
|
2301
|
+
);
|
|
2302
|
+
if (!task) {
|
|
2303
|
+
throw new Error(`Task ${taskName} not found in workflow ${workflowName}`);
|
|
1725
2304
|
}
|
|
1726
|
-
|
|
1727
|
-
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
const workflow = workflows.get(workflowName);
|
|
1733
|
-
if (!workflow) {
|
|
1734
|
-
throw new Error(`Workflow ${workflowName} not found`);
|
|
1735
|
-
}
|
|
1736
|
-
const task = findTaskInTree(
|
|
1737
|
-
workflow.config.startingTask,
|
|
1738
|
-
taskName
|
|
1739
|
-
);
|
|
1740
|
-
if (!task) {
|
|
1741
|
-
throw new Error(`Task ${taskName} not found in workflow ${workflowName}`);
|
|
2305
|
+
return task;
|
|
2306
|
+
};
|
|
2307
|
+
getWebApps2 = async () => {
|
|
2308
|
+
loadIndex();
|
|
2309
|
+
return getMooseInternal().webApps;
|
|
2310
|
+
};
|
|
1742
2311
|
}
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
};
|
|
2312
|
+
});
|
|
2313
|
+
|
|
2314
|
+
// src/moose-runner.ts
|
|
2315
|
+
init_internal();
|
|
2316
|
+
import { register } from "ts-node";
|
|
1749
2317
|
|
|
1750
2318
|
// src/blocks/runner.ts
|
|
1751
2319
|
init_commons();
|
|
1752
2320
|
import fastq from "fastq";
|
|
1753
2321
|
import fs2 from "fs";
|
|
1754
|
-
import
|
|
2322
|
+
import path2 from "path";
|
|
1755
2323
|
var walkDir = (dir, fileExtension, fileList) => {
|
|
1756
2324
|
const files = fs2.readdirSync(dir);
|
|
1757
2325
|
files.forEach((file) => {
|
|
1758
|
-
if (fs2.statSync(
|
|
1759
|
-
fileList = walkDir(
|
|
2326
|
+
if (fs2.statSync(path2.join(dir, file)).isDirectory()) {
|
|
2327
|
+
fileList = walkDir(path2.join(dir, file), fileExtension, fileList);
|
|
1760
2328
|
} else if (file.endsWith(fileExtension)) {
|
|
1761
|
-
fileList.push(
|
|
2329
|
+
fileList.push(path2.join(dir, file));
|
|
1762
2330
|
}
|
|
1763
2331
|
});
|
|
1764
2332
|
return fileList;
|
|
@@ -1769,7 +2337,7 @@ var DependencyError = class extends Error {
|
|
|
1769
2337
|
this.name = "DependencyError";
|
|
1770
2338
|
}
|
|
1771
2339
|
};
|
|
1772
|
-
var
|
|
2340
|
+
var toClientConfig3 = (config) => ({
|
|
1773
2341
|
...config,
|
|
1774
2342
|
useSSL: config.useSSL ? "true" : "false"
|
|
1775
2343
|
});
|
|
@@ -1821,7 +2389,7 @@ var asyncWorker = async (task) => {
|
|
|
1821
2389
|
await createBlocks(task.chClient, task.blocks);
|
|
1822
2390
|
};
|
|
1823
2391
|
var runBlocks = async (config) => {
|
|
1824
|
-
const chClient = getClickhouseClient(
|
|
2392
|
+
const chClient = getClickhouseClient(toClientConfig3(config.clickhouseConfig));
|
|
1825
2393
|
console.log(`Connected`);
|
|
1826
2394
|
const blocksFiles = walkDir(config.blocksDir, ".ts", []);
|
|
1827
2395
|
const numOfBlockFiles = blocksFiles.length;
|
|
@@ -1834,10 +2402,10 @@ var runBlocks = async (config) => {
|
|
|
1834
2402
|
}
|
|
1835
2403
|
}
|
|
1836
2404
|
});
|
|
1837
|
-
for (const
|
|
1838
|
-
console.log(`Adding to queue: ${
|
|
2405
|
+
for (const path4 of blocksFiles) {
|
|
2406
|
+
console.log(`Adding to queue: ${path4}`);
|
|
1839
2407
|
try {
|
|
1840
|
-
const blocks = __require(
|
|
2408
|
+
const blocks = __require(path4).default;
|
|
1841
2409
|
queue.push({
|
|
1842
2410
|
chClient,
|
|
1843
2411
|
blocks,
|
|
@@ -1846,7 +2414,7 @@ var runBlocks = async (config) => {
|
|
|
1846
2414
|
} catch (err) {
|
|
1847
2415
|
cliLog({
|
|
1848
2416
|
action: "Blocks",
|
|
1849
|
-
message: `Failed to import blocks from ${
|
|
2417
|
+
message: `Failed to import blocks from ${path4}: ${err}`,
|
|
1850
2418
|
message_type: "Error"
|
|
1851
2419
|
});
|
|
1852
2420
|
}
|
|
@@ -1856,8 +2424,14 @@ var runBlocks = async (config) => {
|
|
|
1856
2424
|
}
|
|
1857
2425
|
};
|
|
1858
2426
|
|
|
2427
|
+
// src/moose-runner.ts
|
|
2428
|
+
init_runner();
|
|
2429
|
+
|
|
1859
2430
|
// src/streaming-functions/runner.ts
|
|
1860
2431
|
init_commons();
|
|
2432
|
+
init_cluster_utils();
|
|
2433
|
+
init_internal();
|
|
2434
|
+
init_json();
|
|
1861
2435
|
import { Readable as Readable2 } from "stream";
|
|
1862
2436
|
import { KafkaJS as KafkaJS2 } from "@514labs/kafka-javascript";
|
|
1863
2437
|
import { Buffer as Buffer2 } from "buffer";
|
|
@@ -2420,15 +2994,18 @@ async function runApiTypeSerializer(targetModel) {
|
|
|
2420
2994
|
}
|
|
2421
2995
|
|
|
2422
2996
|
// src/scripts/runner.ts
|
|
2997
|
+
init_internal();
|
|
2423
2998
|
import {
|
|
2424
2999
|
NativeConnection,
|
|
2425
3000
|
Worker,
|
|
2426
3001
|
bundleWorkflowCode
|
|
2427
3002
|
} from "@temporalio/worker";
|
|
2428
|
-
import * as
|
|
3003
|
+
import * as path3 from "path";
|
|
2429
3004
|
import * as fs3 from "fs";
|
|
2430
3005
|
|
|
2431
3006
|
// src/scripts/activity.ts
|
|
3007
|
+
init_internal();
|
|
3008
|
+
init_json();
|
|
2432
3009
|
import { log as logger, Context } from "@temporalio/activity";
|
|
2433
3010
|
import { isCancellation } from "@temporalio/workflow";
|
|
2434
3011
|
var activities = {
|
|
@@ -2719,7 +3296,7 @@ async function registerWorkflows(logger2, config) {
|
|
|
2719
3296
|
}
|
|
2720
3297
|
};
|
|
2721
3298
|
const workflowBundle = await bundleWorkflowCode({
|
|
2722
|
-
workflowsPath:
|
|
3299
|
+
workflowsPath: path3.resolve(__dirname, "scripts/workflow.js"),
|
|
2723
3300
|
logger: silentLogger
|
|
2724
3301
|
});
|
|
2725
3302
|
const worker = await Worker.create({
|