@514labs/moose-lib 0.6.295-ci-16-gad4ec11a → 0.6.295-ci-20-gbe187727

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,5 +1,4 @@
1
1
  #!/usr/bin/env node
2
- var __defProp = Object.defineProperty;
3
2
  var __getOwnPropNames = Object.getOwnPropertyNames;
4
3
  var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require : typeof Proxy !== "undefined" ? new Proxy(x, {
5
4
  get: (a, b) => (typeof require !== "undefined" ? require : a)[b]
@@ -10,346 +9,6 @@ var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require
10
9
  var __esm = (fn, res) => function __init() {
11
10
  return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
12
11
  };
13
- var __export = (target, all) => {
14
- for (var name in all)
15
- __defProp(target, name, { get: all[name], enumerable: true });
16
- };
17
-
18
- // src/dmv2/utils/stackTrace.ts
19
- var init_stackTrace = __esm({
20
- "src/dmv2/utils/stackTrace.ts"() {
21
- "use strict";
22
- }
23
- });
24
-
25
- // src/dmv2/typedBase.ts
26
- var init_typedBase = __esm({
27
- "src/dmv2/typedBase.ts"() {
28
- "use strict";
29
- init_stackTrace();
30
- }
31
- });
32
-
33
- // src/dataModels/dataModelTypes.ts
34
- var init_dataModelTypes = __esm({
35
- "src/dataModels/dataModelTypes.ts"() {
36
- "use strict";
37
- }
38
- });
39
-
40
- // src/sqlHelpers.ts
41
- function sql(strings, ...values) {
42
- return new Sql(strings, values);
43
- }
44
- function createClickhouseParameter(parameterIndex, value) {
45
- return `{p${parameterIndex}:${mapToClickHouseType(value)}}`;
46
- }
47
- function emptyIfUndefined(value) {
48
- return value === void 0 ? "" : value;
49
- }
50
- var isTable, isColumn, instanceofSql, Sql, toQuery, toQueryPreview, getValueFromParameter, mapToClickHouseType;
51
- var init_sqlHelpers = __esm({
52
- "src/sqlHelpers.ts"() {
53
- "use strict";
54
- isTable = (value) => typeof value === "object" && value !== null && "kind" in value && value.kind === "OlapTable";
55
- isColumn = (value) => typeof value === "object" && "name" in value && "annotations" in value;
56
- instanceofSql = (value) => typeof value === "object" && "values" in value && "strings" in value;
57
- Sql = class {
58
- values;
59
- strings;
60
- constructor(rawStrings, rawValues) {
61
- if (rawStrings.length - 1 !== rawValues.length) {
62
- if (rawStrings.length === 0) {
63
- throw new TypeError("Expected at least 1 string");
64
- }
65
- throw new TypeError(
66
- `Expected ${rawStrings.length} strings to have ${rawStrings.length - 1} values`
67
- );
68
- }
69
- const valuesLength = rawValues.reduce(
70
- (len, value) => len + (instanceofSql(value) ? value.values.length : isColumn(value) || isTable(value) ? 0 : 1),
71
- 0
72
- );
73
- this.values = new Array(valuesLength);
74
- this.strings = new Array(valuesLength + 1);
75
- this.strings[0] = rawStrings[0];
76
- let i = 0, pos = 0;
77
- while (i < rawValues.length) {
78
- const child = rawValues[i++];
79
- const rawString = rawStrings[i];
80
- if (instanceofSql(child)) {
81
- this.strings[pos] += child.strings[0];
82
- let childIndex = 0;
83
- while (childIndex < child.values.length) {
84
- this.values[pos++] = child.values[childIndex++];
85
- this.strings[pos] = child.strings[childIndex];
86
- }
87
- this.strings[pos] += rawString;
88
- } else if (isColumn(child)) {
89
- const aggregationFunction = child.annotations.find(
90
- ([k, _]) => k === "aggregationFunction"
91
- );
92
- if (aggregationFunction !== void 0) {
93
- this.strings[pos] += `${aggregationFunction[1].functionName}Merge(\`${child.name}\`)`;
94
- } else {
95
- this.strings[pos] += `\`${child.name}\``;
96
- }
97
- this.strings[pos] += rawString;
98
- } else if (isTable(child)) {
99
- if (child.config.database) {
100
- this.strings[pos] += `\`${child.config.database}\`.\`${child.name}\``;
101
- } else {
102
- this.strings[pos] += `\`${child.name}\``;
103
- }
104
- this.strings[pos] += rawString;
105
- } else {
106
- this.values[pos++] = child;
107
- this.strings[pos] = rawString;
108
- }
109
- }
110
- }
111
- };
112
- toQuery = (sql3) => {
113
- const parameterizedStubs = sql3.values.map(
114
- (v, i) => createClickhouseParameter(i, v)
115
- );
116
- const query = sql3.strings.map(
117
- (s, i) => s != "" ? `${s}${emptyIfUndefined(parameterizedStubs[i])}` : ""
118
- ).join("");
119
- const query_params = sql3.values.reduce(
120
- (acc, v, i) => ({
121
- ...acc,
122
- [`p${i}`]: getValueFromParameter(v)
123
- }),
124
- {}
125
- );
126
- return [query, query_params];
127
- };
128
- toQueryPreview = (sql3) => {
129
- try {
130
- const formatValue = (v) => {
131
- if (Array.isArray(v)) {
132
- const [type, val] = v;
133
- if (type === "Identifier") {
134
- return `\`${String(val)}\``;
135
- }
136
- return `[${v.map((x) => formatValue(x)).join(", ")}]`;
137
- }
138
- if (v === null || v === void 0) return "NULL";
139
- if (typeof v === "string") return `'${v.replace(/'/g, "''")}'`;
140
- if (typeof v === "number") return String(v);
141
- if (typeof v === "boolean") return v ? "true" : "false";
142
- if (v instanceof Date)
143
- return `'${v.toISOString().replace("T", " ").slice(0, 19)}'`;
144
- try {
145
- return JSON.stringify(v);
146
- } catch {
147
- return String(v);
148
- }
149
- };
150
- let out = sql3.strings[0] ?? "";
151
- for (let i = 0; i < sql3.values.length; i++) {
152
- const val = getValueFromParameter(sql3.values[i]);
153
- out += formatValue(val);
154
- out += sql3.strings[i + 1] ?? "";
155
- }
156
- return out.replace(/\s+/g, " ").trim();
157
- } catch (error) {
158
- console.log(`toQueryPreview error: ${error}`);
159
- return "/* query preview unavailable */";
160
- }
161
- };
162
- getValueFromParameter = (value) => {
163
- if (Array.isArray(value)) {
164
- const [type, val] = value;
165
- if (type === "Identifier") return val;
166
- }
167
- return value;
168
- };
169
- mapToClickHouseType = (value) => {
170
- if (typeof value === "number") {
171
- return Number.isInteger(value) ? "Int" : "Float";
172
- }
173
- if (typeof value === "boolean") return "Bool";
174
- if (value instanceof Date) return "DateTime";
175
- if (Array.isArray(value)) {
176
- const [type, _] = value;
177
- return type;
178
- }
179
- return "String";
180
- };
181
- }
182
- });
183
-
184
- // src/blocks/helpers.ts
185
- var init_helpers = __esm({
186
- "src/blocks/helpers.ts"() {
187
- "use strict";
188
- init_sqlHelpers();
189
- }
190
- });
191
-
192
- // src/dmv2/sdk/olapTable.ts
193
- import { Readable } from "stream";
194
- import { createHash } from "crypto";
195
- var init_olapTable = __esm({
196
- "src/dmv2/sdk/olapTable.ts"() {
197
- "use strict";
198
- init_typedBase();
199
- init_dataModelTypes();
200
- init_helpers();
201
- init_internal();
202
- init_sqlHelpers();
203
- }
204
- });
205
-
206
- // src/dmv2/sdk/stream.ts
207
- import { createHash as createHash2 } from "crypto";
208
- var init_stream = __esm({
209
- "src/dmv2/sdk/stream.ts"() {
210
- "use strict";
211
- init_typedBase();
212
- init_internal();
213
- init_stackTrace();
214
- }
215
- });
216
-
217
- // src/dmv2/sdk/workflow.ts
218
- var init_workflow = __esm({
219
- "src/dmv2/sdk/workflow.ts"() {
220
- "use strict";
221
- init_internal();
222
- }
223
- });
224
-
225
- // src/dmv2/sdk/ingestApi.ts
226
- var init_ingestApi = __esm({
227
- "src/dmv2/sdk/ingestApi.ts"() {
228
- "use strict";
229
- init_typedBase();
230
- init_internal();
231
- }
232
- });
233
-
234
- // src/dmv2/sdk/consumptionApi.ts
235
- var init_consumptionApi = __esm({
236
- "src/dmv2/sdk/consumptionApi.ts"() {
237
- "use strict";
238
- init_typedBase();
239
- init_internal();
240
- }
241
- });
242
-
243
- // src/dmv2/sdk/ingestPipeline.ts
244
- var init_ingestPipeline = __esm({
245
- "src/dmv2/sdk/ingestPipeline.ts"() {
246
- "use strict";
247
- init_typedBase();
248
- init_stream();
249
- init_olapTable();
250
- init_ingestApi();
251
- init_helpers();
252
- }
253
- });
254
-
255
- // src/dmv2/sdk/etlPipeline.ts
256
- var init_etlPipeline = __esm({
257
- "src/dmv2/sdk/etlPipeline.ts"() {
258
- "use strict";
259
- init_workflow();
260
- }
261
- });
262
-
263
- // src/dmv2/sdk/sqlResource.ts
264
- var init_sqlResource = __esm({
265
- "src/dmv2/sdk/sqlResource.ts"() {
266
- "use strict";
267
- init_internal();
268
- init_sqlHelpers();
269
- init_stackTrace();
270
- }
271
- });
272
-
273
- // src/dmv2/sdk/materializedView.ts
274
- var init_materializedView = __esm({
275
- "src/dmv2/sdk/materializedView.ts"() {
276
- "use strict";
277
- init_helpers();
278
- init_sqlHelpers();
279
- init_olapTable();
280
- init_sqlResource();
281
- }
282
- });
283
-
284
- // src/dmv2/sdk/view.ts
285
- var init_view = __esm({
286
- "src/dmv2/sdk/view.ts"() {
287
- "use strict";
288
- init_helpers();
289
- init_sqlHelpers();
290
- init_sqlResource();
291
- }
292
- });
293
-
294
- // src/dmv2/sdk/lifeCycle.ts
295
- var init_lifeCycle = __esm({
296
- "src/dmv2/sdk/lifeCycle.ts"() {
297
- "use strict";
298
- }
299
- });
300
-
301
- // src/dmv2/sdk/webApp.ts
302
- var init_webApp = __esm({
303
- "src/dmv2/sdk/webApp.ts"() {
304
- "use strict";
305
- init_internal();
306
- }
307
- });
308
-
309
- // src/dmv2/registry.ts
310
- var init_registry = __esm({
311
- "src/dmv2/registry.ts"() {
312
- "use strict";
313
- init_internal();
314
- }
315
- });
316
-
317
- // src/dmv2/index.ts
318
- var init_dmv2 = __esm({
319
- "src/dmv2/index.ts"() {
320
- "use strict";
321
- init_olapTable();
322
- init_stream();
323
- init_workflow();
324
- init_ingestApi();
325
- init_consumptionApi();
326
- init_ingestPipeline();
327
- init_etlPipeline();
328
- init_materializedView();
329
- init_sqlResource();
330
- init_view();
331
- init_lifeCycle();
332
- init_webApp();
333
- init_registry();
334
- }
335
- });
336
-
337
- // src/dataModels/types.ts
338
- var init_types = __esm({
339
- "src/dataModels/types.ts"() {
340
- "use strict";
341
- }
342
- });
343
-
344
- // src/browserCompatible.ts
345
- var init_browserCompatible = __esm({
346
- "src/browserCompatible.ts"() {
347
- "use strict";
348
- init_dmv2();
349
- init_types();
350
- init_sqlHelpers();
351
- }
352
- });
353
12
 
354
13
  // src/commons.ts
355
14
  import http from "http";
@@ -480,12 +139,159 @@ var init_commons = __esm({
480
139
  }
481
140
  });
482
141
 
483
- // src/secrets.ts
484
- var init_secrets = __esm({
485
- "src/secrets.ts"() {
486
- "use strict";
142
+ // src/moose-runner.ts
143
+ import { register } from "ts-node";
144
+
145
+ // src/dmv2/internal.ts
146
+ import process2 from "process";
147
+
148
+ // src/sqlHelpers.ts
149
+ var isTable = (value) => typeof value === "object" && value !== null && "kind" in value && value.kind === "OlapTable";
150
+ var isColumn = (value) => typeof value === "object" && "name" in value && "annotations" in value;
151
+ function sql(strings, ...values) {
152
+ return new Sql(strings, values);
153
+ }
154
+ var instanceofSql = (value) => typeof value === "object" && "values" in value && "strings" in value;
155
+ var Sql = class {
156
+ values;
157
+ strings;
158
+ constructor(rawStrings, rawValues) {
159
+ if (rawStrings.length - 1 !== rawValues.length) {
160
+ if (rawStrings.length === 0) {
161
+ throw new TypeError("Expected at least 1 string");
162
+ }
163
+ throw new TypeError(
164
+ `Expected ${rawStrings.length} strings to have ${rawStrings.length - 1} values`
165
+ );
166
+ }
167
+ const valuesLength = rawValues.reduce(
168
+ (len, value) => len + (instanceofSql(value) ? value.values.length : isColumn(value) || isTable(value) ? 0 : 1),
169
+ 0
170
+ );
171
+ this.values = new Array(valuesLength);
172
+ this.strings = new Array(valuesLength + 1);
173
+ this.strings[0] = rawStrings[0];
174
+ let i = 0, pos = 0;
175
+ while (i < rawValues.length) {
176
+ const child = rawValues[i++];
177
+ const rawString = rawStrings[i];
178
+ if (instanceofSql(child)) {
179
+ this.strings[pos] += child.strings[0];
180
+ let childIndex = 0;
181
+ while (childIndex < child.values.length) {
182
+ this.values[pos++] = child.values[childIndex++];
183
+ this.strings[pos] = child.strings[childIndex];
184
+ }
185
+ this.strings[pos] += rawString;
186
+ } else if (isColumn(child)) {
187
+ const aggregationFunction = child.annotations.find(
188
+ ([k, _]) => k === "aggregationFunction"
189
+ );
190
+ if (aggregationFunction !== void 0) {
191
+ this.strings[pos] += `${aggregationFunction[1].functionName}Merge(\`${child.name}\`)`;
192
+ } else {
193
+ this.strings[pos] += `\`${child.name}\``;
194
+ }
195
+ this.strings[pos] += rawString;
196
+ } else if (isTable(child)) {
197
+ if (child.config.database) {
198
+ this.strings[pos] += `\`${child.config.database}\`.\`${child.name}\``;
199
+ } else {
200
+ this.strings[pos] += `\`${child.name}\``;
201
+ }
202
+ this.strings[pos] += rawString;
203
+ } else {
204
+ this.values[pos++] = child;
205
+ this.strings[pos] = rawString;
206
+ }
207
+ }
487
208
  }
488
- });
209
+ };
210
+ var toQuery = (sql3) => {
211
+ const parameterizedStubs = sql3.values.map(
212
+ (v, i) => createClickhouseParameter(i, v)
213
+ );
214
+ const query = sql3.strings.map(
215
+ (s, i) => s != "" ? `${s}${emptyIfUndefined(parameterizedStubs[i])}` : ""
216
+ ).join("");
217
+ const query_params = sql3.values.reduce(
218
+ (acc, v, i) => ({
219
+ ...acc,
220
+ [`p${i}`]: getValueFromParameter(v)
221
+ }),
222
+ {}
223
+ );
224
+ return [query, query_params];
225
+ };
226
+ var toQueryPreview = (sql3) => {
227
+ try {
228
+ const formatValue = (v) => {
229
+ if (Array.isArray(v)) {
230
+ const [type, val] = v;
231
+ if (type === "Identifier") {
232
+ return `\`${String(val)}\``;
233
+ }
234
+ return `[${v.map((x) => formatValue(x)).join(", ")}]`;
235
+ }
236
+ if (v === null || v === void 0) return "NULL";
237
+ if (typeof v === "string") return `'${v.replace(/'/g, "''")}'`;
238
+ if (typeof v === "number") return String(v);
239
+ if (typeof v === "boolean") return v ? "true" : "false";
240
+ if (v instanceof Date)
241
+ return `'${v.toISOString().replace("T", " ").slice(0, 19)}'`;
242
+ try {
243
+ return JSON.stringify(v);
244
+ } catch {
245
+ return String(v);
246
+ }
247
+ };
248
+ let out = sql3.strings[0] ?? "";
249
+ for (let i = 0; i < sql3.values.length; i++) {
250
+ const val = getValueFromParameter(sql3.values[i]);
251
+ out += formatValue(val);
252
+ out += sql3.strings[i + 1] ?? "";
253
+ }
254
+ return out.replace(/\s+/g, " ").trim();
255
+ } catch (error) {
256
+ console.log(`toQueryPreview error: ${error}`);
257
+ return "/* query preview unavailable */";
258
+ }
259
+ };
260
+ var getValueFromParameter = (value) => {
261
+ if (Array.isArray(value)) {
262
+ const [type, val] = value;
263
+ if (type === "Identifier") return val;
264
+ }
265
+ return value;
266
+ };
267
+ function createClickhouseParameter(parameterIndex, value) {
268
+ return `{p${parameterIndex}:${mapToClickHouseType(value)}}`;
269
+ }
270
+ var mapToClickHouseType = (value) => {
271
+ if (typeof value === "number") {
272
+ return Number.isInteger(value) ? "Int" : "Float";
273
+ }
274
+ if (typeof value === "boolean") return "Bool";
275
+ if (value instanceof Date) return "DateTime";
276
+ if (Array.isArray(value)) {
277
+ const [type, _] = value;
278
+ return type;
279
+ }
280
+ return "String";
281
+ };
282
+ function emptyIfUndefined(value) {
283
+ return value === void 0 ? "" : value;
284
+ }
285
+
286
+ // src/dmv2/sdk/olapTable.ts
287
+ import { Readable } from "stream";
288
+ import { createHash } from "crypto";
289
+
290
+ // src/dmv2/sdk/stream.ts
291
+ import { createHash as createHash2 } from "crypto";
292
+
293
+ // src/index.ts
294
+ init_commons();
489
295
 
490
296
  // src/consumption-apis/helpers.ts
491
297
  import {
@@ -507,6 +313,142 @@ function formatElapsedTime(ms) {
507
313
  const remainingSeconds = seconds % 60;
508
314
  return `${minutes} minutes and ${remainingSeconds.toFixed(2)} seconds`;
509
315
  }
316
+ var MooseClient = class {
317
+ query;
318
+ workflow;
319
+ constructor(queryClient, temporalClient) {
320
+ this.query = queryClient;
321
+ this.workflow = new WorkflowClient(temporalClient);
322
+ }
323
+ };
324
+ var QueryClient = class {
325
+ client;
326
+ query_id_prefix;
327
+ constructor(client, query_id_prefix) {
328
+ this.client = client;
329
+ this.query_id_prefix = query_id_prefix;
330
+ }
331
+ async execute(sql3) {
332
+ const [query, query_params] = toQuery(sql3);
333
+ console.log(`[QueryClient] | Query: ${toQueryPreview(sql3)}`);
334
+ const start = performance.now();
335
+ const result = await this.client.query({
336
+ query,
337
+ query_params,
338
+ format: "JSONEachRow",
339
+ query_id: this.query_id_prefix + randomUUID()
340
+ // Note: wait_end_of_query deliberately NOT set here as this is used for SELECT queries
341
+ // where response buffering would harm streaming performance and concurrency
342
+ });
343
+ const elapsedMs = performance.now() - start;
344
+ console.log(
345
+ `[QueryClient] | Query completed: ${formatElapsedTime(elapsedMs)}`
346
+ );
347
+ return result;
348
+ }
349
+ async command(sql3) {
350
+ const [query, query_params] = toQuery(sql3);
351
+ console.log(`[QueryClient] | Command: ${toQueryPreview(sql3)}`);
352
+ const start = performance.now();
353
+ const result = await this.client.command({
354
+ query,
355
+ query_params,
356
+ query_id: this.query_id_prefix + randomUUID()
357
+ });
358
+ const elapsedMs = performance.now() - start;
359
+ console.log(
360
+ `[QueryClient] | Command completed: ${formatElapsedTime(elapsedMs)}`
361
+ );
362
+ return result;
363
+ }
364
+ };
365
+ var WorkflowClient = class {
366
+ client;
367
+ constructor(temporalClient) {
368
+ this.client = temporalClient;
369
+ }
370
+ async execute(name, input_data) {
371
+ try {
372
+ if (!this.client) {
373
+ return {
374
+ status: 404,
375
+ body: `Temporal client not found. Is the feature flag enabled?`
376
+ };
377
+ }
378
+ const config = await this.getWorkflowConfig(name);
379
+ const [processedInput, workflowId] = this.processInputData(
380
+ name,
381
+ input_data
382
+ );
383
+ console.log(
384
+ `WorkflowClient - starting workflow: ${name} with config ${JSON.stringify(config)} and input_data ${JSON.stringify(processedInput)}`
385
+ );
386
+ const handle = await this.client.workflow.start("ScriptWorkflow", {
387
+ args: [
388
+ { workflow_name: name, execution_mode: "start" },
389
+ processedInput
390
+ ],
391
+ taskQueue: "typescript-script-queue",
392
+ workflowId,
393
+ workflowIdConflictPolicy: "FAIL",
394
+ workflowIdReusePolicy: "ALLOW_DUPLICATE",
395
+ retry: {
396
+ maximumAttempts: config.retries
397
+ },
398
+ workflowRunTimeout: config.timeout
399
+ });
400
+ return {
401
+ status: 200,
402
+ body: `Workflow started: ${name}. View it in the Temporal dashboard: http://localhost:8080/namespaces/default/workflows/${workflowId}/${handle.firstExecutionRunId}/history`
403
+ };
404
+ } catch (error) {
405
+ return {
406
+ status: 400,
407
+ body: `Error starting workflow: ${error}`
408
+ };
409
+ }
410
+ }
411
+ async terminate(workflowId) {
412
+ try {
413
+ if (!this.client) {
414
+ return {
415
+ status: 404,
416
+ body: `Temporal client not found. Is the feature flag enabled?`
417
+ };
418
+ }
419
+ const handle = this.client.workflow.getHandle(workflowId);
420
+ await handle.terminate();
421
+ return {
422
+ status: 200,
423
+ body: `Workflow terminated: ${workflowId}`
424
+ };
425
+ } catch (error) {
426
+ return {
427
+ status: 400,
428
+ body: `Error terminating workflow: ${error}`
429
+ };
430
+ }
431
+ }
432
+ async getWorkflowConfig(name) {
433
+ const workflows = await getWorkflows2();
434
+ const dmv2Workflow = workflows.get(name);
435
+ if (dmv2Workflow) {
436
+ return {
437
+ retries: dmv2Workflow.config.retries || 3,
438
+ timeout: dmv2Workflow.config.timeout || "1h"
439
+ };
440
+ }
441
+ throw new Error(`Workflow config not found for ${name}`);
442
+ }
443
+ processInputData(name, input_data) {
444
+ let workflowId = name;
445
+ if (input_data) {
446
+ const hash = createHash3("sha256").update(JSON.stringify(input_data)).digest("hex").slice(0, 16);
447
+ workflowId = `${name}-${hash}`;
448
+ }
449
+ return [input_data, workflowId];
450
+ }
451
+ };
510
452
  async function getTemporalClient(temporalUrl, namespace, clientCert, clientKey, apiKey) {
511
453
  try {
512
454
  console.info(
@@ -543,984 +485,538 @@ async function getTemporalClient(temporalUrl, namespace, clientCert, clientKey,
543
485
  return void 0;
544
486
  }
545
487
  }
546
- var MooseClient, QueryClient, WorkflowClient;
547
- var init_helpers2 = __esm({
548
- "src/consumption-apis/helpers.ts"() {
549
- "use strict";
550
- init_internal();
551
- init_sqlHelpers();
552
- MooseClient = class {
553
- query;
554
- workflow;
555
- constructor(queryClient, temporalClient) {
556
- this.query = queryClient;
557
- this.workflow = new WorkflowClient(temporalClient);
558
- }
559
- };
560
- QueryClient = class {
561
- client;
562
- query_id_prefix;
563
- constructor(client, query_id_prefix) {
564
- this.client = client;
565
- this.query_id_prefix = query_id_prefix;
566
- }
567
- async execute(sql3) {
568
- const [query, query_params] = toQuery(sql3);
569
- console.log(`[QueryClient] | Query: ${toQueryPreview(sql3)}`);
570
- const start = performance.now();
571
- const result = await this.client.query({
572
- query,
573
- query_params,
574
- format: "JSONEachRow",
575
- query_id: this.query_id_prefix + randomUUID()
576
- // Note: wait_end_of_query deliberately NOT set here as this is used for SELECT queries
577
- // where response buffering would harm streaming performance and concurrency
578
- });
579
- const elapsedMs = performance.now() - start;
580
- console.log(
581
- `[QueryClient] | Query completed: ${formatElapsedTime(elapsedMs)}`
582
- );
583
- return result;
584
- }
585
- async command(sql3) {
586
- const [query, query_params] = toQuery(sql3);
587
- console.log(`[QueryClient] | Command: ${toQueryPreview(sql3)}`);
588
- const start = performance.now();
589
- const result = await this.client.command({
590
- query,
591
- query_params,
592
- query_id: this.query_id_prefix + randomUUID()
593
- });
594
- const elapsedMs = performance.now() - start;
595
- console.log(
596
- `[QueryClient] | Command completed: ${formatElapsedTime(elapsedMs)}`
597
- );
598
- return result;
599
- }
600
- };
601
- WorkflowClient = class {
602
- client;
603
- constructor(temporalClient) {
604
- this.client = temporalClient;
605
- }
606
- async execute(name, input_data) {
607
- try {
608
- if (!this.client) {
609
- return {
610
- status: 404,
611
- body: `Temporal client not found. Is the feature flag enabled?`
612
- };
613
- }
614
- const config = await this.getWorkflowConfig(name);
615
- const [processedInput, workflowId] = this.processInputData(
616
- name,
617
- input_data
618
- );
619
- console.log(
620
- `WorkflowClient - starting workflow: ${name} with config ${JSON.stringify(config)} and input_data ${JSON.stringify(processedInput)}`
621
- );
622
- const handle = await this.client.workflow.start("ScriptWorkflow", {
623
- args: [
624
- { workflow_name: name, execution_mode: "start" },
625
- processedInput
626
- ],
627
- taskQueue: "typescript-script-queue",
628
- workflowId,
629
- workflowIdConflictPolicy: "FAIL",
630
- workflowIdReusePolicy: "ALLOW_DUPLICATE",
631
- retry: {
632
- maximumAttempts: config.retries
633
- },
634
- workflowRunTimeout: config.timeout
635
- });
636
- return {
637
- status: 200,
638
- body: `Workflow started: ${name}. View it in the Temporal dashboard: http://localhost:8080/namespaces/default/workflows/${workflowId}/${handle.firstExecutionRunId}/history`
639
- };
640
- } catch (error) {
641
- return {
642
- status: 400,
643
- body: `Error starting workflow: ${error}`
644
- };
645
- }
646
- }
647
- async terminate(workflowId) {
648
- try {
649
- if (!this.client) {
650
- return {
651
- status: 404,
652
- body: `Temporal client not found. Is the feature flag enabled?`
653
- };
654
- }
655
- const handle = this.client.workflow.getHandle(workflowId);
656
- await handle.terminate();
657
- return {
658
- status: 200,
659
- body: `Workflow terminated: ${workflowId}`
660
- };
661
- } catch (error) {
662
- return {
663
- status: 400,
664
- body: `Error terminating workflow: ${error}`
665
- };
666
- }
667
- }
668
- async getWorkflowConfig(name) {
669
- const workflows = await getWorkflows2();
670
- const dmv2Workflow = workflows.get(name);
671
- if (dmv2Workflow) {
672
- return {
673
- retries: dmv2Workflow.config.retries || 3,
674
- timeout: dmv2Workflow.config.timeout || "1h"
675
- };
676
- }
677
- throw new Error(`Workflow config not found for ${name}`);
678
- }
679
- processInputData(name, input_data) {
680
- let workflowId = name;
681
- if (input_data) {
682
- const hash = createHash3("sha256").update(JSON.stringify(input_data)).digest("hex").slice(0, 16);
683
- workflowId = `${name}-${hash}`;
684
- }
685
- return [input_data, workflowId];
686
- }
687
- };
688
- }
689
- });
690
-
691
- // src/consumption-apis/webAppHelpers.ts
692
- var init_webAppHelpers = __esm({
693
- "src/consumption-apis/webAppHelpers.ts"() {
694
- "use strict";
695
- }
696
- });
697
488
 
698
- // src/scripts/task.ts
699
- var init_task = __esm({
700
- "src/scripts/task.ts"() {
701
- "use strict";
702
- }
703
- });
489
+ // src/consumption-apis/runner.ts
490
+ init_commons();
491
+ import http2 from "http";
492
+ import * as jose from "jose";
704
493
 
705
494
  // src/cluster-utils.ts
706
495
  import cluster from "cluster";
707
496
  import { availableParallelism } from "os";
708
497
  import { exit } from "process";
709
- var DEFAULT_MAX_CPU_USAGE_RATIO, RESTART_TIME_MS, SIGTERM, SIGINT, SHUTDOWN_WORKERS_INTERVAL, Cluster;
710
- var init_cluster_utils = __esm({
711
- "src/cluster-utils.ts"() {
712
- "use strict";
713
- DEFAULT_MAX_CPU_USAGE_RATIO = 0.7;
714
- RESTART_TIME_MS = 1e4;
715
- SIGTERM = "SIGTERM";
716
- SIGINT = "SIGINT";
717
- SHUTDOWN_WORKERS_INTERVAL = 500;
718
- Cluster = class {
719
- // Tracks if shutdown is currently in progress
720
- shutdownInProgress = false;
721
- // Tracks if workers exited cleanly during shutdown
722
- hasCleanWorkerExit = true;
723
- // String identifying if this is primary or worker process
724
- processStr = `${cluster.isPrimary ? "primary" : "worker"} process ${process.pid}`;
725
- // Functions for starting and stopping workers
726
- workerStart;
727
- workerStop;
728
- // Result from starting worker, needed for cleanup
729
- startOutput;
730
- maxCpuUsageRatio;
731
- usedCpuCount;
732
- /**
733
- * Creates a new cluster manager instance.
734
- *
735
- * @param options - Configuration options for the cluster
736
- * @param options.workerStart - Async function to execute when starting a worker
737
- * @param options.workerStop - Async function to execute when stopping a worker
738
- * @param options.maxCpuUsageRatio - Maximum ratio of CPU cores to utilize (0-1)
739
- * @param options.maxWorkerCount - Maximum number of workers to spawn
740
- * @throws {Error} If maxCpuUsageRatio is not between 0 and 1
741
- */
742
- constructor(options) {
743
- this.workerStart = options.workerStart;
744
- this.workerStop = options.workerStop;
745
- if (options.maxCpuUsageRatio && (options.maxCpuUsageRatio > 1 || options.maxCpuUsageRatio < 0)) {
746
- throw new Error("maxCpuUsageRatio must be between 0 and 1");
747
- }
748
- this.maxCpuUsageRatio = options.maxCpuUsageRatio || DEFAULT_MAX_CPU_USAGE_RATIO;
749
- this.usedCpuCount = this.computeCPUUsageCount(
750
- this.maxCpuUsageRatio,
751
- options.maxWorkerCount
752
- );
753
- }
754
- /**
755
- * Calculates the number of CPU cores to utilize based on available parallelism and constraints.
756
- *
757
- * @param cpuUsageRatio - Ratio of CPU cores to use (0-1)
758
- * @param maxWorkerCount - Optional maximum number of workers
759
- * @returns The number of CPU cores to utilize
760
- */
761
- computeCPUUsageCount(cpuUsageRatio, maxWorkerCount) {
762
- const cpuCount = availableParallelism();
763
- const maxWorkers = maxWorkerCount || cpuCount;
764
- return Math.min(
765
- maxWorkers,
766
- Math.max(1, Math.floor(cpuCount * cpuUsageRatio))
767
- );
768
- }
769
- /**
770
- * Initializes the cluster by spawning worker processes and setting up signal handlers.
771
- * For the primary process, spawns workers and monitors parent process.
772
- * For worker processes, executes the worker startup function.
773
- *
774
- * @throws {Error} If worker is undefined in worker process
775
- */
776
- async start() {
777
- process.on(SIGTERM, this.gracefulClusterShutdown(SIGTERM));
778
- process.on(SIGINT, this.gracefulClusterShutdown(SIGINT));
779
- if (cluster.isPrimary) {
780
- const parentPid = process.ppid;
781
- setInterval(() => {
782
- try {
783
- process.kill(parentPid, 0);
784
- } catch (e) {
785
- console.log("Parent process has exited.");
786
- this.gracefulClusterShutdown(SIGTERM)();
787
- }
788
- }, 1e3);
789
- await this.bootWorkers(this.usedCpuCount);
790
- } else {
791
- if (!cluster.worker) {
792
- throw new Error(
793
- "Worker is not defined, it should be defined in worker process"
794
- );
795
- }
796
- this.startOutput = await this.workerStart(
797
- cluster.worker,
798
- this.usedCpuCount
799
- );
800
- }
801
- }
802
- /**
803
- * Spawns worker processes and configures their lifecycle event handlers.
804
- * Handles worker online, exit and disconnect events.
805
- * Automatically restarts failed workers during normal operation.
806
- *
807
- * @param numWorkers - Number of worker processes to spawn
808
- */
809
- bootWorkers = async (numWorkers) => {
810
- console.info(`Setting ${numWorkers} workers...`);
811
- for (let i = 0; i < numWorkers; i++) {
812
- cluster.fork();
813
- }
814
- cluster.on("online", (worker) => {
815
- console.info(`worker process ${worker.process.pid} is online`);
816
- });
817
- cluster.on("exit", (worker, code, signal) => {
818
- console.info(
819
- `worker ${worker.process.pid} exited with code ${code} and signal ${signal}`
820
- );
821
- if (!this.shutdownInProgress) {
822
- setTimeout(() => cluster.fork(), RESTART_TIME_MS);
823
- }
824
- if (this.shutdownInProgress && code != 0) {
825
- this.hasCleanWorkerExit = false;
826
- }
827
- });
828
- cluster.on("disconnect", (worker) => {
829
- console.info(`worker process ${worker.process.pid} has disconnected`);
830
- });
831
- };
832
- /**
833
- * Creates a handler function for graceful shutdown on receipt of a signal.
834
- * Ensures only one shutdown can occur at a time.
835
- * Handles shutdown differently for primary and worker processes.
836
- *
837
- * @param signal - The signal triggering the shutdown (e.g. SIGTERM)
838
- * @returns An async function that performs the shutdown
839
- */
840
- gracefulClusterShutdown = (signal) => async () => {
841
- if (this.shutdownInProgress) {
842
- return;
843
- }
844
- this.shutdownInProgress = true;
845
- this.hasCleanWorkerExit = true;
846
- console.info(
847
- `Got ${signal} on ${this.processStr}. Graceful shutdown start at ${(/* @__PURE__ */ new Date()).toISOString()}`
848
- );
849
- try {
850
- if (cluster.isPrimary) {
851
- await this.shutdownWorkers(signal);
852
- console.info(`${this.processStr} - worker shutdown successful`);
853
- exit(0);
854
- } else {
855
- if (this.startOutput) {
856
- await this.workerStop(this.startOutput);
857
- } else {
858
- console.info(
859
- `${this.processStr} - shutdown before worker fully started`
860
- );
861
- }
862
- console.info(`${this.processStr} shutdown successful`);
863
- this.hasCleanWorkerExit ? exit(0) : exit(1);
864
- }
865
- } catch (e) {
866
- console.error(`${this.processStr} - shutdown failed`, e);
867
- exit(1);
868
- }
869
- };
870
- /**
871
- * Gracefully terminates all worker processes.
872
- * Monitors workers until they all exit or timeout occurs.
873
- * Only relevant for the primary process.
874
- *
875
- * @param signal - The signal to send to worker processes
876
- * @returns A promise that resolves when all workers have terminated
877
- */
878
- shutdownWorkers = (signal) => {
879
- return new Promise((resolve2, reject) => {
880
- if (!cluster.isPrimary) {
881
- return resolve2();
882
- }
883
- if (!cluster.workers) {
884
- return resolve2();
885
- }
886
- const workerIds = Object.keys(cluster.workers);
887
- if (workerIds.length == 0) {
888
- return resolve2();
889
- }
890
- let workersAlive = 0;
891
- let funcRun = 0;
892
- const cleanWorkers = () => {
893
- ++funcRun;
894
- workersAlive = 0;
895
- Object.values(cluster.workers || {}).filter((worker) => !!worker).forEach((worker) => {
896
- if (worker && !worker.isDead()) {
897
- ++workersAlive;
898
- if (funcRun == 1) {
899
- worker.kill(signal);
900
- }
901
- }
902
- });
903
- console.info(workersAlive + " workers alive");
904
- if (workersAlive == 0) {
905
- clearInterval(interval);
906
- return resolve2();
907
- }
908
- };
909
- const interval = setInterval(cleanWorkers, SHUTDOWN_WORKERS_INTERVAL);
910
- });
911
- };
912
- };
913
- }
914
- });
915
-
916
- // src/config/configFile.ts
917
- import path from "path";
918
- import * as toml from "toml";
919
- async function findConfigFile(startDir = process.cwd()) {
920
- const fs4 = await import("fs");
921
- let currentDir = path.resolve(startDir);
922
- while (true) {
923
- const configPath = path.join(currentDir, "moose.config.toml");
924
- if (fs4.existsSync(configPath)) {
925
- return configPath;
926
- }
927
- const parentDir = path.dirname(currentDir);
928
- if (parentDir === currentDir) {
929
- break;
498
+ var DEFAULT_MAX_CPU_USAGE_RATIO = 0.7;
499
+ var RESTART_TIME_MS = 1e4;
500
+ var SIGTERM = "SIGTERM";
501
+ var SIGINT = "SIGINT";
502
+ var SHUTDOWN_WORKERS_INTERVAL = 500;
503
+ var Cluster = class {
504
+ // Tracks if shutdown is currently in progress
505
+ shutdownInProgress = false;
506
+ // Tracks if workers exited cleanly during shutdown
507
+ hasCleanWorkerExit = true;
508
+ // String identifying if this is primary or worker process
509
+ processStr = `${cluster.isPrimary ? "primary" : "worker"} process ${process.pid}`;
510
+ // Functions for starting and stopping workers
511
+ workerStart;
512
+ workerStop;
513
+ // Result from starting worker, needed for cleanup
514
+ startOutput;
515
+ maxCpuUsageRatio;
516
+ usedCpuCount;
517
+ /**
518
+ * Creates a new cluster manager instance.
519
+ *
520
+ * @param options - Configuration options for the cluster
521
+ * @param options.workerStart - Async function to execute when starting a worker
522
+ * @param options.workerStop - Async function to execute when stopping a worker
523
+ * @param options.maxCpuUsageRatio - Maximum ratio of CPU cores to utilize (0-1)
524
+ * @param options.maxWorkerCount - Maximum number of workers to spawn
525
+ * @throws {Error} If maxCpuUsageRatio is not between 0 and 1
526
+ */
527
+ constructor(options) {
528
+ this.workerStart = options.workerStart;
529
+ this.workerStop = options.workerStop;
530
+ if (options.maxCpuUsageRatio && (options.maxCpuUsageRatio > 1 || options.maxCpuUsageRatio < 0)) {
531
+ throw new Error("maxCpuUsageRatio must be between 0 and 1");
930
532
  }
931
- currentDir = parentDir;
932
- }
933
- return null;
934
- }
935
- async function readProjectConfig() {
936
- const fs4 = await import("fs");
937
- const configPath = await findConfigFile();
938
- if (!configPath) {
939
- throw new ConfigError(
940
- "moose.config.toml not found in current directory or any parent directory"
533
+ this.maxCpuUsageRatio = options.maxCpuUsageRatio || DEFAULT_MAX_CPU_USAGE_RATIO;
534
+ this.usedCpuCount = this.computeCPUUsageCount(
535
+ this.maxCpuUsageRatio,
536
+ options.maxWorkerCount
941
537
  );
942
538
  }
943
- try {
944
- const configContent = fs4.readFileSync(configPath, "utf-8");
945
- const config = toml.parse(configContent);
946
- return config;
947
- } catch (error) {
948
- throw new ConfigError(`Failed to parse moose.config.toml: ${error}`);
949
- }
950
- }
951
- var ConfigError;
952
- var init_configFile = __esm({
953
- "src/config/configFile.ts"() {
954
- "use strict";
955
- ConfigError = class extends Error {
956
- constructor(message) {
957
- super(message);
958
- this.name = "ConfigError";
959
- }
960
- };
539
+ /**
540
+ * Calculates the number of CPU cores to utilize based on available parallelism and constraints.
541
+ *
542
+ * @param cpuUsageRatio - Ratio of CPU cores to use (0-1)
543
+ * @param maxWorkerCount - Optional maximum number of workers
544
+ * @returns The number of CPU cores to utilize
545
+ */
546
+ computeCPUUsageCount(cpuUsageRatio, maxWorkerCount) {
547
+ const cpuCount = availableParallelism();
548
+ const maxWorkers = maxWorkerCount || cpuCount;
549
+ return Math.min(
550
+ maxWorkers,
551
+ Math.max(1, Math.floor(cpuCount * cpuUsageRatio))
552
+ );
961
553
  }
962
- });
963
-
964
- // src/config/runtime.ts
965
- var runtime_exports = {};
966
- var ConfigurationRegistry;
967
- var init_runtime = __esm({
968
- "src/config/runtime.ts"() {
969
- "use strict";
970
- init_configFile();
971
- ConfigurationRegistry = class _ConfigurationRegistry {
972
- static instance;
973
- clickhouseConfig;
974
- kafkaConfig;
975
- static getInstance() {
976
- if (!_ConfigurationRegistry.instance) {
977
- _ConfigurationRegistry.instance = new _ConfigurationRegistry();
978
- }
979
- return _ConfigurationRegistry.instance;
980
- }
981
- setClickHouseConfig(config) {
982
- this.clickhouseConfig = config;
983
- }
984
- setKafkaConfig(config) {
985
- this.kafkaConfig = config;
986
- }
987
- _env(name) {
988
- const value = process.env[name];
989
- if (value === void 0) return void 0;
990
- const trimmed = value.trim();
991
- return trimmed.length > 0 ? trimmed : void 0;
992
- }
993
- _parseBool(value) {
994
- if (value === void 0) return void 0;
995
- switch (value.trim().toLowerCase()) {
996
- case "1":
997
- case "true":
998
- case "yes":
999
- case "on":
1000
- return true;
1001
- case "0":
1002
- case "false":
1003
- case "no":
1004
- case "off":
1005
- return false;
1006
- default:
1007
- return void 0;
1008
- }
1009
- }
1010
- async getClickHouseConfig() {
1011
- if (this.clickhouseConfig) {
1012
- return this.clickhouseConfig;
1013
- }
1014
- const projectConfig = await readProjectConfig();
1015
- const envHost = this._env("MOOSE_CLICKHOUSE_CONFIG__HOST");
1016
- const envPort = this._env("MOOSE_CLICKHOUSE_CONFIG__HOST_PORT");
1017
- const envUser = this._env("MOOSE_CLICKHOUSE_CONFIG__USER");
1018
- const envPassword = this._env("MOOSE_CLICKHOUSE_CONFIG__PASSWORD");
1019
- const envDb = this._env("MOOSE_CLICKHOUSE_CONFIG__DB_NAME");
1020
- const envUseSSL = this._parseBool(
1021
- this._env("MOOSE_CLICKHOUSE_CONFIG__USE_SSL")
1022
- );
1023
- return {
1024
- host: envHost ?? projectConfig.clickhouse_config.host,
1025
- port: envPort ?? projectConfig.clickhouse_config.host_port.toString(),
1026
- username: envUser ?? projectConfig.clickhouse_config.user,
1027
- password: envPassword ?? projectConfig.clickhouse_config.password,
1028
- database: envDb ?? projectConfig.clickhouse_config.db_name,
1029
- useSSL: envUseSSL !== void 0 ? envUseSSL : projectConfig.clickhouse_config.use_ssl || false
1030
- };
1031
- }
1032
- async getStandaloneClickhouseConfig(overrides) {
1033
- if (this.clickhouseConfig) {
1034
- return { ...this.clickhouseConfig, ...overrides };
1035
- }
1036
- const envHost = this._env("MOOSE_CLICKHOUSE_CONFIG__HOST");
1037
- const envPort = this._env("MOOSE_CLICKHOUSE_CONFIG__HOST_PORT");
1038
- const envUser = this._env("MOOSE_CLICKHOUSE_CONFIG__USER");
1039
- const envPassword = this._env("MOOSE_CLICKHOUSE_CONFIG__PASSWORD");
1040
- const envDb = this._env("MOOSE_CLICKHOUSE_CONFIG__DB_NAME");
1041
- const envUseSSL = this._parseBool(
1042
- this._env("MOOSE_CLICKHOUSE_CONFIG__USE_SSL")
1043
- );
1044
- let projectConfig;
554
+ /**
555
+ * Initializes the cluster by spawning worker processes and setting up signal handlers.
556
+ * For the primary process, spawns workers and monitors parent process.
557
+ * For worker processes, executes the worker startup function.
558
+ *
559
+ * @throws {Error} If worker is undefined in worker process
560
+ */
561
+ async start() {
562
+ process.on(SIGTERM, this.gracefulClusterShutdown(SIGTERM));
563
+ process.on(SIGINT, this.gracefulClusterShutdown(SIGINT));
564
+ if (cluster.isPrimary) {
565
+ const parentPid = process.ppid;
566
+ setInterval(() => {
1045
567
  try {
1046
- projectConfig = await readProjectConfig();
1047
- } catch (error) {
1048
- projectConfig = null;
1049
- }
1050
- const defaults = {
1051
- host: "localhost",
1052
- port: "18123",
1053
- username: "default",
1054
- password: "",
1055
- database: "local",
1056
- useSSL: false
1057
- };
1058
- return {
1059
- host: overrides?.host ?? envHost ?? projectConfig?.clickhouse_config.host ?? defaults.host,
1060
- port: overrides?.port ?? envPort ?? projectConfig?.clickhouse_config.host_port.toString() ?? defaults.port,
1061
- username: overrides?.username ?? envUser ?? projectConfig?.clickhouse_config.user ?? defaults.username,
1062
- password: overrides?.password ?? envPassword ?? projectConfig?.clickhouse_config.password ?? defaults.password,
1063
- database: overrides?.database ?? envDb ?? projectConfig?.clickhouse_config.db_name ?? defaults.database,
1064
- useSSL: overrides?.useSSL ?? envUseSSL ?? projectConfig?.clickhouse_config.use_ssl ?? defaults.useSSL
1065
- };
1066
- }
1067
- async getKafkaConfig() {
1068
- if (this.kafkaConfig) {
1069
- return this.kafkaConfig;
568
+ process.kill(parentPid, 0);
569
+ } catch (e) {
570
+ console.log("Parent process has exited.");
571
+ this.gracefulClusterShutdown(SIGTERM)();
1070
572
  }
1071
- const projectConfig = await readProjectConfig();
1072
- const envBroker = this._env("MOOSE_REDPANDA_CONFIG__BROKER") ?? this._env("MOOSE_KAFKA_CONFIG__BROKER");
1073
- const envMsgTimeout = this._env("MOOSE_REDPANDA_CONFIG__MESSAGE_TIMEOUT_MS") ?? this._env("MOOSE_KAFKA_CONFIG__MESSAGE_TIMEOUT_MS");
1074
- const envSaslUsername = this._env("MOOSE_REDPANDA_CONFIG__SASL_USERNAME") ?? this._env("MOOSE_KAFKA_CONFIG__SASL_USERNAME");
1075
- const envSaslPassword = this._env("MOOSE_REDPANDA_CONFIG__SASL_PASSWORD") ?? this._env("MOOSE_KAFKA_CONFIG__SASL_PASSWORD");
1076
- const envSaslMechanism = this._env("MOOSE_REDPANDA_CONFIG__SASL_MECHANISM") ?? this._env("MOOSE_KAFKA_CONFIG__SASL_MECHANISM");
1077
- const envSecurityProtocol = this._env("MOOSE_REDPANDA_CONFIG__SECURITY_PROTOCOL") ?? this._env("MOOSE_KAFKA_CONFIG__SECURITY_PROTOCOL");
1078
- const envNamespace = this._env("MOOSE_REDPANDA_CONFIG__NAMESPACE") ?? this._env("MOOSE_KAFKA_CONFIG__NAMESPACE");
1079
- const envSchemaRegistryUrl = this._env("MOOSE_REDPANDA_CONFIG__SCHEMA_REGISTRY_URL") ?? this._env("MOOSE_KAFKA_CONFIG__SCHEMA_REGISTRY_URL");
1080
- const fileKafka = projectConfig.kafka_config ?? projectConfig.redpanda_config;
1081
- return {
1082
- broker: envBroker ?? fileKafka?.broker ?? "localhost:19092",
1083
- messageTimeoutMs: envMsgTimeout ? parseInt(envMsgTimeout, 10) : fileKafka?.message_timeout_ms ?? 1e3,
1084
- saslUsername: envSaslUsername ?? fileKafka?.sasl_username,
1085
- saslPassword: envSaslPassword ?? fileKafka?.sasl_password,
1086
- saslMechanism: envSaslMechanism ?? fileKafka?.sasl_mechanism,
1087
- securityProtocol: envSecurityProtocol ?? fileKafka?.security_protocol,
1088
- namespace: envNamespace ?? fileKafka?.namespace,
1089
- schemaRegistryUrl: envSchemaRegistryUrl ?? fileKafka?.schema_registry_url
1090
- };
1091
- }
1092
- hasRuntimeConfig() {
1093
- return !!this.clickhouseConfig || !!this.kafkaConfig;
573
+ }, 1e3);
574
+ await this.bootWorkers(this.usedCpuCount);
575
+ } else {
576
+ if (!cluster.worker) {
577
+ throw new Error(
578
+ "Worker is not defined, it should be defined in worker process"
579
+ );
1094
580
  }
1095
- };
1096
- globalThis._mooseConfigRegistry = ConfigurationRegistry.getInstance();
1097
- }
1098
- });
1099
-
1100
- // src/consumption-apis/standalone.ts
1101
- var standalone_exports = {};
1102
- __export(standalone_exports, {
1103
- getMooseClients: () => getMooseClients,
1104
- getMooseUtils: () => getMooseUtils
1105
- });
1106
- async function getMooseUtils(req) {
1107
- if (req !== void 0) {
1108
- console.warn(
1109
- "[DEPRECATED] getMooseUtils(req) no longer requires a request parameter. Use getMooseUtils() instead."
1110
- );
1111
- }
1112
- const runtimeContext = globalThis._mooseRuntimeContext;
1113
- if (runtimeContext) {
1114
- return {
1115
- client: runtimeContext.client,
1116
- sql,
1117
- jwt: runtimeContext.jwt
1118
- };
1119
- }
1120
- if (standaloneUtils) {
1121
- return standaloneUtils;
1122
- }
1123
- if (initPromise) {
1124
- return initPromise;
1125
- }
1126
- initPromise = (async () => {
1127
- await Promise.resolve().then(() => (init_runtime(), runtime_exports));
1128
- const configRegistry = globalThis._mooseConfigRegistry;
1129
- if (!configRegistry) {
1130
- throw new Error(
1131
- "Moose not initialized. Ensure you're running within a Moose app or have proper configuration set up."
581
+ this.startOutput = await this.workerStart(
582
+ cluster.worker,
583
+ this.usedCpuCount
1132
584
  );
1133
585
  }
1134
- const clickhouseConfig = await configRegistry.getStandaloneClickhouseConfig();
1135
- const clickhouseClient = getClickhouseClient(
1136
- toClientConfig(clickhouseConfig)
1137
- );
1138
- const queryClient = new QueryClient(clickhouseClient, "standalone");
1139
- const mooseClient = new MooseClient(queryClient);
1140
- standaloneUtils = {
1141
- client: mooseClient,
1142
- sql,
1143
- jwt: void 0
1144
- };
1145
- return standaloneUtils;
1146
- })();
1147
- try {
1148
- return await initPromise;
1149
- } finally {
1150
- initPromise = null;
1151
586
  }
1152
- }
1153
- async function getMooseClients(config) {
1154
- console.warn(
1155
- "[DEPRECATED] getMooseClients() is deprecated. Use getMooseUtils() instead."
1156
- );
1157
- if (config && Object.keys(config).length > 0) {
1158
- await Promise.resolve().then(() => (init_runtime(), runtime_exports));
1159
- const configRegistry = globalThis._mooseConfigRegistry;
1160
- if (!configRegistry) {
1161
- throw new Error(
1162
- "Configuration registry not initialized. Ensure the Moose framework is properly set up."
587
+ /**
588
+ * Spawns worker processes and configures their lifecycle event handlers.
589
+ * Handles worker online, exit and disconnect events.
590
+ * Automatically restarts failed workers during normal operation.
591
+ *
592
+ * @param numWorkers - Number of worker processes to spawn
593
+ */
594
+ bootWorkers = async (numWorkers) => {
595
+ console.info(`Setting ${numWorkers} workers...`);
596
+ for (let i = 0; i < numWorkers; i++) {
597
+ cluster.fork();
598
+ }
599
+ cluster.on("online", (worker) => {
600
+ console.info(`worker process ${worker.process.pid} is online`);
601
+ });
602
+ cluster.on("exit", (worker, code, signal) => {
603
+ console.info(
604
+ `worker ${worker.process.pid} exited with code ${code} and signal ${signal}`
1163
605
  );
606
+ if (!this.shutdownInProgress) {
607
+ setTimeout(() => cluster.fork(), RESTART_TIME_MS);
608
+ }
609
+ if (this.shutdownInProgress && code != 0) {
610
+ this.hasCleanWorkerExit = false;
611
+ }
612
+ });
613
+ cluster.on("disconnect", (worker) => {
614
+ console.info(`worker process ${worker.process.pid} has disconnected`);
615
+ });
616
+ };
617
+ /**
618
+ * Creates a handler function for graceful shutdown on receipt of a signal.
619
+ * Ensures only one shutdown can occur at a time.
620
+ * Handles shutdown differently for primary and worker processes.
621
+ *
622
+ * @param signal - The signal triggering the shutdown (e.g. SIGTERM)
623
+ * @returns An async function that performs the shutdown
624
+ */
625
+ gracefulClusterShutdown = (signal) => async () => {
626
+ if (this.shutdownInProgress) {
627
+ return;
1164
628
  }
1165
- const clickhouseConfig = await configRegistry.getStandaloneClickhouseConfig(config);
1166
- const clickhouseClient = getClickhouseClient(
1167
- toClientConfig(clickhouseConfig)
629
+ this.shutdownInProgress = true;
630
+ this.hasCleanWorkerExit = true;
631
+ console.info(
632
+ `Got ${signal} on ${this.processStr}. Graceful shutdown start at ${(/* @__PURE__ */ new Date()).toISOString()}`
1168
633
  );
1169
- const queryClient = new QueryClient(clickhouseClient, "standalone");
1170
- const mooseClient = new MooseClient(queryClient);
1171
- return { client: mooseClient };
1172
- }
1173
- const utils = await getMooseUtils();
1174
- return { client: utils.client };
1175
- }
1176
- var standaloneUtils, initPromise, toClientConfig;
1177
- var init_standalone = __esm({
1178
- "src/consumption-apis/standalone.ts"() {
1179
- "use strict";
1180
- init_helpers2();
1181
- init_commons();
1182
- init_sqlHelpers();
1183
- standaloneUtils = null;
1184
- initPromise = null;
1185
- toClientConfig = (config) => ({
1186
- ...config,
1187
- useSSL: config.useSSL ? "true" : "false"
634
+ try {
635
+ if (cluster.isPrimary) {
636
+ await this.shutdownWorkers(signal);
637
+ console.info(`${this.processStr} - worker shutdown successful`);
638
+ exit(0);
639
+ } else {
640
+ if (this.startOutput) {
641
+ await this.workerStop(this.startOutput);
642
+ } else {
643
+ console.info(
644
+ `${this.processStr} - shutdown before worker fully started`
645
+ );
646
+ }
647
+ console.info(`${this.processStr} shutdown successful`);
648
+ this.hasCleanWorkerExit ? exit(0) : exit(1);
649
+ }
650
+ } catch (e) {
651
+ console.error(`${this.processStr} - shutdown failed`, e);
652
+ exit(1);
653
+ }
654
+ };
655
+ /**
656
+ * Gracefully terminates all worker processes.
657
+ * Monitors workers until they all exit or timeout occurs.
658
+ * Only relevant for the primary process.
659
+ *
660
+ * @param signal - The signal to send to worker processes
661
+ * @returns A promise that resolves when all workers have terminated
662
+ */
663
+ shutdownWorkers = (signal) => {
664
+ return new Promise((resolve2, reject) => {
665
+ if (!cluster.isPrimary) {
666
+ return resolve2();
667
+ }
668
+ if (!cluster.workers) {
669
+ return resolve2();
670
+ }
671
+ const workerIds = Object.keys(cluster.workers);
672
+ if (workerIds.length == 0) {
673
+ return resolve2();
674
+ }
675
+ let workersAlive = 0;
676
+ let funcRun = 0;
677
+ const cleanWorkers = () => {
678
+ ++funcRun;
679
+ workersAlive = 0;
680
+ Object.values(cluster.workers || {}).filter((worker) => !!worker).forEach((worker) => {
681
+ if (worker && !worker.isDead()) {
682
+ ++workersAlive;
683
+ if (funcRun == 1) {
684
+ worker.kill(signal);
685
+ }
686
+ }
687
+ });
688
+ console.info(workersAlive + " workers alive");
689
+ if (workersAlive == 0) {
690
+ clearInterval(interval);
691
+ return resolve2();
692
+ }
693
+ };
694
+ const interval = setInterval(cleanWorkers, SHUTDOWN_WORKERS_INTERVAL);
1188
695
  });
1189
- }
1190
- });
696
+ };
697
+ };
1191
698
 
1192
699
  // src/consumption-apis/runner.ts
1193
- import http2 from "http";
1194
- import * as jose from "jose";
1195
- var toClientConfig2, createPath, httpLogger, modulesCache, apiHandler, createMainRouter, runApis;
1196
- var init_runner = __esm({
1197
- "src/consumption-apis/runner.ts"() {
1198
- "use strict";
1199
- init_commons();
1200
- init_helpers2();
1201
- init_cluster_utils();
1202
- init_sqlHelpers();
1203
- init_internal();
1204
- toClientConfig2 = (config) => ({
1205
- ...config,
1206
- useSSL: config.useSSL ? "true" : "false"
1207
- });
1208
- createPath = (apisDir, path4) => `${apisDir}${path4}.ts`;
1209
- httpLogger = (req, res, startMs) => {
1210
- console.log(
1211
- `${req.method} ${req.url} ${res.statusCode} ${Date.now() - startMs}ms`
1212
- );
1213
- };
1214
- modulesCache = /* @__PURE__ */ new Map();
1215
- apiHandler = async (publicKey, clickhouseClient, temporalClient, apisDir, enforceAuth, isDmv2, jwtConfig) => {
1216
- const apis = isDmv2 ? await getApis2() : /* @__PURE__ */ new Map();
1217
- return async (req, res) => {
1218
- const start = Date.now();
1219
- try {
1220
- const url = new URL(req.url || "", "http://localhost");
1221
- const fileName = url.pathname;
1222
- let jwtPayload;
1223
- if (publicKey && jwtConfig) {
1224
- const jwt = req.headers.authorization?.split(" ")[1];
1225
- if (jwt) {
1226
- try {
1227
- const { payload } = await jose.jwtVerify(jwt, publicKey, {
1228
- issuer: jwtConfig.issuer,
1229
- audience: jwtConfig.audience
1230
- });
1231
- jwtPayload = payload;
1232
- } catch (error) {
1233
- console.log("JWT verification failed");
1234
- if (enforceAuth) {
1235
- res.writeHead(401, { "Content-Type": "application/json" });
1236
- res.end(JSON.stringify({ error: "Unauthorized" }));
1237
- httpLogger(req, res, start);
1238
- return;
1239
- }
1240
- }
1241
- } else if (enforceAuth) {
700
+ var toClientConfig = (config) => ({
701
+ ...config,
702
+ useSSL: config.useSSL ? "true" : "false"
703
+ });
704
+ var createPath = (apisDir, path3) => `${apisDir}${path3}.ts`;
705
+ var httpLogger = (req, res, startMs) => {
706
+ console.log(
707
+ `${req.method} ${req.url} ${res.statusCode} ${Date.now() - startMs}ms`
708
+ );
709
+ };
710
+ var modulesCache = /* @__PURE__ */ new Map();
711
+ var apiHandler = async (publicKey, clickhouseClient, temporalClient, apisDir, enforceAuth, isDmv2, jwtConfig) => {
712
+ const apis = isDmv2 ? await getApis2() : /* @__PURE__ */ new Map();
713
+ return async (req, res) => {
714
+ const start = Date.now();
715
+ try {
716
+ const url = new URL(req.url || "", "http://localhost");
717
+ const fileName = url.pathname;
718
+ let jwtPayload;
719
+ if (publicKey && jwtConfig) {
720
+ const jwt = req.headers.authorization?.split(" ")[1];
721
+ if (jwt) {
722
+ try {
723
+ const { payload } = await jose.jwtVerify(jwt, publicKey, {
724
+ issuer: jwtConfig.issuer,
725
+ audience: jwtConfig.audience
726
+ });
727
+ jwtPayload = payload;
728
+ } catch (error) {
729
+ console.log("JWT verification failed");
730
+ if (enforceAuth) {
1242
731
  res.writeHead(401, { "Content-Type": "application/json" });
1243
732
  res.end(JSON.stringify({ error: "Unauthorized" }));
1244
733
  httpLogger(req, res, start);
1245
734
  return;
1246
735
  }
1247
- } else if (enforceAuth) {
1248
- res.writeHead(401, { "Content-Type": "application/json" });
1249
- res.end(JSON.stringify({ error: "Unauthorized" }));
1250
- httpLogger(req, res, start);
1251
- return;
1252
736
  }
1253
- const pathName = createPath(apisDir, fileName);
1254
- const paramsObject = Array.from(url.searchParams.entries()).reduce(
1255
- (obj, [key, value]) => {
1256
- const existingValue = obj[key];
1257
- if (existingValue) {
1258
- if (Array.isArray(existingValue)) {
1259
- existingValue.push(value);
1260
- } else {
1261
- obj[key] = [existingValue, value];
1262
- }
1263
- } else {
1264
- obj[key] = value;
1265
- }
1266
- return obj;
1267
- },
1268
- {}
1269
- );
1270
- let userFuncModule = modulesCache.get(pathName);
1271
- if (userFuncModule === void 0) {
1272
- if (isDmv2) {
1273
- let apiName = fileName.replace(/^\/+|\/+$/g, "");
1274
- let version = null;
1275
- userFuncModule = apis.get(apiName);
1276
- if (!userFuncModule) {
1277
- version = url.searchParams.get("version");
1278
- if (!version && apiName.includes("/")) {
1279
- const pathParts = apiName.split("/");
1280
- if (pathParts.length >= 2) {
1281
- userFuncModule = apis.get(apiName);
1282
- if (!userFuncModule) {
1283
- apiName = pathParts[0];
1284
- version = pathParts.slice(1).join("/");
1285
- }
1286
- }
1287
- }
737
+ } else if (enforceAuth) {
738
+ res.writeHead(401, { "Content-Type": "application/json" });
739
+ res.end(JSON.stringify({ error: "Unauthorized" }));
740
+ httpLogger(req, res, start);
741
+ return;
742
+ }
743
+ } else if (enforceAuth) {
744
+ res.writeHead(401, { "Content-Type": "application/json" });
745
+ res.end(JSON.stringify({ error: "Unauthorized" }));
746
+ httpLogger(req, res, start);
747
+ return;
748
+ }
749
+ const pathName = createPath(apisDir, fileName);
750
+ const paramsObject = Array.from(url.searchParams.entries()).reduce(
751
+ (obj, [key, value]) => {
752
+ const existingValue = obj[key];
753
+ if (existingValue) {
754
+ if (Array.isArray(existingValue)) {
755
+ existingValue.push(value);
756
+ } else {
757
+ obj[key] = [existingValue, value];
758
+ }
759
+ } else {
760
+ obj[key] = value;
761
+ }
762
+ return obj;
763
+ },
764
+ {}
765
+ );
766
+ let userFuncModule = modulesCache.get(pathName);
767
+ if (userFuncModule === void 0) {
768
+ if (isDmv2) {
769
+ let apiName = fileName.replace(/^\/+|\/+$/g, "");
770
+ let version = null;
771
+ userFuncModule = apis.get(apiName);
772
+ if (!userFuncModule) {
773
+ version = url.searchParams.get("version");
774
+ if (!version && apiName.includes("/")) {
775
+ const pathParts = apiName.split("/");
776
+ if (pathParts.length >= 2) {
777
+ userFuncModule = apis.get(apiName);
1288
778
  if (!userFuncModule) {
1289
- if (version) {
1290
- const versionedKey = `${apiName}:${version}`;
1291
- userFuncModule = apis.get(versionedKey);
1292
- } else {
1293
- userFuncModule = apis.get(apiName);
1294
- }
779
+ apiName = pathParts[0];
780
+ version = pathParts.slice(1).join("/");
1295
781
  }
1296
782
  }
1297
- if (!userFuncModule) {
1298
- const availableApis = Array.from(apis.keys()).map(
1299
- (key) => key.replace(":", "/")
1300
- );
1301
- const errorMessage = version ? `API ${apiName} with version ${version} not found. Available APIs: ${availableApis.join(", ")}` : `API ${apiName} not found. Available APIs: ${availableApis.join(", ")}`;
1302
- throw new Error(errorMessage);
1303
- }
1304
- modulesCache.set(pathName, userFuncModule);
1305
- console.log(`[API] | Executing API: ${apiName}`);
1306
- } else {
1307
- userFuncModule = __require(pathName);
1308
- modulesCache.set(pathName, userFuncModule);
1309
783
  }
1310
- }
1311
- const queryClient = new QueryClient(clickhouseClient, fileName);
1312
- let result = isDmv2 ? await userFuncModule(paramsObject, {
1313
- client: new MooseClient(queryClient, temporalClient),
1314
- sql,
1315
- jwt: jwtPayload
1316
- }) : await userFuncModule.default(paramsObject, {
1317
- client: new MooseClient(queryClient, temporalClient),
1318
- sql,
1319
- jwt: jwtPayload
1320
- });
1321
- let body;
1322
- let status;
1323
- if (Object.getPrototypeOf(result).constructor.name === "ResultSet") {
1324
- body = JSON.stringify(await result.json());
1325
- } else {
1326
- if ("body" in result && "status" in result) {
1327
- body = JSON.stringify(result.body);
1328
- status = result.status;
1329
- } else {
1330
- body = JSON.stringify(result);
784
+ if (!userFuncModule) {
785
+ if (version) {
786
+ const versionedKey = `${apiName}:${version}`;
787
+ userFuncModule = apis.get(versionedKey);
788
+ } else {
789
+ userFuncModule = apis.get(apiName);
790
+ }
1331
791
  }
1332
792
  }
1333
- if (status) {
1334
- res.writeHead(status, { "Content-Type": "application/json" });
1335
- httpLogger(req, res, start);
1336
- } else {
1337
- res.writeHead(200, { "Content-Type": "application/json" });
1338
- httpLogger(req, res, start);
1339
- }
1340
- res.end(body);
1341
- } catch (error) {
1342
- console.log("error in path ", req.url, error);
1343
- if (Object.getPrototypeOf(error).constructor.name === "TypeGuardError") {
1344
- res.writeHead(400, { "Content-Type": "application/json" });
1345
- res.end(JSON.stringify({ error: error.message }));
1346
- httpLogger(req, res, start);
1347
- }
1348
- if (error instanceof Error) {
1349
- res.writeHead(500, { "Content-Type": "application/json" });
1350
- res.end(JSON.stringify({ error: error.message }));
1351
- httpLogger(req, res, start);
1352
- } else {
1353
- res.writeHead(500, { "Content-Type": "application/json" });
1354
- res.end();
1355
- httpLogger(req, res, start);
793
+ if (!userFuncModule) {
794
+ const availableApis = Array.from(apis.keys()).map(
795
+ (key) => key.replace(":", "/")
796
+ );
797
+ const errorMessage = version ? `API ${apiName} with version ${version} not found. Available APIs: ${availableApis.join(", ")}` : `API ${apiName} not found. Available APIs: ${availableApis.join(", ")}`;
798
+ throw new Error(errorMessage);
1356
799
  }
800
+ modulesCache.set(pathName, userFuncModule);
801
+ console.log(`[API] | Executing API: ${apiName}`);
802
+ } else {
803
+ userFuncModule = __require(pathName);
804
+ modulesCache.set(pathName, userFuncModule);
1357
805
  }
1358
- };
1359
- };
1360
- createMainRouter = async (publicKey, clickhouseClient, temporalClient, apisDir, enforceAuth, isDmv2, jwtConfig) => {
1361
- const apiRequestHandler = await apiHandler(
1362
- publicKey,
1363
- clickhouseClient,
1364
- temporalClient,
1365
- apisDir,
1366
- enforceAuth,
1367
- isDmv2,
1368
- jwtConfig
1369
- );
1370
- const webApps = isDmv2 ? await getWebApps2() : /* @__PURE__ */ new Map();
1371
- const sortedWebApps = Array.from(webApps.values()).sort((a, b) => {
1372
- const pathA = a.config.mountPath || "/";
1373
- const pathB = b.config.mountPath || "/";
1374
- return pathB.length - pathA.length;
806
+ }
807
+ const queryClient = new QueryClient(clickhouseClient, fileName);
808
+ let result = isDmv2 ? await userFuncModule(paramsObject, {
809
+ client: new MooseClient(queryClient, temporalClient),
810
+ sql,
811
+ jwt: jwtPayload
812
+ }) : await userFuncModule.default(paramsObject, {
813
+ client: new MooseClient(queryClient, temporalClient),
814
+ sql,
815
+ jwt: jwtPayload
1375
816
  });
1376
- return async (req, res) => {
1377
- const start = Date.now();
1378
- const url = new URL(req.url || "", "http://localhost");
1379
- const pathname = url.pathname;
1380
- if (pathname === "/_moose_internal/health") {
1381
- res.writeHead(200, { "Content-Type": "application/json" });
1382
- res.end(
1383
- JSON.stringify({
1384
- status: "healthy",
1385
- timestamp: (/* @__PURE__ */ new Date()).toISOString()
1386
- })
1387
- );
1388
- return;
817
+ let body;
818
+ let status;
819
+ if (Object.getPrototypeOf(result).constructor.name === "ResultSet") {
820
+ body = JSON.stringify(await result.json());
821
+ } else {
822
+ if ("body" in result && "status" in result) {
823
+ body = JSON.stringify(result.body);
824
+ status = result.status;
825
+ } else {
826
+ body = JSON.stringify(result);
1389
827
  }
1390
- let jwtPayload;
1391
- if (publicKey && jwtConfig) {
1392
- const jwt = req.headers.authorization?.split(" ")[1];
1393
- if (jwt) {
1394
- try {
1395
- const { payload } = await jose.jwtVerify(jwt, publicKey, {
1396
- issuer: jwtConfig.issuer,
1397
- audience: jwtConfig.audience
1398
- });
1399
- jwtPayload = payload;
1400
- } catch (error) {
1401
- console.log("JWT verification failed for WebApp route");
1402
- }
1403
- }
828
+ }
829
+ if (status) {
830
+ res.writeHead(status, { "Content-Type": "application/json" });
831
+ httpLogger(req, res, start);
832
+ } else {
833
+ res.writeHead(200, { "Content-Type": "application/json" });
834
+ httpLogger(req, res, start);
835
+ }
836
+ res.end(body);
837
+ } catch (error) {
838
+ console.log("error in path ", req.url, error);
839
+ if (Object.getPrototypeOf(error).constructor.name === "TypeGuardError") {
840
+ res.writeHead(400, { "Content-Type": "application/json" });
841
+ res.end(JSON.stringify({ error: error.message }));
842
+ httpLogger(req, res, start);
843
+ }
844
+ if (error instanceof Error) {
845
+ res.writeHead(500, { "Content-Type": "application/json" });
846
+ res.end(JSON.stringify({ error: error.message }));
847
+ httpLogger(req, res, start);
848
+ } else {
849
+ res.writeHead(500, { "Content-Type": "application/json" });
850
+ res.end();
851
+ httpLogger(req, res, start);
852
+ }
853
+ }
854
+ };
855
+ };
856
+ var createMainRouter = async (publicKey, clickhouseClient, temporalClient, apisDir, enforceAuth, isDmv2, jwtConfig) => {
857
+ const apiRequestHandler = await apiHandler(
858
+ publicKey,
859
+ clickhouseClient,
860
+ temporalClient,
861
+ apisDir,
862
+ enforceAuth,
863
+ isDmv2,
864
+ jwtConfig
865
+ );
866
+ const webApps = isDmv2 ? await getWebApps2() : /* @__PURE__ */ new Map();
867
+ const sortedWebApps = Array.from(webApps.values()).sort((a, b) => {
868
+ const pathA = a.config.mountPath || "/";
869
+ const pathB = b.config.mountPath || "/";
870
+ return pathB.length - pathA.length;
871
+ });
872
+ return async (req, res) => {
873
+ const start = Date.now();
874
+ const url = new URL(req.url || "", "http://localhost");
875
+ const pathname = url.pathname;
876
+ if (pathname === "/_moose_internal/health") {
877
+ res.writeHead(200, { "Content-Type": "application/json" });
878
+ res.end(
879
+ JSON.stringify({
880
+ status: "healthy",
881
+ timestamp: (/* @__PURE__ */ new Date()).toISOString()
882
+ })
883
+ );
884
+ return;
885
+ }
886
+ let jwtPayload;
887
+ if (publicKey && jwtConfig) {
888
+ const jwt = req.headers.authorization?.split(" ")[1];
889
+ if (jwt) {
890
+ try {
891
+ const { payload } = await jose.jwtVerify(jwt, publicKey, {
892
+ issuer: jwtConfig.issuer,
893
+ audience: jwtConfig.audience
894
+ });
895
+ jwtPayload = payload;
896
+ } catch (error) {
897
+ console.log("JWT verification failed for WebApp route");
1404
898
  }
1405
- for (const webApp of sortedWebApps) {
1406
- const mountPath = webApp.config.mountPath || "/";
1407
- const normalizedMount = mountPath.endsWith("/") && mountPath !== "/" ? mountPath.slice(0, -1) : mountPath;
1408
- const matches = pathname === normalizedMount || pathname.startsWith(normalizedMount + "/");
1409
- if (matches) {
1410
- if (webApp.config.injectMooseUtils !== false) {
1411
- const { getMooseUtils: getMooseUtils2 } = await Promise.resolve().then(() => (init_standalone(), standalone_exports));
1412
- req.moose = await getMooseUtils2();
1413
- }
1414
- let proxiedUrl = req.url;
1415
- if (normalizedMount !== "/") {
1416
- const pathWithoutMount = pathname.substring(normalizedMount.length) || "/";
1417
- proxiedUrl = pathWithoutMount + url.search;
1418
- }
1419
- try {
1420
- const modifiedReq = Object.assign(
1421
- Object.create(Object.getPrototypeOf(req)),
1422
- req,
1423
- {
1424
- url: proxiedUrl
1425
- }
1426
- );
1427
- await webApp.handler(modifiedReq, res);
1428
- return;
1429
- } catch (error) {
1430
- console.error(`Error in WebApp ${webApp.name}:`, error);
1431
- if (!res.headersSent) {
1432
- res.writeHead(500, { "Content-Type": "application/json" });
1433
- res.end(JSON.stringify({ error: "Internal Server Error" }));
1434
- }
1435
- return;
1436
- }
1437
- }
899
+ }
900
+ }
901
+ for (const webApp of sortedWebApps) {
902
+ const mountPath = webApp.config.mountPath || "/";
903
+ const normalizedMount = mountPath.endsWith("/") && mountPath !== "/" ? mountPath.slice(0, -1) : mountPath;
904
+ const matches = pathname === normalizedMount || pathname.startsWith(normalizedMount + "/");
905
+ if (matches) {
906
+ if (webApp.config.injectMooseUtils !== false) {
907
+ const queryClient = new QueryClient(clickhouseClient, pathname);
908
+ req.moose = {
909
+ client: new MooseClient(queryClient, temporalClient),
910
+ sql,
911
+ jwt: jwtPayload
912
+ };
1438
913
  }
1439
- let apiPath = pathname;
1440
- if (pathname.startsWith("/api/")) {
1441
- apiPath = pathname.substring(4);
1442
- } else if (pathname.startsWith("/consumption/")) {
1443
- apiPath = pathname.substring(13);
914
+ let proxiedUrl = req.url;
915
+ if (normalizedMount !== "/") {
916
+ const pathWithoutMount = pathname.substring(normalizedMount.length) || "/";
917
+ proxiedUrl = pathWithoutMount + url.search;
1444
918
  }
1445
- if (apiPath !== pathname) {
919
+ try {
1446
920
  const modifiedReq = Object.assign(
1447
921
  Object.create(Object.getPrototypeOf(req)),
1448
922
  req,
1449
923
  {
1450
- url: apiPath + url.search
924
+ url: proxiedUrl
1451
925
  }
1452
926
  );
1453
- await apiRequestHandler(modifiedReq, res);
927
+ await webApp.handler(modifiedReq, res);
1454
928
  return;
1455
- }
1456
- res.writeHead(404, { "Content-Type": "application/json" });
1457
- res.end(JSON.stringify({ error: "Not Found" }));
1458
- httpLogger(req, res, start);
1459
- };
1460
- };
1461
- runApis = async (config) => {
1462
- const apisCluster = new Cluster({
1463
- maxWorkerCount: (config.workerCount ?? 0) > 0 ? config.workerCount : void 0,
1464
- workerStart: async () => {
1465
- let temporalClient;
1466
- if (config.temporalConfig) {
1467
- temporalClient = await getTemporalClient(
1468
- config.temporalConfig.url,
1469
- config.temporalConfig.namespace,
1470
- config.temporalConfig.clientCert,
1471
- config.temporalConfig.clientKey,
1472
- config.temporalConfig.apiKey
1473
- );
1474
- }
1475
- const clickhouseClient = getClickhouseClient(
1476
- toClientConfig2(config.clickhouseConfig)
1477
- );
1478
- let publicKey;
1479
- if (config.jwtConfig?.secret) {
1480
- console.log("Importing JWT public key...");
1481
- publicKey = await jose.importSPKI(config.jwtConfig.secret, "RS256");
929
+ } catch (error) {
930
+ console.error(`Error in WebApp ${webApp.name}:`, error);
931
+ if (!res.headersSent) {
932
+ res.writeHead(500, { "Content-Type": "application/json" });
933
+ res.end(JSON.stringify({ error: "Internal Server Error" }));
1482
934
  }
1483
- const runtimeQueryClient = new QueryClient(clickhouseClient, "runtime");
1484
- globalThis._mooseRuntimeContext = {
1485
- client: new MooseClient(runtimeQueryClient, temporalClient)
1486
- };
1487
- const server = http2.createServer(
1488
- await createMainRouter(
1489
- publicKey,
1490
- clickhouseClient,
1491
- temporalClient,
1492
- config.apisDir,
1493
- config.enforceAuth,
1494
- config.isDmv2,
1495
- config.jwtConfig
1496
- )
1497
- );
1498
- const port = config.proxyPort !== void 0 ? config.proxyPort : 4001;
1499
- server.listen(port, "localhost", () => {
1500
- console.log(`Server running on port ${port}`);
1501
- });
1502
- return server;
1503
- },
1504
- workerStop: async (server) => {
1505
- return new Promise((resolve2) => {
1506
- server.close(() => resolve2());
1507
- });
935
+ return;
936
+ }
937
+ }
938
+ }
939
+ let apiPath = pathname;
940
+ if (pathname.startsWith("/api/")) {
941
+ apiPath = pathname.substring(4);
942
+ } else if (pathname.startsWith("/consumption/")) {
943
+ apiPath = pathname.substring(13);
944
+ }
945
+ if (apiPath !== pathname) {
946
+ const modifiedReq = Object.assign(
947
+ Object.create(Object.getPrototypeOf(req)),
948
+ req,
949
+ {
950
+ url: apiPath + url.search
1508
951
  }
952
+ );
953
+ await apiRequestHandler(modifiedReq, res);
954
+ return;
955
+ }
956
+ res.writeHead(404, { "Content-Type": "application/json" });
957
+ res.end(JSON.stringify({ error: "Not Found" }));
958
+ httpLogger(req, res, start);
959
+ };
960
+ };
961
+ var runApis = async (config) => {
962
+ const apisCluster = new Cluster({
963
+ maxWorkerCount: (config.workerCount ?? 0) > 0 ? config.workerCount : void 0,
964
+ workerStart: async () => {
965
+ let temporalClient;
966
+ if (config.temporalConfig) {
967
+ temporalClient = await getTemporalClient(
968
+ config.temporalConfig.url,
969
+ config.temporalConfig.namespace,
970
+ config.temporalConfig.clientCert,
971
+ config.temporalConfig.clientKey,
972
+ config.temporalConfig.apiKey
973
+ );
974
+ }
975
+ const clickhouseClient = getClickhouseClient(
976
+ toClientConfig(config.clickhouseConfig)
977
+ );
978
+ let publicKey;
979
+ if (config.jwtConfig?.secret) {
980
+ console.log("Importing JWT public key...");
981
+ publicKey = await jose.importSPKI(config.jwtConfig.secret, "RS256");
982
+ }
983
+ const server = http2.createServer(
984
+ await createMainRouter(
985
+ publicKey,
986
+ clickhouseClient,
987
+ temporalClient,
988
+ config.apisDir,
989
+ config.enforceAuth,
990
+ config.isDmv2,
991
+ config.jwtConfig
992
+ )
993
+ );
994
+ const port = config.proxyPort !== void 0 ? config.proxyPort : 4001;
995
+ server.listen(port, "localhost", () => {
996
+ console.log(`Server running on port ${port}`);
1509
997
  });
1510
- apisCluster.start();
1511
- };
1512
- }
1513
- });
998
+ return server;
999
+ },
1000
+ workerStop: async (server) => {
1001
+ return new Promise((resolve2) => {
1002
+ server.close(() => resolve2());
1003
+ });
1004
+ }
1005
+ });
1006
+ apisCluster.start();
1007
+ };
1514
1008
 
1515
1009
  // src/clients/redisClient.ts
1516
1010
  import { createClient as createClient2 } from "redis";
1517
- var init_redisClient = __esm({
1518
- "src/clients/redisClient.ts"() {
1519
- "use strict";
1520
- }
1521
- });
1011
+
1012
+ // src/consumption-apis/standalone.ts
1013
+ init_commons();
1014
+
1015
+ // src/utilities/dataParser.ts
1016
+ import { parse } from "csv-parse";
1522
1017
 
1523
1018
  // src/utilities/json.ts
1019
+ var STRING_DATE_ANNOTATION = "stringDate";
1524
1020
  function isNullableType(dt) {
1525
1021
  return typeof dt === "object" && dt !== null && "nullable" in dt && typeof dt.nullable !== "undefined";
1526
1022
  }
@@ -1639,78 +1135,38 @@ function mutateParsedJson(data, fieldMutations) {
1639
1135
  }
1640
1136
  applyFieldMutations(data, fieldMutations);
1641
1137
  }
1642
- var STRING_DATE_ANNOTATION;
1643
- var init_json = __esm({
1644
- "src/utilities/json.ts"() {
1645
- "use strict";
1646
- STRING_DATE_ANNOTATION = "stringDate";
1647
- }
1648
- });
1649
1138
 
1650
1139
  // src/utilities/dataParser.ts
1651
- import { parse as parse2 } from "csv-parse";
1652
- var CSV_DELIMITERS, DEFAULT_CSV_CONFIG;
1653
- var init_dataParser = __esm({
1654
- "src/utilities/dataParser.ts"() {
1655
- "use strict";
1656
- init_json();
1657
- CSV_DELIMITERS = {
1658
- COMMA: ",",
1659
- TAB: " ",
1660
- SEMICOLON: ";",
1661
- PIPE: "|"
1662
- };
1663
- DEFAULT_CSV_CONFIG = {
1664
- delimiter: CSV_DELIMITERS.COMMA,
1665
- columns: true,
1666
- skipEmptyLines: true,
1667
- trim: true
1668
- };
1669
- }
1670
- });
1671
-
1672
- // src/utilities/index.ts
1673
- var init_utilities = __esm({
1674
- "src/utilities/index.ts"() {
1675
- "use strict";
1676
- init_dataParser();
1677
- }
1678
- });
1679
-
1680
- // src/connectors/dataSource.ts
1681
- var init_dataSource = __esm({
1682
- "src/connectors/dataSource.ts"() {
1683
- "use strict";
1684
- }
1685
- });
1686
-
1687
- // src/index.ts
1688
- var init_index = __esm({
1689
- "src/index.ts"() {
1690
- "use strict";
1691
- init_browserCompatible();
1692
- init_helpers();
1693
- init_commons();
1694
- init_secrets();
1695
- init_helpers2();
1696
- init_webAppHelpers();
1697
- init_task();
1698
- init_runner();
1699
- init_redisClient();
1700
- init_helpers2();
1701
- init_standalone();
1702
- init_sqlHelpers();
1703
- init_utilities();
1704
- init_dataSource();
1705
- init_types();
1706
- }
1707
- });
1140
+ var CSV_DELIMITERS = {
1141
+ COMMA: ",",
1142
+ TAB: " ",
1143
+ SEMICOLON: ";",
1144
+ PIPE: "|"
1145
+ };
1146
+ var DEFAULT_CSV_CONFIG = {
1147
+ delimiter: CSV_DELIMITERS.COMMA,
1148
+ columns: true,
1149
+ skipEmptyLines: true,
1150
+ trim: true
1151
+ };
1708
1152
 
1709
1153
  // src/dmv2/internal.ts
1710
- import process2 from "process";
1154
+ init_commons();
1711
1155
  function getSourceDir() {
1712
1156
  return process2.env.MOOSE_SOURCE_DIR || "app";
1713
1157
  }
1158
+ var moose_internal = {
1159
+ tables: /* @__PURE__ */ new Map(),
1160
+ streams: /* @__PURE__ */ new Map(),
1161
+ ingestApis: /* @__PURE__ */ new Map(),
1162
+ apis: /* @__PURE__ */ new Map(),
1163
+ sqlResources: /* @__PURE__ */ new Map(),
1164
+ workflows: /* @__PURE__ */ new Map(),
1165
+ webApps: /* @__PURE__ */ new Map(),
1166
+ materializedViews: /* @__PURE__ */ new Map(),
1167
+ customViews: /* @__PURE__ */ new Map()
1168
+ };
1169
+ var defaultRetentionPeriod = 60 * 60 * 24 * 7;
1714
1170
  function isS3QueueConfig(config) {
1715
1171
  return "engine" in config && config.engine === "S3Queue" /* S3Queue */;
1716
1172
  }
@@ -1947,386 +1403,388 @@ function convertTableConfigToEngineConfig(config) {
1947
1403
  }
1948
1404
  return void 0;
1949
1405
  }
1950
- function findTaskInTree(task, targetName) {
1951
- if (task.name === targetName) {
1952
- return task;
1953
- }
1954
- if (task.config.onComplete?.length) {
1955
- for (const childTask of task.config.onComplete) {
1956
- const found = findTaskInTree(childTask, targetName);
1957
- if (found) {
1958
- return found;
1406
+ var toInfraMap = (registry) => {
1407
+ const tables = {};
1408
+ const topics = {};
1409
+ const ingestApis = {};
1410
+ const apis = {};
1411
+ const sqlResources = {};
1412
+ const workflows = {};
1413
+ const webApps = {};
1414
+ const materializedViews = {};
1415
+ const customViews = {};
1416
+ registry.tables.forEach((table) => {
1417
+ const id = table.config.version ? `${table.name}_${table.config.version}` : table.name;
1418
+ let metadata = table.metadata;
1419
+ if (!metadata && table.config && table.pipelineParent) {
1420
+ metadata = table.pipelineParent.metadata;
1421
+ }
1422
+ const engineConfig = convertTableConfigToEngineConfig(table.config);
1423
+ let tableSettings = void 0;
1424
+ if (table.config.settings) {
1425
+ tableSettings = Object.entries(table.config.settings).reduce(
1426
+ (acc, [key, value]) => {
1427
+ if (value !== void 0) {
1428
+ acc[key] = String(value);
1429
+ }
1430
+ return acc;
1431
+ },
1432
+ {}
1433
+ );
1434
+ }
1435
+ if (engineConfig?.engine === "S3Queue") {
1436
+ if (!tableSettings) {
1437
+ tableSettings = {};
1438
+ }
1439
+ if (!tableSettings.mode) {
1440
+ tableSettings.mode = "unordered";
1959
1441
  }
1960
1442
  }
1961
- }
1962
- return void 0;
1963
- }
1964
- var moose_internal, defaultRetentionPeriod, toInfraMap, getMooseInternal, dumpMooseInternal, loadIndex, getStreamingFunctions, getApis2, getWorkflows2, getTaskForWorkflow, getWebApps2;
1965
- var init_internal = __esm({
1966
- "src/dmv2/internal.ts"() {
1967
- "use strict";
1968
- init_index();
1969
- init_commons();
1970
- moose_internal = {
1971
- tables: /* @__PURE__ */ new Map(),
1972
- streams: /* @__PURE__ */ new Map(),
1973
- ingestApis: /* @__PURE__ */ new Map(),
1974
- apis: /* @__PURE__ */ new Map(),
1975
- sqlResources: /* @__PURE__ */ new Map(),
1976
- workflows: /* @__PURE__ */ new Map(),
1977
- webApps: /* @__PURE__ */ new Map()
1443
+ const hasOrderByFields = "orderByFields" in table.config && Array.isArray(table.config.orderByFields) && table.config.orderByFields.length > 0;
1444
+ const hasOrderByExpression = "orderByExpression" in table.config && typeof table.config.orderByExpression === "string" && table.config.orderByExpression.length > 0;
1445
+ if (hasOrderByFields && hasOrderByExpression) {
1446
+ throw new Error(
1447
+ `Table ${table.name}: Provide either orderByFields or orderByExpression, not both.`
1448
+ );
1449
+ }
1450
+ const orderBy = hasOrderByExpression && "orderByExpression" in table.config ? table.config.orderByExpression ?? "" : "orderByFields" in table.config ? table.config.orderByFields ?? [] : [];
1451
+ tables[id] = {
1452
+ name: table.name,
1453
+ columns: table.columnArray,
1454
+ orderBy,
1455
+ partitionBy: "partitionBy" in table.config ? table.config.partitionBy : void 0,
1456
+ sampleByExpression: "sampleByExpression" in table.config ? table.config.sampleByExpression : void 0,
1457
+ primaryKeyExpression: "primaryKeyExpression" in table.config ? table.config.primaryKeyExpression : void 0,
1458
+ engineConfig,
1459
+ version: table.config.version,
1460
+ metadata,
1461
+ lifeCycle: table.config.lifeCycle,
1462
+ // Map 'settings' to 'tableSettings' for internal use
1463
+ tableSettings: tableSettings && Object.keys(tableSettings).length > 0 ? tableSettings : void 0,
1464
+ indexes: table.config.indexes?.map((i) => ({
1465
+ ...i,
1466
+ granularity: i.granularity === void 0 ? 1 : i.granularity,
1467
+ arguments: i.arguments === void 0 ? [] : i.arguments
1468
+ })) || [],
1469
+ ttl: table.config.ttl,
1470
+ database: table.config.database,
1471
+ cluster: table.config.cluster
1978
1472
  };
1979
- defaultRetentionPeriod = 60 * 60 * 24 * 7;
1980
- toInfraMap = (registry) => {
1981
- const tables = {};
1982
- const topics = {};
1983
- const ingestApis = {};
1984
- const apis = {};
1985
- const sqlResources = {};
1986
- const workflows = {};
1987
- const webApps = {};
1988
- registry.tables.forEach((table) => {
1989
- const id = table.config.version ? `${table.name}_${table.config.version}` : table.name;
1990
- let metadata = table.metadata;
1991
- if (!metadata && table.config && table.pipelineParent) {
1992
- metadata = table.pipelineParent.metadata;
1993
- }
1994
- const engineConfig = convertTableConfigToEngineConfig(table.config);
1995
- let tableSettings = void 0;
1996
- if (table.config.settings) {
1997
- tableSettings = Object.entries(table.config.settings).reduce(
1998
- (acc, [key, value]) => {
1999
- if (value !== void 0) {
2000
- acc[key] = String(value);
2001
- }
2002
- return acc;
2003
- },
2004
- {}
2005
- );
2006
- }
2007
- if (engineConfig?.engine === "S3Queue") {
2008
- if (!tableSettings) {
2009
- tableSettings = {};
2010
- }
2011
- if (!tableSettings.mode) {
2012
- tableSettings.mode = "unordered";
2013
- }
2014
- }
2015
- const hasOrderByFields = "orderByFields" in table.config && Array.isArray(table.config.orderByFields) && table.config.orderByFields.length > 0;
2016
- const hasOrderByExpression = "orderByExpression" in table.config && typeof table.config.orderByExpression === "string" && table.config.orderByExpression.length > 0;
2017
- if (hasOrderByFields && hasOrderByExpression) {
2018
- throw new Error(
2019
- `Table ${table.name}: Provide either orderByFields or orderByExpression, not both.`
2020
- );
2021
- }
2022
- const orderBy = hasOrderByExpression && "orderByExpression" in table.config ? table.config.orderByExpression ?? "" : "orderByFields" in table.config ? table.config.orderByFields ?? [] : [];
2023
- tables[id] = {
2024
- name: table.name,
2025
- columns: table.columnArray,
2026
- orderBy,
2027
- partitionBy: "partitionBy" in table.config ? table.config.partitionBy : void 0,
2028
- sampleByExpression: "sampleByExpression" in table.config ? table.config.sampleByExpression : void 0,
2029
- primaryKeyExpression: "primaryKeyExpression" in table.config ? table.config.primaryKeyExpression : void 0,
2030
- engineConfig,
2031
- version: table.config.version,
2032
- metadata,
2033
- lifeCycle: table.config.lifeCycle,
2034
- // Map 'settings' to 'tableSettings' for internal use
2035
- tableSettings: tableSettings && Object.keys(tableSettings).length > 0 ? tableSettings : void 0,
2036
- indexes: table.config.indexes?.map((i) => ({
2037
- ...i,
2038
- granularity: i.granularity === void 0 ? 1 : i.granularity,
2039
- arguments: i.arguments === void 0 ? [] : i.arguments
2040
- })) || [],
2041
- ttl: table.config.ttl,
2042
- database: table.config.database,
2043
- cluster: table.config.cluster
2044
- };
2045
- });
2046
- registry.streams.forEach((stream) => {
2047
- let metadata = stream.metadata;
2048
- if (!metadata && stream.config && stream.pipelineParent) {
2049
- metadata = stream.pipelineParent.metadata;
2050
- }
2051
- const transformationTargets = [];
2052
- const consumers = [];
2053
- stream._transformations.forEach((transforms, destinationName) => {
2054
- transforms.forEach(([destination, _, config]) => {
2055
- transformationTargets.push({
2056
- kind: "stream",
2057
- name: destinationName,
2058
- version: config.version,
2059
- metadata: config.metadata,
2060
- sourceFile: config.sourceFile
2061
- });
2062
- });
2063
- });
2064
- stream._consumers.forEach((consumer) => {
2065
- consumers.push({
2066
- version: consumer.config.version,
2067
- sourceFile: consumer.config.sourceFile
2068
- });
1473
+ });
1474
+ registry.streams.forEach((stream) => {
1475
+ let metadata = stream.metadata;
1476
+ if (!metadata && stream.config && stream.pipelineParent) {
1477
+ metadata = stream.pipelineParent.metadata;
1478
+ }
1479
+ const transformationTargets = [];
1480
+ const consumers = [];
1481
+ stream._transformations.forEach((transforms, destinationName) => {
1482
+ transforms.forEach(([destination, _, config]) => {
1483
+ transformationTargets.push({
1484
+ kind: "stream",
1485
+ name: destinationName,
1486
+ version: config.version,
1487
+ metadata: config.metadata,
1488
+ sourceFile: config.sourceFile
2069
1489
  });
2070
- topics[stream.name] = {
2071
- name: stream.name,
2072
- columns: stream.columnArray,
2073
- targetTable: stream.config.destination?.name,
2074
- targetTableVersion: stream.config.destination?.config.version,
2075
- retentionPeriod: stream.config.retentionPeriod ?? defaultRetentionPeriod,
2076
- partitionCount: stream.config.parallelism ?? 1,
2077
- version: stream.config.version,
2078
- transformationTargets,
2079
- hasMultiTransform: stream._multipleTransformations === void 0,
2080
- consumers,
2081
- metadata,
2082
- lifeCycle: stream.config.lifeCycle,
2083
- schemaConfig: stream.config.schemaConfig
2084
- };
2085
- });
2086
- registry.ingestApis.forEach((api) => {
2087
- let metadata = api.metadata;
2088
- if (!metadata && api.config && api.pipelineParent) {
2089
- metadata = api.pipelineParent.metadata;
2090
- }
2091
- ingestApis[api.name] = {
2092
- name: api.name,
2093
- columns: api.columnArray,
2094
- version: api.config.version,
2095
- path: api.config.path,
2096
- writeTo: {
2097
- kind: "stream",
2098
- name: api.config.destination.name
2099
- },
2100
- deadLetterQueue: api.config.deadLetterQueue?.name,
2101
- metadata,
2102
- schema: api.schema,
2103
- allowExtraFields: api.allowExtraFields
2104
- };
2105
- });
2106
- registry.apis.forEach((api, key) => {
2107
- const rustKey = api.config.version ? `${api.name}:${api.config.version}` : api.name;
2108
- apis[rustKey] = {
2109
- name: api.name,
2110
- queryParams: api.columnArray,
2111
- responseSchema: api.responseSchema,
2112
- version: api.config.version,
2113
- path: api.config.path,
2114
- metadata: api.metadata
2115
- };
2116
- });
2117
- registry.sqlResources.forEach((sqlResource) => {
2118
- sqlResources[sqlResource.name] = {
2119
- name: sqlResource.name,
2120
- setup: sqlResource.setup,
2121
- teardown: sqlResource.teardown,
2122
- sourceFile: sqlResource.sourceFile,
2123
- sourceLine: sqlResource.sourceLine,
2124
- sourceColumn: sqlResource.sourceColumn,
2125
- pullsDataFrom: sqlResource.pullsDataFrom.map((r) => {
2126
- if (r.kind === "OlapTable") {
2127
- const table = r;
2128
- const id = table.config.version ? `${table.name}_${table.config.version}` : table.name;
2129
- return {
2130
- id,
2131
- kind: "Table"
2132
- };
2133
- } else if (r.kind === "SqlResource") {
2134
- const resource = r;
2135
- return {
2136
- id: resource.name,
2137
- kind: "SqlResource"
2138
- };
2139
- } else {
2140
- throw new Error(`Unknown sql resource dependency type: ${r}`);
2141
- }
2142
- }),
2143
- pushesDataTo: sqlResource.pushesDataTo.map((r) => {
2144
- if (r.kind === "OlapTable") {
2145
- const table = r;
2146
- const id = table.config.version ? `${table.name}_${table.config.version}` : table.name;
2147
- return {
2148
- id,
2149
- kind: "Table"
2150
- };
2151
- } else if (r.kind === "SqlResource") {
2152
- const resource = r;
2153
- return {
2154
- id: resource.name,
2155
- kind: "SqlResource"
2156
- };
2157
- } else {
2158
- throw new Error(`Unknown sql resource dependency type: ${r}`);
2159
- }
2160
- })
2161
- };
2162
- });
2163
- registry.workflows.forEach((workflow) => {
2164
- workflows[workflow.name] = {
2165
- name: workflow.name,
2166
- retries: workflow.config.retries,
2167
- timeout: workflow.config.timeout,
2168
- schedule: workflow.config.schedule
2169
- };
2170
1490
  });
2171
- registry.webApps.forEach((webApp) => {
2172
- webApps[webApp.name] = {
2173
- name: webApp.name,
2174
- mountPath: webApp.config.mountPath || "/",
2175
- metadata: webApp.config.metadata
2176
- };
1491
+ });
1492
+ stream._consumers.forEach((consumer) => {
1493
+ consumers.push({
1494
+ version: consumer.config.version,
1495
+ sourceFile: consumer.config.sourceFile
2177
1496
  });
2178
- return {
2179
- topics,
2180
- tables,
2181
- ingestApis,
2182
- apis,
2183
- sqlResources,
2184
- workflows,
2185
- webApps
2186
- };
1497
+ });
1498
+ topics[stream.name] = {
1499
+ name: stream.name,
1500
+ columns: stream.columnArray,
1501
+ targetTable: stream.config.destination?.name,
1502
+ targetTableVersion: stream.config.destination?.config.version,
1503
+ retentionPeriod: stream.config.retentionPeriod ?? defaultRetentionPeriod,
1504
+ partitionCount: stream.config.parallelism ?? 1,
1505
+ version: stream.config.version,
1506
+ transformationTargets,
1507
+ hasMultiTransform: stream._multipleTransformations === void 0,
1508
+ consumers,
1509
+ metadata,
1510
+ lifeCycle: stream.config.lifeCycle,
1511
+ schemaConfig: stream.config.schemaConfig
2187
1512
  };
2188
- getMooseInternal = () => globalThis.moose_internal;
2189
- if (getMooseInternal() === void 0) {
2190
- globalThis.moose_internal = moose_internal;
1513
+ });
1514
+ registry.ingestApis.forEach((api) => {
1515
+ let metadata = api.metadata;
1516
+ if (!metadata && api.config && api.pipelineParent) {
1517
+ metadata = api.pipelineParent.metadata;
2191
1518
  }
2192
- dumpMooseInternal = async () => {
2193
- loadIndex();
2194
- console.log(
2195
- "___MOOSE_STUFF___start",
2196
- JSON.stringify(toInfraMap(getMooseInternal())),
2197
- "end___MOOSE_STUFF___"
2198
- );
1519
+ ingestApis[api.name] = {
1520
+ name: api.name,
1521
+ columns: api.columnArray,
1522
+ version: api.config.version,
1523
+ path: api.config.path,
1524
+ writeTo: {
1525
+ kind: "stream",
1526
+ name: api.config.destination.name
1527
+ },
1528
+ deadLetterQueue: api.config.deadLetterQueue?.name,
1529
+ metadata,
1530
+ schema: api.schema,
1531
+ allowExtraFields: api.allowExtraFields
2199
1532
  };
2200
- loadIndex = () => {
2201
- const registry = getMooseInternal();
2202
- registry.tables.clear();
2203
- registry.streams.clear();
2204
- registry.ingestApis.clear();
2205
- registry.apis.clear();
2206
- registry.sqlResources.clear();
2207
- registry.workflows.clear();
2208
- registry.webApps.clear();
2209
- const appDir = `${process2.cwd()}/${getSourceDir()}`;
2210
- Object.keys(__require.cache).forEach((key) => {
2211
- if (key.startsWith(appDir)) {
2212
- delete __require.cache[key];
1533
+ });
1534
+ registry.apis.forEach((api, key) => {
1535
+ const rustKey = api.config.version ? `${api.name}:${api.config.version}` : api.name;
1536
+ apis[rustKey] = {
1537
+ name: api.name,
1538
+ queryParams: api.columnArray,
1539
+ responseSchema: api.responseSchema,
1540
+ version: api.config.version,
1541
+ path: api.config.path,
1542
+ metadata: api.metadata
1543
+ };
1544
+ });
1545
+ registry.sqlResources.forEach((sqlResource) => {
1546
+ sqlResources[sqlResource.name] = {
1547
+ name: sqlResource.name,
1548
+ setup: sqlResource.setup,
1549
+ teardown: sqlResource.teardown,
1550
+ sourceFile: sqlResource.sourceFile,
1551
+ sourceLine: sqlResource.sourceLine,
1552
+ sourceColumn: sqlResource.sourceColumn,
1553
+ pullsDataFrom: sqlResource.pullsDataFrom.map((r) => {
1554
+ if (r.kind === "OlapTable") {
1555
+ const table = r;
1556
+ const id = table.config.version ? `${table.name}_${table.config.version}` : table.name;
1557
+ return {
1558
+ id,
1559
+ kind: "Table"
1560
+ };
1561
+ } else if (r.kind === "SqlResource") {
1562
+ const resource = r;
1563
+ return {
1564
+ id: resource.name,
1565
+ kind: "SqlResource"
1566
+ };
1567
+ } else {
1568
+ throw new Error(`Unknown sql resource dependency type: ${r}`);
2213
1569
  }
2214
- });
2215
- try {
2216
- __require(`${process2.cwd()}/${getSourceDir()}/index.ts`);
2217
- } catch (error) {
2218
- let hint;
2219
- const details = error instanceof Error ? error.message : String(error);
2220
- if (details.includes("ERR_REQUIRE_ESM") || details.includes("ES Module")) {
2221
- hint = "The file or its dependencies are ESM-only. Switch to packages that dual-support CJS & ESM, or upgrade to Node 22.12+. If you must use Node 20, you may try Node 20.19\n\n";
1570
+ }),
1571
+ pushesDataTo: sqlResource.pushesDataTo.map((r) => {
1572
+ if (r.kind === "OlapTable") {
1573
+ const table = r;
1574
+ const id = table.config.version ? `${table.name}_${table.config.version}` : table.name;
1575
+ return {
1576
+ id,
1577
+ kind: "Table"
1578
+ };
1579
+ } else if (r.kind === "SqlResource") {
1580
+ const resource = r;
1581
+ return {
1582
+ id: resource.name,
1583
+ kind: "SqlResource"
1584
+ };
1585
+ } else {
1586
+ throw new Error(`Unknown sql resource dependency type: ${r}`);
2222
1587
  }
2223
- const errorMsg = `${hint ?? ""}${details}`;
2224
- const cause = error instanceof Error ? error : void 0;
2225
- throw new Error(errorMsg, { cause });
2226
- }
1588
+ })
2227
1589
  };
2228
- getStreamingFunctions = async () => {
2229
- loadIndex();
2230
- const registry = getMooseInternal();
2231
- const transformFunctions = /* @__PURE__ */ new Map();
2232
- registry.streams.forEach((stream) => {
2233
- stream._transformations.forEach((transforms, destinationName) => {
2234
- transforms.forEach(([_, transform, config]) => {
2235
- const transformFunctionKey = `${stream.name}_${destinationName}${config.version ? `_${config.version}` : ""}`;
2236
- compilerLog(`getStreamingFunctions: ${transformFunctionKey}`);
2237
- transformFunctions.set(transformFunctionKey, [
2238
- transform,
2239
- config,
2240
- stream.columnArray
2241
- ]);
2242
- });
2243
- });
2244
- stream._consumers.forEach((consumer) => {
2245
- const consumerFunctionKey = `${stream.name}_<no-target>${consumer.config.version ? `_${consumer.config.version}` : ""}`;
2246
- transformFunctions.set(consumerFunctionKey, [
2247
- consumer.consumer,
2248
- consumer.config,
2249
- stream.columnArray
2250
- ]);
2251
- });
2252
- });
2253
- return transformFunctions;
1590
+ });
1591
+ registry.workflows.forEach((workflow) => {
1592
+ workflows[workflow.name] = {
1593
+ name: workflow.name,
1594
+ retries: workflow.config.retries,
1595
+ timeout: workflow.config.timeout,
1596
+ schedule: workflow.config.schedule
2254
1597
  };
2255
- getApis2 = async () => {
2256
- loadIndex();
2257
- const apiFunctions = /* @__PURE__ */ new Map();
2258
- const registry = getMooseInternal();
2259
- const versionCountByName = /* @__PURE__ */ new Map();
2260
- const nameToSoleVersionHandler = /* @__PURE__ */ new Map();
2261
- registry.apis.forEach((api, key) => {
2262
- const handler = api.getHandler();
2263
- apiFunctions.set(key, handler);
2264
- if (!api.config.version) {
2265
- if (!apiFunctions.has(api.name)) {
2266
- apiFunctions.set(api.name, handler);
2267
- }
2268
- nameToSoleVersionHandler.delete(api.name);
2269
- versionCountByName.delete(api.name);
2270
- } else if (!apiFunctions.has(api.name)) {
2271
- const count = (versionCountByName.get(api.name) ?? 0) + 1;
2272
- versionCountByName.set(api.name, count);
2273
- if (count === 1) {
2274
- nameToSoleVersionHandler.set(api.name, handler);
2275
- } else {
2276
- nameToSoleVersionHandler.delete(api.name);
2277
- }
2278
- }
2279
- });
2280
- nameToSoleVersionHandler.forEach((handler, name) => {
2281
- if (!apiFunctions.has(name)) {
2282
- apiFunctions.set(name, handler);
2283
- }
2284
- });
2285
- return apiFunctions;
1598
+ });
1599
+ registry.webApps.forEach((webApp) => {
1600
+ webApps[webApp.name] = {
1601
+ name: webApp.name,
1602
+ mountPath: webApp.config.mountPath || "/",
1603
+ metadata: webApp.config.metadata
1604
+ };
1605
+ });
1606
+ registry.materializedViews.forEach((mv) => {
1607
+ materializedViews[mv.name] = {
1608
+ name: mv.name,
1609
+ selectSql: mv.selectSql,
1610
+ sourceTables: mv.sourceTables,
1611
+ targetTable: mv.targetTable.name,
1612
+ targetDatabase: mv.targetTable.config.database,
1613
+ sourceFile: mv.sourceFile
2286
1614
  };
2287
- getWorkflows2 = async () => {
2288
- loadIndex();
2289
- const registry = getMooseInternal();
2290
- return registry.workflows;
1615
+ });
1616
+ registry.customViews.forEach((view) => {
1617
+ customViews[view.name] = {
1618
+ name: view.name,
1619
+ selectSql: view.selectSql,
1620
+ sourceTables: view.sourceTables,
1621
+ sourceFile: view.sourceFile
2291
1622
  };
2292
- getTaskForWorkflow = async (workflowName, taskName) => {
2293
- const workflows = await getWorkflows2();
2294
- const workflow = workflows.get(workflowName);
2295
- if (!workflow) {
2296
- throw new Error(`Workflow ${workflowName} not found`);
1623
+ });
1624
+ return {
1625
+ topics,
1626
+ tables,
1627
+ ingestApis,
1628
+ apis,
1629
+ sqlResources,
1630
+ workflows,
1631
+ webApps,
1632
+ materializedViews,
1633
+ customViews
1634
+ };
1635
+ };
1636
+ var getMooseInternal = () => globalThis.moose_internal;
1637
+ if (getMooseInternal() === void 0) {
1638
+ globalThis.moose_internal = moose_internal;
1639
+ }
1640
+ var dumpMooseInternal = async () => {
1641
+ loadIndex();
1642
+ console.log(
1643
+ "___MOOSE_STUFF___start",
1644
+ JSON.stringify(toInfraMap(getMooseInternal())),
1645
+ "end___MOOSE_STUFF___"
1646
+ );
1647
+ };
1648
+ var loadIndex = () => {
1649
+ const registry = getMooseInternal();
1650
+ registry.tables.clear();
1651
+ registry.streams.clear();
1652
+ registry.ingestApis.clear();
1653
+ registry.apis.clear();
1654
+ registry.sqlResources.clear();
1655
+ registry.workflows.clear();
1656
+ registry.webApps.clear();
1657
+ registry.materializedViews.clear();
1658
+ registry.customViews.clear();
1659
+ const appDir = `${process2.cwd()}/${getSourceDir()}`;
1660
+ Object.keys(__require.cache).forEach((key) => {
1661
+ if (key.startsWith(appDir)) {
1662
+ delete __require.cache[key];
1663
+ }
1664
+ });
1665
+ try {
1666
+ __require(`${process2.cwd()}/${getSourceDir()}/index.ts`);
1667
+ } catch (error) {
1668
+ let hint;
1669
+ const details = error instanceof Error ? error.message : String(error);
1670
+ if (details.includes("ERR_REQUIRE_ESM") || details.includes("ES Module")) {
1671
+ hint = "The file or its dependencies are ESM-only. Switch to packages that dual-support CJS & ESM, or upgrade to Node 22.12+. If you must use Node 20, you may try Node 20.19\n\n";
1672
+ }
1673
+ const errorMsg = `${hint ?? ""}${details}`;
1674
+ const cause = error instanceof Error ? error : void 0;
1675
+ throw new Error(errorMsg, { cause });
1676
+ }
1677
+ };
1678
+ var getStreamingFunctions = async () => {
1679
+ loadIndex();
1680
+ const registry = getMooseInternal();
1681
+ const transformFunctions = /* @__PURE__ */ new Map();
1682
+ registry.streams.forEach((stream) => {
1683
+ stream._transformations.forEach((transforms, destinationName) => {
1684
+ transforms.forEach(([_, transform, config]) => {
1685
+ const transformFunctionKey = `${stream.name}_${destinationName}${config.version ? `_${config.version}` : ""}`;
1686
+ compilerLog(`getStreamingFunctions: ${transformFunctionKey}`);
1687
+ transformFunctions.set(transformFunctionKey, [
1688
+ transform,
1689
+ config,
1690
+ stream.columnArray
1691
+ ]);
1692
+ });
1693
+ });
1694
+ stream._consumers.forEach((consumer) => {
1695
+ const consumerFunctionKey = `${stream.name}_<no-target>${consumer.config.version ? `_${consumer.config.version}` : ""}`;
1696
+ transformFunctions.set(consumerFunctionKey, [
1697
+ consumer.consumer,
1698
+ consumer.config,
1699
+ stream.columnArray
1700
+ ]);
1701
+ });
1702
+ });
1703
+ return transformFunctions;
1704
+ };
1705
+ var getApis2 = async () => {
1706
+ loadIndex();
1707
+ const apiFunctions = /* @__PURE__ */ new Map();
1708
+ const registry = getMooseInternal();
1709
+ const versionCountByName = /* @__PURE__ */ new Map();
1710
+ const nameToSoleVersionHandler = /* @__PURE__ */ new Map();
1711
+ registry.apis.forEach((api, key) => {
1712
+ const handler = api.getHandler();
1713
+ apiFunctions.set(key, handler);
1714
+ if (!api.config.version) {
1715
+ if (!apiFunctions.has(api.name)) {
1716
+ apiFunctions.set(api.name, handler);
2297
1717
  }
2298
- const task = findTaskInTree(
2299
- workflow.config.startingTask,
2300
- taskName
2301
- );
2302
- if (!task) {
2303
- throw new Error(`Task ${taskName} not found in workflow ${workflowName}`);
1718
+ nameToSoleVersionHandler.delete(api.name);
1719
+ versionCountByName.delete(api.name);
1720
+ } else if (!apiFunctions.has(api.name)) {
1721
+ const count = (versionCountByName.get(api.name) ?? 0) + 1;
1722
+ versionCountByName.set(api.name, count);
1723
+ if (count === 1) {
1724
+ nameToSoleVersionHandler.set(api.name, handler);
1725
+ } else {
1726
+ nameToSoleVersionHandler.delete(api.name);
2304
1727
  }
2305
- return task;
2306
- };
2307
- getWebApps2 = async () => {
2308
- loadIndex();
2309
- return getMooseInternal().webApps;
2310
- };
1728
+ }
1729
+ });
1730
+ nameToSoleVersionHandler.forEach((handler, name) => {
1731
+ if (!apiFunctions.has(name)) {
1732
+ apiFunctions.set(name, handler);
1733
+ }
1734
+ });
1735
+ return apiFunctions;
1736
+ };
1737
+ var getWorkflows2 = async () => {
1738
+ loadIndex();
1739
+ const registry = getMooseInternal();
1740
+ return registry.workflows;
1741
+ };
1742
+ function findTaskInTree(task, targetName) {
1743
+ if (task.name === targetName) {
1744
+ return task;
2311
1745
  }
2312
- });
2313
-
2314
- // src/moose-runner.ts
2315
- init_internal();
2316
- import { register } from "ts-node";
1746
+ if (task.config.onComplete?.length) {
1747
+ for (const childTask of task.config.onComplete) {
1748
+ const found = findTaskInTree(childTask, targetName);
1749
+ if (found) {
1750
+ return found;
1751
+ }
1752
+ }
1753
+ }
1754
+ return void 0;
1755
+ }
1756
+ var getTaskForWorkflow = async (workflowName, taskName) => {
1757
+ const workflows = await getWorkflows2();
1758
+ const workflow = workflows.get(workflowName);
1759
+ if (!workflow) {
1760
+ throw new Error(`Workflow ${workflowName} not found`);
1761
+ }
1762
+ const task = findTaskInTree(
1763
+ workflow.config.startingTask,
1764
+ taskName
1765
+ );
1766
+ if (!task) {
1767
+ throw new Error(`Task ${taskName} not found in workflow ${workflowName}`);
1768
+ }
1769
+ return task;
1770
+ };
1771
+ var getWebApps2 = async () => {
1772
+ loadIndex();
1773
+ return getMooseInternal().webApps;
1774
+ };
2317
1775
 
2318
1776
  // src/blocks/runner.ts
2319
1777
  init_commons();
2320
1778
  import fastq from "fastq";
2321
1779
  import fs2 from "fs";
2322
- import path2 from "path";
1780
+ import path from "path";
2323
1781
  var walkDir = (dir, fileExtension, fileList) => {
2324
1782
  const files = fs2.readdirSync(dir);
2325
1783
  files.forEach((file) => {
2326
- if (fs2.statSync(path2.join(dir, file)).isDirectory()) {
2327
- fileList = walkDir(path2.join(dir, file), fileExtension, fileList);
1784
+ if (fs2.statSync(path.join(dir, file)).isDirectory()) {
1785
+ fileList = walkDir(path.join(dir, file), fileExtension, fileList);
2328
1786
  } else if (file.endsWith(fileExtension)) {
2329
- fileList.push(path2.join(dir, file));
1787
+ fileList.push(path.join(dir, file));
2330
1788
  }
2331
1789
  });
2332
1790
  return fileList;
@@ -2337,7 +1795,7 @@ var DependencyError = class extends Error {
2337
1795
  this.name = "DependencyError";
2338
1796
  }
2339
1797
  };
2340
- var toClientConfig3 = (config) => ({
1798
+ var toClientConfig2 = (config) => ({
2341
1799
  ...config,
2342
1800
  useSSL: config.useSSL ? "true" : "false"
2343
1801
  });
@@ -2389,7 +1847,7 @@ var asyncWorker = async (task) => {
2389
1847
  await createBlocks(task.chClient, task.blocks);
2390
1848
  };
2391
1849
  var runBlocks = async (config) => {
2392
- const chClient = getClickhouseClient(toClientConfig3(config.clickhouseConfig));
1850
+ const chClient = getClickhouseClient(toClientConfig2(config.clickhouseConfig));
2393
1851
  console.log(`Connected`);
2394
1852
  const blocksFiles = walkDir(config.blocksDir, ".ts", []);
2395
1853
  const numOfBlockFiles = blocksFiles.length;
@@ -2402,10 +1860,10 @@ var runBlocks = async (config) => {
2402
1860
  }
2403
1861
  }
2404
1862
  });
2405
- for (const path4 of blocksFiles) {
2406
- console.log(`Adding to queue: ${path4}`);
1863
+ for (const path3 of blocksFiles) {
1864
+ console.log(`Adding to queue: ${path3}`);
2407
1865
  try {
2408
- const blocks = __require(path4).default;
1866
+ const blocks = __require(path3).default;
2409
1867
  queue.push({
2410
1868
  chClient,
2411
1869
  blocks,
@@ -2414,7 +1872,7 @@ var runBlocks = async (config) => {
2414
1872
  } catch (err) {
2415
1873
  cliLog({
2416
1874
  action: "Blocks",
2417
- message: `Failed to import blocks from ${path4}: ${err}`,
1875
+ message: `Failed to import blocks from ${path3}: ${err}`,
2418
1876
  message_type: "Error"
2419
1877
  });
2420
1878
  }
@@ -2424,14 +1882,8 @@ var runBlocks = async (config) => {
2424
1882
  }
2425
1883
  };
2426
1884
 
2427
- // src/moose-runner.ts
2428
- init_runner();
2429
-
2430
1885
  // src/streaming-functions/runner.ts
2431
1886
  init_commons();
2432
- init_cluster_utils();
2433
- init_internal();
2434
- init_json();
2435
1887
  import { Readable as Readable2 } from "stream";
2436
1888
  import { KafkaJS as KafkaJS2 } from "@514labs/kafka-javascript";
2437
1889
  import { Buffer as Buffer2 } from "buffer";
@@ -2994,18 +2446,15 @@ async function runApiTypeSerializer(targetModel) {
2994
2446
  }
2995
2447
 
2996
2448
  // src/scripts/runner.ts
2997
- init_internal();
2998
2449
  import {
2999
2450
  NativeConnection,
3000
2451
  Worker,
3001
2452
  bundleWorkflowCode
3002
2453
  } from "@temporalio/worker";
3003
- import * as path3 from "path";
2454
+ import * as path2 from "path";
3004
2455
  import * as fs3 from "fs";
3005
2456
 
3006
2457
  // src/scripts/activity.ts
3007
- init_internal();
3008
- init_json();
3009
2458
  import { log as logger, Context } from "@temporalio/activity";
3010
2459
  import { isCancellation } from "@temporalio/workflow";
3011
2460
  var activities = {
@@ -3296,7 +2745,7 @@ async function registerWorkflows(logger2, config) {
3296
2745
  }
3297
2746
  };
3298
2747
  const workflowBundle = await bundleWorkflowCode({
3299
- workflowsPath: path3.resolve(__dirname, "scripts/workflow.js"),
2748
+ workflowsPath: path2.resolve(__dirname, "scripts/workflow.js"),
3300
2749
  logger: silentLogger
3301
2750
  });
3302
2751
  const worker = await Worker.create({