@514labs/moose-lib 0.6.320 → 0.6.321-ci-5-ga23d35fe
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browserCompatible.d.mts +7 -3
- package/dist/browserCompatible.d.ts +7 -3
- package/dist/browserCompatible.js +197 -180
- package/dist/browserCompatible.js.map +1 -1
- package/dist/browserCompatible.mjs +196 -180
- package/dist/browserCompatible.mjs.map +1 -1
- package/dist/compilerPlugin.js.map +1 -1
- package/dist/compilerPlugin.mjs.map +1 -1
- package/dist/dataModels/toDataModels.js.map +1 -1
- package/dist/dataModels/toDataModels.mjs.map +1 -1
- package/dist/dmv2/index.d.mts +2 -1
- package/dist/dmv2/index.d.ts +2 -1
- package/dist/dmv2/index.js +41 -24
- package/dist/dmv2/index.js.map +1 -1
- package/dist/dmv2/index.mjs +40 -24
- package/dist/dmv2/index.mjs.map +1 -1
- package/dist/{index-C4miZc-A.d.mts → index-Aq9KzsRd.d.mts} +181 -33
- package/dist/{index-C4miZc-A.d.ts → index-Aq9KzsRd.d.ts} +181 -33
- package/dist/index.d.mts +4 -4
- package/dist/index.d.ts +4 -4
- package/dist/index.js +176 -202
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +176 -199
- package/dist/index.mjs.map +1 -1
- package/dist/moose-runner.js +88 -257
- package/dist/moose-runner.js.map +1 -1
- package/dist/moose-runner.mjs +89 -258
- package/dist/moose-runner.mjs.map +1 -1
- package/dist/scripts/workflow.js +9 -11
- package/dist/scripts/workflow.js.map +1 -1
- package/dist/scripts/workflow.mjs +9 -11
- package/dist/scripts/workflow.mjs.map +1 -1
- package/package.json +1 -1
- package/dist/browserCompatible-BVw4gSAN.d.mts +0 -184
- package/dist/browserCompatible-rK8ei0bt.d.ts +0 -184
package/dist/moose-runner.js
CHANGED
|
@@ -52,6 +52,13 @@ var init_dataModelTypes = __esm({
|
|
|
52
52
|
}
|
|
53
53
|
});
|
|
54
54
|
|
|
55
|
+
// src/dataModels/types.ts
|
|
56
|
+
var init_types = __esm({
|
|
57
|
+
"src/dataModels/types.ts"() {
|
|
58
|
+
"use strict";
|
|
59
|
+
}
|
|
60
|
+
});
|
|
61
|
+
|
|
55
62
|
// src/sqlHelpers.ts
|
|
56
63
|
function sql(strings, ...values) {
|
|
57
64
|
return new Sql(strings, values);
|
|
@@ -200,14 +207,6 @@ var init_sqlHelpers = __esm({
|
|
|
200
207
|
}
|
|
201
208
|
});
|
|
202
209
|
|
|
203
|
-
// src/blocks/helpers.ts
|
|
204
|
-
var init_helpers = __esm({
|
|
205
|
-
"src/blocks/helpers.ts"() {
|
|
206
|
-
"use strict";
|
|
207
|
-
init_sqlHelpers();
|
|
208
|
-
}
|
|
209
|
-
});
|
|
210
|
-
|
|
211
210
|
// src/dmv2/sdk/olapTable.ts
|
|
212
211
|
var import_node_stream, import_node_crypto;
|
|
213
212
|
var init_olapTable = __esm({
|
|
@@ -215,7 +214,7 @@ var init_olapTable = __esm({
|
|
|
215
214
|
"use strict";
|
|
216
215
|
init_typedBase();
|
|
217
216
|
init_dataModelTypes();
|
|
218
|
-
|
|
217
|
+
init_types();
|
|
219
218
|
init_internal();
|
|
220
219
|
import_node_stream = require("stream");
|
|
221
220
|
import_node_crypto = require("crypto");
|
|
@@ -269,7 +268,7 @@ var init_ingestPipeline = __esm({
|
|
|
269
268
|
init_stream();
|
|
270
269
|
init_olapTable();
|
|
271
270
|
init_ingestApi();
|
|
272
|
-
|
|
271
|
+
init_types();
|
|
273
272
|
}
|
|
274
273
|
});
|
|
275
274
|
|
|
@@ -285,7 +284,7 @@ var init_etlPipeline = __esm({
|
|
|
285
284
|
var init_materializedView = __esm({
|
|
286
285
|
"src/dmv2/sdk/materializedView.ts"() {
|
|
287
286
|
"use strict";
|
|
288
|
-
|
|
287
|
+
init_types();
|
|
289
288
|
init_sqlHelpers();
|
|
290
289
|
init_olapTable();
|
|
291
290
|
init_internal();
|
|
@@ -342,6 +341,7 @@ var init_dmv2 = __esm({
|
|
|
342
341
|
"src/dmv2/index.ts"() {
|
|
343
342
|
"use strict";
|
|
344
343
|
init_olapTable();
|
|
344
|
+
init_types();
|
|
345
345
|
init_stream();
|
|
346
346
|
init_workflow();
|
|
347
347
|
init_ingestApi();
|
|
@@ -357,13 +357,6 @@ var init_dmv2 = __esm({
|
|
|
357
357
|
}
|
|
358
358
|
});
|
|
359
359
|
|
|
360
|
-
// src/dataModels/types.ts
|
|
361
|
-
var init_types = __esm({
|
|
362
|
-
"src/dataModels/types.ts"() {
|
|
363
|
-
"use strict";
|
|
364
|
-
}
|
|
365
|
-
});
|
|
366
|
-
|
|
367
360
|
// src/browserCompatible.ts
|
|
368
361
|
var init_browserCompatible = __esm({
|
|
369
362
|
"src/browserCompatible.ts"() {
|
|
@@ -560,7 +553,7 @@ async function getTemporalClient(temporalUrl, namespace, clientCert, clientKey,
|
|
|
560
553
|
}
|
|
561
554
|
}
|
|
562
555
|
var import_client2, import_node_crypto3, import_perf_hooks, fs, MooseClient, QueryClient, WorkflowClient;
|
|
563
|
-
var
|
|
556
|
+
var init_helpers = __esm({
|
|
564
557
|
"src/consumption-apis/helpers.ts"() {
|
|
565
558
|
"use strict";
|
|
566
559
|
import_client2 = require("@temporalio/client");
|
|
@@ -687,11 +680,11 @@ var init_helpers2 = __esm({
|
|
|
687
680
|
}
|
|
688
681
|
async getWorkflowConfig(name) {
|
|
689
682
|
const workflows = await getWorkflows2();
|
|
690
|
-
const
|
|
691
|
-
if (
|
|
683
|
+
const workflow = workflows.get(name);
|
|
684
|
+
if (workflow) {
|
|
692
685
|
return {
|
|
693
|
-
retries:
|
|
694
|
-
timeout:
|
|
686
|
+
retries: workflow.config.retries || 3,
|
|
687
|
+
timeout: workflow.config.timeout || "1h"
|
|
695
688
|
};
|
|
696
689
|
}
|
|
697
690
|
throw new Error(`Workflow config not found for ${name}`);
|
|
@@ -935,11 +928,11 @@ var init_cluster_utils = __esm({
|
|
|
935
928
|
|
|
936
929
|
// src/config/configFile.ts
|
|
937
930
|
async function findConfigFile(startDir = process.cwd()) {
|
|
938
|
-
const
|
|
931
|
+
const fs4 = await import("fs");
|
|
939
932
|
let currentDir = import_node_path.default.resolve(startDir);
|
|
940
933
|
while (true) {
|
|
941
934
|
const configPath = import_node_path.default.join(currentDir, "moose.config.toml");
|
|
942
|
-
if (
|
|
935
|
+
if (fs4.existsSync(configPath)) {
|
|
943
936
|
return configPath;
|
|
944
937
|
}
|
|
945
938
|
const parentDir = import_node_path.default.dirname(currentDir);
|
|
@@ -951,7 +944,7 @@ async function findConfigFile(startDir = process.cwd()) {
|
|
|
951
944
|
return null;
|
|
952
945
|
}
|
|
953
946
|
async function readProjectConfig() {
|
|
954
|
-
const
|
|
947
|
+
const fs4 = await import("fs");
|
|
955
948
|
const configPath = await findConfigFile();
|
|
956
949
|
if (!configPath) {
|
|
957
950
|
throw new ConfigError(
|
|
@@ -959,7 +952,7 @@ async function readProjectConfig() {
|
|
|
959
952
|
);
|
|
960
953
|
}
|
|
961
954
|
try {
|
|
962
|
-
const configContent =
|
|
955
|
+
const configContent = fs4.readFileSync(configPath, "utf-8");
|
|
963
956
|
const config = toml.parse(configContent);
|
|
964
957
|
return config;
|
|
965
958
|
} catch (error) {
|
|
@@ -1197,7 +1190,7 @@ var standaloneUtils, initPromise, toClientConfig;
|
|
|
1197
1190
|
var init_standalone = __esm({
|
|
1198
1191
|
"src/consumption-apis/standalone.ts"() {
|
|
1199
1192
|
"use strict";
|
|
1200
|
-
|
|
1193
|
+
init_helpers();
|
|
1201
1194
|
init_commons();
|
|
1202
1195
|
init_sqlHelpers();
|
|
1203
1196
|
standaloneUtils = null;
|
|
@@ -1210,13 +1203,13 @@ var init_standalone = __esm({
|
|
|
1210
1203
|
});
|
|
1211
1204
|
|
|
1212
1205
|
// src/consumption-apis/runner.ts
|
|
1213
|
-
var import_http2, jose, toClientConfig2,
|
|
1206
|
+
var import_http2, jose, toClientConfig2, httpLogger, modulesCache, apiHandler, createMainRouter, runApis;
|
|
1214
1207
|
var init_runner = __esm({
|
|
1215
1208
|
"src/consumption-apis/runner.ts"() {
|
|
1216
1209
|
"use strict";
|
|
1217
1210
|
import_http2 = __toESM(require("http"));
|
|
1218
1211
|
init_commons();
|
|
1219
|
-
|
|
1212
|
+
init_helpers();
|
|
1220
1213
|
jose = __toESM(require("jose"));
|
|
1221
1214
|
init_cluster_utils();
|
|
1222
1215
|
init_sqlHelpers();
|
|
@@ -1225,15 +1218,14 @@ var init_runner = __esm({
|
|
|
1225
1218
|
...config,
|
|
1226
1219
|
useSSL: config.useSSL ? "true" : "false"
|
|
1227
1220
|
});
|
|
1228
|
-
createPath = (apisDir, path5) => `${apisDir}${path5}.ts`;
|
|
1229
1221
|
httpLogger = (req, res, startMs) => {
|
|
1230
1222
|
console.log(
|
|
1231
1223
|
`${req.method} ${req.url} ${res.statusCode} ${Date.now() - startMs}ms`
|
|
1232
1224
|
);
|
|
1233
1225
|
};
|
|
1234
1226
|
modulesCache = /* @__PURE__ */ new Map();
|
|
1235
|
-
apiHandler = async (publicKey, clickhouseClient, temporalClient,
|
|
1236
|
-
const apis =
|
|
1227
|
+
apiHandler = async (publicKey, clickhouseClient, temporalClient, enforceAuth, jwtConfig) => {
|
|
1228
|
+
const apis = await getApis2();
|
|
1237
1229
|
return async (req, res) => {
|
|
1238
1230
|
const start = Date.now();
|
|
1239
1231
|
try {
|
|
@@ -1270,7 +1262,6 @@ var init_runner = __esm({
|
|
|
1270
1262
|
httpLogger(req, res, start);
|
|
1271
1263
|
return;
|
|
1272
1264
|
}
|
|
1273
|
-
const pathName = createPath(apisDir, fileName);
|
|
1274
1265
|
const paramsObject = Array.from(url.searchParams.entries()).reduce(
|
|
1275
1266
|
(obj, [key, value]) => {
|
|
1276
1267
|
const existingValue = obj[key];
|
|
@@ -1287,53 +1278,44 @@ var init_runner = __esm({
|
|
|
1287
1278
|
},
|
|
1288
1279
|
{}
|
|
1289
1280
|
);
|
|
1290
|
-
let userFuncModule = modulesCache.get(
|
|
1281
|
+
let userFuncModule = modulesCache.get(fileName);
|
|
1291
1282
|
if (userFuncModule === void 0) {
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
version = pathParts.slice(1).join("/");
|
|
1305
|
-
}
|
|
1306
|
-
}
|
|
1307
|
-
}
|
|
1308
|
-
if (!userFuncModule) {
|
|
1309
|
-
if (version) {
|
|
1310
|
-
const versionedKey = `${apiName}:${version}`;
|
|
1311
|
-
userFuncModule = apis.get(versionedKey);
|
|
1312
|
-
} else {
|
|
1313
|
-
userFuncModule = apis.get(apiName);
|
|
1283
|
+
let apiName = fileName.replace(/^\/+|\/+$/g, "");
|
|
1284
|
+
let version = null;
|
|
1285
|
+
userFuncModule = apis.get(apiName);
|
|
1286
|
+
if (!userFuncModule) {
|
|
1287
|
+
version = url.searchParams.get("version");
|
|
1288
|
+
if (!version && apiName.includes("/")) {
|
|
1289
|
+
const pathParts = apiName.split("/");
|
|
1290
|
+
if (pathParts.length >= 2) {
|
|
1291
|
+
userFuncModule = apis.get(apiName);
|
|
1292
|
+
if (!userFuncModule) {
|
|
1293
|
+
apiName = pathParts[0];
|
|
1294
|
+
version = pathParts.slice(1).join("/");
|
|
1314
1295
|
}
|
|
1315
1296
|
}
|
|
1316
1297
|
}
|
|
1317
1298
|
if (!userFuncModule) {
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1299
|
+
if (version) {
|
|
1300
|
+
const versionedKey = `${apiName}:${version}`;
|
|
1301
|
+
userFuncModule = apis.get(versionedKey);
|
|
1302
|
+
} else {
|
|
1303
|
+
userFuncModule = apis.get(apiName);
|
|
1304
|
+
}
|
|
1323
1305
|
}
|
|
1324
|
-
modulesCache.set(pathName, userFuncModule);
|
|
1325
|
-
console.log(`[API] | Executing API: ${apiName}`);
|
|
1326
|
-
} else {
|
|
1327
|
-
userFuncModule = require(pathName);
|
|
1328
|
-
modulesCache.set(pathName, userFuncModule);
|
|
1329
1306
|
}
|
|
1307
|
+
if (!userFuncModule) {
|
|
1308
|
+
const availableApis = Array.from(apis.keys()).map(
|
|
1309
|
+
(key) => key.replace(":", "/")
|
|
1310
|
+
);
|
|
1311
|
+
const errorMessage = version ? `API ${apiName} with version ${version} not found. Available APIs: ${availableApis.join(", ")}` : `API ${apiName} not found. Available APIs: ${availableApis.join(", ")}`;
|
|
1312
|
+
throw new Error(errorMessage);
|
|
1313
|
+
}
|
|
1314
|
+
modulesCache.set(fileName, userFuncModule);
|
|
1315
|
+
console.log(`[API] | Executing API: ${apiName}`);
|
|
1330
1316
|
}
|
|
1331
1317
|
const queryClient = new QueryClient(clickhouseClient, fileName);
|
|
1332
|
-
let result =
|
|
1333
|
-
client: new MooseClient(queryClient, temporalClient),
|
|
1334
|
-
sql,
|
|
1335
|
-
jwt: jwtPayload
|
|
1336
|
-
}) : await userFuncModule.default(paramsObject, {
|
|
1318
|
+
let result = await userFuncModule(paramsObject, {
|
|
1337
1319
|
client: new MooseClient(queryClient, temporalClient),
|
|
1338
1320
|
sql,
|
|
1339
1321
|
jwt: jwtPayload
|
|
@@ -1377,17 +1359,15 @@ var init_runner = __esm({
|
|
|
1377
1359
|
}
|
|
1378
1360
|
};
|
|
1379
1361
|
};
|
|
1380
|
-
createMainRouter = async (publicKey, clickhouseClient, temporalClient,
|
|
1362
|
+
createMainRouter = async (publicKey, clickhouseClient, temporalClient, enforceAuth, jwtConfig) => {
|
|
1381
1363
|
const apiRequestHandler = await apiHandler(
|
|
1382
1364
|
publicKey,
|
|
1383
1365
|
clickhouseClient,
|
|
1384
1366
|
temporalClient,
|
|
1385
|
-
apisDir,
|
|
1386
1367
|
enforceAuth,
|
|
1387
|
-
isDmv2,
|
|
1388
1368
|
jwtConfig
|
|
1389
1369
|
);
|
|
1390
|
-
const webApps =
|
|
1370
|
+
const webApps = await getWebApps2();
|
|
1391
1371
|
const sortedWebApps = Array.from(webApps.values()).sort((a, b) => {
|
|
1392
1372
|
const pathA = a.config.mountPath || "/";
|
|
1393
1373
|
const pathB = b.config.mountPath || "/";
|
|
@@ -1509,9 +1489,7 @@ var init_runner = __esm({
|
|
|
1509
1489
|
publicKey,
|
|
1510
1490
|
clickhouseClient,
|
|
1511
1491
|
temporalClient,
|
|
1512
|
-
config.apisDir,
|
|
1513
1492
|
config.enforceAuth,
|
|
1514
|
-
config.isDmv2,
|
|
1515
1493
|
config.jwtConfig
|
|
1516
1494
|
)
|
|
1517
1495
|
);
|
|
@@ -1710,15 +1688,14 @@ var init_index = __esm({
|
|
|
1710
1688
|
"src/index.ts"() {
|
|
1711
1689
|
"use strict";
|
|
1712
1690
|
init_browserCompatible();
|
|
1713
|
-
init_helpers();
|
|
1714
1691
|
init_commons();
|
|
1715
1692
|
init_secrets();
|
|
1716
|
-
|
|
1693
|
+
init_helpers();
|
|
1717
1694
|
init_webAppHelpers();
|
|
1718
1695
|
init_task();
|
|
1719
1696
|
init_runner();
|
|
1720
1697
|
init_redisClient();
|
|
1721
|
-
|
|
1698
|
+
init_helpers();
|
|
1722
1699
|
init_standalone();
|
|
1723
1700
|
init_sqlHelpers();
|
|
1724
1701
|
init_utilities();
|
|
@@ -2434,117 +2411,6 @@ var init_internal = __esm({
|
|
|
2434
2411
|
// src/moose-runner.ts
|
|
2435
2412
|
var import_ts_node = require("ts-node");
|
|
2436
2413
|
init_internal();
|
|
2437
|
-
|
|
2438
|
-
// src/blocks/runner.ts
|
|
2439
|
-
var import_fastq = __toESM(require("fastq"));
|
|
2440
|
-
init_commons();
|
|
2441
|
-
var import_node_fs = __toESM(require("fs"));
|
|
2442
|
-
var import_node_path2 = __toESM(require("path"));
|
|
2443
|
-
var walkDir = (dir, fileExtension, fileList) => {
|
|
2444
|
-
const files = import_node_fs.default.readdirSync(dir);
|
|
2445
|
-
files.forEach((file) => {
|
|
2446
|
-
if (import_node_fs.default.statSync(import_node_path2.default.join(dir, file)).isDirectory()) {
|
|
2447
|
-
fileList = walkDir(import_node_path2.default.join(dir, file), fileExtension, fileList);
|
|
2448
|
-
} else if (file.endsWith(fileExtension)) {
|
|
2449
|
-
fileList.push(import_node_path2.default.join(dir, file));
|
|
2450
|
-
}
|
|
2451
|
-
});
|
|
2452
|
-
return fileList;
|
|
2453
|
-
};
|
|
2454
|
-
var DependencyError = class extends Error {
|
|
2455
|
-
constructor(message) {
|
|
2456
|
-
super(message);
|
|
2457
|
-
this.name = "DependencyError";
|
|
2458
|
-
}
|
|
2459
|
-
};
|
|
2460
|
-
var toClientConfig3 = (config) => ({
|
|
2461
|
-
...config,
|
|
2462
|
-
useSSL: config.useSSL ? "true" : "false"
|
|
2463
|
-
});
|
|
2464
|
-
var createBlocks = async (chClient, blocks) => {
|
|
2465
|
-
for (const query of blocks.setup) {
|
|
2466
|
-
try {
|
|
2467
|
-
console.log(`Creating block using query ${query}`);
|
|
2468
|
-
await chClient.command({
|
|
2469
|
-
query,
|
|
2470
|
-
clickhouse_settings: {
|
|
2471
|
-
wait_end_of_query: 1
|
|
2472
|
-
// Ensure at least once delivery and DDL acknowledgment
|
|
2473
|
-
}
|
|
2474
|
-
});
|
|
2475
|
-
} catch (err) {
|
|
2476
|
-
cliLog({
|
|
2477
|
-
action: "Blocks",
|
|
2478
|
-
message: `Failed to create blocks: ${err}`,
|
|
2479
|
-
message_type: "Error"
|
|
2480
|
-
});
|
|
2481
|
-
if (err && JSON.stringify(err).includes(`UNKNOWN_TABLE`)) {
|
|
2482
|
-
throw new DependencyError(err.toString());
|
|
2483
|
-
}
|
|
2484
|
-
}
|
|
2485
|
-
}
|
|
2486
|
-
};
|
|
2487
|
-
var deleteBlocks = async (chClient, blocks) => {
|
|
2488
|
-
for (const query of blocks.teardown) {
|
|
2489
|
-
try {
|
|
2490
|
-
console.log(`Deleting block using query ${query}`);
|
|
2491
|
-
await chClient.command({
|
|
2492
|
-
query,
|
|
2493
|
-
clickhouse_settings: {
|
|
2494
|
-
wait_end_of_query: 1
|
|
2495
|
-
// Ensure at least once delivery and DDL acknowledgment
|
|
2496
|
-
}
|
|
2497
|
-
});
|
|
2498
|
-
} catch (err) {
|
|
2499
|
-
cliLog({
|
|
2500
|
-
action: "Blocks",
|
|
2501
|
-
message: `Failed to delete blocks: ${err}`,
|
|
2502
|
-
message_type: "Error"
|
|
2503
|
-
});
|
|
2504
|
-
}
|
|
2505
|
-
}
|
|
2506
|
-
};
|
|
2507
|
-
var asyncWorker = async (task) => {
|
|
2508
|
-
await deleteBlocks(task.chClient, task.blocks);
|
|
2509
|
-
await createBlocks(task.chClient, task.blocks);
|
|
2510
|
-
};
|
|
2511
|
-
var runBlocks = async (config) => {
|
|
2512
|
-
const chClient = getClickhouseClient(toClientConfig3(config.clickhouseConfig));
|
|
2513
|
-
console.log(`Connected`);
|
|
2514
|
-
const blocksFiles = walkDir(config.blocksDir, ".ts", []);
|
|
2515
|
-
const numOfBlockFiles = blocksFiles.length;
|
|
2516
|
-
console.log(`Found ${numOfBlockFiles} blocks files`);
|
|
2517
|
-
const queue = import_fastq.default.promise(asyncWorker, 1);
|
|
2518
|
-
queue.error((err, task) => {
|
|
2519
|
-
if (err && task.retries > 0) {
|
|
2520
|
-
if (err instanceof DependencyError) {
|
|
2521
|
-
queue.push({ ...task, retries: task.retries - 1 });
|
|
2522
|
-
}
|
|
2523
|
-
}
|
|
2524
|
-
});
|
|
2525
|
-
for (const path5 of blocksFiles) {
|
|
2526
|
-
console.log(`Adding to queue: ${path5}`);
|
|
2527
|
-
try {
|
|
2528
|
-
const blocks = require(path5).default;
|
|
2529
|
-
queue.push({
|
|
2530
|
-
chClient,
|
|
2531
|
-
blocks,
|
|
2532
|
-
retries: numOfBlockFiles
|
|
2533
|
-
});
|
|
2534
|
-
} catch (err) {
|
|
2535
|
-
cliLog({
|
|
2536
|
-
action: "Blocks",
|
|
2537
|
-
message: `Failed to import blocks from ${path5}: ${err}`,
|
|
2538
|
-
message_type: "Error"
|
|
2539
|
-
});
|
|
2540
|
-
}
|
|
2541
|
-
}
|
|
2542
|
-
while (!queue.idle()) {
|
|
2543
|
-
await new Promise((resolve3) => setTimeout(resolve3, 1e3));
|
|
2544
|
-
}
|
|
2545
|
-
};
|
|
2546
|
-
|
|
2547
|
-
// src/moose-runner.ts
|
|
2548
2414
|
init_runner();
|
|
2549
2415
|
|
|
2550
2416
|
// src/streaming-functions/runner.ts
|
|
@@ -2829,17 +2695,7 @@ var sendMessageMetrics = (logger2, metrics) => {
|
|
|
2829
2695
|
metrics.count_out = 0;
|
|
2830
2696
|
setTimeout(() => sendMessageMetrics(logger2, metrics), 1e3);
|
|
2831
2697
|
};
|
|
2832
|
-
function loadStreamingFunction(
|
|
2833
|
-
let streamingFunctionImport;
|
|
2834
|
-
try {
|
|
2835
|
-
streamingFunctionImport = require(functionFilePath.substring(0, functionFilePath.length - 3));
|
|
2836
|
-
} catch (e) {
|
|
2837
|
-
cliLog({ action: "Function", message: `${e}`, message_type: "Error" });
|
|
2838
|
-
throw e;
|
|
2839
|
-
}
|
|
2840
|
-
return streamingFunctionImport.default;
|
|
2841
|
-
}
|
|
2842
|
-
async function loadStreamingFunctionV2(sourceTopic, targetTopic) {
|
|
2698
|
+
async function loadStreamingFunction(sourceTopic, targetTopic) {
|
|
2843
2699
|
const transformFunctions = await getStreamingFunctions();
|
|
2844
2700
|
const transformFunctionKey = `${topicNameToStreamName(sourceTopic)}_${targetTopic ? topicNameToStreamName(targetTopic) : "<no-target>"}`;
|
|
2845
2701
|
const matchingEntries = Array.from(transformFunctions.entries()).filter(
|
|
@@ -2884,17 +2740,12 @@ var startConsumer = async (args, logger2, metrics, _parallelism, consumer, produ
|
|
|
2884
2740
|
);
|
|
2885
2741
|
let streamingFunctions;
|
|
2886
2742
|
let fieldMutations;
|
|
2887
|
-
|
|
2888
|
-
|
|
2889
|
-
|
|
2890
|
-
|
|
2891
|
-
|
|
2892
|
-
|
|
2893
|
-
fieldMutations = result.fieldMutations;
|
|
2894
|
-
} else {
|
|
2895
|
-
streamingFunctions = [[loadStreamingFunction(args.functionFilePath), {}]];
|
|
2896
|
-
fieldMutations = void 0;
|
|
2897
|
-
}
|
|
2743
|
+
const result = await loadStreamingFunction(
|
|
2744
|
+
args.sourceTopic,
|
|
2745
|
+
args.targetTopic
|
|
2746
|
+
);
|
|
2747
|
+
streamingFunctions = result.functions;
|
|
2748
|
+
fieldMutations = result.fieldMutations;
|
|
2898
2749
|
await consumer.subscribe({
|
|
2899
2750
|
topics: [args.sourceTopic.name]
|
|
2900
2751
|
// Use full topic name for Kafka operations
|
|
@@ -3128,8 +2979,8 @@ async function runApiTypeSerializer(targetModel) {
|
|
|
3128
2979
|
|
|
3129
2980
|
// src/scripts/runner.ts
|
|
3130
2981
|
var import_worker2 = require("@temporalio/worker");
|
|
3131
|
-
var
|
|
3132
|
-
var
|
|
2982
|
+
var path3 = __toESM(require("path"));
|
|
2983
|
+
var fs3 = __toESM(require("fs"));
|
|
3133
2984
|
init_internal();
|
|
3134
2985
|
|
|
3135
2986
|
// src/scripts/activity.ts
|
|
@@ -3138,7 +2989,7 @@ var import_workflow3 = require("@temporalio/workflow");
|
|
|
3138
2989
|
init_internal();
|
|
3139
2990
|
init_json();
|
|
3140
2991
|
var activities = {
|
|
3141
|
-
async
|
|
2992
|
+
async hasWorkflow(name) {
|
|
3142
2993
|
try {
|
|
3143
2994
|
const workflows = await getWorkflows2();
|
|
3144
2995
|
const hasWorkflow = workflows.has(name);
|
|
@@ -3149,7 +3000,7 @@ var activities = {
|
|
|
3149
3000
|
return false;
|
|
3150
3001
|
}
|
|
3151
3002
|
},
|
|
3152
|
-
async
|
|
3003
|
+
async getWorkflowByName(name) {
|
|
3153
3004
|
try {
|
|
3154
3005
|
import_activity.log.info(`Getting workflow ${name}`);
|
|
3155
3006
|
const workflows = await getWorkflows2();
|
|
@@ -3194,7 +3045,7 @@ var activities = {
|
|
|
3194
3045
|
throw new Error(errorMsg);
|
|
3195
3046
|
}
|
|
3196
3047
|
},
|
|
3197
|
-
async
|
|
3048
|
+
async executeTask(workflow, task, inputData) {
|
|
3198
3049
|
const context = import_activity.Context.current();
|
|
3199
3050
|
const taskState = {};
|
|
3200
3051
|
let heartbeatInterval = null;
|
|
@@ -3255,7 +3106,7 @@ var activities = {
|
|
|
3255
3106
|
};
|
|
3256
3107
|
function createActivityForScript(scriptName) {
|
|
3257
3108
|
return {
|
|
3258
|
-
[scriptName]: activities.
|
|
3109
|
+
[scriptName]: activities.executeTask
|
|
3259
3110
|
};
|
|
3260
3111
|
}
|
|
3261
3112
|
|
|
@@ -3293,12 +3144,12 @@ var initializeLogger = LoggerSingleton.initializeLogger;
|
|
|
3293
3144
|
|
|
3294
3145
|
// src/scripts/runner.ts
|
|
3295
3146
|
var ALREADY_REGISTERED = /* @__PURE__ */ new Set();
|
|
3296
|
-
function
|
|
3297
|
-
logger2.info(`<
|
|
3147
|
+
function collectActivities(logger2, workflows) {
|
|
3148
|
+
logger2.info(`<WF> Collecting tasks from workflows`);
|
|
3298
3149
|
const scriptNames = [];
|
|
3299
3150
|
for (const [name, workflow] of workflows.entries()) {
|
|
3300
3151
|
logger2.info(
|
|
3301
|
-
`<
|
|
3152
|
+
`<WF> Registering workflow: ${name} with starting task: ${workflow.config.startingTask.name}`
|
|
3302
3153
|
);
|
|
3303
3154
|
scriptNames.push(`${name}/${workflow.config.startingTask.name}`);
|
|
3304
3155
|
}
|
|
@@ -3313,8 +3164,8 @@ async function createTemporalConnection(logger2, temporalConfig) {
|
|
|
3313
3164
|
};
|
|
3314
3165
|
if (temporalConfig.clientCert && temporalConfig.clientKey) {
|
|
3315
3166
|
logger2.info("Using TLS for secure Temporal");
|
|
3316
|
-
const cert = await
|
|
3317
|
-
const key = await
|
|
3167
|
+
const cert = await fs3.readFileSync(temporalConfig.clientCert);
|
|
3168
|
+
const key = await fs3.readFileSync(temporalConfig.clientKey);
|
|
3318
3169
|
connectionOptions.tls = {
|
|
3319
3170
|
clientCertPair: {
|
|
3320
3171
|
crt: cert,
|
|
@@ -3361,29 +3212,27 @@ async function registerWorkflows(logger2, config) {
|
|
|
3361
3212
|
try {
|
|
3362
3213
|
const workflows = await getWorkflows2();
|
|
3363
3214
|
if (workflows.size > 0) {
|
|
3364
|
-
logger2.info(`<
|
|
3365
|
-
allScriptPaths.push(...
|
|
3215
|
+
logger2.info(`<WF> Found ${workflows.size} workflows`);
|
|
3216
|
+
allScriptPaths.push(...collectActivities(logger2, workflows));
|
|
3366
3217
|
if (allScriptPaths.length === 0) {
|
|
3367
|
-
logger2.info(`<
|
|
3218
|
+
logger2.info(`<WF> No tasks found in workflows`);
|
|
3368
3219
|
return null;
|
|
3369
3220
|
}
|
|
3370
|
-
logger2.info(
|
|
3371
|
-
`<DMV2WF> Found ${allScriptPaths.length} tasks in dmv2 workflows`
|
|
3372
|
-
);
|
|
3221
|
+
logger2.info(`<WF> Found ${allScriptPaths.length} tasks in workflows`);
|
|
3373
3222
|
for (const activityName of allScriptPaths) {
|
|
3374
3223
|
if (!ALREADY_REGISTERED.has(activityName)) {
|
|
3375
3224
|
const activity = await createActivityForScript(activityName);
|
|
3376
3225
|
dynamicActivities.push(activity);
|
|
3377
3226
|
ALREADY_REGISTERED.add(activityName);
|
|
3378
|
-
logger2.info(`<
|
|
3227
|
+
logger2.info(`<WF> Registered task ${activityName}`);
|
|
3379
3228
|
}
|
|
3380
3229
|
}
|
|
3381
3230
|
if (dynamicActivities.length === 0) {
|
|
3382
|
-
logger2.info(`<
|
|
3231
|
+
logger2.info(`<WF> No dynamic activities found in workflows`);
|
|
3383
3232
|
return null;
|
|
3384
3233
|
}
|
|
3385
3234
|
logger2.info(
|
|
3386
|
-
`<
|
|
3235
|
+
`<WF> Found ${dynamicActivities.length} dynamic activities in workflows`
|
|
3387
3236
|
);
|
|
3388
3237
|
}
|
|
3389
3238
|
if (allScriptPaths.length === 0) {
|
|
@@ -3421,7 +3270,7 @@ async function registerWorkflows(logger2, config) {
|
|
|
3421
3270
|
}
|
|
3422
3271
|
};
|
|
3423
3272
|
const workflowBundle = await (0, import_worker2.bundleWorkflowCode)({
|
|
3424
|
-
workflowsPath:
|
|
3273
|
+
workflowsPath: path3.resolve(__dirname, "scripts/workflow.js"),
|
|
3425
3274
|
logger: silentLogger
|
|
3426
3275
|
});
|
|
3427
3276
|
const worker = await import_worker2.Worker.create({
|
|
@@ -3536,33 +3385,17 @@ program.command("dmv2-serializer").description("Load DMv2 index").action(() => {
|
|
|
3536
3385
|
program.command("export-serializer").description("Run export serializer").argument("<target-model>", "Target model to serialize").action((targetModel) => {
|
|
3537
3386
|
runExportSerializer(targetModel);
|
|
3538
3387
|
});
|
|
3539
|
-
program.command("
|
|
3540
|
-
(blocksDir, clickhouseDb, clickhouseHost, clickhousePort, clickhouseUsername, clickhousePassword, options) => {
|
|
3541
|
-
runBlocks({
|
|
3542
|
-
blocksDir,
|
|
3543
|
-
clickhouseConfig: {
|
|
3544
|
-
database: clickhouseDb,
|
|
3545
|
-
host: clickhouseHost,
|
|
3546
|
-
port: clickhousePort,
|
|
3547
|
-
username: clickhouseUsername,
|
|
3548
|
-
password: clickhousePassword,
|
|
3549
|
-
useSSL: options.clickhouseUseSsl
|
|
3550
|
-
}
|
|
3551
|
-
});
|
|
3552
|
-
}
|
|
3553
|
-
);
|
|
3554
|
-
program.command("consumption-apis").description("Run consumption APIs").argument("<consumption-dir>", "Directory containing consumption APIs").argument("<clickhouse-db>", "Clickhouse database name").argument("<clickhouse-host>", "Clickhouse host").argument("<clickhouse-port>", "Clickhouse port").argument("<clickhouse-username>", "Clickhouse username").argument("<clickhouse-password>", "Clickhouse password").option("--clickhouse-use-ssl", "Use SSL for Clickhouse connection", false).option("--jwt-secret <secret>", "JWT public key for verification").option("--jwt-issuer <issuer>", "Expected JWT issuer").option("--jwt-audience <audience>", "Expected JWT audience").option(
|
|
3388
|
+
program.command("consumption-apis").description("Run consumption APIs").argument("<clickhouse-db>", "Clickhouse database name").argument("<clickhouse-host>", "Clickhouse host").argument("<clickhouse-port>", "Clickhouse port").argument("<clickhouse-username>", "Clickhouse username").argument("<clickhouse-password>", "Clickhouse password").option("--clickhouse-use-ssl", "Use SSL for Clickhouse connection", false).option("--jwt-secret <secret>", "JWT public key for verification").option("--jwt-issuer <issuer>", "Expected JWT issuer").option("--jwt-audience <audience>", "Expected JWT audience").option(
|
|
3555
3389
|
"--enforce-auth",
|
|
3556
3390
|
"Enforce authentication on all consumption APIs",
|
|
3557
3391
|
false
|
|
3558
|
-
).option("--temporal-url <url>", "Temporal server URL").option("--temporal-namespace <namespace>", "Temporal namespace").option("--client-cert <path>", "Path to client certificate").option("--client-key <path>", "Path to client key").option("--api-key <key>", "API key for authentication").option("--
|
|
3392
|
+
).option("--temporal-url <url>", "Temporal server URL").option("--temporal-namespace <namespace>", "Temporal namespace").option("--client-cert <path>", "Path to client certificate").option("--client-key <path>", "Path to client key").option("--api-key <key>", "API key for authentication").option("--proxy-port <port>", "Port to run the proxy server on", parseInt).option(
|
|
3559
3393
|
"--worker-count <count>",
|
|
3560
3394
|
"Number of worker processes for the consumption API cluster",
|
|
3561
3395
|
parseInt
|
|
3562
3396
|
).action(
|
|
3563
|
-
(
|
|
3397
|
+
(clickhouseDb, clickhouseHost, clickhousePort, clickhouseUsername, clickhousePassword, options) => {
|
|
3564
3398
|
runApis({
|
|
3565
|
-
apisDir,
|
|
3566
3399
|
clickhouseConfig: {
|
|
3567
3400
|
database: clickhouseDb,
|
|
3568
3401
|
host: clickhouseHost,
|
|
@@ -3584,7 +3417,6 @@ program.command("consumption-apis").description("Run consumption APIs").argument
|
|
|
3584
3417
|
apiKey: options.apiKey
|
|
3585
3418
|
},
|
|
3586
3419
|
enforceAuth: options.enforceAuth,
|
|
3587
|
-
isDmv2: options.isDmv2,
|
|
3588
3420
|
proxyPort: options.proxyPort,
|
|
3589
3421
|
workerCount: options.workerCount
|
|
3590
3422
|
});
|
|
@@ -3593,7 +3425,7 @@ program.command("consumption-apis").description("Run consumption APIs").argument
|
|
|
3593
3425
|
program.command("streaming-functions").description("Run streaming functions").argument("<source-topic>", "Source topic configuration as JSON").argument("<function-file-path>", "Path to the function file").argument(
|
|
3594
3426
|
"<broker>",
|
|
3595
3427
|
"Kafka broker address(es) - comma-separated for multiple brokers (e.g., 'broker1:9092, broker2:9092'). Whitespace around commas is automatically trimmed."
|
|
3596
|
-
).argument("<max-subscriber-count>", "Maximum number of subscribers").option("--target-topic <target-topic>", "Target topic configuration as JSON").option("--sasl-username <username>", "SASL username").option("--sasl-password <password>", "SASL password").option("--sasl-mechanism <mechanism>", "SASL mechanism").option("--security-protocol <protocol>", "Security protocol").option("--
|
|
3428
|
+
).argument("<max-subscriber-count>", "Maximum number of subscribers").option("--target-topic <target-topic>", "Target topic configuration as JSON").option("--sasl-username <username>", "SASL username").option("--sasl-password <password>", "SASL password").option("--sasl-mechanism <mechanism>", "SASL mechanism").option("--security-protocol <protocol>", "Security protocol").option("--log-payloads", "Log payloads for debugging", false).action(
|
|
3597
3429
|
(sourceTopic, functionFilePath, broker, maxSubscriberCount, options) => {
|
|
3598
3430
|
const config = {
|
|
3599
3431
|
sourceTopic: JSON.parse(sourceTopic),
|
|
@@ -3601,7 +3433,6 @@ program.command("streaming-functions").description("Run streaming functions").ar
|
|
|
3601
3433
|
functionFilePath,
|
|
3602
3434
|
broker,
|
|
3603
3435
|
maxSubscriberCount: parseInt(maxSubscriberCount),
|
|
3604
|
-
isDmv2: options.isDmv2,
|
|
3605
3436
|
logPayloads: options.logPayloads,
|
|
3606
3437
|
saslUsername: options.saslUsername,
|
|
3607
3438
|
saslPassword: options.saslPassword,
|