@514labs/moose-lib 0.6.320 → 0.6.321-ci-5-ga23d35fe
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browserCompatible.d.mts +7 -3
- package/dist/browserCompatible.d.ts +7 -3
- package/dist/browserCompatible.js +197 -180
- package/dist/browserCompatible.js.map +1 -1
- package/dist/browserCompatible.mjs +196 -180
- package/dist/browserCompatible.mjs.map +1 -1
- package/dist/compilerPlugin.js.map +1 -1
- package/dist/compilerPlugin.mjs.map +1 -1
- package/dist/dataModels/toDataModels.js.map +1 -1
- package/dist/dataModels/toDataModels.mjs.map +1 -1
- package/dist/dmv2/index.d.mts +2 -1
- package/dist/dmv2/index.d.ts +2 -1
- package/dist/dmv2/index.js +41 -24
- package/dist/dmv2/index.js.map +1 -1
- package/dist/dmv2/index.mjs +40 -24
- package/dist/dmv2/index.mjs.map +1 -1
- package/dist/{index-C4miZc-A.d.mts → index-Aq9KzsRd.d.mts} +181 -33
- package/dist/{index-C4miZc-A.d.ts → index-Aq9KzsRd.d.ts} +181 -33
- package/dist/index.d.mts +4 -4
- package/dist/index.d.ts +4 -4
- package/dist/index.js +176 -202
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +176 -199
- package/dist/index.mjs.map +1 -1
- package/dist/moose-runner.js +88 -257
- package/dist/moose-runner.js.map +1 -1
- package/dist/moose-runner.mjs +89 -258
- package/dist/moose-runner.mjs.map +1 -1
- package/dist/scripts/workflow.js +9 -11
- package/dist/scripts/workflow.js.map +1 -1
- package/dist/scripts/workflow.mjs +9 -11
- package/dist/scripts/workflow.mjs.map +1 -1
- package/package.json +1 -1
- package/dist/browserCompatible-BVw4gSAN.d.mts +0 -184
- package/dist/browserCompatible-rK8ei0bt.d.ts +0 -184
package/dist/moose-runner.mjs
CHANGED
|
@@ -37,6 +37,13 @@ var init_dataModelTypes = __esm({
|
|
|
37
37
|
}
|
|
38
38
|
});
|
|
39
39
|
|
|
40
|
+
// src/dataModels/types.ts
|
|
41
|
+
var init_types = __esm({
|
|
42
|
+
"src/dataModels/types.ts"() {
|
|
43
|
+
"use strict";
|
|
44
|
+
}
|
|
45
|
+
});
|
|
46
|
+
|
|
40
47
|
// src/sqlHelpers.ts
|
|
41
48
|
function sql(strings, ...values) {
|
|
42
49
|
return new Sql(strings, values);
|
|
@@ -185,14 +192,6 @@ var init_sqlHelpers = __esm({
|
|
|
185
192
|
}
|
|
186
193
|
});
|
|
187
194
|
|
|
188
|
-
// src/blocks/helpers.ts
|
|
189
|
-
var init_helpers = __esm({
|
|
190
|
-
"src/blocks/helpers.ts"() {
|
|
191
|
-
"use strict";
|
|
192
|
-
init_sqlHelpers();
|
|
193
|
-
}
|
|
194
|
-
});
|
|
195
|
-
|
|
196
195
|
// src/dmv2/sdk/olapTable.ts
|
|
197
196
|
import { Readable } from "stream";
|
|
198
197
|
import { createHash } from "crypto";
|
|
@@ -201,7 +200,7 @@ var init_olapTable = __esm({
|
|
|
201
200
|
"use strict";
|
|
202
201
|
init_typedBase();
|
|
203
202
|
init_dataModelTypes();
|
|
204
|
-
|
|
203
|
+
init_types();
|
|
205
204
|
init_internal();
|
|
206
205
|
init_sqlHelpers();
|
|
207
206
|
}
|
|
@@ -252,7 +251,7 @@ var init_ingestPipeline = __esm({
|
|
|
252
251
|
init_stream();
|
|
253
252
|
init_olapTable();
|
|
254
253
|
init_ingestApi();
|
|
255
|
-
|
|
254
|
+
init_types();
|
|
256
255
|
}
|
|
257
256
|
});
|
|
258
257
|
|
|
@@ -268,7 +267,7 @@ var init_etlPipeline = __esm({
|
|
|
268
267
|
var init_materializedView = __esm({
|
|
269
268
|
"src/dmv2/sdk/materializedView.ts"() {
|
|
270
269
|
"use strict";
|
|
271
|
-
|
|
270
|
+
init_types();
|
|
272
271
|
init_sqlHelpers();
|
|
273
272
|
init_olapTable();
|
|
274
273
|
init_internal();
|
|
@@ -325,6 +324,7 @@ var init_dmv2 = __esm({
|
|
|
325
324
|
"src/dmv2/index.ts"() {
|
|
326
325
|
"use strict";
|
|
327
326
|
init_olapTable();
|
|
327
|
+
init_types();
|
|
328
328
|
init_stream();
|
|
329
329
|
init_workflow();
|
|
330
330
|
init_ingestApi();
|
|
@@ -340,13 +340,6 @@ var init_dmv2 = __esm({
|
|
|
340
340
|
}
|
|
341
341
|
});
|
|
342
342
|
|
|
343
|
-
// src/dataModels/types.ts
|
|
344
|
-
var init_types = __esm({
|
|
345
|
-
"src/dataModels/types.ts"() {
|
|
346
|
-
"use strict";
|
|
347
|
-
}
|
|
348
|
-
});
|
|
349
|
-
|
|
350
343
|
// src/browserCompatible.ts
|
|
351
344
|
var init_browserCompatible = __esm({
|
|
352
345
|
"src/browserCompatible.ts"() {
|
|
@@ -550,7 +543,7 @@ async function getTemporalClient(temporalUrl, namespace, clientCert, clientKey,
|
|
|
550
543
|
}
|
|
551
544
|
}
|
|
552
545
|
var MooseClient, QueryClient, WorkflowClient;
|
|
553
|
-
var
|
|
546
|
+
var init_helpers = __esm({
|
|
554
547
|
"src/consumption-apis/helpers.ts"() {
|
|
555
548
|
"use strict";
|
|
556
549
|
init_internal();
|
|
@@ -673,11 +666,11 @@ var init_helpers2 = __esm({
|
|
|
673
666
|
}
|
|
674
667
|
async getWorkflowConfig(name) {
|
|
675
668
|
const workflows = await getWorkflows2();
|
|
676
|
-
const
|
|
677
|
-
if (
|
|
669
|
+
const workflow = workflows.get(name);
|
|
670
|
+
if (workflow) {
|
|
678
671
|
return {
|
|
679
|
-
retries:
|
|
680
|
-
timeout:
|
|
672
|
+
retries: workflow.config.retries || 3,
|
|
673
|
+
timeout: workflow.config.timeout || "1h"
|
|
681
674
|
};
|
|
682
675
|
}
|
|
683
676
|
throw new Error(`Workflow config not found for ${name}`);
|
|
@@ -923,11 +916,11 @@ var init_cluster_utils = __esm({
|
|
|
923
916
|
import path from "path";
|
|
924
917
|
import * as toml from "toml";
|
|
925
918
|
async function findConfigFile(startDir = process.cwd()) {
|
|
926
|
-
const
|
|
919
|
+
const fs4 = await import("fs");
|
|
927
920
|
let currentDir = path.resolve(startDir);
|
|
928
921
|
while (true) {
|
|
929
922
|
const configPath = path.join(currentDir, "moose.config.toml");
|
|
930
|
-
if (
|
|
923
|
+
if (fs4.existsSync(configPath)) {
|
|
931
924
|
return configPath;
|
|
932
925
|
}
|
|
933
926
|
const parentDir = path.dirname(currentDir);
|
|
@@ -939,7 +932,7 @@ async function findConfigFile(startDir = process.cwd()) {
|
|
|
939
932
|
return null;
|
|
940
933
|
}
|
|
941
934
|
async function readProjectConfig() {
|
|
942
|
-
const
|
|
935
|
+
const fs4 = await import("fs");
|
|
943
936
|
const configPath = await findConfigFile();
|
|
944
937
|
if (!configPath) {
|
|
945
938
|
throw new ConfigError(
|
|
@@ -947,7 +940,7 @@ async function readProjectConfig() {
|
|
|
947
940
|
);
|
|
948
941
|
}
|
|
949
942
|
try {
|
|
950
|
-
const configContent =
|
|
943
|
+
const configContent = fs4.readFileSync(configPath, "utf-8");
|
|
951
944
|
const config = toml.parse(configContent);
|
|
952
945
|
return config;
|
|
953
946
|
} catch (error) {
|
|
@@ -1183,7 +1176,7 @@ var standaloneUtils, initPromise, toClientConfig;
|
|
|
1183
1176
|
var init_standalone = __esm({
|
|
1184
1177
|
"src/consumption-apis/standalone.ts"() {
|
|
1185
1178
|
"use strict";
|
|
1186
|
-
|
|
1179
|
+
init_helpers();
|
|
1187
1180
|
init_commons();
|
|
1188
1181
|
init_sqlHelpers();
|
|
1189
1182
|
standaloneUtils = null;
|
|
@@ -1198,12 +1191,12 @@ var init_standalone = __esm({
|
|
|
1198
1191
|
// src/consumption-apis/runner.ts
|
|
1199
1192
|
import http2 from "http";
|
|
1200
1193
|
import * as jose from "jose";
|
|
1201
|
-
var toClientConfig2,
|
|
1194
|
+
var toClientConfig2, httpLogger, modulesCache, apiHandler, createMainRouter, runApis;
|
|
1202
1195
|
var init_runner = __esm({
|
|
1203
1196
|
"src/consumption-apis/runner.ts"() {
|
|
1204
1197
|
"use strict";
|
|
1205
1198
|
init_commons();
|
|
1206
|
-
|
|
1199
|
+
init_helpers();
|
|
1207
1200
|
init_cluster_utils();
|
|
1208
1201
|
init_sqlHelpers();
|
|
1209
1202
|
init_internal();
|
|
@@ -1211,15 +1204,14 @@ var init_runner = __esm({
|
|
|
1211
1204
|
...config,
|
|
1212
1205
|
useSSL: config.useSSL ? "true" : "false"
|
|
1213
1206
|
});
|
|
1214
|
-
createPath = (apisDir, path5) => `${apisDir}${path5}.ts`;
|
|
1215
1207
|
httpLogger = (req, res, startMs) => {
|
|
1216
1208
|
console.log(
|
|
1217
1209
|
`${req.method} ${req.url} ${res.statusCode} ${Date.now() - startMs}ms`
|
|
1218
1210
|
);
|
|
1219
1211
|
};
|
|
1220
1212
|
modulesCache = /* @__PURE__ */ new Map();
|
|
1221
|
-
apiHandler = async (publicKey, clickhouseClient, temporalClient,
|
|
1222
|
-
const apis =
|
|
1213
|
+
apiHandler = async (publicKey, clickhouseClient, temporalClient, enforceAuth, jwtConfig) => {
|
|
1214
|
+
const apis = await getApis2();
|
|
1223
1215
|
return async (req, res) => {
|
|
1224
1216
|
const start = Date.now();
|
|
1225
1217
|
try {
|
|
@@ -1256,7 +1248,6 @@ var init_runner = __esm({
|
|
|
1256
1248
|
httpLogger(req, res, start);
|
|
1257
1249
|
return;
|
|
1258
1250
|
}
|
|
1259
|
-
const pathName = createPath(apisDir, fileName);
|
|
1260
1251
|
const paramsObject = Array.from(url.searchParams.entries()).reduce(
|
|
1261
1252
|
(obj, [key, value]) => {
|
|
1262
1253
|
const existingValue = obj[key];
|
|
@@ -1273,53 +1264,44 @@ var init_runner = __esm({
|
|
|
1273
1264
|
},
|
|
1274
1265
|
{}
|
|
1275
1266
|
);
|
|
1276
|
-
let userFuncModule = modulesCache.get(
|
|
1267
|
+
let userFuncModule = modulesCache.get(fileName);
|
|
1277
1268
|
if (userFuncModule === void 0) {
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
version = pathParts.slice(1).join("/");
|
|
1291
|
-
}
|
|
1292
|
-
}
|
|
1293
|
-
}
|
|
1294
|
-
if (!userFuncModule) {
|
|
1295
|
-
if (version) {
|
|
1296
|
-
const versionedKey = `${apiName}:${version}`;
|
|
1297
|
-
userFuncModule = apis.get(versionedKey);
|
|
1298
|
-
} else {
|
|
1299
|
-
userFuncModule = apis.get(apiName);
|
|
1269
|
+
let apiName = fileName.replace(/^\/+|\/+$/g, "");
|
|
1270
|
+
let version = null;
|
|
1271
|
+
userFuncModule = apis.get(apiName);
|
|
1272
|
+
if (!userFuncModule) {
|
|
1273
|
+
version = url.searchParams.get("version");
|
|
1274
|
+
if (!version && apiName.includes("/")) {
|
|
1275
|
+
const pathParts = apiName.split("/");
|
|
1276
|
+
if (pathParts.length >= 2) {
|
|
1277
|
+
userFuncModule = apis.get(apiName);
|
|
1278
|
+
if (!userFuncModule) {
|
|
1279
|
+
apiName = pathParts[0];
|
|
1280
|
+
version = pathParts.slice(1).join("/");
|
|
1300
1281
|
}
|
|
1301
1282
|
}
|
|
1302
1283
|
}
|
|
1303
1284
|
if (!userFuncModule) {
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1285
|
+
if (version) {
|
|
1286
|
+
const versionedKey = `${apiName}:${version}`;
|
|
1287
|
+
userFuncModule = apis.get(versionedKey);
|
|
1288
|
+
} else {
|
|
1289
|
+
userFuncModule = apis.get(apiName);
|
|
1290
|
+
}
|
|
1309
1291
|
}
|
|
1310
|
-
modulesCache.set(pathName, userFuncModule);
|
|
1311
|
-
console.log(`[API] | Executing API: ${apiName}`);
|
|
1312
|
-
} else {
|
|
1313
|
-
userFuncModule = __require(pathName);
|
|
1314
|
-
modulesCache.set(pathName, userFuncModule);
|
|
1315
1292
|
}
|
|
1293
|
+
if (!userFuncModule) {
|
|
1294
|
+
const availableApis = Array.from(apis.keys()).map(
|
|
1295
|
+
(key) => key.replace(":", "/")
|
|
1296
|
+
);
|
|
1297
|
+
const errorMessage = version ? `API ${apiName} with version ${version} not found. Available APIs: ${availableApis.join(", ")}` : `API ${apiName} not found. Available APIs: ${availableApis.join(", ")}`;
|
|
1298
|
+
throw new Error(errorMessage);
|
|
1299
|
+
}
|
|
1300
|
+
modulesCache.set(fileName, userFuncModule);
|
|
1301
|
+
console.log(`[API] | Executing API: ${apiName}`);
|
|
1316
1302
|
}
|
|
1317
1303
|
const queryClient = new QueryClient(clickhouseClient, fileName);
|
|
1318
|
-
let result =
|
|
1319
|
-
client: new MooseClient(queryClient, temporalClient),
|
|
1320
|
-
sql,
|
|
1321
|
-
jwt: jwtPayload
|
|
1322
|
-
}) : await userFuncModule.default(paramsObject, {
|
|
1304
|
+
let result = await userFuncModule(paramsObject, {
|
|
1323
1305
|
client: new MooseClient(queryClient, temporalClient),
|
|
1324
1306
|
sql,
|
|
1325
1307
|
jwt: jwtPayload
|
|
@@ -1363,17 +1345,15 @@ var init_runner = __esm({
|
|
|
1363
1345
|
}
|
|
1364
1346
|
};
|
|
1365
1347
|
};
|
|
1366
|
-
createMainRouter = async (publicKey, clickhouseClient, temporalClient,
|
|
1348
|
+
createMainRouter = async (publicKey, clickhouseClient, temporalClient, enforceAuth, jwtConfig) => {
|
|
1367
1349
|
const apiRequestHandler = await apiHandler(
|
|
1368
1350
|
publicKey,
|
|
1369
1351
|
clickhouseClient,
|
|
1370
1352
|
temporalClient,
|
|
1371
|
-
apisDir,
|
|
1372
1353
|
enforceAuth,
|
|
1373
|
-
isDmv2,
|
|
1374
1354
|
jwtConfig
|
|
1375
1355
|
);
|
|
1376
|
-
const webApps =
|
|
1356
|
+
const webApps = await getWebApps2();
|
|
1377
1357
|
const sortedWebApps = Array.from(webApps.values()).sort((a, b) => {
|
|
1378
1358
|
const pathA = a.config.mountPath || "/";
|
|
1379
1359
|
const pathB = b.config.mountPath || "/";
|
|
@@ -1495,9 +1475,7 @@ var init_runner = __esm({
|
|
|
1495
1475
|
publicKey,
|
|
1496
1476
|
clickhouseClient,
|
|
1497
1477
|
temporalClient,
|
|
1498
|
-
config.apisDir,
|
|
1499
1478
|
config.enforceAuth,
|
|
1500
|
-
config.isDmv2,
|
|
1501
1479
|
config.jwtConfig
|
|
1502
1480
|
)
|
|
1503
1481
|
);
|
|
@@ -1695,15 +1673,14 @@ var init_index = __esm({
|
|
|
1695
1673
|
"src/index.ts"() {
|
|
1696
1674
|
"use strict";
|
|
1697
1675
|
init_browserCompatible();
|
|
1698
|
-
init_helpers();
|
|
1699
1676
|
init_commons();
|
|
1700
1677
|
init_secrets();
|
|
1701
|
-
|
|
1678
|
+
init_helpers();
|
|
1702
1679
|
init_webAppHelpers();
|
|
1703
1680
|
init_task();
|
|
1704
1681
|
init_runner();
|
|
1705
1682
|
init_redisClient();
|
|
1706
|
-
|
|
1683
|
+
init_helpers();
|
|
1707
1684
|
init_standalone();
|
|
1708
1685
|
init_sqlHelpers();
|
|
1709
1686
|
init_utilities();
|
|
@@ -2418,119 +2395,8 @@ var init_internal = __esm({
|
|
|
2418
2395
|
|
|
2419
2396
|
// src/moose-runner.ts
|
|
2420
2397
|
init_internal();
|
|
2421
|
-
import { register } from "ts-node";
|
|
2422
|
-
|
|
2423
|
-
// src/blocks/runner.ts
|
|
2424
|
-
init_commons();
|
|
2425
|
-
import fastq from "fastq";
|
|
2426
|
-
import fs3 from "fs";
|
|
2427
|
-
import path3 from "path";
|
|
2428
|
-
var walkDir = (dir, fileExtension, fileList) => {
|
|
2429
|
-
const files = fs3.readdirSync(dir);
|
|
2430
|
-
files.forEach((file) => {
|
|
2431
|
-
if (fs3.statSync(path3.join(dir, file)).isDirectory()) {
|
|
2432
|
-
fileList = walkDir(path3.join(dir, file), fileExtension, fileList);
|
|
2433
|
-
} else if (file.endsWith(fileExtension)) {
|
|
2434
|
-
fileList.push(path3.join(dir, file));
|
|
2435
|
-
}
|
|
2436
|
-
});
|
|
2437
|
-
return fileList;
|
|
2438
|
-
};
|
|
2439
|
-
var DependencyError = class extends Error {
|
|
2440
|
-
constructor(message) {
|
|
2441
|
-
super(message);
|
|
2442
|
-
this.name = "DependencyError";
|
|
2443
|
-
}
|
|
2444
|
-
};
|
|
2445
|
-
var toClientConfig3 = (config) => ({
|
|
2446
|
-
...config,
|
|
2447
|
-
useSSL: config.useSSL ? "true" : "false"
|
|
2448
|
-
});
|
|
2449
|
-
var createBlocks = async (chClient, blocks) => {
|
|
2450
|
-
for (const query of blocks.setup) {
|
|
2451
|
-
try {
|
|
2452
|
-
console.log(`Creating block using query ${query}`);
|
|
2453
|
-
await chClient.command({
|
|
2454
|
-
query,
|
|
2455
|
-
clickhouse_settings: {
|
|
2456
|
-
wait_end_of_query: 1
|
|
2457
|
-
// Ensure at least once delivery and DDL acknowledgment
|
|
2458
|
-
}
|
|
2459
|
-
});
|
|
2460
|
-
} catch (err) {
|
|
2461
|
-
cliLog({
|
|
2462
|
-
action: "Blocks",
|
|
2463
|
-
message: `Failed to create blocks: ${err}`,
|
|
2464
|
-
message_type: "Error"
|
|
2465
|
-
});
|
|
2466
|
-
if (err && JSON.stringify(err).includes(`UNKNOWN_TABLE`)) {
|
|
2467
|
-
throw new DependencyError(err.toString());
|
|
2468
|
-
}
|
|
2469
|
-
}
|
|
2470
|
-
}
|
|
2471
|
-
};
|
|
2472
|
-
var deleteBlocks = async (chClient, blocks) => {
|
|
2473
|
-
for (const query of blocks.teardown) {
|
|
2474
|
-
try {
|
|
2475
|
-
console.log(`Deleting block using query ${query}`);
|
|
2476
|
-
await chClient.command({
|
|
2477
|
-
query,
|
|
2478
|
-
clickhouse_settings: {
|
|
2479
|
-
wait_end_of_query: 1
|
|
2480
|
-
// Ensure at least once delivery and DDL acknowledgment
|
|
2481
|
-
}
|
|
2482
|
-
});
|
|
2483
|
-
} catch (err) {
|
|
2484
|
-
cliLog({
|
|
2485
|
-
action: "Blocks",
|
|
2486
|
-
message: `Failed to delete blocks: ${err}`,
|
|
2487
|
-
message_type: "Error"
|
|
2488
|
-
});
|
|
2489
|
-
}
|
|
2490
|
-
}
|
|
2491
|
-
};
|
|
2492
|
-
var asyncWorker = async (task) => {
|
|
2493
|
-
await deleteBlocks(task.chClient, task.blocks);
|
|
2494
|
-
await createBlocks(task.chClient, task.blocks);
|
|
2495
|
-
};
|
|
2496
|
-
var runBlocks = async (config) => {
|
|
2497
|
-
const chClient = getClickhouseClient(toClientConfig3(config.clickhouseConfig));
|
|
2498
|
-
console.log(`Connected`);
|
|
2499
|
-
const blocksFiles = walkDir(config.blocksDir, ".ts", []);
|
|
2500
|
-
const numOfBlockFiles = blocksFiles.length;
|
|
2501
|
-
console.log(`Found ${numOfBlockFiles} blocks files`);
|
|
2502
|
-
const queue = fastq.promise(asyncWorker, 1);
|
|
2503
|
-
queue.error((err, task) => {
|
|
2504
|
-
if (err && task.retries > 0) {
|
|
2505
|
-
if (err instanceof DependencyError) {
|
|
2506
|
-
queue.push({ ...task, retries: task.retries - 1 });
|
|
2507
|
-
}
|
|
2508
|
-
}
|
|
2509
|
-
});
|
|
2510
|
-
for (const path5 of blocksFiles) {
|
|
2511
|
-
console.log(`Adding to queue: ${path5}`);
|
|
2512
|
-
try {
|
|
2513
|
-
const blocks = __require(path5).default;
|
|
2514
|
-
queue.push({
|
|
2515
|
-
chClient,
|
|
2516
|
-
blocks,
|
|
2517
|
-
retries: numOfBlockFiles
|
|
2518
|
-
});
|
|
2519
|
-
} catch (err) {
|
|
2520
|
-
cliLog({
|
|
2521
|
-
action: "Blocks",
|
|
2522
|
-
message: `Failed to import blocks from ${path5}: ${err}`,
|
|
2523
|
-
message_type: "Error"
|
|
2524
|
-
});
|
|
2525
|
-
}
|
|
2526
|
-
}
|
|
2527
|
-
while (!queue.idle()) {
|
|
2528
|
-
await new Promise((resolve3) => setTimeout(resolve3, 1e3));
|
|
2529
|
-
}
|
|
2530
|
-
};
|
|
2531
|
-
|
|
2532
|
-
// src/moose-runner.ts
|
|
2533
2398
|
init_runner();
|
|
2399
|
+
import { register } from "ts-node";
|
|
2534
2400
|
|
|
2535
2401
|
// src/streaming-functions/runner.ts
|
|
2536
2402
|
init_commons();
|
|
@@ -2814,17 +2680,7 @@ var sendMessageMetrics = (logger2, metrics) => {
|
|
|
2814
2680
|
metrics.count_out = 0;
|
|
2815
2681
|
setTimeout(() => sendMessageMetrics(logger2, metrics), 1e3);
|
|
2816
2682
|
};
|
|
2817
|
-
function loadStreamingFunction(
|
|
2818
|
-
let streamingFunctionImport;
|
|
2819
|
-
try {
|
|
2820
|
-
streamingFunctionImport = __require(functionFilePath.substring(0, functionFilePath.length - 3));
|
|
2821
|
-
} catch (e) {
|
|
2822
|
-
cliLog({ action: "Function", message: `${e}`, message_type: "Error" });
|
|
2823
|
-
throw e;
|
|
2824
|
-
}
|
|
2825
|
-
return streamingFunctionImport.default;
|
|
2826
|
-
}
|
|
2827
|
-
async function loadStreamingFunctionV2(sourceTopic, targetTopic) {
|
|
2683
|
+
async function loadStreamingFunction(sourceTopic, targetTopic) {
|
|
2828
2684
|
const transformFunctions = await getStreamingFunctions();
|
|
2829
2685
|
const transformFunctionKey = `${topicNameToStreamName(sourceTopic)}_${targetTopic ? topicNameToStreamName(targetTopic) : "<no-target>"}`;
|
|
2830
2686
|
const matchingEntries = Array.from(transformFunctions.entries()).filter(
|
|
@@ -2869,17 +2725,12 @@ var startConsumer = async (args, logger2, metrics, _parallelism, consumer, produ
|
|
|
2869
2725
|
);
|
|
2870
2726
|
let streamingFunctions;
|
|
2871
2727
|
let fieldMutations;
|
|
2872
|
-
|
|
2873
|
-
|
|
2874
|
-
|
|
2875
|
-
|
|
2876
|
-
|
|
2877
|
-
|
|
2878
|
-
fieldMutations = result.fieldMutations;
|
|
2879
|
-
} else {
|
|
2880
|
-
streamingFunctions = [[loadStreamingFunction(args.functionFilePath), {}]];
|
|
2881
|
-
fieldMutations = void 0;
|
|
2882
|
-
}
|
|
2728
|
+
const result = await loadStreamingFunction(
|
|
2729
|
+
args.sourceTopic,
|
|
2730
|
+
args.targetTopic
|
|
2731
|
+
);
|
|
2732
|
+
streamingFunctions = result.functions;
|
|
2733
|
+
fieldMutations = result.fieldMutations;
|
|
2883
2734
|
await consumer.subscribe({
|
|
2884
2735
|
topics: [args.sourceTopic.name]
|
|
2885
2736
|
// Use full topic name for Kafka operations
|
|
@@ -3118,8 +2969,8 @@ import {
|
|
|
3118
2969
|
Worker,
|
|
3119
2970
|
bundleWorkflowCode
|
|
3120
2971
|
} from "@temporalio/worker";
|
|
3121
|
-
import * as
|
|
3122
|
-
import * as
|
|
2972
|
+
import * as path3 from "path";
|
|
2973
|
+
import * as fs3 from "fs";
|
|
3123
2974
|
|
|
3124
2975
|
// src/scripts/activity.ts
|
|
3125
2976
|
init_internal();
|
|
@@ -3127,7 +2978,7 @@ init_json();
|
|
|
3127
2978
|
import { log as logger, Context } from "@temporalio/activity";
|
|
3128
2979
|
import { isCancellation } from "@temporalio/workflow";
|
|
3129
2980
|
var activities = {
|
|
3130
|
-
async
|
|
2981
|
+
async hasWorkflow(name) {
|
|
3131
2982
|
try {
|
|
3132
2983
|
const workflows = await getWorkflows2();
|
|
3133
2984
|
const hasWorkflow = workflows.has(name);
|
|
@@ -3138,7 +2989,7 @@ var activities = {
|
|
|
3138
2989
|
return false;
|
|
3139
2990
|
}
|
|
3140
2991
|
},
|
|
3141
|
-
async
|
|
2992
|
+
async getWorkflowByName(name) {
|
|
3142
2993
|
try {
|
|
3143
2994
|
logger.info(`Getting workflow ${name}`);
|
|
3144
2995
|
const workflows = await getWorkflows2();
|
|
@@ -3183,7 +3034,7 @@ var activities = {
|
|
|
3183
3034
|
throw new Error(errorMsg);
|
|
3184
3035
|
}
|
|
3185
3036
|
},
|
|
3186
|
-
async
|
|
3037
|
+
async executeTask(workflow, task, inputData) {
|
|
3187
3038
|
const context = Context.current();
|
|
3188
3039
|
const taskState = {};
|
|
3189
3040
|
let heartbeatInterval = null;
|
|
@@ -3244,7 +3095,7 @@ var activities = {
|
|
|
3244
3095
|
};
|
|
3245
3096
|
function createActivityForScript(scriptName) {
|
|
3246
3097
|
return {
|
|
3247
|
-
[scriptName]: activities.
|
|
3098
|
+
[scriptName]: activities.executeTask
|
|
3248
3099
|
};
|
|
3249
3100
|
}
|
|
3250
3101
|
|
|
@@ -3286,12 +3137,12 @@ var initializeLogger = LoggerSingleton.initializeLogger;
|
|
|
3286
3137
|
|
|
3287
3138
|
// src/scripts/runner.ts
|
|
3288
3139
|
var ALREADY_REGISTERED = /* @__PURE__ */ new Set();
|
|
3289
|
-
function
|
|
3290
|
-
logger2.info(`<
|
|
3140
|
+
function collectActivities(logger2, workflows) {
|
|
3141
|
+
logger2.info(`<WF> Collecting tasks from workflows`);
|
|
3291
3142
|
const scriptNames = [];
|
|
3292
3143
|
for (const [name, workflow] of workflows.entries()) {
|
|
3293
3144
|
logger2.info(
|
|
3294
|
-
`<
|
|
3145
|
+
`<WF> Registering workflow: ${name} with starting task: ${workflow.config.startingTask.name}`
|
|
3295
3146
|
);
|
|
3296
3147
|
scriptNames.push(`${name}/${workflow.config.startingTask.name}`);
|
|
3297
3148
|
}
|
|
@@ -3306,8 +3157,8 @@ async function createTemporalConnection(logger2, temporalConfig) {
|
|
|
3306
3157
|
};
|
|
3307
3158
|
if (temporalConfig.clientCert && temporalConfig.clientKey) {
|
|
3308
3159
|
logger2.info("Using TLS for secure Temporal");
|
|
3309
|
-
const cert = await
|
|
3310
|
-
const key = await
|
|
3160
|
+
const cert = await fs3.readFileSync(temporalConfig.clientCert);
|
|
3161
|
+
const key = await fs3.readFileSync(temporalConfig.clientKey);
|
|
3311
3162
|
connectionOptions.tls = {
|
|
3312
3163
|
clientCertPair: {
|
|
3313
3164
|
crt: cert,
|
|
@@ -3354,29 +3205,27 @@ async function registerWorkflows(logger2, config) {
|
|
|
3354
3205
|
try {
|
|
3355
3206
|
const workflows = await getWorkflows2();
|
|
3356
3207
|
if (workflows.size > 0) {
|
|
3357
|
-
logger2.info(`<
|
|
3358
|
-
allScriptPaths.push(...
|
|
3208
|
+
logger2.info(`<WF> Found ${workflows.size} workflows`);
|
|
3209
|
+
allScriptPaths.push(...collectActivities(logger2, workflows));
|
|
3359
3210
|
if (allScriptPaths.length === 0) {
|
|
3360
|
-
logger2.info(`<
|
|
3211
|
+
logger2.info(`<WF> No tasks found in workflows`);
|
|
3361
3212
|
return null;
|
|
3362
3213
|
}
|
|
3363
|
-
logger2.info(
|
|
3364
|
-
`<DMV2WF> Found ${allScriptPaths.length} tasks in dmv2 workflows`
|
|
3365
|
-
);
|
|
3214
|
+
logger2.info(`<WF> Found ${allScriptPaths.length} tasks in workflows`);
|
|
3366
3215
|
for (const activityName of allScriptPaths) {
|
|
3367
3216
|
if (!ALREADY_REGISTERED.has(activityName)) {
|
|
3368
3217
|
const activity = await createActivityForScript(activityName);
|
|
3369
3218
|
dynamicActivities.push(activity);
|
|
3370
3219
|
ALREADY_REGISTERED.add(activityName);
|
|
3371
|
-
logger2.info(`<
|
|
3220
|
+
logger2.info(`<WF> Registered task ${activityName}`);
|
|
3372
3221
|
}
|
|
3373
3222
|
}
|
|
3374
3223
|
if (dynamicActivities.length === 0) {
|
|
3375
|
-
logger2.info(`<
|
|
3224
|
+
logger2.info(`<WF> No dynamic activities found in workflows`);
|
|
3376
3225
|
return null;
|
|
3377
3226
|
}
|
|
3378
3227
|
logger2.info(
|
|
3379
|
-
`<
|
|
3228
|
+
`<WF> Found ${dynamicActivities.length} dynamic activities in workflows`
|
|
3380
3229
|
);
|
|
3381
3230
|
}
|
|
3382
3231
|
if (allScriptPaths.length === 0) {
|
|
@@ -3414,7 +3263,7 @@ async function registerWorkflows(logger2, config) {
|
|
|
3414
3263
|
}
|
|
3415
3264
|
};
|
|
3416
3265
|
const workflowBundle = await bundleWorkflowCode({
|
|
3417
|
-
workflowsPath:
|
|
3266
|
+
workflowsPath: path3.resolve(__dirname, "scripts/workflow.js"),
|
|
3418
3267
|
logger: silentLogger
|
|
3419
3268
|
});
|
|
3420
3269
|
const worker = await Worker.create({
|
|
@@ -3529,33 +3378,17 @@ program.command("dmv2-serializer").description("Load DMv2 index").action(() => {
|
|
|
3529
3378
|
program.command("export-serializer").description("Run export serializer").argument("<target-model>", "Target model to serialize").action((targetModel) => {
|
|
3530
3379
|
runExportSerializer(targetModel);
|
|
3531
3380
|
});
|
|
3532
|
-
program.command("
|
|
3533
|
-
(blocksDir, clickhouseDb, clickhouseHost, clickhousePort, clickhouseUsername, clickhousePassword, options) => {
|
|
3534
|
-
runBlocks({
|
|
3535
|
-
blocksDir,
|
|
3536
|
-
clickhouseConfig: {
|
|
3537
|
-
database: clickhouseDb,
|
|
3538
|
-
host: clickhouseHost,
|
|
3539
|
-
port: clickhousePort,
|
|
3540
|
-
username: clickhouseUsername,
|
|
3541
|
-
password: clickhousePassword,
|
|
3542
|
-
useSSL: options.clickhouseUseSsl
|
|
3543
|
-
}
|
|
3544
|
-
});
|
|
3545
|
-
}
|
|
3546
|
-
);
|
|
3547
|
-
program.command("consumption-apis").description("Run consumption APIs").argument("<consumption-dir>", "Directory containing consumption APIs").argument("<clickhouse-db>", "Clickhouse database name").argument("<clickhouse-host>", "Clickhouse host").argument("<clickhouse-port>", "Clickhouse port").argument("<clickhouse-username>", "Clickhouse username").argument("<clickhouse-password>", "Clickhouse password").option("--clickhouse-use-ssl", "Use SSL for Clickhouse connection", false).option("--jwt-secret <secret>", "JWT public key for verification").option("--jwt-issuer <issuer>", "Expected JWT issuer").option("--jwt-audience <audience>", "Expected JWT audience").option(
|
|
3381
|
+
program.command("consumption-apis").description("Run consumption APIs").argument("<clickhouse-db>", "Clickhouse database name").argument("<clickhouse-host>", "Clickhouse host").argument("<clickhouse-port>", "Clickhouse port").argument("<clickhouse-username>", "Clickhouse username").argument("<clickhouse-password>", "Clickhouse password").option("--clickhouse-use-ssl", "Use SSL for Clickhouse connection", false).option("--jwt-secret <secret>", "JWT public key for verification").option("--jwt-issuer <issuer>", "Expected JWT issuer").option("--jwt-audience <audience>", "Expected JWT audience").option(
|
|
3548
3382
|
"--enforce-auth",
|
|
3549
3383
|
"Enforce authentication on all consumption APIs",
|
|
3550
3384
|
false
|
|
3551
|
-
).option("--temporal-url <url>", "Temporal server URL").option("--temporal-namespace <namespace>", "Temporal namespace").option("--client-cert <path>", "Path to client certificate").option("--client-key <path>", "Path to client key").option("--api-key <key>", "API key for authentication").option("--
|
|
3385
|
+
).option("--temporal-url <url>", "Temporal server URL").option("--temporal-namespace <namespace>", "Temporal namespace").option("--client-cert <path>", "Path to client certificate").option("--client-key <path>", "Path to client key").option("--api-key <key>", "API key for authentication").option("--proxy-port <port>", "Port to run the proxy server on", parseInt).option(
|
|
3552
3386
|
"--worker-count <count>",
|
|
3553
3387
|
"Number of worker processes for the consumption API cluster",
|
|
3554
3388
|
parseInt
|
|
3555
3389
|
).action(
|
|
3556
|
-
(
|
|
3390
|
+
(clickhouseDb, clickhouseHost, clickhousePort, clickhouseUsername, clickhousePassword, options) => {
|
|
3557
3391
|
runApis({
|
|
3558
|
-
apisDir,
|
|
3559
3392
|
clickhouseConfig: {
|
|
3560
3393
|
database: clickhouseDb,
|
|
3561
3394
|
host: clickhouseHost,
|
|
@@ -3577,7 +3410,6 @@ program.command("consumption-apis").description("Run consumption APIs").argument
|
|
|
3577
3410
|
apiKey: options.apiKey
|
|
3578
3411
|
},
|
|
3579
3412
|
enforceAuth: options.enforceAuth,
|
|
3580
|
-
isDmv2: options.isDmv2,
|
|
3581
3413
|
proxyPort: options.proxyPort,
|
|
3582
3414
|
workerCount: options.workerCount
|
|
3583
3415
|
});
|
|
@@ -3586,7 +3418,7 @@ program.command("consumption-apis").description("Run consumption APIs").argument
|
|
|
3586
3418
|
program.command("streaming-functions").description("Run streaming functions").argument("<source-topic>", "Source topic configuration as JSON").argument("<function-file-path>", "Path to the function file").argument(
|
|
3587
3419
|
"<broker>",
|
|
3588
3420
|
"Kafka broker address(es) - comma-separated for multiple brokers (e.g., 'broker1:9092, broker2:9092'). Whitespace around commas is automatically trimmed."
|
|
3589
|
-
).argument("<max-subscriber-count>", "Maximum number of subscribers").option("--target-topic <target-topic>", "Target topic configuration as JSON").option("--sasl-username <username>", "SASL username").option("--sasl-password <password>", "SASL password").option("--sasl-mechanism <mechanism>", "SASL mechanism").option("--security-protocol <protocol>", "Security protocol").option("--
|
|
3421
|
+
).argument("<max-subscriber-count>", "Maximum number of subscribers").option("--target-topic <target-topic>", "Target topic configuration as JSON").option("--sasl-username <username>", "SASL username").option("--sasl-password <password>", "SASL password").option("--sasl-mechanism <mechanism>", "SASL mechanism").option("--security-protocol <protocol>", "Security protocol").option("--log-payloads", "Log payloads for debugging", false).action(
|
|
3590
3422
|
(sourceTopic, functionFilePath, broker, maxSubscriberCount, options) => {
|
|
3591
3423
|
const config = {
|
|
3592
3424
|
sourceTopic: JSON.parse(sourceTopic),
|
|
@@ -3594,7 +3426,6 @@ program.command("streaming-functions").description("Run streaming functions").ar
|
|
|
3594
3426
|
functionFilePath,
|
|
3595
3427
|
broker,
|
|
3596
3428
|
maxSubscriberCount: parseInt(maxSubscriberCount),
|
|
3597
|
-
isDmv2: options.isDmv2,
|
|
3598
3429
|
logPayloads: options.logPayloads,
|
|
3599
3430
|
saslUsername: options.saslUsername,
|
|
3600
3431
|
saslPassword: options.saslPassword,
|