@bayoudhi/moose-lib-serverless 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{helpers-C3Yr4RzN.d.mts → index.d.mts} +373 -39
- package/dist/{helpers-C3Yr4RzN.d.ts → index.d.ts} +373 -39
- package/dist/{serverless.js → index.js} +184 -176
- package/dist/index.js.map +1 -0
- package/dist/{serverless.mjs → index.mjs} +120 -91
- package/dist/index.mjs.map +1 -0
- package/package.json +15 -8
- package/dist/browserCompatible.d.mts +0 -14
- package/dist/browserCompatible.d.ts +0 -14
- package/dist/serverless.d.mts +0 -383
- package/dist/serverless.d.ts +0 -383
- package/dist/serverless.js.map +0 -1
- package/dist/serverless.mjs.map +0 -1
|
@@ -5,8 +5,8 @@ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
|
5
5
|
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
6
|
var __getProtoOf = Object.getPrototypeOf;
|
|
7
7
|
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
8
|
-
var
|
|
9
|
-
return
|
|
8
|
+
var __commonJS = (cb, mod) => function __require() {
|
|
9
|
+
return mod || (0, cb[__getOwnPropNames(cb)[0]])((mod = { exports: {} }).exports, mod), mod.exports;
|
|
10
10
|
};
|
|
11
11
|
var __export = (target, all) => {
|
|
12
12
|
for (var name in all)
|
|
@@ -30,7 +30,115 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
|
|
|
30
30
|
));
|
|
31
31
|
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
32
32
|
|
|
33
|
-
//
|
|
33
|
+
// stub-native:@514labs/kafka-javascript
|
|
34
|
+
var require_kafka_javascript = __commonJS({
|
|
35
|
+
"stub-native:@514labs/kafka-javascript"(exports2, module2) {
|
|
36
|
+
"use strict";
|
|
37
|
+
module2.exports = {};
|
|
38
|
+
}
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
// stub-native:@kafkajs/confluent-schema-registry
|
|
42
|
+
var require_confluent_schema_registry = __commonJS({
|
|
43
|
+
"stub-native:@kafkajs/confluent-schema-registry"(exports2, module2) {
|
|
44
|
+
"use strict";
|
|
45
|
+
module2.exports = {};
|
|
46
|
+
}
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
// src/index.ts
|
|
50
|
+
var index_exports = {};
|
|
51
|
+
__export(index_exports, {
|
|
52
|
+
ACKs: () => ACKs,
|
|
53
|
+
Api: () => Api,
|
|
54
|
+
CSV_DELIMITERS: () => CSV_DELIMITERS,
|
|
55
|
+
ClickHouseEngines: () => ClickHouseEngines,
|
|
56
|
+
ConsumptionApi: () => ConsumptionApi,
|
|
57
|
+
DEFAULT_CSV_CONFIG: () => DEFAULT_CSV_CONFIG,
|
|
58
|
+
DEFAULT_JSON_CONFIG: () => DEFAULT_JSON_CONFIG,
|
|
59
|
+
DataSource: () => DataSource,
|
|
60
|
+
DeadLetterQueue: () => DeadLetterQueue,
|
|
61
|
+
ETLPipeline: () => ETLPipeline,
|
|
62
|
+
IngestApi: () => IngestApi,
|
|
63
|
+
IngestPipeline: () => IngestPipeline,
|
|
64
|
+
LifeCycle: () => LifeCycle,
|
|
65
|
+
MAX_RETRIES: () => MAX_RETRIES,
|
|
66
|
+
MAX_RETRIES_PRODUCER: () => MAX_RETRIES_PRODUCER,
|
|
67
|
+
MAX_RETRY_TIME_MS: () => MAX_RETRY_TIME_MS,
|
|
68
|
+
MOOSE_RUNTIME_ENV_PREFIX: () => MOOSE_RUNTIME_ENV_PREFIX,
|
|
69
|
+
MaterializedView: () => MaterializedView,
|
|
70
|
+
OlapTable: () => OlapTable,
|
|
71
|
+
RETRY_FACTOR_PRODUCER: () => RETRY_FACTOR_PRODUCER,
|
|
72
|
+
RETRY_INITIAL_TIME_MS: () => RETRY_INITIAL_TIME_MS,
|
|
73
|
+
Sql: () => Sql,
|
|
74
|
+
SqlResource: () => SqlResource,
|
|
75
|
+
Stream: () => Stream,
|
|
76
|
+
Task: () => Task,
|
|
77
|
+
View: () => View,
|
|
78
|
+
WebApp: () => WebApp,
|
|
79
|
+
Workflow: () => Workflow,
|
|
80
|
+
antiCachePath: () => antiCachePath,
|
|
81
|
+
cliLog: () => cliLog,
|
|
82
|
+
compilerLog: () => compilerLog,
|
|
83
|
+
createClickhouseParameter: () => createClickhouseParameter,
|
|
84
|
+
getApi: () => getApi,
|
|
85
|
+
getApis: () => getApis,
|
|
86
|
+
getFileName: () => getFileName,
|
|
87
|
+
getIngestApi: () => getIngestApi,
|
|
88
|
+
getIngestApis: () => getIngestApis,
|
|
89
|
+
getMaterializedView: () => getMaterializedView,
|
|
90
|
+
getMaterializedViews: () => getMaterializedViews,
|
|
91
|
+
getSqlResource: () => getSqlResource,
|
|
92
|
+
getSqlResources: () => getSqlResources,
|
|
93
|
+
getStream: () => getStream,
|
|
94
|
+
getStreams: () => getStreams,
|
|
95
|
+
getTable: () => getTable,
|
|
96
|
+
getTables: () => getTables,
|
|
97
|
+
getValueFromParameter: () => getValueFromParameter,
|
|
98
|
+
getView: () => getView,
|
|
99
|
+
getViews: () => getViews,
|
|
100
|
+
getWebApp: () => getWebApp,
|
|
101
|
+
getWebApps: () => getWebApps,
|
|
102
|
+
getWorkflow: () => getWorkflow,
|
|
103
|
+
getWorkflows: () => getWorkflows,
|
|
104
|
+
isValidCSVDelimiter: () => isValidCSVDelimiter,
|
|
105
|
+
logError: () => logError,
|
|
106
|
+
mapToClickHouseType: () => mapToClickHouseType,
|
|
107
|
+
mapTstoJs: () => mapTstoJs,
|
|
108
|
+
mooseEnvSecrets: () => mooseEnvSecrets,
|
|
109
|
+
mooseRuntimeEnv: () => mooseRuntimeEnv,
|
|
110
|
+
parseCSV: () => parseCSV,
|
|
111
|
+
parseJSON: () => parseJSON,
|
|
112
|
+
parseJSONWithDates: () => parseJSONWithDates,
|
|
113
|
+
quoteIdentifier: () => quoteIdentifier,
|
|
114
|
+
sql: () => sql,
|
|
115
|
+
toQuery: () => toQuery,
|
|
116
|
+
toQueryPreview: () => toQueryPreview,
|
|
117
|
+
toStaticQuery: () => toStaticQuery
|
|
118
|
+
});
|
|
119
|
+
module.exports = __toCommonJS(index_exports);
|
|
120
|
+
|
|
121
|
+
// ../ts-moose-lib/dist/serverless.mjs
|
|
122
|
+
var import_path = __toESM(require("path"), 1);
|
|
123
|
+
var toml = __toESM(require("toml"), 1);
|
|
124
|
+
var import_kafka_javascript = __toESM(require_kafka_javascript(), 1);
|
|
125
|
+
var import_client = require("@clickhouse/client");
|
|
126
|
+
var import_fs = require("fs");
|
|
127
|
+
var import_path2 = __toESM(require("path"), 1);
|
|
128
|
+
var import_process = __toESM(require("process"), 1);
|
|
129
|
+
var import_stream = require("stream");
|
|
130
|
+
var import_crypto = require("crypto");
|
|
131
|
+
var import_crypto2 = require("crypto");
|
|
132
|
+
var import_csv_parse = require("csv-parse");
|
|
133
|
+
var __defProp2 = Object.defineProperty;
|
|
134
|
+
var __getOwnPropNames2 = Object.getOwnPropertyNames;
|
|
135
|
+
var __esm = (fn, res) => function __init() {
|
|
136
|
+
return fn && (res = (0, fn[__getOwnPropNames2(fn)[0]])(fn = 0)), res;
|
|
137
|
+
};
|
|
138
|
+
var __export2 = (target, all) => {
|
|
139
|
+
for (var name in all)
|
|
140
|
+
__defProp2(target, name, { get: all[name], enumerable: true });
|
|
141
|
+
};
|
|
34
142
|
function isTruthy(value) {
|
|
35
143
|
if (!value) return false;
|
|
36
144
|
switch (value.trim().toLowerCase()) {
|
|
@@ -46,7 +154,17 @@ function isTruthy(value) {
|
|
|
46
154
|
function mapTstoJs(filePath) {
|
|
47
155
|
return filePath.replace(/\.ts$/, ".js").replace(/\.cts$/, ".cjs").replace(/\.mts$/, ".mjs");
|
|
48
156
|
}
|
|
49
|
-
var compilerLog
|
|
157
|
+
var compilerLog;
|
|
158
|
+
var antiCachePath;
|
|
159
|
+
var getFileName;
|
|
160
|
+
var cliLog;
|
|
161
|
+
var MAX_RETRIES;
|
|
162
|
+
var MAX_RETRY_TIME_MS;
|
|
163
|
+
var RETRY_INITIAL_TIME_MS;
|
|
164
|
+
var MAX_RETRIES_PRODUCER;
|
|
165
|
+
var RETRY_FACTOR_PRODUCER;
|
|
166
|
+
var ACKs;
|
|
167
|
+
var logError;
|
|
50
168
|
var init_commons_types = __esm({
|
|
51
169
|
"src/commons-types.ts"() {
|
|
52
170
|
"use strict";
|
|
@@ -92,17 +210,15 @@ var init_commons_types = __esm({
|
|
|
92
210
|
};
|
|
93
211
|
}
|
|
94
212
|
});
|
|
95
|
-
|
|
96
|
-
// src/config/configFile.ts
|
|
97
213
|
async function findConfigFile(startDir = process.cwd()) {
|
|
98
214
|
const fs = await import("fs");
|
|
99
|
-
let currentDir =
|
|
215
|
+
let currentDir = import_path.default.resolve(startDir);
|
|
100
216
|
while (true) {
|
|
101
|
-
const configPath =
|
|
217
|
+
const configPath = import_path.default.join(currentDir, "moose.config.toml");
|
|
102
218
|
if (fs.existsSync(configPath)) {
|
|
103
219
|
return configPath;
|
|
104
220
|
}
|
|
105
|
-
const parentDir =
|
|
221
|
+
const parentDir = import_path.default.dirname(currentDir);
|
|
106
222
|
if (parentDir === currentDir) {
|
|
107
223
|
break;
|
|
108
224
|
}
|
|
@@ -126,12 +242,10 @@ async function readProjectConfig() {
|
|
|
126
242
|
throw new ConfigError(`Failed to parse moose.config.toml: ${error}`);
|
|
127
243
|
}
|
|
128
244
|
}
|
|
129
|
-
var
|
|
245
|
+
var ConfigError;
|
|
130
246
|
var init_configFile = __esm({
|
|
131
247
|
"src/config/configFile.ts"() {
|
|
132
248
|
"use strict";
|
|
133
|
-
import_node_path = __toESM(require("path"));
|
|
134
|
-
toml = __toESM(require("toml"));
|
|
135
249
|
ConfigError = class extends Error {
|
|
136
250
|
constructor(message) {
|
|
137
251
|
super(message);
|
|
@@ -140,8 +254,6 @@ var init_configFile = __esm({
|
|
|
140
254
|
};
|
|
141
255
|
}
|
|
142
256
|
});
|
|
143
|
-
|
|
144
|
-
// src/config/runtime.ts
|
|
145
257
|
var runtime_exports = {};
|
|
146
258
|
var ConfigurationRegistry;
|
|
147
259
|
var init_runtime = __esm({
|
|
@@ -276,10 +388,8 @@ var init_runtime = __esm({
|
|
|
276
388
|
globalThis._mooseConfigRegistry = ConfigurationRegistry.getInstance();
|
|
277
389
|
}
|
|
278
390
|
});
|
|
279
|
-
|
|
280
|
-
// src/commons.ts
|
|
281
391
|
var commons_exports = {};
|
|
282
|
-
|
|
392
|
+
__export2(commons_exports, {
|
|
283
393
|
ACKs: () => ACKs,
|
|
284
394
|
MAX_RETRIES: () => MAX_RETRIES,
|
|
285
395
|
MAX_RETRIES_PRODUCER: () => MAX_RETRIES_PRODUCER,
|
|
@@ -306,13 +416,13 @@ function walkDirectory(dir, extensions) {
|
|
|
306
416
|
try {
|
|
307
417
|
const entries = (0, import_fs.readdirSync)(dir, { withFileTypes: true });
|
|
308
418
|
for (const entry of entries) {
|
|
309
|
-
const fullPath =
|
|
419
|
+
const fullPath = import_path2.default.join(dir, entry.name);
|
|
310
420
|
if (entry.isDirectory()) {
|
|
311
421
|
if (entry.name !== "node_modules") {
|
|
312
422
|
results.push(...walkDirectory(fullPath, extensions));
|
|
313
423
|
}
|
|
314
424
|
} else if (entry.isFile()) {
|
|
315
|
-
const ext =
|
|
425
|
+
const ext = import_path2.default.extname(entry.name);
|
|
316
426
|
if (extensions.includes(ext)) {
|
|
317
427
|
results.push(fullPath);
|
|
318
428
|
}
|
|
@@ -331,12 +441,18 @@ function addJsExtensionToImports(content, fileDir) {
|
|
|
331
441
|
result = result.replace(fromPattern, (match, prefix, importPath, quote) => {
|
|
332
442
|
return rewriteImportPath(match, prefix, importPath, quote, fileDir);
|
|
333
443
|
});
|
|
334
|
-
result = result.replace(
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
444
|
+
result = result.replace(
|
|
445
|
+
bareImportPattern,
|
|
446
|
+
(match, prefix, importPath, quote) => {
|
|
447
|
+
return rewriteImportPath(match, prefix, importPath, quote, fileDir);
|
|
448
|
+
}
|
|
449
|
+
);
|
|
450
|
+
result = result.replace(
|
|
451
|
+
dynamicPattern,
|
|
452
|
+
(match, prefix, importPath, quote) => {
|
|
453
|
+
return rewriteImportPath(match, prefix, importPath, quote, fileDir);
|
|
454
|
+
}
|
|
455
|
+
);
|
|
340
456
|
return result;
|
|
341
457
|
}
|
|
342
458
|
function rewriteImportPath(match, prefix, importPath, quote, fileDir) {
|
|
@@ -347,23 +463,23 @@ function rewriteImportPath(match, prefix, importPath, quote, fileDir) {
|
|
|
347
463
|
return match;
|
|
348
464
|
}
|
|
349
465
|
if (fileDir) {
|
|
350
|
-
const resolvedPath =
|
|
466
|
+
const resolvedPath = import_path2.default.resolve(fileDir, importPath);
|
|
351
467
|
if ((0, import_fs.existsSync)(`${resolvedPath}.js`)) {
|
|
352
468
|
return `${prefix}${importPath}.js${quote}`;
|
|
353
469
|
}
|
|
354
|
-
if ((0, import_fs.existsSync)(
|
|
470
|
+
if ((0, import_fs.existsSync)(import_path2.default.join(resolvedPath, "index.js"))) {
|
|
355
471
|
return `${prefix}${importPath}/index.js${quote}`;
|
|
356
472
|
}
|
|
357
473
|
if ((0, import_fs.existsSync)(`${resolvedPath}.mjs`)) {
|
|
358
474
|
return `${prefix}${importPath}.mjs${quote}`;
|
|
359
475
|
}
|
|
360
|
-
if ((0, import_fs.existsSync)(
|
|
476
|
+
if ((0, import_fs.existsSync)(import_path2.default.join(resolvedPath, "index.mjs"))) {
|
|
361
477
|
return `${prefix}${importPath}/index.mjs${quote}`;
|
|
362
478
|
}
|
|
363
479
|
if ((0, import_fs.existsSync)(`${resolvedPath}.cjs`)) {
|
|
364
480
|
return `${prefix}${importPath}.cjs${quote}`;
|
|
365
481
|
}
|
|
366
|
-
if ((0, import_fs.existsSync)(
|
|
482
|
+
if ((0, import_fs.existsSync)(import_path2.default.join(resolvedPath, "index.cjs"))) {
|
|
367
483
|
return `${prefix}${importPath}/index.cjs${quote}`;
|
|
368
484
|
}
|
|
369
485
|
}
|
|
@@ -373,7 +489,7 @@ function rewriteImportExtensions(outDir) {
|
|
|
373
489
|
const files = walkDirectory(outDir, [".js", ".mjs"]);
|
|
374
490
|
for (const filePath of files) {
|
|
375
491
|
const content = (0, import_fs.readFileSync)(filePath, "utf-8");
|
|
376
|
-
const fileDir =
|
|
492
|
+
const fileDir = import_path2.default.dirname(filePath);
|
|
377
493
|
const rewritten = addJsExtensionToImports(content, fileDir);
|
|
378
494
|
if (content !== rewritten) {
|
|
379
495
|
(0, import_fs.writeFileSync)(filePath, rewritten, "utf-8");
|
|
@@ -402,15 +518,15 @@ async function getKafkaProducer(cfg, logger, maxMessageBytes) {
|
|
|
402
518
|
await producer.connect();
|
|
403
519
|
return producer;
|
|
404
520
|
}
|
|
405
|
-
var
|
|
521
|
+
var Kafka;
|
|
522
|
+
var getClickhouseClient;
|
|
523
|
+
var parseBrokerString;
|
|
524
|
+
var buildSaslConfig;
|
|
525
|
+
var getKafkaClient;
|
|
406
526
|
var init_commons = __esm({
|
|
407
527
|
"src/commons.ts"() {
|
|
408
528
|
"use strict";
|
|
409
529
|
init_commons_types();
|
|
410
|
-
import_kafka_javascript = require("@514labs/kafka-javascript");
|
|
411
|
-
import_client = require("@clickhouse/client");
|
|
412
|
-
import_fs = require("fs");
|
|
413
|
-
import_path = __toESM(require("path"));
|
|
414
530
|
init_commons_types();
|
|
415
531
|
({ Kafka } = import_kafka_javascript.KafkaJS);
|
|
416
532
|
getClickhouseClient = ({
|
|
@@ -475,80 +591,6 @@ var init_commons = __esm({
|
|
|
475
591
|
};
|
|
476
592
|
}
|
|
477
593
|
});
|
|
478
|
-
|
|
479
|
-
// src/serverless.ts
|
|
480
|
-
var serverless_exports = {};
|
|
481
|
-
__export(serverless_exports, {
|
|
482
|
-
ACKs: () => ACKs,
|
|
483
|
-
Api: () => Api,
|
|
484
|
-
CSV_DELIMITERS: () => CSV_DELIMITERS,
|
|
485
|
-
ClickHouseEngines: () => ClickHouseEngines,
|
|
486
|
-
ConsumptionApi: () => ConsumptionApi,
|
|
487
|
-
DEFAULT_CSV_CONFIG: () => DEFAULT_CSV_CONFIG,
|
|
488
|
-
DEFAULT_JSON_CONFIG: () => DEFAULT_JSON_CONFIG,
|
|
489
|
-
DataSource: () => DataSource,
|
|
490
|
-
DeadLetterQueue: () => DeadLetterQueue,
|
|
491
|
-
ETLPipeline: () => ETLPipeline,
|
|
492
|
-
IngestApi: () => IngestApi,
|
|
493
|
-
IngestPipeline: () => IngestPipeline,
|
|
494
|
-
LifeCycle: () => LifeCycle,
|
|
495
|
-
MAX_RETRIES: () => MAX_RETRIES,
|
|
496
|
-
MAX_RETRIES_PRODUCER: () => MAX_RETRIES_PRODUCER,
|
|
497
|
-
MAX_RETRY_TIME_MS: () => MAX_RETRY_TIME_MS,
|
|
498
|
-
MOOSE_RUNTIME_ENV_PREFIX: () => MOOSE_RUNTIME_ENV_PREFIX,
|
|
499
|
-
MaterializedView: () => MaterializedView,
|
|
500
|
-
OlapTable: () => OlapTable,
|
|
501
|
-
RETRY_FACTOR_PRODUCER: () => RETRY_FACTOR_PRODUCER,
|
|
502
|
-
RETRY_INITIAL_TIME_MS: () => RETRY_INITIAL_TIME_MS,
|
|
503
|
-
Sql: () => Sql,
|
|
504
|
-
SqlResource: () => SqlResource,
|
|
505
|
-
Stream: () => Stream,
|
|
506
|
-
Task: () => Task,
|
|
507
|
-
View: () => View,
|
|
508
|
-
WebApp: () => WebApp,
|
|
509
|
-
Workflow: () => Workflow,
|
|
510
|
-
antiCachePath: () => antiCachePath,
|
|
511
|
-
cliLog: () => cliLog,
|
|
512
|
-
compilerLog: () => compilerLog,
|
|
513
|
-
createClickhouseParameter: () => createClickhouseParameter,
|
|
514
|
-
getApi: () => getApi,
|
|
515
|
-
getApis: () => getApis,
|
|
516
|
-
getFileName: () => getFileName,
|
|
517
|
-
getIngestApi: () => getIngestApi,
|
|
518
|
-
getIngestApis: () => getIngestApis,
|
|
519
|
-
getMaterializedView: () => getMaterializedView,
|
|
520
|
-
getMaterializedViews: () => getMaterializedViews,
|
|
521
|
-
getSqlResource: () => getSqlResource,
|
|
522
|
-
getSqlResources: () => getSqlResources,
|
|
523
|
-
getStream: () => getStream,
|
|
524
|
-
getStreams: () => getStreams,
|
|
525
|
-
getTable: () => getTable,
|
|
526
|
-
getTables: () => getTables,
|
|
527
|
-
getValueFromParameter: () => getValueFromParameter,
|
|
528
|
-
getView: () => getView,
|
|
529
|
-
getViews: () => getViews,
|
|
530
|
-
getWebApp: () => getWebApp,
|
|
531
|
-
getWebApps: () => getWebApps,
|
|
532
|
-
getWorkflow: () => getWorkflow,
|
|
533
|
-
getWorkflows: () => getWorkflows,
|
|
534
|
-
isValidCSVDelimiter: () => isValidCSVDelimiter,
|
|
535
|
-
logError: () => logError,
|
|
536
|
-
mapToClickHouseType: () => mapToClickHouseType,
|
|
537
|
-
mapTstoJs: () => mapTstoJs,
|
|
538
|
-
mooseEnvSecrets: () => mooseEnvSecrets,
|
|
539
|
-
mooseRuntimeEnv: () => mooseRuntimeEnv,
|
|
540
|
-
parseCSV: () => parseCSV,
|
|
541
|
-
parseJSON: () => parseJSON,
|
|
542
|
-
parseJSONWithDates: () => parseJSONWithDates,
|
|
543
|
-
quoteIdentifier: () => quoteIdentifier,
|
|
544
|
-
sql: () => sql,
|
|
545
|
-
toQuery: () => toQuery,
|
|
546
|
-
toQueryPreview: () => toQueryPreview,
|
|
547
|
-
toStaticQuery: () => toStaticQuery
|
|
548
|
-
});
|
|
549
|
-
module.exports = __toCommonJS(serverless_exports);
|
|
550
|
-
|
|
551
|
-
// src/dataModels/types.ts
|
|
552
594
|
var ClickHouseEngines = /* @__PURE__ */ ((ClickHouseEngines2) => {
|
|
553
595
|
ClickHouseEngines2["MergeTree"] = "MergeTree";
|
|
554
596
|
ClickHouseEngines2["ReplacingMergeTree"] = "ReplacingMergeTree";
|
|
@@ -571,8 +613,6 @@ var ClickHouseEngines = /* @__PURE__ */ ((ClickHouseEngines2) => {
|
|
|
571
613
|
ClickHouseEngines2["ReplicatedVersionedCollapsingMergeTree"] = "ReplicatedVersionedCollapsingMergeTree";
|
|
572
614
|
return ClickHouseEngines2;
|
|
573
615
|
})(ClickHouseEngines || {});
|
|
574
|
-
|
|
575
|
-
// src/dmv2/utils/stackTrace.ts
|
|
576
616
|
function shouldSkipStackLine(line) {
|
|
577
617
|
return line.includes("node_modules") || // Skip npm installed packages (prod)
|
|
578
618
|
line.includes("node:internal") || // Skip Node.js internals (modern format)
|
|
@@ -633,8 +673,6 @@ function getSourceFileFromStack(stack) {
|
|
|
633
673
|
const location = getSourceLocationFromStack(stack);
|
|
634
674
|
return location?.file;
|
|
635
675
|
}
|
|
636
|
-
|
|
637
|
-
// src/dmv2/typedBase.ts
|
|
638
676
|
var TypedBase = class {
|
|
639
677
|
/** The JSON schema representation of type T. Injected by the compiler plugin. */
|
|
640
678
|
schema;
|
|
@@ -693,17 +731,12 @@ var TypedBase = class {
|
|
|
693
731
|
}
|
|
694
732
|
}
|
|
695
733
|
};
|
|
696
|
-
|
|
697
|
-
// src/dataModels/dataModelTypes.ts
|
|
698
734
|
function isArrayNestedType(dt) {
|
|
699
735
|
return typeof dt === "object" && dt !== null && dt.elementType !== null && typeof dt.elementType === "object" && dt.elementType.hasOwnProperty("columns") && Array.isArray(dt.elementType.columns);
|
|
700
736
|
}
|
|
701
737
|
function isNestedType(dt) {
|
|
702
738
|
return typeof dt === "object" && dt !== null && Array.isArray(dt.columns);
|
|
703
739
|
}
|
|
704
|
-
|
|
705
|
-
// src/dmv2/internal.ts
|
|
706
|
-
var import_process = __toESM(require("process"));
|
|
707
740
|
init_commons_types();
|
|
708
741
|
var isClientOnlyMode = () => import_process.default.env.MOOSE_CLIENT_ONLY === "true";
|
|
709
742
|
var moose_internal = {
|
|
@@ -756,7 +789,13 @@ var dlqSchema = {
|
|
|
756
789
|
]
|
|
757
790
|
}
|
|
758
791
|
},
|
|
759
|
-
required: [
|
|
792
|
+
required: [
|
|
793
|
+
"originalRecord",
|
|
794
|
+
"errorMessage",
|
|
795
|
+
"errorType",
|
|
796
|
+
"failedAt",
|
|
797
|
+
"source"
|
|
798
|
+
]
|
|
760
799
|
},
|
|
761
800
|
Recordstringany: {
|
|
762
801
|
type: "object",
|
|
@@ -840,12 +879,6 @@ var dlqColumns = [
|
|
|
840
879
|
comment: null
|
|
841
880
|
}
|
|
842
881
|
];
|
|
843
|
-
|
|
844
|
-
// src/dmv2/sdk/olapTable.ts
|
|
845
|
-
var import_node_stream = require("stream");
|
|
846
|
-
var import_node_crypto = require("crypto");
|
|
847
|
-
|
|
848
|
-
// src/sqlHelpers.ts
|
|
849
882
|
var quoteIdentifier = (name) => {
|
|
850
883
|
return name.startsWith("`") && name.endsWith("`") ? name : `\`${name}\``;
|
|
851
884
|
};
|
|
@@ -1016,8 +1049,6 @@ var mapToClickHouseType = (value) => {
|
|
|
1016
1049
|
function emptyIfUndefined(value) {
|
|
1017
1050
|
return value === void 0 ? "" : value;
|
|
1018
1051
|
}
|
|
1019
|
-
|
|
1020
|
-
// src/dmv2/sdk/olapTable.ts
|
|
1021
1052
|
var OlapTable = class extends TypedBase {
|
|
1022
1053
|
name;
|
|
1023
1054
|
/** @internal */
|
|
@@ -1029,7 +1060,14 @@ var OlapTable = class extends TypedBase {
|
|
|
1029
1060
|
/** @internal Cached table name to avoid repeated generation */
|
|
1030
1061
|
_cachedTableName;
|
|
1031
1062
|
constructor(name, config, schema, columns, validators) {
|
|
1032
|
-
const resolvedConfig = config ? "engine" in config ? config : {
|
|
1063
|
+
const resolvedConfig = config ? "engine" in config ? config : {
|
|
1064
|
+
...config,
|
|
1065
|
+
engine: "MergeTree"
|
|
1066
|
+
/* MergeTree */
|
|
1067
|
+
} : {
|
|
1068
|
+
engine: "MergeTree"
|
|
1069
|
+
/* MergeTree */
|
|
1070
|
+
};
|
|
1033
1071
|
const hasFields = Array.isArray(resolvedConfig.orderByFields) && resolvedConfig.orderByFields.length > 0;
|
|
1034
1072
|
const hasExpr = typeof resolvedConfig.orderByExpression === "string" && resolvedConfig.orderByExpression.length > 0;
|
|
1035
1073
|
if (hasFields && hasExpr) {
|
|
@@ -1082,7 +1120,7 @@ var OlapTable = class extends TypedBase {
|
|
|
1082
1120
|
createConfigHash(clickhouseConfig) {
|
|
1083
1121
|
const effectiveDatabase = this.config.database ?? clickhouseConfig.database;
|
|
1084
1122
|
const configString = `${clickhouseConfig.host}:${clickhouseConfig.port}:${clickhouseConfig.username}:${clickhouseConfig.password}:${effectiveDatabase}:${clickhouseConfig.useSSL}`;
|
|
1085
|
-
return (0,
|
|
1123
|
+
return (0, import_crypto.createHash)("sha256").update(configString).digest("hex").substring(0, 16);
|
|
1086
1124
|
}
|
|
1087
1125
|
/**
|
|
1088
1126
|
* Gets or creates a memoized ClickHouse client.
|
|
@@ -1292,7 +1330,7 @@ var OlapTable = class extends TypedBase {
|
|
|
1292
1330
|
* @private
|
|
1293
1331
|
*/
|
|
1294
1332
|
validateInsertParameters(data, options) {
|
|
1295
|
-
const isStream = data instanceof
|
|
1333
|
+
const isStream = data instanceof import_stream.Readable;
|
|
1296
1334
|
const strategy = options?.strategy || "fail-fast";
|
|
1297
1335
|
const shouldValidate = options?.validate !== false;
|
|
1298
1336
|
if (isStream && strategy === "isolate") {
|
|
@@ -1699,9 +1737,6 @@ var OlapTable = class extends TypedBase {
|
|
|
1699
1737
|
// were removed in ENG-856. Use direct configuration instead, e.g.:
|
|
1700
1738
|
// new OlapTable(name, { engine: ClickHouseEngines.ReplacingMergeTree, orderByFields: ["id"], ver: "updated_at" })
|
|
1701
1739
|
};
|
|
1702
|
-
|
|
1703
|
-
// src/dmv2/sdk/stream.ts
|
|
1704
|
-
var import_node_crypto2 = require("crypto");
|
|
1705
1740
|
var RoutedMessage = class {
|
|
1706
1741
|
/** The destination stream for the message */
|
|
1707
1742
|
destination;
|
|
@@ -1775,7 +1810,7 @@ var Stream = class extends TypedBase {
|
|
|
1775
1810
|
kafkaConfig.securityProtocol,
|
|
1776
1811
|
kafkaConfig.namespace
|
|
1777
1812
|
].join(":");
|
|
1778
|
-
return (0,
|
|
1813
|
+
return (0, import_crypto2.createHash)("sha256").update(configString).digest("hex").substring(0, 16);
|
|
1779
1814
|
}
|
|
1780
1815
|
/**
|
|
1781
1816
|
* Gets or creates a memoized KafkaJS producer using runtime configuration.
|
|
@@ -1855,7 +1890,7 @@ var Stream = class extends TypedBase {
|
|
|
1855
1890
|
}
|
|
1856
1891
|
const {
|
|
1857
1892
|
default: { SchemaRegistry }
|
|
1858
|
-
} = await
|
|
1893
|
+
} = await Promise.resolve().then(() => __toESM(require_confluent_schema_registry(), 1));
|
|
1859
1894
|
const registry = new SchemaRegistry({ host: schemaRegistryUrl });
|
|
1860
1895
|
let schemaId;
|
|
1861
1896
|
if ("id" in sr.reference) {
|
|
@@ -1863,13 +1898,18 @@ var Stream = class extends TypedBase {
|
|
|
1863
1898
|
} else if ("subjectLatest" in sr.reference) {
|
|
1864
1899
|
schemaId = await registry.getLatestSchemaId(sr.reference.subjectLatest);
|
|
1865
1900
|
} else if ("subject" in sr.reference) {
|
|
1866
|
-
schemaId = await registry.getRegistryId(
|
|
1901
|
+
schemaId = await registry.getRegistryId(
|
|
1902
|
+
sr.reference.subject,
|
|
1903
|
+
sr.reference.version
|
|
1904
|
+
);
|
|
1867
1905
|
}
|
|
1868
1906
|
if (schemaId === void 0) {
|
|
1869
1907
|
throw new Error("Malformed schema reference.");
|
|
1870
1908
|
}
|
|
1871
1909
|
const encoded = await Promise.all(
|
|
1872
|
-
flat.map(
|
|
1910
|
+
flat.map(
|
|
1911
|
+
(v) => registry.encode(schemaId, v)
|
|
1912
|
+
)
|
|
1873
1913
|
);
|
|
1874
1914
|
await producer.send({
|
|
1875
1915
|
topic,
|
|
@@ -1912,7 +1952,9 @@ var Stream = class extends TypedBase {
|
|
|
1912
1952
|
existingTransforms.push([destination, transformation, transformConfig]);
|
|
1913
1953
|
}
|
|
1914
1954
|
} else {
|
|
1915
|
-
this._transformations.set(destination.name, [
|
|
1955
|
+
this._transformations.set(destination.name, [
|
|
1956
|
+
[destination, transformation, transformConfig]
|
|
1957
|
+
]);
|
|
1916
1958
|
}
|
|
1917
1959
|
}
|
|
1918
1960
|
/**
|
|
@@ -2028,8 +2070,6 @@ var DeadLetterQueue = class extends Stream {
|
|
|
2028
2070
|
super.addMultiTransform(withValidate);
|
|
2029
2071
|
}
|
|
2030
2072
|
};
|
|
2031
|
-
|
|
2032
|
-
// src/dmv2/sdk/workflow.ts
|
|
2033
2073
|
var Task = class {
|
|
2034
2074
|
/**
|
|
2035
2075
|
* Creates a new Task instance.
|
|
@@ -2137,8 +2177,6 @@ var Workflow = class {
|
|
|
2137
2177
|
validateTask(startingTask, []);
|
|
2138
2178
|
}
|
|
2139
2179
|
};
|
|
2140
|
-
|
|
2141
|
-
// src/dmv2/sdk/ingestApi.ts
|
|
2142
2180
|
var IngestApi = class extends TypedBase {
|
|
2143
2181
|
constructor(name, config, schema, columns, validators, allowExtraFields) {
|
|
2144
2182
|
super(name, config, schema, columns, void 0, allowExtraFields);
|
|
@@ -2149,8 +2187,6 @@ var IngestApi = class extends TypedBase {
|
|
|
2149
2187
|
ingestApis.set(name, this);
|
|
2150
2188
|
}
|
|
2151
2189
|
};
|
|
2152
|
-
|
|
2153
|
-
// src/dmv2/sdk/consumptionApi.ts
|
|
2154
2190
|
var Api = class extends TypedBase {
|
|
2155
2191
|
/** @internal The handler function that processes requests and generates responses. */
|
|
2156
2192
|
_handler;
|
|
@@ -2257,8 +2293,6 @@ var Api = class extends TypedBase {
|
|
|
2257
2293
|
}
|
|
2258
2294
|
};
|
|
2259
2295
|
var ConsumptionApi = Api;
|
|
2260
|
-
|
|
2261
|
-
// src/dmv2/sdk/ingestPipeline.ts
|
|
2262
2296
|
var IngestPipeline = class extends TypedBase {
|
|
2263
2297
|
/**
|
|
2264
2298
|
* The OLAP table component of the pipeline, if configured.
|
|
@@ -2297,7 +2331,7 @@ var IngestPipeline = class extends TypedBase {
|
|
|
2297
2331
|
...config.version && { version: config.version }
|
|
2298
2332
|
} : {
|
|
2299
2333
|
lifeCycle: config.lifeCycle,
|
|
2300
|
-
engine: "MergeTree"
|
|
2334
|
+
engine: "MergeTree",
|
|
2301
2335
|
...config.version && { version: config.version }
|
|
2302
2336
|
};
|
|
2303
2337
|
this.table = new OlapTable(
|
|
@@ -2367,8 +2401,6 @@ var IngestPipeline = class extends TypedBase {
|
|
|
2367
2401
|
}
|
|
2368
2402
|
}
|
|
2369
2403
|
};
|
|
2370
|
-
|
|
2371
|
-
// src/dmv2/sdk/etlPipeline.ts
|
|
2372
2404
|
var InternalBatcher = class {
|
|
2373
2405
|
iterator;
|
|
2374
2406
|
batchSize;
|
|
@@ -2509,8 +2541,6 @@ var ETLPipeline = class {
|
|
|
2509
2541
|
console.log(`Completed ETL Pipeline: ${this.name}`);
|
|
2510
2542
|
}
|
|
2511
2543
|
};
|
|
2512
|
-
|
|
2513
|
-
// src/dmv2/sdk/materializedView.ts
|
|
2514
2544
|
function formatTableReference(table) {
|
|
2515
2545
|
const database = table instanceof OlapTable ? table.config.database : void 0;
|
|
2516
2546
|
if (database) {
|
|
@@ -2554,7 +2584,8 @@ var MaterializedView = class {
|
|
|
2554
2584
|
),
|
|
2555
2585
|
{
|
|
2556
2586
|
orderByFields: options.targetTable?.orderByFields ?? options.orderByFields,
|
|
2557
|
-
engine: options.targetTable?.engine ?? options.engine ?? "MergeTree"
|
|
2587
|
+
engine: options.targetTable?.engine ?? options.engine ?? "MergeTree"
|
|
2588
|
+
/* MergeTree */
|
|
2558
2589
|
},
|
|
2559
2590
|
targetSchema,
|
|
2560
2591
|
targetColumns
|
|
@@ -2585,8 +2616,6 @@ var MaterializedView = class {
|
|
|
2585
2616
|
materializedViews.set(this.name, this);
|
|
2586
2617
|
}
|
|
2587
2618
|
};
|
|
2588
|
-
|
|
2589
|
-
// src/dmv2/sdk/sqlResource.ts
|
|
2590
2619
|
var SqlResource = class {
|
|
2591
2620
|
/** @internal */
|
|
2592
2621
|
kind = "SqlResource";
|
|
@@ -2639,8 +2668,6 @@ var SqlResource = class {
|
|
|
2639
2668
|
}
|
|
2640
2669
|
}
|
|
2641
2670
|
};
|
|
2642
|
-
|
|
2643
|
-
// src/dmv2/sdk/view.ts
|
|
2644
2671
|
function formatTableReference2(table) {
|
|
2645
2672
|
const database = table instanceof OlapTable ? table.config.database : void 0;
|
|
2646
2673
|
if (database) {
|
|
@@ -2688,16 +2715,12 @@ var View = class {
|
|
|
2688
2715
|
views.set(this.name, this);
|
|
2689
2716
|
}
|
|
2690
2717
|
};
|
|
2691
|
-
|
|
2692
|
-
// src/dmv2/sdk/lifeCycle.ts
|
|
2693
2718
|
var LifeCycle = /* @__PURE__ */ ((LifeCycle2) => {
|
|
2694
2719
|
LifeCycle2["FULLY_MANAGED"] = "FULLY_MANAGED";
|
|
2695
2720
|
LifeCycle2["DELETION_PROTECTED"] = "DELETION_PROTECTED";
|
|
2696
2721
|
LifeCycle2["EXTERNALLY_MANAGED"] = "EXTERNALLY_MANAGED";
|
|
2697
2722
|
return LifeCycle2;
|
|
2698
2723
|
})(LifeCycle || {});
|
|
2699
|
-
|
|
2700
|
-
// src/dmv2/sdk/webApp.ts
|
|
2701
2724
|
var RESERVED_MOUNT_PATHS = [
|
|
2702
2725
|
"/admin",
|
|
2703
2726
|
"/api",
|
|
@@ -2810,8 +2833,6 @@ Examples:
|
|
|
2810
2833
|
return this._rawApp;
|
|
2811
2834
|
}
|
|
2812
2835
|
};
|
|
2813
|
-
|
|
2814
|
-
// src/dmv2/registry.ts
|
|
2815
2836
|
function getTables() {
|
|
2816
2837
|
return getMooseInternal().tables;
|
|
2817
2838
|
}
|
|
@@ -2887,11 +2908,7 @@ function getViews() {
|
|
|
2887
2908
|
function getView(name) {
|
|
2888
2909
|
return getMooseInternal().views.get(name);
|
|
2889
2910
|
}
|
|
2890
|
-
|
|
2891
|
-
// src/serverless.ts
|
|
2892
2911
|
init_commons_types();
|
|
2893
|
-
|
|
2894
|
-
// src/connectors/dataSource.ts
|
|
2895
2912
|
var DataSource = class {
|
|
2896
2913
|
name;
|
|
2897
2914
|
supportsIncremental;
|
|
@@ -2900,8 +2917,6 @@ var DataSource = class {
|
|
|
2900
2917
|
this.supportsIncremental = config.supportsIncremental ?? false;
|
|
2901
2918
|
}
|
|
2902
2919
|
};
|
|
2903
|
-
|
|
2904
|
-
// src/secrets.ts
|
|
2905
2920
|
var MOOSE_RUNTIME_ENV_PREFIX = "__MOOSE_RUNTIME_ENV__:";
|
|
2906
2921
|
var mooseRuntimeEnv = {
|
|
2907
2922
|
/**
|
|
@@ -2946,11 +2961,6 @@ var mooseRuntimeEnv = {
|
|
|
2946
2961
|
}
|
|
2947
2962
|
};
|
|
2948
2963
|
var mooseEnvSecrets = mooseRuntimeEnv;
|
|
2949
|
-
|
|
2950
|
-
// src/utilities/dataParser.ts
|
|
2951
|
-
var import_csv_parse = require("csv-parse");
|
|
2952
|
-
|
|
2953
|
-
// src/utilities/json.ts
|
|
2954
2964
|
function jsonDateReviver(key, value) {
|
|
2955
2965
|
const iso8601Format = /^([\+-]?\d{4}(?!\d{2}\b))((-?)((0[1-9]|1[0-2])(\3([12]\d|0[1-9]|3[01]))?|W([0-4]\d|5[0-2])(-?[1-7])?|(00[1-9]|0[1-9]\d|[12]\d{2}|3([0-5]\d|6[1-6])))([T\s]((([01]\d|2[0-3])((:?)[0-5]\d)?|24\:?00)([\.,]\d+(?!:))?)?(\17[0-5]\d([\.,]\d+)?)?([zZ]|([\+-])([01]\d|2[0-3]):?([0-5]\d)?)?)?)$/;
|
|
2956
2966
|
if (typeof value === "string" && iso8601Format.test(value)) {
|
|
@@ -2958,8 +2968,6 @@ function jsonDateReviver(key, value) {
|
|
|
2958
2968
|
}
|
|
2959
2969
|
return value;
|
|
2960
2970
|
}
|
|
2961
|
-
|
|
2962
|
-
// src/utilities/dataParser.ts
|
|
2963
2971
|
function parseCSV(content, config) {
|
|
2964
2972
|
return new Promise((resolve, reject) => {
|
|
2965
2973
|
const results = [];
|
|
@@ -3081,4 +3089,4 @@ var DEFAULT_JSON_CONFIG = {
|
|
|
3081
3089
|
toQueryPreview,
|
|
3082
3090
|
toStaticQuery
|
|
3083
3091
|
});
|
|
3084
|
-
//# sourceMappingURL=
|
|
3092
|
+
//# sourceMappingURL=index.js.map
|