@514labs/moose-lib 0.6.348 → 0.6.349

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -8,183 +8,6 @@ var __export = (target, all) => {
8
8
  __defProp(target, name, { get: all[name], enumerable: true });
9
9
  };
10
10
 
11
- // src/dmv2/utils/stackTrace.ts
12
- function shouldSkipStackLine(line) {
13
- return line.includes("node_modules") || // Skip npm installed packages (prod)
14
- line.includes("node:internal") || // Skip Node.js internals (modern format)
15
- line.includes("internal/modules") || // Skip Node.js internals (older format)
16
- line.includes("ts-node") || // Skip TypeScript execution
17
- line.includes("/ts-moose-lib/src/") || // Skip dev/linked moose-lib src (Unix)
18
- line.includes("\\ts-moose-lib\\src\\") || // Skip dev/linked moose-lib src (Windows)
19
- line.includes("/ts-moose-lib/dist/") || // Skip dev/linked moose-lib dist (Unix)
20
- line.includes("\\ts-moose-lib\\dist\\");
21
- }
22
- function parseStackLine(line) {
23
- const match = line.match(/\((.*):(\d+):(\d+)\)/) || line.match(/at (.*):(\d+):(\d+)/);
24
- if (match && match[1]) {
25
- return {
26
- file: match[1],
27
- line: match[2]
28
- };
29
- }
30
- return void 0;
31
- }
32
- function getSourceFileInfo(stack) {
33
- if (!stack) return {};
34
- const lines = stack.split("\n");
35
- for (const line of lines) {
36
- if (shouldSkipStackLine(line)) continue;
37
- const info = parseStackLine(line);
38
- if (info) return info;
39
- }
40
- return {};
41
- }
42
- function getSourceLocationFromStack(stack) {
43
- if (!stack) return void 0;
44
- const lines = stack.split("\n");
45
- for (const line of lines.slice(1)) {
46
- if (shouldSkipStackLine(line)) {
47
- continue;
48
- }
49
- const v8Match = line.match(/at\s+(?:.*?\s+\()?(.+):(\d+):(\d+)\)?/);
50
- if (v8Match) {
51
- return {
52
- file: v8Match[1],
53
- line: parseInt(v8Match[2], 10),
54
- column: parseInt(v8Match[3], 10)
55
- };
56
- }
57
- const smMatch = line.match(/(?:.*@)?(.+):(\d+):(\d+)/);
58
- if (smMatch) {
59
- return {
60
- file: smMatch[1],
61
- line: parseInt(smMatch[2], 10),
62
- column: parseInt(smMatch[3], 10)
63
- };
64
- }
65
- }
66
- return void 0;
67
- }
68
- function getSourceFileFromStack(stack) {
69
- const location = getSourceLocationFromStack(stack);
70
- return location?.file;
71
- }
72
- var init_stackTrace = __esm({
73
- "src/dmv2/utils/stackTrace.ts"() {
74
- "use strict";
75
- }
76
- });
77
-
78
- // src/dmv2/typedBase.ts
79
- var TypedBase;
80
- var init_typedBase = __esm({
81
- "src/dmv2/typedBase.ts"() {
82
- "use strict";
83
- init_stackTrace();
84
- TypedBase = class {
85
- /** The JSON schema representation of type T. Injected by the compiler plugin. */
86
- schema;
87
- /** The name assigned to this resource instance. */
88
- name;
89
- /** A dictionary mapping column names (keys of T) to their Column definitions. */
90
- columns;
91
- /** An array containing the Column definitions for this resource. Injected by the compiler plugin. */
92
- columnArray;
93
- /** The configuration object specific to this resource type. */
94
- config;
95
- /** Typia validation functions for type T. Injected by the compiler plugin for OlapTable. */
96
- validators;
97
- /** Optional metadata for the resource, always present as an object. */
98
- metadata;
99
- /**
100
- * Whether this resource allows extra fields beyond the defined columns.
101
- * When true, extra fields in payloads are passed through to streaming functions.
102
- * Injected by the compiler plugin when the type has an index signature.
103
- */
104
- allowExtraFields;
105
- /**
106
- * @internal Constructor intended for internal use by subclasses and the compiler plugin.
107
- * It expects the schema and columns to be provided, typically injected by the compiler.
108
- *
109
- * @param name The name for the resource instance.
110
- * @param config The configuration object for the resource.
111
- * @param schema The JSON schema for the resource's data type T (injected).
112
- * @param columns The array of Column definitions for T (injected).
113
- * @param allowExtraFields Whether extra fields are allowed (injected when type has index signature).
114
- */
115
- constructor(name, config, schema, columns, validators, allowExtraFields) {
116
- if (schema === void 0 || columns === void 0) {
117
- throw new Error(
118
- "Supply the type param T so that the schema is inserted by the compiler plugin."
119
- );
120
- }
121
- this.schema = schema;
122
- this.columnArray = columns;
123
- const columnsObj = {};
124
- columns.forEach((column) => {
125
- columnsObj[column.name] = column;
126
- });
127
- this.columns = columnsObj;
128
- this.name = name;
129
- this.config = config;
130
- this.validators = validators;
131
- this.allowExtraFields = allowExtraFields ?? false;
132
- this.metadata = config?.metadata ? { ...config.metadata } : {};
133
- if (!this.metadata.source) {
134
- const stack = new Error().stack;
135
- if (stack) {
136
- const info = getSourceFileInfo(stack);
137
- this.metadata.source = { file: info.file, line: info.line };
138
- }
139
- }
140
- }
141
- };
142
- }
143
- });
144
-
145
- // src/dataModels/dataModelTypes.ts
146
- function isArrayNestedType(dt) {
147
- return typeof dt === "object" && dt !== null && dt.elementType !== null && typeof dt.elementType === "object" && dt.elementType.hasOwnProperty("columns") && Array.isArray(dt.elementType.columns);
148
- }
149
- function isNestedType(dt) {
150
- return typeof dt === "object" && dt !== null && Array.isArray(dt.columns);
151
- }
152
- var init_dataModelTypes = __esm({
153
- "src/dataModels/dataModelTypes.ts"() {
154
- "use strict";
155
- }
156
- });
157
-
158
- // src/dataModels/types.ts
159
- var ClickHouseEngines;
160
- var init_types = __esm({
161
- "src/dataModels/types.ts"() {
162
- "use strict";
163
- ClickHouseEngines = /* @__PURE__ */ ((ClickHouseEngines2) => {
164
- ClickHouseEngines2["MergeTree"] = "MergeTree";
165
- ClickHouseEngines2["ReplacingMergeTree"] = "ReplacingMergeTree";
166
- ClickHouseEngines2["SummingMergeTree"] = "SummingMergeTree";
167
- ClickHouseEngines2["AggregatingMergeTree"] = "AggregatingMergeTree";
168
- ClickHouseEngines2["CollapsingMergeTree"] = "CollapsingMergeTree";
169
- ClickHouseEngines2["VersionedCollapsingMergeTree"] = "VersionedCollapsingMergeTree";
170
- ClickHouseEngines2["GraphiteMergeTree"] = "GraphiteMergeTree";
171
- ClickHouseEngines2["S3Queue"] = "S3Queue";
172
- ClickHouseEngines2["S3"] = "S3";
173
- ClickHouseEngines2["Buffer"] = "Buffer";
174
- ClickHouseEngines2["Distributed"] = "Distributed";
175
- ClickHouseEngines2["IcebergS3"] = "IcebergS3";
176
- ClickHouseEngines2["Kafka"] = "Kafka";
177
- ClickHouseEngines2["ReplicatedMergeTree"] = "ReplicatedMergeTree";
178
- ClickHouseEngines2["ReplicatedReplacingMergeTree"] = "ReplicatedReplacingMergeTree";
179
- ClickHouseEngines2["ReplicatedAggregatingMergeTree"] = "ReplicatedAggregatingMergeTree";
180
- ClickHouseEngines2["ReplicatedSummingMergeTree"] = "ReplicatedSummingMergeTree";
181
- ClickHouseEngines2["ReplicatedCollapsingMergeTree"] = "ReplicatedCollapsingMergeTree";
182
- ClickHouseEngines2["ReplicatedVersionedCollapsingMergeTree"] = "ReplicatedVersionedCollapsingMergeTree";
183
- return ClickHouseEngines2;
184
- })(ClickHouseEngines || {});
185
- }
186
- });
187
-
188
11
  // src/commons.ts
189
12
  var commons_exports = {};
190
13
  __export(commons_exports, {
@@ -448,486 +271,6 @@ var init_commons = __esm({
448
271
  }
449
272
  });
450
273
 
451
- // src/secrets.ts
452
- var init_secrets = __esm({
453
- "src/secrets.ts"() {
454
- "use strict";
455
- }
456
- });
457
-
458
- // src/sqlHelpers.ts
459
- function sqlImpl(strings, ...values) {
460
- return new Sql(strings, values);
461
- }
462
- function createClickhouseParameter(parameterIndex, value) {
463
- return `{p${parameterIndex}:${mapToClickHouseType(value)}}`;
464
- }
465
- function emptyIfUndefined(value) {
466
- return value === void 0 ? "" : value;
467
- }
468
- var quoteIdentifier, isTable, isView, isColumn, sql, instanceofSql, Sql, toStaticQuery, toQuery, toQueryPreview, getValueFromParameter, mapToClickHouseType;
469
- var init_sqlHelpers = __esm({
470
- "src/sqlHelpers.ts"() {
471
- "use strict";
472
- quoteIdentifier = (name) => {
473
- return name.startsWith("`") && name.endsWith("`") ? name : `\`${name}\``;
474
- };
475
- isTable = (value) => typeof value === "object" && value !== null && "kind" in value && value.kind === "OlapTable";
476
- isView = (value) => typeof value === "object" && value !== null && "kind" in value && value.kind === "View";
477
- isColumn = (value) => typeof value === "object" && value !== null && !("kind" in value) && "name" in value && "annotations" in value;
478
- sql = sqlImpl;
479
- instanceofSql = (value) => typeof value === "object" && "values" in value && "strings" in value;
480
- Sql = class _Sql {
481
- values;
482
- strings;
483
- constructor(rawStrings, rawValues) {
484
- if (rawStrings.length - 1 !== rawValues.length) {
485
- if (rawStrings.length === 0) {
486
- throw new TypeError("Expected at least 1 string");
487
- }
488
- throw new TypeError(
489
- `Expected ${rawStrings.length} strings to have ${rawStrings.length - 1} values`
490
- );
491
- }
492
- const valuesLength = rawValues.reduce(
493
- (len, value) => len + (instanceofSql(value) ? value.values.length : isColumn(value) || isTable(value) || isView(value) ? 0 : 1),
494
- 0
495
- );
496
- this.values = new Array(valuesLength);
497
- this.strings = new Array(valuesLength + 1);
498
- this.strings[0] = rawStrings[0];
499
- let i = 0, pos = 0;
500
- while (i < rawValues.length) {
501
- const child = rawValues[i++];
502
- const rawString = rawStrings[i];
503
- if (instanceofSql(child)) {
504
- this.strings[pos] += child.strings[0];
505
- let childIndex = 0;
506
- while (childIndex < child.values.length) {
507
- this.values[pos++] = child.values[childIndex++];
508
- this.strings[pos] = child.strings[childIndex];
509
- }
510
- this.strings[pos] += rawString;
511
- } else if (isColumn(child)) {
512
- const aggregationFunction = child.annotations.find(
513
- ([k, _]) => k === "aggregationFunction"
514
- );
515
- if (aggregationFunction !== void 0) {
516
- this.strings[pos] += `${aggregationFunction[1].functionName}Merge(\`${child.name}\`)`;
517
- } else {
518
- this.strings[pos] += `\`${child.name}\``;
519
- }
520
- this.strings[pos] += rawString;
521
- } else if (isTable(child)) {
522
- if (child.config.database) {
523
- this.strings[pos] += `\`${child.config.database}\`.\`${child.name}\``;
524
- } else {
525
- this.strings[pos] += `\`${child.name}\``;
526
- }
527
- this.strings[pos] += rawString;
528
- } else if (isView(child)) {
529
- this.strings[pos] += `\`${child.name}\``;
530
- this.strings[pos] += rawString;
531
- } else {
532
- this.values[pos++] = child;
533
- this.strings[pos] = rawString;
534
- }
535
- }
536
- }
537
- /**
538
- * Append another Sql fragment, returning a new Sql instance.
539
- */
540
- append(other) {
541
- return new _Sql([...this.strings, ""], [...this.values, other]);
542
- }
543
- };
544
- sql.join = function(fragments, separator) {
545
- if (fragments.length === 0) return new Sql([""], []);
546
- if (fragments.length === 1) return fragments[0];
547
- const sep = separator ?? ", ";
548
- const normalized = sep.includes(" ") ? sep : ` ${sep} `;
549
- const strings = ["", ...Array(fragments.length - 1).fill(normalized), ""];
550
- return new Sql(strings, fragments);
551
- };
552
- sql.raw = function(text) {
553
- return new Sql([text], []);
554
- };
555
- toStaticQuery = (sql3) => {
556
- const [query, params] = toQuery(sql3);
557
- if (Object.keys(params).length !== 0) {
558
- throw new Error(
559
- "Dynamic SQL is not allowed in the select statement in view creation."
560
- );
561
- }
562
- return query;
563
- };
564
- toQuery = (sql3) => {
565
- const parameterizedStubs = sql3.values.map(
566
- (v, i) => createClickhouseParameter(i, v)
567
- );
568
- const query = sql3.strings.map(
569
- (s, i) => s != "" ? `${s}${emptyIfUndefined(parameterizedStubs[i])}` : ""
570
- ).join("");
571
- const query_params = sql3.values.reduce(
572
- (acc, v, i) => ({
573
- ...acc,
574
- [`p${i}`]: getValueFromParameter(v)
575
- }),
576
- {}
577
- );
578
- return [query, query_params];
579
- };
580
- toQueryPreview = (sql3) => {
581
- try {
582
- const formatValue = (v) => {
583
- if (Array.isArray(v)) {
584
- const [type, val] = v;
585
- if (type === "Identifier") {
586
- return `\`${String(val)}\``;
587
- }
588
- return `[${v.map((x) => formatValue(x)).join(", ")}]`;
589
- }
590
- if (v === null || v === void 0) return "NULL";
591
- if (typeof v === "string") return `'${v.replace(/'/g, "''")}'`;
592
- if (typeof v === "number") return String(v);
593
- if (typeof v === "boolean") return v ? "true" : "false";
594
- if (v instanceof Date)
595
- return `'${v.toISOString().replace("T", " ").slice(0, 19)}'`;
596
- try {
597
- return JSON.stringify(v);
598
- } catch {
599
- return String(v);
600
- }
601
- };
602
- let out = sql3.strings[0] ?? "";
603
- for (let i = 0; i < sql3.values.length; i++) {
604
- const val = getValueFromParameter(sql3.values[i]);
605
- out += formatValue(val);
606
- out += sql3.strings[i + 1] ?? "";
607
- }
608
- return out.replace(/\s+/g, " ").trim();
609
- } catch (error) {
610
- console.log(`toQueryPreview error: ${error}`);
611
- return "/* query preview unavailable */";
612
- }
613
- };
614
- getValueFromParameter = (value) => {
615
- if (Array.isArray(value)) {
616
- const [type, val] = value;
617
- if (type === "Identifier") return val;
618
- }
619
- return value;
620
- };
621
- mapToClickHouseType = (value) => {
622
- if (typeof value === "number") {
623
- return Number.isInteger(value) ? "Int" : "Float";
624
- }
625
- if (typeof value === "boolean") return "Bool";
626
- if (value instanceof Date) return "DateTime";
627
- if (Array.isArray(value)) {
628
- const [type, _] = value;
629
- return type;
630
- }
631
- return "String";
632
- };
633
- }
634
- });
635
-
636
- // src/consumption-apis/helpers.ts
637
- import {
638
- Client as TemporalClient,
639
- Connection
640
- } from "@temporalio/client";
641
- import { createHash, randomUUID } from "crypto";
642
- var init_helpers = __esm({
643
- "src/consumption-apis/helpers.ts"() {
644
- "use strict";
645
- init_internal();
646
- init_sqlHelpers();
647
- }
648
- });
649
-
650
- // src/consumption-apis/webAppHelpers.ts
651
- var init_webAppHelpers = __esm({
652
- "src/consumption-apis/webAppHelpers.ts"() {
653
- "use strict";
654
- }
655
- });
656
-
657
- // src/scripts/task.ts
658
- var init_task = __esm({
659
- "src/scripts/task.ts"() {
660
- "use strict";
661
- }
662
- });
663
-
664
- // src/cluster-utils.ts
665
- import cluster from "cluster";
666
- import { availableParallelism } from "os";
667
- import { exit } from "process";
668
- var init_cluster_utils = __esm({
669
- "src/cluster-utils.ts"() {
670
- "use strict";
671
- }
672
- });
673
-
674
- // src/compiler-config.ts
675
- var init_compiler_config = __esm({
676
- "src/compiler-config.ts"() {
677
- "use strict";
678
- }
679
- });
680
-
681
- // src/consumption-apis/runner.ts
682
- import * as jose from "jose";
683
- var init_runner = __esm({
684
- "src/consumption-apis/runner.ts"() {
685
- "use strict";
686
- init_commons();
687
- init_helpers();
688
- init_cluster_utils();
689
- init_sqlHelpers();
690
- init_internal();
691
- init_compiler_config();
692
- }
693
- });
694
-
695
- // src/clients/redisClient.ts
696
- import { createClient as createClient2 } from "redis";
697
- var init_redisClient = __esm({
698
- "src/clients/redisClient.ts"() {
699
- "use strict";
700
- }
701
- });
702
-
703
- // src/consumption-apis/standalone.ts
704
- var init_standalone = __esm({
705
- "src/consumption-apis/standalone.ts"() {
706
- "use strict";
707
- init_helpers();
708
- init_commons();
709
- init_sqlHelpers();
710
- }
711
- });
712
-
713
- // src/utilities/json.ts
714
- var init_json = __esm({
715
- "src/utilities/json.ts"() {
716
- "use strict";
717
- }
718
- });
719
-
720
- // src/utilities/dataParser.ts
721
- import { parse } from "csv-parse";
722
- var CSV_DELIMITERS, DEFAULT_CSV_CONFIG;
723
- var init_dataParser = __esm({
724
- "src/utilities/dataParser.ts"() {
725
- "use strict";
726
- init_json();
727
- CSV_DELIMITERS = {
728
- COMMA: ",",
729
- TAB: " ",
730
- SEMICOLON: ";",
731
- PIPE: "|"
732
- };
733
- DEFAULT_CSV_CONFIG = {
734
- delimiter: CSV_DELIMITERS.COMMA,
735
- columns: true,
736
- skipEmptyLines: true,
737
- trim: true
738
- };
739
- }
740
- });
741
-
742
- // src/utilities/index.ts
743
- var init_utilities = __esm({
744
- "src/utilities/index.ts"() {
745
- "use strict";
746
- init_dataParser();
747
- }
748
- });
749
-
750
- // src/connectors/dataSource.ts
751
- var init_dataSource = __esm({
752
- "src/connectors/dataSource.ts"() {
753
- "use strict";
754
- }
755
- });
756
-
757
- // src/index.ts
758
- var init_index = __esm({
759
- "src/index.ts"() {
760
- "use strict";
761
- init_browserCompatible();
762
- init_commons();
763
- init_secrets();
764
- init_helpers();
765
- init_webAppHelpers();
766
- init_task();
767
- init_runner();
768
- init_redisClient();
769
- init_helpers();
770
- init_standalone();
771
- init_sqlHelpers();
772
- init_utilities();
773
- init_dataSource();
774
- init_types();
775
- }
776
- });
777
-
778
- // src/dmv2/internal.ts
779
- import process2 from "process";
780
- var isClientOnlyMode, moose_internal, defaultRetentionPeriod, getMooseInternal, dlqSchema, dlqColumns;
781
- var init_internal = __esm({
782
- "src/dmv2/internal.ts"() {
783
- "use strict";
784
- init_index();
785
- init_commons();
786
- init_compiler_config();
787
- isClientOnlyMode = () => process2.env.MOOSE_CLIENT_ONLY === "true";
788
- moose_internal = {
789
- tables: /* @__PURE__ */ new Map(),
790
- streams: /* @__PURE__ */ new Map(),
791
- ingestApis: /* @__PURE__ */ new Map(),
792
- apis: /* @__PURE__ */ new Map(),
793
- sqlResources: /* @__PURE__ */ new Map(),
794
- workflows: /* @__PURE__ */ new Map(),
795
- webApps: /* @__PURE__ */ new Map(),
796
- materializedViews: /* @__PURE__ */ new Map(),
797
- views: /* @__PURE__ */ new Map()
798
- };
799
- defaultRetentionPeriod = 60 * 60 * 24 * 7;
800
- getMooseInternal = () => globalThis.moose_internal;
801
- if (getMooseInternal() === void 0) {
802
- globalThis.moose_internal = moose_internal;
803
- }
804
- dlqSchema = {
805
- version: "3.1",
806
- components: {
807
- schemas: {
808
- DeadLetterModel: {
809
- type: "object",
810
- properties: {
811
- originalRecord: {
812
- $ref: "#/components/schemas/Recordstringany"
813
- },
814
- errorMessage: {
815
- type: "string"
816
- },
817
- errorType: {
818
- type: "string"
819
- },
820
- failedAt: {
821
- type: "string",
822
- format: "date-time"
823
- },
824
- source: {
825
- oneOf: [
826
- {
827
- const: "api"
828
- },
829
- {
830
- const: "transform"
831
- },
832
- {
833
- const: "table"
834
- }
835
- ]
836
- }
837
- },
838
- required: [
839
- "originalRecord",
840
- "errorMessage",
841
- "errorType",
842
- "failedAt",
843
- "source"
844
- ]
845
- },
846
- Recordstringany: {
847
- type: "object",
848
- properties: {},
849
- required: [],
850
- description: "Construct a type with a set of properties K of type T",
851
- additionalProperties: {}
852
- }
853
- }
854
- },
855
- schemas: [
856
- {
857
- $ref: "#/components/schemas/DeadLetterModel"
858
- }
859
- ]
860
- };
861
- dlqColumns = [
862
- {
863
- name: "originalRecord",
864
- data_type: "Json",
865
- primary_key: false,
866
- required: true,
867
- unique: false,
868
- default: null,
869
- annotations: [],
870
- ttl: null,
871
- codec: null,
872
- materialized: null,
873
- comment: null
874
- },
875
- {
876
- name: "errorMessage",
877
- data_type: "String",
878
- primary_key: false,
879
- required: true,
880
- unique: false,
881
- default: null,
882
- annotations: [],
883
- ttl: null,
884
- codec: null,
885
- materialized: null,
886
- comment: null
887
- },
888
- {
889
- name: "errorType",
890
- data_type: "String",
891
- primary_key: false,
892
- required: true,
893
- unique: false,
894
- default: null,
895
- annotations: [],
896
- ttl: null,
897
- codec: null,
898
- materialized: null,
899
- comment: null
900
- },
901
- {
902
- name: "failedAt",
903
- data_type: "DateTime",
904
- primary_key: false,
905
- required: true,
906
- unique: false,
907
- default: null,
908
- annotations: [],
909
- ttl: null,
910
- codec: null,
911
- materialized: null,
912
- comment: null
913
- },
914
- {
915
- name: "source",
916
- data_type: "String",
917
- primary_key: false,
918
- required: true,
919
- unique: false,
920
- default: null,
921
- annotations: [],
922
- ttl: null,
923
- codec: null,
924
- materialized: null,
925
- comment: null
926
- }
927
- ];
928
- }
929
- });
930
-
931
274
  // src/config/configFile.ts
932
275
  import path from "path";
933
276
  import * as toml from "toml";
@@ -1112,1568 +455,2011 @@ var init_runtime = __esm({
1112
455
  }
1113
456
  });
1114
457
 
1115
- // src/dmv2/sdk/olapTable.ts
1116
- import { Readable } from "stream";
1117
- import { createHash as createHash2 } from "crypto";
1118
- var OlapTable;
1119
- var init_olapTable = __esm({
1120
- "src/dmv2/sdk/olapTable.ts"() {
1121
- "use strict";
1122
- init_typedBase();
1123
- init_dataModelTypes();
1124
- init_types();
1125
- init_internal();
1126
- init_sqlHelpers();
1127
- OlapTable = class extends TypedBase {
1128
- name;
1129
- /** @internal */
1130
- kind = "OlapTable";
1131
- /** @internal Memoized ClickHouse client for reusing connections across insert calls */
1132
- _memoizedClient;
1133
- /** @internal Hash of the configuration used to create the memoized client */
1134
- _configHash;
1135
- /** @internal Cached table name to avoid repeated generation */
1136
- _cachedTableName;
1137
- constructor(name, config, schema, columns, validators) {
1138
- const resolvedConfig = config ? "engine" in config ? config : { ...config, engine: "MergeTree" /* MergeTree */ } : { engine: "MergeTree" /* MergeTree */ };
1139
- const hasFields = Array.isArray(resolvedConfig.orderByFields) && resolvedConfig.orderByFields.length > 0;
1140
- const hasExpr = typeof resolvedConfig.orderByExpression === "string" && resolvedConfig.orderByExpression.length > 0;
1141
- if (hasFields && hasExpr) {
1142
- throw new Error(
1143
- `OlapTable ${name}: Provide either orderByFields or orderByExpression, not both.`
1144
- );
1145
- }
1146
- const hasCluster = typeof resolvedConfig.cluster === "string";
1147
- const hasKeeperPath = typeof resolvedConfig.keeperPath === "string";
1148
- const hasReplicaName = typeof resolvedConfig.replicaName === "string";
1149
- if (hasCluster && (hasKeeperPath || hasReplicaName)) {
1150
- throw new Error(
1151
- `OlapTable ${name}: Cannot specify both 'cluster' and explicit replication params ('keeperPath' or 'replicaName'). Use 'cluster' for auto-injected params, or use explicit 'keeperPath' and 'replicaName' without 'cluster'.`
1152
- );
1153
- }
1154
- super(name, resolvedConfig, schema, columns, validators);
1155
- this.name = name;
1156
- const tables = getMooseInternal().tables;
1157
- const registryKey = this.config.version ? `${name}_${this.config.version}` : name;
1158
- if (!isClientOnlyMode() && tables.has(registryKey)) {
1159
- throw new Error(
1160
- `OlapTable with name ${name} and version ${config?.version ?? "unversioned"} already exists`
1161
- );
1162
- }
1163
- tables.set(registryKey, this);
458
+ // src/dmv2/utils/stackTrace.ts
459
+ function shouldSkipStackLine(line) {
460
+ return line.includes("node_modules") || // Skip npm installed packages (prod)
461
+ line.includes("node:internal") || // Skip Node.js internals (modern format)
462
+ line.includes("internal/modules") || // Skip Node.js internals (older format)
463
+ line.includes("ts-node") || // Skip TypeScript execution
464
+ line.includes("/ts-moose-lib/src/") || // Skip dev/linked moose-lib src (Unix)
465
+ line.includes("\\ts-moose-lib\\src\\") || // Skip dev/linked moose-lib src (Windows)
466
+ line.includes("/ts-moose-lib/dist/") || // Skip dev/linked moose-lib dist (Unix)
467
+ line.includes("\\ts-moose-lib\\dist\\");
468
+ }
469
+ function parseStackLine(line) {
470
+ const match = line.match(/\((.*):(\d+):(\d+)\)/) || line.match(/at (.*):(\d+):(\d+)/);
471
+ if (match && match[1]) {
472
+ return {
473
+ file: match[1],
474
+ line: match[2]
475
+ };
476
+ }
477
+ return void 0;
478
+ }
479
+ function getSourceFileInfo(stack) {
480
+ if (!stack) return {};
481
+ const lines = stack.split("\n");
482
+ for (const line of lines) {
483
+ if (shouldSkipStackLine(line)) continue;
484
+ const info = parseStackLine(line);
485
+ if (info) return info;
486
+ }
487
+ return {};
488
+ }
489
+ function getSourceLocationFromStack(stack) {
490
+ if (!stack) return void 0;
491
+ const lines = stack.split("\n");
492
+ for (const line of lines.slice(1)) {
493
+ if (shouldSkipStackLine(line)) {
494
+ continue;
495
+ }
496
+ const v8Match = line.match(/at\s+(?:.*?\s+\()?(.+):(\d+):(\d+)\)?/);
497
+ if (v8Match) {
498
+ return {
499
+ file: v8Match[1],
500
+ line: parseInt(v8Match[2], 10),
501
+ column: parseInt(v8Match[3], 10)
502
+ };
503
+ }
504
+ const smMatch = line.match(/(?:.*@)?(.+):(\d+):(\d+)/);
505
+ if (smMatch) {
506
+ return {
507
+ file: smMatch[1],
508
+ line: parseInt(smMatch[2], 10),
509
+ column: parseInt(smMatch[3], 10)
510
+ };
511
+ }
512
+ }
513
+ return void 0;
514
+ }
515
+ function getSourceFileFromStack(stack) {
516
+ const location = getSourceLocationFromStack(stack);
517
+ return location?.file;
518
+ }
519
+
520
+ // src/dmv2/typedBase.ts
521
+ var TypedBase = class {
522
+ /** The JSON schema representation of type T. Injected by the compiler plugin. */
523
+ schema;
524
+ /** The name assigned to this resource instance. */
525
+ name;
526
+ /** A dictionary mapping column names (keys of T) to their Column definitions. */
527
+ columns;
528
+ /** An array containing the Column definitions for this resource. Injected by the compiler plugin. */
529
+ columnArray;
530
+ /** The configuration object specific to this resource type. */
531
+ config;
532
+ /** Typia validation functions for type T. Injected by the compiler plugin for OlapTable. */
533
+ validators;
534
+ /** Optional metadata for the resource, always present as an object. */
535
+ metadata;
536
+ /**
537
+ * Whether this resource allows extra fields beyond the defined columns.
538
+ * When true, extra fields in payloads are passed through to streaming functions.
539
+ * Injected by the compiler plugin when the type has an index signature.
540
+ */
541
+ allowExtraFields;
542
+ /**
543
+ * @internal Constructor intended for internal use by subclasses and the compiler plugin.
544
+ * It expects the schema and columns to be provided, typically injected by the compiler.
545
+ *
546
+ * @param name The name for the resource instance.
547
+ * @param config The configuration object for the resource.
548
+ * @param schema The JSON schema for the resource's data type T (injected).
549
+ * @param columns The array of Column definitions for T (injected).
550
+ * @param allowExtraFields Whether extra fields are allowed (injected when type has index signature).
551
+ */
552
+ constructor(name, config, schema, columns, validators, allowExtraFields) {
553
+ if (schema === void 0 || columns === void 0) {
554
+ throw new Error(
555
+ "Supply the type param T so that the schema is inserted by the compiler plugin."
556
+ );
557
+ }
558
+ this.schema = schema;
559
+ this.columnArray = columns;
560
+ const columnsObj = {};
561
+ columns.forEach((column) => {
562
+ columnsObj[column.name] = column;
563
+ });
564
+ this.columns = columnsObj;
565
+ this.name = name;
566
+ this.config = config;
567
+ this.validators = validators;
568
+ this.allowExtraFields = allowExtraFields ?? false;
569
+ this.metadata = config?.metadata ? { ...config.metadata } : {};
570
+ if (!this.metadata.source) {
571
+ const stack = new Error().stack;
572
+ if (stack) {
573
+ const info = getSourceFileInfo(stack);
574
+ this.metadata.source = { file: info.file, line: info.line };
1164
575
  }
1165
- /**
1166
- * Generates the versioned table name following Moose's naming convention
1167
- * Format: {tableName}_{version_with_dots_replaced_by_underscores}
1168
- */
1169
- generateTableName() {
1170
- if (this._cachedTableName) {
1171
- return this._cachedTableName;
576
+ }
577
+ }
578
+ };
579
+
580
+ // src/dataModels/dataModelTypes.ts
581
+ function isArrayNestedType(dt) {
582
+ return typeof dt === "object" && dt !== null && dt.elementType !== null && typeof dt.elementType === "object" && dt.elementType.hasOwnProperty("columns") && Array.isArray(dt.elementType.columns);
583
+ }
584
+ function isNestedType(dt) {
585
+ return typeof dt === "object" && dt !== null && Array.isArray(dt.columns);
586
+ }
587
+
588
+ // src/dataModels/types.ts
589
+ var ClickHouseEngines = /* @__PURE__ */ ((ClickHouseEngines2) => {
590
+ ClickHouseEngines2["MergeTree"] = "MergeTree";
591
+ ClickHouseEngines2["ReplacingMergeTree"] = "ReplacingMergeTree";
592
+ ClickHouseEngines2["SummingMergeTree"] = "SummingMergeTree";
593
+ ClickHouseEngines2["AggregatingMergeTree"] = "AggregatingMergeTree";
594
+ ClickHouseEngines2["CollapsingMergeTree"] = "CollapsingMergeTree";
595
+ ClickHouseEngines2["VersionedCollapsingMergeTree"] = "VersionedCollapsingMergeTree";
596
+ ClickHouseEngines2["GraphiteMergeTree"] = "GraphiteMergeTree";
597
+ ClickHouseEngines2["S3Queue"] = "S3Queue";
598
+ ClickHouseEngines2["S3"] = "S3";
599
+ ClickHouseEngines2["Buffer"] = "Buffer";
600
+ ClickHouseEngines2["Distributed"] = "Distributed";
601
+ ClickHouseEngines2["IcebergS3"] = "IcebergS3";
602
+ ClickHouseEngines2["Kafka"] = "Kafka";
603
+ ClickHouseEngines2["ReplicatedMergeTree"] = "ReplicatedMergeTree";
604
+ ClickHouseEngines2["ReplicatedReplacingMergeTree"] = "ReplicatedReplacingMergeTree";
605
+ ClickHouseEngines2["ReplicatedAggregatingMergeTree"] = "ReplicatedAggregatingMergeTree";
606
+ ClickHouseEngines2["ReplicatedSummingMergeTree"] = "ReplicatedSummingMergeTree";
607
+ ClickHouseEngines2["ReplicatedCollapsingMergeTree"] = "ReplicatedCollapsingMergeTree";
608
+ ClickHouseEngines2["ReplicatedVersionedCollapsingMergeTree"] = "ReplicatedVersionedCollapsingMergeTree";
609
+ return ClickHouseEngines2;
610
+ })(ClickHouseEngines || {});
611
+
612
+ // src/dmv2/internal.ts
613
+ import process2 from "process";
614
+
615
+ // src/index.ts
616
+ init_commons();
617
+
618
+ // src/consumption-apis/helpers.ts
619
+ import {
620
+ Client as TemporalClient,
621
+ Connection
622
+ } from "@temporalio/client";
623
+ import { createHash, randomUUID } from "crypto";
624
+
625
+ // src/sqlHelpers.ts
626
+ var quoteIdentifier = (name) => {
627
+ return name.startsWith("`") && name.endsWith("`") ? name : `\`${name}\``;
628
+ };
629
+ var isTable = (value) => typeof value === "object" && value !== null && "kind" in value && value.kind === "OlapTable";
630
+ var isView = (value) => typeof value === "object" && value !== null && "kind" in value && value.kind === "View";
631
+ var isColumn = (value) => typeof value === "object" && value !== null && !("kind" in value) && "name" in value && "annotations" in value;
632
+ function sqlImpl(strings, ...values) {
633
+ return new Sql(strings, values);
634
+ }
635
+ var sql = sqlImpl;
636
+ var instanceofSql = (value) => typeof value === "object" && "values" in value && "strings" in value;
637
+ var Sql = class _Sql {
638
+ values;
639
+ strings;
640
+ constructor(rawStrings, rawValues) {
641
+ if (rawStrings.length - 1 !== rawValues.length) {
642
+ if (rawStrings.length === 0) {
643
+ throw new TypeError("Expected at least 1 string");
644
+ }
645
+ throw new TypeError(
646
+ `Expected ${rawStrings.length} strings to have ${rawStrings.length - 1} values`
647
+ );
648
+ }
649
+ const valuesLength = rawValues.reduce(
650
+ (len, value) => len + (instanceofSql(value) ? value.values.length : isColumn(value) || isTable(value) || isView(value) ? 0 : 1),
651
+ 0
652
+ );
653
+ this.values = new Array(valuesLength);
654
+ this.strings = new Array(valuesLength + 1);
655
+ this.strings[0] = rawStrings[0];
656
+ let i = 0, pos = 0;
657
+ while (i < rawValues.length) {
658
+ const child = rawValues[i++];
659
+ const rawString = rawStrings[i];
660
+ if (instanceofSql(child)) {
661
+ this.strings[pos] += child.strings[0];
662
+ let childIndex = 0;
663
+ while (childIndex < child.values.length) {
664
+ this.values[pos++] = child.values[childIndex++];
665
+ this.strings[pos] = child.strings[childIndex];
666
+ }
667
+ this.strings[pos] += rawString;
668
+ } else if (isColumn(child)) {
669
+ const aggregationFunction = child.annotations.find(
670
+ ([k, _]) => k === "aggregationFunction"
671
+ );
672
+ if (aggregationFunction !== void 0) {
673
+ this.strings[pos] += `${aggregationFunction[1].functionName}Merge(\`${child.name}\`)`;
674
+ } else {
675
+ this.strings[pos] += `\`${child.name}\``;
1172
676
  }
1173
- const tableVersion = this.config.version;
1174
- if (!tableVersion) {
1175
- this._cachedTableName = this.name;
677
+ this.strings[pos] += rawString;
678
+ } else if (isTable(child)) {
679
+ if (child.config.database) {
680
+ this.strings[pos] += `\`${child.config.database}\`.\`${child.name}\``;
1176
681
  } else {
1177
- const versionSuffix = tableVersion.replace(/\./g, "_");
1178
- this._cachedTableName = `${this.name}_${versionSuffix}`;
682
+ this.strings[pos] += `\`${child.name}\``;
1179
683
  }
1180
- return this._cachedTableName;
1181
- }
1182
- /**
1183
- * Creates a fast hash of the ClickHouse configuration.
1184
- * Uses crypto.createHash for better performance than JSON.stringify.
1185
- *
1186
- * @private
1187
- */
1188
- createConfigHash(clickhouseConfig) {
1189
- const effectiveDatabase = this.config.database ?? clickhouseConfig.database;
1190
- const configString = `${clickhouseConfig.host}:${clickhouseConfig.port}:${clickhouseConfig.username}:${clickhouseConfig.password}:${effectiveDatabase}:${clickhouseConfig.useSSL}`;
1191
- return createHash2("sha256").update(configString).digest("hex").substring(0, 16);
684
+ this.strings[pos] += rawString;
685
+ } else if (isView(child)) {
686
+ this.strings[pos] += `\`${child.name}\``;
687
+ this.strings[pos] += rawString;
688
+ } else {
689
+ this.values[pos++] = child;
690
+ this.strings[pos] = rawString;
1192
691
  }
1193
- /**
1194
- * Gets or creates a memoized ClickHouse client.
1195
- * The client is cached and reused across multiple insert calls for better performance.
1196
- * If the configuration changes, a new client will be created.
1197
- *
1198
- * @private
1199
- */
1200
- async getMemoizedClient() {
1201
- await Promise.resolve().then(() => (init_runtime(), runtime_exports));
1202
- const configRegistry = globalThis._mooseConfigRegistry;
1203
- const { getClickhouseClient: getClickhouseClient2 } = await Promise.resolve().then(() => (init_commons(), commons_exports));
1204
- const clickhouseConfig = await configRegistry.getClickHouseConfig();
1205
- const currentConfigHash = this.createConfigHash(clickhouseConfig);
1206
- if (this._memoizedClient && this._configHash === currentConfigHash) {
1207
- return { client: this._memoizedClient, config: clickhouseConfig };
1208
- }
1209
- if (this._memoizedClient && this._configHash !== currentConfigHash) {
1210
- try {
1211
- await this._memoizedClient.close();
1212
- } catch (error) {
1213
- }
1214
- }
1215
- const effectiveDatabase = this.config.database ?? clickhouseConfig.database;
1216
- const client = getClickhouseClient2({
1217
- username: clickhouseConfig.username,
1218
- password: clickhouseConfig.password,
1219
- database: effectiveDatabase,
1220
- useSSL: clickhouseConfig.useSSL ? "true" : "false",
1221
- host: clickhouseConfig.host,
1222
- port: clickhouseConfig.port
1223
- });
1224
- this._memoizedClient = client;
1225
- this._configHash = currentConfigHash;
1226
- return { client, config: clickhouseConfig };
692
+ }
693
+ }
694
+ /**
695
+ * Append another Sql fragment, returning a new Sql instance.
696
+ */
697
+ append(other) {
698
+ return new _Sql([...this.strings, ""], [...this.values, other]);
699
+ }
700
+ };
701
+ sql.join = function(fragments, separator) {
702
+ if (fragments.length === 0) return new Sql([""], []);
703
+ if (fragments.length === 1) return fragments[0];
704
+ const sep = separator ?? ", ";
705
+ const normalized = sep.includes(" ") ? sep : ` ${sep} `;
706
+ const strings = ["", ...Array(fragments.length - 1).fill(normalized), ""];
707
+ return new Sql(strings, fragments);
708
+ };
709
+ sql.raw = function(text) {
710
+ return new Sql([text], []);
711
+ };
712
+ var toStaticQuery = (sql3) => {
713
+ const [query, params] = toQuery(sql3);
714
+ if (Object.keys(params).length !== 0) {
715
+ throw new Error(
716
+ "Dynamic SQL is not allowed in the select statement in view creation."
717
+ );
718
+ }
719
+ return query;
720
+ };
721
+ var toQuery = (sql3) => {
722
+ const parameterizedStubs = sql3.values.map(
723
+ (v, i) => createClickhouseParameter(i, v)
724
+ );
725
+ const query = sql3.strings.map(
726
+ (s, i) => s != "" ? `${s}${emptyIfUndefined(parameterizedStubs[i])}` : ""
727
+ ).join("");
728
+ const query_params = sql3.values.reduce(
729
+ (acc, v, i) => ({
730
+ ...acc,
731
+ [`p${i}`]: getValueFromParameter(v)
732
+ }),
733
+ {}
734
+ );
735
+ return [query, query_params];
736
+ };
737
+ var toQueryPreview = (sql3) => {
738
+ try {
739
+ const formatValue = (v) => {
740
+ if (Array.isArray(v)) {
741
+ const [type, val] = v;
742
+ if (type === "Identifier") {
743
+ return `\`${String(val)}\``;
744
+ }
745
+ return `[${v.map((x) => formatValue(x)).join(", ")}]`;
746
+ }
747
+ if (v === null || v === void 0) return "NULL";
748
+ if (typeof v === "string") return `'${v.replace(/'/g, "''")}'`;
749
+ if (typeof v === "number") return String(v);
750
+ if (typeof v === "boolean") return v ? "true" : "false";
751
+ if (v instanceof Date)
752
+ return `'${v.toISOString().replace("T", " ").slice(0, 19)}'`;
753
+ try {
754
+ return JSON.stringify(v);
755
+ } catch {
756
+ return String(v);
1227
757
  }
1228
- /**
1229
- * Closes the memoized ClickHouse client if it exists.
1230
- * This is useful for cleaning up connections when the table instance is no longer needed.
1231
- * The client will be automatically recreated on the next insert call if needed.
1232
- */
1233
- async closeClient() {
1234
- if (this._memoizedClient) {
1235
- try {
1236
- await this._memoizedClient.close();
1237
- } catch (error) {
1238
- } finally {
1239
- this._memoizedClient = void 0;
1240
- this._configHash = void 0;
758
+ };
759
+ let out = sql3.strings[0] ?? "";
760
+ for (let i = 0; i < sql3.values.length; i++) {
761
+ const val = getValueFromParameter(sql3.values[i]);
762
+ out += formatValue(val);
763
+ out += sql3.strings[i + 1] ?? "";
764
+ }
765
+ return out.replace(/\s+/g, " ").trim();
766
+ } catch (error) {
767
+ console.log(`toQueryPreview error: ${error}`);
768
+ return "/* query preview unavailable */";
769
+ }
770
+ };
771
+ var getValueFromParameter = (value) => {
772
+ if (Array.isArray(value)) {
773
+ const [type, val] = value;
774
+ if (type === "Identifier") return val;
775
+ }
776
+ return value;
777
+ };
778
+ function createClickhouseParameter(parameterIndex, value) {
779
+ return `{p${parameterIndex}:${mapToClickHouseType(value)}}`;
780
+ }
781
+ var mapToClickHouseType = (value) => {
782
+ if (typeof value === "number") {
783
+ return Number.isInteger(value) ? "Int" : "Float";
784
+ }
785
+ if (typeof value === "boolean") return "Bool";
786
+ if (value instanceof Date) return "DateTime";
787
+ if (Array.isArray(value)) {
788
+ const [type, _] = value;
789
+ return type;
790
+ }
791
+ return "String";
792
+ };
793
+ function emptyIfUndefined(value) {
794
+ return value === void 0 ? "" : value;
795
+ }
796
+
797
+ // src/clients/redisClient.ts
798
+ import { createClient as createClient2 } from "redis";
799
+
800
+ // src/consumption-apis/standalone.ts
801
+ init_commons();
802
+
803
+ // src/utilities/dataParser.ts
804
+ import { parse } from "csv-parse";
805
+ var CSV_DELIMITERS = {
806
+ COMMA: ",",
807
+ TAB: " ",
808
+ SEMICOLON: ";",
809
+ PIPE: "|"
810
+ };
811
+ var DEFAULT_CSV_CONFIG = {
812
+ delimiter: CSV_DELIMITERS.COMMA,
813
+ columns: true,
814
+ skipEmptyLines: true,
815
+ trim: true
816
+ };
817
+
818
+ // src/dmv2/internal.ts
819
+ init_commons();
820
+ var isClientOnlyMode = () => process2.env.MOOSE_CLIENT_ONLY === "true";
821
+ var moose_internal = {
822
+ tables: /* @__PURE__ */ new Map(),
823
+ streams: /* @__PURE__ */ new Map(),
824
+ ingestApis: /* @__PURE__ */ new Map(),
825
+ apis: /* @__PURE__ */ new Map(),
826
+ sqlResources: /* @__PURE__ */ new Map(),
827
+ workflows: /* @__PURE__ */ new Map(),
828
+ webApps: /* @__PURE__ */ new Map(),
829
+ materializedViews: /* @__PURE__ */ new Map(),
830
+ views: /* @__PURE__ */ new Map()
831
+ };
832
+ var defaultRetentionPeriod = 60 * 60 * 24 * 7;
833
+ var getMooseInternal = () => globalThis.moose_internal;
834
+ if (getMooseInternal() === void 0) {
835
+ globalThis.moose_internal = moose_internal;
836
+ }
837
+ var dlqSchema = {
838
+ version: "3.1",
839
+ components: {
840
+ schemas: {
841
+ DeadLetterModel: {
842
+ type: "object",
843
+ properties: {
844
+ originalRecord: {
845
+ $ref: "#/components/schemas/Recordstringany"
846
+ },
847
+ errorMessage: {
848
+ type: "string"
849
+ },
850
+ errorType: {
851
+ type: "string"
852
+ },
853
+ failedAt: {
854
+ type: "string",
855
+ format: "date-time"
856
+ },
857
+ source: {
858
+ oneOf: [
859
+ {
860
+ const: "api"
861
+ },
862
+ {
863
+ const: "transform"
864
+ },
865
+ {
866
+ const: "table"
867
+ }
868
+ ]
1241
869
  }
1242
- }
870
+ },
871
+ required: [
872
+ "originalRecord",
873
+ "errorMessage",
874
+ "errorType",
875
+ "failedAt",
876
+ "source"
877
+ ]
878
+ },
879
+ Recordstringany: {
880
+ type: "object",
881
+ properties: {},
882
+ required: [],
883
+ description: "Construct a type with a set of properties K of type T",
884
+ additionalProperties: {}
1243
885
  }
1244
- /**
1245
- * Validates a single record using typia's comprehensive type checking.
1246
- * This provides the most accurate validation as it uses the exact TypeScript type information.
1247
- *
1248
- * @param record The record to validate
1249
- * @returns Validation result with detailed error information
1250
- */
1251
- validateRecord(record) {
1252
- if (this.validators?.validate) {
1253
- try {
1254
- const result = this.validators.validate(record);
1255
- return {
1256
- success: result.success,
1257
- data: result.data,
1258
- errors: result.errors?.map(
1259
- (err) => typeof err === "string" ? err : JSON.stringify(err)
1260
- )
1261
- };
1262
- } catch (error) {
1263
- return {
1264
- success: false,
1265
- errors: [error instanceof Error ? error.message : String(error)]
1266
- };
1267
- }
1268
- }
1269
- throw new Error("No typia validator found");
886
+ }
887
+ },
888
+ schemas: [
889
+ {
890
+ $ref: "#/components/schemas/DeadLetterModel"
891
+ }
892
+ ]
893
+ };
894
+ var dlqColumns = [
895
+ {
896
+ name: "originalRecord",
897
+ data_type: "Json",
898
+ primary_key: false,
899
+ required: true,
900
+ unique: false,
901
+ default: null,
902
+ annotations: [],
903
+ ttl: null,
904
+ codec: null,
905
+ materialized: null,
906
+ comment: null
907
+ },
908
+ {
909
+ name: "errorMessage",
910
+ data_type: "String",
911
+ primary_key: false,
912
+ required: true,
913
+ unique: false,
914
+ default: null,
915
+ annotations: [],
916
+ ttl: null,
917
+ codec: null,
918
+ materialized: null,
919
+ comment: null
920
+ },
921
+ {
922
+ name: "errorType",
923
+ data_type: "String",
924
+ primary_key: false,
925
+ required: true,
926
+ unique: false,
927
+ default: null,
928
+ annotations: [],
929
+ ttl: null,
930
+ codec: null,
931
+ materialized: null,
932
+ comment: null
933
+ },
934
+ {
935
+ name: "failedAt",
936
+ data_type: "DateTime",
937
+ primary_key: false,
938
+ required: true,
939
+ unique: false,
940
+ default: null,
941
+ annotations: [],
942
+ ttl: null,
943
+ codec: null,
944
+ materialized: null,
945
+ comment: null
946
+ },
947
+ {
948
+ name: "source",
949
+ data_type: "String",
950
+ primary_key: false,
951
+ required: true,
952
+ unique: false,
953
+ default: null,
954
+ annotations: [],
955
+ ttl: null,
956
+ codec: null,
957
+ materialized: null,
958
+ comment: null
959
+ }
960
+ ];
961
+
962
+ // src/dmv2/sdk/olapTable.ts
963
+ import { Readable } from "stream";
964
+ import { createHash as createHash2 } from "crypto";
965
+ var OlapTable = class extends TypedBase {
966
+ name;
967
+ /** @internal */
968
+ kind = "OlapTable";
969
+ /** @internal Memoized ClickHouse client for reusing connections across insert calls */
970
+ _memoizedClient;
971
+ /** @internal Hash of the configuration used to create the memoized client */
972
+ _configHash;
973
+ /** @internal Cached table name to avoid repeated generation */
974
+ _cachedTableName;
975
+ constructor(name, config, schema, columns, validators) {
976
+ const resolvedConfig = config ? "engine" in config ? config : { ...config, engine: "MergeTree" /* MergeTree */ } : { engine: "MergeTree" /* MergeTree */ };
977
+ const hasFields = Array.isArray(resolvedConfig.orderByFields) && resolvedConfig.orderByFields.length > 0;
978
+ const hasExpr = typeof resolvedConfig.orderByExpression === "string" && resolvedConfig.orderByExpression.length > 0;
979
+ if (hasFields && hasExpr) {
980
+ throw new Error(
981
+ `OlapTable ${name}: Provide either orderByFields or orderByExpression, not both.`
982
+ );
983
+ }
984
+ const hasCluster = typeof resolvedConfig.cluster === "string";
985
+ const hasKeeperPath = typeof resolvedConfig.keeperPath === "string";
986
+ const hasReplicaName = typeof resolvedConfig.replicaName === "string";
987
+ if (hasCluster && (hasKeeperPath || hasReplicaName)) {
988
+ throw new Error(
989
+ `OlapTable ${name}: Cannot specify both 'cluster' and explicit replication params ('keeperPath' or 'replicaName'). Use 'cluster' for auto-injected params, or use explicit 'keeperPath' and 'replicaName' without 'cluster'.`
990
+ );
991
+ }
992
+ super(name, resolvedConfig, schema, columns, validators);
993
+ this.name = name;
994
+ const tables = getMooseInternal().tables;
995
+ const registryKey = this.config.version ? `${name}_${this.config.version}` : name;
996
+ if (!isClientOnlyMode() && tables.has(registryKey)) {
997
+ throw new Error(
998
+ `OlapTable with name ${name} and version ${config?.version ?? "unversioned"} already exists`
999
+ );
1000
+ }
1001
+ tables.set(registryKey, this);
1002
+ }
1003
+ /**
1004
+ * Generates the versioned table name following Moose's naming convention
1005
+ * Format: {tableName}_{version_with_dots_replaced_by_underscores}
1006
+ */
1007
+ generateTableName() {
1008
+ if (this._cachedTableName) {
1009
+ return this._cachedTableName;
1010
+ }
1011
+ const tableVersion = this.config.version;
1012
+ if (!tableVersion) {
1013
+ this._cachedTableName = this.name;
1014
+ } else {
1015
+ const versionSuffix = tableVersion.replace(/\./g, "_");
1016
+ this._cachedTableName = `${this.name}_${versionSuffix}`;
1017
+ }
1018
+ return this._cachedTableName;
1019
+ }
1020
+ /**
1021
+ * Creates a fast hash of the ClickHouse configuration.
1022
+ * Uses crypto.createHash for better performance than JSON.stringify.
1023
+ *
1024
+ * @private
1025
+ */
1026
+ createConfigHash(clickhouseConfig) {
1027
+ const effectiveDatabase = this.config.database ?? clickhouseConfig.database;
1028
+ const configString = `${clickhouseConfig.host}:${clickhouseConfig.port}:${clickhouseConfig.username}:${clickhouseConfig.password}:${effectiveDatabase}:${clickhouseConfig.useSSL}`;
1029
+ return createHash2("sha256").update(configString).digest("hex").substring(0, 16);
1030
+ }
1031
+ /**
1032
+ * Gets or creates a memoized ClickHouse client.
1033
+ * The client is cached and reused across multiple insert calls for better performance.
1034
+ * If the configuration changes, a new client will be created.
1035
+ *
1036
+ * @private
1037
+ */
1038
+ async getMemoizedClient() {
1039
+ await Promise.resolve().then(() => (init_runtime(), runtime_exports));
1040
+ const configRegistry = globalThis._mooseConfigRegistry;
1041
+ const { getClickhouseClient: getClickhouseClient2 } = await Promise.resolve().then(() => (init_commons(), commons_exports));
1042
+ const clickhouseConfig = await configRegistry.getClickHouseConfig();
1043
+ const currentConfigHash = this.createConfigHash(clickhouseConfig);
1044
+ if (this._memoizedClient && this._configHash === currentConfigHash) {
1045
+ return { client: this._memoizedClient, config: clickhouseConfig };
1046
+ }
1047
+ if (this._memoizedClient && this._configHash !== currentConfigHash) {
1048
+ try {
1049
+ await this._memoizedClient.close();
1050
+ } catch (error) {
1270
1051
  }
1271
- /**
1272
- * Type guard function using typia's is() function.
1273
- * Provides compile-time type narrowing for TypeScript.
1274
- *
1275
- * @param record The record to check
1276
- * @returns True if record matches type T, with type narrowing
1277
- */
1278
- isValidRecord(record) {
1279
- if (this.validators?.is) {
1280
- return this.validators.is(record);
1281
- }
1282
- throw new Error("No typia validator found");
1052
+ }
1053
+ const effectiveDatabase = this.config.database ?? clickhouseConfig.database;
1054
+ const client = getClickhouseClient2({
1055
+ username: clickhouseConfig.username,
1056
+ password: clickhouseConfig.password,
1057
+ database: effectiveDatabase,
1058
+ useSSL: clickhouseConfig.useSSL ? "true" : "false",
1059
+ host: clickhouseConfig.host,
1060
+ port: clickhouseConfig.port
1061
+ });
1062
+ this._memoizedClient = client;
1063
+ this._configHash = currentConfigHash;
1064
+ return { client, config: clickhouseConfig };
1065
+ }
1066
+ /**
1067
+ * Closes the memoized ClickHouse client if it exists.
1068
+ * This is useful for cleaning up connections when the table instance is no longer needed.
1069
+ * The client will be automatically recreated on the next insert call if needed.
1070
+ */
1071
+ async closeClient() {
1072
+ if (this._memoizedClient) {
1073
+ try {
1074
+ await this._memoizedClient.close();
1075
+ } catch (error) {
1076
+ } finally {
1077
+ this._memoizedClient = void 0;
1078
+ this._configHash = void 0;
1283
1079
  }
1284
- /**
1285
- * Assert that a record matches type T, throwing detailed errors if not.
1286
- * Uses typia's assert() function for the most detailed error reporting.
1287
- *
1288
- * @param record The record to assert
1289
- * @returns The validated and typed record
1290
- * @throws Detailed validation error if record doesn't match type T
1291
- */
1292
- assertValidRecord(record) {
1293
- if (this.validators?.assert) {
1294
- return this.validators.assert(record);
1295
- }
1296
- throw new Error("No typia validator found");
1080
+ }
1081
+ }
1082
+ /**
1083
+ * Validates a single record using typia's comprehensive type checking.
1084
+ * This provides the most accurate validation as it uses the exact TypeScript type information.
1085
+ *
1086
+ * @param record The record to validate
1087
+ * @returns Validation result with detailed error information
1088
+ */
1089
+ validateRecord(record) {
1090
+ if (this.validators?.validate) {
1091
+ try {
1092
+ const result = this.validators.validate(record);
1093
+ return {
1094
+ success: result.success,
1095
+ data: result.data,
1096
+ errors: result.errors?.map(
1097
+ (err) => typeof err === "string" ? err : JSON.stringify(err)
1098
+ )
1099
+ };
1100
+ } catch (error) {
1101
+ return {
1102
+ success: false,
1103
+ errors: [error instanceof Error ? error.message : String(error)]
1104
+ };
1297
1105
  }
1298
- /**
1299
- * Validates an array of records with comprehensive error reporting.
1300
- * Uses the most appropriate validation method available (typia or basic).
1301
- *
1302
- * @param data Array of records to validate
1303
- * @returns Detailed validation results
1304
- */
1305
- async validateRecords(data) {
1306
- const valid = [];
1307
- const invalid = [];
1308
- valid.length = 0;
1309
- invalid.length = 0;
1310
- const dataLength = data.length;
1311
- for (let i = 0; i < dataLength; i++) {
1312
- const record = data[i];
1313
- try {
1314
- if (this.isValidRecord(record)) {
1315
- valid.push(this.mapToClickhouseRecord(record));
1316
- } else {
1317
- const result = this.validateRecord(record);
1318
- if (result.success) {
1319
- valid.push(this.mapToClickhouseRecord(record));
1320
- } else {
1321
- invalid.push({
1322
- record,
1323
- error: result.errors?.join(", ") || "Validation failed",
1324
- index: i,
1325
- path: "root"
1326
- });
1327
- }
1328
- }
1329
- } catch (error) {
1106
+ }
1107
+ throw new Error("No typia validator found");
1108
+ }
1109
+ /**
1110
+ * Type guard function using typia's is() function.
1111
+ * Provides compile-time type narrowing for TypeScript.
1112
+ *
1113
+ * @param record The record to check
1114
+ * @returns True if record matches type T, with type narrowing
1115
+ */
1116
+ isValidRecord(record) {
1117
+ if (this.validators?.is) {
1118
+ return this.validators.is(record);
1119
+ }
1120
+ throw new Error("No typia validator found");
1121
+ }
1122
+ /**
1123
+ * Assert that a record matches type T, throwing detailed errors if not.
1124
+ * Uses typia's assert() function for the most detailed error reporting.
1125
+ *
1126
+ * @param record The record to assert
1127
+ * @returns The validated and typed record
1128
+ * @throws Detailed validation error if record doesn't match type T
1129
+ */
1130
+ assertValidRecord(record) {
1131
+ if (this.validators?.assert) {
1132
+ return this.validators.assert(record);
1133
+ }
1134
+ throw new Error("No typia validator found");
1135
+ }
1136
+ /**
1137
+ * Validates an array of records with comprehensive error reporting.
1138
+ * Uses the most appropriate validation method available (typia or basic).
1139
+ *
1140
+ * @param data Array of records to validate
1141
+ * @returns Detailed validation results
1142
+ */
1143
+ async validateRecords(data) {
1144
+ const valid = [];
1145
+ const invalid = [];
1146
+ valid.length = 0;
1147
+ invalid.length = 0;
1148
+ const dataLength = data.length;
1149
+ for (let i = 0; i < dataLength; i++) {
1150
+ const record = data[i];
1151
+ try {
1152
+ if (this.isValidRecord(record)) {
1153
+ valid.push(this.mapToClickhouseRecord(record));
1154
+ } else {
1155
+ const result = this.validateRecord(record);
1156
+ if (result.success) {
1157
+ valid.push(this.mapToClickhouseRecord(record));
1158
+ } else {
1330
1159
  invalid.push({
1331
1160
  record,
1332
- error: error instanceof Error ? error.message : String(error),
1161
+ error: result.errors?.join(", ") || "Validation failed",
1333
1162
  index: i,
1334
1163
  path: "root"
1335
1164
  });
1336
1165
  }
1337
1166
  }
1338
- return {
1339
- valid,
1340
- invalid,
1341
- total: dataLength
1342
- };
1167
+ } catch (error) {
1168
+ invalid.push({
1169
+ record,
1170
+ error: error instanceof Error ? error.message : String(error),
1171
+ index: i,
1172
+ path: "root"
1173
+ });
1343
1174
  }
1344
- /**
1345
- * Optimized batch retry that minimizes individual insert operations.
1346
- * Groups records into smaller batches to reduce round trips while still isolating failures.
1347
- *
1348
- * @private
1349
- */
1350
- async retryIndividualRecords(client, tableName, records) {
1351
- const successful = [];
1352
- const failed = [];
1353
- const RETRY_BATCH_SIZE = 10;
1354
- const totalRecords = records.length;
1355
- for (let i = 0; i < totalRecords; i += RETRY_BATCH_SIZE) {
1356
- const batchEnd = Math.min(i + RETRY_BATCH_SIZE, totalRecords);
1357
- const batch = records.slice(i, batchEnd);
1175
+ }
1176
+ return {
1177
+ valid,
1178
+ invalid,
1179
+ total: dataLength
1180
+ };
1181
+ }
1182
+ /**
1183
+ * Optimized batch retry that minimizes individual insert operations.
1184
+ * Groups records into smaller batches to reduce round trips while still isolating failures.
1185
+ *
1186
+ * @private
1187
+ */
1188
+ async retryIndividualRecords(client, tableName, records) {
1189
+ const successful = [];
1190
+ const failed = [];
1191
+ const RETRY_BATCH_SIZE = 10;
1192
+ const totalRecords = records.length;
1193
+ for (let i = 0; i < totalRecords; i += RETRY_BATCH_SIZE) {
1194
+ const batchEnd = Math.min(i + RETRY_BATCH_SIZE, totalRecords);
1195
+ const batch = records.slice(i, batchEnd);
1196
+ try {
1197
+ await client.insert({
1198
+ table: quoteIdentifier(tableName),
1199
+ values: batch,
1200
+ format: "JSONEachRow",
1201
+ clickhouse_settings: {
1202
+ date_time_input_format: "best_effort",
1203
+ // Add performance settings for retries
1204
+ max_insert_block_size: RETRY_BATCH_SIZE,
1205
+ max_block_size: RETRY_BATCH_SIZE
1206
+ }
1207
+ });
1208
+ successful.push(...batch);
1209
+ } catch (batchError) {
1210
+ for (let j = 0; j < batch.length; j++) {
1211
+ const record = batch[j];
1358
1212
  try {
1359
1213
  await client.insert({
1360
1214
  table: quoteIdentifier(tableName),
1361
- values: batch,
1215
+ values: [record],
1362
1216
  format: "JSONEachRow",
1363
1217
  clickhouse_settings: {
1364
- date_time_input_format: "best_effort",
1365
- // Add performance settings for retries
1366
- max_insert_block_size: RETRY_BATCH_SIZE,
1367
- max_block_size: RETRY_BATCH_SIZE
1218
+ date_time_input_format: "best_effort"
1368
1219
  }
1369
1220
  });
1370
- successful.push(...batch);
1371
- } catch (batchError) {
1372
- for (let j = 0; j < batch.length; j++) {
1373
- const record = batch[j];
1374
- try {
1375
- await client.insert({
1376
- table: quoteIdentifier(tableName),
1377
- values: [record],
1378
- format: "JSONEachRow",
1379
- clickhouse_settings: {
1380
- date_time_input_format: "best_effort"
1381
- }
1382
- });
1383
- successful.push(record);
1384
- } catch (error) {
1385
- failed.push({
1386
- record,
1387
- error: error instanceof Error ? error.message : String(error),
1388
- index: i + j
1389
- });
1390
- }
1391
- }
1392
- }
1393
- }
1394
- return { successful, failed };
1395
- }
1396
- /**
1397
- * Validates input parameters and strategy compatibility
1398
- * @private
1399
- */
1400
- validateInsertParameters(data, options) {
1401
- const isStream = data instanceof Readable;
1402
- const strategy = options?.strategy || "fail-fast";
1403
- const shouldValidate = options?.validate !== false;
1404
- if (isStream && strategy === "isolate") {
1405
- throw new Error(
1406
- "The 'isolate' error strategy is not supported with stream input. Use 'fail-fast' or 'discard' instead."
1407
- );
1408
- }
1409
- if (isStream && shouldValidate) {
1410
- console.warn(
1411
- "Validation is not supported with stream input. Validation will be skipped."
1412
- );
1413
- }
1414
- return { isStream, strategy, shouldValidate };
1415
- }
1416
- /**
1417
- * Handles early return cases for empty data
1418
- * @private
1419
- */
1420
- handleEmptyData(data, isStream) {
1421
- if (isStream && !data) {
1422
- return {
1423
- successful: 0,
1424
- failed: 0,
1425
- total: 0
1426
- };
1427
- }
1428
- if (!isStream && (!data || data.length === 0)) {
1429
- return {
1430
- successful: 0,
1431
- failed: 0,
1432
- total: 0
1433
- };
1434
- }
1435
- return null;
1436
- }
1437
- /**
1438
- * Performs pre-insertion validation for array data
1439
- * @private
1440
- */
1441
- async performPreInsertionValidation(data, shouldValidate, strategy, options) {
1442
- if (!shouldValidate) {
1443
- return { validatedData: data, validationErrors: [] };
1444
- }
1445
- try {
1446
- const validationResult = await this.validateRecords(data);
1447
- const validatedData = validationResult.valid;
1448
- const validationErrors = validationResult.invalid;
1449
- if (validationErrors.length > 0) {
1450
- this.handleValidationErrors(validationErrors, strategy, data, options);
1451
- switch (strategy) {
1452
- case "discard":
1453
- return { validatedData, validationErrors };
1454
- case "isolate":
1455
- return { validatedData: data, validationErrors };
1456
- default:
1457
- return { validatedData, validationErrors };
1458
- }
1459
- }
1460
- return { validatedData, validationErrors };
1461
- } catch (validationError) {
1462
- if (strategy === "fail-fast") {
1463
- throw validationError;
1464
- }
1465
- console.warn("Validation error:", validationError);
1466
- return { validatedData: data, validationErrors: [] };
1467
- }
1468
- }
1469
- /**
1470
- * Handles validation errors based on the specified strategy
1471
- * @private
1472
- */
1473
- handleValidationErrors(validationErrors, strategy, data, options) {
1474
- switch (strategy) {
1475
- case "fail-fast":
1476
- const firstError = validationErrors[0];
1477
- throw new Error(
1478
- `Validation failed for record at index ${firstError.index}: ${firstError.error}`
1479
- );
1480
- case "discard":
1481
- this.checkValidationThresholds(validationErrors, data.length, options);
1482
- break;
1483
- case "isolate":
1484
- break;
1485
- }
1486
- }
1487
- /**
1488
- * Checks if validation errors exceed configured thresholds
1489
- * @private
1490
- */
1491
- checkValidationThresholds(validationErrors, totalRecords, options) {
1492
- const validationFailedCount = validationErrors.length;
1493
- const validationFailedRatio = validationFailedCount / totalRecords;
1494
- if (options?.allowErrors !== void 0 && validationFailedCount > options.allowErrors) {
1495
- throw new Error(
1496
- `Too many validation failures: ${validationFailedCount} > ${options.allowErrors}. Errors: ${validationErrors.map((e) => e.error).join(", ")}`
1497
- );
1498
- }
1499
- if (options?.allowErrorsRatio !== void 0 && validationFailedRatio > options.allowErrorsRatio) {
1500
- throw new Error(
1501
- `Validation failure ratio too high: ${validationFailedRatio.toFixed(3)} > ${options.allowErrorsRatio}. Errors: ${validationErrors.map((e) => e.error).join(", ")}`
1502
- );
1503
- }
1504
- }
1505
- /**
1506
- * Optimized insert options preparation with better memory management
1507
- * @private
1508
- */
1509
- prepareInsertOptions(tableName, data, validatedData, isStream, strategy, options) {
1510
- const insertOptions = {
1511
- table: quoteIdentifier(tableName),
1512
- format: "JSONEachRow",
1513
- clickhouse_settings: {
1514
- date_time_input_format: "best_effort",
1515
- wait_end_of_query: 1,
1516
- // Ensure at least once delivery for INSERT operations
1517
- // Performance optimizations
1518
- max_insert_block_size: isStream ? 1e5 : Math.min(validatedData.length, 1e5),
1519
- max_block_size: 65536,
1520
- // Use async inserts for better performance with large datasets
1521
- async_insert: validatedData.length > 1e3 ? 1 : 0,
1522
- wait_for_async_insert: 1
1523
- // For at least once delivery
1524
- }
1525
- };
1526
- if (isStream) {
1527
- insertOptions.values = data;
1528
- } else {
1529
- insertOptions.values = validatedData;
1530
- }
1531
- if (strategy === "discard" && (options?.allowErrors !== void 0 || options?.allowErrorsRatio !== void 0)) {
1532
- if (options.allowErrors !== void 0) {
1533
- insertOptions.clickhouse_settings.input_format_allow_errors_num = options.allowErrors;
1534
- }
1535
- if (options.allowErrorsRatio !== void 0) {
1536
- insertOptions.clickhouse_settings.input_format_allow_errors_ratio = options.allowErrorsRatio;
1221
+ successful.push(record);
1222
+ } catch (error) {
1223
+ failed.push({
1224
+ record,
1225
+ error: error instanceof Error ? error.message : String(error),
1226
+ index: i + j
1227
+ });
1537
1228
  }
1538
1229
  }
1539
- return insertOptions;
1540
1230
  }
1541
- /**
1542
- * Creates success result for completed insertions
1543
- * @private
1544
- */
1545
- createSuccessResult(data, validatedData, validationErrors, isStream, shouldValidate, strategy) {
1546
- if (isStream) {
1547
- return {
1548
- successful: -1,
1549
- // -1 indicates stream mode where count is unknown
1550
- failed: 0,
1551
- total: -1
1552
- };
1553
- }
1554
- const insertedCount = validatedData.length;
1555
- const totalProcessed = shouldValidate ? data.length : insertedCount;
1556
- const result = {
1557
- successful: insertedCount,
1558
- failed: shouldValidate ? validationErrors.length : 0,
1559
- total: totalProcessed
1560
- };
1561
- if (shouldValidate && validationErrors.length > 0 && strategy === "discard") {
1562
- result.failedRecords = validationErrors.map((ve) => ({
1563
- record: ve.record,
1564
- error: `Validation error: ${ve.error}`,
1565
- index: ve.index
1566
- }));
1567
- }
1568
- return result;
1569
- }
1570
- /**
1571
- * Handles insertion errors based on the specified strategy
1572
- * @private
1573
- */
1574
- async handleInsertionError(batchError, strategy, tableName, data, validatedData, validationErrors, isStream, shouldValidate, options) {
1231
+ }
1232
+ return { successful, failed };
1233
+ }
1234
+ /**
1235
+ * Validates input parameters and strategy compatibility
1236
+ * @private
1237
+ */
1238
+ validateInsertParameters(data, options) {
1239
+ const isStream = data instanceof Readable;
1240
+ const strategy = options?.strategy || "fail-fast";
1241
+ const shouldValidate = options?.validate !== false;
1242
+ if (isStream && strategy === "isolate") {
1243
+ throw new Error(
1244
+ "The 'isolate' error strategy is not supported with stream input. Use 'fail-fast' or 'discard' instead."
1245
+ );
1246
+ }
1247
+ if (isStream && shouldValidate) {
1248
+ console.warn(
1249
+ "Validation is not supported with stream input. Validation will be skipped."
1250
+ );
1251
+ }
1252
+ return { isStream, strategy, shouldValidate };
1253
+ }
1254
+ /**
1255
+ * Handles early return cases for empty data
1256
+ * @private
1257
+ */
1258
+ handleEmptyData(data, isStream) {
1259
+ if (isStream && !data) {
1260
+ return {
1261
+ successful: 0,
1262
+ failed: 0,
1263
+ total: 0
1264
+ };
1265
+ }
1266
+ if (!isStream && (!data || data.length === 0)) {
1267
+ return {
1268
+ successful: 0,
1269
+ failed: 0,
1270
+ total: 0
1271
+ };
1272
+ }
1273
+ return null;
1274
+ }
1275
+ /**
1276
+ * Performs pre-insertion validation for array data
1277
+ * @private
1278
+ */
1279
+ async performPreInsertionValidation(data, shouldValidate, strategy, options) {
1280
+ if (!shouldValidate) {
1281
+ return { validatedData: data, validationErrors: [] };
1282
+ }
1283
+ try {
1284
+ const validationResult = await this.validateRecords(data);
1285
+ const validatedData = validationResult.valid;
1286
+ const validationErrors = validationResult.invalid;
1287
+ if (validationErrors.length > 0) {
1288
+ this.handleValidationErrors(validationErrors, strategy, data, options);
1575
1289
  switch (strategy) {
1576
- case "fail-fast":
1577
- throw new Error(
1578
- `Failed to insert data into table ${tableName}: ${batchError}`
1579
- );
1580
1290
  case "discard":
1581
- throw new Error(
1582
- `Too many errors during insert into table ${tableName}. Error threshold exceeded: ${batchError}`
1583
- );
1291
+ return { validatedData, validationErrors };
1584
1292
  case "isolate":
1585
- return await this.handleIsolateStrategy(
1586
- batchError,
1587
- tableName,
1588
- data,
1589
- validatedData,
1590
- validationErrors,
1591
- isStream,
1592
- shouldValidate,
1593
- options
1594
- );
1293
+ return { validatedData: data, validationErrors };
1595
1294
  default:
1596
- throw new Error(`Unknown error strategy: ${strategy}`);
1295
+ return { validatedData, validationErrors };
1597
1296
  }
1598
1297
  }
1599
- /**
1600
- * Handles the isolate strategy for insertion errors
1601
- * @private
1602
- */
1603
- async handleIsolateStrategy(batchError, tableName, data, validatedData, validationErrors, isStream, shouldValidate, options) {
1604
- if (isStream) {
1605
- throw new Error(
1606
- `Isolate strategy is not supported with stream input: ${batchError}`
1607
- );
1608
- }
1609
- try {
1610
- const { client } = await this.getMemoizedClient();
1611
- const skipValidationOnRetry = options?.skipValidationOnRetry || false;
1612
- const retryData = skipValidationOnRetry ? data : validatedData;
1613
- const { successful, failed } = await this.retryIndividualRecords(
1614
- client,
1615
- tableName,
1616
- retryData
1617
- );
1618
- const allFailedRecords = [
1619
- // Validation errors (if any and not skipping validation on retry)
1620
- ...shouldValidate && !skipValidationOnRetry ? validationErrors.map((ve) => ({
1621
- record: ve.record,
1622
- error: `Validation error: ${ve.error}`,
1623
- index: ve.index
1624
- })) : [],
1625
- // Insertion errors
1626
- ...failed
1627
- ];
1628
- this.checkInsertionThresholds(
1629
- allFailedRecords,
1630
- data.length,
1631
- options
1632
- );
1633
- return {
1634
- successful: successful.length,
1635
- failed: allFailedRecords.length,
1636
- total: data.length,
1637
- failedRecords: allFailedRecords
1638
- };
1639
- } catch (isolationError) {
1640
- throw new Error(
1641
- `Failed to insert data into table ${tableName} during record isolation: ${isolationError}`
1642
- );
1643
- }
1298
+ return { validatedData, validationErrors };
1299
+ } catch (validationError) {
1300
+ if (strategy === "fail-fast") {
1301
+ throw validationError;
1644
1302
  }
1645
- /**
1646
- * Checks if insertion errors exceed configured thresholds
1647
- * @private
1648
- */
1649
- checkInsertionThresholds(failedRecords, totalRecords, options) {
1650
- const totalFailed = failedRecords.length;
1651
- const failedRatio = totalFailed / totalRecords;
1652
- if (options?.allowErrors !== void 0 && totalFailed > options.allowErrors) {
1653
- throw new Error(
1654
- `Too many failed records: ${totalFailed} > ${options.allowErrors}. Failed records: ${failedRecords.map((f) => f.error).join(", ")}`
1655
- );
1656
- }
1657
- if (options?.allowErrorsRatio !== void 0 && failedRatio > options.allowErrorsRatio) {
1658
- throw new Error(
1659
- `Failed record ratio too high: ${failedRatio.toFixed(3)} > ${options.allowErrorsRatio}. Failed records: ${failedRecords.map((f) => f.error).join(", ")}`
1660
- );
1661
- }
1303
+ console.warn("Validation error:", validationError);
1304
+ return { validatedData: data, validationErrors: [] };
1305
+ }
1306
+ }
1307
+ /**
1308
+ * Handles validation errors based on the specified strategy
1309
+ * @private
1310
+ */
1311
+ handleValidationErrors(validationErrors, strategy, data, options) {
1312
+ switch (strategy) {
1313
+ case "fail-fast":
1314
+ const firstError = validationErrors[0];
1315
+ throw new Error(
1316
+ `Validation failed for record at index ${firstError.index}: ${firstError.error}`
1317
+ );
1318
+ case "discard":
1319
+ this.checkValidationThresholds(validationErrors, data.length, options);
1320
+ break;
1321
+ case "isolate":
1322
+ break;
1323
+ }
1324
+ }
1325
+ /**
1326
+ * Checks if validation errors exceed configured thresholds
1327
+ * @private
1328
+ */
1329
+ checkValidationThresholds(validationErrors, totalRecords, options) {
1330
+ const validationFailedCount = validationErrors.length;
1331
+ const validationFailedRatio = validationFailedCount / totalRecords;
1332
+ if (options?.allowErrors !== void 0 && validationFailedCount > options.allowErrors) {
1333
+ throw new Error(
1334
+ `Too many validation failures: ${validationFailedCount} > ${options.allowErrors}. Errors: ${validationErrors.map((e) => e.error).join(", ")}`
1335
+ );
1336
+ }
1337
+ if (options?.allowErrorsRatio !== void 0 && validationFailedRatio > options.allowErrorsRatio) {
1338
+ throw new Error(
1339
+ `Validation failure ratio too high: ${validationFailedRatio.toFixed(3)} > ${options.allowErrorsRatio}. Errors: ${validationErrors.map((e) => e.error).join(", ")}`
1340
+ );
1341
+ }
1342
+ }
1343
+ /**
1344
+ * Optimized insert options preparation with better memory management
1345
+ * @private
1346
+ */
1347
+ prepareInsertOptions(tableName, data, validatedData, isStream, strategy, options) {
1348
+ const insertOptions = {
1349
+ table: quoteIdentifier(tableName),
1350
+ format: "JSONEachRow",
1351
+ clickhouse_settings: {
1352
+ date_time_input_format: "best_effort",
1353
+ wait_end_of_query: 1,
1354
+ // Ensure at least once delivery for INSERT operations
1355
+ // Performance optimizations
1356
+ max_insert_block_size: isStream ? 1e5 : Math.min(validatedData.length, 1e5),
1357
+ max_block_size: 65536,
1358
+ // Use async inserts for better performance with large datasets
1359
+ async_insert: validatedData.length > 1e3 ? 1 : 0,
1360
+ wait_for_async_insert: 1
1361
+ // For at least once delivery
1662
1362
  }
1663
- /**
1664
- * Recursively transforms a record to match ClickHouse's JSONEachRow requirements
1665
- *
1666
- * - For every Array(Nested(...)) field at any depth, each item is wrapped in its own array and recursively processed.
1667
- * - For every Nested struct (not array), it recurses into the struct.
1668
- * - This ensures compatibility with kafka_clickhouse_sync
1669
- *
1670
- * @param record The input record to transform (may be deeply nested)
1671
- * @param columns The schema columns for this level (defaults to this.columnArray at the top level)
1672
- * @returns The transformed record, ready for ClickHouse JSONEachRow insertion
1673
- */
1674
- mapToClickhouseRecord(record, columns = this.columnArray) {
1675
- const result = { ...record };
1676
- for (const col of columns) {
1677
- const value = record[col.name];
1678
- const dt = col.data_type;
1679
- if (isArrayNestedType(dt)) {
1680
- if (Array.isArray(value) && (value.length === 0 || typeof value[0] === "object")) {
1681
- result[col.name] = value.map((item) => [
1682
- this.mapToClickhouseRecord(item, dt.elementType.columns)
1683
- ]);
1684
- }
1685
- } else if (isNestedType(dt)) {
1686
- if (value && typeof value === "object") {
1687
- result[col.name] = this.mapToClickhouseRecord(value, dt.columns);
1688
- }
1689
- }
1690
- }
1691
- return result;
1363
+ };
1364
+ if (isStream) {
1365
+ insertOptions.values = data;
1366
+ } else {
1367
+ insertOptions.values = validatedData;
1368
+ }
1369
+ if (strategy === "discard" && (options?.allowErrors !== void 0 || options?.allowErrorsRatio !== void 0)) {
1370
+ if (options.allowErrors !== void 0) {
1371
+ insertOptions.clickhouse_settings.input_format_allow_errors_num = options.allowErrors;
1692
1372
  }
1693
- /**
1694
- * Inserts data directly into the ClickHouse table with enhanced error handling and validation.
1695
- * This method establishes a direct connection to ClickHouse using the project configuration
1696
- * and inserts the provided data into the versioned table.
1697
- *
1698
- * PERFORMANCE OPTIMIZATIONS:
1699
- * - Memoized client connections with fast config hashing
1700
- * - Single-pass validation with pre-allocated arrays
1701
- * - Batch-optimized retry strategy (batches of 10, then individual)
1702
- * - Optimized ClickHouse settings for large datasets
1703
- * - Reduced memory allocations and object creation
1704
- *
1705
- * Uses advanced typia validation when available for comprehensive type checking,
1706
- * with fallback to basic validation for compatibility.
1707
- *
1708
- * The ClickHouse client is memoized and reused across multiple insert calls for better performance.
1709
- * If the configuration changes, a new client will be automatically created.
1710
- *
1711
- * @param data Array of objects conforming to the table schema, or a Node.js Readable stream
1712
- * @param options Optional configuration for error handling, validation, and insertion behavior
1713
- * @returns Promise resolving to detailed insertion results
1714
- * @throws {ConfigError} When configuration cannot be read or parsed
1715
- * @throws {ClickHouseError} When insertion fails based on the error strategy
1716
- * @throws {ValidationError} When validation fails and strategy is 'fail-fast'
1717
- *
1718
- * @example
1719
- * ```typescript
1720
- * // Create an OlapTable instance (typia validators auto-injected)
1721
- * const userTable = new OlapTable<User>('users');
1722
- *
1723
- * // Insert with comprehensive typia validation
1724
- * const result1 = await userTable.insert([
1725
- * { id: 1, name: 'John', email: 'john@example.com' },
1726
- * { id: 2, name: 'Jane', email: 'jane@example.com' }
1727
- * ]);
1728
- *
1729
- * // Insert data with stream input (validation not available for streams)
1730
- * const dataStream = new Readable({
1731
- * objectMode: true,
1732
- * read() { // Stream implementation }
1733
- * });
1734
- * const result2 = await userTable.insert(dataStream, { strategy: 'fail-fast' });
1735
- *
1736
- * // Insert with validation disabled for performance
1737
- * const result3 = await userTable.insert(data, { validate: false });
1738
- *
1739
- * // Insert with error handling strategies
1740
- * const result4 = await userTable.insert(mixedData, {
1741
- * strategy: 'isolate',
1742
- * allowErrorsRatio: 0.1,
1743
- * validate: true // Use typia validation (default)
1744
- * });
1745
- *
1746
- * // Optional: Clean up connection when completely done
1747
- * await userTable.closeClient();
1748
- * ```
1749
- */
1750
- async insert(data, options) {
1751
- const { isStream, strategy, shouldValidate } = this.validateInsertParameters(data, options);
1752
- const emptyResult = this.handleEmptyData(data, isStream);
1753
- if (emptyResult) {
1754
- return emptyResult;
1755
- }
1756
- let validatedData = [];
1757
- let validationErrors = [];
1758
- if (!isStream && shouldValidate) {
1759
- const validationResult = await this.performPreInsertionValidation(
1760
- data,
1761
- shouldValidate,
1762
- strategy,
1763
- options
1764
- );
1765
- validatedData = validationResult.validatedData;
1766
- validationErrors = validationResult.validationErrors;
1767
- } else {
1768
- validatedData = isStream ? [] : data;
1373
+ if (options.allowErrorsRatio !== void 0) {
1374
+ insertOptions.clickhouse_settings.input_format_allow_errors_ratio = options.allowErrorsRatio;
1375
+ }
1376
+ }
1377
+ return insertOptions;
1378
+ }
1379
+ /**
1380
+ * Creates success result for completed insertions
1381
+ * @private
1382
+ */
1383
+ createSuccessResult(data, validatedData, validationErrors, isStream, shouldValidate, strategy) {
1384
+ if (isStream) {
1385
+ return {
1386
+ successful: -1,
1387
+ // -1 indicates stream mode where count is unknown
1388
+ failed: 0,
1389
+ total: -1
1390
+ };
1391
+ }
1392
+ const insertedCount = validatedData.length;
1393
+ const totalProcessed = shouldValidate ? data.length : insertedCount;
1394
+ const result = {
1395
+ successful: insertedCount,
1396
+ failed: shouldValidate ? validationErrors.length : 0,
1397
+ total: totalProcessed
1398
+ };
1399
+ if (shouldValidate && validationErrors.length > 0 && strategy === "discard") {
1400
+ result.failedRecords = validationErrors.map((ve) => ({
1401
+ record: ve.record,
1402
+ error: `Validation error: ${ve.error}`,
1403
+ index: ve.index
1404
+ }));
1405
+ }
1406
+ return result;
1407
+ }
1408
+ /**
1409
+ * Handles insertion errors based on the specified strategy
1410
+ * @private
1411
+ */
1412
+ async handleInsertionError(batchError, strategy, tableName, data, validatedData, validationErrors, isStream, shouldValidate, options) {
1413
+ switch (strategy) {
1414
+ case "fail-fast":
1415
+ throw new Error(
1416
+ `Failed to insert data into table ${tableName}: ${batchError}`
1417
+ );
1418
+ case "discard":
1419
+ throw new Error(
1420
+ `Too many errors during insert into table ${tableName}. Error threshold exceeded: ${batchError}`
1421
+ );
1422
+ case "isolate":
1423
+ return await this.handleIsolateStrategy(
1424
+ batchError,
1425
+ tableName,
1426
+ data,
1427
+ validatedData,
1428
+ validationErrors,
1429
+ isStream,
1430
+ shouldValidate,
1431
+ options
1432
+ );
1433
+ default:
1434
+ throw new Error(`Unknown error strategy: ${strategy}`);
1435
+ }
1436
+ }
1437
+ /**
1438
+ * Handles the isolate strategy for insertion errors
1439
+ * @private
1440
+ */
1441
+ async handleIsolateStrategy(batchError, tableName, data, validatedData, validationErrors, isStream, shouldValidate, options) {
1442
+ if (isStream) {
1443
+ throw new Error(
1444
+ `Isolate strategy is not supported with stream input: ${batchError}`
1445
+ );
1446
+ }
1447
+ try {
1448
+ const { client } = await this.getMemoizedClient();
1449
+ const skipValidationOnRetry = options?.skipValidationOnRetry || false;
1450
+ const retryData = skipValidationOnRetry ? data : validatedData;
1451
+ const { successful, failed } = await this.retryIndividualRecords(
1452
+ client,
1453
+ tableName,
1454
+ retryData
1455
+ );
1456
+ const allFailedRecords = [
1457
+ // Validation errors (if any and not skipping validation on retry)
1458
+ ...shouldValidate && !skipValidationOnRetry ? validationErrors.map((ve) => ({
1459
+ record: ve.record,
1460
+ error: `Validation error: ${ve.error}`,
1461
+ index: ve.index
1462
+ })) : [],
1463
+ // Insertion errors
1464
+ ...failed
1465
+ ];
1466
+ this.checkInsertionThresholds(
1467
+ allFailedRecords,
1468
+ data.length,
1469
+ options
1470
+ );
1471
+ return {
1472
+ successful: successful.length,
1473
+ failed: allFailedRecords.length,
1474
+ total: data.length,
1475
+ failedRecords: allFailedRecords
1476
+ };
1477
+ } catch (isolationError) {
1478
+ throw new Error(
1479
+ `Failed to insert data into table ${tableName} during record isolation: ${isolationError}`
1480
+ );
1481
+ }
1482
+ }
1483
+ /**
1484
+ * Checks if insertion errors exceed configured thresholds
1485
+ * @private
1486
+ */
1487
+ checkInsertionThresholds(failedRecords, totalRecords, options) {
1488
+ const totalFailed = failedRecords.length;
1489
+ const failedRatio = totalFailed / totalRecords;
1490
+ if (options?.allowErrors !== void 0 && totalFailed > options.allowErrors) {
1491
+ throw new Error(
1492
+ `Too many failed records: ${totalFailed} > ${options.allowErrors}. Failed records: ${failedRecords.map((f) => f.error).join(", ")}`
1493
+ );
1494
+ }
1495
+ if (options?.allowErrorsRatio !== void 0 && failedRatio > options.allowErrorsRatio) {
1496
+ throw new Error(
1497
+ `Failed record ratio too high: ${failedRatio.toFixed(3)} > ${options.allowErrorsRatio}. Failed records: ${failedRecords.map((f) => f.error).join(", ")}`
1498
+ );
1499
+ }
1500
+ }
1501
+ /**
1502
+ * Recursively transforms a record to match ClickHouse's JSONEachRow requirements
1503
+ *
1504
+ * - For every Array(Nested(...)) field at any depth, each item is wrapped in its own array and recursively processed.
1505
+ * - For every Nested struct (not array), it recurses into the struct.
1506
+ * - This ensures compatibility with kafka_clickhouse_sync
1507
+ *
1508
+ * @param record The input record to transform (may be deeply nested)
1509
+ * @param columns The schema columns for this level (defaults to this.columnArray at the top level)
1510
+ * @returns The transformed record, ready for ClickHouse JSONEachRow insertion
1511
+ */
1512
+ mapToClickhouseRecord(record, columns = this.columnArray) {
1513
+ const result = { ...record };
1514
+ for (const col of columns) {
1515
+ const value = record[col.name];
1516
+ const dt = col.data_type;
1517
+ if (isArrayNestedType(dt)) {
1518
+ if (Array.isArray(value) && (value.length === 0 || typeof value[0] === "object")) {
1519
+ result[col.name] = value.map((item) => [
1520
+ this.mapToClickhouseRecord(item, dt.elementType.columns)
1521
+ ]);
1769
1522
  }
1770
- const { client } = await this.getMemoizedClient();
1771
- const tableName = this.generateTableName();
1772
- try {
1773
- const insertOptions = this.prepareInsertOptions(
1774
- tableName,
1775
- data,
1776
- validatedData,
1777
- isStream,
1778
- strategy,
1779
- options
1780
- );
1781
- await client.insert(insertOptions);
1782
- return this.createSuccessResult(
1783
- data,
1784
- validatedData,
1785
- validationErrors,
1786
- isStream,
1787
- shouldValidate,
1788
- strategy
1789
- );
1790
- } catch (batchError) {
1791
- return await this.handleInsertionError(
1792
- batchError,
1793
- strategy,
1794
- tableName,
1795
- data,
1796
- validatedData,
1797
- validationErrors,
1798
- isStream,
1799
- shouldValidate,
1800
- options
1801
- );
1523
+ } else if (isNestedType(dt)) {
1524
+ if (value && typeof value === "object") {
1525
+ result[col.name] = this.mapToClickhouseRecord(value, dt.columns);
1802
1526
  }
1803
1527
  }
1804
- // Note: Static factory methods (withS3Queue, withReplacingMergeTree, withMergeTree)
1805
- // were removed in ENG-856. Use direct configuration instead, e.g.:
1806
- // new OlapTable(name, { engine: ClickHouseEngines.ReplacingMergeTree, orderByFields: ["id"], ver: "updated_at" })
1807
- };
1528
+ }
1529
+ return result;
1808
1530
  }
1809
- });
1531
+ /**
1532
+ * Inserts data directly into the ClickHouse table with enhanced error handling and validation.
1533
+ * This method establishes a direct connection to ClickHouse using the project configuration
1534
+ * and inserts the provided data into the versioned table.
1535
+ *
1536
+ * PERFORMANCE OPTIMIZATIONS:
1537
+ * - Memoized client connections with fast config hashing
1538
+ * - Single-pass validation with pre-allocated arrays
1539
+ * - Batch-optimized retry strategy (batches of 10, then individual)
1540
+ * - Optimized ClickHouse settings for large datasets
1541
+ * - Reduced memory allocations and object creation
1542
+ *
1543
+ * Uses advanced typia validation when available for comprehensive type checking,
1544
+ * with fallback to basic validation for compatibility.
1545
+ *
1546
+ * The ClickHouse client is memoized and reused across multiple insert calls for better performance.
1547
+ * If the configuration changes, a new client will be automatically created.
1548
+ *
1549
+ * @param data Array of objects conforming to the table schema, or a Node.js Readable stream
1550
+ * @param options Optional configuration for error handling, validation, and insertion behavior
1551
+ * @returns Promise resolving to detailed insertion results
1552
+ * @throws {ConfigError} When configuration cannot be read or parsed
1553
+ * @throws {ClickHouseError} When insertion fails based on the error strategy
1554
+ * @throws {ValidationError} When validation fails and strategy is 'fail-fast'
1555
+ *
1556
+ * @example
1557
+ * ```typescript
1558
+ * // Create an OlapTable instance (typia validators auto-injected)
1559
+ * const userTable = new OlapTable<User>('users');
1560
+ *
1561
+ * // Insert with comprehensive typia validation
1562
+ * const result1 = await userTable.insert([
1563
+ * { id: 1, name: 'John', email: 'john@example.com' },
1564
+ * { id: 2, name: 'Jane', email: 'jane@example.com' }
1565
+ * ]);
1566
+ *
1567
+ * // Insert data with stream input (validation not available for streams)
1568
+ * const dataStream = new Readable({
1569
+ * objectMode: true,
1570
+ * read() { // Stream implementation }
1571
+ * });
1572
+ * const result2 = await userTable.insert(dataStream, { strategy: 'fail-fast' });
1573
+ *
1574
+ * // Insert with validation disabled for performance
1575
+ * const result3 = await userTable.insert(data, { validate: false });
1576
+ *
1577
+ * // Insert with error handling strategies
1578
+ * const result4 = await userTable.insert(mixedData, {
1579
+ * strategy: 'isolate',
1580
+ * allowErrorsRatio: 0.1,
1581
+ * validate: true // Use typia validation (default)
1582
+ * });
1583
+ *
1584
+ * // Optional: Clean up connection when completely done
1585
+ * await userTable.closeClient();
1586
+ * ```
1587
+ */
1588
+ async insert(data, options) {
1589
+ const { isStream, strategy, shouldValidate } = this.validateInsertParameters(data, options);
1590
+ const emptyResult = this.handleEmptyData(data, isStream);
1591
+ if (emptyResult) {
1592
+ return emptyResult;
1593
+ }
1594
+ let validatedData = [];
1595
+ let validationErrors = [];
1596
+ if (!isStream && shouldValidate) {
1597
+ const validationResult = await this.performPreInsertionValidation(
1598
+ data,
1599
+ shouldValidate,
1600
+ strategy,
1601
+ options
1602
+ );
1603
+ validatedData = validationResult.validatedData;
1604
+ validationErrors = validationResult.validationErrors;
1605
+ } else {
1606
+ validatedData = isStream ? [] : data;
1607
+ }
1608
+ const { client } = await this.getMemoizedClient();
1609
+ const tableName = this.generateTableName();
1610
+ try {
1611
+ const insertOptions = this.prepareInsertOptions(
1612
+ tableName,
1613
+ data,
1614
+ validatedData,
1615
+ isStream,
1616
+ strategy,
1617
+ options
1618
+ );
1619
+ await client.insert(insertOptions);
1620
+ return this.createSuccessResult(
1621
+ data,
1622
+ validatedData,
1623
+ validationErrors,
1624
+ isStream,
1625
+ shouldValidate,
1626
+ strategy
1627
+ );
1628
+ } catch (batchError) {
1629
+ return await this.handleInsertionError(
1630
+ batchError,
1631
+ strategy,
1632
+ tableName,
1633
+ data,
1634
+ validatedData,
1635
+ validationErrors,
1636
+ isStream,
1637
+ shouldValidate,
1638
+ options
1639
+ );
1640
+ }
1641
+ }
1642
+ // Note: Static factory methods (withS3Queue, withReplacingMergeTree, withMergeTree)
1643
+ // were removed in ENG-856. Use direct configuration instead, e.g.:
1644
+ // new OlapTable(name, { engine: ClickHouseEngines.ReplacingMergeTree, orderByFields: ["id"], ver: "updated_at" })
1645
+ };
1810
1646
 
1811
1647
  // src/dmv2/sdk/stream.ts
1812
1648
  import { createHash as createHash3 } from "crypto";
1813
- function attachTypeGuard(dl, typeGuard) {
1814
- dl.asTyped = () => typeGuard(dl.originalRecord);
1815
- }
1816
- var RoutedMessage, Stream, DeadLetterQueue;
1817
- var init_stream = __esm({
1818
- "src/dmv2/sdk/stream.ts"() {
1819
- "use strict";
1820
- init_typedBase();
1821
- init_internal();
1822
- init_stackTrace();
1823
- RoutedMessage = class {
1824
- /** The destination stream for the message */
1825
- destination;
1826
- /** The message value(s) to send */
1827
- values;
1828
- /**
1829
- * Creates a new routed message.
1830
- *
1831
- * @param destination The target stream
1832
- * @param values The message(s) to route
1833
- */
1834
- constructor(destination, values) {
1835
- this.destination = destination;
1836
- this.values = values;
1837
- }
1838
- };
1839
- Stream = class extends TypedBase {
1840
- defaultDeadLetterQueue;
1841
- /** @internal Memoized KafkaJS producer for reusing connections across sends */
1842
- _memoizedProducer;
1843
- /** @internal Hash of the configuration used to create the memoized Kafka producer */
1844
- _kafkaConfigHash;
1845
- constructor(name, config, schema, columns, validators, allowExtraFields) {
1846
- super(name, config ?? {}, schema, columns, void 0, allowExtraFields);
1847
- const streams = getMooseInternal().streams;
1848
- if (streams.has(name)) {
1849
- throw new Error(`Stream with name ${name} already exists`);
1850
- }
1851
- streams.set(name, this);
1852
- this.defaultDeadLetterQueue = this.config.defaultDeadLetterQueue;
1853
- }
1854
- /**
1855
- * Internal map storing transformation configurations.
1856
- * Maps destination stream names to arrays of transformation functions and their configs.
1857
- *
1858
- * @internal
1859
- */
1860
- _transformations = /* @__PURE__ */ new Map();
1861
- /**
1862
- * Internal function for multi-stream transformations.
1863
- * Allows a single transformation to route messages to multiple destinations.
1864
- *
1865
- * @internal
1866
- */
1867
- _multipleTransformations;
1868
- /**
1869
- * Internal array storing consumer configurations.
1870
- *
1871
- * @internal
1872
- */
1873
- _consumers = new Array();
1874
- /**
1875
- * Builds the full Kafka topic name including optional namespace and version suffix.
1876
- * Version suffix is appended as _x_y_z where dots in version are replaced with underscores.
1877
- */
1878
- buildFullTopicName(namespace) {
1879
- const versionSuffix = this.config.version ? `_${this.config.version.replace(/\./g, "_")}` : "";
1880
- const base = `${this.name}${versionSuffix}`;
1881
- return namespace !== void 0 && namespace.length > 0 ? `${namespace}.${base}` : base;
1882
- }
1883
- /**
1884
- * Creates a fast hash string from relevant Kafka configuration fields.
1885
- */
1886
- createConfigHash(kafkaConfig) {
1887
- const configString = [
1888
- kafkaConfig.broker,
1889
- kafkaConfig.messageTimeoutMs,
1890
- kafkaConfig.saslUsername,
1891
- kafkaConfig.saslPassword,
1892
- kafkaConfig.saslMechanism,
1893
- kafkaConfig.securityProtocol,
1894
- kafkaConfig.namespace
1895
- ].join(":");
1896
- return createHash3("sha256").update(configString).digest("hex").substring(0, 16);
1897
- }
1898
- /**
1899
- * Gets or creates a memoized KafkaJS producer using runtime configuration.
1900
- */
1901
- async getMemoizedProducer() {
1902
- await Promise.resolve().then(() => (init_runtime(), runtime_exports));
1903
- const configRegistry = globalThis._mooseConfigRegistry;
1904
- const { getKafkaProducer: getKafkaProducer2 } = await Promise.resolve().then(() => (init_commons(), commons_exports));
1905
- const kafkaConfig = await configRegistry.getKafkaConfig();
1906
- const currentHash = this.createConfigHash(kafkaConfig);
1907
- if (this._memoizedProducer && this._kafkaConfigHash === currentHash) {
1908
- return { producer: this._memoizedProducer, kafkaConfig };
1909
- }
1910
- if (this._memoizedProducer && this._kafkaConfigHash !== currentHash) {
1911
- try {
1912
- await this._memoizedProducer.disconnect();
1913
- } catch {
1914
- }
1915
- this._memoizedProducer = void 0;
1916
- }
1917
- const clientId = `moose-sdk-stream-${this.name}`;
1918
- const logger = {
1919
- logPrefix: clientId,
1920
- log: (message) => {
1921
- console.log(`${clientId}: ${message}`);
1922
- },
1923
- error: (message) => {
1924
- console.error(`${clientId}: ${message}`);
1925
- },
1926
- warn: (message) => {
1927
- console.warn(`${clientId}: ${message}`);
1928
- }
1929
- };
1930
- const producer = await getKafkaProducer2(
1931
- {
1932
- clientId,
1933
- broker: kafkaConfig.broker,
1934
- securityProtocol: kafkaConfig.securityProtocol,
1935
- saslUsername: kafkaConfig.saslUsername,
1936
- saslPassword: kafkaConfig.saslPassword,
1937
- saslMechanism: kafkaConfig.saslMechanism
1938
- },
1939
- logger
1940
- );
1941
- this._memoizedProducer = producer;
1942
- this._kafkaConfigHash = currentHash;
1943
- return { producer, kafkaConfig };
1944
- }
1945
- /**
1946
- * Closes the memoized Kafka producer if it exists.
1947
- */
1948
- async closeProducer() {
1949
- if (this._memoizedProducer) {
1950
- try {
1951
- await this._memoizedProducer.disconnect();
1952
- } catch {
1953
- } finally {
1954
- this._memoizedProducer = void 0;
1955
- this._kafkaConfigHash = void 0;
1956
- }
1957
- }
1649
+ var RoutedMessage = class {
1650
+ /** The destination stream for the message */
1651
+ destination;
1652
+ /** The message value(s) to send */
1653
+ values;
1654
+ /**
1655
+ * Creates a new routed message.
1656
+ *
1657
+ * @param destination The target stream
1658
+ * @param values The message(s) to route
1659
+ */
1660
+ constructor(destination, values) {
1661
+ this.destination = destination;
1662
+ this.values = values;
1663
+ }
1664
+ };
1665
+ var Stream = class extends TypedBase {
1666
+ defaultDeadLetterQueue;
1667
+ /** @internal Memoized KafkaJS producer for reusing connections across sends */
1668
+ _memoizedProducer;
1669
+ /** @internal Hash of the configuration used to create the memoized Kafka producer */
1670
+ _kafkaConfigHash;
1671
+ constructor(name, config, schema, columns, validators, allowExtraFields) {
1672
+ super(name, config ?? {}, schema, columns, void 0, allowExtraFields);
1673
+ const streams = getMooseInternal().streams;
1674
+ if (streams.has(name)) {
1675
+ throw new Error(`Stream with name ${name} already exists`);
1676
+ }
1677
+ streams.set(name, this);
1678
+ this.defaultDeadLetterQueue = this.config.defaultDeadLetterQueue;
1679
+ }
1680
+ /**
1681
+ * Internal map storing transformation configurations.
1682
+ * Maps destination stream names to arrays of transformation functions and their configs.
1683
+ *
1684
+ * @internal
1685
+ */
1686
+ _transformations = /* @__PURE__ */ new Map();
1687
+ /**
1688
+ * Internal function for multi-stream transformations.
1689
+ * Allows a single transformation to route messages to multiple destinations.
1690
+ *
1691
+ * @internal
1692
+ */
1693
+ _multipleTransformations;
1694
+ /**
1695
+ * Internal array storing consumer configurations.
1696
+ *
1697
+ * @internal
1698
+ */
1699
+ _consumers = new Array();
1700
+ /**
1701
+ * Builds the full Kafka topic name including optional namespace and version suffix.
1702
+ * Version suffix is appended as _x_y_z where dots in version are replaced with underscores.
1703
+ */
1704
+ buildFullTopicName(namespace) {
1705
+ const versionSuffix = this.config.version ? `_${this.config.version.replace(/\./g, "_")}` : "";
1706
+ const base = `${this.name}${versionSuffix}`;
1707
+ return namespace !== void 0 && namespace.length > 0 ? `${namespace}.${base}` : base;
1708
+ }
1709
+ /**
1710
+ * Creates a fast hash string from relevant Kafka configuration fields.
1711
+ */
1712
+ createConfigHash(kafkaConfig) {
1713
+ const configString = [
1714
+ kafkaConfig.broker,
1715
+ kafkaConfig.messageTimeoutMs,
1716
+ kafkaConfig.saslUsername,
1717
+ kafkaConfig.saslPassword,
1718
+ kafkaConfig.saslMechanism,
1719
+ kafkaConfig.securityProtocol,
1720
+ kafkaConfig.namespace
1721
+ ].join(":");
1722
+ return createHash3("sha256").update(configString).digest("hex").substring(0, 16);
1723
+ }
1724
+ /**
1725
+ * Gets or creates a memoized KafkaJS producer using runtime configuration.
1726
+ */
1727
+ async getMemoizedProducer() {
1728
+ await Promise.resolve().then(() => (init_runtime(), runtime_exports));
1729
+ const configRegistry = globalThis._mooseConfigRegistry;
1730
+ const { getKafkaProducer: getKafkaProducer2 } = await Promise.resolve().then(() => (init_commons(), commons_exports));
1731
+ const kafkaConfig = await configRegistry.getKafkaConfig();
1732
+ const currentHash = this.createConfigHash(kafkaConfig);
1733
+ if (this._memoizedProducer && this._kafkaConfigHash === currentHash) {
1734
+ return { producer: this._memoizedProducer, kafkaConfig };
1735
+ }
1736
+ if (this._memoizedProducer && this._kafkaConfigHash !== currentHash) {
1737
+ try {
1738
+ await this._memoizedProducer.disconnect();
1739
+ } catch {
1958
1740
  }
1959
- /**
1960
- * Sends one or more records to this stream's Kafka topic.
1961
- * Values are JSON-serialized as message values.
1962
- */
1963
- async send(values) {
1964
- const flat = Array.isArray(values) ? values : values !== void 0 && values !== null ? [values] : [];
1965
- if (flat.length === 0) return;
1966
- const { producer, kafkaConfig } = await this.getMemoizedProducer();
1967
- const topic = this.buildFullTopicName(kafkaConfig.namespace);
1968
- const sr = this.config.schemaConfig;
1969
- if (sr && sr.kind === "JSON") {
1970
- const schemaRegistryUrl = kafkaConfig.schemaRegistryUrl;
1971
- if (!schemaRegistryUrl) {
1972
- throw new Error("Schema Registry URL not configured");
1973
- }
1974
- const {
1975
- default: { SchemaRegistry }
1976
- } = await import("@kafkajs/confluent-schema-registry");
1977
- const registry = new SchemaRegistry({ host: schemaRegistryUrl });
1978
- let schemaId = void 0;
1979
- if ("id" in sr.reference) {
1980
- schemaId = sr.reference.id;
1981
- } else if ("subjectLatest" in sr.reference) {
1982
- schemaId = await registry.getLatestSchemaId(sr.reference.subjectLatest);
1983
- } else if ("subject" in sr.reference) {
1984
- schemaId = await registry.getRegistryId(
1985
- sr.reference.subject,
1986
- sr.reference.version
1987
- );
1988
- }
1989
- if (schemaId === void 0) {
1990
- throw new Error("Malformed schema reference.");
1991
- }
1992
- const encoded = await Promise.all(
1993
- flat.map(
1994
- (v) => registry.encode(schemaId, v)
1995
- )
1996
- );
1997
- await producer.send({
1998
- topic,
1999
- messages: encoded.map((value) => ({ value }))
2000
- });
2001
- return;
2002
- } else if (sr !== void 0) {
2003
- throw new Error("Currently only JSON Schema is supported.");
2004
- }
2005
- await producer.send({
2006
- topic,
2007
- messages: flat.map((v) => ({ value: JSON.stringify(v) }))
2008
- });
1741
+ this._memoizedProducer = void 0;
1742
+ }
1743
+ const clientId = `moose-sdk-stream-${this.name}`;
1744
+ const logger = {
1745
+ logPrefix: clientId,
1746
+ log: (message) => {
1747
+ console.log(`${clientId}: ${message}`);
1748
+ },
1749
+ error: (message) => {
1750
+ console.error(`${clientId}: ${message}`);
1751
+ },
1752
+ warn: (message) => {
1753
+ console.warn(`${clientId}: ${message}`);
2009
1754
  }
2010
- /**
2011
- * Adds a transformation step that processes messages from this stream and sends the results to a destination stream.
2012
- * Multiple transformations to the same destination stream can be added if they have distinct `version` identifiers in their config.
2013
- *
2014
- * @template U The data type of the messages in the destination stream.
2015
- * @param destination The destination stream for the transformed messages.
2016
- * @param transformation A function that takes a message of type T and returns zero or more messages of type U (or a Promise thereof).
2017
- * Return `null` or `undefined` or an empty array `[]` to filter out a message. Return an array to emit multiple messages.
2018
- * @param config Optional configuration for this specific transformation step, like a version.
2019
- */
2020
- addTransform(destination, transformation, config) {
2021
- const sourceFile = getSourceFileFromStack(new Error().stack);
2022
- const transformConfig = {
2023
- ...config ?? {},
2024
- sourceFile
2025
- };
2026
- if (transformConfig.deadLetterQueue === void 0) {
2027
- transformConfig.deadLetterQueue = this.defaultDeadLetterQueue;
2028
- }
2029
- if (this._transformations.has(destination.name)) {
2030
- const existingTransforms = this._transformations.get(destination.name);
2031
- const hasVersion = existingTransforms.some(
2032
- ([_, __, cfg]) => cfg.version === transformConfig.version
2033
- );
2034
- if (!hasVersion) {
2035
- existingTransforms.push([destination, transformation, transformConfig]);
2036
- }
2037
- } else {
2038
- this._transformations.set(destination.name, [
2039
- [destination, transformation, transformConfig]
2040
- ]);
2041
- }
1755
+ };
1756
+ const producer = await getKafkaProducer2(
1757
+ {
1758
+ clientId,
1759
+ broker: kafkaConfig.broker,
1760
+ securityProtocol: kafkaConfig.securityProtocol,
1761
+ saslUsername: kafkaConfig.saslUsername,
1762
+ saslPassword: kafkaConfig.saslPassword,
1763
+ saslMechanism: kafkaConfig.saslMechanism
1764
+ },
1765
+ logger
1766
+ );
1767
+ this._memoizedProducer = producer;
1768
+ this._kafkaConfigHash = currentHash;
1769
+ return { producer, kafkaConfig };
1770
+ }
1771
+ /**
1772
+ * Closes the memoized Kafka producer if it exists.
1773
+ */
1774
+ async closeProducer() {
1775
+ if (this._memoizedProducer) {
1776
+ try {
1777
+ await this._memoizedProducer.disconnect();
1778
+ } catch {
1779
+ } finally {
1780
+ this._memoizedProducer = void 0;
1781
+ this._kafkaConfigHash = void 0;
2042
1782
  }
2043
- /**
2044
- * Adds a consumer function that processes messages from this stream.
2045
- * Multiple consumers can be added if they have distinct `version` identifiers in their config.
2046
- *
2047
- * @param consumer A function that takes a message of type T and performs an action (e.g., side effect, logging). Should return void or Promise<void>.
2048
- * @param config Optional configuration for this specific consumer, like a version.
2049
- */
2050
- addConsumer(consumer, config) {
2051
- const sourceFile = getSourceFileFromStack(new Error().stack);
2052
- const consumerConfig = {
2053
- ...config ?? {},
2054
- sourceFile
2055
- };
2056
- if (consumerConfig.deadLetterQueue === void 0) {
2057
- consumerConfig.deadLetterQueue = this.defaultDeadLetterQueue;
2058
- }
2059
- const hasVersion = this._consumers.some(
2060
- (existing) => existing.config.version === consumerConfig.version
1783
+ }
1784
+ }
1785
+ /**
1786
+ * Sends one or more records to this stream's Kafka topic.
1787
+ * Values are JSON-serialized as message values.
1788
+ */
1789
+ async send(values) {
1790
+ const flat = Array.isArray(values) ? values : values !== void 0 && values !== null ? [values] : [];
1791
+ if (flat.length === 0) return;
1792
+ const { producer, kafkaConfig } = await this.getMemoizedProducer();
1793
+ const topic = this.buildFullTopicName(kafkaConfig.namespace);
1794
+ const sr = this.config.schemaConfig;
1795
+ if (sr && sr.kind === "JSON") {
1796
+ const schemaRegistryUrl = kafkaConfig.schemaRegistryUrl;
1797
+ if (!schemaRegistryUrl) {
1798
+ throw new Error("Schema Registry URL not configured");
1799
+ }
1800
+ const {
1801
+ default: { SchemaRegistry }
1802
+ } = await import("@kafkajs/confluent-schema-registry");
1803
+ const registry = new SchemaRegistry({ host: schemaRegistryUrl });
1804
+ let schemaId = void 0;
1805
+ if ("id" in sr.reference) {
1806
+ schemaId = sr.reference.id;
1807
+ } else if ("subjectLatest" in sr.reference) {
1808
+ schemaId = await registry.getLatestSchemaId(sr.reference.subjectLatest);
1809
+ } else if ("subject" in sr.reference) {
1810
+ schemaId = await registry.getRegistryId(
1811
+ sr.reference.subject,
1812
+ sr.reference.version
2061
1813
  );
2062
- if (!hasVersion) {
2063
- this._consumers.push({ consumer, config: consumerConfig });
2064
- }
2065
1814
  }
2066
- /**
2067
- * Helper method for `addMultiTransform` to specify the destination and values for a routed message.
2068
- * @param values The value or values to send to this stream.
2069
- * @returns A `RoutedMessage` object associating the values with this stream.
2070
- *
2071
- * @example
2072
- * ```typescript
2073
- * sourceStream.addMultiTransform((record) => [
2074
- * destinationStream1.routed(transformedRecord1),
2075
- * destinationStream2.routed([record2a, record2b])
2076
- * ]);
2077
- * ```
2078
- */
2079
- routed = (values) => new RoutedMessage(this, values);
2080
- /**
2081
- * Adds a single transformation function that can route messages to multiple destination streams.
2082
- * This is an alternative to adding multiple individual `addTransform` calls.
2083
- * Only one multi-transform function can be added per stream.
2084
- *
2085
- * @param transformation A function that takes a message of type T and returns an array of `RoutedMessage` objects,
2086
- * each specifying a destination stream and the message(s) to send to it.
2087
- */
2088
- addMultiTransform(transformation) {
2089
- this._multipleTransformations = transformation;
1815
+ if (schemaId === void 0) {
1816
+ throw new Error("Malformed schema reference.");
2090
1817
  }
1818
+ const encoded = await Promise.all(
1819
+ flat.map(
1820
+ (v) => registry.encode(schemaId, v)
1821
+ )
1822
+ );
1823
+ await producer.send({
1824
+ topic,
1825
+ messages: encoded.map((value) => ({ value }))
1826
+ });
1827
+ return;
1828
+ } else if (sr !== void 0) {
1829
+ throw new Error("Currently only JSON Schema is supported.");
1830
+ }
1831
+ await producer.send({
1832
+ topic,
1833
+ messages: flat.map((v) => ({ value: JSON.stringify(v) }))
1834
+ });
1835
+ }
1836
+ /**
1837
+ * Adds a transformation step that processes messages from this stream and sends the results to a destination stream.
1838
+ * Multiple transformations to the same destination stream can be added if they have distinct `version` identifiers in their config.
1839
+ *
1840
+ * @template U The data type of the messages in the destination stream.
1841
+ * @param destination The destination stream for the transformed messages.
1842
+ * @param transformation A function that takes a message of type T and returns zero or more messages of type U (or a Promise thereof).
1843
+ * Return `null` or `undefined` or an empty array `[]` to filter out a message. Return an array to emit multiple messages.
1844
+ * @param config Optional configuration for this specific transformation step, like a version.
1845
+ */
1846
+ addTransform(destination, transformation, config) {
1847
+ const sourceFile = getSourceFileFromStack(new Error().stack);
1848
+ const transformConfig = {
1849
+ ...config ?? {},
1850
+ sourceFile
2091
1851
  };
2092
- DeadLetterQueue = class extends Stream {
2093
- constructor(name, config, typeGuard) {
2094
- if (typeGuard === void 0) {
2095
- throw new Error(
2096
- "Supply the type param T so that the schema is inserted by the compiler plugin."
2097
- );
2098
- }
2099
- super(name, config ?? {}, dlqSchema, dlqColumns, void 0, false);
2100
- this.typeGuard = typeGuard;
2101
- getMooseInternal().streams.set(name, this);
2102
- }
2103
- /**
2104
- * Internal type guard function for validating and casting original records.
2105
- *
2106
- * @internal
2107
- */
2108
- typeGuard;
2109
- /**
2110
- * Adds a transformation step for dead letter records.
2111
- * The transformation function receives a DeadLetter<T> with type recovery capabilities.
2112
- *
2113
- * @template U The output type for the transformation
2114
- * @param destination The destination stream for transformed messages
2115
- * @param transformation Function to transform dead letter records
2116
- * @param config Optional transformation configuration
2117
- */
2118
- addTransform(destination, transformation, config) {
2119
- const withValidate = (deadLetter) => {
2120
- attachTypeGuard(deadLetter, this.typeGuard);
2121
- return transformation(deadLetter);
2122
- };
2123
- super.addTransform(destination, withValidate, config);
2124
- }
2125
- /**
2126
- * Adds a consumer for dead letter records.
2127
- * The consumer function receives a DeadLetter<T> with type recovery capabilities.
2128
- *
2129
- * @param consumer Function to process dead letter records
2130
- * @param config Optional consumer configuration
2131
- */
2132
- addConsumer(consumer, config) {
2133
- const withValidate = (deadLetter) => {
2134
- attachTypeGuard(deadLetter, this.typeGuard);
2135
- return consumer(deadLetter);
2136
- };
2137
- super.addConsumer(withValidate, config);
2138
- }
2139
- /**
2140
- * Adds a multi-stream transformation for dead letter records.
2141
- * The transformation function receives a DeadLetter<T> with type recovery capabilities.
2142
- *
2143
- * @param transformation Function to route dead letter records to multiple destinations
2144
- */
2145
- addMultiTransform(transformation) {
2146
- const withValidate = (deadLetter) => {
2147
- attachTypeGuard(deadLetter, this.typeGuard);
2148
- return transformation(deadLetter);
2149
- };
2150
- super.addMultiTransform(withValidate);
1852
+ if (transformConfig.deadLetterQueue === void 0) {
1853
+ transformConfig.deadLetterQueue = this.defaultDeadLetterQueue;
1854
+ }
1855
+ if (this._transformations.has(destination.name)) {
1856
+ const existingTransforms = this._transformations.get(destination.name);
1857
+ const hasVersion = existingTransforms.some(
1858
+ ([_, __, cfg]) => cfg.version === transformConfig.version
1859
+ );
1860
+ if (!hasVersion) {
1861
+ existingTransforms.push([destination, transformation, transformConfig]);
2151
1862
  }
1863
+ } else {
1864
+ this._transformations.set(destination.name, [
1865
+ [destination, transformation, transformConfig]
1866
+ ]);
1867
+ }
1868
+ }
1869
+ /**
1870
+ * Adds a consumer function that processes messages from this stream.
1871
+ * Multiple consumers can be added if they have distinct `version` identifiers in their config.
1872
+ *
1873
+ * @param consumer A function that takes a message of type T and performs an action (e.g., side effect, logging). Should return void or Promise<void>.
1874
+ * @param config Optional configuration for this specific consumer, like a version.
1875
+ */
1876
+ addConsumer(consumer, config) {
1877
+ const sourceFile = getSourceFileFromStack(new Error().stack);
1878
+ const consumerConfig = {
1879
+ ...config ?? {},
1880
+ sourceFile
2152
1881
  };
1882
+ if (consumerConfig.deadLetterQueue === void 0) {
1883
+ consumerConfig.deadLetterQueue = this.defaultDeadLetterQueue;
1884
+ }
1885
+ const hasVersion = this._consumers.some(
1886
+ (existing) => existing.config.version === consumerConfig.version
1887
+ );
1888
+ if (!hasVersion) {
1889
+ this._consumers.push({ consumer, config: consumerConfig });
1890
+ }
2153
1891
  }
2154
- });
2155
-
2156
- // src/dmv2/sdk/workflow.ts
2157
- var Task, Workflow;
2158
- var init_workflow = __esm({
2159
- "src/dmv2/sdk/workflow.ts"() {
2160
- "use strict";
2161
- init_internal();
2162
- Task = class {
2163
- /**
2164
- * Creates a new Task instance.
2165
- *
2166
- * @param name - Unique identifier for the task
2167
- * @param config - Configuration object defining the task behavior
2168
- *
2169
- * @example
2170
- * ```typescript
2171
- * // No input, no output
2172
- * const task1 = new Task<null, void>("task1", {
2173
- * run: async () => {
2174
- * console.log("No input/output");
2175
- * }
2176
- * });
2177
- *
2178
- * // No input, but has output
2179
- * const task2 = new Task<null, OutputType>("task2", {
2180
- * run: async () => {
2181
- * return someOutput;
2182
- * }
2183
- * });
2184
- *
2185
- * // Has input, no output
2186
- * const task3 = new Task<InputType, void>("task3", {
2187
- * run: async (input: InputType) => {
2188
- * // process input but return nothing
2189
- * }
2190
- * });
2191
- *
2192
- * // Has both input and output
2193
- * const task4 = new Task<InputType, OutputType>("task4", {
2194
- * run: async (input: InputType) => {
2195
- * return process(input);
2196
- * }
2197
- * });
2198
- * ```
2199
- */
2200
- constructor(name, config) {
2201
- this.name = name;
2202
- this.config = config;
2203
- }
1892
+ /**
1893
+ * Helper method for `addMultiTransform` to specify the destination and values for a routed message.
1894
+ * @param values The value or values to send to this stream.
1895
+ * @returns A `RoutedMessage` object associating the values with this stream.
1896
+ *
1897
+ * @example
1898
+ * ```typescript
1899
+ * sourceStream.addMultiTransform((record) => [
1900
+ * destinationStream1.routed(transformedRecord1),
1901
+ * destinationStream2.routed([record2a, record2b])
1902
+ * ]);
1903
+ * ```
1904
+ */
1905
+ routed = (values) => new RoutedMessage(this, values);
1906
+ /**
1907
+ * Adds a single transformation function that can route messages to multiple destination streams.
1908
+ * This is an alternative to adding multiple individual `addTransform` calls.
1909
+ * Only one multi-transform function can be added per stream.
1910
+ *
1911
+ * @param transformation A function that takes a message of type T and returns an array of `RoutedMessage` objects,
1912
+ * each specifying a destination stream and the message(s) to send to it.
1913
+ */
1914
+ addMultiTransform(transformation) {
1915
+ this._multipleTransformations = transformation;
1916
+ }
1917
+ };
1918
+ function attachTypeGuard(dl, typeGuard) {
1919
+ dl.asTyped = () => typeGuard(dl.originalRecord);
1920
+ }
1921
+ var DeadLetterQueue = class extends Stream {
1922
+ constructor(name, config, typeGuard) {
1923
+ if (typeGuard === void 0) {
1924
+ throw new Error(
1925
+ "Supply the type param T so that the schema is inserted by the compiler plugin."
1926
+ );
1927
+ }
1928
+ super(name, config ?? {}, dlqSchema, dlqColumns, void 0, false);
1929
+ this.typeGuard = typeGuard;
1930
+ getMooseInternal().streams.set(name, this);
1931
+ }
1932
+ /**
1933
+ * Internal type guard function for validating and casting original records.
1934
+ *
1935
+ * @internal
1936
+ */
1937
+ typeGuard;
1938
+ /**
1939
+ * Adds a transformation step for dead letter records.
1940
+ * The transformation function receives a DeadLetter<T> with type recovery capabilities.
1941
+ *
1942
+ * @template U The output type for the transformation
1943
+ * @param destination The destination stream for transformed messages
1944
+ * @param transformation Function to transform dead letter records
1945
+ * @param config Optional transformation configuration
1946
+ */
1947
+ addTransform(destination, transformation, config) {
1948
+ const withValidate = (deadLetter) => {
1949
+ attachTypeGuard(deadLetter, this.typeGuard);
1950
+ return transformation(deadLetter);
2204
1951
  };
2205
- Workflow = class {
2206
- /**
2207
- * Creates a new Workflow instance and registers it with the Moose system.
2208
- *
2209
- * @param name - Unique identifier for the workflow
2210
- * @param config - Configuration object defining the workflow behavior and task orchestration
2211
- * @throws {Error} When the workflow contains null/undefined tasks or infinite loops
2212
- */
2213
- constructor(name, config) {
2214
- this.name = name;
2215
- this.config = config;
2216
- const workflows = getMooseInternal().workflows;
2217
- if (workflows.has(name)) {
2218
- throw new Error(`Workflow with name ${name} already exists`);
2219
- }
2220
- this.validateTaskGraph(config.startingTask, name);
2221
- workflows.set(name, this);
2222
- }
2223
- /**
2224
- * Validates the task graph to ensure there are no null tasks or infinite loops.
2225
- *
2226
- * @private
2227
- * @param startingTask - The starting task to begin validation from
2228
- * @param workflowName - The name of the workflow being validated (for error messages)
2229
- * @throws {Error} When null/undefined tasks are found or infinite loops are detected
2230
- */
2231
- validateTaskGraph(startingTask, workflowName) {
2232
- if (startingTask === null || startingTask === void 0) {
2233
- throw new Error(
2234
- `Workflow "${workflowName}" has a null or undefined starting task`
2235
- );
2236
- }
2237
- const visited = /* @__PURE__ */ new Set();
2238
- const recursionStack = /* @__PURE__ */ new Set();
2239
- const validateTask = (task, currentPath) => {
2240
- if (task === null || task === void 0) {
2241
- const pathStr = currentPath.length > 0 ? currentPath.join(" -> ") + " -> " : "";
2242
- throw new Error(
2243
- `Workflow "${workflowName}" contains a null or undefined task in the task chain: ${pathStr}null`
2244
- );
2245
- }
2246
- const taskName = task.name;
2247
- if (recursionStack.has(taskName)) {
2248
- const cycleStartIndex = currentPath.indexOf(taskName);
2249
- const cyclePath = cycleStartIndex >= 0 ? currentPath.slice(cycleStartIndex).concat(taskName) : currentPath.concat(taskName);
2250
- throw new Error(
2251
- `Workflow "${workflowName}" contains an infinite loop in task chain: ${cyclePath.join(" -> ")}`
2252
- );
2253
- }
2254
- if (visited.has(taskName)) {
2255
- return;
2256
- }
2257
- visited.add(taskName);
2258
- recursionStack.add(taskName);
2259
- if (task.config.onComplete) {
2260
- for (const nextTask of task.config.onComplete) {
2261
- validateTask(nextTask, [...currentPath, taskName]);
2262
- }
2263
- }
2264
- recursionStack.delete(taskName);
2265
- };
2266
- validateTask(startingTask, []);
2267
- }
1952
+ super.addTransform(destination, withValidate, config);
1953
+ }
1954
+ /**
1955
+ * Adds a consumer for dead letter records.
1956
+ * The consumer function receives a DeadLetter<T> with type recovery capabilities.
1957
+ *
1958
+ * @param consumer Function to process dead letter records
1959
+ * @param config Optional consumer configuration
1960
+ */
1961
+ addConsumer(consumer, config) {
1962
+ const withValidate = (deadLetter) => {
1963
+ attachTypeGuard(deadLetter, this.typeGuard);
1964
+ return consumer(deadLetter);
2268
1965
  };
1966
+ super.addConsumer(withValidate, config);
2269
1967
  }
2270
- });
2271
-
2272
- // src/dmv2/sdk/ingestApi.ts
2273
- var IngestApi;
2274
- var init_ingestApi = __esm({
2275
- "src/dmv2/sdk/ingestApi.ts"() {
2276
- "use strict";
2277
- init_typedBase();
2278
- init_internal();
2279
- IngestApi = class extends TypedBase {
2280
- constructor(name, config, schema, columns, validators, allowExtraFields) {
2281
- super(name, config, schema, columns, void 0, allowExtraFields);
2282
- const ingestApis = getMooseInternal().ingestApis;
2283
- if (ingestApis.has(name)) {
2284
- throw new Error(`Ingest API with name ${name} already exists`);
2285
- }
2286
- ingestApis.set(name, this);
2287
- }
1968
+ /**
1969
+ * Adds a multi-stream transformation for dead letter records.
1970
+ * The transformation function receives a DeadLetter<T> with type recovery capabilities.
1971
+ *
1972
+ * @param transformation Function to route dead letter records to multiple destinations
1973
+ */
1974
+ addMultiTransform(transformation) {
1975
+ const withValidate = (deadLetter) => {
1976
+ attachTypeGuard(deadLetter, this.typeGuard);
1977
+ return transformation(deadLetter);
2288
1978
  };
1979
+ super.addMultiTransform(withValidate);
2289
1980
  }
2290
- });
1981
+ };
2291
1982
 
2292
- // src/dmv2/sdk/consumptionApi.ts
2293
- var Api, ConsumptionApi;
2294
- var init_consumptionApi = __esm({
2295
- "src/dmv2/sdk/consumptionApi.ts"() {
2296
- "use strict";
2297
- init_typedBase();
2298
- init_internal();
2299
- Api = class extends TypedBase {
2300
- /** @internal The handler function that processes requests and generates responses. */
2301
- _handler;
2302
- /** @internal The JSON schema definition for the response type R. */
2303
- responseSchema;
2304
- constructor(name, handler, config, schema, columns, responseSchema) {
2305
- super(name, config ?? {}, schema, columns);
2306
- this._handler = handler;
2307
- this.responseSchema = responseSchema ?? {
2308
- version: "3.1",
2309
- schemas: [{ type: "array", items: { type: "object" } }],
2310
- components: { schemas: {} }
2311
- };
2312
- const apis = getMooseInternal().apis;
2313
- const key = `${name}${config?.version ? `:${config.version}` : ""}`;
2314
- if (apis.has(key)) {
2315
- throw new Error(
2316
- `Consumption API with name ${name} and version ${config?.version} already exists`
2317
- );
2318
- }
2319
- apis.set(key, this);
2320
- if (config?.path) {
2321
- if (config.version) {
2322
- const pathEndsWithVersion = config.path.endsWith(`/${config.version}`) || config.path === config.version || config.path.endsWith(config.version) && config.path.length > config.version.length && config.path[config.path.length - config.version.length - 1] === "/";
2323
- if (pathEndsWithVersion) {
2324
- if (apis.has(config.path)) {
2325
- const existing = apis.get(config.path);
2326
- throw new Error(
2327
- `Cannot register API "${name}" with path "${config.path}" - this path is already used by API "${existing.name}"`
2328
- );
2329
- }
2330
- apis.set(config.path, this);
2331
- } else {
2332
- const versionedPath = `${config.path.replace(/\/$/, "")}/${config.version}`;
2333
- if (apis.has(versionedPath)) {
2334
- const existing = apis.get(versionedPath);
2335
- throw new Error(
2336
- `Cannot register API "${name}" with path "${versionedPath}" - this path is already used by API "${existing.name}"`
2337
- );
2338
- }
2339
- apis.set(versionedPath, this);
2340
- if (!apis.has(config.path)) {
2341
- apis.set(config.path, this);
2342
- }
2343
- }
2344
- } else {
2345
- if (apis.has(config.path)) {
2346
- const existing = apis.get(config.path);
2347
- throw new Error(
2348
- `Cannot register API "${name}" with custom path "${config.path}" - this path is already used by API "${existing.name}"`
2349
- );
2350
- }
2351
- apis.set(config.path, this);
2352
- }
2353
- }
2354
- }
2355
- /**
2356
- * Retrieves the handler function associated with this Consumption API.
2357
- * @returns The handler function.
2358
- */
2359
- getHandler = () => {
2360
- return this._handler;
2361
- };
2362
- async call(baseUrl, queryParams) {
2363
- let path2;
2364
- if (this.config?.path) {
2365
- if (this.config.version) {
2366
- const pathEndsWithVersion = this.config.path.endsWith(`/${this.config.version}`) || this.config.path === this.config.version || this.config.path.endsWith(this.config.version) && this.config.path.length > this.config.version.length && this.config.path[this.config.path.length - this.config.version.length - 1] === "/";
2367
- if (pathEndsWithVersion) {
2368
- path2 = this.config.path;
2369
- } else {
2370
- path2 = `${this.config.path.replace(/\/$/, "")}/${this.config.version}`;
2371
- }
2372
- } else {
2373
- path2 = this.config.path;
2374
- }
2375
- } else {
2376
- path2 = this.config?.version ? `${this.name}/${this.config.version}` : this.name;
2377
- }
2378
- const url = new URL(`${baseUrl.replace(/\/$/, "")}/api/${path2}`);
2379
- const searchParams = url.searchParams;
2380
- for (const [key, value] of Object.entries(queryParams)) {
2381
- if (Array.isArray(value)) {
2382
- for (const item of value) {
2383
- if (item !== null && item !== void 0) {
2384
- searchParams.append(key, String(item));
2385
- }
2386
- }
2387
- } else if (value !== null && value !== void 0) {
2388
- searchParams.append(key, String(value));
2389
- }
2390
- }
2391
- const response = await fetch(url, {
2392
- method: "GET",
2393
- headers: {
2394
- Accept: "application/json"
2395
- }
2396
- });
2397
- if (!response.ok) {
2398
- throw new Error(`HTTP error! status: ${response.status}`);
1983
+ // src/dmv2/sdk/workflow.ts
1984
+ var Task = class {
1985
+ /**
1986
+ * Creates a new Task instance.
1987
+ *
1988
+ * @param name - Unique identifier for the task
1989
+ * @param config - Configuration object defining the task behavior
1990
+ *
1991
+ * @example
1992
+ * ```typescript
1993
+ * // No input, no output
1994
+ * const task1 = new Task<null, void>("task1", {
1995
+ * run: async () => {
1996
+ * console.log("No input/output");
1997
+ * }
1998
+ * });
1999
+ *
2000
+ * // No input, but has output
2001
+ * const task2 = new Task<null, OutputType>("task2", {
2002
+ * run: async () => {
2003
+ * return someOutput;
2004
+ * }
2005
+ * });
2006
+ *
2007
+ * // Has input, no output
2008
+ * const task3 = new Task<InputType, void>("task3", {
2009
+ * run: async (input: InputType) => {
2010
+ * // process input but return nothing
2011
+ * }
2012
+ * });
2013
+ *
2014
+ * // Has both input and output
2015
+ * const task4 = new Task<InputType, OutputType>("task4", {
2016
+ * run: async (input: InputType) => {
2017
+ * return process(input);
2018
+ * }
2019
+ * });
2020
+ * ```
2021
+ */
2022
+ constructor(name, config) {
2023
+ this.name = name;
2024
+ this.config = config;
2025
+ }
2026
+ };
2027
+ var Workflow = class {
2028
+ /**
2029
+ * Creates a new Workflow instance and registers it with the Moose system.
2030
+ *
2031
+ * @param name - Unique identifier for the workflow
2032
+ * @param config - Configuration object defining the workflow behavior and task orchestration
2033
+ * @throws {Error} When the workflow contains null/undefined tasks or infinite loops
2034
+ */
2035
+ constructor(name, config) {
2036
+ this.name = name;
2037
+ this.config = config;
2038
+ const workflows = getMooseInternal().workflows;
2039
+ if (workflows.has(name)) {
2040
+ throw new Error(`Workflow with name ${name} already exists`);
2041
+ }
2042
+ this.validateTaskGraph(config.startingTask, name);
2043
+ workflows.set(name, this);
2044
+ }
2045
+ /**
2046
+ * Validates the task graph to ensure there are no null tasks or infinite loops.
2047
+ *
2048
+ * @private
2049
+ * @param startingTask - The starting task to begin validation from
2050
+ * @param workflowName - The name of the workflow being validated (for error messages)
2051
+ * @throws {Error} When null/undefined tasks are found or infinite loops are detected
2052
+ */
2053
+ validateTaskGraph(startingTask, workflowName) {
2054
+ if (startingTask === null || startingTask === void 0) {
2055
+ throw new Error(
2056
+ `Workflow "${workflowName}" has a null or undefined starting task`
2057
+ );
2058
+ }
2059
+ const visited = /* @__PURE__ */ new Set();
2060
+ const recursionStack = /* @__PURE__ */ new Set();
2061
+ const validateTask = (task, currentPath) => {
2062
+ if (task === null || task === void 0) {
2063
+ const pathStr = currentPath.length > 0 ? currentPath.join(" -> ") + " -> " : "";
2064
+ throw new Error(
2065
+ `Workflow "${workflowName}" contains a null or undefined task in the task chain: ${pathStr}null`
2066
+ );
2067
+ }
2068
+ const taskName = task.name;
2069
+ if (recursionStack.has(taskName)) {
2070
+ const cycleStartIndex = currentPath.indexOf(taskName);
2071
+ const cyclePath = cycleStartIndex >= 0 ? currentPath.slice(cycleStartIndex).concat(taskName) : currentPath.concat(taskName);
2072
+ throw new Error(
2073
+ `Workflow "${workflowName}" contains an infinite loop in task chain: ${cyclePath.join(" -> ")}`
2074
+ );
2075
+ }
2076
+ if (visited.has(taskName)) {
2077
+ return;
2078
+ }
2079
+ visited.add(taskName);
2080
+ recursionStack.add(taskName);
2081
+ if (task.config.onComplete) {
2082
+ for (const nextTask of task.config.onComplete) {
2083
+ validateTask(nextTask, [...currentPath, taskName]);
2399
2084
  }
2400
- const data = await response.json();
2401
- return data;
2402
2085
  }
2086
+ recursionStack.delete(taskName);
2403
2087
  };
2404
- ConsumptionApi = Api;
2088
+ validateTask(startingTask, []);
2405
2089
  }
2406
- });
2090
+ };
2407
2091
 
2408
- // src/dmv2/sdk/ingestPipeline.ts
2409
- var IngestPipeline;
2410
- var init_ingestPipeline = __esm({
2411
- "src/dmv2/sdk/ingestPipeline.ts"() {
2412
- "use strict";
2413
- init_typedBase();
2414
- init_stream();
2415
- init_olapTable();
2416
- init_ingestApi();
2417
- init_types();
2418
- IngestPipeline = class extends TypedBase {
2419
- /**
2420
- * The OLAP table component of the pipeline, if configured.
2421
- * Provides analytical query capabilities for the ingested data.
2422
- * Only present when `config.table` is not `false`.
2423
- */
2424
- table;
2425
- /**
2426
- * The stream component of the pipeline, if configured.
2427
- * Handles real-time data flow and processing between components.
2428
- * Only present when `config.stream` is not `false`.
2429
- */
2430
- stream;
2431
- /**
2432
- * The ingest API component of the pipeline, if configured.
2433
- * Provides HTTP endpoints for data ingestion.
2434
- * Only present when `config.ingestApi` is not `false`.
2435
- */
2436
- ingestApi;
2437
- /** The dead letter queue of the pipeline, if configured. */
2438
- deadLetterQueue;
2439
- constructor(name, config, schema, columns, validators, allowExtraFields) {
2440
- super(name, config, schema, columns, validators, allowExtraFields);
2441
- if (config.ingest !== void 0) {
2442
- console.warn(
2443
- "\u26A0\uFE0F DEPRECATION WARNING: The 'ingest' parameter is deprecated and will be removed in a future version. Please use 'ingestApi' instead."
2444
- );
2445
- if (config.ingestApi === void 0) {
2446
- config.ingestApi = config.ingest;
2092
+ // src/dmv2/sdk/ingestApi.ts
2093
+ var IngestApi = class extends TypedBase {
2094
+ constructor(name, config, schema, columns, validators, allowExtraFields) {
2095
+ super(name, config, schema, columns, void 0, allowExtraFields);
2096
+ const ingestApis = getMooseInternal().ingestApis;
2097
+ if (ingestApis.has(name)) {
2098
+ throw new Error(`Ingest API with name ${name} already exists`);
2099
+ }
2100
+ ingestApis.set(name, this);
2101
+ }
2102
+ };
2103
+
2104
+ // src/dmv2/sdk/consumptionApi.ts
2105
+ var Api = class extends TypedBase {
2106
+ /** @internal The handler function that processes requests and generates responses. */
2107
+ _handler;
2108
+ /** @internal The JSON schema definition for the response type R. */
2109
+ responseSchema;
2110
+ constructor(name, handler, config, schema, columns, responseSchema) {
2111
+ super(name, config ?? {}, schema, columns);
2112
+ this._handler = handler;
2113
+ this.responseSchema = responseSchema ?? {
2114
+ version: "3.1",
2115
+ schemas: [{ type: "array", items: { type: "object" } }],
2116
+ components: { schemas: {} }
2117
+ };
2118
+ const apis = getMooseInternal().apis;
2119
+ const key = `${name}${config?.version ? `:${config.version}` : ""}`;
2120
+ if (apis.has(key)) {
2121
+ throw new Error(
2122
+ `Consumption API with name ${name} and version ${config?.version} already exists`
2123
+ );
2124
+ }
2125
+ apis.set(key, this);
2126
+ if (config?.path) {
2127
+ if (config.version) {
2128
+ const pathEndsWithVersion = config.path.endsWith(`/${config.version}`) || config.path === config.version || config.path.endsWith(config.version) && config.path.length > config.version.length && config.path[config.path.length - config.version.length - 1] === "/";
2129
+ if (pathEndsWithVersion) {
2130
+ if (apis.has(config.path)) {
2131
+ const existing = apis.get(config.path);
2132
+ throw new Error(
2133
+ `Cannot register API "${name}" with path "${config.path}" - this path is already used by API "${existing.name}"`
2134
+ );
2135
+ }
2136
+ apis.set(config.path, this);
2137
+ } else {
2138
+ const versionedPath = `${config.path.replace(/\/$/, "")}/${config.version}`;
2139
+ if (apis.has(versionedPath)) {
2140
+ const existing = apis.get(versionedPath);
2141
+ throw new Error(
2142
+ `Cannot register API "${name}" with path "${versionedPath}" - this path is already used by API "${existing.name}"`
2143
+ );
2144
+ }
2145
+ apis.set(versionedPath, this);
2146
+ if (!apis.has(config.path)) {
2147
+ apis.set(config.path, this);
2447
2148
  }
2448
2149
  }
2449
- if (config.table) {
2450
- const tableConfig = typeof config.table === "object" ? {
2451
- ...config.table,
2452
- lifeCycle: config.table.lifeCycle ?? config.lifeCycle,
2453
- ...config.version && { version: config.version }
2454
- } : {
2455
- lifeCycle: config.lifeCycle,
2456
- engine: "MergeTree" /* MergeTree */,
2457
- ...config.version && { version: config.version }
2458
- };
2459
- this.table = new OlapTable(
2460
- name,
2461
- tableConfig,
2462
- this.schema,
2463
- this.columnArray,
2464
- this.validators
2465
- );
2466
- }
2467
- if (config.deadLetterQueue) {
2468
- const streamConfig = {
2469
- destination: void 0,
2470
- ...typeof config.deadLetterQueue === "object" ? {
2471
- ...config.deadLetterQueue,
2472
- lifeCycle: config.deadLetterQueue.lifeCycle ?? config.lifeCycle
2473
- } : { lifeCycle: config.lifeCycle },
2474
- ...config.version && { version: config.version }
2475
- };
2476
- this.deadLetterQueue = new DeadLetterQueue(
2477
- `${name}DeadLetterQueue`,
2478
- streamConfig,
2479
- validators.assert
2150
+ } else {
2151
+ if (apis.has(config.path)) {
2152
+ const existing = apis.get(config.path);
2153
+ throw new Error(
2154
+ `Cannot register API "${name}" with custom path "${config.path}" - this path is already used by API "${existing.name}"`
2480
2155
  );
2481
2156
  }
2482
- if (config.stream) {
2483
- const streamConfig = {
2484
- destination: this.table,
2485
- defaultDeadLetterQueue: this.deadLetterQueue,
2486
- ...typeof config.stream === "object" ? {
2487
- ...config.stream,
2488
- lifeCycle: config.stream.lifeCycle ?? config.lifeCycle
2489
- } : { lifeCycle: config.lifeCycle },
2490
- ...config.version && { version: config.version }
2491
- };
2492
- this.stream = new Stream(
2493
- name,
2494
- streamConfig,
2495
- this.schema,
2496
- this.columnArray,
2497
- void 0,
2498
- this.allowExtraFields
2499
- );
2500
- this.stream.pipelineParent = this;
2157
+ apis.set(config.path, this);
2158
+ }
2159
+ }
2160
+ }
2161
+ /**
2162
+ * Retrieves the handler function associated with this Consumption API.
2163
+ * @returns The handler function.
2164
+ */
2165
+ getHandler = () => {
2166
+ return this._handler;
2167
+ };
2168
+ async call(baseUrl, queryParams) {
2169
+ let path2;
2170
+ if (this.config?.path) {
2171
+ if (this.config.version) {
2172
+ const pathEndsWithVersion = this.config.path.endsWith(`/${this.config.version}`) || this.config.path === this.config.version || this.config.path.endsWith(this.config.version) && this.config.path.length > this.config.version.length && this.config.path[this.config.path.length - this.config.version.length - 1] === "/";
2173
+ if (pathEndsWithVersion) {
2174
+ path2 = this.config.path;
2175
+ } else {
2176
+ path2 = `${this.config.path.replace(/\/$/, "")}/${this.config.version}`;
2501
2177
  }
2502
- const effectiveIngestAPI = config.ingestApi !== void 0 ? config.ingestApi : config.ingest;
2503
- if (effectiveIngestAPI) {
2504
- if (!this.stream) {
2505
- throw new Error("Ingest API needs a stream to write to.");
2178
+ } else {
2179
+ path2 = this.config.path;
2180
+ }
2181
+ } else {
2182
+ path2 = this.config?.version ? `${this.name}/${this.config.version}` : this.name;
2183
+ }
2184
+ const url = new URL(`${baseUrl.replace(/\/$/, "")}/api/${path2}`);
2185
+ const searchParams = url.searchParams;
2186
+ for (const [key, value] of Object.entries(queryParams)) {
2187
+ if (Array.isArray(value)) {
2188
+ for (const item of value) {
2189
+ if (item !== null && item !== void 0) {
2190
+ searchParams.append(key, String(item));
2506
2191
  }
2507
- const ingestConfig = {
2508
- destination: this.stream,
2509
- deadLetterQueue: this.deadLetterQueue,
2510
- ...typeof effectiveIngestAPI === "object" ? effectiveIngestAPI : {},
2511
- ...config.version && { version: config.version },
2512
- ...config.path && { path: config.path }
2513
- };
2514
- this.ingestApi = new IngestApi(
2515
- name,
2516
- ingestConfig,
2517
- this.schema,
2518
- this.columnArray,
2519
- void 0,
2520
- this.allowExtraFields
2521
- );
2522
- this.ingestApi.pipelineParent = this;
2523
2192
  }
2193
+ } else if (value !== null && value !== void 0) {
2194
+ searchParams.append(key, String(value));
2524
2195
  }
2525
- };
2196
+ }
2197
+ const response = await fetch(url, {
2198
+ method: "GET",
2199
+ headers: {
2200
+ Accept: "application/json"
2201
+ }
2202
+ });
2203
+ if (!response.ok) {
2204
+ throw new Error(`HTTP error! status: ${response.status}`);
2205
+ }
2206
+ const data = await response.json();
2207
+ return data;
2526
2208
  }
2527
- });
2209
+ };
2210
+ var ConsumptionApi = Api;
2528
2211
 
2529
- // src/dmv2/sdk/etlPipeline.ts
2530
- var InternalBatcher, ETLPipeline;
2531
- var init_etlPipeline = __esm({
2532
- "src/dmv2/sdk/etlPipeline.ts"() {
2533
- "use strict";
2534
- init_workflow();
2535
- InternalBatcher = class {
2536
- iterator;
2537
- batchSize;
2538
- constructor(asyncIterable, batchSize = 20) {
2539
- this.iterator = asyncIterable[Symbol.asyncIterator]();
2540
- this.batchSize = batchSize;
2541
- }
2542
- async getNextBatch() {
2543
- const items = [];
2544
- for (let i = 0; i < this.batchSize; i++) {
2545
- const { value, done } = await this.iterator.next();
2546
- if (done) {
2547
- return { items, hasMore: false };
2548
- }
2549
- items.push(value);
2550
- }
2551
- return { items, hasMore: true };
2212
+ // src/dmv2/sdk/ingestPipeline.ts
2213
+ var IngestPipeline = class extends TypedBase {
2214
+ /**
2215
+ * The OLAP table component of the pipeline, if configured.
2216
+ * Provides analytical query capabilities for the ingested data.
2217
+ * Only present when `config.table` is not `false`.
2218
+ */
2219
+ table;
2220
+ /**
2221
+ * The stream component of the pipeline, if configured.
2222
+ * Handles real-time data flow and processing between components.
2223
+ * Only present when `config.stream` is not `false`.
2224
+ */
2225
+ stream;
2226
+ /**
2227
+ * The ingest API component of the pipeline, if configured.
2228
+ * Provides HTTP endpoints for data ingestion.
2229
+ * Only present when `config.ingestApi` is not `false`.
2230
+ */
2231
+ ingestApi;
2232
+ /** The dead letter queue of the pipeline, if configured. */
2233
+ deadLetterQueue;
2234
+ constructor(name, config, schema, columns, validators, allowExtraFields) {
2235
+ super(name, config, schema, columns, validators, allowExtraFields);
2236
+ if (config.ingest !== void 0) {
2237
+ console.warn(
2238
+ "\u26A0\uFE0F DEPRECATION WARNING: The 'ingest' parameter is deprecated and will be removed in a future version. Please use 'ingestApi' instead."
2239
+ );
2240
+ if (config.ingestApi === void 0) {
2241
+ config.ingestApi = config.ingest;
2552
2242
  }
2243
+ }
2244
+ if (config.table) {
2245
+ const tableConfig = typeof config.table === "object" ? {
2246
+ ...config.table,
2247
+ lifeCycle: config.table.lifeCycle ?? config.lifeCycle,
2248
+ ...config.version && { version: config.version }
2249
+ } : {
2250
+ lifeCycle: config.lifeCycle,
2251
+ engine: "MergeTree" /* MergeTree */,
2252
+ ...config.version && { version: config.version }
2253
+ };
2254
+ this.table = new OlapTable(
2255
+ name,
2256
+ tableConfig,
2257
+ this.schema,
2258
+ this.columnArray,
2259
+ this.validators
2260
+ );
2261
+ }
2262
+ if (config.deadLetterQueue) {
2263
+ const streamConfig = {
2264
+ destination: void 0,
2265
+ ...typeof config.deadLetterQueue === "object" ? {
2266
+ ...config.deadLetterQueue,
2267
+ lifeCycle: config.deadLetterQueue.lifeCycle ?? config.lifeCycle
2268
+ } : { lifeCycle: config.lifeCycle },
2269
+ ...config.version && { version: config.version }
2270
+ };
2271
+ this.deadLetterQueue = new DeadLetterQueue(
2272
+ `${name}DeadLetterQueue`,
2273
+ streamConfig,
2274
+ validators.assert
2275
+ );
2276
+ }
2277
+ if (config.stream) {
2278
+ const streamConfig = {
2279
+ destination: this.table,
2280
+ defaultDeadLetterQueue: this.deadLetterQueue,
2281
+ ...typeof config.stream === "object" ? {
2282
+ ...config.stream,
2283
+ lifeCycle: config.stream.lifeCycle ?? config.lifeCycle
2284
+ } : { lifeCycle: config.lifeCycle },
2285
+ ...config.version && { version: config.version }
2286
+ };
2287
+ this.stream = new Stream(
2288
+ name,
2289
+ streamConfig,
2290
+ this.schema,
2291
+ this.columnArray,
2292
+ void 0,
2293
+ this.allowExtraFields
2294
+ );
2295
+ this.stream.pipelineParent = this;
2296
+ }
2297
+ const effectiveIngestAPI = config.ingestApi !== void 0 ? config.ingestApi : config.ingest;
2298
+ if (effectiveIngestAPI) {
2299
+ if (!this.stream) {
2300
+ throw new Error("Ingest API needs a stream to write to.");
2301
+ }
2302
+ const ingestConfig = {
2303
+ destination: this.stream,
2304
+ deadLetterQueue: this.deadLetterQueue,
2305
+ ...typeof effectiveIngestAPI === "object" ? effectiveIngestAPI : {},
2306
+ ...config.version && { version: config.version },
2307
+ ...config.path && { path: config.path }
2308
+ };
2309
+ this.ingestApi = new IngestApi(
2310
+ name,
2311
+ ingestConfig,
2312
+ this.schema,
2313
+ this.columnArray,
2314
+ void 0,
2315
+ this.allowExtraFields
2316
+ );
2317
+ this.ingestApi.pipelineParent = this;
2318
+ }
2319
+ }
2320
+ };
2321
+
2322
+ // src/dmv2/sdk/etlPipeline.ts
2323
+ var InternalBatcher = class {
2324
+ iterator;
2325
+ batchSize;
2326
+ constructor(asyncIterable, batchSize = 20) {
2327
+ this.iterator = asyncIterable[Symbol.asyncIterator]();
2328
+ this.batchSize = batchSize;
2329
+ }
2330
+ async getNextBatch() {
2331
+ const items = [];
2332
+ for (let i = 0; i < this.batchSize; i++) {
2333
+ const { value, done } = await this.iterator.next();
2334
+ if (done) {
2335
+ return { items, hasMore: false };
2336
+ }
2337
+ items.push(value);
2338
+ }
2339
+ return { items, hasMore: true };
2340
+ }
2341
+ };
2342
+ var ETLPipeline = class {
2343
+ constructor(name, config) {
2344
+ this.name = name;
2345
+ this.config = config;
2346
+ this.setupPipeline();
2347
+ }
2348
+ batcher;
2349
+ setupPipeline() {
2350
+ this.batcher = this.createBatcher();
2351
+ const tasks = this.createAllTasks();
2352
+ tasks.extract.config.onComplete = [tasks.transform];
2353
+ tasks.transform.config.onComplete = [tasks.load];
2354
+ new Workflow(this.name, {
2355
+ startingTask: tasks.extract,
2356
+ retries: 1,
2357
+ timeout: "30m"
2358
+ });
2359
+ }
2360
+ createBatcher() {
2361
+ const iterable = typeof this.config.extract === "function" ? this.config.extract() : this.config.extract;
2362
+ return new InternalBatcher(iterable);
2363
+ }
2364
+ getDefaultTaskConfig() {
2365
+ return {
2366
+ retries: 1,
2367
+ timeout: "30m"
2553
2368
  };
2554
- ETLPipeline = class {
2555
- constructor(name, config) {
2556
- this.name = name;
2557
- this.config = config;
2558
- this.setupPipeline();
2559
- }
2560
- batcher;
2561
- setupPipeline() {
2562
- this.batcher = this.createBatcher();
2563
- const tasks = this.createAllTasks();
2564
- tasks.extract.config.onComplete = [tasks.transform];
2565
- tasks.transform.config.onComplete = [tasks.load];
2566
- new Workflow(this.name, {
2567
- startingTask: tasks.extract,
2568
- retries: 1,
2569
- timeout: "30m"
2570
- });
2571
- }
2572
- createBatcher() {
2573
- const iterable = typeof this.config.extract === "function" ? this.config.extract() : this.config.extract;
2574
- return new InternalBatcher(iterable);
2575
- }
2576
- getDefaultTaskConfig() {
2577
- return {
2578
- retries: 1,
2579
- timeout: "30m"
2580
- };
2581
- }
2582
- createAllTasks() {
2583
- const taskConfig = this.getDefaultTaskConfig();
2584
- return {
2585
- extract: this.createExtractTask(taskConfig),
2586
- transform: this.createTransformTask(taskConfig),
2587
- load: this.createLoadTask(taskConfig)
2588
- };
2589
- }
2590
- createExtractTask(taskConfig) {
2591
- return new Task(`${this.name}_extract`, {
2592
- run: async ({}) => {
2593
- console.log(`Running extract task for ${this.name}...`);
2594
- const batch = await this.batcher.getNextBatch();
2595
- console.log(`Extract task completed with ${batch.items.length} items`);
2596
- return batch;
2597
- },
2598
- retries: taskConfig.retries,
2599
- timeout: taskConfig.timeout
2600
- });
2601
- }
2602
- createTransformTask(taskConfig) {
2603
- return new Task(
2604
- `${this.name}_transform`,
2605
- {
2606
- // Use new single-parameter context API for handlers
2607
- run: async ({ input }) => {
2608
- const batch = input;
2609
- console.log(
2610
- `Running transform task for ${this.name} with ${batch.items.length} items...`
2611
- );
2612
- const transformedItems = [];
2613
- for (const item of batch.items) {
2614
- const transformed = await this.config.transform(item);
2615
- transformedItems.push(transformed);
2616
- }
2617
- console.log(
2618
- `Transform task completed with ${transformedItems.length} items`
2619
- );
2620
- return { items: transformedItems };
2621
- },
2622
- retries: taskConfig.retries,
2623
- timeout: taskConfig.timeout
2624
- }
2625
- );
2626
- }
2627
- createLoadTask(taskConfig) {
2628
- return new Task(`${this.name}_load`, {
2629
- run: async ({ input: transformedItems }) => {
2630
- console.log(
2631
- `Running load task for ${this.name} with ${transformedItems.items.length} items...`
2632
- );
2633
- if ("insert" in this.config.load) {
2634
- await this.config.load.insert(transformedItems.items);
2635
- } else {
2636
- await this.config.load(transformedItems.items);
2637
- }
2638
- console.log(`Load task completed`);
2639
- },
2640
- retries: taskConfig.retries,
2641
- timeout: taskConfig.timeout
2642
- });
2643
- }
2644
- // Execute the entire ETL pipeline
2645
- async run() {
2646
- console.log(`Starting ETL Pipeline: ${this.name}`);
2647
- let batchNumber = 1;
2648
- do {
2649
- console.log(`Processing batch ${batchNumber}...`);
2650
- const batch = await this.batcher.getNextBatch();
2651
- if (batch.items.length === 0) {
2652
- break;
2653
- }
2369
+ }
2370
+ createAllTasks() {
2371
+ const taskConfig = this.getDefaultTaskConfig();
2372
+ return {
2373
+ extract: this.createExtractTask(taskConfig),
2374
+ transform: this.createTransformTask(taskConfig),
2375
+ load: this.createLoadTask(taskConfig)
2376
+ };
2377
+ }
2378
+ createExtractTask(taskConfig) {
2379
+ return new Task(`${this.name}_extract`, {
2380
+ run: async ({}) => {
2381
+ console.log(`Running extract task for ${this.name}...`);
2382
+ const batch = await this.batcher.getNextBatch();
2383
+ console.log(`Extract task completed with ${batch.items.length} items`);
2384
+ return batch;
2385
+ },
2386
+ retries: taskConfig.retries,
2387
+ timeout: taskConfig.timeout
2388
+ });
2389
+ }
2390
+ createTransformTask(taskConfig) {
2391
+ return new Task(
2392
+ `${this.name}_transform`,
2393
+ {
2394
+ // Use new single-parameter context API for handlers
2395
+ run: async ({ input }) => {
2396
+ const batch = input;
2397
+ console.log(
2398
+ `Running transform task for ${this.name} with ${batch.items.length} items...`
2399
+ );
2654
2400
  const transformedItems = [];
2655
- for (const extractedData of batch.items) {
2656
- const transformedData = await this.config.transform(extractedData);
2657
- transformedItems.push(transformedData);
2658
- }
2659
- if ("insert" in this.config.load) {
2660
- await this.config.load.insert(transformedItems);
2661
- } else {
2662
- await this.config.load(transformedItems);
2401
+ for (const item of batch.items) {
2402
+ const transformed = await this.config.transform(item);
2403
+ transformedItems.push(transformed);
2663
2404
  }
2664
2405
  console.log(
2665
- `Completed batch ${batchNumber} with ${batch.items.length} items`
2406
+ `Transform task completed with ${transformedItems.length} items`
2666
2407
  );
2667
- batchNumber++;
2668
- if (!batch.hasMore) {
2669
- break;
2670
- }
2671
- } while (true);
2672
- console.log(`Completed ETL Pipeline: ${this.name}`);
2408
+ return { items: transformedItems };
2409
+ },
2410
+ retries: taskConfig.retries,
2411
+ timeout: taskConfig.timeout
2673
2412
  }
2674
- };
2413
+ );
2675
2414
  }
2676
- });
2415
+ createLoadTask(taskConfig) {
2416
+ return new Task(`${this.name}_load`, {
2417
+ run: async ({ input: transformedItems }) => {
2418
+ console.log(
2419
+ `Running load task for ${this.name} with ${transformedItems.items.length} items...`
2420
+ );
2421
+ if ("insert" in this.config.load) {
2422
+ await this.config.load.insert(transformedItems.items);
2423
+ } else {
2424
+ await this.config.load(transformedItems.items);
2425
+ }
2426
+ console.log(`Load task completed`);
2427
+ },
2428
+ retries: taskConfig.retries,
2429
+ timeout: taskConfig.timeout
2430
+ });
2431
+ }
2432
+ // Execute the entire ETL pipeline
2433
+ async run() {
2434
+ console.log(`Starting ETL Pipeline: ${this.name}`);
2435
+ let batchNumber = 1;
2436
+ do {
2437
+ console.log(`Processing batch ${batchNumber}...`);
2438
+ const batch = await this.batcher.getNextBatch();
2439
+ if (batch.items.length === 0) {
2440
+ break;
2441
+ }
2442
+ const transformedItems = [];
2443
+ for (const extractedData of batch.items) {
2444
+ const transformedData = await this.config.transform(extractedData);
2445
+ transformedItems.push(transformedData);
2446
+ }
2447
+ if ("insert" in this.config.load) {
2448
+ await this.config.load.insert(transformedItems);
2449
+ } else {
2450
+ await this.config.load(transformedItems);
2451
+ }
2452
+ console.log(
2453
+ `Completed batch ${batchNumber} with ${batch.items.length} items`
2454
+ );
2455
+ batchNumber++;
2456
+ if (!batch.hasMore) {
2457
+ break;
2458
+ }
2459
+ } while (true);
2460
+ console.log(`Completed ETL Pipeline: ${this.name}`);
2461
+ }
2462
+ };
2677
2463
 
2678
2464
  // src/dmv2/sdk/materializedView.ts
2679
2465
  function formatTableReference(table) {
@@ -2683,147 +2469,127 @@ function formatTableReference(table) {
2683
2469
  }
2684
2470
  return `\`${table.name}\``;
2685
2471
  }
2686
- var requireTargetTableName, MaterializedView;
2687
- var init_materializedView = __esm({
2688
- "src/dmv2/sdk/materializedView.ts"() {
2689
- "use strict";
2690
- init_types();
2691
- init_sqlHelpers();
2692
- init_olapTable();
2693
- init_internal();
2694
- init_stackTrace();
2695
- requireTargetTableName = (tableName) => {
2696
- if (typeof tableName === "string") {
2697
- return tableName;
2698
- } else {
2699
- throw new Error("Name of targetTable is not specified.");
2700
- }
2701
- };
2702
- MaterializedView = class {
2703
- /** @internal */
2704
- kind = "MaterializedView";
2705
- /** The name of the materialized view */
2706
- name;
2707
- /** The target OlapTable instance where the materialized data is stored. */
2708
- targetTable;
2709
- /** The SELECT SQL statement */
2710
- selectSql;
2711
- /** Names of source tables that the SELECT reads from */
2712
- sourceTables;
2713
- /** Optional metadata for the materialized view */
2714
- metadata;
2715
- constructor(options, targetSchema, targetColumns) {
2716
- let selectStatement = options.selectStatement;
2717
- if (typeof selectStatement !== "string") {
2718
- selectStatement = toStaticQuery(selectStatement);
2719
- }
2720
- if (targetSchema === void 0 || targetColumns === void 0) {
2721
- throw new Error(
2722
- "Supply the type param T so that the schema is inserted by the compiler plugin."
2723
- );
2724
- }
2725
- const targetTable = options.targetTable instanceof OlapTable ? options.targetTable : new OlapTable(
2726
- requireTargetTableName(
2727
- options.targetTable?.name ?? options.tableName
2728
- ),
2729
- {
2730
- orderByFields: options.targetTable?.orderByFields ?? options.orderByFields,
2731
- engine: options.targetTable?.engine ?? options.engine ?? "MergeTree" /* MergeTree */
2732
- },
2733
- targetSchema,
2734
- targetColumns
2735
- );
2736
- if (targetTable.name === options.materializedViewName) {
2737
- throw new Error(
2738
- "Materialized view name cannot be the same as the target table name."
2739
- );
2740
- }
2741
- this.name = options.materializedViewName;
2742
- this.targetTable = targetTable;
2743
- this.selectSql = selectStatement;
2744
- this.sourceTables = options.selectTables.map(
2745
- (t) => formatTableReference(t)
2746
- );
2747
- this.metadata = options.metadata ? { ...options.metadata } : {};
2748
- if (!this.metadata.source) {
2749
- const stack = new Error().stack;
2750
- const sourceInfo = getSourceFileFromStack(stack);
2751
- if (sourceInfo) {
2752
- this.metadata.source = { file: sourceInfo };
2753
- }
2754
- }
2755
- const materializedViews = getMooseInternal().materializedViews;
2756
- if (!isClientOnlyMode() && materializedViews.has(this.name)) {
2757
- throw new Error(`MaterializedView with name ${this.name} already exists`);
2758
- }
2759
- materializedViews.set(this.name, this);
2472
+ var requireTargetTableName = (tableName) => {
2473
+ if (typeof tableName === "string") {
2474
+ return tableName;
2475
+ } else {
2476
+ throw new Error("Name of targetTable is not specified.");
2477
+ }
2478
+ };
2479
+ var MaterializedView = class {
2480
+ /** @internal */
2481
+ kind = "MaterializedView";
2482
+ /** The name of the materialized view */
2483
+ name;
2484
+ /** The target OlapTable instance where the materialized data is stored. */
2485
+ targetTable;
2486
+ /** The SELECT SQL statement */
2487
+ selectSql;
2488
+ /** Names of source tables that the SELECT reads from */
2489
+ sourceTables;
2490
+ /** Optional metadata for the materialized view */
2491
+ metadata;
2492
+ constructor(options, targetSchema, targetColumns) {
2493
+ let selectStatement = options.selectStatement;
2494
+ if (typeof selectStatement !== "string") {
2495
+ selectStatement = toStaticQuery(selectStatement);
2496
+ }
2497
+ if (targetSchema === void 0 || targetColumns === void 0) {
2498
+ throw new Error(
2499
+ "Supply the type param T so that the schema is inserted by the compiler plugin."
2500
+ );
2501
+ }
2502
+ const targetTable = options.targetTable instanceof OlapTable ? options.targetTable : new OlapTable(
2503
+ requireTargetTableName(
2504
+ options.targetTable?.name ?? options.tableName
2505
+ ),
2506
+ {
2507
+ orderByFields: options.targetTable?.orderByFields ?? options.orderByFields,
2508
+ engine: options.targetTable?.engine ?? options.engine ?? "MergeTree" /* MergeTree */
2509
+ },
2510
+ targetSchema,
2511
+ targetColumns
2512
+ );
2513
+ if (targetTable.name === options.materializedViewName) {
2514
+ throw new Error(
2515
+ "Materialized view name cannot be the same as the target table name."
2516
+ );
2517
+ }
2518
+ this.name = options.materializedViewName;
2519
+ this.targetTable = targetTable;
2520
+ this.selectSql = selectStatement;
2521
+ this.sourceTables = options.selectTables.map(
2522
+ (t) => formatTableReference(t)
2523
+ );
2524
+ this.metadata = options.metadata ? { ...options.metadata } : {};
2525
+ if (!this.metadata.source) {
2526
+ const stack = new Error().stack;
2527
+ const sourceInfo = getSourceFileFromStack(stack);
2528
+ if (sourceInfo) {
2529
+ this.metadata.source = { file: sourceInfo };
2760
2530
  }
2761
- };
2531
+ }
2532
+ const materializedViews = getMooseInternal().materializedViews;
2533
+ if (!isClientOnlyMode() && materializedViews.has(this.name)) {
2534
+ throw new Error(`MaterializedView with name ${this.name} already exists`);
2535
+ }
2536
+ materializedViews.set(this.name, this);
2762
2537
  }
2763
- });
2538
+ };
2764
2539
 
2765
2540
  // src/dmv2/sdk/sqlResource.ts
2766
- var SqlResource;
2767
- var init_sqlResource = __esm({
2768
- "src/dmv2/sdk/sqlResource.ts"() {
2769
- "use strict";
2770
- init_internal();
2771
- init_sqlHelpers();
2772
- init_stackTrace();
2773
- SqlResource = class {
2774
- /** @internal */
2775
- kind = "SqlResource";
2776
- /** Array of SQL statements to execute for setting up the resource. */
2777
- setup;
2778
- /** Array of SQL statements to execute for tearing down the resource. */
2779
- teardown;
2780
- /** The name of the SQL resource (e.g., view name, materialized view name). */
2781
- name;
2782
- /** List of OlapTables or Views that this resource reads data from. */
2783
- pullsDataFrom;
2784
- /** List of OlapTables or Views that this resource writes data to. */
2785
- pushesDataTo;
2786
- /** @internal Source file path where this resource was defined */
2787
- sourceFile;
2788
- /** @internal Source line number where this resource was defined */
2789
- sourceLine;
2790
- /** @internal Source column number where this resource was defined */
2791
- sourceColumn;
2792
- /**
2793
- * Creates a new SqlResource instance.
2794
- * @param name The name of the resource.
2795
- * @param setup An array of SQL DDL statements to create the resource.
2796
- * @param teardown An array of SQL DDL statements to drop the resource.
2797
- * @param options Optional configuration for specifying data dependencies.
2798
- * @param options.pullsDataFrom Tables/Views this resource reads from.
2799
- * @param options.pushesDataTo Tables/Views this resource writes to.
2800
- */
2801
- constructor(name, setup, teardown, options) {
2802
- const sqlResources = getMooseInternal().sqlResources;
2803
- if (!isClientOnlyMode() && sqlResources.has(name)) {
2804
- throw new Error(`SqlResource with name ${name} already exists`);
2805
- }
2806
- sqlResources.set(name, this);
2807
- this.name = name;
2808
- this.setup = setup.map(
2809
- (sql3) => typeof sql3 === "string" ? sql3 : toStaticQuery(sql3)
2810
- );
2811
- this.teardown = teardown.map(
2812
- (sql3) => typeof sql3 === "string" ? sql3 : toStaticQuery(sql3)
2813
- );
2814
- this.pullsDataFrom = options?.pullsDataFrom ?? [];
2815
- this.pushesDataTo = options?.pushesDataTo ?? [];
2816
- const stack = new Error().stack;
2817
- const location = getSourceLocationFromStack(stack);
2818
- if (location) {
2819
- this.sourceFile = location.file;
2820
- this.sourceLine = location.line;
2821
- this.sourceColumn = location.column;
2822
- }
2823
- }
2824
- };
2541
+ var SqlResource = class {
2542
+ /** @internal */
2543
+ kind = "SqlResource";
2544
+ /** Array of SQL statements to execute for setting up the resource. */
2545
+ setup;
2546
+ /** Array of SQL statements to execute for tearing down the resource. */
2547
+ teardown;
2548
+ /** The name of the SQL resource (e.g., view name, materialized view name). */
2549
+ name;
2550
+ /** List of OlapTables or Views that this resource reads data from. */
2551
+ pullsDataFrom;
2552
+ /** List of OlapTables or Views that this resource writes data to. */
2553
+ pushesDataTo;
2554
+ /** @internal Source file path where this resource was defined */
2555
+ sourceFile;
2556
+ /** @internal Source line number where this resource was defined */
2557
+ sourceLine;
2558
+ /** @internal Source column number where this resource was defined */
2559
+ sourceColumn;
2560
+ /**
2561
+ * Creates a new SqlResource instance.
2562
+ * @param name The name of the resource.
2563
+ * @param setup An array of SQL DDL statements to create the resource.
2564
+ * @param teardown An array of SQL DDL statements to drop the resource.
2565
+ * @param options Optional configuration for specifying data dependencies.
2566
+ * @param options.pullsDataFrom Tables/Views this resource reads from.
2567
+ * @param options.pushesDataTo Tables/Views this resource writes to.
2568
+ */
2569
+ constructor(name, setup, teardown, options) {
2570
+ const sqlResources = getMooseInternal().sqlResources;
2571
+ if (!isClientOnlyMode() && sqlResources.has(name)) {
2572
+ throw new Error(`SqlResource with name ${name} already exists`);
2573
+ }
2574
+ sqlResources.set(name, this);
2575
+ this.name = name;
2576
+ this.setup = setup.map(
2577
+ (sql3) => typeof sql3 === "string" ? sql3 : toStaticQuery(sql3)
2578
+ );
2579
+ this.teardown = teardown.map(
2580
+ (sql3) => typeof sql3 === "string" ? sql3 : toStaticQuery(sql3)
2581
+ );
2582
+ this.pullsDataFrom = options?.pullsDataFrom ?? [];
2583
+ this.pushesDataTo = options?.pushesDataTo ?? [];
2584
+ const stack = new Error().stack;
2585
+ const location = getSourceLocationFromStack(stack);
2586
+ if (location) {
2587
+ this.sourceFile = location.file;
2588
+ this.sourceLine = location.line;
2589
+ this.sourceColumn = location.column;
2590
+ }
2825
2591
  }
2826
- });
2592
+ };
2827
2593
 
2828
2594
  // src/dmv2/sdk/view.ts
2829
2595
  function formatTableReference2(table) {
@@ -2833,171 +2599,150 @@ function formatTableReference2(table) {
2833
2599
  }
2834
2600
  return `\`${table.name}\``;
2835
2601
  }
2836
- var View;
2837
- var init_view = __esm({
2838
- "src/dmv2/sdk/view.ts"() {
2839
- "use strict";
2840
- init_sqlHelpers();
2841
- init_olapTable();
2842
- init_internal();
2843
- init_stackTrace();
2844
- View = class {
2845
- /** @internal */
2846
- kind = "View";
2847
- /** The name of the view */
2848
- name;
2849
- /** The SELECT SQL statement that defines the view */
2850
- selectSql;
2851
- /** Names of source tables/views that the SELECT reads from */
2852
- sourceTables;
2853
- /** Optional metadata for the view */
2854
- metadata;
2855
- /**
2856
- * Creates a new View instance.
2857
- * @param name The name of the view to be created.
2858
- * @param selectStatement The SQL SELECT statement that defines the view's logic.
2859
- * @param baseTables An array of OlapTable or View objects that the `selectStatement` reads from. Used for dependency tracking.
2860
- * @param metadata Optional metadata for the view (e.g., description, source file).
2861
- */
2862
- constructor(name, selectStatement, baseTables, metadata) {
2863
- if (typeof selectStatement !== "string") {
2864
- selectStatement = toStaticQuery(selectStatement);
2865
- }
2866
- this.name = name;
2867
- this.selectSql = selectStatement;
2868
- this.sourceTables = baseTables.map((t) => formatTableReference2(t));
2869
- this.metadata = metadata ? { ...metadata } : {};
2870
- if (!this.metadata.source) {
2871
- const stack = new Error().stack;
2872
- const sourceInfo = getSourceFileFromStack(stack);
2873
- if (sourceInfo) {
2874
- this.metadata.source = { file: sourceInfo };
2875
- }
2876
- }
2877
- const views = getMooseInternal().views;
2878
- if (!isClientOnlyMode() && views.has(this.name)) {
2879
- throw new Error(`View with name ${this.name} already exists`);
2880
- }
2881
- views.set(this.name, this);
2602
+ var View = class {
2603
+ /** @internal */
2604
+ kind = "View";
2605
+ /** The name of the view */
2606
+ name;
2607
+ /** The SELECT SQL statement that defines the view */
2608
+ selectSql;
2609
+ /** Names of source tables/views that the SELECT reads from */
2610
+ sourceTables;
2611
+ /** Optional metadata for the view */
2612
+ metadata;
2613
+ /**
2614
+ * Creates a new View instance.
2615
+ * @param name The name of the view to be created.
2616
+ * @param selectStatement The SQL SELECT statement that defines the view's logic.
2617
+ * @param baseTables An array of OlapTable or View objects that the `selectStatement` reads from. Used for dependency tracking.
2618
+ * @param metadata Optional metadata for the view (e.g., description, source file).
2619
+ */
2620
+ constructor(name, selectStatement, baseTables, metadata) {
2621
+ if (typeof selectStatement !== "string") {
2622
+ selectStatement = toStaticQuery(selectStatement);
2623
+ }
2624
+ this.name = name;
2625
+ this.selectSql = selectStatement;
2626
+ this.sourceTables = baseTables.map((t) => formatTableReference2(t));
2627
+ this.metadata = metadata ? { ...metadata } : {};
2628
+ if (!this.metadata.source) {
2629
+ const stack = new Error().stack;
2630
+ const sourceInfo = getSourceFileFromStack(stack);
2631
+ if (sourceInfo) {
2632
+ this.metadata.source = { file: sourceInfo };
2882
2633
  }
2883
- };
2634
+ }
2635
+ const views = getMooseInternal().views;
2636
+ if (!isClientOnlyMode() && views.has(this.name)) {
2637
+ throw new Error(`View with name ${this.name} already exists`);
2638
+ }
2639
+ views.set(this.name, this);
2884
2640
  }
2885
- });
2641
+ };
2886
2642
 
2887
2643
  // src/dmv2/sdk/lifeCycle.ts
2888
- var LifeCycle;
2889
- var init_lifeCycle = __esm({
2890
- "src/dmv2/sdk/lifeCycle.ts"() {
2891
- "use strict";
2892
- LifeCycle = /* @__PURE__ */ ((LifeCycle2) => {
2893
- LifeCycle2["FULLY_MANAGED"] = "FULLY_MANAGED";
2894
- LifeCycle2["DELETION_PROTECTED"] = "DELETION_PROTECTED";
2895
- LifeCycle2["EXTERNALLY_MANAGED"] = "EXTERNALLY_MANAGED";
2896
- return LifeCycle2;
2897
- })(LifeCycle || {});
2898
- }
2899
- });
2644
+ var LifeCycle = /* @__PURE__ */ ((LifeCycle2) => {
2645
+ LifeCycle2["FULLY_MANAGED"] = "FULLY_MANAGED";
2646
+ LifeCycle2["DELETION_PROTECTED"] = "DELETION_PROTECTED";
2647
+ LifeCycle2["EXTERNALLY_MANAGED"] = "EXTERNALLY_MANAGED";
2648
+ return LifeCycle2;
2649
+ })(LifeCycle || {});
2900
2650
 
2901
2651
  // src/dmv2/sdk/webApp.ts
2902
- var RESERVED_MOUNT_PATHS, WebApp;
2903
- var init_webApp = __esm({
2904
- "src/dmv2/sdk/webApp.ts"() {
2905
- "use strict";
2906
- init_internal();
2907
- RESERVED_MOUNT_PATHS = [
2908
- "/admin",
2909
- "/api",
2910
- "/consumption",
2911
- "/health",
2912
- "/ingest",
2913
- "/moose",
2914
- // reserved for future use
2915
- "/ready",
2916
- "/workflows"
2917
- ];
2918
- WebApp = class {
2919
- name;
2920
- handler;
2921
- config;
2922
- _rawApp;
2923
- constructor(name, appOrHandler, config) {
2924
- this.name = name;
2925
- this.config = config;
2926
- if (!this.config.mountPath) {
2927
- throw new Error(
2928
- `mountPath is required. Please specify a mount path for your WebApp (e.g., "/myapi").`
2929
- );
2930
- }
2931
- const mountPath = this.config.mountPath;
2932
- if (mountPath === "/") {
2933
- throw new Error(
2934
- `mountPath cannot be "/" as it would allow routes to overlap with reserved paths: ${RESERVED_MOUNT_PATHS.join(", ")}`
2935
- );
2936
- }
2937
- if (mountPath.endsWith("/")) {
2652
+ var RESERVED_MOUNT_PATHS = [
2653
+ "/admin",
2654
+ "/api",
2655
+ "/consumption",
2656
+ "/health",
2657
+ "/ingest",
2658
+ "/moose",
2659
+ // reserved for future use
2660
+ "/ready",
2661
+ "/workflows"
2662
+ ];
2663
+ var WebApp = class {
2664
+ name;
2665
+ handler;
2666
+ config;
2667
+ _rawApp;
2668
+ constructor(name, appOrHandler, config) {
2669
+ this.name = name;
2670
+ this.config = config;
2671
+ if (!this.config.mountPath) {
2672
+ throw new Error(
2673
+ `mountPath is required. Please specify a mount path for your WebApp (e.g., "/myapi").`
2674
+ );
2675
+ }
2676
+ const mountPath = this.config.mountPath;
2677
+ if (mountPath === "/") {
2678
+ throw new Error(
2679
+ `mountPath cannot be "/" as it would allow routes to overlap with reserved paths: ${RESERVED_MOUNT_PATHS.join(", ")}`
2680
+ );
2681
+ }
2682
+ if (mountPath.endsWith("/")) {
2683
+ throw new Error(
2684
+ `mountPath cannot end with a trailing slash. Remove the '/' from: "${mountPath}"`
2685
+ );
2686
+ }
2687
+ for (const reserved of RESERVED_MOUNT_PATHS) {
2688
+ if (mountPath === reserved || mountPath.startsWith(`${reserved}/`)) {
2689
+ throw new Error(
2690
+ `mountPath cannot begin with a reserved path: ${RESERVED_MOUNT_PATHS.join(", ")}. Got: "${mountPath}"`
2691
+ );
2692
+ }
2693
+ }
2694
+ this.handler = this.toHandler(appOrHandler);
2695
+ this._rawApp = typeof appOrHandler === "function" ? void 0 : appOrHandler;
2696
+ const webApps = getMooseInternal().webApps;
2697
+ if (webApps.has(name)) {
2698
+ throw new Error(`WebApp with name ${name} already exists`);
2699
+ }
2700
+ if (this.config.mountPath) {
2701
+ for (const [existingName, existingApp] of webApps) {
2702
+ if (existingApp.config.mountPath === this.config.mountPath) {
2938
2703
  throw new Error(
2939
- `mountPath cannot end with a trailing slash. Remove the '/' from: "${mountPath}"`
2704
+ `WebApp with mountPath "${this.config.mountPath}" already exists (used by WebApp "${existingName}")`
2940
2705
  );
2941
2706
  }
2942
- for (const reserved of RESERVED_MOUNT_PATHS) {
2943
- if (mountPath === reserved || mountPath.startsWith(`${reserved}/`)) {
2944
- throw new Error(
2945
- `mountPath cannot begin with a reserved path: ${RESERVED_MOUNT_PATHS.join(", ")}. Got: "${mountPath}"`
2946
- );
2947
- }
2948
- }
2949
- this.handler = this.toHandler(appOrHandler);
2950
- this._rawApp = typeof appOrHandler === "function" ? void 0 : appOrHandler;
2951
- const webApps = getMooseInternal().webApps;
2952
- if (webApps.has(name)) {
2953
- throw new Error(`WebApp with name ${name} already exists`);
2954
- }
2955
- if (this.config.mountPath) {
2956
- for (const [existingName, existingApp] of webApps) {
2957
- if (existingApp.config.mountPath === this.config.mountPath) {
2958
- throw new Error(
2959
- `WebApp with mountPath "${this.config.mountPath}" already exists (used by WebApp "${existingName}")`
2960
- );
2961
- }
2962
- }
2963
- }
2964
- webApps.set(name, this);
2965
2707
  }
2966
- toHandler(appOrHandler) {
2967
- if (typeof appOrHandler === "function") {
2968
- return appOrHandler;
2969
- }
2970
- const app = appOrHandler;
2971
- if (typeof app.handle === "function") {
2972
- return (req, res) => {
2973
- app.handle(req, res, (err) => {
2974
- if (err) {
2975
- console.error("WebApp handler error:", err);
2976
- if (!res.headersSent) {
2977
- res.writeHead(500, { "Content-Type": "application/json" });
2978
- res.end(JSON.stringify({ error: "Internal Server Error" }));
2979
- }
2980
- }
2981
- });
2982
- };
2983
- }
2984
- if (typeof app.callback === "function") {
2985
- return app.callback();
2986
- }
2987
- if (typeof app.routing === "function") {
2988
- const routing = app.routing;
2989
- const appWithReady = app;
2990
- let readyPromise = null;
2991
- return async (req, res) => {
2992
- if (readyPromise === null) {
2993
- readyPromise = typeof appWithReady.ready === "function" ? appWithReady.ready() : Promise.resolve();
2708
+ }
2709
+ webApps.set(name, this);
2710
+ }
2711
+ toHandler(appOrHandler) {
2712
+ if (typeof appOrHandler === "function") {
2713
+ return appOrHandler;
2714
+ }
2715
+ const app = appOrHandler;
2716
+ if (typeof app.handle === "function") {
2717
+ return (req, res) => {
2718
+ app.handle(req, res, (err) => {
2719
+ if (err) {
2720
+ console.error("WebApp handler error:", err);
2721
+ if (!res.headersSent) {
2722
+ res.writeHead(500, { "Content-Type": "application/json" });
2723
+ res.end(JSON.stringify({ error: "Internal Server Error" }));
2994
2724
  }
2995
- await readyPromise;
2996
- routing(req, res);
2997
- };
2998
- }
2999
- throw new Error(
3000
- `Unable to convert app to handler. The provided object must be:
2725
+ }
2726
+ });
2727
+ };
2728
+ }
2729
+ if (typeof app.callback === "function") {
2730
+ return app.callback();
2731
+ }
2732
+ if (typeof app.routing === "function") {
2733
+ const routing = app.routing;
2734
+ const appWithReady = app;
2735
+ let readyPromise = null;
2736
+ return async (req, res) => {
2737
+ if (readyPromise === null) {
2738
+ readyPromise = typeof appWithReady.ready === "function" ? appWithReady.ready() : Promise.resolve();
2739
+ }
2740
+ await readyPromise;
2741
+ routing(req, res);
2742
+ };
2743
+ }
2744
+ throw new Error(
2745
+ `Unable to convert app to handler. The provided object must be:
3001
2746
  - A function (raw Node.js handler)
3002
2747
  - An object with .handle() method (Express, Connect)
3003
2748
  - An object with .callback() method (Koa)
@@ -3009,14 +2754,12 @@ Examples:
3009
2754
  Fastify: new WebApp("name", fastifyApp)
3010
2755
  Raw: new WebApp("name", (req, res) => { ... })
3011
2756
  `
3012
- );
3013
- }
3014
- getRawApp() {
3015
- return this._rawApp;
3016
- }
3017
- };
2757
+ );
3018
2758
  }
3019
- });
2759
+ getRawApp() {
2760
+ return this._rawApp;
2761
+ }
2762
+ };
3020
2763
 
3021
2764
  // src/dmv2/registry.ts
3022
2765
  function getTables() {
@@ -3037,7 +2780,7 @@ function getIngestApis() {
3037
2780
  function getIngestApi(name) {
3038
2781
  return getMooseInternal().ingestApis.get(name);
3039
2782
  }
3040
- function getApis2() {
2783
+ function getApis() {
3041
2784
  return getMooseInternal().apis;
3042
2785
  }
3043
2786
  function getApi(nameOrPath) {
@@ -3076,7 +2819,7 @@ function getWorkflows2() {
3076
2819
  function getWorkflow(name) {
3077
2820
  return getMooseInternal().workflows.get(name);
3078
2821
  }
3079
- function getWebApps2() {
2822
+ function getWebApps() {
3080
2823
  return getMooseInternal().webApps;
3081
2824
  }
3082
2825
  function getWebApp(name) {
@@ -3094,43 +2837,6 @@ function getViews() {
3094
2837
  function getView(name) {
3095
2838
  return getMooseInternal().views.get(name);
3096
2839
  }
3097
- var init_registry = __esm({
3098
- "src/dmv2/registry.ts"() {
3099
- "use strict";
3100
- init_internal();
3101
- }
3102
- });
3103
-
3104
- // src/dmv2/index.ts
3105
- var init_dmv2 = __esm({
3106
- "src/dmv2/index.ts"() {
3107
- "use strict";
3108
- init_olapTable();
3109
- init_types();
3110
- init_stream();
3111
- init_workflow();
3112
- init_ingestApi();
3113
- init_consumptionApi();
3114
- init_ingestPipeline();
3115
- init_etlPipeline();
3116
- init_materializedView();
3117
- init_sqlResource();
3118
- init_view();
3119
- init_lifeCycle();
3120
- init_webApp();
3121
- init_registry();
3122
- }
3123
- });
3124
-
3125
- // src/browserCompatible.ts
3126
- var init_browserCompatible = __esm({
3127
- "src/browserCompatible.ts"() {
3128
- init_dmv2();
3129
- init_types();
3130
- init_sqlHelpers();
3131
- }
3132
- });
3133
- init_browserCompatible();
3134
2840
  export {
3135
2841
  Api,
3136
2842
  ClickHouseEngines,
@@ -3151,7 +2857,7 @@ export {
3151
2857
  Workflow,
3152
2858
  createClickhouseParameter,
3153
2859
  getApi,
3154
- getApis2 as getApis,
2860
+ getApis,
3155
2861
  getIngestApi,
3156
2862
  getIngestApis,
3157
2863
  getMaterializedView,
@@ -3166,7 +2872,7 @@ export {
3166
2872
  getView,
3167
2873
  getViews,
3168
2874
  getWebApp,
3169
- getWebApps2 as getWebApps,
2875
+ getWebApps,
3170
2876
  getWorkflow,
3171
2877
  getWorkflows2 as getWorkflows,
3172
2878
  mapToClickHouseType,