@bayoudhi/moose-lib-serverless 0.7.2 → 0.7.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browserCompatible.d.ts +12 -0
- package/dist/index-DdE-_e4q.d.ts +2412 -0
- package/dist/index.d.mts +300 -2387
- package/dist/index.d.ts +300 -2387
- package/dist/index.js +1027 -269
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1021 -275
- package/dist/index.mjs.map +1 -1
- package/dist/moose-runner.js +221 -218
- package/dist/moose-tspc.js +267 -61
- package/package.json +2 -2
package/dist/index.d.mts
CHANGED
|
@@ -1,2349 +1,29 @@
|
|
|
1
|
-
|
|
2
|
-
import
|
|
3
|
-
import {
|
|
4
|
-
|
|
5
|
-
import
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
8
|
-
import
|
|
9
|
-
import
|
|
10
|
-
|
|
11
|
-
/**
|
|
12
|
-
* Quote a ClickHouse identifier with backticks if not already quoted.
|
|
13
|
-
* Backticks allow special characters (e.g., hyphens) in identifiers.
|
|
14
|
-
*/
|
|
15
|
-
declare const quoteIdentifier: (name: string) => string;
|
|
16
|
-
type IdentifierBrandedString = string & {
|
|
17
|
-
readonly __identifier_brand?: unique symbol;
|
|
18
|
-
};
|
|
19
|
-
type NonIdentifierBrandedString = string & {
|
|
20
|
-
readonly __identifier_brand?: unique symbol;
|
|
21
|
-
};
|
|
22
|
-
/**
|
|
23
|
-
* Values supported by SQL engine.
|
|
24
|
-
*/
|
|
25
|
-
type Value = NonIdentifierBrandedString | number | boolean | Date | [string, string];
|
|
26
|
-
/**
|
|
27
|
-
* Supported value or SQL instance.
|
|
28
|
-
*/
|
|
29
|
-
type RawValue = Value | Sql;
|
|
30
|
-
/**
|
|
31
|
-
* Sql template tag interface with attached helper methods.
|
|
32
|
-
*/
|
|
33
|
-
interface SqlTemplateTag {
|
|
34
|
-
(strings: readonly string[], ...values: readonly (RawValue | Column | OlapTable<any> | View)[]): Sql;
|
|
35
|
-
/**
|
|
36
|
-
* Join an array of Sql fragments with a separator.
|
|
37
|
-
* @param fragments - Array of Sql fragments to join
|
|
38
|
-
* @param separator - Optional separator string (defaults to ", ")
|
|
39
|
-
*/
|
|
40
|
-
join(fragments: Sql[], separator?: string): Sql;
|
|
41
|
-
/**
|
|
42
|
-
* Create raw SQL from a string without parameterization.
|
|
43
|
-
* WARNING: SQL injection risk if used with untrusted input.
|
|
44
|
-
*/
|
|
45
|
-
raw(text: string): Sql;
|
|
46
|
-
}
|
|
47
|
-
declare const sql: SqlTemplateTag;
|
|
48
|
-
/**
|
|
49
|
-
* A SQL instance can be nested within each other to build SQL strings.
|
|
50
|
-
*/
|
|
51
|
-
declare class Sql {
|
|
52
|
-
readonly values: Value[];
|
|
53
|
-
readonly strings: string[];
|
|
54
|
-
constructor(rawStrings: readonly string[], rawValues: readonly (RawValue | Column | OlapTable<any> | View | Sql)[]);
|
|
55
|
-
/**
|
|
56
|
-
* Append another Sql fragment, returning a new Sql instance.
|
|
57
|
-
*/
|
|
58
|
-
append(other: Sql): Sql;
|
|
59
|
-
}
|
|
60
|
-
declare const toStaticQuery: (sql: Sql) => string;
|
|
61
|
-
declare const toQuery: (sql: Sql) => [string, {
|
|
62
|
-
[pN: string]: any;
|
|
63
|
-
}];
|
|
64
|
-
/**
|
|
65
|
-
* Build a display-only SQL string with values inlined for logging/debugging.
|
|
66
|
-
* Does not alter execution behavior; use toQuery for actual execution.
|
|
67
|
-
*/
|
|
68
|
-
declare const toQueryPreview: (sql: Sql) => string;
|
|
69
|
-
declare const getValueFromParameter: (value: any) => any;
|
|
70
|
-
declare function createClickhouseParameter(parameterIndex: number, value: Value): string;
|
|
71
|
-
/**
|
|
72
|
-
* Convert the JS type (source is JSON format by API query parameter) to the corresponding ClickHouse type for generating named placeholder of parameterized query.
|
|
73
|
-
* Only support to convert number to Int or Float, boolean to Bool, string to String, other types will convert to String.
|
|
74
|
-
* If exist complex type e.g: object, Array, null, undefined, Date, Record.. etc, just convert to string type by ClickHouse function in SQL.
|
|
75
|
-
* ClickHouse support converting string to other types function.
|
|
76
|
-
* Please see Each section of the https://clickhouse.com/docs/en/sql-reference/functions and https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions
|
|
77
|
-
* @param value
|
|
78
|
-
* @returns 'Float', 'Int', 'Bool', 'String'
|
|
79
|
-
*/
|
|
80
|
-
declare const mapToClickHouseType: (value: Value) => string;
|
|
81
|
-
|
|
82
|
-
type EnumValues = {
|
|
83
|
-
name: string;
|
|
84
|
-
value: {
|
|
85
|
-
Int: number;
|
|
86
|
-
};
|
|
87
|
-
}[] | {
|
|
88
|
-
name: string;
|
|
89
|
-
value: {
|
|
90
|
-
String: string;
|
|
91
|
-
};
|
|
92
|
-
}[];
|
|
93
|
-
type DataEnum = {
|
|
94
|
-
name: string;
|
|
95
|
-
values: EnumValues;
|
|
96
|
-
};
|
|
97
|
-
type Nested = {
|
|
98
|
-
name: string;
|
|
99
|
-
columns: Column[];
|
|
100
|
-
jwt: boolean;
|
|
101
|
-
};
|
|
102
|
-
type ArrayType = {
|
|
103
|
-
elementType: DataType;
|
|
104
|
-
elementNullable: boolean;
|
|
105
|
-
};
|
|
106
|
-
type NamedTupleType = {
|
|
107
|
-
fields: Array<[string, DataType]>;
|
|
108
|
-
};
|
|
109
|
-
type MapType = {
|
|
110
|
-
keyType: DataType;
|
|
111
|
-
valueType: DataType;
|
|
112
|
-
};
|
|
113
|
-
type JsonOptions = {
|
|
114
|
-
max_dynamic_paths?: number;
|
|
115
|
-
max_dynamic_types?: number;
|
|
116
|
-
typed_paths?: Array<[string, DataType]>;
|
|
117
|
-
skip_paths?: string[];
|
|
118
|
-
skip_regexps?: string[];
|
|
119
|
-
};
|
|
120
|
-
type DataType = string | DataEnum | ArrayType | Nested | NamedTupleType | MapType | JsonOptions | {
|
|
121
|
-
nullable: DataType;
|
|
122
|
-
};
|
|
123
|
-
interface Column {
|
|
124
|
-
name: IdentifierBrandedString;
|
|
125
|
-
data_type: DataType;
|
|
126
|
-
required: boolean;
|
|
127
|
-
unique: false;
|
|
128
|
-
primary_key: boolean;
|
|
129
|
-
default: string | null;
|
|
130
|
-
materialized: string | null;
|
|
131
|
-
ttl: string | null;
|
|
132
|
-
codec: string | null;
|
|
133
|
-
annotations: [string, any][];
|
|
134
|
-
comment: string | null;
|
|
135
|
-
}
|
|
136
|
-
|
|
137
|
-
/**
|
|
138
|
-
* Type definition for typia validation functions
|
|
139
|
-
*/
|
|
140
|
-
interface TypiaValidators<T> {
|
|
141
|
-
/** Typia validator function: returns { success: boolean, data?: T, errors?: any[] } */
|
|
142
|
-
validate?: (data: unknown) => {
|
|
143
|
-
success: boolean;
|
|
144
|
-
data?: T;
|
|
145
|
-
errors?: any[];
|
|
146
|
-
};
|
|
147
|
-
/** Typia assert function: throws on validation failure, returns T on success */
|
|
148
|
-
assert?: (data: unknown) => T;
|
|
149
|
-
/** Typia is function: returns boolean indicating if data matches type T */
|
|
150
|
-
is?: (data: unknown) => data is T;
|
|
151
|
-
}
|
|
152
|
-
/**
|
|
153
|
-
* Base class for all typed Moose dmv2 resources (OlapTable, Stream, etc.).
|
|
154
|
-
* Handles the storage and injection of schema information (JSON schema and Column array)
|
|
155
|
-
* provided by the Moose compiler plugin.
|
|
156
|
-
*
|
|
157
|
-
* @template T The data type (interface or type alias) defining the schema of the resource.
|
|
158
|
-
* @template C The specific configuration type for the resource (e.g., OlapConfig, StreamConfig).
|
|
159
|
-
*/
|
|
160
|
-
declare class TypedBase<T, C> {
|
|
161
|
-
/** The JSON schema representation of type T. Injected by the compiler plugin. */
|
|
162
|
-
schema: IJsonSchemaCollection.IV3_1;
|
|
163
|
-
/** The name assigned to this resource instance. */
|
|
164
|
-
name: string;
|
|
165
|
-
/** A dictionary mapping column names (keys of T) to their Column definitions. */
|
|
166
|
-
columns: {
|
|
167
|
-
[columnName in keyof Required<T>]: Column;
|
|
168
|
-
};
|
|
169
|
-
/** An array containing the Column definitions for this resource. Injected by the compiler plugin. */
|
|
170
|
-
columnArray: Column[];
|
|
171
|
-
/** The configuration object specific to this resource type. */
|
|
172
|
-
config: C;
|
|
173
|
-
/** Typia validation functions for type T. Injected by the compiler plugin for OlapTable. */
|
|
174
|
-
validators?: TypiaValidators<T>;
|
|
175
|
-
/** Optional metadata for the resource, always present as an object. */
|
|
176
|
-
metadata: {
|
|
177
|
-
[key: string]: any;
|
|
178
|
-
};
|
|
179
|
-
/**
|
|
180
|
-
* Whether this resource allows extra fields beyond the defined columns.
|
|
181
|
-
* When true, extra fields in payloads are passed through to streaming functions.
|
|
182
|
-
* Injected by the compiler plugin when the type has an index signature.
|
|
183
|
-
*/
|
|
184
|
-
allowExtraFields: boolean;
|
|
185
|
-
/**
|
|
186
|
-
* @internal Constructor intended for internal use by subclasses and the compiler plugin.
|
|
187
|
-
* It expects the schema and columns to be provided, typically injected by the compiler.
|
|
188
|
-
*
|
|
189
|
-
* @param name The name for the resource instance.
|
|
190
|
-
* @param config The configuration object for the resource.
|
|
191
|
-
* @param schema The JSON schema for the resource's data type T (injected).
|
|
192
|
-
* @param columns The array of Column definitions for T (injected).
|
|
193
|
-
* @param allowExtraFields Whether extra fields are allowed (injected when type has index signature).
|
|
194
|
-
*/
|
|
195
|
-
constructor(name: string, config: C, schema?: IJsonSchemaCollection.IV3_1, columns?: Column[], validators?: TypiaValidators<T>, allowExtraFields?: boolean);
|
|
196
|
-
}
|
|
197
|
-
|
|
198
|
-
type ClickHousePrecision<P extends number> = {
|
|
199
|
-
_clickhouse_precision?: P;
|
|
200
|
-
};
|
|
201
|
-
declare const DecimalRegex: "^-?\\d+(\\.\\d+)?$";
|
|
202
|
-
type ClickHouseDecimal<P extends number, S extends number> = {
|
|
203
|
-
_clickhouse_precision?: P;
|
|
204
|
-
_clickhouse_scale?: S;
|
|
205
|
-
} & Pattern<typeof DecimalRegex>;
|
|
206
|
-
type ClickHouseFixedStringSize<N extends number> = {
|
|
207
|
-
_clickhouse_fixed_string_size?: N;
|
|
208
|
-
};
|
|
209
|
-
/**
|
|
210
|
-
* FixedString(N) - Fixed-length string of exactly N bytes.
|
|
211
|
-
*
|
|
212
|
-
* ClickHouse stores exactly N bytes, padding shorter values with null bytes.
|
|
213
|
-
* Values exceeding N bytes will throw an exception.
|
|
214
|
-
*
|
|
215
|
-
* Use for binary data: hashes, IP addresses, UUIDs, MAC addresses.
|
|
216
|
-
*
|
|
217
|
-
* @example
|
|
218
|
-
* interface BinaryData {
|
|
219
|
-
* md5_hash: string & FixedString<16>; // 16-byte MD5
|
|
220
|
-
* sha256_hash: string & FixedString<32>; // 32-byte SHA256
|
|
221
|
-
* }
|
|
222
|
-
*/
|
|
223
|
-
type FixedString<N extends number> = string & ClickHouseFixedStringSize<N>;
|
|
224
|
-
type ClickHouseByteSize<N extends number> = {
|
|
225
|
-
_clickhouse_byte_size?: N;
|
|
226
|
-
};
|
|
227
|
-
type LowCardinality = {
|
|
228
|
-
_LowCardinality?: true;
|
|
229
|
-
};
|
|
230
|
-
type DateTime = Date;
|
|
231
|
-
type DateTime64<P extends number> = Date & ClickHousePrecision<P>;
|
|
232
|
-
type DateTimeString = string & tags.Format<"date-time">;
|
|
233
|
-
/**
|
|
234
|
-
* JS Date objects cannot hold microsecond precision.
|
|
235
|
-
* Use string as the runtime type to avoid losing information.
|
|
236
|
-
*/
|
|
237
|
-
type DateTime64String<P extends number> = string & tags.Format<"date-time"> & ClickHousePrecision<P>;
|
|
238
|
-
type Float32 = number & ClickHouseFloat<"float32">;
|
|
239
|
-
type Float64 = number & ClickHouseFloat<"float64">;
|
|
240
|
-
type Int8 = number & ClickHouseInt<"int8">;
|
|
241
|
-
type Int16 = number & ClickHouseInt<"int16">;
|
|
242
|
-
type Int32 = number & ClickHouseInt<"int32">;
|
|
243
|
-
type Int64 = number & ClickHouseInt<"int64">;
|
|
244
|
-
type UInt8 = number & ClickHouseInt<"uint8">;
|
|
245
|
-
type UInt16 = number & ClickHouseInt<"uint16">;
|
|
246
|
-
type UInt32 = number & ClickHouseInt<"uint32">;
|
|
247
|
-
type UInt64 = number & ClickHouseInt<"uint64">;
|
|
248
|
-
type Decimal<P extends number, S extends number> = string & ClickHouseDecimal<P, S>;
|
|
249
|
-
/**
|
|
250
|
-
* Attach compression codec to a column type.
|
|
251
|
-
*
|
|
252
|
-
* Any valid ClickHouse codec expression is allowed. ClickHouse validates the codec at runtime.
|
|
253
|
-
*
|
|
254
|
-
* @template T The base data type
|
|
255
|
-
* @template CodecExpr The codec expression (single codec or chain)
|
|
256
|
-
*
|
|
257
|
-
* @example
|
|
258
|
-
* interface Metrics {
|
|
259
|
-
* // Single codec
|
|
260
|
-
* log_blob: string & ClickHouseCodec<"ZSTD(3)">;
|
|
261
|
-
*
|
|
262
|
-
* // Codec chain (processed left-to-right)
|
|
263
|
-
* timestamp: Date & ClickHouseCodec<"Delta, LZ4">;
|
|
264
|
-
* temperature: number & ClickHouseCodec<"Gorilla, ZSTD">;
|
|
265
|
-
*
|
|
266
|
-
* // Specialized codecs
|
|
267
|
-
* counter: number & ClickHouseCodec<"DoubleDelta">;
|
|
268
|
-
*
|
|
269
|
-
* // Can combine with other annotations
|
|
270
|
-
* count: UInt64 & ClickHouseCodec<"DoubleDelta, LZ4">;
|
|
271
|
-
* }
|
|
272
|
-
*/
|
|
273
|
-
type ClickHouseCodec<CodecExpr extends string> = {
|
|
274
|
-
_clickhouse_codec?: CodecExpr;
|
|
275
|
-
};
|
|
276
|
-
type ClickHouseFloat<Value extends "float32" | "float64"> = tags.Type<Value extends "float32" ? "float" : "double">;
|
|
277
|
-
type ClickHouseInt<Value extends "int8" | "int16" | "int32" | "int64" | "uint8" | "uint16" | "uint32" | "uint64"> = Value extends "int32" | "int64" | "uint32" | "uint64" ? tags.Type<Value> : TagBase<{
|
|
278
|
-
target: "number";
|
|
279
|
-
kind: "type";
|
|
280
|
-
value: Value;
|
|
281
|
-
validate: Value extends "int8" ? "-128 <= $input && $input <= 127" : Value extends "int16" ? "-32768 <= $input && $input <= 32767" : Value extends "uint8" ? "0 <= $input && $input <= 255" : Value extends "uint16" ? "0 <= $input && $input <= 65535" : never;
|
|
282
|
-
exclusive: true;
|
|
283
|
-
schema: {
|
|
284
|
-
type: "integer";
|
|
285
|
-
};
|
|
286
|
-
}>;
|
|
287
|
-
/**
|
|
288
|
-
* By default, nested objects map to the `Nested` type in clickhouse.
|
|
289
|
-
* Write `nestedObject: AnotherInterfaceType & ClickHouseNamedTuple`
|
|
290
|
-
* to map AnotherInterfaceType to the named tuple type.
|
|
291
|
-
*/
|
|
292
|
-
type ClickHouseNamedTuple = {
|
|
293
|
-
_clickhouse_mapped_type?: "namedTuple";
|
|
294
|
-
};
|
|
295
|
-
type ClickHouseJson<maxDynamicPaths extends number | undefined = undefined, maxDynamicTypes extends number | undefined = undefined, skipPaths extends string[] = [], skipRegexes extends string[] = []> = {
|
|
296
|
-
_clickhouse_mapped_type?: "JSON";
|
|
297
|
-
_clickhouse_json_settings?: {
|
|
298
|
-
maxDynamicPaths?: maxDynamicPaths;
|
|
299
|
-
maxDynamicTypes?: maxDynamicTypes;
|
|
300
|
-
skipPaths?: skipPaths;
|
|
301
|
-
skipRegexes?: skipRegexes;
|
|
302
|
-
};
|
|
303
|
-
};
|
|
304
|
-
/**
|
|
305
|
-
* typia may have trouble handling this type.
|
|
306
|
-
* In which case, use {@link WithDefault} as a workaround
|
|
307
|
-
*
|
|
308
|
-
* @example
|
|
309
|
-
* { field: number & ClickHouseDefault<"0"> }
|
|
310
|
-
*/
|
|
311
|
-
type ClickHouseDefault<SqlExpression extends string> = {
|
|
312
|
-
_clickhouse_default?: SqlExpression;
|
|
313
|
-
};
|
|
314
|
-
/**
|
|
315
|
-
* @example
|
|
316
|
-
* {
|
|
317
|
-
* ...
|
|
318
|
-
* timestamp: Date;
|
|
319
|
-
* debugMessage: string & ClickHouseTTL<"timestamp + INTERVAL 1 WEEK">;
|
|
320
|
-
* }
|
|
321
|
-
*/
|
|
322
|
-
type ClickHouseTTL<SqlExpression extends string> = {
|
|
323
|
-
_clickhouse_ttl?: SqlExpression;
|
|
324
|
-
};
|
|
325
|
-
/**
|
|
326
|
-
* ClickHouse MATERIALIZED column annotation.
|
|
327
|
-
* The column value is computed at INSERT time and physically stored.
|
|
328
|
-
* Cannot be explicitly inserted by users.
|
|
329
|
-
*
|
|
330
|
-
* @example
|
|
331
|
-
* interface Events {
|
|
332
|
-
* eventTime: DateTime;
|
|
333
|
-
* // Extract date component - computed and stored at insert time
|
|
334
|
-
* eventDate: Date & ClickHouseMaterialized<"toDate(event_time)">;
|
|
335
|
-
*
|
|
336
|
-
* userId: string;
|
|
337
|
-
* // Precompute hash for fast lookups
|
|
338
|
-
* userHash: UInt64 & ClickHouseMaterialized<"cityHash64(userId)">;
|
|
339
|
-
* }
|
|
340
|
-
*
|
|
341
|
-
* @remarks
|
|
342
|
-
* - MATERIALIZED and DEFAULT are mutually exclusive
|
|
343
|
-
* - Can be combined with ClickHouseCodec for compression
|
|
344
|
-
* - Changing the expression modifies the column in-place (existing values preserved)
|
|
345
|
-
*/
|
|
346
|
-
type ClickHouseMaterialized<SqlExpression extends string> = {
|
|
347
|
-
_clickhouse_materialized?: SqlExpression;
|
|
348
|
-
};
|
|
349
|
-
/**
|
|
350
|
-
* See also {@link ClickHouseDefault}
|
|
351
|
-
*
|
|
352
|
-
* @example{ updated_at: WithDefault<Date, "now()"> }
|
|
353
|
-
*/
|
|
354
|
-
type WithDefault<T, _SqlExpression extends string> = T;
|
|
355
|
-
/**
|
|
356
|
-
* ClickHouse table engine types supported by Moose.
|
|
357
|
-
*/
|
|
358
|
-
declare enum ClickHouseEngines {
|
|
359
|
-
MergeTree = "MergeTree",
|
|
360
|
-
ReplacingMergeTree = "ReplacingMergeTree",
|
|
361
|
-
SummingMergeTree = "SummingMergeTree",
|
|
362
|
-
AggregatingMergeTree = "AggregatingMergeTree",
|
|
363
|
-
CollapsingMergeTree = "CollapsingMergeTree",
|
|
364
|
-
VersionedCollapsingMergeTree = "VersionedCollapsingMergeTree",
|
|
365
|
-
GraphiteMergeTree = "GraphiteMergeTree",
|
|
366
|
-
S3Queue = "S3Queue",
|
|
367
|
-
S3 = "S3",
|
|
368
|
-
Buffer = "Buffer",
|
|
369
|
-
Distributed = "Distributed",
|
|
370
|
-
IcebergS3 = "IcebergS3",
|
|
371
|
-
Kafka = "Kafka",
|
|
372
|
-
ReplicatedMergeTree = "ReplicatedMergeTree",
|
|
373
|
-
ReplicatedReplacingMergeTree = "ReplicatedReplacingMergeTree",
|
|
374
|
-
ReplicatedAggregatingMergeTree = "ReplicatedAggregatingMergeTree",
|
|
375
|
-
ReplicatedSummingMergeTree = "ReplicatedSummingMergeTree",
|
|
376
|
-
ReplicatedCollapsingMergeTree = "ReplicatedCollapsingMergeTree",
|
|
377
|
-
ReplicatedVersionedCollapsingMergeTree = "ReplicatedVersionedCollapsingMergeTree"
|
|
378
|
-
}
|
|
379
|
-
|
|
380
|
-
/**
|
|
381
|
-
* Defines how Moose manages the lifecycle of database resources when your code changes.
|
|
382
|
-
*
|
|
383
|
-
* This enum controls the behavior when there are differences between your code definitions
|
|
384
|
-
* and the actual database schema or structure.
|
|
385
|
-
*/
|
|
386
|
-
declare enum LifeCycle {
|
|
387
|
-
/**
|
|
388
|
-
* Full automatic management (default behavior).
|
|
389
|
-
* Moose will automatically modify database resources to match your code definitions,
|
|
390
|
-
* including potentially destructive operations like dropping columns or tables.
|
|
391
|
-
*/
|
|
392
|
-
FULLY_MANAGED = "FULLY_MANAGED",
|
|
393
|
-
/**
|
|
394
|
-
* Deletion-protected automatic management.
|
|
395
|
-
* Moose will modify resources to match your code but will avoid destructive actions
|
|
396
|
-
* such as dropping columns, or tables. Only additive changes are applied.
|
|
397
|
-
*/
|
|
398
|
-
DELETION_PROTECTED = "DELETION_PROTECTED",
|
|
399
|
-
/**
|
|
400
|
-
* External management - no automatic changes.
|
|
401
|
-
* Moose will not modify the database resources. You are responsible for managing
|
|
402
|
-
* the schema and ensuring it matches your code definitions manually.
|
|
403
|
-
*/
|
|
404
|
-
EXTERNALLY_MANAGED = "EXTERNALLY_MANAGED"
|
|
405
|
-
}
|
|
406
|
-
|
|
407
|
-
interface TableIndex {
|
|
408
|
-
name: string;
|
|
409
|
-
expression: string;
|
|
410
|
-
type: string;
|
|
411
|
-
arguments?: string[];
|
|
412
|
-
granularity?: number;
|
|
413
|
-
}
|
|
414
|
-
/**
|
|
415
|
-
* Represents a failed record during insertion with error details
|
|
416
|
-
*/
|
|
417
|
-
interface FailedRecord<T> {
|
|
418
|
-
/** The original record that failed to insert */
|
|
419
|
-
record: T;
|
|
420
|
-
/** The error message describing why the insertion failed */
|
|
421
|
-
error: string;
|
|
422
|
-
/** Optional: The index of this record in the original batch */
|
|
423
|
-
index?: number;
|
|
424
|
-
}
|
|
425
|
-
/**
|
|
426
|
-
* Result of an insert operation with detailed success/failure information
|
|
427
|
-
*/
|
|
428
|
-
interface InsertResult<T> {
|
|
429
|
-
/** Number of records successfully inserted */
|
|
430
|
-
successful: number;
|
|
431
|
-
/** Number of records that failed to insert */
|
|
432
|
-
failed: number;
|
|
433
|
-
/** Total number of records processed */
|
|
434
|
-
total: number;
|
|
435
|
-
/** Detailed information about failed records (if record isolation was used) */
|
|
436
|
-
failedRecords?: FailedRecord<T>[];
|
|
437
|
-
}
|
|
438
|
-
/**
|
|
439
|
-
* Error handling strategy for insert operations
|
|
440
|
-
*/
|
|
441
|
-
type ErrorStrategy = "fail-fast" | "discard" | "isolate";
|
|
442
|
-
/**
|
|
443
|
-
* Options for insert operations
|
|
444
|
-
*/
|
|
445
|
-
interface InsertOptions {
|
|
446
|
-
/** Maximum number of bad records to tolerate before failing */
|
|
447
|
-
allowErrors?: number;
|
|
448
|
-
/** Maximum ratio of bad records to tolerate (0.0 to 1.0) before failing */
|
|
449
|
-
allowErrorsRatio?: number;
|
|
450
|
-
/** Error handling strategy */
|
|
451
|
-
strategy?: ErrorStrategy;
|
|
452
|
-
/** Whether to enable dead letter queue for failed records (future feature) */
|
|
453
|
-
deadLetterQueue?: boolean;
|
|
454
|
-
/** Whether to validate data against schema before insertion (default: true) */
|
|
455
|
-
validate?: boolean;
|
|
456
|
-
/** Whether to skip validation for individual records during 'isolate' strategy retries (default: false) */
|
|
457
|
-
skipValidationOnRetry?: boolean;
|
|
458
|
-
}
|
|
459
|
-
/**
|
|
460
|
-
* Validation result for a record with detailed error information
|
|
461
|
-
*/
|
|
462
|
-
interface ValidationError {
|
|
463
|
-
/** The original record that failed validation */
|
|
464
|
-
record: any;
|
|
465
|
-
/** Detailed validation error message */
|
|
466
|
-
error: string;
|
|
467
|
-
/** Optional: The index of this record in the original batch */
|
|
468
|
-
index?: number;
|
|
469
|
-
/** The path to the field that failed validation */
|
|
470
|
-
path?: string;
|
|
471
|
-
}
|
|
472
|
-
/**
|
|
473
|
-
* Result of data validation with success/failure breakdown
|
|
474
|
-
*/
|
|
475
|
-
interface ValidationResult<T> {
|
|
476
|
-
/** Records that passed validation */
|
|
477
|
-
valid: T[];
|
|
478
|
-
/** Records that failed validation with detailed error information */
|
|
479
|
-
invalid: ValidationError[];
|
|
480
|
-
/** Total number of records processed */
|
|
481
|
-
total: number;
|
|
482
|
-
}
|
|
483
|
-
/**
|
|
484
|
-
* S3Queue-specific table settings that can be modified with ALTER TABLE MODIFY SETTING
|
|
485
|
-
* Note: Since ClickHouse 24.7, settings no longer require the 's3queue_' prefix
|
|
486
|
-
*/
|
|
487
|
-
interface S3QueueTableSettings {
|
|
488
|
-
/** Processing mode: "ordered" for sequential or "unordered" for parallel processing */
|
|
489
|
-
mode?: "ordered" | "unordered";
|
|
490
|
-
/** What to do with files after processing: 'keep' or 'delete' */
|
|
491
|
-
after_processing?: "keep" | "delete";
|
|
492
|
-
/** ZooKeeper/Keeper path for coordination between replicas */
|
|
493
|
-
keeper_path?: string;
|
|
494
|
-
/** Number of retry attempts for failed files */
|
|
495
|
-
loading_retries?: string;
|
|
496
|
-
/** Number of threads for parallel processing */
|
|
497
|
-
processing_threads_num?: string;
|
|
498
|
-
/** Enable parallel inserts */
|
|
499
|
-
parallel_inserts?: string;
|
|
500
|
-
/** Enable logging to system.s3queue_log table */
|
|
501
|
-
enable_logging_to_queue_log?: string;
|
|
502
|
-
/** Last processed file path (for ordered mode) */
|
|
503
|
-
last_processed_path?: string;
|
|
504
|
-
/** Maximum number of tracked files in ZooKeeper */
|
|
505
|
-
tracked_files_limit?: string;
|
|
506
|
-
/** TTL for tracked files in seconds */
|
|
507
|
-
tracked_file_ttl_sec?: string;
|
|
508
|
-
/** Minimum polling timeout in milliseconds */
|
|
509
|
-
polling_min_timeout_ms?: string;
|
|
510
|
-
/** Maximum polling timeout in milliseconds */
|
|
511
|
-
polling_max_timeout_ms?: string;
|
|
512
|
-
/** Polling backoff in milliseconds */
|
|
513
|
-
polling_backoff_ms?: string;
|
|
514
|
-
/** Minimum cleanup interval in milliseconds */
|
|
515
|
-
cleanup_interval_min_ms?: string;
|
|
516
|
-
/** Maximum cleanup interval in milliseconds */
|
|
517
|
-
cleanup_interval_max_ms?: string;
|
|
518
|
-
/** Number of buckets for sharding (0 = disabled) */
|
|
519
|
-
buckets?: string;
|
|
520
|
-
/** Batch size for listing objects */
|
|
521
|
-
list_objects_batch_size?: string;
|
|
522
|
-
/** Enable hash ring filtering for distributed processing */
|
|
523
|
-
enable_hash_ring_filtering?: string;
|
|
524
|
-
/** Maximum files to process before committing */
|
|
525
|
-
max_processed_files_before_commit?: string;
|
|
526
|
-
/** Maximum rows to process before committing */
|
|
527
|
-
max_processed_rows_before_commit?: string;
|
|
528
|
-
/** Maximum bytes to process before committing */
|
|
529
|
-
max_processed_bytes_before_commit?: string;
|
|
530
|
-
/** Maximum processing time in seconds before committing */
|
|
531
|
-
max_processing_time_sec_before_commit?: string;
|
|
532
|
-
/** Use persistent processing nodes (available from 25.8) */
|
|
533
|
-
use_persistent_processing_nodes?: string;
|
|
534
|
-
/** TTL for persistent processing nodes in seconds */
|
|
535
|
-
persistent_processing_nodes_ttl_seconds?: string;
|
|
536
|
-
/** Additional settings */
|
|
537
|
-
[key: string]: string | undefined;
|
|
538
|
-
}
|
|
539
|
-
/**
|
|
540
|
-
* Base configuration shared by all table engines
|
|
541
|
-
* @template T The data type of the records stored in the table.
|
|
542
|
-
*/
|
|
543
|
-
type BaseOlapConfig<T> = ({
|
|
544
|
-
/**
|
|
545
|
-
* Specifies the fields to use for ordering data within the ClickHouse table.
|
|
546
|
-
* This is crucial for optimizing query performance.
|
|
547
|
-
*/
|
|
548
|
-
orderByFields: (keyof T & string)[];
|
|
549
|
-
orderByExpression?: undefined;
|
|
550
|
-
} | {
|
|
551
|
-
orderByFields?: undefined;
|
|
552
|
-
/**
|
|
553
|
-
* An arbitrary ClickHouse SQL expression for the order by clause.
|
|
554
|
-
*
|
|
555
|
-
* `orderByExpression: "(id, name)"` is equivalent to `orderByFields: ["id", "name"]`
|
|
556
|
-
* `orderByExpression: "tuple()"` means no sorting
|
|
557
|
-
*/
|
|
558
|
-
orderByExpression: string;
|
|
559
|
-
} | {
|
|
560
|
-
orderByFields?: undefined;
|
|
561
|
-
orderByExpression?: undefined;
|
|
562
|
-
}) & {
|
|
563
|
-
partitionBy?: string;
|
|
564
|
-
/**
|
|
565
|
-
* SAMPLE BY expression for approximate query processing.
|
|
566
|
-
*
|
|
567
|
-
* Examples:
|
|
568
|
-
* ```typescript
|
|
569
|
-
* // Single unsigned integer field
|
|
570
|
-
* sampleByExpression: "userId"
|
|
571
|
-
*
|
|
572
|
-
* // Hash function on any field type
|
|
573
|
-
* sampleByExpression: "cityHash64(id)"
|
|
574
|
-
*
|
|
575
|
-
* // Multiple fields with hash
|
|
576
|
-
* sampleByExpression: "cityHash64(userId, timestamp)"
|
|
577
|
-
* ```
|
|
578
|
-
*
|
|
579
|
-
* Requirements:
|
|
580
|
-
* - Expression must evaluate to an unsigned integer (UInt8/16/32/64)
|
|
581
|
-
* - Expression must be present in the ORDER BY clause
|
|
582
|
-
* - If using hash functions, the same expression must appear in orderByExpression
|
|
583
|
-
*/
|
|
584
|
-
sampleByExpression?: string;
|
|
585
|
-
/**
|
|
586
|
-
* Optional PRIMARY KEY expression.
|
|
587
|
-
* When specified, this overrides the primary key inferred from Key<T> column annotations.
|
|
588
|
-
*
|
|
589
|
-
* This allows for:
|
|
590
|
-
* - Complex primary keys using functions (e.g., "cityHash64(id)")
|
|
591
|
-
* - Different column ordering in primary key vs schema definition
|
|
592
|
-
* - Primary keys that differ from ORDER BY
|
|
593
|
-
*
|
|
594
|
-
* Example: primaryKeyExpression: "(userId, cityHash64(eventId))"
|
|
595
|
-
*
|
|
596
|
-
* Note: When this is set, any Key<T> annotations on columns are ignored for PRIMARY KEY generation.
|
|
597
|
-
*/
|
|
598
|
-
primaryKeyExpression?: string;
|
|
599
|
-
version?: string;
|
|
600
|
-
lifeCycle?: LifeCycle;
|
|
601
|
-
settings?: {
|
|
602
|
-
[key: string]: string;
|
|
603
|
-
};
|
|
604
|
-
/**
|
|
605
|
-
* Optional TTL configuration for the table.
|
|
606
|
-
* e.g., "TTL timestamp + INTERVAL 90 DAY DELETE"
|
|
607
|
-
*
|
|
608
|
-
* Use the {@link ClickHouseTTL} type to configure column level TTL
|
|
609
|
-
*/
|
|
610
|
-
ttl?: string;
|
|
611
|
-
/** Optional secondary/data-skipping indexes */
|
|
612
|
-
indexes?: TableIndex[];
|
|
613
|
-
/**
|
|
614
|
-
* Optional database name for multi-database support.
|
|
615
|
-
* When not specified, uses the global ClickHouse config database.
|
|
616
|
-
*/
|
|
617
|
-
database?: string;
|
|
618
|
-
/**
|
|
619
|
-
* Optional cluster name for ON CLUSTER support.
|
|
620
|
-
* Use this to enable replicated tables across ClickHouse clusters.
|
|
621
|
-
* The cluster must be defined in config.toml (dev environment only).
|
|
622
|
-
* Example: cluster: "prod_cluster"
|
|
623
|
-
*/
|
|
624
|
-
cluster?: string;
|
|
625
|
-
};
|
|
626
|
-
/**
|
|
627
|
-
* Configuration for MergeTree engine
|
|
628
|
-
* @template T The data type of the records stored in the table.
|
|
629
|
-
*/
|
|
630
|
-
type MergeTreeConfig<T> = BaseOlapConfig<T> & {
|
|
631
|
-
engine: ClickHouseEngines.MergeTree;
|
|
632
|
-
};
|
|
633
|
-
/**
|
|
634
|
-
* Configuration for ReplacingMergeTree engine (deduplication)
|
|
635
|
-
* @template T The data type of the records stored in the table.
|
|
636
|
-
*/
|
|
637
|
-
type ReplacingMergeTreeConfig<T> = BaseOlapConfig<T> & {
|
|
638
|
-
engine: ClickHouseEngines.ReplacingMergeTree;
|
|
639
|
-
ver?: keyof T & string;
|
|
640
|
-
isDeleted?: keyof T & string;
|
|
641
|
-
};
|
|
642
|
-
/**
|
|
643
|
-
* Configuration for AggregatingMergeTree engine
|
|
644
|
-
* @template T The data type of the records stored in the table.
|
|
645
|
-
*/
|
|
646
|
-
type AggregatingMergeTreeConfig<T> = BaseOlapConfig<T> & {
|
|
647
|
-
engine: ClickHouseEngines.AggregatingMergeTree;
|
|
648
|
-
};
|
|
649
|
-
/**
|
|
650
|
-
* Configuration for SummingMergeTree engine
|
|
651
|
-
* @template T The data type of the records stored in the table.
|
|
652
|
-
*/
|
|
653
|
-
type SummingMergeTreeConfig<T> = BaseOlapConfig<T> & {
|
|
654
|
-
engine: ClickHouseEngines.SummingMergeTree;
|
|
655
|
-
columns?: string[];
|
|
656
|
-
};
|
|
657
|
-
/**
|
|
658
|
-
* Configuration for CollapsingMergeTree engine
|
|
659
|
-
* @template T The data type of the records stored in the table.
|
|
660
|
-
*/
|
|
661
|
-
type CollapsingMergeTreeConfig<T> = BaseOlapConfig<T> & {
|
|
662
|
-
engine: ClickHouseEngines.CollapsingMergeTree;
|
|
663
|
-
sign: keyof T & string;
|
|
664
|
-
};
|
|
665
|
-
/**
|
|
666
|
-
* Configuration for VersionedCollapsingMergeTree engine
|
|
667
|
-
* @template T The data type of the records stored in the table.
|
|
668
|
-
*/
|
|
669
|
-
type VersionedCollapsingMergeTreeConfig<T> = BaseOlapConfig<T> & {
|
|
670
|
-
engine: ClickHouseEngines.VersionedCollapsingMergeTree;
|
|
671
|
-
sign: keyof T & string;
|
|
672
|
-
ver: keyof T & string;
|
|
673
|
-
};
|
|
674
|
-
interface ReplicatedEngineProperties {
|
|
675
|
-
keeperPath?: string;
|
|
676
|
-
replicaName?: string;
|
|
677
|
-
}
|
|
678
|
-
/**
|
|
679
|
-
* Configuration for ReplicatedMergeTree engine
|
|
680
|
-
* @template T The data type of the records stored in the table.
|
|
681
|
-
*
|
|
682
|
-
* Note: keeperPath and replicaName are optional. Omit them for ClickHouse Cloud,
|
|
683
|
-
* which manages replication automatically. For self-hosted with ClickHouse Keeper,
|
|
684
|
-
* provide both parameters or neither (to use server defaults).
|
|
685
|
-
*/
|
|
686
|
-
type ReplicatedMergeTreeConfig<T> = Omit<MergeTreeConfig<T>, "engine"> & ReplicatedEngineProperties & {
|
|
687
|
-
engine: ClickHouseEngines.ReplicatedMergeTree;
|
|
688
|
-
};
|
|
689
|
-
/**
|
|
690
|
-
* Configuration for ReplicatedReplacingMergeTree engine
|
|
691
|
-
* @template T The data type of the records stored in the table.
|
|
692
|
-
*
|
|
693
|
-
* Note: keeperPath and replicaName are optional. Omit them for ClickHouse Cloud,
|
|
694
|
-
* which manages replication automatically. For self-hosted with ClickHouse Keeper,
|
|
695
|
-
* provide both parameters or neither (to use server defaults).
|
|
696
|
-
*/
|
|
697
|
-
type ReplicatedReplacingMergeTreeConfig<T> = Omit<ReplacingMergeTreeConfig<T>, "engine"> & ReplicatedEngineProperties & {
|
|
698
|
-
engine: ClickHouseEngines.ReplicatedReplacingMergeTree;
|
|
699
|
-
};
|
|
700
|
-
/**
|
|
701
|
-
* Configuration for ReplicatedAggregatingMergeTree engine
|
|
702
|
-
* @template T The data type of the records stored in the table.
|
|
703
|
-
*
|
|
704
|
-
* Note: keeperPath and replicaName are optional. Omit them for ClickHouse Cloud,
|
|
705
|
-
* which manages replication automatically. For self-hosted with ClickHouse Keeper,
|
|
706
|
-
* provide both parameters or neither (to use server defaults).
|
|
707
|
-
*/
|
|
708
|
-
type ReplicatedAggregatingMergeTreeConfig<T> = Omit<AggregatingMergeTreeConfig<T>, "engine"> & ReplicatedEngineProperties & {
|
|
709
|
-
engine: ClickHouseEngines.ReplicatedAggregatingMergeTree;
|
|
710
|
-
};
|
|
711
|
-
/**
|
|
712
|
-
* Configuration for ReplicatedSummingMergeTree engine
|
|
713
|
-
* @template T The data type of the records stored in the table.
|
|
714
|
-
*
|
|
715
|
-
* Note: keeperPath and replicaName are optional. Omit them for ClickHouse Cloud,
|
|
716
|
-
* which manages replication automatically. For self-hosted with ClickHouse Keeper,
|
|
717
|
-
* provide both parameters or neither (to use server defaults).
|
|
718
|
-
*/
|
|
719
|
-
type ReplicatedSummingMergeTreeConfig<T> = Omit<SummingMergeTreeConfig<T>, "engine"> & ReplicatedEngineProperties & {
|
|
720
|
-
engine: ClickHouseEngines.ReplicatedSummingMergeTree;
|
|
721
|
-
};
|
|
722
|
-
/**
|
|
723
|
-
* Configuration for ReplicatedCollapsingMergeTree engine
|
|
724
|
-
* @template T The data type of the records stored in the table.
|
|
725
|
-
*
|
|
726
|
-
* Note: keeperPath and replicaName are optional. Omit them for ClickHouse Cloud,
|
|
727
|
-
* which manages replication automatically. For self-hosted with ClickHouse Keeper,
|
|
728
|
-
* provide both parameters or neither (to use server defaults).
|
|
729
|
-
*/
|
|
730
|
-
type ReplicatedCollapsingMergeTreeConfig<T> = Omit<CollapsingMergeTreeConfig<T>, "engine"> & ReplicatedEngineProperties & {
|
|
731
|
-
engine: ClickHouseEngines.ReplicatedCollapsingMergeTree;
|
|
732
|
-
};
|
|
733
|
-
/**
|
|
734
|
-
* Configuration for ReplicatedVersionedCollapsingMergeTree engine
|
|
735
|
-
* @template T The data type of the records stored in the table.
|
|
736
|
-
*
|
|
737
|
-
* Note: keeperPath and replicaName are optional. Omit them for ClickHouse Cloud,
|
|
738
|
-
* which manages replication automatically. For self-hosted with ClickHouse Keeper,
|
|
739
|
-
* provide both parameters or neither (to use server defaults).
|
|
740
|
-
*/
|
|
741
|
-
type ReplicatedVersionedCollapsingMergeTreeConfig<T> = Omit<VersionedCollapsingMergeTreeConfig<T>, "engine"> & ReplicatedEngineProperties & {
|
|
742
|
-
engine: ClickHouseEngines.ReplicatedVersionedCollapsingMergeTree;
|
|
743
|
-
};
|
|
744
|
-
/**
|
|
745
|
-
* Configuration for S3Queue engine - only non-alterable constructor parameters.
|
|
746
|
-
* S3Queue-specific settings like 'mode', 'keeper_path', etc. should be specified
|
|
747
|
-
* in the settings field, not here.
|
|
748
|
-
* @template T The data type of the records stored in the table.
|
|
749
|
-
*/
|
|
750
|
-
type S3QueueConfig<T> = Omit<BaseOlapConfig<T>, "settings" | "orderByFields" | "partitionBy" | "sampleByExpression"> & {
|
|
751
|
-
engine: ClickHouseEngines.S3Queue;
|
|
752
|
-
/** S3 bucket path with wildcards (e.g., 's3://bucket/data/*.json') */
|
|
753
|
-
s3Path: string;
|
|
754
|
-
/** Data format (e.g., 'JSONEachRow', 'CSV', 'Parquet') */
|
|
755
|
-
format: string;
|
|
756
|
-
/** AWS access key ID (optional, omit for NOSIGN/public buckets) */
|
|
757
|
-
awsAccessKeyId?: string;
|
|
758
|
-
/** AWS secret access key */
|
|
759
|
-
awsSecretAccessKey?: string;
|
|
760
|
-
/** Compression type (e.g., 'gzip', 'zstd') */
|
|
761
|
-
compression?: string;
|
|
762
|
-
/** Custom HTTP headers */
|
|
763
|
-
headers?: {
|
|
764
|
-
[key: string]: string;
|
|
765
|
-
};
|
|
766
|
-
/**
|
|
767
|
-
* S3Queue-specific table settings that can be modified with ALTER TABLE MODIFY SETTING.
|
|
768
|
-
* These settings control the behavior of the S3Queue engine.
|
|
769
|
-
*/
|
|
770
|
-
settings?: S3QueueTableSettings;
|
|
771
|
-
};
|
|
772
|
-
/**
|
|
773
|
-
* Configuration for S3 engine
|
|
774
|
-
* Note: S3 engine supports ORDER BY clause, unlike S3Queue, Buffer, and Distributed engines
|
|
775
|
-
* @template T The data type of the records stored in the table.
|
|
776
|
-
*/
|
|
777
|
-
type S3Config<T> = Omit<BaseOlapConfig<T>, "sampleByExpression"> & {
|
|
778
|
-
engine: ClickHouseEngines.S3;
|
|
779
|
-
/** S3 path (e.g., 's3://bucket/path/file.json') */
|
|
780
|
-
path: string;
|
|
781
|
-
/** Data format (e.g., 'JSONEachRow', 'CSV', 'Parquet') */
|
|
782
|
-
format: string;
|
|
783
|
-
/** AWS access key ID (optional, omit for NOSIGN/public buckets) */
|
|
784
|
-
awsAccessKeyId?: string;
|
|
785
|
-
/** AWS secret access key */
|
|
786
|
-
awsSecretAccessKey?: string;
|
|
787
|
-
/** Compression type (e.g., 'gzip', 'zstd', 'auto') */
|
|
788
|
-
compression?: string;
|
|
789
|
-
/** Partition strategy (optional) */
|
|
790
|
-
partitionStrategy?: string;
|
|
791
|
-
/** Partition columns in data file (optional) */
|
|
792
|
-
partitionColumnsInDataFile?: string;
|
|
793
|
-
};
|
|
794
|
-
/**
|
|
795
|
-
* Configuration for Buffer engine
|
|
796
|
-
* @template T The data type of the records stored in the table.
|
|
797
|
-
*/
|
|
798
|
-
type BufferConfig<T> = Omit<BaseOlapConfig<T>, "orderByFields" | "orderByExpression" | "partitionBy" | "sampleByExpression"> & {
|
|
799
|
-
engine: ClickHouseEngines.Buffer;
|
|
800
|
-
/** Target database name for the destination table */
|
|
801
|
-
targetDatabase: string;
|
|
802
|
-
/** Target table name where data will be flushed */
|
|
803
|
-
targetTable: string;
|
|
804
|
-
/** Number of buffer layers (typically 16) */
|
|
805
|
-
numLayers: number;
|
|
806
|
-
/** Minimum time in seconds before flushing */
|
|
807
|
-
minTime: number;
|
|
808
|
-
/** Maximum time in seconds before flushing */
|
|
809
|
-
maxTime: number;
|
|
810
|
-
/** Minimum number of rows before flushing */
|
|
811
|
-
minRows: number;
|
|
812
|
-
/** Maximum number of rows before flushing */
|
|
813
|
-
maxRows: number;
|
|
814
|
-
/** Minimum bytes before flushing */
|
|
815
|
-
minBytes: number;
|
|
816
|
-
/** Maximum bytes before flushing */
|
|
817
|
-
maxBytes: number;
|
|
818
|
-
/** Optional: Flush time in seconds */
|
|
819
|
-
flushTime?: number;
|
|
820
|
-
/** Optional: Flush number of rows */
|
|
821
|
-
flushRows?: number;
|
|
822
|
-
/** Optional: Flush number of bytes */
|
|
823
|
-
flushBytes?: number;
|
|
824
|
-
};
|
|
825
|
-
/**
|
|
826
|
-
* Configuration for Distributed engine
|
|
827
|
-
* @template T The data type of the records stored in the table.
|
|
828
|
-
*/
|
|
829
|
-
type DistributedConfig<T> = Omit<BaseOlapConfig<T>, "orderByFields" | "orderByExpression" | "partitionBy" | "sampleByExpression"> & {
|
|
830
|
-
engine: ClickHouseEngines.Distributed;
|
|
831
|
-
/** Cluster name from the ClickHouse configuration */
|
|
832
|
-
cluster: string;
|
|
833
|
-
/** Database name on the cluster */
|
|
834
|
-
targetDatabase: string;
|
|
835
|
-
/** Table name on the cluster */
|
|
836
|
-
targetTable: string;
|
|
837
|
-
/** Optional: Sharding key expression for data distribution */
|
|
838
|
-
shardingKey?: string;
|
|
839
|
-
/** Optional: Policy name for data distribution */
|
|
840
|
-
policyName?: string;
|
|
841
|
-
};
|
|
842
|
-
/** Kafka table settings. See: https://clickhouse.com/docs/engines/table-engines/integrations/kafka */
|
|
843
|
-
interface KafkaTableSettings {
|
|
844
|
-
kafka_security_protocol?: "PLAINTEXT" | "SSL" | "SASL_PLAINTEXT" | "SASL_SSL";
|
|
845
|
-
kafka_sasl_mechanism?: "GSSAPI" | "PLAIN" | "SCRAM-SHA-256" | "SCRAM-SHA-512" | "OAUTHBEARER";
|
|
846
|
-
kafka_sasl_username?: string;
|
|
847
|
-
kafka_sasl_password?: string;
|
|
848
|
-
kafka_schema?: string;
|
|
849
|
-
kafka_num_consumers?: string;
|
|
850
|
-
kafka_max_block_size?: string;
|
|
851
|
-
kafka_skip_broken_messages?: string;
|
|
852
|
-
kafka_commit_every_batch?: string;
|
|
853
|
-
kafka_client_id?: string;
|
|
854
|
-
kafka_poll_timeout_ms?: string;
|
|
855
|
-
kafka_poll_max_batch_size?: string;
|
|
856
|
-
kafka_flush_interval_ms?: string;
|
|
857
|
-
kafka_consumer_reschedule_ms?: string;
|
|
858
|
-
kafka_thread_per_consumer?: string;
|
|
859
|
-
kafka_handle_error_mode?: "default" | "stream";
|
|
860
|
-
kafka_commit_on_select?: string;
|
|
861
|
-
kafka_max_rows_per_message?: string;
|
|
862
|
-
kafka_compression_codec?: string;
|
|
863
|
-
kafka_compression_level?: string;
|
|
864
|
-
}
|
|
865
|
-
/** Kafka engine for streaming data from Kafka topics. Additional settings go in `settings`. */
|
|
866
|
-
type KafkaConfig<T> = Omit<BaseOlapConfig<T>, "orderByFields" | "orderByExpression" | "partitionBy" | "sampleByExpression"> & {
|
|
867
|
-
engine: ClickHouseEngines.Kafka;
|
|
868
|
-
brokerList: string;
|
|
869
|
-
topicList: string;
|
|
870
|
-
groupName: string;
|
|
871
|
-
format: string;
|
|
872
|
-
settings?: KafkaTableSettings;
|
|
873
|
-
};
|
|
874
|
-
/**
|
|
875
|
-
* Configuration for IcebergS3 engine - read-only Iceberg table access
|
|
876
|
-
*
|
|
877
|
-
* Provides direct querying of Apache Iceberg tables stored on S3.
|
|
878
|
-
* Data is not copied; queries stream directly from Parquet/ORC files.
|
|
879
|
-
*
|
|
880
|
-
* @template T The data type of the records stored in the table.
|
|
881
|
-
*
|
|
882
|
-
* @example
|
|
883
|
-
* ```typescript
|
|
884
|
-
* const lakeEvents = new OlapTable<Event>("lake_events", {
|
|
885
|
-
* engine: ClickHouseEngines.IcebergS3,
|
|
886
|
-
* path: "s3://datalake/events/",
|
|
887
|
-
* format: "Parquet",
|
|
888
|
-
* awsAccessKeyId: mooseRuntimeEnv.get("AWS_ACCESS_KEY_ID"),
|
|
889
|
-
* awsSecretAccessKey: mooseRuntimeEnv.get("AWS_SECRET_ACCESS_KEY")
|
|
890
|
-
* });
|
|
891
|
-
* ```
|
|
892
|
-
*
|
|
893
|
-
* @remarks
|
|
894
|
-
* - IcebergS3 engine is read-only
|
|
895
|
-
* - Does not support ORDER BY, PARTITION BY, or SAMPLE BY clauses
|
|
896
|
-
* - Queries always see the latest Iceberg snapshot (with metadata cache)
|
|
897
|
-
*/
|
|
898
|
-
type IcebergS3Config<T> = Omit<BaseOlapConfig<T>, "orderByFields" | "orderByExpression" | "partitionBy" | "sampleByExpression"> & {
|
|
899
|
-
engine: ClickHouseEngines.IcebergS3;
|
|
900
|
-
/** S3 path to Iceberg table root (e.g., 's3://bucket/warehouse/events/') */
|
|
901
|
-
path: string;
|
|
902
|
-
/** Data format - 'Parquet' or 'ORC' */
|
|
903
|
-
format: "Parquet" | "ORC";
|
|
904
|
-
/** AWS access key ID (optional, omit for NOSIGN/public buckets) */
|
|
905
|
-
awsAccessKeyId?: string;
|
|
906
|
-
/** AWS secret access key (optional) */
|
|
907
|
-
awsSecretAccessKey?: string;
|
|
908
|
-
/** Compression type (optional: 'gzip', 'zstd', 'auto') */
|
|
909
|
-
compression?: string;
|
|
910
|
-
};
|
|
911
|
-
/**
|
|
912
|
-
* Legacy configuration (backward compatibility) - defaults to MergeTree engine
|
|
913
|
-
* @template T The data type of the records stored in the table.
|
|
914
|
-
*/
|
|
915
|
-
type LegacyOlapConfig<T> = BaseOlapConfig<T>;
|
|
916
|
-
type EngineConfig<T> = MergeTreeConfig<T> | ReplacingMergeTreeConfig<T> | AggregatingMergeTreeConfig<T> | SummingMergeTreeConfig<T> | CollapsingMergeTreeConfig<T> | VersionedCollapsingMergeTreeConfig<T> | ReplicatedMergeTreeConfig<T> | ReplicatedReplacingMergeTreeConfig<T> | ReplicatedAggregatingMergeTreeConfig<T> | ReplicatedSummingMergeTreeConfig<T> | ReplicatedCollapsingMergeTreeConfig<T> | ReplicatedVersionedCollapsingMergeTreeConfig<T> | S3QueueConfig<T> | S3Config<T> | BufferConfig<T> | DistributedConfig<T> | IcebergS3Config<T> | KafkaConfig<T>;
|
|
917
|
-
/**
|
|
918
|
-
* Union of all engine-specific configurations (new API)
|
|
919
|
-
* @template T The data type of the records stored in the table.
|
|
920
|
-
*/
|
|
921
|
-
type OlapConfig<T> = EngineConfig<T> | LegacyOlapConfig<T>;
|
|
922
|
-
/**
|
|
923
|
-
* Represents an OLAP (Online Analytical Processing) table, typically corresponding to a ClickHouse table.
|
|
924
|
-
* Provides a typed interface for interacting with the table.
|
|
925
|
-
*
|
|
926
|
-
* @template T The data type of the records stored in the table. The structure of T defines the table schema.
|
|
927
|
-
*/
|
|
928
|
-
declare class OlapTable<T> extends TypedBase<T, OlapConfig<T>> {
|
|
929
|
-
name: IdentifierBrandedString;
|
|
930
|
-
/** @internal */
|
|
931
|
-
readonly kind = "OlapTable";
|
|
932
|
-
/** @internal Memoized ClickHouse client for reusing connections across insert calls */
|
|
933
|
-
private _memoizedClient?;
|
|
934
|
-
/** @internal Hash of the configuration used to create the memoized client */
|
|
935
|
-
private _configHash?;
|
|
936
|
-
/** @internal Cached table name to avoid repeated generation */
|
|
937
|
-
private _cachedTableName?;
|
|
938
|
-
/**
|
|
939
|
-
* Creates a new OlapTable instance.
|
|
940
|
-
* @param name The name of the table. This name is used for the underlying ClickHouse table.
|
|
941
|
-
* @param config Optional configuration for the OLAP table.
|
|
942
|
-
*/
|
|
943
|
-
constructor(name: string, config?: OlapConfig<T>);
|
|
944
|
-
/** @internal **/
|
|
945
|
-
constructor(name: string, config: OlapConfig<T>, schema: IJsonSchemaCollection$1.IV3_1, columns: Column[], validators?: TypiaValidators<T>);
|
|
946
|
-
/**
|
|
947
|
-
* Generates the versioned table name following Moose's naming convention
|
|
948
|
-
* Format: {tableName}_{version_with_dots_replaced_by_underscores}
|
|
949
|
-
*/
|
|
950
|
-
private generateTableName;
|
|
951
|
-
/**
|
|
952
|
-
* Creates a fast hash of the ClickHouse configuration.
|
|
953
|
-
* Uses crypto.createHash for better performance than JSON.stringify.
|
|
954
|
-
*
|
|
955
|
-
* @private
|
|
956
|
-
*/
|
|
957
|
-
private createConfigHash;
|
|
958
|
-
/**
|
|
959
|
-
* Gets or creates a memoized ClickHouse client.
|
|
960
|
-
* The client is cached and reused across multiple insert calls for better performance.
|
|
961
|
-
* If the configuration changes, a new client will be created.
|
|
962
|
-
*
|
|
963
|
-
* @private
|
|
964
|
-
*/
|
|
965
|
-
private getMemoizedClient;
|
|
966
|
-
/**
|
|
967
|
-
* Closes the memoized ClickHouse client if it exists.
|
|
968
|
-
* This is useful for cleaning up connections when the table instance is no longer needed.
|
|
969
|
-
* The client will be automatically recreated on the next insert call if needed.
|
|
970
|
-
*/
|
|
971
|
-
closeClient(): Promise<void>;
|
|
972
|
-
/**
|
|
973
|
-
* Validates a single record using typia's comprehensive type checking.
|
|
974
|
-
* This provides the most accurate validation as it uses the exact TypeScript type information.
|
|
975
|
-
*
|
|
976
|
-
* @param record The record to validate
|
|
977
|
-
* @returns Validation result with detailed error information
|
|
978
|
-
*/
|
|
979
|
-
validateRecord(record: unknown): {
|
|
980
|
-
success: boolean;
|
|
981
|
-
data?: T;
|
|
982
|
-
errors?: string[];
|
|
983
|
-
};
|
|
984
|
-
/**
|
|
985
|
-
* Type guard function using typia's is() function.
|
|
986
|
-
* Provides compile-time type narrowing for TypeScript.
|
|
987
|
-
*
|
|
988
|
-
* @param record The record to check
|
|
989
|
-
* @returns True if record matches type T, with type narrowing
|
|
990
|
-
*/
|
|
991
|
-
isValidRecord(record: unknown): record is T;
|
|
992
|
-
/**
|
|
993
|
-
* Assert that a record matches type T, throwing detailed errors if not.
|
|
994
|
-
* Uses typia's assert() function for the most detailed error reporting.
|
|
995
|
-
*
|
|
996
|
-
* @param record The record to assert
|
|
997
|
-
* @returns The validated and typed record
|
|
998
|
-
* @throws Detailed validation error if record doesn't match type T
|
|
999
|
-
*/
|
|
1000
|
-
assertValidRecord(record: unknown): T;
|
|
1001
|
-
/**
|
|
1002
|
-
* Validates an array of records with comprehensive error reporting.
|
|
1003
|
-
* Uses the most appropriate validation method available (typia or basic).
|
|
1004
|
-
*
|
|
1005
|
-
* @param data Array of records to validate
|
|
1006
|
-
* @returns Detailed validation results
|
|
1007
|
-
*/
|
|
1008
|
-
validateRecords(data: unknown[]): Promise<ValidationResult<T>>;
|
|
1009
|
-
/**
|
|
1010
|
-
* Optimized batch retry that minimizes individual insert operations.
|
|
1011
|
-
* Groups records into smaller batches to reduce round trips while still isolating failures.
|
|
1012
|
-
*
|
|
1013
|
-
* @private
|
|
1014
|
-
*/
|
|
1015
|
-
private retryIndividualRecords;
|
|
1016
|
-
/**
|
|
1017
|
-
* Validates input parameters and strategy compatibility
|
|
1018
|
-
* @private
|
|
1019
|
-
*/
|
|
1020
|
-
private validateInsertParameters;
|
|
1021
|
-
/**
|
|
1022
|
-
* Handles early return cases for empty data
|
|
1023
|
-
* @private
|
|
1024
|
-
*/
|
|
1025
|
-
private handleEmptyData;
|
|
1026
|
-
/**
|
|
1027
|
-
* Performs pre-insertion validation for array data
|
|
1028
|
-
* @private
|
|
1029
|
-
*/
|
|
1030
|
-
private performPreInsertionValidation;
|
|
1031
|
-
/**
|
|
1032
|
-
* Handles validation errors based on the specified strategy
|
|
1033
|
-
* @private
|
|
1034
|
-
*/
|
|
1035
|
-
private handleValidationErrors;
|
|
1036
|
-
/**
|
|
1037
|
-
* Checks if validation errors exceed configured thresholds
|
|
1038
|
-
* @private
|
|
1039
|
-
*/
|
|
1040
|
-
private checkValidationThresholds;
|
|
1041
|
-
/**
|
|
1042
|
-
* Optimized insert options preparation with better memory management
|
|
1043
|
-
* @private
|
|
1044
|
-
*/
|
|
1045
|
-
private prepareInsertOptions;
|
|
1046
|
-
/**
|
|
1047
|
-
* Creates success result for completed insertions
|
|
1048
|
-
* @private
|
|
1049
|
-
*/
|
|
1050
|
-
private createSuccessResult;
|
|
1051
|
-
/**
|
|
1052
|
-
* Handles insertion errors based on the specified strategy
|
|
1053
|
-
* @private
|
|
1054
|
-
*/
|
|
1055
|
-
private handleInsertionError;
|
|
1056
|
-
/**
|
|
1057
|
-
* Handles the isolate strategy for insertion errors
|
|
1058
|
-
* @private
|
|
1059
|
-
*/
|
|
1060
|
-
private handleIsolateStrategy;
|
|
1061
|
-
/**
|
|
1062
|
-
* Checks if insertion errors exceed configured thresholds
|
|
1063
|
-
* @private
|
|
1064
|
-
*/
|
|
1065
|
-
private checkInsertionThresholds;
|
|
1066
|
-
/**
|
|
1067
|
-
* Recursively transforms a record to match ClickHouse's JSONEachRow requirements
|
|
1068
|
-
*
|
|
1069
|
-
* - For every Array(Nested(...)) field at any depth, each item is wrapped in its own array and recursively processed.
|
|
1070
|
-
* - For every Nested struct (not array), it recurses into the struct.
|
|
1071
|
-
* - This ensures compatibility with kafka_clickhouse_sync
|
|
1072
|
-
*
|
|
1073
|
-
* @param record The input record to transform (may be deeply nested)
|
|
1074
|
-
* @param columns The schema columns for this level (defaults to this.columnArray at the top level)
|
|
1075
|
-
* @returns The transformed record, ready for ClickHouse JSONEachRow insertion
|
|
1076
|
-
*/
|
|
1077
|
-
private mapToClickhouseRecord;
|
|
1078
|
-
/**
|
|
1079
|
-
* Inserts data directly into the ClickHouse table with enhanced error handling and validation.
|
|
1080
|
-
* This method establishes a direct connection to ClickHouse using the project configuration
|
|
1081
|
-
* and inserts the provided data into the versioned table.
|
|
1082
|
-
*
|
|
1083
|
-
* PERFORMANCE OPTIMIZATIONS:
|
|
1084
|
-
* - Memoized client connections with fast config hashing
|
|
1085
|
-
* - Single-pass validation with pre-allocated arrays
|
|
1086
|
-
* - Batch-optimized retry strategy (batches of 10, then individual)
|
|
1087
|
-
* - Optimized ClickHouse settings for large datasets
|
|
1088
|
-
* - Reduced memory allocations and object creation
|
|
1089
|
-
*
|
|
1090
|
-
* Uses advanced typia validation when available for comprehensive type checking,
|
|
1091
|
-
* with fallback to basic validation for compatibility.
|
|
1092
|
-
*
|
|
1093
|
-
* The ClickHouse client is memoized and reused across multiple insert calls for better performance.
|
|
1094
|
-
* If the configuration changes, a new client will be automatically created.
|
|
1095
|
-
*
|
|
1096
|
-
* @param data Array of objects conforming to the table schema, or a Node.js Readable stream
|
|
1097
|
-
* @param options Optional configuration for error handling, validation, and insertion behavior
|
|
1098
|
-
* @returns Promise resolving to detailed insertion results
|
|
1099
|
-
* @throws {ConfigError} When configuration cannot be read or parsed
|
|
1100
|
-
* @throws {ClickHouseError} When insertion fails based on the error strategy
|
|
1101
|
-
* @throws {ValidationError} When validation fails and strategy is 'fail-fast'
|
|
1102
|
-
*
|
|
1103
|
-
* @example
|
|
1104
|
-
* ```typescript
|
|
1105
|
-
* // Create an OlapTable instance (typia validators auto-injected)
|
|
1106
|
-
* const userTable = new OlapTable<User>('users');
|
|
1107
|
-
*
|
|
1108
|
-
* // Insert with comprehensive typia validation
|
|
1109
|
-
* const result1 = await userTable.insert([
|
|
1110
|
-
* { id: 1, name: 'John', email: 'john@example.com' },
|
|
1111
|
-
* { id: 2, name: 'Jane', email: 'jane@example.com' }
|
|
1112
|
-
* ]);
|
|
1113
|
-
*
|
|
1114
|
-
* // Insert data with stream input (validation not available for streams)
|
|
1115
|
-
* const dataStream = new Readable({
|
|
1116
|
-
* objectMode: true,
|
|
1117
|
-
* read() { // Stream implementation }
|
|
1118
|
-
* });
|
|
1119
|
-
* const result2 = await userTable.insert(dataStream, { strategy: 'fail-fast' });
|
|
1120
|
-
*
|
|
1121
|
-
* // Insert with validation disabled for performance
|
|
1122
|
-
* const result3 = await userTable.insert(data, { validate: false });
|
|
1123
|
-
*
|
|
1124
|
-
* // Insert with error handling strategies
|
|
1125
|
-
* const result4 = await userTable.insert(mixedData, {
|
|
1126
|
-
* strategy: 'isolate',
|
|
1127
|
-
* allowErrorsRatio: 0.1,
|
|
1128
|
-
* validate: true // Use typia validation (default)
|
|
1129
|
-
* });
|
|
1130
|
-
*
|
|
1131
|
-
* // Optional: Clean up connection when completely done
|
|
1132
|
-
* await userTable.closeClient();
|
|
1133
|
-
* ```
|
|
1134
|
-
*/
|
|
1135
|
-
insert(data: T[] | Readable, options?: InsertOptions): Promise<InsertResult<T>>;
|
|
1136
|
-
}
|
|
1137
|
-
|
|
1138
|
-
/**
|
|
1139
|
-
* @fileoverview Stream SDK for data streaming operations in Moose.
|
|
1140
|
-
*
|
|
1141
|
-
* This module provides the core streaming functionality including:
|
|
1142
|
-
* - Stream creation and configuration
|
|
1143
|
-
* - Message transformations between streams
|
|
1144
|
-
* - Consumer registration for message processing
|
|
1145
|
-
* - Dead letter queue handling for error recovery
|
|
1146
|
-
*
|
|
1147
|
-
* @module Stream
|
|
1148
|
-
*/
|
|
1149
|
-
|
|
1150
|
-
/**
|
|
1151
|
-
* Represents zero, one, or many values of type T.
|
|
1152
|
-
* Used for flexible return types in transformations where a single input
|
|
1153
|
-
* can produce no output, one output, or multiple outputs.
|
|
1154
|
-
*
|
|
1155
|
-
* @template T The type of the value(s)
|
|
1156
|
-
* @example
|
|
1157
|
-
* ```typescript
|
|
1158
|
-
* // Can return a single value
|
|
1159
|
-
* const single: ZeroOrMany<string> = "hello";
|
|
1160
|
-
*
|
|
1161
|
-
* // Can return an array
|
|
1162
|
-
* const multiple: ZeroOrMany<string> = ["hello", "world"];
|
|
1163
|
-
*
|
|
1164
|
-
* // Can return null/undefined to filter out
|
|
1165
|
-
* const filtered: ZeroOrMany<string> = null;
|
|
1166
|
-
* ```
|
|
1167
|
-
*/
|
|
1168
|
-
type ZeroOrMany<T> = T | T[] | undefined | null;
|
|
1169
|
-
/**
|
|
1170
|
-
* Function type for transforming records from one type to another.
|
|
1171
|
-
* Supports both synchronous and asynchronous transformations.
|
|
1172
|
-
*
|
|
1173
|
-
* @template T The input record type
|
|
1174
|
-
* @template U The output record type
|
|
1175
|
-
* @param record The input record to transform
|
|
1176
|
-
* @returns The transformed record(s), or null/undefined to filter out
|
|
1177
|
-
*
|
|
1178
|
-
* @example
|
|
1179
|
-
* ```typescript
|
|
1180
|
-
* const transform: SyncOrAsyncTransform<InputType, OutputType> = (record) => {
|
|
1181
|
-
* return { ...record, processed: true };
|
|
1182
|
-
* };
|
|
1183
|
-
* ```
|
|
1184
|
-
*/
|
|
1185
|
-
type SyncOrAsyncTransform<T, U> = (record: T) => ZeroOrMany<U> | Promise<ZeroOrMany<U>>;
|
|
1186
|
-
/**
|
|
1187
|
-
* Function type for consuming records without producing output.
|
|
1188
|
-
* Used for side effects like logging, external API calls, or database writes.
|
|
1189
|
-
*
|
|
1190
|
-
* @template T The record type to consume
|
|
1191
|
-
* @param record The record to process
|
|
1192
|
-
* @returns Promise<void> or void
|
|
1193
|
-
*
|
|
1194
|
-
* @example
|
|
1195
|
-
* ```typescript
|
|
1196
|
-
* const consumer: Consumer<UserEvent> = async (event) => {
|
|
1197
|
-
* await sendToAnalytics(event);
|
|
1198
|
-
* };
|
|
1199
|
-
* ```
|
|
1200
|
-
*/
|
|
1201
|
-
type Consumer<T> = (record: T) => Promise<void> | void;
|
|
1202
|
-
/**
|
|
1203
|
-
* Configuration options for stream transformations.
|
|
1204
|
-
*
|
|
1205
|
-
* @template T The type of records being transformed
|
|
1206
|
-
*/
|
|
1207
|
-
interface TransformConfig<T> {
|
|
1208
|
-
/**
|
|
1209
|
-
* Optional version identifier for this transformation.
|
|
1210
|
-
* Multiple transformations to the same destination can coexist with different versions.
|
|
1211
|
-
*/
|
|
1212
|
-
version?: string;
|
|
1213
|
-
/**
|
|
1214
|
-
* Optional metadata for documentation and tracking purposes.
|
|
1215
|
-
*/
|
|
1216
|
-
metadata?: {
|
|
1217
|
-
description?: string;
|
|
1218
|
-
};
|
|
1219
|
-
/**
|
|
1220
|
-
* Optional dead letter queue for handling transformation failures.
|
|
1221
|
-
* Failed records will be sent to this queue for manual inspection or reprocessing.
|
|
1222
|
-
* Uses {@link Stream.defaultDeadLetterQueue} by default
|
|
1223
|
-
* unless a DeadLetterQueue is provided, or it is explicitly disabled with a null value
|
|
1224
|
-
*/
|
|
1225
|
-
deadLetterQueue?: DeadLetterQueue<T> | null;
|
|
1226
|
-
/**
|
|
1227
|
-
* @internal Source file path where this transform was declared.
|
|
1228
|
-
* Automatically captured from stack trace.
|
|
1229
|
-
*/
|
|
1230
|
-
sourceFile?: string;
|
|
1231
|
-
}
|
|
1232
|
-
/**
|
|
1233
|
-
* Configuration options for stream consumers.
|
|
1234
|
-
*
|
|
1235
|
-
* @template T The type of records being consumed
|
|
1236
|
-
*/
|
|
1237
|
-
interface ConsumerConfig<T> {
|
|
1238
|
-
/**
|
|
1239
|
-
* Optional version identifier for this consumer.
|
|
1240
|
-
* Multiple consumers can coexist with different versions.
|
|
1241
|
-
*/
|
|
1242
|
-
version?: string;
|
|
1243
|
-
/**
|
|
1244
|
-
* Optional dead letter queue for handling consumer failures.
|
|
1245
|
-
* Failed records will be sent to this queue for manual inspection or reprocessing.
|
|
1246
|
-
* Uses {@link Stream.defaultDeadLetterQueue} by default
|
|
1247
|
-
* unless a DeadLetterQueue is provided, or it is explicitly disabled with a null value
|
|
1248
|
-
*/
|
|
1249
|
-
deadLetterQueue?: DeadLetterQueue<T> | null;
|
|
1250
|
-
/**
|
|
1251
|
-
* @internal Source file path where this consumer was declared.
|
|
1252
|
-
* Automatically captured from stack trace.
|
|
1253
|
-
*/
|
|
1254
|
-
sourceFile?: string;
|
|
1255
|
-
}
|
|
1256
|
-
type SchemaRegistryEncoding = "JSON" | "AVRO" | "PROTOBUF";
|
|
1257
|
-
type SchemaRegistryReference = {
|
|
1258
|
-
id: number;
|
|
1259
|
-
} | {
|
|
1260
|
-
subjectLatest: string;
|
|
1261
|
-
} | {
|
|
1262
|
-
subject: string;
|
|
1263
|
-
version: number;
|
|
1264
|
-
};
|
|
1265
|
-
interface KafkaSchemaConfig {
|
|
1266
|
-
kind: SchemaRegistryEncoding;
|
|
1267
|
-
reference: SchemaRegistryReference;
|
|
1268
|
-
}
|
|
1269
|
-
/**
|
|
1270
|
-
* Represents a message routed to a specific destination stream.
|
|
1271
|
-
* Used internally by the multi-transform functionality to specify
|
|
1272
|
-
* where transformed messages should be sent.
|
|
1273
|
-
*
|
|
1274
|
-
* @internal
|
|
1275
|
-
*/
|
|
1276
|
-
declare class RoutedMessage {
|
|
1277
|
-
/** The destination stream for the message */
|
|
1278
|
-
destination: Stream<any>;
|
|
1279
|
-
/** The message value(s) to send */
|
|
1280
|
-
values: ZeroOrMany<any>;
|
|
1281
|
-
/**
|
|
1282
|
-
* Creates a new routed message.
|
|
1283
|
-
*
|
|
1284
|
-
* @param destination The target stream
|
|
1285
|
-
* @param values The message(s) to route
|
|
1286
|
-
*/
|
|
1287
|
-
constructor(destination: Stream<any>, values: ZeroOrMany<any>);
|
|
1288
|
-
}
|
|
1289
|
-
/**
|
|
1290
|
-
* Configuration options for a data stream (e.g., a Redpanda topic).
|
|
1291
|
-
* @template T The data type of the messages in the stream.
|
|
1292
|
-
*/
|
|
1293
|
-
interface StreamConfig<T> {
|
|
1294
|
-
/**
|
|
1295
|
-
* Specifies the number of partitions for the stream. Affects parallelism and throughput.
|
|
1296
|
-
*/
|
|
1297
|
-
parallelism?: number;
|
|
1298
|
-
/**
|
|
1299
|
-
* Specifies the data retention period for the stream in seconds. Messages older than this may be deleted.
|
|
1300
|
-
*/
|
|
1301
|
-
retentionPeriod?: number;
|
|
1302
|
-
/**
|
|
1303
|
-
* An optional destination OLAP table where messages from this stream should be automatically ingested.
|
|
1304
|
-
*/
|
|
1305
|
-
destination?: OlapTable<T>;
|
|
1306
|
-
/**
|
|
1307
|
-
* An optional version string for this configuration. Can be used for tracking changes or managing deployments.
|
|
1308
|
-
*/
|
|
1309
|
-
version?: string;
|
|
1310
|
-
metadata?: {
|
|
1311
|
-
description?: string;
|
|
1312
|
-
};
|
|
1313
|
-
lifeCycle?: LifeCycle;
|
|
1314
|
-
defaultDeadLetterQueue?: DeadLetterQueue<T>;
|
|
1315
|
-
/** Optional Schema Registry configuration for this stream */
|
|
1316
|
-
schemaConfig?: KafkaSchemaConfig;
|
|
1317
|
-
}
|
|
1318
|
-
/**
|
|
1319
|
-
* Represents a data stream, typically corresponding to a Redpanda topic.
|
|
1320
|
-
* Provides a typed interface for producing to and consuming from the stream, and defining transformations.
|
|
1321
|
-
*
|
|
1322
|
-
* @template T The data type of the messages flowing through the stream. The structure of T defines the message schema.
|
|
1323
|
-
*/
|
|
1324
|
-
declare class Stream<T> extends TypedBase<T, StreamConfig<T>> {
|
|
1325
|
-
defaultDeadLetterQueue?: DeadLetterQueue<T>;
|
|
1326
|
-
/** @internal Memoized KafkaJS producer for reusing connections across sends */
|
|
1327
|
-
private _memoizedProducer?;
|
|
1328
|
-
/** @internal Hash of the configuration used to create the memoized Kafka producer */
|
|
1329
|
-
private _kafkaConfigHash?;
|
|
1330
|
-
/**
|
|
1331
|
-
* Creates a new Stream instance.
|
|
1332
|
-
* @param name The name of the stream. This name is used for the underlying Redpanda topic.
|
|
1333
|
-
* @param config Optional configuration for the stream.
|
|
1334
|
-
*/
|
|
1335
|
-
constructor(name: string, config?: StreamConfig<T>);
|
|
1336
|
-
/**
|
|
1337
|
-
* @internal
|
|
1338
|
-
* Note: `validators` parameter is a positional placeholder (always undefined for Stream).
|
|
1339
|
-
* It exists because TypedBase has validators as the 5th param, and we need to pass
|
|
1340
|
-
* allowExtraFields as the 6th param. Stream doesn't use validators.
|
|
1341
|
-
*/
|
|
1342
|
-
constructor(name: string, config: StreamConfig<T>, schema: IJsonSchemaCollection$1.IV3_1, columns: Column[], validators: undefined, allowExtraFields: boolean);
|
|
1343
|
-
/**
|
|
1344
|
-
* Internal map storing transformation configurations.
|
|
1345
|
-
* Maps destination stream names to arrays of transformation functions and their configs.
|
|
1346
|
-
*
|
|
1347
|
-
* @internal
|
|
1348
|
-
*/
|
|
1349
|
-
_transformations: Map<string, [Stream<any>, SyncOrAsyncTransform<T, any>, TransformConfig<T>][]>;
|
|
1350
|
-
/**
|
|
1351
|
-
* Internal function for multi-stream transformations.
|
|
1352
|
-
* Allows a single transformation to route messages to multiple destinations.
|
|
1353
|
-
*
|
|
1354
|
-
* @internal
|
|
1355
|
-
*/
|
|
1356
|
-
_multipleTransformations?: (record: T) => [RoutedMessage];
|
|
1357
|
-
/**
|
|
1358
|
-
* Internal array storing consumer configurations.
|
|
1359
|
-
*
|
|
1360
|
-
* @internal
|
|
1361
|
-
*/
|
|
1362
|
-
_consumers: {
|
|
1363
|
-
consumer: Consumer<T>;
|
|
1364
|
-
config: ConsumerConfig<T>;
|
|
1365
|
-
}[];
|
|
1366
|
-
/**
|
|
1367
|
-
* Builds the full Kafka topic name including optional namespace and version suffix.
|
|
1368
|
-
* Version suffix is appended as _x_y_z where dots in version are replaced with underscores.
|
|
1369
|
-
*/
|
|
1370
|
-
private buildFullTopicName;
|
|
1371
|
-
/**
|
|
1372
|
-
* Creates a fast hash string from relevant Kafka configuration fields.
|
|
1373
|
-
*/
|
|
1374
|
-
private createConfigHash;
|
|
1375
|
-
/**
|
|
1376
|
-
* Gets or creates a memoized KafkaJS producer using runtime configuration.
|
|
1377
|
-
*/
|
|
1378
|
-
private getMemoizedProducer;
|
|
1379
|
-
/**
|
|
1380
|
-
* Closes the memoized Kafka producer if it exists.
|
|
1381
|
-
*/
|
|
1382
|
-
closeProducer(): Promise<void>;
|
|
1383
|
-
/**
|
|
1384
|
-
* Sends one or more records to this stream's Kafka topic.
|
|
1385
|
-
* Values are JSON-serialized as message values.
|
|
1386
|
-
*/
|
|
1387
|
-
send(values: ZeroOrMany<T>): Promise<void>;
|
|
1388
|
-
/**
|
|
1389
|
-
* Adds a transformation step that processes messages from this stream and sends the results to a destination stream.
|
|
1390
|
-
* Multiple transformations to the same destination stream can be added if they have distinct `version` identifiers in their config.
|
|
1391
|
-
*
|
|
1392
|
-
* @template U The data type of the messages in the destination stream.
|
|
1393
|
-
* @param destination The destination stream for the transformed messages.
|
|
1394
|
-
* @param transformation A function that takes a message of type T and returns zero or more messages of type U (or a Promise thereof).
|
|
1395
|
-
* Return `null` or `undefined` or an empty array `[]` to filter out a message. Return an array to emit multiple messages.
|
|
1396
|
-
* @param config Optional configuration for this specific transformation step, like a version.
|
|
1397
|
-
*/
|
|
1398
|
-
addTransform<U>(destination: Stream<U>, transformation: SyncOrAsyncTransform<T, U>, config?: TransformConfig<T>): void;
|
|
1399
|
-
/**
|
|
1400
|
-
* Adds a consumer function that processes messages from this stream.
|
|
1401
|
-
* Multiple consumers can be added if they have distinct `version` identifiers in their config.
|
|
1402
|
-
*
|
|
1403
|
-
* @param consumer A function that takes a message of type T and performs an action (e.g., side effect, logging). Should return void or Promise<void>.
|
|
1404
|
-
* @param config Optional configuration for this specific consumer, like a version.
|
|
1405
|
-
*/
|
|
1406
|
-
addConsumer(consumer: Consumer<T>, config?: ConsumerConfig<T>): void;
|
|
1407
|
-
/**
|
|
1408
|
-
* Helper method for `addMultiTransform` to specify the destination and values for a routed message.
|
|
1409
|
-
* @param values The value or values to send to this stream.
|
|
1410
|
-
* @returns A `RoutedMessage` object associating the values with this stream.
|
|
1411
|
-
*
|
|
1412
|
-
* @example
|
|
1413
|
-
* ```typescript
|
|
1414
|
-
* sourceStream.addMultiTransform((record) => [
|
|
1415
|
-
* destinationStream1.routed(transformedRecord1),
|
|
1416
|
-
* destinationStream2.routed([record2a, record2b])
|
|
1417
|
-
* ]);
|
|
1418
|
-
* ```
|
|
1419
|
-
*/
|
|
1420
|
-
routed: (values: ZeroOrMany<T>) => RoutedMessage;
|
|
1421
|
-
/**
|
|
1422
|
-
* Adds a single transformation function that can route messages to multiple destination streams.
|
|
1423
|
-
* This is an alternative to adding multiple individual `addTransform` calls.
|
|
1424
|
-
* Only one multi-transform function can be added per stream.
|
|
1425
|
-
*
|
|
1426
|
-
* @param transformation A function that takes a message of type T and returns an array of `RoutedMessage` objects,
|
|
1427
|
-
* each specifying a destination stream and the message(s) to send to it.
|
|
1428
|
-
*/
|
|
1429
|
-
addMultiTransform(transformation: (record: T) => [RoutedMessage]): void;
|
|
1430
|
-
}
|
|
1431
|
-
/**
|
|
1432
|
-
* Base model for dead letter queue entries.
|
|
1433
|
-
* Contains the original failed record along with error information.
|
|
1434
|
-
*/
|
|
1435
|
-
interface DeadLetterModel {
|
|
1436
|
-
/** The original record that failed processing */
|
|
1437
|
-
originalRecord: Record<string, any>;
|
|
1438
|
-
/** Human-readable error message describing the failure */
|
|
1439
|
-
errorMessage: string;
|
|
1440
|
-
/** Classification of the error type (e.g., "ValidationError", "TransformError") */
|
|
1441
|
-
errorType: string;
|
|
1442
|
-
/** Timestamp when the failure occurred */
|
|
1443
|
-
failedAt: Date;
|
|
1444
|
-
/** The source component where the failure occurred */
|
|
1445
|
-
source: "api" | "transform" | "table";
|
|
1446
|
-
}
|
|
1447
|
-
/**
|
|
1448
|
-
* Enhanced dead letter model with type recovery functionality.
|
|
1449
|
-
* Extends the base model with the ability to recover the original typed record.
|
|
1450
|
-
*
|
|
1451
|
-
* @template T The original record type before failure
|
|
1452
|
-
*/
|
|
1453
|
-
interface DeadLetter<T> extends DeadLetterModel {
|
|
1454
|
-
/**
|
|
1455
|
-
* Recovers the original record as its typed form.
|
|
1456
|
-
* Useful for reprocessing failed records with proper type safety.
|
|
1457
|
-
*
|
|
1458
|
-
* @returns The original record cast to type T
|
|
1459
|
-
*/
|
|
1460
|
-
asTyped: () => T;
|
|
1461
|
-
}
|
|
1462
|
-
/**
|
|
1463
|
-
* Specialized stream for handling failed records (dead letters).
|
|
1464
|
-
* Provides type-safe access to failed records for reprocessing or analysis.
|
|
1465
|
-
*
|
|
1466
|
-
* @template T The original record type that failed processing
|
|
1467
|
-
*
|
|
1468
|
-
* @example
|
|
1469
|
-
* ```typescript
|
|
1470
|
-
* const dlq = new DeadLetterQueue<UserEvent>("user-events-dlq");
|
|
1471
|
-
*
|
|
1472
|
-
* dlq.addConsumer(async (deadLetter) => {
|
|
1473
|
-
* const originalEvent = deadLetter.asTyped();
|
|
1474
|
-
* console.log(`Failed event: ${deadLetter.errorMessage}`);
|
|
1475
|
-
* // Potentially reprocess or alert
|
|
1476
|
-
* });
|
|
1477
|
-
* ```
|
|
1478
|
-
*/
|
|
1479
|
-
declare class DeadLetterQueue<T> extends Stream<DeadLetterModel> {
|
|
1480
|
-
/**
|
|
1481
|
-
* Creates a new DeadLetterQueue instance.
|
|
1482
|
-
* @param name The name of the dead letter queue stream
|
|
1483
|
-
* @param config Optional configuration for the stream. The metadata property is always present and includes stackTrace.
|
|
1484
|
-
*/
|
|
1485
|
-
constructor(name: string, config?: StreamConfig<DeadLetterModel>);
|
|
1486
|
-
/** @internal **/
|
|
1487
|
-
constructor(name: string, config: StreamConfig<DeadLetterModel>, validate: (originalRecord: any) => T);
|
|
1488
|
-
/**
|
|
1489
|
-
* Internal type guard function for validating and casting original records.
|
|
1490
|
-
*
|
|
1491
|
-
* @internal
|
|
1492
|
-
*/
|
|
1493
|
-
private typeGuard;
|
|
1494
|
-
/**
|
|
1495
|
-
* Adds a transformation step for dead letter records.
|
|
1496
|
-
* The transformation function receives a DeadLetter<T> with type recovery capabilities.
|
|
1497
|
-
*
|
|
1498
|
-
* @template U The output type for the transformation
|
|
1499
|
-
* @param destination The destination stream for transformed messages
|
|
1500
|
-
* @param transformation Function to transform dead letter records
|
|
1501
|
-
* @param config Optional transformation configuration
|
|
1502
|
-
*/
|
|
1503
|
-
addTransform<U>(destination: Stream<U>, transformation: SyncOrAsyncTransform<DeadLetter<T>, U>, config?: TransformConfig<DeadLetterModel>): void;
|
|
1504
|
-
/**
|
|
1505
|
-
* Adds a consumer for dead letter records.
|
|
1506
|
-
* The consumer function receives a DeadLetter<T> with type recovery capabilities.
|
|
1507
|
-
*
|
|
1508
|
-
* @param consumer Function to process dead letter records
|
|
1509
|
-
* @param config Optional consumer configuration
|
|
1510
|
-
*/
|
|
1511
|
-
addConsumer(consumer: Consumer<DeadLetter<T>>, config?: ConsumerConfig<DeadLetterModel>): void;
|
|
1512
|
-
/**
|
|
1513
|
-
* Adds a multi-stream transformation for dead letter records.
|
|
1514
|
-
* The transformation function receives a DeadLetter<T> with type recovery capabilities.
|
|
1515
|
-
*
|
|
1516
|
-
* @param transformation Function to route dead letter records to multiple destinations
|
|
1517
|
-
*/
|
|
1518
|
-
addMultiTransform(transformation: (record: DeadLetter<T>) => [RoutedMessage]): void;
|
|
1519
|
-
}
|
|
1520
|
-
|
|
1521
|
-
/**
|
|
1522
|
-
* Context passed to task handlers. Single param to future-proof API changes.
|
|
1523
|
-
*
|
|
1524
|
-
* - state: shared mutable state for the task and its lifecycle hooks
|
|
1525
|
-
* - input: optional typed input for the task (undefined when task has no input)
|
|
1526
|
-
*/
|
|
1527
|
-
/**
|
|
1528
|
-
* Task handler context. If the task declares an input type (T != null),
|
|
1529
|
-
* `input` is required and strongly typed. For no-input tasks (T = null),
|
|
1530
|
-
* `input` is omitted/optional.
|
|
1531
|
-
*/
|
|
1532
|
-
type TaskContext<TInput> = TInput extends null ? {
|
|
1533
|
-
state: any;
|
|
1534
|
-
input?: null;
|
|
1535
|
-
} : {
|
|
1536
|
-
state: any;
|
|
1537
|
-
input: TInput;
|
|
1538
|
-
};
|
|
1539
|
-
/**
|
|
1540
|
-
* Configuration options for defining a task within a workflow.
|
|
1541
|
-
*
|
|
1542
|
-
* @template T - The input type for the task
|
|
1543
|
-
* @template R - The return type for the task
|
|
1544
|
-
*/
|
|
1545
|
-
interface TaskConfig<T, R> {
|
|
1546
|
-
/** The main function that executes the task logic */
|
|
1547
|
-
run: (context: TaskContext<T>) => Promise<R>;
|
|
1548
|
-
/**
|
|
1549
|
-
* Optional array of tasks to execute after this task completes successfully.
|
|
1550
|
-
* Supports all combinations of input types (real type or null) and output types (real type or void).
|
|
1551
|
-
* When this task returns void, onComplete tasks expect null as input.
|
|
1552
|
-
* When this task returns a real type, onComplete tasks expect that type as input.
|
|
1553
|
-
*/
|
|
1554
|
-
onComplete?: (Task<R extends void ? null : R, any> | Task<R extends void ? null : R, void>)[];
|
|
1555
|
-
/**
|
|
1556
|
-
* Optional function that is called when the task is cancelled.
|
|
1557
|
-
*/
|
|
1558
|
-
/** Optional function that is called when the task is cancelled. */
|
|
1559
|
-
onCancel?: (context: TaskContext<T>) => Promise<void>;
|
|
1560
|
-
/** Optional timeout duration for the task execution (e.g., "30s", "5m") */
|
|
1561
|
-
timeout?: string;
|
|
1562
|
-
/** Optional number of retry attempts if the task fails */
|
|
1563
|
-
retries?: number;
|
|
1564
|
-
}
|
|
1565
|
-
/**
|
|
1566
|
-
* Represents a single task within a workflow system.
|
|
1567
|
-
*
|
|
1568
|
-
* A Task encapsulates the execution logic, completion handlers, and configuration
|
|
1569
|
-
* for a unit of work that can be chained with other tasks in a workflow.
|
|
1570
|
-
*
|
|
1571
|
-
* @template T - The input type that this task expects
|
|
1572
|
-
* @template R - The return type that this task produces
|
|
1573
|
-
*/
|
|
1574
|
-
declare class Task<T, R> {
|
|
1575
|
-
readonly name: string;
|
|
1576
|
-
readonly config: TaskConfig<T, R>;
|
|
1577
|
-
/**
|
|
1578
|
-
* Creates a new Task instance.
|
|
1579
|
-
*
|
|
1580
|
-
* @param name - Unique identifier for the task
|
|
1581
|
-
* @param config - Configuration object defining the task behavior
|
|
1582
|
-
*
|
|
1583
|
-
* @example
|
|
1584
|
-
* ```typescript
|
|
1585
|
-
* // No input, no output
|
|
1586
|
-
* const task1 = new Task<null, void>("task1", {
|
|
1587
|
-
* run: async () => {
|
|
1588
|
-
* console.log("No input/output");
|
|
1589
|
-
* }
|
|
1590
|
-
* });
|
|
1591
|
-
*
|
|
1592
|
-
* // No input, but has output
|
|
1593
|
-
* const task2 = new Task<null, OutputType>("task2", {
|
|
1594
|
-
* run: async () => {
|
|
1595
|
-
* return someOutput;
|
|
1596
|
-
* }
|
|
1597
|
-
* });
|
|
1598
|
-
*
|
|
1599
|
-
* // Has input, no output
|
|
1600
|
-
* const task3 = new Task<InputType, void>("task3", {
|
|
1601
|
-
* run: async (input: InputType) => {
|
|
1602
|
-
* // process input but return nothing
|
|
1603
|
-
* }
|
|
1604
|
-
* });
|
|
1605
|
-
*
|
|
1606
|
-
* // Has both input and output
|
|
1607
|
-
* const task4 = new Task<InputType, OutputType>("task4", {
|
|
1608
|
-
* run: async (input: InputType) => {
|
|
1609
|
-
* return process(input);
|
|
1610
|
-
* }
|
|
1611
|
-
* });
|
|
1612
|
-
* ```
|
|
1613
|
-
*/
|
|
1614
|
-
constructor(name: string, config: TaskConfig<T, R>);
|
|
1615
|
-
}
|
|
1616
|
-
/**
|
|
1617
|
-
* Configuration options for defining a workflow.
|
|
1618
|
-
*
|
|
1619
|
-
* A workflow orchestrates the execution of multiple tasks in a defined sequence
|
|
1620
|
-
* or pattern, with support for scheduling, retries, and timeouts.
|
|
1621
|
-
*/
|
|
1622
|
-
interface WorkflowConfig {
|
|
1623
|
-
/**
|
|
1624
|
-
* The initial task that begins the workflow execution.
|
|
1625
|
-
* Supports all combinations of input types (real type or null) and output types (real type or void):
|
|
1626
|
-
* - Task<null, OutputType>: No input, returns a type
|
|
1627
|
-
* - Task<null, void>: No input, returns nothing
|
|
1628
|
-
* - Task<InputType, OutputType>: Has input, returns a type
|
|
1629
|
-
* - Task<InputType, void>: Has input, returns nothing
|
|
1630
|
-
*/
|
|
1631
|
-
startingTask: Task<null, any> | Task<null, void> | Task<any, any> | Task<any, void>;
|
|
1632
|
-
/** Optional number of retry attempts if the entire workflow fails */
|
|
1633
|
-
retries?: number;
|
|
1634
|
-
/** Optional timeout duration for the entire workflow execution (e.g., "10m", "1h") */
|
|
1635
|
-
timeout?: string;
|
|
1636
|
-
/** Optional cron-style schedule string for automated workflow execution */
|
|
1637
|
-
schedule?: string;
|
|
1638
|
-
}
|
|
1639
|
-
/**
|
|
1640
|
-
* Represents a complete workflow composed of interconnected tasks.
|
|
1641
|
-
*
|
|
1642
|
-
* A Workflow manages the execution flow of multiple tasks, handling scheduling,
|
|
1643
|
-
* error recovery, and task orchestration. Once created, workflows are automatically
|
|
1644
|
-
* registered with the internal Moose system.
|
|
1645
|
-
*
|
|
1646
|
-
* @example
|
|
1647
|
-
* ```typescript
|
|
1648
|
-
* const dataProcessingWorkflow = new Workflow("dataProcessing", {
|
|
1649
|
-
* startingTask: extractDataTask,
|
|
1650
|
-
* schedule: "0 2 * * *", // Run daily at 2 AM
|
|
1651
|
-
* timeout: "1h",
|
|
1652
|
-
* retries: 2
|
|
1653
|
-
* });
|
|
1654
|
-
* ```
|
|
1655
|
-
*/
|
|
1656
|
-
declare class Workflow {
|
|
1657
|
-
readonly name: string;
|
|
1658
|
-
readonly config: WorkflowConfig;
|
|
1659
|
-
/**
|
|
1660
|
-
* Creates a new Workflow instance and registers it with the Moose system.
|
|
1661
|
-
*
|
|
1662
|
-
* @param name - Unique identifier for the workflow
|
|
1663
|
-
* @param config - Configuration object defining the workflow behavior and task orchestration
|
|
1664
|
-
* @throws {Error} When the workflow contains null/undefined tasks or infinite loops
|
|
1665
|
-
*/
|
|
1666
|
-
constructor(name: string, config: WorkflowConfig);
|
|
1667
|
-
/**
|
|
1668
|
-
* Validates the task graph to ensure there are no null tasks or infinite loops.
|
|
1669
|
-
*
|
|
1670
|
-
* @private
|
|
1671
|
-
* @param startingTask - The starting task to begin validation from
|
|
1672
|
-
* @param workflowName - The name of the workflow being validated (for error messages)
|
|
1673
|
-
* @throws {Error} When null/undefined tasks are found or infinite loops are detected
|
|
1674
|
-
*/
|
|
1675
|
-
private validateTaskGraph;
|
|
1676
|
-
}
|
|
1677
|
-
|
|
1678
|
-
/**
|
|
1679
|
-
* @template T The data type of the messages expected by the destination stream.
|
|
1680
|
-
*/
|
|
1681
|
-
interface IngestConfig<T> {
|
|
1682
|
-
/**
|
|
1683
|
-
* The destination stream where the ingested data should be sent.
|
|
1684
|
-
*/
|
|
1685
|
-
destination: Stream<T>;
|
|
1686
|
-
deadLetterQueue?: DeadLetterQueue<T>;
|
|
1687
|
-
/**
|
|
1688
|
-
* An optional version string for this configuration.
|
|
1689
|
-
*/
|
|
1690
|
-
version?: string;
|
|
1691
|
-
/**
|
|
1692
|
-
* An optional custom path for the ingestion endpoint.
|
|
1693
|
-
*/
|
|
1694
|
-
path?: string;
|
|
1695
|
-
metadata?: {
|
|
1696
|
-
description?: string;
|
|
1697
|
-
};
|
|
1698
|
-
}
|
|
1699
|
-
/**
|
|
1700
|
-
* Represents an Ingest API endpoint, used for sending data into a Moose system, typically writing to a Stream.
|
|
1701
|
-
* Provides a typed interface for the expected data format.
|
|
1702
|
-
*
|
|
1703
|
-
* @template T The data type of the records that this API endpoint accepts. The structure of T defines the expected request body schema.
|
|
1704
|
-
*/
|
|
1705
|
-
declare class IngestApi<T> extends TypedBase<T, IngestConfig<T>> {
|
|
1706
|
-
/**
|
|
1707
|
-
* Creates a new IngestApi instance.
|
|
1708
|
-
* @param name The name of the ingest API endpoint.
|
|
1709
|
-
* @param config Optional configuration for the ingest API.
|
|
1710
|
-
*/
|
|
1711
|
-
constructor(name: string, config?: IngestConfig<T>);
|
|
1712
|
-
/**
|
|
1713
|
-
* @internal
|
|
1714
|
-
* Note: `validators` parameter is a positional placeholder (always undefined for IngestApi).
|
|
1715
|
-
* It exists because TypedBase has validators as the 5th param, and we need to pass
|
|
1716
|
-
* allowExtraFields as the 6th param. IngestApi doesn't use validators.
|
|
1717
|
-
*/
|
|
1718
|
-
constructor(name: string, config: IngestConfig<T>, schema: IJsonSchemaCollection$1.IV3_1, columns: Column[], validators: undefined, allowExtraFields: boolean);
|
|
1719
|
-
}
|
|
1720
|
-
|
|
1721
|
-
/**
|
|
1722
|
-
* Utilities provided by getMooseUtils() for database access and SQL queries.
|
|
1723
|
-
* Works in both Moose runtime and standalone contexts.
|
|
1724
|
-
*/
|
|
1725
|
-
interface MooseUtils {
|
|
1726
|
-
client: MooseClient;
|
|
1727
|
-
sql: typeof sql;
|
|
1728
|
-
jwt?: JWTPayload;
|
|
1729
|
-
}
|
|
1730
|
-
/**
|
|
1731
|
-
* @deprecated Use MooseUtils instead. ApiUtil is now a type alias to MooseUtils
|
|
1732
|
-
* and will be removed in a future version.
|
|
1733
|
-
*
|
|
1734
|
-
* Migration: Replace `ApiUtil` with `MooseUtils` in your type annotations.
|
|
1735
|
-
*/
|
|
1736
|
-
type ApiUtil = MooseUtils;
|
|
1737
|
-
/** @deprecated Use MooseUtils instead. */
|
|
1738
|
-
type ConsumptionUtil = MooseUtils;
|
|
1739
|
-
declare class MooseClient {
|
|
1740
|
-
query: QueryClient;
|
|
1741
|
-
workflow: WorkflowClient;
|
|
1742
|
-
constructor(queryClient: QueryClient, temporalClient?: Client);
|
|
1743
|
-
}
|
|
1744
|
-
declare class QueryClient {
|
|
1745
|
-
client: ClickHouseClient;
|
|
1746
|
-
query_id_prefix: string;
|
|
1747
|
-
constructor(client: ClickHouseClient, query_id_prefix: string);
|
|
1748
|
-
execute<T = any>(sql: Sql): Promise<ResultSet<"JSONEachRow"> & {
|
|
1749
|
-
__query_result_t?: T[];
|
|
1750
|
-
}>;
|
|
1751
|
-
command(sql: Sql): Promise<CommandResult>;
|
|
1752
|
-
}
|
|
1753
|
-
declare class WorkflowClient {
|
|
1754
|
-
client: Client | undefined;
|
|
1755
|
-
constructor(temporalClient?: Client);
|
|
1756
|
-
execute(name: string, input_data: any): Promise<{
|
|
1757
|
-
status: number;
|
|
1758
|
-
body: string;
|
|
1759
|
-
}>;
|
|
1760
|
-
terminate(workflowId: string): Promise<{
|
|
1761
|
-
status: number;
|
|
1762
|
-
body: string;
|
|
1763
|
-
}>;
|
|
1764
|
-
private getWorkflowConfig;
|
|
1765
|
-
private processInputData;
|
|
1766
|
-
}
|
|
1767
|
-
|
|
1768
|
-
/**
|
|
1769
|
-
* Defines the signature for a handler function used by a Consumption API.
|
|
1770
|
-
* @template T The expected type of the request parameters or query parameters.
|
|
1771
|
-
* @template R The expected type of the response data.
|
|
1772
|
-
* @param params An object containing the validated request parameters, matching the structure of T.
|
|
1773
|
-
* @param utils Utility functions provided to the handler, e.g., for database access (`runSql`).
|
|
1774
|
-
* @returns A Promise resolving to the response data of type R.
|
|
1775
|
-
*/
|
|
1776
|
-
type ApiHandler<T, R> = (params: T, utils: ApiUtil) => Promise<R>;
|
|
1777
|
-
/**
|
|
1778
|
-
* @template T The data type of the request parameters.
|
|
1779
|
-
*/
|
|
1780
|
-
interface ApiConfig<T> {
|
|
1781
|
-
/**
|
|
1782
|
-
* An optional version string for this configuration.
|
|
1783
|
-
*/
|
|
1784
|
-
version?: string;
|
|
1785
|
-
/**
|
|
1786
|
-
* An optional custom path for the API endpoint.
|
|
1787
|
-
* If not specified, defaults to the API name.
|
|
1788
|
-
*/
|
|
1789
|
-
path?: string;
|
|
1790
|
-
metadata?: {
|
|
1791
|
-
description?: string;
|
|
1792
|
-
};
|
|
1793
|
-
}
|
|
1794
|
-
/**
|
|
1795
|
-
* Represents a Consumption API endpoint (API), used for querying data from a Moose system.
|
|
1796
|
-
* Exposes data, often from an OlapTable or derived through a custom handler function.
|
|
1797
|
-
*
|
|
1798
|
-
* @template T The data type defining the expected structure of the API's query parameters.
|
|
1799
|
-
* @template R The data type defining the expected structure of the API's response body. Defaults to `any`.
|
|
1800
|
-
*/
|
|
1801
|
-
declare class Api<T, R = any> extends TypedBase<T, ApiConfig<T>> {
|
|
1802
|
-
/** @internal The handler function that processes requests and generates responses. */
|
|
1803
|
-
_handler: ApiHandler<T, R>;
|
|
1804
|
-
/** @internal The JSON schema definition for the response type R. */
|
|
1805
|
-
responseSchema: IJsonSchemaCollection$1.IV3_1;
|
|
1806
|
-
/**
|
|
1807
|
-
* Creates a new Api instance.
|
|
1808
|
-
* @param name The name of the consumption API endpoint.
|
|
1809
|
-
* @param handler The function to execute when the endpoint is called. It receives validated query parameters and utility functions.
|
|
1810
|
-
* @param config Optional configuration for the consumption API.
|
|
1811
|
-
*/
|
|
1812
|
-
constructor(name: string, handler: ApiHandler<T, R>, config?: {});
|
|
1813
|
-
/** @internal **/
|
|
1814
|
-
constructor(name: string, handler: ApiHandler<T, R>, config: ApiConfig<T>, schema: IJsonSchemaCollection$1.IV3_1, columns: Column[], responseSchema: IJsonSchemaCollection$1.IV3_1);
|
|
1815
|
-
/**
|
|
1816
|
-
* Retrieves the handler function associated with this Consumption API.
|
|
1817
|
-
* @returns The handler function.
|
|
1818
|
-
*/
|
|
1819
|
-
getHandler: () => ApiHandler<T, R>;
|
|
1820
|
-
call(baseUrl: string, queryParams: T): Promise<R>;
|
|
1821
|
-
}
|
|
1822
|
-
/** @deprecated Use ApiConfig<T> directly instead. */
|
|
1823
|
-
type EgressConfig<T> = ApiConfig<T>;
|
|
1824
|
-
/** @deprecated Use Api directly instead. */
|
|
1825
|
-
declare const ConsumptionApi: typeof Api;
|
|
1826
|
-
|
|
1827
|
-
/**
|
|
1828
|
-
* Configuration options for a complete ingestion pipeline, potentially including an Ingest API, a Stream, and an OLAP Table.
|
|
1829
|
-
*
|
|
1830
|
-
* @template T The data type of the records being ingested.
|
|
1831
|
-
*
|
|
1832
|
-
* @example
|
|
1833
|
-
* ```typescript
|
|
1834
|
-
* // Simple pipeline with all components enabled
|
|
1835
|
-
* const pipelineConfig: IngestPipelineConfig<UserData> = {
|
|
1836
|
-
* table: true,
|
|
1837
|
-
* stream: true,
|
|
1838
|
-
* ingestApi: true
|
|
1839
|
-
* };
|
|
1840
|
-
*
|
|
1841
|
-
* // Advanced pipeline with custom configurations
|
|
1842
|
-
* const advancedConfig: IngestPipelineConfig<UserData> = {
|
|
1843
|
-
* table: { orderByFields: ['timestamp', 'userId'], engine: ClickHouseEngines.ReplacingMergeTree },
|
|
1844
|
-
* stream: { parallelism: 4, retentionPeriod: 86400 },
|
|
1845
|
-
* ingestApi: true,
|
|
1846
|
-
* version: '1.2.0',
|
|
1847
|
-
* metadata: { description: 'User data ingestion pipeline' }
|
|
1848
|
-
* };
|
|
1849
|
-
* ```
|
|
1850
|
-
*/
|
|
1851
|
-
type IngestPipelineConfig<T> = {
|
|
1852
|
-
/**
|
|
1853
|
-
* Configuration for the OLAP table component of the pipeline.
|
|
1854
|
-
*
|
|
1855
|
-
* - If `true`, a table with default settings is created.
|
|
1856
|
-
* - If an `OlapConfig` object is provided, it specifies the table's configuration.
|
|
1857
|
-
* - If `false`, no OLAP table is created.
|
|
1858
|
-
*
|
|
1859
|
-
* @default false
|
|
1860
|
-
*/
|
|
1861
|
-
table: boolean | OlapConfig<T>;
|
|
1862
|
-
/**
|
|
1863
|
-
* Configuration for the stream component of the pipeline.
|
|
1864
|
-
*
|
|
1865
|
-
* - If `true`, a stream with default settings is created.
|
|
1866
|
-
* - Pass a config object to specify the stream's configuration.
|
|
1867
|
-
* - The stream's destination will automatically be set to the pipeline's table if one exists.
|
|
1868
|
-
* - If `false`, no stream is created.
|
|
1869
|
-
*
|
|
1870
|
-
* @default false
|
|
1871
|
-
*/
|
|
1872
|
-
stream: boolean | Omit<StreamConfig<T>, "destination">;
|
|
1873
|
-
/**
|
|
1874
|
-
* Configuration for the ingest API component of the pipeline.
|
|
1875
|
-
*
|
|
1876
|
-
* - If `true`, an ingest API with default settings is created.
|
|
1877
|
-
* - If a partial `IngestConfig` object (excluding `destination`) is provided, it specifies the API's configuration.
|
|
1878
|
-
* - The API's destination will automatically be set to the pipeline's stream if one exists.
|
|
1879
|
-
* - If `false`, no ingest API is created.
|
|
1880
|
-
*
|
|
1881
|
-
* **Note:** Requires a stream to be configured when enabled.
|
|
1882
|
-
*
|
|
1883
|
-
* @default false
|
|
1884
|
-
*/
|
|
1885
|
-
ingestApi: boolean | Omit<IngestConfig<T>, "destination">;
|
|
1886
|
-
/**
|
|
1887
|
-
* @deprecated Use `ingestApi` instead. This parameter will be removed in a future version.
|
|
1888
|
-
*/
|
|
1889
|
-
ingest?: boolean | Omit<IngestConfig<T>, "destination">;
|
|
1890
|
-
/**
|
|
1891
|
-
* Configuration for the dead letter queue of the pipeline.
|
|
1892
|
-
* If `true`, a dead letter queue with default settings is created.
|
|
1893
|
-
* If a partial `StreamConfig` object (excluding `destination`) is provided, it specifies the dead letter queue's configuration.
|
|
1894
|
-
* The API's destination will automatically be set to the pipeline's stream if one exists.
|
|
1895
|
-
* If `false` or `undefined`, no dead letter queue is created.
|
|
1896
|
-
*/
|
|
1897
|
-
deadLetterQueue?: boolean | StreamConfig<DeadLetterModel>;
|
|
1898
|
-
/**
|
|
1899
|
-
* An optional version string applying to all components (table, stream, ingest) created by this pipeline configuration.
|
|
1900
|
-
* This version will be used for schema versioning and component identification.
|
|
1901
|
-
*
|
|
1902
|
-
* @example "v1.0.0", "2023-12", "prod"
|
|
1903
|
-
*/
|
|
1904
|
-
version?: string;
|
|
1905
|
-
/**
|
|
1906
|
-
* An optional custom path for the ingestion API endpoint.
|
|
1907
|
-
* This will be used as the HTTP path for the ingest API if one is created.
|
|
1908
|
-
*
|
|
1909
|
-
* @example "pipelines/analytics", "data/events"
|
|
1910
|
-
*/
|
|
1911
|
-
path?: string;
|
|
1912
|
-
/**
|
|
1913
|
-
* Optional metadata for the pipeline.
|
|
1914
|
-
*/
|
|
1915
|
-
metadata?: {
|
|
1916
|
-
/** Human-readable description of the pipeline's purpose */
|
|
1917
|
-
description?: string;
|
|
1918
|
-
};
|
|
1919
|
-
/** Determines how changes in code will propagate to the resources. */
|
|
1920
|
-
lifeCycle?: LifeCycle;
|
|
1921
|
-
};
|
|
1922
|
-
/**
|
|
1923
|
-
* Represents a complete ingestion pipeline, potentially combining an Ingest API, a Stream, and an OLAP Table
|
|
1924
|
-
* under a single name and configuration. Simplifies the setup of common ingestion patterns.
|
|
1925
|
-
*
|
|
1926
|
-
* This class provides a high-level abstraction for creating data ingestion workflows that can include:
|
|
1927
|
-
* - An HTTP API endpoint for receiving data
|
|
1928
|
-
* - A streaming component for real-time data processing
|
|
1929
|
-
* - An OLAP table for analytical queries
|
|
1930
|
-
*
|
|
1931
|
-
* @template T The data type of the records flowing through the pipeline. This type defines the schema for the
|
|
1932
|
-
* Ingest API input, the Stream messages, and the OLAP Table rows.
|
|
1933
|
-
*
|
|
1934
|
-
* @example
|
|
1935
|
-
* ```typescript
|
|
1936
|
-
* // Create a complete pipeline with all components
|
|
1937
|
-
* const userDataPipeline = new IngestPipeline('userData', {
|
|
1938
|
-
* table: true,
|
|
1939
|
-
* stream: true,
|
|
1940
|
-
* ingestApi: true,
|
|
1941
|
-
* version: '1.0.0',
|
|
1942
|
-
* metadata: { description: 'Pipeline for user registration data' }
|
|
1943
|
-
* });
|
|
1944
|
-
*
|
|
1945
|
-
* // Create a pipeline with only table and stream
|
|
1946
|
-
* const analyticsStream = new IngestPipeline('analytics', {
|
|
1947
|
-
* table: { orderByFields: ['timestamp'], engine: ClickHouseEngines.ReplacingMergeTree },
|
|
1948
|
-
* stream: { parallelism: 8, retentionPeriod: 604800 },
|
|
1949
|
-
* ingestApi: false
|
|
1950
|
-
* });
|
|
1951
|
-
* ```
|
|
1952
|
-
*/
|
|
1953
|
-
declare class IngestPipeline<T> extends TypedBase<T, IngestPipelineConfig<T>> {
|
|
1954
|
-
/**
|
|
1955
|
-
* The OLAP table component of the pipeline, if configured.
|
|
1956
|
-
* Provides analytical query capabilities for the ingested data.
|
|
1957
|
-
* Only present when `config.table` is not `false`.
|
|
1958
|
-
*/
|
|
1959
|
-
table?: OlapTable<T>;
|
|
1960
|
-
/**
|
|
1961
|
-
* The stream component of the pipeline, if configured.
|
|
1962
|
-
* Handles real-time data flow and processing between components.
|
|
1963
|
-
* Only present when `config.stream` is not `false`.
|
|
1964
|
-
*/
|
|
1965
|
-
stream?: Stream<T>;
|
|
1966
|
-
/**
|
|
1967
|
-
* The ingest API component of the pipeline, if configured.
|
|
1968
|
-
* Provides HTTP endpoints for data ingestion.
|
|
1969
|
-
* Only present when `config.ingestApi` is not `false`.
|
|
1970
|
-
*/
|
|
1971
|
-
ingestApi?: IngestApi<T>;
|
|
1972
|
-
/** The dead letter queue of the pipeline, if configured. */
|
|
1973
|
-
deadLetterQueue?: DeadLetterQueue<T>;
|
|
1974
|
-
/**
|
|
1975
|
-
* Creates a new IngestPipeline instance.
|
|
1976
|
-
* Based on the configuration, it automatically creates and links the IngestApi, Stream, and OlapTable components.
|
|
1977
|
-
*
|
|
1978
|
-
* @param name The base name for the pipeline components (e.g., "userData" could create "userData" table, "userData" stream, "userData" ingest API).
|
|
1979
|
-
* @param config Optional configuration for the ingestion pipeline.
|
|
1980
|
-
*
|
|
1981
|
-
* @throws {Error} When ingest API is enabled but no stream is configured, since the API requires a stream destination.
|
|
1982
|
-
*
|
|
1983
|
-
* @example
|
|
1984
|
-
* ```typescript
|
|
1985
|
-
* const pipeline = new IngestPipeline('events', {
|
|
1986
|
-
* table: { orderByFields: ['timestamp'], engine: ClickHouseEngines.ReplacingMergeTree },
|
|
1987
|
-
* stream: { parallelism: 2 },
|
|
1988
|
-
* ingestApi: true
|
|
1989
|
-
* });
|
|
1990
|
-
* ```
|
|
1991
|
-
*/
|
|
1992
|
-
constructor(name: string, config: IngestPipelineConfig<T>);
|
|
1993
|
-
/**
|
|
1994
|
-
* Internal constructor used by the framework for advanced initialization.
|
|
1995
|
-
*
|
|
1996
|
-
* @internal
|
|
1997
|
-
* @param name The base name for the pipeline components.
|
|
1998
|
-
* @param config Configuration specifying which components to create and their settings.
|
|
1999
|
-
* @param schema JSON schema collection for type validation.
|
|
2000
|
-
* @param columns Column definitions for the data model.
|
|
2001
|
-
* @param validators Typia validation functions.
|
|
2002
|
-
* @param allowExtraFields Whether extra fields are allowed (injected when type has index signature).
|
|
2003
|
-
*/
|
|
2004
|
-
constructor(name: string, config: IngestPipelineConfig<T>, schema: IJsonSchemaCollection$1.IV3_1, columns: Column[], validators: TypiaValidators<T>, allowExtraFields: boolean);
|
|
2005
|
-
}
|
|
2006
|
-
|
|
2007
|
-
interface ETLPipelineConfig<T, U> {
|
|
2008
|
-
extract: AsyncIterable<T> | (() => AsyncIterable<T>);
|
|
2009
|
-
transform: (sourceData: T) => Promise<U>;
|
|
2010
|
-
load: ((data: U[]) => Promise<void>) | OlapTable<U>;
|
|
2011
|
-
}
|
|
2012
|
-
declare class ETLPipeline<T, U> {
|
|
2013
|
-
readonly name: string;
|
|
2014
|
-
readonly config: ETLPipelineConfig<T, U>;
|
|
2015
|
-
private batcher;
|
|
2016
|
-
constructor(name: string, config: ETLPipelineConfig<T, U>);
|
|
2017
|
-
private setupPipeline;
|
|
2018
|
-
private createBatcher;
|
|
2019
|
-
private getDefaultTaskConfig;
|
|
2020
|
-
private createAllTasks;
|
|
2021
|
-
private createExtractTask;
|
|
2022
|
-
private createTransformTask;
|
|
2023
|
-
private createLoadTask;
|
|
2024
|
-
run(): Promise<void>;
|
|
2025
|
-
}
|
|
2026
|
-
|
|
2027
|
-
/**
|
|
2028
|
-
* Represents a database View, defined by a SQL SELECT statement based on one or more base tables or other views.
|
|
2029
|
-
* Emits structured data for the Moose infrastructure system.
|
|
2030
|
-
*/
|
|
2031
|
-
declare class View {
|
|
2032
|
-
/** @internal */
|
|
2033
|
-
readonly kind = "View";
|
|
2034
|
-
/** The name of the view */
|
|
2035
|
-
name: string;
|
|
2036
|
-
/** The SELECT SQL statement that defines the view */
|
|
2037
|
-
selectSql: string;
|
|
2038
|
-
/** Names of source tables/views that the SELECT reads from */
|
|
2039
|
-
sourceTables: string[];
|
|
2040
|
-
/** Optional metadata for the view */
|
|
2041
|
-
metadata: {
|
|
2042
|
-
[key: string]: any;
|
|
2043
|
-
};
|
|
2044
|
-
/**
|
|
2045
|
-
* Creates a new View instance.
|
|
2046
|
-
* @param name The name of the view to be created.
|
|
2047
|
-
* @param selectStatement The SQL SELECT statement that defines the view's logic.
|
|
2048
|
-
* @param baseTables An array of OlapTable or View objects that the `selectStatement` reads from. Used for dependency tracking.
|
|
2049
|
-
* @param metadata Optional metadata for the view (e.g., description, source file).
|
|
2050
|
-
*/
|
|
2051
|
-
constructor(name: string, selectStatement: string | Sql, baseTables: (OlapTable<any> | View)[], metadata?: {
|
|
2052
|
-
[key: string]: any;
|
|
2053
|
-
});
|
|
2054
|
-
}
|
|
2055
|
-
|
|
2056
|
-
/**
|
|
2057
|
-
* Configuration options for creating a Materialized View.
|
|
2058
|
-
* @template T The data type of the records stored in the target table of the materialized view.
|
|
2059
|
-
*/
|
|
2060
|
-
interface MaterializedViewConfig<T> {
|
|
2061
|
-
/** The SQL SELECT statement or `Sql` object defining the data to be materialized. Dynamic SQL (with parameters) is not allowed here. */
|
|
2062
|
-
selectStatement: string | Sql;
|
|
2063
|
-
/** An array of OlapTable or View objects that the `selectStatement` reads from. */
|
|
2064
|
-
selectTables: (OlapTable<any> | View)[];
|
|
2065
|
-
/** @deprecated See {@link targetTable}
|
|
2066
|
-
* The name for the underlying target OlapTable that stores the materialized data. */
|
|
2067
|
-
tableName?: string;
|
|
2068
|
-
/** The name for the ClickHouse MATERIALIZED VIEW object itself. */
|
|
2069
|
-
materializedViewName: string;
|
|
2070
|
-
/** @deprecated See {@link targetTable}
|
|
2071
|
-
* Optional ClickHouse engine for the target table (e.g., ReplacingMergeTree). Defaults to MergeTree. */
|
|
2072
|
-
engine?: ClickHouseEngines;
|
|
2073
|
-
targetTable?: OlapTable<T> /** Target table if the OlapTable object is already constructed. */ | {
|
|
2074
|
-
/** The name for the underlying target OlapTable that stores the materialized data. */
|
|
2075
|
-
name: string;
|
|
2076
|
-
/** Optional ClickHouse engine for the target table (e.g., ReplacingMergeTree). Defaults to MergeTree. */
|
|
2077
|
-
engine?: ClickHouseEngines;
|
|
2078
|
-
/** Optional ordering fields for the target table. Crucial if using ReplacingMergeTree. */
|
|
2079
|
-
orderByFields?: (keyof T & string)[];
|
|
2080
|
-
};
|
|
2081
|
-
/** @deprecated See {@link targetTable}
|
|
2082
|
-
* Optional ordering fields for the target table. Crucial if using ReplacingMergeTree. */
|
|
2083
|
-
orderByFields?: (keyof T & string)[];
|
|
2084
|
-
/** Optional metadata for the materialized view (e.g., description, source file). */
|
|
2085
|
-
metadata?: {
|
|
2086
|
-
[key: string]: any;
|
|
2087
|
-
};
|
|
2088
|
-
}
|
|
2089
|
-
/**
|
|
2090
|
-
* Represents a Materialized View in ClickHouse.
|
|
2091
|
-
* This encapsulates both the target OlapTable that stores the data and the MATERIALIZED VIEW definition
|
|
2092
|
-
* that populates the table based on inserts into the source tables.
|
|
2093
|
-
*
|
|
2094
|
-
* @template TargetTable The data type of the records stored in the underlying target OlapTable. The structure of T defines the target table schema.
|
|
2095
|
-
*/
|
|
2096
|
-
declare class MaterializedView<TargetTable> {
|
|
2097
|
-
/** @internal */
|
|
2098
|
-
readonly kind = "MaterializedView";
|
|
2099
|
-
/** The name of the materialized view */
|
|
2100
|
-
name: string;
|
|
2101
|
-
/** The target OlapTable instance where the materialized data is stored. */
|
|
2102
|
-
targetTable: OlapTable<TargetTable>;
|
|
2103
|
-
/** The SELECT SQL statement */
|
|
2104
|
-
selectSql: string;
|
|
2105
|
-
/** Names of source tables that the SELECT reads from */
|
|
2106
|
-
sourceTables: string[];
|
|
2107
|
-
/** Optional metadata for the materialized view */
|
|
2108
|
-
metadata: {
|
|
2109
|
-
[key: string]: any;
|
|
2110
|
-
};
|
|
2111
|
-
/**
|
|
2112
|
-
* Creates a new MaterializedView instance.
|
|
2113
|
-
* Requires the `TargetTable` type parameter to be explicitly provided or inferred,
|
|
2114
|
-
* as it's needed to define the schema of the underlying target table.
|
|
2115
|
-
*
|
|
2116
|
-
* @param options Configuration options for the materialized view.
|
|
2117
|
-
*/
|
|
2118
|
-
constructor(options: MaterializedViewConfig<TargetTable>);
|
|
2119
|
-
/** @internal **/
|
|
2120
|
-
constructor(options: MaterializedViewConfig<TargetTable>, targetSchema: IJsonSchemaCollection$1.IV3_1, targetColumns: Column[]);
|
|
2121
|
-
}
|
|
2122
|
-
|
|
2123
|
-
type SqlObject = OlapTable<any> | SqlResource | View | MaterializedView<any>;
|
|
2124
|
-
/**
|
|
2125
|
-
* Represents a generic SQL resource that requires setup and teardown commands.
|
|
2126
|
-
* Base class for constructs like Views and Materialized Views. Tracks dependencies.
|
|
2127
|
-
*/
|
|
2128
|
-
declare class SqlResource {
|
|
2129
|
-
/** @internal */
|
|
2130
|
-
readonly kind = "SqlResource";
|
|
2131
|
-
/** Array of SQL statements to execute for setting up the resource. */
|
|
2132
|
-
setup: readonly string[];
|
|
2133
|
-
/** Array of SQL statements to execute for tearing down the resource. */
|
|
2134
|
-
teardown: readonly string[];
|
|
2135
|
-
/** The name of the SQL resource (e.g., view name, materialized view name). */
|
|
2136
|
-
name: string;
|
|
2137
|
-
/** List of OlapTables or Views that this resource reads data from. */
|
|
2138
|
-
pullsDataFrom: SqlObject[];
|
|
2139
|
-
/** List of OlapTables or Views that this resource writes data to. */
|
|
2140
|
-
pushesDataTo: SqlObject[];
|
|
2141
|
-
/** @internal Source file path where this resource was defined */
|
|
2142
|
-
sourceFile?: string;
|
|
2143
|
-
/** @internal Source line number where this resource was defined */
|
|
2144
|
-
sourceLine?: number;
|
|
2145
|
-
/** @internal Source column number where this resource was defined */
|
|
2146
|
-
sourceColumn?: number;
|
|
2147
|
-
/**
|
|
2148
|
-
* Creates a new SqlResource instance.
|
|
2149
|
-
* @param name The name of the resource.
|
|
2150
|
-
* @param setup An array of SQL DDL statements to create the resource.
|
|
2151
|
-
* @param teardown An array of SQL DDL statements to drop the resource.
|
|
2152
|
-
* @param options Optional configuration for specifying data dependencies.
|
|
2153
|
-
* @param options.pullsDataFrom Tables/Views this resource reads from.
|
|
2154
|
-
* @param options.pushesDataTo Tables/Views this resource writes to.
|
|
2155
|
-
*/
|
|
2156
|
-
constructor(name: string, setup: readonly (string | Sql)[], teardown: readonly (string | Sql)[], options?: {
|
|
2157
|
-
pullsDataFrom?: SqlObject[];
|
|
2158
|
-
pushesDataTo?: SqlObject[];
|
|
2159
|
-
});
|
|
2160
|
-
}
|
|
2161
|
-
|
|
2162
|
-
type WebAppHandler = (req: http.IncomingMessage, res: http.ServerResponse) => void | Promise<void>;
|
|
2163
|
-
interface FrameworkApp {
|
|
2164
|
-
handle?: (req: http.IncomingMessage, res: http.ServerResponse, next?: (err?: any) => void) => void;
|
|
2165
|
-
callback?: () => WebAppHandler;
|
|
2166
|
-
routing?: (req: http.IncomingMessage, res: http.ServerResponse) => void;
|
|
2167
|
-
ready?: () => PromiseLike<unknown>;
|
|
2168
|
-
}
|
|
2169
|
-
interface WebAppConfig {
|
|
2170
|
-
mountPath: string;
|
|
2171
|
-
metadata?: {
|
|
2172
|
-
description?: string;
|
|
2173
|
-
};
|
|
2174
|
-
injectMooseUtils?: boolean;
|
|
2175
|
-
}
|
|
2176
|
-
declare class WebApp {
|
|
2177
|
-
name: string;
|
|
2178
|
-
handler: WebAppHandler;
|
|
2179
|
-
config: WebAppConfig;
|
|
2180
|
-
private _rawApp?;
|
|
2181
|
-
constructor(name: string, appOrHandler: FrameworkApp | WebAppHandler, config: WebAppConfig);
|
|
2182
|
-
private toHandler;
|
|
2183
|
-
getRawApp(): FrameworkApp | undefined;
|
|
2184
|
-
}
|
|
2185
|
-
|
|
2186
|
-
/**
|
|
2187
|
-
* @module registry
|
|
2188
|
-
* Public registry functions for accessing Moose Data Model v2 (dmv2) resources.
|
|
2189
|
-
*
|
|
2190
|
-
* This module provides functions to retrieve registered resources like tables, streams,
|
|
2191
|
-
* APIs, and more. These functions are part of the public API and can be used by
|
|
2192
|
-
* user applications to inspect and interact with registered Moose resources.
|
|
2193
|
-
*/
|
|
2194
|
-
|
|
2195
|
-
/**
|
|
2196
|
-
* Get all registered OLAP tables.
|
|
2197
|
-
* @returns A Map of table name to OlapTable instance
|
|
2198
|
-
*/
|
|
2199
|
-
declare function getTables(): Map<string, OlapTable<any>>;
|
|
2200
|
-
/**
|
|
2201
|
-
* Get a registered OLAP table by name.
|
|
2202
|
-
* @param name - The name of the table
|
|
2203
|
-
* @returns The OlapTable instance or undefined if not found
|
|
2204
|
-
*/
|
|
2205
|
-
declare function getTable(name: string): OlapTable<any> | undefined;
|
|
2206
|
-
/**
|
|
2207
|
-
* Get all registered streams.
|
|
2208
|
-
* @returns A Map of stream name to Stream instance
|
|
2209
|
-
*/
|
|
2210
|
-
declare function getStreams(): Map<string, Stream<any>>;
|
|
2211
|
-
/**
|
|
2212
|
-
* Get a registered stream by name.
|
|
2213
|
-
* @param name - The name of the stream
|
|
2214
|
-
* @returns The Stream instance or undefined if not found
|
|
2215
|
-
*/
|
|
2216
|
-
declare function getStream(name: string): Stream<any> | undefined;
|
|
2217
|
-
/**
|
|
2218
|
-
* Get all registered ingestion APIs.
|
|
2219
|
-
* @returns A Map of API name to IngestApi instance
|
|
2220
|
-
*/
|
|
2221
|
-
declare function getIngestApis(): Map<string, IngestApi<any>>;
|
|
2222
|
-
/**
|
|
2223
|
-
* Get a registered ingestion API by name.
|
|
2224
|
-
* @param name - The name of the ingestion API
|
|
2225
|
-
* @returns The IngestApi instance or undefined if not found
|
|
2226
|
-
*/
|
|
2227
|
-
declare function getIngestApi(name: string): IngestApi<any> | undefined;
|
|
2228
|
-
/**
|
|
2229
|
-
* Get all registered APIs (consumption/egress APIs).
|
|
2230
|
-
* @returns A Map of API key to Api instance
|
|
2231
|
-
*/
|
|
2232
|
-
declare function getApis(): Map<string, Api<any>>;
|
|
2233
|
-
/**
|
|
2234
|
-
* Get a registered API by name, version, or path.
|
|
2235
|
-
*
|
|
2236
|
-
* Supports multiple lookup strategies:
|
|
2237
|
-
* 1. Direct lookup by full key (name:version or name for unversioned)
|
|
2238
|
-
* 2. Lookup by name with automatic version aliasing when only one versioned API exists
|
|
2239
|
-
* 3. Lookup by custom path (if configured)
|
|
2240
|
-
*
|
|
2241
|
-
* @param nameOrPath - The name, name:version, or custom path of the API
|
|
2242
|
-
* @returns The Api instance or undefined if not found
|
|
2243
|
-
*/
|
|
2244
|
-
declare function getApi(nameOrPath: string): Api<any> | undefined;
|
|
2245
|
-
/**
|
|
2246
|
-
* Get all registered SQL resources.
|
|
2247
|
-
* @returns A Map of resource name to SqlResource instance
|
|
2248
|
-
*/
|
|
2249
|
-
declare function getSqlResources(): Map<string, SqlResource>;
|
|
2250
|
-
/**
|
|
2251
|
-
* Get a registered SQL resource by name.
|
|
2252
|
-
* @param name - The name of the SQL resource
|
|
2253
|
-
* @returns The SqlResource instance or undefined if not found
|
|
2254
|
-
*/
|
|
2255
|
-
declare function getSqlResource(name: string): SqlResource | undefined;
|
|
2256
|
-
/**
|
|
2257
|
-
* Get all registered workflows.
|
|
2258
|
-
* @returns A Map of workflow name to Workflow instance
|
|
2259
|
-
*/
|
|
2260
|
-
declare function getWorkflows(): Map<string, Workflow>;
|
|
2261
|
-
/**
|
|
2262
|
-
* Get a registered workflow by name.
|
|
2263
|
-
* @param name - The name of the workflow
|
|
2264
|
-
* @returns The Workflow instance or undefined if not found
|
|
2265
|
-
*/
|
|
2266
|
-
declare function getWorkflow(name: string): Workflow | undefined;
|
|
2267
|
-
/**
|
|
2268
|
-
* Get all registered web apps.
|
|
2269
|
-
* @returns A Map of web app name to WebApp instance
|
|
2270
|
-
*/
|
|
2271
|
-
declare function getWebApps(): Map<string, WebApp>;
|
|
2272
|
-
/**
|
|
2273
|
-
* Get a registered web app by name.
|
|
2274
|
-
* @param name - The name of the web app
|
|
2275
|
-
* @returns The WebApp instance or undefined if not found
|
|
2276
|
-
*/
|
|
2277
|
-
declare function getWebApp(name: string): WebApp | undefined;
|
|
2278
|
-
/**
|
|
2279
|
-
* Get all registered materialized views.
|
|
2280
|
-
* @returns A Map of MV name to MaterializedView instance
|
|
2281
|
-
*/
|
|
2282
|
-
declare function getMaterializedViews(): Map<string, MaterializedView<any>>;
|
|
2283
|
-
/**
|
|
2284
|
-
* Get a registered materialized view by name.
|
|
2285
|
-
* @param name - The name of the materialized view
|
|
2286
|
-
* @returns The MaterializedView instance or undefined if not found
|
|
2287
|
-
*/
|
|
2288
|
-
declare function getMaterializedView(name: string): MaterializedView<any> | undefined;
|
|
2289
|
-
/**
|
|
2290
|
-
* Get all registered views.
|
|
2291
|
-
* @returns A Map of view name to View instance
|
|
2292
|
-
*/
|
|
2293
|
-
declare function getViews(): Map<string, View>;
|
|
2294
|
-
/**
|
|
2295
|
-
* Get a registered view by name.
|
|
2296
|
-
* @param name - The name of the view
|
|
2297
|
-
* @returns The View instance or undefined if not found
|
|
2298
|
-
*/
|
|
2299
|
-
declare function getView(name: string): View | undefined;
|
|
2300
|
-
|
|
2301
|
-
/**
|
|
2302
|
-
* @module dmv2
|
|
2303
|
-
* This module defines the core Moose v2 data model constructs, including OlapTable, Stream, IngestApi, Api,
|
|
2304
|
-
* IngestPipeline, View, and MaterializedView. These classes provide a typed interface for defining and managing
|
|
2305
|
-
* data infrastructure components like ClickHouse tables, Redpanda streams, and data processing pipelines.
|
|
2306
|
-
*/
|
|
2307
|
-
/**
|
|
2308
|
-
* A helper type used potentially for indicating aggregated fields in query results or schemas.
|
|
2309
|
-
* Captures the aggregation function name and argument types.
|
|
2310
|
-
* (Usage context might be specific to query builders or ORM features).
|
|
2311
|
-
*
|
|
2312
|
-
* @template AggregationFunction The name of the aggregation function (e.g., 'sum', 'avg', 'count').
|
|
2313
|
-
* @template ArgTypes An array type representing the types of the arguments passed to the aggregation function.
|
|
2314
|
-
*/
|
|
2315
|
-
type Aggregated<AggregationFunction extends string, ArgTypes extends any[] = []> = {
|
|
2316
|
-
_aggregationFunction?: AggregationFunction;
|
|
2317
|
-
_argTypes?: ArgTypes;
|
|
2318
|
-
};
|
|
2319
|
-
/**
|
|
2320
|
-
* A helper type for SimpleAggregateFunction in ClickHouse.
|
|
2321
|
-
* SimpleAggregateFunction stores the aggregated value directly instead of intermediate states,
|
|
2322
|
-
* offering better performance for functions like sum, max, min, any, anyLast, etc.
|
|
2323
|
-
*
|
|
2324
|
-
* @template AggregationFunction The name of the simple aggregation function (e.g., 'sum', 'max', 'anyLast').
|
|
2325
|
-
* @template ArgType The type of the argument (and result) of the aggregation function.
|
|
2326
|
-
*
|
|
2327
|
-
* @example
|
|
2328
|
-
* ```typescript
|
|
2329
|
-
* interface Stats {
|
|
2330
|
-
* rowCount: number & SimpleAggregated<'sum', number>;
|
|
2331
|
-
* maxValue: number & SimpleAggregated<'max', number>;
|
|
2332
|
-
* lastStatus: string & SimpleAggregated<'anyLast', string>;
|
|
2333
|
-
* }
|
|
2334
|
-
* ```
|
|
2335
|
-
*/
|
|
2336
|
-
type SimpleAggregated<AggregationFunction extends string, ArgType = any> = {
|
|
2337
|
-
_simpleAggregationFunction?: AggregationFunction;
|
|
2338
|
-
_argType?: ArgType;
|
|
2339
|
-
};
|
|
2340
|
-
|
|
2341
|
-
type Key<T extends string | number | Date> = T;
|
|
2342
|
-
type JWT<T extends object> = T;
|
|
1
|
+
export { JWT, Key } from './browserCompatible';
|
|
2
|
+
import * as _clickhouse_client from '@clickhouse/client';
|
|
3
|
+
import { aE as MooseUtils, aF as MooseClient } from './index-DdE-_e4q';
|
|
4
|
+
export { A as Aggregated, h as Api, i as ApiConfig, aP as ApiHelpers, ao as ApiUtil, Y as ClickHouseByteSize, a7 as ClickHouseCodec, X as ClickHouseDecimal, a3 as ClickHouseDefault, C as ClickHouseEngines, Z as ClickHouseFixedStringSize, _ as ClickHouseFloat, $ as ClickHouseInt, a0 as ClickHouseJson, aI as ClickHouseLineString, a5 as ClickHouseMaterialized, aJ as ClickHouseMultiLineString, aL as ClickHouseMultiPolygon, a2 as ClickHouseNamedTuple, aG as ClickHousePoint, aK as ClickHousePolygon, U as ClickHousePrecision, aH as ClickHouseRing, a4 as ClickHouseTTL, j as ConsumptionApi, aQ as ConsumptionHelpers, ap as ConsumptionUtil, a8 as DateTime, a9 as DateTime64, ab as DateTime64String, aa as DateTimeString, e as DeadLetter, D as DeadLetterModel, f as DeadLetterQueue, an as Decimal, m as ETLPipeline, n as ETLPipelineConfig, E as EgressConfig, ac as FixedString, ad as Float32, ae as Float64, F as FrameworkApp, ar as IdentifierBrandedString, I as IngestApi, g as IngestConfig, k as IngestPipeline, ag as Int16, ah as Int32, ai as Int64, af as Int8, L as LifeCycle, a1 as LowCardinality, M as MaterializedView, as as NonIdentifierBrandedString, a as OlapConfig, O as OlapTable, aM as QueryClient, au as RawValue, b as S3QueueTableSettings, S as SimpleAggregated, ax as Sql, l as SqlResource, av as SqlTemplateTag, c as Stream, d as StreamConfig, T as Task, ak as UInt16, al as UInt32, am as UInt64, aj as UInt8, at as Value, V as View, o as WebApp, p as WebAppConfig, q as WebAppHandler, a6 as WithDefault, W as Workflow, aN as WorkflowClient, aC as createClickhouseParameter, y as getApi, x as getApis, w as getIngestApi, v as getIngestApis, Q as getMaterializedView, R as getMaterializedViews, B as getSqlResource, z as getSqlResources, u as getStream, t as getStreams, s as getTable, r as getTables, aO as getTemporalClient, aB as getValueFromParameter, N as getView, P as getViews, K as getWebApp, J as getWebApps, H as getWorkflow, G as getWorkflows, aR as joinQueries, aD as mapToClickHouseType, aq as quoteIdentifier, aw as sql, az as toQuery, aA as toQueryPreview, ay as toStaticQuery } from './index-DdE-_e4q';
|
|
5
|
+
import http from 'http';
|
|
6
|
+
import { IsTuple } from 'typia/lib/typings/IsTuple';
|
|
7
|
+
import { Readable } from 'node:stream';
|
|
8
|
+
import 'typia';
|
|
9
|
+
import 'typia/lib/tags';
|
|
10
|
+
import 'jose';
|
|
2343
11
|
|
|
12
|
+
declare const Kafka: any;
|
|
13
|
+
type Kafka = any;
|
|
14
|
+
type Producer = any;
|
|
2344
15
|
declare const compilerLog: (message: string) => void;
|
|
2345
16
|
declare const antiCachePath: (path: string) => string;
|
|
2346
17
|
declare const getFileName: (filePath: string) => string;
|
|
18
|
+
interface ClientConfig {
|
|
19
|
+
username: string;
|
|
20
|
+
password: string;
|
|
21
|
+
database: string;
|
|
22
|
+
useSSL: string;
|
|
23
|
+
host: string;
|
|
24
|
+
port: string;
|
|
25
|
+
}
|
|
26
|
+
declare const getClickhouseClient: ({ username, password, database, useSSL, host, port, }: ClientConfig) => _clickhouse_client.ClickHouseClient;
|
|
2347
27
|
type CliLogData = {
|
|
2348
28
|
message_type?: "Info" | "Success" | "Warning" | "Error" | "Highlight";
|
|
2349
29
|
action: string;
|
|
@@ -2355,12 +35,48 @@ declare const cliLog: (log: CliLogData) => void;
|
|
|
2355
35
|
* This is needed because 'import' does not support .ts, .cts, and .mts
|
|
2356
36
|
*/
|
|
2357
37
|
declare function mapTstoJs(filePath: string): string;
|
|
38
|
+
/**
|
|
39
|
+
* Rewrites relative import paths in JavaScript files to include .js extensions.
|
|
40
|
+
* This is required for Node.js ESM which requires explicit extensions.
|
|
41
|
+
*
|
|
42
|
+
* Handles:
|
|
43
|
+
* - import statements: import { foo } from './bar' -> import { foo } from './bar'
|
|
44
|
+
* - dynamic imports: import('./bar') -> import('./bar')
|
|
45
|
+
* - re-exports: export { foo } from './bar' -> export { foo } from './bar'
|
|
46
|
+
*
|
|
47
|
+
* Does NOT modify:
|
|
48
|
+
* - Package imports (no leading . or ..)
|
|
49
|
+
* - Imports that already have extensions
|
|
50
|
+
* - Imports from node_modules
|
|
51
|
+
*
|
|
52
|
+
* @param outDir - Directory containing compiled JavaScript files
|
|
53
|
+
*/
|
|
54
|
+
declare function rewriteImportExtensions(outDir: string): void;
|
|
2358
55
|
declare const MAX_RETRIES = 150;
|
|
2359
56
|
declare const MAX_RETRY_TIME_MS = 1000;
|
|
2360
57
|
declare const RETRY_INITIAL_TIME_MS = 100;
|
|
2361
58
|
declare const MAX_RETRIES_PRODUCER = 150;
|
|
2362
59
|
declare const RETRY_FACTOR_PRODUCER = 0.2;
|
|
2363
60
|
declare const ACKs = -1;
|
|
61
|
+
/**
|
|
62
|
+
* Creates the base producer configuration for Kafka.
|
|
63
|
+
* Used by both the SDK stream publishing and streaming function workers.
|
|
64
|
+
*
|
|
65
|
+
* @param maxMessageBytes - Optional max message size in bytes (synced with topic config)
|
|
66
|
+
* @returns Producer configuration object for the Confluent Kafka client
|
|
67
|
+
*/
|
|
68
|
+
declare function createProducerConfig(maxMessageBytes?: number): {
|
|
69
|
+
"message.max.bytes"?: number | undefined;
|
|
70
|
+
kafkaJS: {
|
|
71
|
+
idempotent: boolean;
|
|
72
|
+
acks: number;
|
|
73
|
+
retry: {
|
|
74
|
+
retries: number;
|
|
75
|
+
maxRetryTime: number;
|
|
76
|
+
};
|
|
77
|
+
};
|
|
78
|
+
"linger.ms": number;
|
|
79
|
+
};
|
|
2364
80
|
type KafkaClientConfig = {
|
|
2365
81
|
clientId: string;
|
|
2366
82
|
broker: string;
|
|
@@ -2369,6 +85,15 @@ type KafkaClientConfig = {
|
|
|
2369
85
|
saslPassword?: string;
|
|
2370
86
|
saslMechanism?: string;
|
|
2371
87
|
};
|
|
88
|
+
/**
|
|
89
|
+
* Dynamically creates and connects a KafkaJS producer using the provided configuration.
|
|
90
|
+
* Returns a connected producer instance.
|
|
91
|
+
*
|
|
92
|
+
* @param cfg - Kafka client configuration
|
|
93
|
+
* @param logger - Logger instance
|
|
94
|
+
* @param maxMessageBytes - Optional max message size in bytes (synced with topic config)
|
|
95
|
+
*/
|
|
96
|
+
declare function getKafkaProducer(cfg: KafkaClientConfig, logger: Logger, maxMessageBytes?: number): Promise<Producer>;
|
|
2372
97
|
/**
|
|
2373
98
|
* Interface for logging functionality
|
|
2374
99
|
*/
|
|
@@ -2379,6 +104,11 @@ interface Logger {
|
|
|
2379
104
|
warn: (message: string) => void;
|
|
2380
105
|
}
|
|
2381
106
|
declare const logError: (logger: Logger, e: Error) => void;
|
|
107
|
+
/**
|
|
108
|
+
* Dynamically creates a KafkaJS client configured with provided settings.
|
|
109
|
+
* Use this to construct producers/consumers with custom options.
|
|
110
|
+
*/
|
|
111
|
+
declare const getKafkaClient: (cfg: KafkaClientConfig, logger: Logger) => Promise<Kafka>;
|
|
2382
112
|
|
|
2383
113
|
/**
|
|
2384
114
|
* @module secrets
|
|
@@ -2477,6 +207,205 @@ declare const mooseEnvSecrets: {
|
|
|
2477
207
|
get(envVarName: string): string;
|
|
2478
208
|
};
|
|
2479
209
|
|
|
210
|
+
/**
|
|
211
|
+
* @deprecated Use `getMooseUtils()` from '@514labs/moose-lib' instead.
|
|
212
|
+
*
|
|
213
|
+
* This synchronous function extracts MooseUtils from a request object that was
|
|
214
|
+
* injected by Moose runtime middleware. It returns undefined if not running
|
|
215
|
+
* in a Moose-managed context.
|
|
216
|
+
*
|
|
217
|
+
* Migration: Replace with the async version:
|
|
218
|
+
* ```typescript
|
|
219
|
+
* // Old (sync, deprecated):
|
|
220
|
+
* import { getMooseUtilsFromRequest } from '@514labs/moose-lib';
|
|
221
|
+
* const moose = getMooseUtilsFromRequest(req);
|
|
222
|
+
*
|
|
223
|
+
* // New (async, recommended):
|
|
224
|
+
* import { getMooseUtils } from '@514labs/moose-lib';
|
|
225
|
+
* const moose = await getMooseUtils();
|
|
226
|
+
* ```
|
|
227
|
+
*
|
|
228
|
+
* @param req - The HTTP request object containing injected moose utilities
|
|
229
|
+
* @returns MooseUtils if available on the request, undefined otherwise
|
|
230
|
+
*/
|
|
231
|
+
declare function getMooseUtilsFromRequest(req: http.IncomingMessage | any): MooseUtils | undefined;
|
|
232
|
+
/**
|
|
233
|
+
* @deprecated Use `getMooseUtils()` from '@514labs/moose-lib' instead.
|
|
234
|
+
*
|
|
235
|
+
* This is a legacy alias for getMooseUtilsFromRequest. The main getMooseUtils
|
|
236
|
+
* export from '@514labs/moose-lib' is now async and does not require a request parameter.
|
|
237
|
+
*
|
|
238
|
+
* BREAKING CHANGE WARNING: The new getMooseUtils() returns Promise<MooseUtils>,
|
|
239
|
+
* not MooseUtils | undefined. You must await the result:
|
|
240
|
+
* ```typescript
|
|
241
|
+
* const moose = await getMooseUtils(); // New async API
|
|
242
|
+
* ```
|
|
243
|
+
*/
|
|
244
|
+
declare const getLegacyMooseUtils: typeof getMooseUtilsFromRequest;
|
|
245
|
+
/**
|
|
246
|
+
* @deprecated No longer needed. Use getMooseUtils() directly instead.
|
|
247
|
+
* Moose now handles utility injection automatically when injectMooseUtils is true.
|
|
248
|
+
*/
|
|
249
|
+
declare function expressMiddleware(): (req: any, res: any, next: any) => void;
|
|
250
|
+
/**
|
|
251
|
+
* @deprecated Use MooseUtils from helpers.ts instead.
|
|
252
|
+
*/
|
|
253
|
+
interface ExpressRequestWithMoose {
|
|
254
|
+
moose?: MooseUtils;
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
interface TaskFunction {
|
|
258
|
+
(input?: any): Promise<{
|
|
259
|
+
task: string;
|
|
260
|
+
data: any;
|
|
261
|
+
}>;
|
|
262
|
+
}
|
|
263
|
+
interface TaskConfig {
|
|
264
|
+
retries: number;
|
|
265
|
+
}
|
|
266
|
+
interface TaskDefinition {
|
|
267
|
+
task: TaskFunction;
|
|
268
|
+
config?: TaskConfig;
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
type SupportedTypes = string | object;
|
|
272
|
+
declare class MooseCache {
|
|
273
|
+
private client;
|
|
274
|
+
private isConnected;
|
|
275
|
+
private readonly keyPrefix;
|
|
276
|
+
private disconnectTimer;
|
|
277
|
+
private readonly idleTimeout;
|
|
278
|
+
private connectPromise;
|
|
279
|
+
private constructor();
|
|
280
|
+
private clearDisconnectTimer;
|
|
281
|
+
private resetDisconnectTimer;
|
|
282
|
+
private ensureConnected;
|
|
283
|
+
private connect;
|
|
284
|
+
private gracefulShutdown;
|
|
285
|
+
private getPrefixedKey;
|
|
286
|
+
/**
|
|
287
|
+
* Gets the singleton instance of MooseCache. Creates a new instance if one doesn't exist.
|
|
288
|
+
* The client will automatically connect to Redis and handle reconnection if needed.
|
|
289
|
+
*
|
|
290
|
+
* @returns Promise<MooseCache> The singleton instance of MooseCache
|
|
291
|
+
* @example
|
|
292
|
+
* const cache = await MooseCache.get();
|
|
293
|
+
*/
|
|
294
|
+
static get(): Promise<MooseCache>;
|
|
295
|
+
/**
|
|
296
|
+
* Sets a value in the cache. Objects are automatically JSON stringified.
|
|
297
|
+
*
|
|
298
|
+
* @param key - The key to store the value under
|
|
299
|
+
* @param value - The value to store. Can be a string or any object (will be JSON stringified)
|
|
300
|
+
* @param ttlSeconds - Optional time-to-live in seconds. If not provided, defaults to 1 hour (3600 seconds).
|
|
301
|
+
* Must be a non-negative number. If 0, the key will expire immediately.
|
|
302
|
+
* @example
|
|
303
|
+
* // Store a string
|
|
304
|
+
* await cache.set("foo", "bar");
|
|
305
|
+
*
|
|
306
|
+
* // Store an object with custom TTL
|
|
307
|
+
* await cache.set("foo:config", { baz: 123, qux: true }, 60); // expires in 1 minute
|
|
308
|
+
*
|
|
309
|
+
* // This is essentially a get-set, which returns the previous value if it exists.
|
|
310
|
+
* // You can create logic to only do work for the first time.
|
|
311
|
+
* const value = await cache.set("testSessionId", "true");
|
|
312
|
+
* if (value) {
|
|
313
|
+
* // Cache was set before, return
|
|
314
|
+
* } else {
|
|
315
|
+
* // Cache was set for first time, do work
|
|
316
|
+
* }
|
|
317
|
+
*/
|
|
318
|
+
set(key: string, value: string | object, ttlSeconds?: number): Promise<string | null>;
|
|
319
|
+
/**
|
|
320
|
+
* Retrieves a value from the cache. Attempts to parse the value as JSON if possible.
|
|
321
|
+
*
|
|
322
|
+
* @param key - The key to retrieve
|
|
323
|
+
* @returns Promise<T | null> The value, parsed as type T if it was JSON, or as string if not. Returns null if key doesn't exist
|
|
324
|
+
* @example
|
|
325
|
+
* // Get a string
|
|
326
|
+
* const value = await cache.get("foo");
|
|
327
|
+
*
|
|
328
|
+
* // Get and parse an object with type safety
|
|
329
|
+
* interface Config { baz: number; qux: boolean; }
|
|
330
|
+
* const config = await cache.get<Config>("foo:config");
|
|
331
|
+
*/
|
|
332
|
+
get<T extends SupportedTypes = string>(key: string): Promise<T | null>;
|
|
333
|
+
/**
|
|
334
|
+
* Deletes a specific key from the cache.
|
|
335
|
+
*
|
|
336
|
+
* @param key - The key to delete
|
|
337
|
+
* @example
|
|
338
|
+
* await cache.delete("foo");
|
|
339
|
+
*/
|
|
340
|
+
delete(key: string): Promise<void>;
|
|
341
|
+
/**
|
|
342
|
+
* Deletes all keys that start with the given prefix.
|
|
343
|
+
*
|
|
344
|
+
* @param keyPrefix - The prefix of keys to delete
|
|
345
|
+
* @example
|
|
346
|
+
* // Delete all keys starting with "foo"
|
|
347
|
+
* await cache.clearKeys("foo");
|
|
348
|
+
*/
|
|
349
|
+
clearKeys(keyPrefix: string): Promise<void>;
|
|
350
|
+
/**
|
|
351
|
+
* Deletes all keys in the cache
|
|
352
|
+
*
|
|
353
|
+
* @example
|
|
354
|
+
* await cache.clear();
|
|
355
|
+
*/
|
|
356
|
+
clear(): Promise<void>;
|
|
357
|
+
/**
|
|
358
|
+
* Manually disconnects the Redis client. The client will automatically reconnect
|
|
359
|
+
* when the next operation is performed.
|
|
360
|
+
*
|
|
361
|
+
* @example
|
|
362
|
+
* await cache.disconnect();
|
|
363
|
+
*/
|
|
364
|
+
disconnect(): Promise<void>;
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
interface RuntimeClickHouseConfig {
|
|
368
|
+
host: string;
|
|
369
|
+
port: string;
|
|
370
|
+
username: string;
|
|
371
|
+
password: string;
|
|
372
|
+
database: string;
|
|
373
|
+
useSSL: boolean;
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
/**
|
|
377
|
+
* Get Moose utilities for database access and SQL queries.
|
|
378
|
+
* Works in both Moose runtime and standalone contexts.
|
|
379
|
+
*
|
|
380
|
+
* **IMPORTANT**: This function is async and returns a Promise. You must await the result:
|
|
381
|
+
* ```typescript
|
|
382
|
+
* const moose = await getMooseUtils(); // Correct
|
|
383
|
+
* const moose = getMooseUtils(); // WRONG - returns Promise, not MooseUtils!
|
|
384
|
+
* ```
|
|
385
|
+
*
|
|
386
|
+
* **Breaking Change from v1.x**: This function signature changed from sync to async.
|
|
387
|
+
* If you were using the old sync API that extracted utils from a request object,
|
|
388
|
+
* use `getMooseUtilsFromRequest(req)` for backward compatibility (deprecated).
|
|
389
|
+
*
|
|
390
|
+
* @param req - DEPRECATED: Request parameter is no longer needed and will be ignored.
|
|
391
|
+
* If you need to extract moose from a request, use getMooseUtilsFromRequest().
|
|
392
|
+
* @returns Promise resolving to MooseUtils with client and sql utilities.
|
|
393
|
+
*
|
|
394
|
+
* @example
|
|
395
|
+
* ```typescript
|
|
396
|
+
* const { client, sql } = await getMooseUtils();
|
|
397
|
+
* const result = await client.query.execute(sql`SELECT * FROM table`);
|
|
398
|
+
* ```
|
|
399
|
+
*/
|
|
400
|
+
declare function getMooseUtils(req?: any): Promise<MooseUtils>;
|
|
401
|
+
/**
|
|
402
|
+
* @deprecated Use getMooseUtils() instead.
|
|
403
|
+
* Creates a Moose client for database access.
|
|
404
|
+
*/
|
|
405
|
+
declare function getMooseClients(config?: Partial<RuntimeClickHouseConfig>): Promise<{
|
|
406
|
+
client: MooseClient;
|
|
407
|
+
}>;
|
|
408
|
+
|
|
2480
409
|
/**
|
|
2481
410
|
* Configuration for CSV parsing options
|
|
2482
411
|
*/
|
|
@@ -2698,18 +627,20 @@ interface ExtractionResult<T = any> {
|
|
|
2698
627
|
metadata: Record<string, any>;
|
|
2699
628
|
}
|
|
2700
629
|
|
|
2701
|
-
|
|
2702
|
-
|
|
2703
|
-
|
|
2704
|
-
|
|
2705
|
-
|
|
2706
|
-
|
|
2707
|
-
|
|
2708
|
-
|
|
2709
|
-
|
|
2710
|
-
|
|
2711
|
-
|
|
2712
|
-
|
|
630
|
+
type DataModelConfig<T> = Partial<{
|
|
631
|
+
ingestion: true;
|
|
632
|
+
storage: {
|
|
633
|
+
enabled?: boolean;
|
|
634
|
+
order_by_fields?: (keyof T)[];
|
|
635
|
+
deduplicate?: boolean;
|
|
636
|
+
name?: string;
|
|
637
|
+
};
|
|
638
|
+
parallelism?: number;
|
|
639
|
+
}>;
|
|
640
|
+
|
|
641
|
+
export { ACKs, type CSVParsingConfig, CSV_DELIMITERS, type CliLogData, DEFAULT_CSV_CONFIG, DEFAULT_JSON_CONFIG, type DataModelConfig, DataSource, type DataSourceConfig, type ExpressRequestWithMoose, type ExtractionResult, type JSONParsingConfig, type KafkaClientConfig, type Logger, MAX_RETRIES, MAX_RETRIES_PRODUCER, MAX_RETRY_TIME_MS, MOOSE_RUNTIME_ENV_PREFIX, MooseCache, MooseClient, MooseUtils, type Producer, RETRY_FACTOR_PRODUCER, RETRY_INITIAL_TIME_MS, type StripDateIntersection, type TaskConfig, type TaskDefinition, type TaskFunction, antiCachePath, cliLog, compilerLog, createProducerConfig, expressMiddleware, getClickhouseClient, getFileName, getKafkaClient, getKafkaProducer, getLegacyMooseUtils, getMooseClients, getMooseUtils, getMooseUtilsFromRequest, isValidCSVDelimiter, logError, mapTstoJs, mooseEnvSecrets, mooseRuntimeEnv, parseCSV, parseJSON, parseJSONWithDates, rewriteImportExtensions };
|
|
642
|
+
|
|
643
|
+
// ── @bayoudhi/moose-lib-serverless additions ──────────────────────────────
|
|
2713
644
|
|
|
2714
645
|
/**
|
|
2715
646
|
* ClickHouse connection configuration for serverless environments.
|
|
@@ -2719,20 +650,21 @@ interface ExtractionResult<T = any> {
|
|
|
2719
650
|
* this file doesn't exist, so you must provide the config programmatically
|
|
2720
651
|
* via {@link configureClickHouse} before calling `.insert()` on any table.
|
|
2721
652
|
*/
|
|
2722
|
-
interface ClickHouseConfig {
|
|
2723
|
-
|
|
2724
|
-
|
|
2725
|
-
|
|
2726
|
-
|
|
2727
|
-
|
|
2728
|
-
|
|
2729
|
-
|
|
2730
|
-
|
|
2731
|
-
|
|
2732
|
-
|
|
2733
|
-
|
|
2734
|
-
|
|
653
|
+
export interface ClickHouseConfig {
|
|
654
|
+
/** ClickHouse host (e.g. "clickhouse.example.com") */
|
|
655
|
+
host: string;
|
|
656
|
+
/** ClickHouse HTTP port as a string (e.g. "8443") */
|
|
657
|
+
port: string;
|
|
658
|
+
/** ClickHouse username (e.g. "default") */
|
|
659
|
+
username: string;
|
|
660
|
+
/** ClickHouse password */
|
|
661
|
+
password: string;
|
|
662
|
+
/** ClickHouse database name */
|
|
663
|
+
database: string;
|
|
664
|
+
/** Whether to use HTTPS/SSL for the connection */
|
|
665
|
+
useSSL: boolean;
|
|
2735
666
|
}
|
|
667
|
+
|
|
2736
668
|
/**
|
|
2737
669
|
* Configure the ClickHouse connection for serverless environments.
|
|
2738
670
|
*
|
|
@@ -2740,24 +672,5 @@ interface ClickHouseConfig {
|
|
|
2740
672
|
* ClickHouse connection details. This bypasses the `moose.config.toml` file
|
|
2741
673
|
* lookup that would otherwise fail in environments without a Moose project
|
|
2742
674
|
* structure.
|
|
2743
|
-
*
|
|
2744
|
-
* @example
|
|
2745
|
-
* ```typescript
|
|
2746
|
-
* import { configureClickHouse, OlapTable } from "@bayoudhi/moose-lib-serverless";
|
|
2747
|
-
*
|
|
2748
|
-
* configureClickHouse({
|
|
2749
|
-
* host: process.env.CLICKHOUSE_HOST!,
|
|
2750
|
-
* port: process.env.CLICKHOUSE_PORT!,
|
|
2751
|
-
* username: process.env.CLICKHOUSE_USER!,
|
|
2752
|
-
* password: process.env.CLICKHOUSE_PASSWORD!,
|
|
2753
|
-
* database: process.env.CLICKHOUSE_DATABASE!,
|
|
2754
|
-
* useSSL: true,
|
|
2755
|
-
* });
|
|
2756
|
-
*
|
|
2757
|
-
* // Now .insert() works without moose.config.toml
|
|
2758
|
-
* await myTable.insert(data);
|
|
2759
|
-
* ```
|
|
2760
675
|
*/
|
|
2761
|
-
declare function configureClickHouse(config: ClickHouseConfig): void;
|
|
2762
|
-
|
|
2763
|
-
export { ACKs, type Aggregated, Api, type ApiConfig, type ApiUtil, type CSVParsingConfig, CSV_DELIMITERS, type CliLogData, type ClickHouseByteSize, type ClickHouseCodec, type ClickHouseConfig, type ClickHouseDecimal, type ClickHouseDefault, ClickHouseEngines, type ClickHouseFixedStringSize, type ClickHouseFloat, type ClickHouseInt, type ClickHouseJson, type ClickHouseMaterialized, type ClickHouseNamedTuple, type ClickHousePrecision, type ClickHouseTTL, ConsumptionApi, type ConsumptionUtil, DEFAULT_CSV_CONFIG, DEFAULT_JSON_CONFIG, DataSource, type DataSourceConfig, type DateTime, type DateTime64, type DateTime64String, type DateTimeString, type DeadLetter, type DeadLetterModel, DeadLetterQueue, type Decimal, ETLPipeline, type ETLPipelineConfig, type EgressConfig, type ExtractionResult, type FixedString, type Float32, type Float64, type FrameworkApp, type IdentifierBrandedString, IngestApi, type IngestConfig, IngestPipeline, type Int16, type Int32, type Int64, type Int8, type JSONParsingConfig, type JWT, type KafkaClientConfig, type Key, LifeCycle, type Logger, type LowCardinality, MAX_RETRIES, MAX_RETRIES_PRODUCER, MAX_RETRY_TIME_MS, MOOSE_RUNTIME_ENV_PREFIX, MaterializedView, type NonIdentifierBrandedString, type OlapConfig, OlapTable, RETRY_FACTOR_PRODUCER, RETRY_INITIAL_TIME_MS, type RawValue, type S3QueueTableSettings, type SimpleAggregated, Sql, SqlResource, type SqlTemplateTag, Stream, type StreamConfig, type StripDateIntersection, Task, type UInt16, type UInt32, type UInt64, type UInt8, type Value, View, WebApp, type WebAppConfig, type WebAppHandler, type WithDefault, Workflow, antiCachePath, cliLog, compilerLog, configureClickHouse, createClickhouseParameter, getApi, getApis, getFileName, getIngestApi, getIngestApis, getMaterializedView, getMaterializedViews, getSqlResource, getSqlResources, getStream, getStreams, getTable, getTables, getValueFromParameter, getView, getViews, getWebApp, getWebApps, getWorkflow, getWorkflows, isValidCSVDelimiter, logError, mapToClickHouseType, mapTstoJs, mooseEnvSecrets, mooseRuntimeEnv, parseCSV, parseJSON, parseJSONWithDates, quoteIdentifier, sql, toQuery, toQueryPreview, toStaticQuery };
|
|
676
|
+
export declare function configureClickHouse(config: ClickHouseConfig): void;
|