@bayoudhi/moose-lib-serverless 0.1.1 → 0.2.0

package/dist/{helpers-C3Yr4RzN.d.mts → index.d.mts}

@@ -1,11 +1,12 @@
-import { tags, IJsonSchemaCollection as IJsonSchemaCollection$1 } from 'typia';
+import { Readable } from 'node:stream';
+import { IsTuple } from 'typia/lib/typings/IsTuple';
+import { IJsonSchemaCollection as IJsonSchemaCollection$1, tags } from 'typia';
 import { ClickHouseClient, ResultSet, CommandResult } from '@clickhouse/client';
 import { Client } from '@temporalio/client';
 import { JWTPayload } from 'jose';
 import http from 'http';
 import { IJsonSchemaCollection } from 'typia/src/schemas/json/IJsonSchemaCollection';
 import { Pattern, TagBase } from 'typia/lib/tags';
-import { Readable } from 'node:stream';
 
 type EnumValues = {
     name: string;
@@ -229,24 +230,6 @@ type ClickHouseJson<maxDynamicPaths extends number | undefined = undefined, maxD
         skipRegexes?: skipRegexes;
     };
 };
-type ClickHousePoint = [number, number] & {
-    _clickhouse_mapped_type?: "Point";
-};
-type ClickHouseRing = ClickHousePoint[] & {
-    _clickhouse_mapped_type?: "Ring";
-};
-type ClickHouseLineString = ClickHousePoint[] & {
-    _clickhouse_mapped_type?: "LineString";
-};
-type ClickHouseMultiLineString = ClickHouseLineString[] & {
-    _clickhouse_mapped_type?: "MultiLineString";
-};
-type ClickHousePolygon = ClickHouseRing[] & {
-    _clickhouse_mapped_type?: "Polygon";
-};
-type ClickHouseMultiPolygon = ClickHousePolygon[] & {
-    _clickhouse_mapped_type?: "MultiPolygon";
-};
 /**
  * typia may have trouble handling this type.
  * In which case, use {@link WithDefault} as a workaround
@@ -2354,27 +2337,378 @@ declare class WorkflowClient {
     private getWorkflowConfig;
     private processInputData;
 }
+
+type Key<T extends string | number | Date> = T;
+type JWT<T extends object> = T;
+
 /**
- * This looks similar to the client in runner.ts which is a worker.
- * Temporal SDK uses similar looking connection options & client,
- * but there are different libraries for a worker & client like this one
- * that triggers workflows.
+ * @fileoverview Pure TypeScript types, interfaces, constants, and utility functions
+ * extracted from commons.ts. This module has NO native dependencies (no Kafka,
+ * no ClickHouse client, no fs) and is safe to import in serverless/Lambda environments.
+ *
+ * The full commons.ts re-exports everything from this module for backward compatibility.
+ *
+ * @module commons-types
  */
-declare function getTemporalClient(temporalUrl: string, namespace: string, clientCert: string, clientKey: string, apiKey: string): Promise<Client | undefined>;
-declare const ApiHelpers: {
-    column: (value: string) => [string, string];
-    table: (value: string) => [string, string];
+/**
+ * Utility function for compiler-related logging that can be disabled via environment variable.
+ * Set MOOSE_DISABLE_COMPILER_LOGS=true to suppress these logs (useful for testing environments).
+ */
+declare const compilerLog: (message: string) => void;
+declare const antiCachePath: (path: string) => string;
+declare const getFileName: (filePath: string) => string;
+type CliLogData = {
+    message_type?: "Info" | "Success" | "Warning" | "Error" | "Highlight";
+    action: string;
+    message: string;
 };
-/** @deprecated Use ApiHelpers instead. */
-declare const ConsumptionHelpers: {
-    column: (value: string) => [string, string];
-    table: (value: string) => [string, string];
+declare const cliLog: (log: CliLogData) => void;
+/**
+ * Method to change .ts, .cts, and .mts to .js, .cjs, and .mjs
+ * This is needed because 'import' does not support .ts, .cts, and .mts
+ */
+declare function mapTstoJs(filePath: string): string;
+declare const MAX_RETRIES = 150;
+declare const MAX_RETRY_TIME_MS = 1000;
+declare const RETRY_INITIAL_TIME_MS = 100;
+declare const MAX_RETRIES_PRODUCER = 150;
+declare const RETRY_FACTOR_PRODUCER = 0.2;
+declare const ACKs = -1;
+/**
+ * Interface for logging functionality
+ */
+interface Logger {
+    logPrefix: string;
+    log: (message: string) => void;
+    error: (message: string) => void;
+    warn: (message: string) => void;
+}
+declare const logError: (logger: Logger, e: Error) => void;
+type KafkaClientConfig = {
+    clientId: string;
+    broker: string;
+    securityProtocol?: string;
+    saslUsername?: string;
+    saslPassword?: string;
+    saslMechanism?: string;
 };
-declare function joinQueries({ values, separator, prefix, suffix, }: {
-    values: readonly RawValue[];
-    separator?: string;
-    prefix?: string;
-    suffix?: string;
-}): Sql;
 
-export { getStreams as $, type ApiUtil as A, type ApiConfig as B, type ConsumptionUtil as C, type DateTime as D, ClickHouseEngines as E, type FixedString as F, ConsumptionApi as G, type DeadLetter as H, type Int8 as I, type DeadLetterModel as J, DeadLetterQueue as K, type LowCardinality as L, type EgressConfig as M, ETLPipeline as N, type ETLPipelineConfig as O, type FrameworkApp as P, getApi as Q, getApis as R, getIngestApi as S, getIngestApis as T, type UInt8 as U, getMaterializedView as V, type WithDefault as W, getMaterializedViews as X, getSqlResource as Y, getSqlResources as Z, getStream as _, type ClickHouseByteSize as a, getTable as a0, getTables as a1, getView as a2, getViews as a3, getWebApp as a4, getWebApps as a5, getWorkflow as a6, getWorkflows as a7, IngestApi as a8, type IngestConfig as a9, toQueryPreview as aA, getValueFromParameter as aB, createClickhouseParameter as aC, mapToClickHouseType as aD, type MooseUtils as aE, MooseClient as aF, type ClickHousePoint as aG, type ClickHouseRing as aH, type ClickHouseLineString as aI, type ClickHouseMultiLineString as aJ, type ClickHousePolygon as aK, type ClickHouseMultiPolygon as aL, QueryClient as aM, WorkflowClient as aN, getTemporalClient as aO, ApiHelpers as aP, ConsumptionHelpers as aQ, joinQueries as aR, type ConsumerConfig as aS, type TransformConfig as aT, type TaskContext as aU, type TaskConfig as aV, type IngestPipelineConfig as aW, type MaterializedViewConfig as aX, IngestPipeline as aa, LifeCycle as ab, MaterializedView as ac, type OlapConfig as ad, OlapTable as ae, type S3QueueTableSettings as af, type SimpleAggregated as ag, SqlResource as ah, Stream as ai, type StreamConfig as aj, Task as ak, View as al, WebApp as am, type WebAppConfig as an, type WebAppHandler as ao, Workflow as ap, quoteIdentifier as aq, type IdentifierBrandedString as ar, type NonIdentifierBrandedString as as, type Value as at, type RawValue as au, type SqlTemplateTag as av, sql as aw, Sql as ax, toStaticQuery as ay, toQuery as az, type ClickHouseCodec as b, type ClickHouseDecimal as c, type ClickHouseDefault as d, type ClickHouseFixedStringSize as e, type ClickHouseFloat as f, type ClickHouseInt as g, type ClickHouseJson as h, type ClickHouseMaterialized as i, type ClickHouseNamedTuple as j, type ClickHousePrecision as k, type ClickHouseTTL as l, type DateTime64 as m, type DateTime64String as n, type DateTimeString as o, type Decimal as p, type Float32 as q, type Float64 as r, type Int16 as s, type Int32 as t, type Int64 as u, type UInt16 as v, type UInt32 as w, type UInt64 as x, type Aggregated as y, Api as z };
+/**
+ * @module secrets
+ * Utilities for runtime environment variable resolution.
+ *
+ * This module provides functionality to mark values that should be resolved
+ * from environment variables at runtime by the Moose CLI, rather than being
+ * embedded at build time.
+ *
+ * @example
+ * ```typescript
+ * import { S3QueueEngine, mooseRuntimeEnv } from 'moose-lib';
+ *
+ * const table = OlapTable<MyData>(
+ *   "MyTable",
+ *   OlapConfig({
+ *     engine: S3QueueEngine({
+ *       s3_path: "s3://bucket/data/*.json",
+ *       format: "JSONEachRow",
+ *       awsAccessKeyId: mooseRuntimeEnv.get("AWS_ACCESS_KEY_ID"),
+ *       awsSecretAccessKey: mooseRuntimeEnv.get("AWS_SECRET_ACCESS_KEY")
+ *     })
+ *   })
+ * );
+ * ```
+ */
+/**
+ * Prefix used to mark values for runtime environment variable resolution.
+ * @internal
+ */
+declare const MOOSE_RUNTIME_ENV_PREFIX = "__MOOSE_RUNTIME_ENV__:";
+/**
+ * Utilities for marking values to be resolved from environment variables at runtime.
+ *
+ * When you use `mooseRuntimeEnv.get()`, the behavior depends on the context:
+ * - During infrastructure map loading: Returns a marker string for later resolution
+ * - During function/workflow execution: Returns the actual environment variable value
+ *
+ * This is useful for:
+ * - Credentials that should never be embedded in Docker images
+ * - Configuration that can be rotated without rebuilding
+ * - Different values for different environments (dev, staging, prod)
+ * - Any runtime configuration in infrastructure elements (Tables, Topics, etc.)
+ */
+declare const mooseRuntimeEnv: {
+    /**
+     * Gets a value from an environment variable, with behavior depending on context.
+     *
+     * When IS_LOADING_INFRA_MAP=true (infrastructure loading):
+     *   Returns a marker string that Moose CLI will resolve later
+     *
+     * When IS_LOADING_INFRA_MAP is unset (function/workflow runtime):
+     *   Returns the actual value from the environment variable
+     *
+     * @param envVarName - Name of the environment variable to resolve
+     * @returns Either a marker string or the actual environment variable value
+     * @throws {Error} If the environment variable name is empty
+     * @throws {Error} If the environment variable is not set (runtime mode only)
+     *
+     * @example
+     * ```typescript
+     * // Instead of this (evaluated at build time):
+     * awsAccessKeyId: process.env.AWS_ACCESS_KEY_ID
+     *
+     * // Use this (evaluated at runtime):
+     * awsAccessKeyId: mooseRuntimeEnv.get("AWS_ACCESS_KEY_ID")
+     * ```
+     */
+    get(envVarName: string): string;
+};
+/** @deprecated Use mooseRuntimeEnv instead */
+declare const mooseEnvSecrets: {
+    /**
+     * Gets a value from an environment variable, with behavior depending on context.
+     *
+     * When IS_LOADING_INFRA_MAP=true (infrastructure loading):
+     *   Returns a marker string that Moose CLI will resolve later
+     *
+     * When IS_LOADING_INFRA_MAP is unset (function/workflow runtime):
+     *   Returns the actual value from the environment variable
+     *
+     * @param envVarName - Name of the environment variable to resolve
+     * @returns Either a marker string or the actual environment variable value
+     * @throws {Error} If the environment variable name is empty
+     * @throws {Error} If the environment variable is not set (runtime mode only)
+     *
+     * @example
+     * ```typescript
+     * // Instead of this (evaluated at build time):
+     * awsAccessKeyId: process.env.AWS_ACCESS_KEY_ID
+     *
+     * // Use this (evaluated at runtime):
+     * awsAccessKeyId: mooseRuntimeEnv.get("AWS_ACCESS_KEY_ID")
+     * ```
+     */
+    get(envVarName: string): string;
+};
+
+/**
+ * Configuration for CSV parsing options
+ */
+interface CSVParsingConfig {
+    /** CSV delimiter character */
+    delimiter: string;
+    /** Whether to treat first row as headers */
+    columns?: boolean;
+    /** Whether to skip empty lines */
+    skipEmptyLines?: boolean;
+    /** Whether to trim whitespace from values */
+    trim?: boolean;
+}
+/**
+ * Configuration for JSON parsing options
+ */
+interface JSONParsingConfig {
+    /** Custom reviver function for JSON.parse */
+    reviver?: (key: string, value: any) => any;
+}
+/**
+ * Parses CSV content into an array of objects
+ *
+ * @param content - The CSV content as a string
+ * @param config - CSV parsing configuration
+ * @returns Promise resolving to an array of parsed objects
+ */
+declare function parseCSV<T = Record<string, any>>(content: string, config: CSVParsingConfig): Promise<T[]>;
+/**
+ * Parses JSON content into an array of objects
+ *
+ * @param content - The JSON content as a string
+ * @param config - JSON parsing configuration
+ * @returns Array of parsed objects
+ */
+declare function parseJSON<T = any>(content: string, config?: JSONParsingConfig): T[];
+/**
+ * Parses JSON content with automatic date revival
+ *
+ * @param content - The JSON content as a string
+ * @returns Array of parsed objects with Date objects for ISO 8601 strings
+ */
+declare function parseJSONWithDates<T = any>(content: string): T[];
+/**
+ * Type guard to check if a value is a valid CSV delimiter
+ */
+declare function isValidCSVDelimiter(delimiter: string): boolean;
+/**
+ * Common CSV delimiters
+ */
+declare const CSV_DELIMITERS: {
+    readonly COMMA: ",";
+    readonly TAB: "\t";
+    readonly SEMICOLON: ";";
+    readonly PIPE: "|";
+};
+/**
+ * Default CSV parsing configuration
+ */
+declare const DEFAULT_CSV_CONFIG: CSVParsingConfig;
+/**
+ * Default JSON parsing configuration with date revival
+ */
+declare const DEFAULT_JSON_CONFIG: JSONParsingConfig;
+
+type HasFunctionField<T> = T extends object ? {
+    [K in keyof T]: T[K] extends Function ? true : false;
+}[keyof T] extends false | undefined ? false : true : false;
+type OptionalToUndefinedable<T> = {
+    [K in {} & keyof T]: T[K];
+};
+type StripInterfaceFields<T> = {
+    [K in keyof T]: StripDateIntersection<T[K]>;
+};
+/**
+ * `Date & ...` is considered "nonsensible intersection" by typia,
+ * causing JSON schema to fail.
+ * This helper type recursively cleans up the intersection type tagging.
+ */
+type StripDateIntersection<T> = T extends Date ? Date extends T ? Date : T : T extends ReadonlyArray<unknown> ? IsTuple<T> extends true ? StripDateFromTuple<T> : T extends ReadonlyArray<infer U> ? ReadonlyArray<U> extends T ? ReadonlyArray<StripDateIntersection<U>> : Array<StripDateIntersection<U>> : T extends Array<infer U> ? Array<StripDateIntersection<U>> : T : true extends HasFunctionField<T> ? T : T extends object ? StripInterfaceFields<OptionalToUndefinedable<T>> : T;
+type StripDateFromTuple<T extends readonly any[]> = T extends ([
+    infer T1,
+    infer T2,
+    infer T3,
+    infer T4,
+    infer T5,
+    infer T6,
+    infer T7,
+    infer T8,
+    infer T9,
+    infer T10
+]) ? [
+    StripDateIntersection<T1>,
+    StripDateIntersection<T2>,
+    StripDateIntersection<T3>,
+    StripDateIntersection<T4>,
+    StripDateIntersection<T5>,
+    StripDateIntersection<T6>,
+    StripDateIntersection<T7>,
+    StripDateIntersection<T8>,
+    StripDateIntersection<T9>,
+    StripDateIntersection<T10>
+] : T extends ([
+    infer T1,
+    infer T2,
+    infer T3,
+    infer T4,
+    infer T5,
+    infer T6,
+    infer T7,
+    infer T8,
+    infer T9
+]) ? [
+    StripDateIntersection<T1>,
+    StripDateIntersection<T2>,
+    StripDateIntersection<T3>,
+    StripDateIntersection<T4>,
+    StripDateIntersection<T5>,
+    StripDateIntersection<T6>,
+    StripDateIntersection<T7>,
+    StripDateIntersection<T8>,
+    StripDateIntersection<T9>
+] : T extends ([
+    infer T1,
+    infer T2,
+    infer T3,
+    infer T4,
+    infer T5,
+    infer T6,
+    infer T7,
+    infer T8
+]) ? [
+    StripDateIntersection<T1>,
+    StripDateIntersection<T2>,
+    StripDateIntersection<T3>,
+    StripDateIntersection<T4>,
+    StripDateIntersection<T5>,
+    StripDateIntersection<T6>,
+    StripDateIntersection<T7>,
+    StripDateIntersection<T8>
+] : T extends ([
+    infer T1,
+    infer T2,
+    infer T3,
+    infer T4,
+    infer T5,
+    infer T6,
+    infer T7
+]) ? [
+    StripDateIntersection<T1>,
+    StripDateIntersection<T2>,
+    StripDateIntersection<T3>,
+    StripDateIntersection<T4>,
+    StripDateIntersection<T5>,
+    StripDateIntersection<T6>,
+    StripDateIntersection<T7>
+] : T extends [infer T1, infer T2, infer T3, infer T4, infer T5, infer T6] ? [
+    StripDateIntersection<T1>,
+    StripDateIntersection<T2>,
+    StripDateIntersection<T3>,
+    StripDateIntersection<T4>,
+    StripDateIntersection<T5>,
+    StripDateIntersection<T6>
+] : T extends [infer T1, infer T2, infer T3, infer T4, infer T5] ? [
+    StripDateIntersection<T1>,
+    StripDateIntersection<T2>,
+    StripDateIntersection<T3>,
+    StripDateIntersection<T4>,
+    StripDateIntersection<T5>
+] : T extends [infer T1, infer T2, infer T3, infer T4] ? [
+    StripDateIntersection<T1>,
+    StripDateIntersection<T2>,
+    StripDateIntersection<T3>,
+    StripDateIntersection<T4>
+] : T extends [infer T1, infer T2, infer T3] ? [
+    StripDateIntersection<T1>,
+    StripDateIntersection<T2>,
+    StripDateIntersection<T3>
+] : T extends [infer T1, infer T2] ? [
+    StripDateIntersection<T1>,
+    StripDateIntersection<T2>
+] : T extends [infer T1] ? [StripDateIntersection<T1>] : [];
+
+/**
+ * Configuration for a data source
+ */
+interface DataSourceConfig {
+    name: string;
+    supportsIncremental?: boolean;
+}
+/**
+ * DataSource is an abstract class that defines the interface for all data sources.
+ * It is used to extract data from a source and test the connection to the source.
+ */
+declare abstract class DataSource<T = any, ItemType = any> {
+    protected name: string;
+    protected supportsIncremental: boolean;
+    constructor(config: DataSourceConfig);
+    /**
+     * Extract data from the source
+     * Returns either ItemType (for single requests) or Readable (for paginated requests)
+     */
+    abstract extract(): Promise<ItemType | Readable>;
+    /**
+     * Test connection to the source
+     */
+    abstract testConnection(): Promise<{
+        success: boolean;
+        message?: string;
+    }>;
+}
+/**
+ * Result returned from extraction
+ * For single requests: data is of type T
+ * For paginated requests: data is a Readable stream yielding items of type T
+ */
+interface ExtractionResult<T = any> {
+    data: T | Readable;
+    metadata: Record<string, any>;
+}
+
+export { ACKs, type Aggregated, Api, type ApiConfig, type ApiUtil, type CSVParsingConfig, CSV_DELIMITERS, type CliLogData, type ClickHouseByteSize, type ClickHouseCodec, type ClickHouseDecimal, type ClickHouseDefault, ClickHouseEngines, type ClickHouseFixedStringSize, type ClickHouseFloat, type ClickHouseInt, type ClickHouseJson, type ClickHouseMaterialized, type ClickHouseNamedTuple, type ClickHousePrecision, type ClickHouseTTL, ConsumptionApi, type ConsumptionUtil, DEFAULT_CSV_CONFIG, DEFAULT_JSON_CONFIG, DataSource, type DataSourceConfig, type DateTime, type DateTime64, type DateTime64String, type DateTimeString, type DeadLetter, type DeadLetterModel, DeadLetterQueue, type Decimal, ETLPipeline, type ETLPipelineConfig, type EgressConfig, type ExtractionResult, type FixedString, type Float32, type Float64, type FrameworkApp, type IdentifierBrandedString, IngestApi, type IngestConfig, IngestPipeline, type Int16, type Int32, type Int64, type Int8, type JSONParsingConfig, type JWT, type KafkaClientConfig, type Key, LifeCycle, type Logger, type LowCardinality, MAX_RETRIES, MAX_RETRIES_PRODUCER, MAX_RETRY_TIME_MS, MOOSE_RUNTIME_ENV_PREFIX, MaterializedView, type NonIdentifierBrandedString, type OlapConfig, OlapTable, RETRY_FACTOR_PRODUCER, RETRY_INITIAL_TIME_MS, type RawValue, type S3QueueTableSettings, type SimpleAggregated, Sql, SqlResource, type SqlTemplateTag, Stream, type StreamConfig, type StripDateIntersection, Task, type UInt16, type UInt32, type UInt64, type UInt8, type Value, View, WebApp, type WebAppConfig, type WebAppHandler, type WithDefault, Workflow, antiCachePath, cliLog, compilerLog, createClickhouseParameter, getApi, getApis, getFileName, getIngestApi, getIngestApis, getMaterializedView, getMaterializedViews, getSqlResource, getSqlResources, getStream, getStreams, getTable, getTables, getValueFromParameter, getView, getViews, getWebApp, getWebApps, getWorkflow, getWorkflows, isValidCSVDelimiter, logError, mapToClickHouseType, mapTstoJs, mooseEnvSecrets, mooseRuntimeEnv, parseCSV, parseJSON, parseJSONWithDates, quoteIdentifier, sql, toQuery, toQueryPreview, toStaticQuery };