@whaly/connector-sdk 0.2.0 → 0.3.1

package/dist/index.d.ts

@@ -1,7 +1,12 @@
 import { AxiosInstance, AxiosRequestHeaders, AxiosResponse, AxiosRequestConfig, AxiosError } from 'axios';
-import { Validator, ValidationError } from 'jsonschema';
+import Ajv, { ValidateFunction, ErrorObject } from 'ajv';
 import { Dayjs } from 'dayjs';
 import { WriteStream } from 'fs';
+import { WorkBook } from 'xlsx';
+export { WorkBook } from 'xlsx';
+import Client from 'ssh2-sftp-client';
+export { default as SftpClient, ConnectOptions as SftpConnectOptions } from 'ssh2-sftp-client';
+import { File } from '@google-cloud/storage';
 import { BigQuery, Dataset, Table } from '@google-cloud/bigquery';
 
 declare const loadJson: (fileName: string) => any;
@@ -73,7 +78,8 @@ declare function gracefulExit(exitCode: number): Promise<void>;
 
 declare enum ReplicationMethod {
     INCREMENTAL = "INCREMENTAL",
-    FULL_TABLE = "FULL_TABLE"
+    FULL_TABLE = "FULL_TABLE",
+    APPEND = "APPEND"
 }
 
 interface HTTPHeaders {
@@ -225,6 +231,7 @@ declare abstract class StreamWarehouseSyncService {
     getcolumnNameWithTypeSuffix(colName: string, warehouseType: string | undefined): string;
     abstract getMergeQueries: () => string[];
     abstract getReplaceQueries: () => string[];
+    abstract getAppendQueries: () => string[];
     abstract createDatabaseAndSchemaIfNotExists: (retryCount: number) => Promise<void>;
     abstract createTable: () => Promise<void>;
     abstract addColumns: (fields: InternalTableField[]) => Promise<void>;
@@ -266,9 +273,12 @@ declare class StreamState$1 {
     keyProperties: string[];
     _hasBeenLoadedYetDuringThisSync: boolean;
     replicationMethod: ReplicationMethod;
+    compiledValidateFn?: ValidateFunction;
     constructor(streamId: StreamId, dbSync: StreamWarehouseSyncService, replicationMethod: ReplicationMethod);
     setSchema(schema: FlattenedSchema): void;
     getSchema(): FlattenedSchema | undefined;
+    setCompiledValidateFn(fn: ValidateFunction): void;
+    getCompiledValidateFn(): ValidateFunction | undefined;
     getBatchDate(): string;
     getBatchedRowCount(): number;
     incrementBatchedRowCount(): void;
@@ -301,7 +311,7 @@ declare abstract class ITarget<C extends BaseConfig = BaseConfig> {
     streams: {
         [streamName: string]: StreamState$1;
     };
-    validator: Validator;
+    ajv: Ajv;
     constructor(config: C, stateProvider: StateProvider);
     static requiredConfigKeys: string[];
     abstract newDBSyncInstance: (streamId: StreamId, database: string, schema: string, table: string) => StreamWarehouseSyncService;
@@ -407,6 +417,7 @@ declare abstract class Stream<O = any, C = any, P = undefined> {
     isSorted: boolean;
     isSilent: boolean;
     childConcurrency: number;
+    totalRows?: number;
     rowsSyncedMetricsConf: MetricConfiguration;
     executionTimeMetricsConf: MetricConfiguration;
     constructor(config: C, tapState: InputTapState, target: ITarget, childConcurrency?: number);
@@ -588,8 +599,8 @@ declare abstract class Tap<C> {
 
 declare class SchemaValidationError extends Error {
     streamId: StreamId;
-    validationErrors: ValidationError[];
-    constructor(message: string, streamId: StreamId, errors: ValidationError[]);
+    validationErrors: ErrorObject[];
+    constructor(message: string, streamId: StreamId, errors: ErrorObject[]);
 }
 declare class MissingFieldInSchemaError extends Error {
     streamId: StreamId;
@@ -608,6 +619,461 @@ declare const flattenSchema: <T>(streamId: StreamId, schema: T) => FlattenedSche
 declare const createTemporaryFileStream: (streamId: StreamId) => TempFile;
 declare function safePath(path: string): string;
 
+/**
+ * Field types for file processing columns.
+ * Maps to JSON Schema types via fieldTypeToJsonSchema().
+ */
+type FieldType = "STRING" | "FLOAT" | "TIMESTAMP";
+declare enum FileFormat {
+    CSV = "CSV",
+    EXCEL = "EXCEL"
+}
+type ExcelFieldSpec = ExcelSourceColumn | ExcelDerivedField;
+interface ExcelSourceColumn {
+    type: FieldType;
+    column: string;
+    primaryKey?: boolean;
+}
+interface ExcelDerivedField {
+    variableName: string;
+    primaryKey?: boolean;
+    type: FieldType;
+}
+interface ExcelFieldMapping {
+    [key: string]: ExcelFieldSpec;
+}
+type ExcelExtractionConfig = ExcelSingleSheetExtractionConfig | ExcelCustomExtractorConfig;
+interface ExcelExtractionBaseConfig {
+    type: "single-sheet-extraction" | "processor";
+    extension: string;
+    fileNameValidator: (fileName: string) => boolean;
+    fileNameVariablesExtractor: (fileName: string, workbook?: WorkBook) => {
+        [key: string]: string | undefined;
+    };
+    tableName: string | ((fileName: string, workbook?: WorkBook, variables?: {
+        [key: string]: string | undefined;
+    }) => string);
+    replicationMethod: ReplicationMethod;
+}
+interface ExcelCustomExtractorConfig extends ExcelExtractionBaseConfig {
+    type: "processor";
+    processor: (workbook: WorkBook) => Promise<Record<string, string>[]>;
+}
+interface ExcelSingleSheetExtractionConfig extends ExcelExtractionBaseConfig {
+    type: "single-sheet-extraction";
+    columns: ExcelFieldMapping;
+    sheetName?: string;
+    numberOfRowsToSkip: number;
+}
+type CsvFieldsConfig = CsvFieldsArrayConfig | CsvFieldsDictConfig;
+type CsvFieldsArrayConfig = Array<{
+    key: string;
+    valueTransformer?: (val: any) => string;
+    type: FieldType;
+}>;
+interface CsvFieldsDictConfig {
+    [key: string]: {
+        key: string;
+        valueTransformer?: (val: any) => string;
+        type: FieldType;
+    };
+}
+interface CsvFileConfig {
+    encoding?: string;
+    separator: string;
+    fields: CsvFieldsConfig;
+    addSyncedAtColumn?: boolean;
+}
+interface FileStreamBaseConfig {
+    streamId: string;
+    replicationMethod: ReplicationMethod;
+    primaryKeys: string[];
+}
+interface CsvStreamConfig extends FileStreamBaseConfig {
+    format: FileFormat.CSV;
+    csv: CsvFileConfig;
+}
+interface ExcelStreamConfig extends FileStreamBaseConfig {
+    format: FileFormat.EXCEL;
+    excel: ExcelExtractionConfig;
+}
+type FileStreamConfig = CsvStreamConfig | ExcelStreamConfig;
+interface FileStreamEntry {
+    config: FileStreamConfig;
+    filePath: string | string[];
+}
+
+type JsonSchemaProperty = {
+    type: (string | "null")[];
+    format?: string;
+};
+/**
+ * Convert a FieldType to a JSON Schema property definition.
+ */
+declare function fieldTypeToJsonSchema(fieldType: FieldType): JsonSchemaProperty;
+/**
+ * Convert an ExcelFieldMapping to a JSON Schema properties object.
+ */
+declare function excelFieldsToJsonSchema(fields: ExcelFieldMapping): {
+    type: "object";
+    properties: Record<string, JsonSchemaProperty>;
+};
+/**
+ * Convert a CsvFieldsConfig to a JSON Schema properties object.
+ */
+declare function csvFieldsToJsonSchema(config: CsvFieldsConfig): {
+    type: "object";
+    properties: Record<string, JsonSchemaProperty>;
+};
+/**
+ * Extract all output keys from a CsvFieldsConfig to use as composite primary keys.
+ * Useful for FULL_TABLE replication where all fields form the PK (since TRUNCATE + MERGE = INSERT).
+ */
+declare function extractPrimaryKeysFromCsvConfig(config: CsvFieldsConfig): string[];
+/**
+ * Extract primary keys from an ExcelFieldMapping.
+ */
+declare function extractPrimaryKeysFromExcelFields(fields: ExcelFieldMapping): string[];
+
+/**
+ * Converts an Excel column letter (A, B, C, AA, AZ, etc.) to its zero-based index.
+ */
+declare const excelColumnToIndex: (columnLetter: string) => number;
+/**
+ * Converts a zero-based column index to Excel column letter(s).
+ */
+declare const indexToExcelColumn: (columnIndex: number) => string;
+/**
+ * Converts an Excel row number (1-based) to zero-based array index.
+ */
+declare const excelRowToIndex: (rowNumber: string | number) => number;
+/**
+ * Converts a zero-based array index to Excel row number (1-based).
+ */
+declare const indexToExcelRow: (index: number) => number;
+/**
+ * Parses an Excel cell reference (like "A1", "Z24", "AA100") into column and row indices.
+ */
+declare const parseCellReference: (cellRef: string) => {
+    columnIndex: number;
+    rowIndex: number;
+};
+/**
+ * Creates an Excel cell reference from column and row indices.
+ */
+declare const createCellReference: (columnIndex: number, rowIndex: number) => string;
+/**
+ * Low-level Excel extractor: returns array of row records from a workbook.
+ */
+declare const extractExcelRows: (localFilePath: string, conf: ExcelExtractionConfig) => Promise<Record<string, string>[]>;
+/**
+ * Validates an Excel file configuration.
+ */
+declare const validateExcelConfig: (config: ExcelExtractionConfig) => void;
+/**
+ * Finds all matching configurations for a given filename.
+ */
+declare const findAllMatchingConfigs: (filename: string, configs: ExcelExtractionConfig[]) => ExcelExtractionConfig[];
+/**
+ * Creates an async generator from Excel data.
+ */
+declare function createExcelGenerator(data: Record<string, string>[]): AsyncGenerator<Record<string, any>, void, unknown>;
+
+/**
+ * Builder pattern for creating Excel configurations.
+ *
+ * Sensible defaults:
+ * - fileNameValidator: accepts all files
+ * - fileNameVariablesExtractor: extracts nothing
+ * - replicationMethod: FULL_TABLE
+ */
+declare class ExcelExtractionConfigBuilder {
+    private config;
+    static create(): ExcelExtractionConfigBuilder;
+    extension(ext: string): ExcelExtractionConfigBuilder;
+    tableName(name: string | ((fileName: string, workbook?: WorkBook, variables?: {
+        [key: string]: string | undefined;
+    }) => string)): ExcelExtractionConfigBuilder;
+    fileValidator(validator: (filename: string) => boolean): ExcelExtractionConfigBuilder;
+    variablesExtractor(extractor: (filename: string, workbook?: WorkBook) => {
+        [key: string]: string | undefined;
+    }): ExcelExtractionConfigBuilder;
+    replicationMethod(method: ReplicationMethod): ExcelExtractionConfigBuilder;
+    columns(cols: {
+        [key: string]: ExcelFieldSpec;
+    }): ExcelExtractionConfigBuilder;
+    singleSheet(sheetName?: string, skipRows?: number): ExcelExtractionConfigBuilder;
+    processor(processorFn: (workbook: WorkBook) => Promise<Record<string, string>[]>): ExcelExtractionConfigBuilder;
+    build(): ExcelExtractionConfig;
+}
+
+/**
+ * Async generator that yields parsed rows from a CSV file.
+ * Supports custom separators, encoding, BOM stripping, and field transformers.
+ */
+declare function rowGeneratorFromCsv(path: string, fileConfig: CsvFileConfig): AsyncGenerator<Record<string, any>>;
+/**
+ * Counts data rows in a CSV file by scanning for newline bytes.
+ * Subtracts 1 for the header row. Fast: no CSV parsing required.
+ */
+declare function countCsvLines(filePath: string): Promise<number>;
+/**
+ * Validates CSV header row against expected configuration.
+ */
+declare const checkCsvHeaderRow: (path: string, fileConfig: CsvFileConfig) => Promise<void>;
+
+/**
+ * Writes an array of objects directly to a CSV file.
+ */
+declare const writeDataToCsv: (fileName: string, data: any[]) => Promise<void>;
+/**
+ * Writes data from an async generator to CSV with batch writing (10k row batches).
+ */
+declare const writeGeneratorToCSV: (generator: AsyncGenerator, outputFileName: string) => Promise<number>;
+
+/**
+ * Builder pattern for creating CSV configurations.
+ *
+ * Usage:
+ * ```ts
+ * const config = CsvExtractionConfigBuilder.create()
+ *   .separator(";")
+ *   .encoding("latin1")
+ *   .addSyncedAtColumn()
+ *   .fieldsFromDict({
+ *     "Product ID":   { key: "product_id",   type: "STRING" },
+ *     "Product Name": { key: "product_name", type: "STRING" },
+ *     "Price":        { key: "price",        type: "FLOAT" },
+ *   })
+ *   .build();
+ * ```
+ */
+declare class CsvExtractionConfigBuilder {
+    private _separator;
+    private _encoding?;
+    private _addSyncedAtColumn?;
+    private _fields?;
+    private _fieldsMode?;
+    private _replicationMethod?;
+    private _primaryKeys?;
+    static create(): CsvExtractionConfigBuilder;
+    separator(sep: string): CsvExtractionConfigBuilder;
+    encoding(enc: string): CsvExtractionConfigBuilder;
+    addSyncedAtColumn(enabled?: boolean): CsvExtractionConfigBuilder;
+    fieldsFromDict(mapping: {
+        [header: string]: {
+            key: string;
+            valueTransformer?: (val: any) => string;
+            type: FieldType;
+        };
+    }): CsvExtractionConfigBuilder;
+    fieldsFromArray(fields: Array<{
+        key: string;
+        valueTransformer?: (val: any) => string;
+        type: FieldType;
+    }>): CsvExtractionConfigBuilder;
+    replicationMethod(method: ReplicationMethod): CsvExtractionConfigBuilder;
+    primaryKeys(keys: string[]): CsvExtractionConfigBuilder;
+    build(): CsvFileConfig;
+    buildStreamConfig(streamId: string): FileStreamConfig;
+}
+
+/**
+ * FileStream bridges file processing (Excel/CSV) into the connector SDK pipeline.
+ *
+ * It extends Stream so it integrates with the standard Tap -> Stream -> Target flow.
+ * Each FileStream reads one file with one config and yields records.
+ */
+declare class FileStream extends Stream<Record<string, any>, FileStreamConfig> {
+    private localFilePaths;
+    constructor(config: FileStreamConfig, localFilePath: string | string[], tapState: InputTapState, target: ITarget);
+    /**
+     * Build JSON Schema from the file config's field definitions.
+     */
+    getSchema(): Promise<Schema | undefined>;
+    /**
+     * Yield records from all file(s).
+     * When multiple file paths are provided, records are yielded sequentially from each file.
+     * Pre-counts total rows across all files to enable percentage progress logging.
+     */
+    _getRecords(): AsyncIterable<Record<string, any>>;
+}
+/**
+ * Create a FileStreamConfig from an ExcelExtractionConfig.
+ * Handles both single-sheet-extraction and processor config types.
+ *
+ * For single-sheet configs, primary keys are extracted from column definitions.
+ * For processor configs, primary keys default to an empty array.
+ *
+ * @param excelConfig - The Excel extraction config (single-sheet or processor)
+ * @param fileName - The base filename (used for variable extraction and table name resolution)
+ * @param localFilePath - Optional local file path; required for processor configs with dynamic table names
+ */
+declare function createExcelStreamConfig(excelConfig: ExcelExtractionConfig, fileName: string, localFilePath?: string): FileStreamConfig;
+/**
+ * Create a FileStreamConfig from a CsvFileConfig.
+ *
+ * By default, uses FULL_TABLE replication and extracts all field keys as composite primary keys.
+ * Override via the options parameter.
+ *
+ * @param streamId - The stream/table name
+ * @param csvConfig - The CSV file configuration
+ * @param options - Optional overrides for replicationMethod and primaryKeys
+ */
+declare function createCsvStreamConfig(streamId: string, csvConfig: CsvFileConfig, options?: {
+    replicationMethod?: ReplicationMethod;
+    primaryKeys?: string[];
+}): FileStreamConfig;
+/**
+ * Process an array of FileStreamEntry sequentially, then signal completion to the target.
+ *
+ * This is the recommended pattern for SFTP and similar multi-file ingestion flows
+ * where FileTap isn't needed.
+ *
+ * @param entries - Array of { config, filePath } pairs
+ * @param tapState - The tap state (typically `{ bookmarks: {} }`)
+ * @param target - The target (e.g. BigQueryTarget)
+ */
+declare function processFileStreams(entries: FileStreamEntry[], tapState: InputTapState, target: ITarget): Promise<void>;
+
+interface FileTapConfig {
+    [key: string]: any;
+}
+/**
+ * FileTap orchestrates multiple FileStreams.
+ *
+ * It extends Tap so it integrates with the standard Tap -> Stream -> Target flow.
+ * Provide file stream entries (config + file path pairs), and it creates a FileStream per entry.
+ *
+ * For simple sequential file processing without the Tap lifecycle, use processFileStreams() instead.
+ */
+declare class FileTap extends Tap<FileTapConfig> {
+    private entries;
+    constructor(target: ITarget, config: FileTapConfig, stateProvider: StateProvider, entries: FileStreamEntry[]);
+    /**
+     * Convenience constructor for the common case where all configs share the same file path.
+     */
+    static fromConfigs(target: ITarget, config: FileTapConfig, stateProvider: StateProvider, fileConfigs: FileStreamConfig[], sharedFilePath: string): FileTap;
+    init(): Promise<void>;
+}
+
+/**
+ * Variable extraction utilities for common filename patterns.
+ */
+declare class VariableExtractors {
+    /**
+     * Returns the filename as a variable.
+     */
+    static filename(): (filename: string) => {
+        [key: string]: string | undefined;
+    };
+    /**
+     * Extracts named capture groups from a regex pattern.
+     */
+    static regex(pattern: RegExp): (filename: string) => {
+        [key: string]: string | undefined;
+    };
+    /**
+     * Combines multiple extraction functions.
+     */
+    static combine(...extractors: ((filename: string) => {
+        [key: string]: string | undefined;
+    })[]): (filename: string) => {
+        [key: string]: string | undefined;
+    };
+}
+/**
+ * File pattern utilities for common filename validation patterns.
+ */
+declare class FilePatterns {
+    /**
+     * Creates a validator for files starting with a specific prefix (case-insensitive).
+     */
+    static startsWith(prefix: string): (filename: string) => boolean;
+    /**
+     * Creates a validator for files matching a regex pattern.
+     */
+    static regex(pattern: RegExp): (filename: string) => boolean;
+    /**
+     * Combines multiple validators with AND logic.
+     */
+    static and(...validators: ((filename: string) => boolean)[]): (filename: string) => boolean;
+    /**
+     * Combines multiple validators with OR logic.
+     */
+    static or(...validators: ((filename: string) => boolean)[]): (filename: string) => boolean;
+}
+
+/**
+ * Pre-instantiated SFTP client singleton.
+ * @deprecated Prefer creating your own SftpClient instance for better lifecycle control.
+ */
+declare const SftpService: Client;
+
+interface CloudStorageServiceOptions {
+    processedSuffix?: string;
+    supportedExtensions?: string[];
+}
+/**
+ * Service for interacting with Google Cloud Storage.
+ * Supports file download, upload, listing, and marker-file-based processing tracking.
+ */
+declare class CloudStorageService {
+    private storage;
+    private bucket;
+    private processedSuffix;
+    private supportedExtensions;
+    private path;
+    constructor(bucketName: string, path?: string, opts?: CloudStorageServiceOptions);
+    /**
+     * Lists all files in the bucket with an optional prefix.
+     */
+    listFiles(prefix?: string): Promise<string[]>;
+    /**
+     * Returns files that haven't been marked as processed.
+     * Filters by supported extensions if configured.
+     */
+    getUnprocessedFiles(): Promise<string[]>;
+    /**
+     * Creates a marker file to indicate a file has been processed.
+     */
+    createMarkerFile(fileName: string): Promise<void>;
+    /**
+     * Downloads a file from GCS to a local tmp directory.
+     * Returns the local path of the downloaded file.
+     */
+    downloadFile(filePath: string, fileName: string): Promise<string>;
+    /**
+     * Uploads a local file to GCS.
+     */
+    uploadFile(localPath: string, destPath: string): Promise<File>;
+    /**
+     * Reads a GCS object and returns its contents as a UTF-8 string.
+     */
+    readObjectAsString(objectPath: string): Promise<string>;
+    /**
+     * Writes a string to a GCS object with JSON content type.
+     */
+    writeStringObject(objectPath: string, contents: string): Promise<void>;
+    /**
+     * Uploads a local file to the bucket with a unique name based on prefix, streamId, and UUID.
+     * Files are stored under `<prefix>/<run-id>/` when the RUN_ID env var is set,
+     * or `<prefix>/default/` otherwise.
+     * Returns the GCS File reference.
+     */
+    uploadFileWithUniqueName(filePath: string, prefix: string, streamId: string): Promise<File>;
+}
+
+/**
+ * Extract a ZIP (or other archive) file to a target directory.
+ * Uses the `decompress` library which supports zip, tar, gz, bz2, etc.
+ *
+ * @param zipFilePath - Path to the archive file
+ * @param extractedPath - Directory to extract into (created if it doesn't exist)
+ * @returns The output directory path
+ */
+declare const unzip: (zipFilePath: string, extractedPath: string) => Promise<string>;
+
 interface BigQueryConfig extends BaseConfig {
     schema: string;
     project_id: string;
@@ -632,6 +1098,7 @@ declare class BigQueryDBSync extends StreamWarehouseSyncService {
     getWarehouseTypeFromJSONSchema: (definition: JSONSchemaFieldDefinition) => string;
     getMergeQueries: () => string[];
     getReplaceQueries: () => string[];
+    getAppendQueries: () => string[];
     createDatabaseAndSchemaIfNotExists: (retryCount: number) => Promise<void>;
     createTable: () => Promise<void>;
     addColumns: (tableFields: InternalTableField[]) => Promise<void>;
@@ -669,10 +1136,11 @@ declare class BigQueryTarget extends ITarget<BigQueryConfig> {
 declare class GCSStateProvider implements StateProvider {
     private bucketName;
     private connectorId;
+    private service;
     constructor(connectorId: string, bucketName: string);
     private getObjectPath;
     getState(): Promise<StateHolder>;
     writeState(state: string): Promise<void>;
 }
 
-export { ALL_STREAM_ID_KEYWORD, API_CALLS_METRIC_NAME, Authenticator, BATCH_INTERVAL_MS, type BaseConfig, type BigQueryConfig, BigQueryTarget, type BigqueryLocation, type Bookmark, type Bookmarks, type ColumnMappingStoreUnsafeToSafe, CounterMetric, DEFAULT_MAX_CONCURRENT_STREAMS, EXECUTION_TIME_METRIC_NAME, type ErrorType, type FlattenedSchema, GCSStateProvider, type GetConfig, type HTTPHeaders, type HTTPMethod, ITarget, type InputTapState, type InternalTableField, type JSONSchemaFieldDefinition, type Message, type MessageType, type MetricConfiguration, MissingFieldInSchemaError, MissingSchemaError, type PostConfig, type ProgressMarkers, type PropertiesMetadata, RESTStream, ROWS_SYNCED_METRIC_NAME, RecordMessage, RenameColumnStore, ReplicationMethod, ReplicationMethodMessage, type SafeColumnNameConverterFn, type Schema, SchemaMessage, SchemaValidationError, type State, type StateHolder, StateMessage, type StateProvider, StateService, Stream, type StreamId, type StreamState, StreamWarehouseSyncService, type StreamWithTempFile, type SyncOptions, Tap, type TapState, type TargetSchemaHook, type TargetSchemaHookInput, type TempFile, type URLParams, type WarehouseTableField, _greaterThan, addWhalyFields, createTemporaryFileStream, extractStateForStream, finalizeStateProgressMarkers, flattenSchema, getAllMetrics, getAxiosInstance, getCounterMetric, getCounterMetrics, getDownloadFileApiCall, getJSONApiCall, getJSONApiCallWithFullResponse, getRawJSONApiCall, gracefulExit, haltAndCatchFire, incrementStreamState, loadJson, loadSchema, logger, postFormDataApiCall, postJSONApiCall, postUrlEncodedApiCall, printMemoryFootprint, removeParasiteProperties, safePath };
+export { ALL_STREAM_ID_KEYWORD, API_CALLS_METRIC_NAME, Authenticator, BATCH_INTERVAL_MS, type BaseConfig, type BigQueryConfig, BigQueryTarget, type BigqueryLocation, type Bookmark, type Bookmarks, CloudStorageService, type CloudStorageServiceOptions, type ColumnMappingStoreUnsafeToSafe, CounterMetric, CsvExtractionConfigBuilder, type CsvFieldsArrayConfig, type CsvFieldsConfig, type CsvFieldsDictConfig, type CsvFileConfig, type CsvStreamConfig, DEFAULT_MAX_CONCURRENT_STREAMS, EXECUTION_TIME_METRIC_NAME, type ErrorType, type ExcelCustomExtractorConfig, type ExcelDerivedField, type ExcelExtractionBaseConfig, type ExcelExtractionConfig, ExcelExtractionConfigBuilder, type ExcelFieldMapping, type ExcelFieldSpec, type ExcelSingleSheetExtractionConfig, type ExcelSourceColumn, type ExcelStreamConfig, type FieldType, FileFormat, FilePatterns, FileStream, type FileStreamConfig, type FileStreamEntry, FileTap, type FileTapConfig, type FlattenedSchema, GCSStateProvider, type GetConfig, type HTTPHeaders, type HTTPMethod, ITarget, type InputTapState, type InternalTableField, type JSONSchemaFieldDefinition, type Message, type MessageType, type MetricConfiguration, MissingFieldInSchemaError, MissingSchemaError, type PostConfig, type ProgressMarkers, type PropertiesMetadata, RESTStream, ROWS_SYNCED_METRIC_NAME, RecordMessage, RenameColumnStore, ReplicationMethod, ReplicationMethodMessage, type SafeColumnNameConverterFn, type Schema, SchemaMessage, SchemaValidationError, SftpService, type State, type StateHolder, StateMessage, type StateProvider, StateService, Stream, type StreamId, type StreamState, StreamWarehouseSyncService, type StreamWithTempFile, type SyncOptions, Tap, type TapState, type TargetSchemaHook, type TargetSchemaHookInput, type TempFile, type URLParams, VariableExtractors, type WarehouseTableField, _greaterThan, addWhalyFields, checkCsvHeaderRow, countCsvLines, createCellReference, createCsvStreamConfig, createExcelGenerator, createExcelStreamConfig, createTemporaryFileStream, csvFieldsToJsonSchema, excelColumnToIndex, excelFieldsToJsonSchema, excelRowToIndex, extractExcelRows, extractPrimaryKeysFromCsvConfig, extractPrimaryKeysFromExcelFields, extractStateForStream, fieldTypeToJsonSchema, finalizeStateProgressMarkers, findAllMatchingConfigs, flattenSchema, getAllMetrics, getAxiosInstance, getCounterMetric, getCounterMetrics, getDownloadFileApiCall, getJSONApiCall, getJSONApiCallWithFullResponse, getRawJSONApiCall, gracefulExit, haltAndCatchFire, incrementStreamState, indexToExcelColumn, indexToExcelRow, loadJson, loadSchema, logger, parseCellReference, postFormDataApiCall, postJSONApiCall, postUrlEncodedApiCall, printMemoryFootprint, processFileStreams, removeParasiteProperties, rowGeneratorFromCsv, safePath, unzip, validateExcelConfig, writeDataToCsv, writeGeneratorToCSV };