@whaly/connector-sdk 0.1.1 → 0.3.0

package/dist/index.d.mts

@@ -1,49 +1,96 @@
-import winston from 'winston';
 import { AxiosInstance, AxiosRequestHeaders, AxiosResponse, AxiosRequestConfig, AxiosError } from 'axios';
-import moment from 'moment';
 import { Validator, ValidationError } from 'jsonschema';
+import { Dayjs } from 'dayjs';
 import { WriteStream } from 'fs';
+import { WorkBook } from 'xlsx';
+export { WorkBook } from 'xlsx';
+import Client from 'ssh2-sftp-client';
+export { default as SftpClient, ConnectOptions as SftpConnectOptions } from 'ssh2-sftp-client';
+import { File } from '@google-cloud/storage';
 import { BigQuery, Dataset, Table } from '@google-cloud/bigquery';
 
-type StreamId = string;
-type StreamName = string;
-interface CatalogStreamMetadata {
-    breadcrumb: string[];
-    metadata: Record<string, any>;
-}
-interface CatalogStream {
-    tap_stream_id: StreamId;
-    stream: StreamName;
-    schema: any;
-    metadata: CatalogStreamMetadata[];
+declare const loadJson: (fileName: string) => any;
+declare const loadSchema: (streamId: string) => any;
+
+declare const getAxiosInstance: (retryCount?: number) => AxiosInstance;
+interface GetConfig {
+    qs?: {
+        [key: string]: any;
+    };
+    headers?: AxiosRequestHeaders;
+    retryCount?: number;
 }
-interface CatalogFile {
-    streams: CatalogStream[];
+declare const getRawJSONApiCall: <T>(endpoint: string, config: GetConfig, privateLogging?: boolean) => Promise<AxiosResponse<T>>;
+declare const getJSONApiCallWithFullResponse: <T>(endpoint: string, config: GetConfig, privateLogging?: boolean) => Promise<AxiosResponse<any, any, {}>>;
+declare const getJSONApiCall: <T>(endpoint: string, config: GetConfig, privateLogging?: boolean) => Promise<T>;
+declare const getDownloadFileApiCall: <T>(endpoint: string, config: GetConfig, output: string, privateLogging?: boolean) => Promise<{
+    outputDir: string;
+}>;
+interface PostConfig {
+    qs?: {
+        [key: string]: any;
+    };
+    headers?: AxiosRequestHeaders;
+    retryCount?: number;
 }
+declare const postJSONApiCall: <T>(endpoint: string, config: PostConfig, payload: any) => Promise<T>;
+declare const postFormDataApiCall: <T>(endpoint: string, config: PostConfig, payload: any) => Promise<T>;
+declare const postUrlEncodedApiCall: (endpoint: string, config: PostConfig, payload: any) => Promise<any>;
 
-type ReplicationMethod = "INCREMENTAL" | "FULL_TABLE";
-type DestinationType = `GOOGLE_BIGQUERY` | `SNOWFLAKE`;
-interface StateHolder {
-    state?: any;
+declare const BATCH_INTERVAL_MS = 100;
+declare const logger: {
+    info: (message: string | any, ...args: any[]) => void;
+    error: (message: string | any, ...args: any[]) => void;
+    debug: (message: string | any, ...args: any[]) => void;
+    warn: (message: string | any, ...args: any[]) => void;
+    closeWinston: (cb: () => void) => void;
+};
+
+declare const ALL_STREAM_ID_KEYWORD = "all";
+declare const ROWS_SYNCED_METRIC_NAME = "rows_synced_count";
+declare const API_CALLS_METRIC_NAME = "api_calls_count";
+declare const EXECUTION_TIME_METRIC_NAME = "execution_time_ms";
+interface MetricConfiguration {
+    isEnabled: () => boolean;
+    getStreamIds?: () => string[];
+}
+declare const getCounterMetrics: (name: string, streamIds: string[]) => CounterMetric[];
+declare const getCounterMetric: (name: string, streamId?: string) => CounterMetric;
+declare const getAllMetrics: () => CounterMetric[];
+declare class CounterMetric {
+    private value;
+    name: string;
+    streamId: string;
+    constructor(name: string, streamId: string);
+    increment(inc?: number): void;
+    getStreamId(): string;
+    getName(): string;
+    getValue(): number;
 }
 
-interface StreamRelationship {
-    left: StreamId;
-    right: StreamId;
-    from: string;
-    to: string;
-    type: "1-1" | "1-N" | "N-1";
+declare const printMemoryFootprint: (prefix: string) => void;
+
+type ErrorType = "unauthorized" | "unknown";
+
+declare const haltAndCatchFire: (errorType: ErrorType, errorText: string, errorDebugText: string) => Promise<void>;
+
+declare function gracefulExit(exitCode: number): Promise<void>;
+
+declare enum ReplicationMethod {
+    INCREMENTAL = "INCREMENTAL",
+    FULL_TABLE = "FULL_TABLE",
+    APPEND = "APPEND"
 }
-interface DirectionalRelationship {
-    type: "1-N" | "N-1" | "1-1";
-    streamId: StreamId;
-    from: string;
-    to: string;
+
+interface HTTPHeaders {
+    [headerName: string]: any;
 }
-interface DirectionalRelationships {
-    left: DirectionalRelationship[];
-    right: DirectionalRelationship[];
+interface URLParams {
+    [paramName: string]: any;
 }
+type HTTPMethod = "GET" | "POST";
+
+type StreamId = string;
 
 interface PropertiesMetadata {
     [propName: string]: {
@@ -93,7 +140,6 @@ declare class SchemaMessage implements Message {
     displayLabel: string | undefined;
     description: string | undefined;
     propertiesMetadata: PropertiesMetadata | undefined;
-    relationships: DirectionalRelationships;
     constructor(opts: {
         stream: StreamId;
         schema: any;
@@ -101,7 +147,6 @@ declare class SchemaMessage implements Message {
         bookmarkProperties?: string[];
         displayLabel?: string;
         description?: string;
-        relationships: DirectionalRelationships;
         propertiesMetadata?: PropertiesMetadata | undefined;
     });
     toString(): string;
@@ -117,150 +162,6 @@ interface TargetSchemaHook {
     writeSchema: (input: TargetSchemaHookInput) => Promise<void>;
 }
 
-type ErrorType = "unauthorized" | "unknown";
-
-declare abstract class Resolver {
-    constructor();
-    abstract checkIfCanSync(): Promise<boolean>;
-    abstract markSyncStarted(): Promise<void>;
-    abstract getState(): Promise<StateHolder>;
-    abstract writeCatalog(catalog: CatalogFile): Promise<void>;
-    abstract writeState(state: string): Promise<void>;
-    abstract markSyncFailed(): Promise<void>;
-    abstract markSyncComplete(): Promise<void>;
-    abstract markDiscoveryComplete(): Promise<void>;
-    abstract flushMetrics(): Promise<void>;
-    abstract onError(errorType: ErrorType, errorText: string, errorDebugText: string): Promise<void>;
-    abstract getSchemaHooks(): TargetSchemaHook[];
-    abstract updateSourceValue(optionKey: string, optionValue: string): Promise<void>;
-    abstract getLogFormat(): winston.Logform.Format;
-}
-
-declare const writeStateFile: (resolver: Resolver, state: string) => Promise<void>;
-declare const loadJson: (fileName: string) => any;
-declare const loadSchema: (streamId: string) => any;
-
-declare const getAxiosInstance: (retryCount?: number) => AxiosInstance;
-interface GetConfig {
-    qs?: {
-        [key: string]: any;
-    };
-    headers?: AxiosRequestHeaders;
-    retryCount?: number;
-}
-declare const getRawJSONApiCall: <T>(endpoint: string, config: GetConfig, privateLogging?: boolean) => Promise<AxiosResponse<T>>;
-declare const getJSONApiCallWithFullResponse: <T>(endpoint: string, config: GetConfig, privateLogging?: boolean) => Promise<AxiosResponse<any, any, {}>>;
-declare const getJSONApiCall: <T>(endpoint: string, config: GetConfig, privateLogging?: boolean) => Promise<T>;
-declare const getDownloadFileApiCall: <T>(endpoint: string, config: GetConfig, output: string, privateLogging?: boolean) => Promise<{
-    outputDir: string;
-}>;
-interface PostConfig {
-    qs?: {
-        [key: string]: any;
-    };
-    headers?: AxiosRequestHeaders;
-    retryCount?: number;
-}
-declare const postJSONApiCall: <T>(endpoint: string, config: PostConfig, payload: any) => Promise<T>;
-declare const postFormDataApiCall: <T>(endpoint: string, config: PostConfig, payload: any) => Promise<T>;
-declare const postUrlEncodedApiCall: (endpoint: string, config: PostConfig, payload: any) => Promise<any>;
-
-declare const BATCH_INTERVAL_MS = 100;
-declare const logger: {
-    info: (message: string | any, ...args: any[]) => void;
-    error: (message: string | any, ...args: any[]) => void;
-    debug: (message: string | any, ...args: any[]) => void;
-    warn: (message: string | any, ...args: any[]) => void;
-    closeWinston: (cb: () => void) => void;
-};
-
-declare const ALL_STREAM_ID_KEYWORD = "all";
-declare const ROWS_SYNCED_METRIC_NAME = "rows_synced_count";
-declare const API_CALLS_METRIC_NAME = "api_calls_count";
-declare const EXECUTION_TIME_METRIC_NAME = "execution_time_ms";
-interface MetricConfiguration {
-    isEnabled: () => boolean;
-    getStreamIds?: () => string[];
-}
-declare const getCounterMetrics: (name: string, streamIds: string[]) => CounterMetric[];
-declare const getCounterMetric: (name: string, streamId?: string) => CounterMetric;
-declare const getAllMetrics: () => CounterMetric[];
-declare class CounterMetric {
-    private value;
-    name: string;
-    streamId: string;
-    constructor(name: string, streamId: string);
-    increment(inc?: number): void;
-    getStreamId(): string;
-    getName(): string;
-    getValue(): number;
-}
-
-declare const printMemoryFootprint: (prefix: string) => void;
-
-declare const haltAndCatchFire: (errorType: ErrorType, errorText: string, errorDebugText: string) => Promise<void>;
-
-declare function gracefulExit(exitCode: number): Promise<void>;
-
-declare const findRelatedRelationships: (streamId: StreamId, allRels: StreamRelationship[]) => DirectionalRelationships;
-
-declare const loadResolver: () => Resolver;
-
-interface HTTPHeaders {
-    [headerName: string]: any;
-}
-interface URLParams {
-    [paramName: string]: any;
-}
-type HTTPMethod = "GET" | "POST";
-
-type InclusionMode = "available" | "automatic" | "unsupported";
-interface MetadataEntry {
-    selected?: boolean;
-    "replication-method"?: ReplicationMethod;
-    "replication-key"?: string;
-    "view-key-properties"?: string[];
-    "inclusion"?: InclusionMode;
-    "selected-by-default"?: boolean;
-    "valid-replication-keys"?: string[];
-    "forced-replication-method"?: ReplicationMethod;
-    "forced-replication-key"?: string;
-    "table-key-properties"?: string[];
-    "schema-name"?: string;
-    "is-view"?: boolean;
-    "row-count"?: number;
-    "database-name"?: string;
-    "sql-datatype"?: string;
-    "whaly-display-label"?: string;
-    "whaly-description"?: string;
-    "whaly-parent-stream"?: StreamId;
-}
-interface CatalogMetadata {
-    breadcrumb: string[];
-    metadata: MetadataEntry;
-}
-declare class MetadataMap extends Map<string[], MetadataEntry> {
-    private areEquals;
-    private findExistingEntry;
-    get(key: string[]): MetadataEntry | undefined;
-    set(key: string[], value: MetadataEntry): this;
-}
-declare class StreamMetadata {
-    metadataByBreadcrumb: MetadataMap;
-    constructor();
-    fromCatalogStreamMetadata(metadataArr: CatalogMetadata[]): this;
-    toString(): MetadataMap;
-    get(breadcrumb: string[]): MetadataEntry | undefined;
-    write(breadcrumb: string[], key: string, value: any): this;
-    private getRootBreadcrumbMetadata;
-    isStreamSelected(): boolean;
-    getKeyProperties(): string[];
-    getReplicationKey(): string | undefined;
-    getReplicationMethod(): ReplicationMethod | undefined;
-    getForcedReplicationMethod(): ReplicationMethod | undefined;
-    toList(): CatalogMetadata[];
-}
-
 interface FlattenedSchema {
     [unsafePropertyName: string]: JSONSchemaFieldDefinition;
 }
@@ -287,8 +188,9 @@ interface StreamWithTempFile {
     tempFile: TempFile;
 }
 interface BaseConfig {
-    schema: string;
+    connector_id: string;
     database: string;
+    schema: string;
 }
 
 interface ColumnMappingStoreUnsafeToSafe {
@@ -329,6 +231,7 @@ declare abstract class StreamWarehouseSyncService {
     getcolumnNameWithTypeSuffix(colName: string, warehouseType: string | undefined): string;
     abstract getMergeQueries: () => string[];
     abstract getReplaceQueries: () => string[];
+    abstract getAppendQueries: () => string[];
     abstract createDatabaseAndSchemaIfNotExists: (retryCount: number) => Promise<void>;
     abstract createTable: () => Promise<void>;
     abstract addColumns: (fields: InternalTableField[]) => Promise<void>;
@@ -363,7 +266,7 @@ declare class StreamState$1 {
     streamId: StreamId;
     schema?: FlattenedSchema;
     dbSync: StreamWarehouseSyncService;
-    batchDate: moment.Moment;
+    batchDate: Dayjs;
     batchedRowCount: number;
     fileToLoad: TempFile;
     syncedRowCount: number;
@@ -386,11 +289,19 @@ declare class StreamState$1 {
     setHasBeenLoadedYet(): void;
 }
 
+interface StateHolder {
+    state?: any;
+}
+interface StateProvider {
+    getState(): Promise<StateHolder>;
+    writeState(state: string): Promise<void>;
+}
+
 declare abstract class ITarget<C extends BaseConfig = BaseConfig> {
     config: C;
     schemaHooks: TargetSchemaHook[];
-    resolver: Resolver;
-    syncTime: moment.Moment;
+    syncTime: Dayjs;
+    stateProvider: StateProvider;
     batchedState: any;
     flushedState: any;
     renameColumnStore: RenameColumnStore;
@@ -398,7 +309,7 @@ declare abstract class ITarget<C extends BaseConfig = BaseConfig> {
         [streamName: string]: StreamState$1;
     };
     validator: Validator;
-    constructor(config: C, resolver: Resolver);
+    constructor(config: C, stateProvider: StateProvider);
     static requiredConfigKeys: string[];
     abstract newDBSyncInstance: (streamId: StreamId, database: string, schema: string, table: string) => StreamWarehouseSyncService;
     abstract genTableName: (streamId: string) => string;
@@ -462,11 +373,9 @@ declare class StateService {
     bookmarks: Bookmarks;
     constructor();
     static getInstance(): StateService;
-    setBookmark(streamId: StreamId, ts: moment.Moment): void;
-    setBookmarkV2(streamId: StreamId, streamState: StreamState): void;
-    setBookmarkSignpostV2(streamId: StreamId, signpostValue: string): void;
-    getBookmark(streamId: StreamId): moment.Moment;
-    getBookmarkV2(streamId: StreamId): StreamState;
+    setBookmark(streamId: StreamId, streamState: StreamState): void;
+    setBookmarkSignpost(streamId: StreamId, signpostValue: string): void;
+    getBookmark(streamId: StreamId): StreamState;
     clearState(): void;
     get(): State;
     forwardStateToTarget(target: ITarget): Promise<void>;
@@ -491,7 +400,7 @@ declare abstract class Stream<O = any, C = any, P = undefined> {
     config: C;
     tapState: InputTapState;
     target: ITarget;
-    forcedReplicationMethod: ReplicationMethod | undefined;
+    replicationMethod: ReplicationMethod | undefined;
     selectedByDefault: boolean;
     STATE_MSG_FREQUENCY: number;
     streamId: StreamId;
@@ -499,16 +408,15 @@ declare abstract class Stream<O = any, C = any, P = undefined> {
     displayLabel: string | undefined;
     description: string | undefined;
     primaryKey: string[];
-    forcedReplicationKey: string | undefined;
+    replicationKey: string | undefined;
     useStateFromStreamId: string | undefined;
-    relationships: StreamRelationship[];
     children: Stream<any, any, O | O[]>[];
     isSorted: boolean;
     isSilent: boolean;
     childConcurrency: number;
     rowsSyncedMetricsConf: MetricConfiguration;
     executionTimeMetricsConf: MetricConfiguration;
-    constructor(config: C, state: InputTapState, target: ITarget, childConcurrency?: number);
+    constructor(config: C, tapState: InputTapState, target: ITarget, childConcurrency?: number);
     /**
      * Sync this stream
      * Called only for streams with no parents, aka. "root tap streams"
@@ -545,17 +453,17 @@ declare abstract class Stream<O = any, C = any, P = undefined> {
      * or `start_date` config if set,
      * or the UNIX Epoch
      **/
-    getStartingTimestamp(): moment.Moment;
+    getStartingTimestamp(): Dayjs;
     /**
      * Return the max allowable bookmark value for this stream's replication key.
      *
-     * For timestamp-based replication keys, this defaults to `moment()`. For
+     * For timestamp-based replication keys, this defaults to `dayjs()`. For
      * non-timestamp replication keys, default to `undefined`.
      *
      * Override this value to prevent bookmarks from being advanced in cases where we
      * may only have a partial set of records.
      */
-    getReplicationKeySignpost: () => Promise<moment.Moment | undefined>;
+    getReplicationKeySignpost: () => Promise<Dayjs | undefined>;
     /**
      * Return the default replication method for the stream that will be used to write the catalog.
      */
@@ -657,7 +565,7 @@ declare abstract class RESTStream<R, O, NPT, C, P = undefined> extends Stream<O,
      * TODO: Make URL + Path templatisable and replace those with Extractor (or Tap) config
      * @returns
      */
-    getUrl(parent?: P): string;
+    getNextUrl(previousToken?: NPT, parent?: P): string;
     /**
      * Parse the response and return an iterator of result rows.
      *
@@ -677,8 +585,10 @@ declare abstract class Tap<C> {
     config: C;
     streams: Stream[];
     concurrency: number;
+    stateProvider: StateProvider;
+    tapState: InputTapState;
     abstract init(): Promise<void>;
-    constructor(target: ITarget, config: C);
+    constructor(target: ITarget, config: C, stateProvider: StateProvider);
     sync: (options?: SyncOptions) => Promise<void>;
     end(): Promise<void>;
 }
@@ -705,6 +615,453 @@ declare const flattenSchema: <T>(streamId: StreamId, schema: T) => FlattenedSche
 declare const createTemporaryFileStream: (streamId: StreamId) => TempFile;
 declare function safePath(path: string): string;
 
+/**
+ * Field types for file processing columns.
+ * Maps to JSON Schema types via fieldTypeToJsonSchema().
+ */
+type FieldType = "STRING" | "FLOAT" | "TIMESTAMP";
+declare enum FileFormat {
+    CSV = "CSV",
+    EXCEL = "EXCEL"
+}
+type ExcelFieldSpec = ExcelSourceColumn | ExcelDerivedField;
+interface ExcelSourceColumn {
+    type: FieldType;
+    column: string;
+    primaryKey?: boolean;
+}
+interface ExcelDerivedField {
+    variableName: string;
+    primaryKey?: boolean;
+    type: FieldType;
+}
+interface ExcelFieldMapping {
+    [key: string]: ExcelFieldSpec;
+}
+type ExcelExtractionConfig = ExcelSingleSheetExtractionConfig | ExcelCustomExtractorConfig;
+interface ExcelExtractionBaseConfig {
+    type: "single-sheet-extraction" | "processor";
+    extension: string;
+    fileNameValidator: (fileName: string) => boolean;
+    fileNameVariablesExtractor: (fileName: string, workbook?: WorkBook) => {
+        [key: string]: string | undefined;
+    };
+    tableName: string | ((fileName: string, workbook?: WorkBook, variables?: {
+        [key: string]: string | undefined;
+    }) => string);
+    replicationMethod: ReplicationMethod;
+}
+interface ExcelCustomExtractorConfig extends ExcelExtractionBaseConfig {
+    type: "processor";
+    processor: (workbook: WorkBook) => Promise<Record<string, string>[]>;
+}
+interface ExcelSingleSheetExtractionConfig extends ExcelExtractionBaseConfig {
+    type: "single-sheet-extraction";
+    columns: ExcelFieldMapping;
+    sheetName?: string;
+    numberOfRowsToSkip: number;
+}
+type CsvFieldsConfig = CsvFieldsArrayConfig | CsvFieldsDictConfig;
+type CsvFieldsArrayConfig = Array<{
+    key: string;
+    valueTransformer?: (val: any) => string;
+    type: FieldType;
+}>;
+interface CsvFieldsDictConfig {
+    [key: string]: {
+        key: string;
+        valueTransformer?: (val: any) => string;
+        type: FieldType;
+    };
+}
+interface CsvFileConfig {
+    encoding?: string;
+    separator: string;
+    fields: CsvFieldsConfig;
+    addSyncedAtColumn?: boolean;
+}
+interface FileStreamBaseConfig {
+    streamId: string;
+    replicationMethod: ReplicationMethod;
+    primaryKeys: string[];
+}
+interface CsvStreamConfig extends FileStreamBaseConfig {
+    format: FileFormat.CSV;
+    csv: CsvFileConfig;
+}
+interface ExcelStreamConfig extends FileStreamBaseConfig {
+    format: FileFormat.EXCEL;
+    excel: ExcelExtractionConfig;
+}
+type FileStreamConfig = CsvStreamConfig | ExcelStreamConfig;
+interface FileStreamEntry {
+    config: FileStreamConfig;
+    filePath: string | string[];
+}
+
+type JsonSchemaProperty = {
+    type: (string | "null")[];
+    format?: string;
+};
+/**
+ * Convert a FieldType to a JSON Schema property definition.
+ */
+declare function fieldTypeToJsonSchema(fieldType: FieldType): JsonSchemaProperty;
+/**
+ * Convert an ExcelFieldMapping to a JSON Schema properties object.
+ */
+declare function excelFieldsToJsonSchema(fields: ExcelFieldMapping): {
+    type: "object";
+    properties: Record<string, JsonSchemaProperty>;
+};
+/**
+ * Convert a CsvFieldsConfig to a JSON Schema properties object.
+ */
+declare function csvFieldsToJsonSchema(config: CsvFieldsConfig): {
+    type: "object";
+    properties: Record<string, JsonSchemaProperty>;
+};
+/**
+ * Extract all output keys from a CsvFieldsConfig to use as composite primary keys.
+ * Useful for FULL_TABLE replication where all fields form the PK (since TRUNCATE + MERGE = INSERT).
+ */
+declare function extractPrimaryKeysFromCsvConfig(config: CsvFieldsConfig): string[];
+/**
+ * Extract primary keys from an ExcelFieldMapping.
+ */
+declare function extractPrimaryKeysFromExcelFields(fields: ExcelFieldMapping): string[];
+
+/**
+ * Converts an Excel column letter (A, B, C, AA, AZ, etc.) to its zero-based index.
+ */
+declare const excelColumnToIndex: (columnLetter: string) => number;
+/**
+ * Converts a zero-based column index to Excel column letter(s).
+ */
+declare const indexToExcelColumn: (columnIndex: number) => string;
+/**
+ * Converts an Excel row number (1-based) to zero-based array index.
+ */
+declare const excelRowToIndex: (rowNumber: string | number) => number;
+/**
+ * Converts a zero-based array index to Excel row number (1-based).
+ */
+declare const indexToExcelRow: (index: number) => number;
+/**
+ * Parses an Excel cell reference (like "A1", "Z24", "AA100") into column and row indices.
+ */
+declare const parseCellReference: (cellRef: string) => {
+    columnIndex: number;
+    rowIndex: number;
+};
+/**
+ * Creates an Excel cell reference from column and row indices.
+ */
+declare const createCellReference: (columnIndex: number, rowIndex: number) => string;
+/**
+ * Low-level Excel extractor: returns array of row records from a workbook.
+ */
+declare const extractExcelRows: (localFilePath: string, conf: ExcelExtractionConfig) => Promise<Record<string, string>[]>;
+/**
+ * Validates an Excel file configuration.
+ */
+declare const validateExcelConfig: (config: ExcelExtractionConfig) => void;
+/**
+ * Finds all matching configurations for a given filename.
+ */
+declare const findAllMatchingConfigs: (filename: string, configs: ExcelExtractionConfig[]) => ExcelExtractionConfig[];
+/**
+ * Creates an async generator from Excel data.
+ */
+declare function createExcelGenerator(data: Record<string, string>[]): AsyncGenerator<Record<string, any>, void, unknown>;
+
+/**
+ * Builder pattern for creating Excel configurations.
+ *
+ * Sensible defaults:
+ * - fileNameValidator: accepts all files
+ * - fileNameVariablesExtractor: extracts nothing
+ * - replicationMethod: FULL_TABLE
+ */
+declare class ExcelExtractionConfigBuilder {
+    private config;
+    static create(): ExcelExtractionConfigBuilder;
+    extension(ext: string): ExcelExtractionConfigBuilder;
+    tableName(name: string | ((fileName: string, workbook?: WorkBook, variables?: {
+        [key: string]: string | undefined;
+    }) => string)): ExcelExtractionConfigBuilder;
+    fileValidator(validator: (filename: string) => boolean): ExcelExtractionConfigBuilder;
+    variablesExtractor(extractor: (filename: string, workbook?: WorkBook) => {
+        [key: string]: string | undefined;
+    }): ExcelExtractionConfigBuilder;
+    replicationMethod(method: ReplicationMethod): ExcelExtractionConfigBuilder;
+    columns(cols: {
+        [key: string]: ExcelFieldSpec;
+    }): ExcelExtractionConfigBuilder;
+    singleSheet(sheetName?: string, skipRows?: number): ExcelExtractionConfigBuilder;
+    processor(processorFn: (workbook: WorkBook) => Promise<Record<string, string>[]>): ExcelExtractionConfigBuilder;
+    build(): ExcelExtractionConfig;
+}
+
+/**
+ * Async generator that yields parsed rows from a CSV file.
+ * Supports custom separators, encoding, BOM stripping, and field transformers.
+ */
+declare function rowGeneratorFromCsv(path: string, fileConfig: CsvFileConfig): AsyncGenerator<Record<string, any>>;
+/**
+ * Validates CSV header row against expected configuration.
+ */
+declare const checkCsvHeaderRow: (path: string, fileConfig: CsvFileConfig) => Promise<void>;
+
+/**
+ * Writes an array of objects directly to a CSV file.
+ */
+declare const writeDataToCsv: (fileName: string, data: any[]) => Promise<void>;
+/**
+ * Writes data from an async generator to CSV with batch writing (10k row batches).
+ */
+declare const writeGeneratorToCSV: (generator: AsyncGenerator, outputFileName: string) => Promise<number>;
+
+/**
+ * Builder pattern for creating CSV configurations.
+ *
+ * Usage:
+ * ```ts
+ * const config = CsvExtractionConfigBuilder.create()
+ *   .separator(";")
+ *   .encoding("latin1")
+ *   .addSyncedAtColumn()
+ *   .fieldsFromDict({
+ *     "Product ID":   { key: "product_id",   type: "STRING" },
+ *     "Product Name": { key: "product_name", type: "STRING" },
+ *     "Price":        { key: "price",        type: "FLOAT" },
+ *   })
+ *   .build();
+ * ```
+ */
+declare class CsvExtractionConfigBuilder {
+    private _separator;
+    private _encoding?;
+    private _addSyncedAtColumn?;
+    private _fields?;
+    private _fieldsMode?;
+    private _replicationMethod?;
+    private _primaryKeys?;
+    static create(): CsvExtractionConfigBuilder;
+    separator(sep: string): CsvExtractionConfigBuilder;
+    encoding(enc: string): CsvExtractionConfigBuilder;
+    addSyncedAtColumn(enabled?: boolean): CsvExtractionConfigBuilder;
+    fieldsFromDict(mapping: {
+        [header: string]: {
+            key: string;
+            valueTransformer?: (val: any) => string;
+            type: FieldType;
+        };
+    }): CsvExtractionConfigBuilder;
+    fieldsFromArray(fields: Array<{
+        key: string;
+        valueTransformer?: (val: any) => string;
+        type: FieldType;
+    }>): CsvExtractionConfigBuilder;
+    replicationMethod(method: ReplicationMethod): CsvExtractionConfigBuilder;
+    primaryKeys(keys: string[]): CsvExtractionConfigBuilder;
+    build(): CsvFileConfig;
+    buildStreamConfig(streamId: string): FileStreamConfig;
+}
+
+/**
+ * FileStream bridges file processing (Excel/CSV) into the connector SDK pipeline.
+ *
+ * It extends Stream so it integrates with the standard Tap -> Stream -> Target flow.
+ * Each FileStream reads one file with one config and yields records.
+ */
+declare class FileStream extends Stream<Record<string, any>, FileStreamConfig> {
+    private localFilePaths;
+    constructor(config: FileStreamConfig, localFilePath: string | string[], tapState: InputTapState, target: ITarget);
+    /**
+     * Build JSON Schema from the file config's field definitions.
+     */
+    getSchema(): Promise<Schema | undefined>;
+    /**
+     * Yield records from all file(s).
+     * When multiple file paths are provided, records are yielded sequentially from each file.
+     */
+    _getRecords(): AsyncIterable<Record<string, any>>;
+}
+/**
+ * Create a FileStreamConfig from an ExcelExtractionConfig.
+ * Handles both single-sheet-extraction and processor config types.
+ *
+ * For single-sheet configs, primary keys are extracted from column definitions.
+ * For processor configs, primary keys default to an empty array.
+ *
+ * @param excelConfig - The Excel extraction config (single-sheet or processor)
+ * @param fileName - The base filename (used for variable extraction and table name resolution)
+ * @param localFilePath - Optional local file path; required for processor configs with dynamic table names
+ */
+declare function createExcelStreamConfig(excelConfig: ExcelExtractionConfig, fileName: string, localFilePath?: string): FileStreamConfig;
+/**
+ * Create a FileStreamConfig from a CsvFileConfig.
+ *
+ * By default, uses FULL_TABLE replication and extracts all field keys as composite primary keys.
+ * Override via the options parameter.
+ *
+ * @param streamId - The stream/table name
+ * @param csvConfig - The CSV file configuration
+ * @param options - Optional overrides for replicationMethod and primaryKeys
+ */
+declare function createCsvStreamConfig(streamId: string, csvConfig: CsvFileConfig, options?: {
+    replicationMethod?: ReplicationMethod;
+    primaryKeys?: string[];
+}): FileStreamConfig;
+/**
+ * Process an array of FileStreamEntry sequentially, then signal completion to the target.
+ *
+ * This is the recommended pattern for SFTP and similar multi-file ingestion flows
+ * where FileTap isn't needed.
+ *
+ * @param entries - Array of { config, filePath } pairs
+ * @param tapState - The tap state (typically `{ bookmarks: {} }`)
+ * @param target - The target (e.g. BigQueryTarget)
+ */
+declare function processFileStreams(entries: FileStreamEntry[], tapState: InputTapState, target: ITarget): Promise<void>;
+
+interface FileTapConfig {
+    [key: string]: any;
+}
+/**
+ * FileTap orchestrates multiple FileStreams.
+ *
+ * It extends Tap so it integrates with the standard Tap -> Stream -> Target flow.
+ * Provide file stream entries (config + file path pairs), and it creates a FileStream per entry.
+ *
+ * For simple sequential file processing without the Tap lifecycle, use processFileStreams() instead.
+ */
+declare class FileTap extends Tap<FileTapConfig> {
+    private entries;
+    constructor(target: ITarget, config: FileTapConfig, stateProvider: StateProvider, entries: FileStreamEntry[]);
+    /**
+     * Convenience constructor for the common case where all configs share the same file path.
+     */
+    static fromConfigs(target: ITarget, config: FileTapConfig, stateProvider: StateProvider, fileConfigs: FileStreamConfig[], sharedFilePath: string): FileTap;
+    init(): Promise<void>;
+}
+
+/**
+ * Variable extraction utilities for common filename patterns.
+ */
+declare class VariableExtractors {
+    /**
+     * Returns the filename as a variable.
+     */
+    static filename(): (filename: string) => {
+        [key: string]: string | undefined;
+    };
+    /**
+     * Extracts named capture groups from a regex pattern.
+     */
+    static regex(pattern: RegExp): (filename: string) => {
+        [key: string]: string | undefined;
+    };
+    /**
+     * Combines multiple extraction functions.
+     */
+    static combine(...extractors: ((filename: string) => {
+        [key: string]: string | undefined;
+    })[]): (filename: string) => {
+        [key: string]: string | undefined;
+    };
+}
+/**
+ * File pattern utilities for common filename validation patterns.
+ */
+declare class FilePatterns {
+    /**
+     * Creates a validator for files starting with a specific prefix (case-insensitive).
+     */
+    static startsWith(prefix: string): (filename: string) => boolean;
+    /**
+     * Creates a validator for files matching a regex pattern.
+     */
+    static regex(pattern: RegExp): (filename: string) => boolean;
+    /**
+     * Combines multiple validators with AND logic.
+     */
+    static and(...validators: ((filename: string) => boolean)[]): (filename: string) => boolean;
+    /**
+     * Combines multiple validators with OR logic.
+     */
+    static or(...validators: ((filename: string) => boolean)[]): (filename: string) => boolean;
+}
+
+/**
+ * Pre-instantiated SFTP client singleton.
+ * @deprecated Prefer creating your own SftpClient instance for better lifecycle control.
+ */
+declare const SftpService: Client;
+
+interface CloudStorageServiceOptions {
+    processedSuffix?: string;
+    supportedExtensions?: string[];
+}
+/**
+ * Service for interacting with Google Cloud Storage.
+ * Supports file download, upload, listing, and marker-file-based processing tracking.
+ */
+declare class CloudStorageService {
+    private storage;
+    private bucket;
+    private processedSuffix;
+    private supportedExtensions;
+    private path;
+    constructor(bucketName: string, path?: string, opts?: CloudStorageServiceOptions);
+    /**
+     * Lists all files in the bucket with an optional prefix.
+     */
+    listFiles(prefix?: string): Promise<string[]>;
+    /**
+     * Returns files that haven't been marked as processed.
+     * Filters by supported extensions if configured.
+     */
+    getUnprocessedFiles(): Promise<string[]>;
+    /**
+     * Creates a marker file to indicate a file has been processed.
+     */
+    createMarkerFile(fileName: string): Promise<void>;
+    /**
+     * Downloads a file from GCS to a local tmp directory.
+     * Returns the local path of the downloaded file.
+     */
+    downloadFile(filePath: string, fileName: string): Promise<string>;
+    /**
+     * Uploads a local file to GCS.
+     */
+    uploadFile(localPath: string, destPath: string): Promise<File>;
+    /**
+     * Reads a GCS object and returns its contents as a UTF-8 string.
+     */
+    readObjectAsString(objectPath: string): Promise<string>;
+    /**
+     * Writes a string to a GCS object with JSON content type.
+     */
+    writeStringObject(objectPath: string, contents: string): Promise<void>;
+    /**
+     * Uploads a local file to the bucket with a unique name based on prefix, streamId, and UUID.
+     * Returns the GCS File reference.
+     */
+    uploadFileWithUniqueName(filePath: string, prefix: string, streamId: string): Promise<File>;
+}
+
+/**
+ * Extract a ZIP (or other archive) file to a target directory.
+ * Uses the `decompress` library which supports zip, tar, gz, bz2, etc.
+ *
+ * @param zipFilePath - Path to the archive file
+ * @param extractedPath - Directory to extract into (created if it doesn't exist)
+ * @returns The output directory path
+ */
+declare const unzip: (zipFilePath: string, extractedPath: string) => Promise<string>;
+
 interface BigQueryConfig extends BaseConfig {
     schema: string;
     project_id: string;
@@ -729,6 +1086,7 @@ declare class BigQueryDBSync extends StreamWarehouseSyncService {
     getWarehouseTypeFromJSONSchema: (definition: JSONSchemaFieldDefinition) => string;
     getMergeQueries: () => string[];
     getReplaceQueries: () => string[];
+    getAppendQueries: () => string[];
     createDatabaseAndSchemaIfNotExists: (retryCount: number) => Promise<void>;
     createTable: () => Promise<void>;
     addColumns: (tableFields: InternalTableField[]) => Promise<void>;
@@ -754,7 +1112,7 @@ declare class BigQueryDBSync extends StreamWarehouseSyncService {
 }
 
 declare class BigQueryTarget extends ITarget<BigQueryConfig> {
-    constructor(config: BigQueryConfig, resolver: Resolver);
+    constructor(config: BigQueryConfig, stateProvider: StateProvider);
     static requiredConfigKeys: string[];
     newDBSyncInstance: (streamId: StreamId, database: string, schema: string, table: string) => BigQueryDBSync;
     genTableName: (key: string) => string;
@@ -763,4 +1121,14 @@ declare class BigQueryTarget extends ITarget<BigQueryConfig> {
     convertNumberIntoDecimal: (record: any, schema: FlattenedSchema) => any;
 }
 
-export { ALL_STREAM_ID_KEYWORD, API_CALLS_METRIC_NAME, Authenticator, BATCH_INTERVAL_MS, type BaseConfig, type BigQueryConfig, BigQueryTarget, type BigqueryLocation, type Bookmark, type Bookmarks, type CatalogFile, type CatalogMetadata, type CatalogStream, type CatalogStreamMetadata, type ColumnMappingStoreUnsafeToSafe, CounterMetric, DEFAULT_MAX_CONCURRENT_STREAMS, type DestinationType, type DirectionalRelationship, type DirectionalRelationships, EXECUTION_TIME_METRIC_NAME, type ErrorType, type FlattenedSchema, type GetConfig, type HTTPHeaders, type HTTPMethod, ITarget, type InclusionMode, type InputTapState, type InternalTableField, type JSONSchemaFieldDefinition, type Message, type MessageType, type MetadataEntry, type MetricConfiguration, MissingFieldInSchemaError, MissingSchemaError, type PostConfig, type ProgressMarkers, type PropertiesMetadata, RESTStream, ROWS_SYNCED_METRIC_NAME, RecordMessage, RenameColumnStore, type ReplicationMethod, ReplicationMethodMessage, Resolver, type SafeColumnNameConverterFn, type Schema, SchemaMessage, SchemaValidationError, type State, type StateHolder, StateMessage, StateService, Stream, type StreamId, StreamMetadata, type StreamName, type StreamRelationship, type StreamState, StreamWarehouseSyncService, type StreamWithTempFile, type SyncOptions, Tap, type TapState, type TargetSchemaHook, type TargetSchemaHookInput, type TempFile, type URLParams, type WarehouseTableField, _greaterThan, addWhalyFields, createTemporaryFileStream, extractStateForStream, finalizeStateProgressMarkers, findRelatedRelationships, flattenSchema, getAllMetrics, getAxiosInstance, getCounterMetric, getCounterMetrics, getDownloadFileApiCall, getJSONApiCall, getJSONApiCallWithFullResponse, getRawJSONApiCall, gracefulExit, haltAndCatchFire, incrementStreamState, loadJson, loadResolver, loadSchema, logger, postFormDataApiCall, postJSONApiCall, postUrlEncodedApiCall, printMemoryFootprint, removeParasiteProperties, safePath, writeStateFile };
+declare class GCSStateProvider implements StateProvider {
+    private bucketName;
+    private connectorId;
+    private service;
+    constructor(connectorId: string, bucketName: string);
+    private getObjectPath;
+    getState(): Promise<StateHolder>;
+    writeState(state: string): Promise<void>;
+}
+
+export { ALL_STREAM_ID_KEYWORD, API_CALLS_METRIC_NAME, Authenticator, BATCH_INTERVAL_MS, type BaseConfig, type BigQueryConfig, BigQueryTarget, type BigqueryLocation, type Bookmark, type Bookmarks, CloudStorageService, type CloudStorageServiceOptions, type ColumnMappingStoreUnsafeToSafe, CounterMetric, CsvExtractionConfigBuilder, type CsvFieldsArrayConfig, type CsvFieldsConfig, type CsvFieldsDictConfig, type CsvFileConfig, type CsvStreamConfig, DEFAULT_MAX_CONCURRENT_STREAMS, EXECUTION_TIME_METRIC_NAME, type ErrorType, type ExcelCustomExtractorConfig, type ExcelDerivedField, type ExcelExtractionBaseConfig, type ExcelExtractionConfig, ExcelExtractionConfigBuilder, type ExcelFieldMapping, type ExcelFieldSpec, type ExcelSingleSheetExtractionConfig, type ExcelSourceColumn, type ExcelStreamConfig, type FieldType, FileFormat, FilePatterns, FileStream, type FileStreamConfig, type FileStreamEntry, FileTap, type FileTapConfig, type FlattenedSchema, GCSStateProvider, type GetConfig, type HTTPHeaders, type HTTPMethod, ITarget, type InputTapState, type InternalTableField, type JSONSchemaFieldDefinition, type Message, type MessageType, type MetricConfiguration, MissingFieldInSchemaError, MissingSchemaError, type PostConfig, type ProgressMarkers, type PropertiesMetadata, RESTStream, ROWS_SYNCED_METRIC_NAME, RecordMessage, RenameColumnStore, ReplicationMethod, ReplicationMethodMessage, type SafeColumnNameConverterFn, type Schema, SchemaMessage, SchemaValidationError, SftpService, type State, type StateHolder, StateMessage, type StateProvider, StateService, Stream, type StreamId, type StreamState, StreamWarehouseSyncService, type StreamWithTempFile, type SyncOptions, Tap, type TapState, type TargetSchemaHook, type TargetSchemaHookInput, type TempFile, type URLParams, VariableExtractors, type WarehouseTableField, _greaterThan, addWhalyFields, checkCsvHeaderRow, createCellReference, createCsvStreamConfig, createExcelGenerator, createExcelStreamConfig, createTemporaryFileStream, csvFieldsToJsonSchema, excelColumnToIndex, excelFieldsToJsonSchema, excelRowToIndex, extractExcelRows, extractPrimaryKeysFromCsvConfig, extractPrimaryKeysFromExcelFields, extractStateForStream, fieldTypeToJsonSchema, finalizeStateProgressMarkers, findAllMatchingConfigs, flattenSchema, getAllMetrics, getAxiosInstance, getCounterMetric, getCounterMetrics, getDownloadFileApiCall, getJSONApiCall, getJSONApiCallWithFullResponse, getRawJSONApiCall, gracefulExit, haltAndCatchFire, incrementStreamState, indexToExcelColumn, indexToExcelRow, loadJson, loadSchema, logger, parseCellReference, postFormDataApiCall, postJSONApiCall, postUrlEncodedApiCall, printMemoryFootprint, processFileStreams, removeParasiteProperties, rowGeneratorFromCsv, safePath, unzip, validateExcelConfig, writeDataToCsv, writeGeneratorToCSV };