@whaly/connector-sdk 0.1.0 → 0.2.0

package/dist/index.d.mts

@@ -1,26 +1,90 @@
-import winston from 'winston';
+import { AxiosInstance, AxiosRequestHeaders, AxiosResponse, AxiosRequestConfig, AxiosError } from 'axios';
 import { Validator, ValidationError } from 'jsonschema';
-import moment from 'moment';
+import { Dayjs } from 'dayjs';
 import { WriteStream } from 'fs';
-import { AxiosInstance, AxiosRequestHeaders, AxiosResponse, AxiosRequestConfig, AxiosError } from 'axios';
+import { BigQuery, Dataset, Table } from '@google-cloud/bigquery';
 
-interface StreamRelationship {
-    left: StreamId;
-    right: StreamId;
-    from: string;
-    to: string;
-    type: "1-1" | "1-N" | "N-1";
+declare const loadJson: (fileName: string) => any;
+declare const loadSchema: (streamId: string) => any;
+
+declare const getAxiosInstance: (retryCount?: number) => AxiosInstance;
+interface GetConfig {
+    qs?: {
+        [key: string]: any;
+    };
+    headers?: AxiosRequestHeaders;
+    retryCount?: number;
 }
-interface DirectionalRelationship {
-    type: "1-N" | "N-1" | "1-1";
-    streamId: StreamId;
-    from: string;
-    to: string;
+declare const getRawJSONApiCall: <T>(endpoint: string, config: GetConfig, privateLogging?: boolean) => Promise<AxiosResponse<T>>;
+declare const getJSONApiCallWithFullResponse: <T>(endpoint: string, config: GetConfig, privateLogging?: boolean) => Promise<AxiosResponse<any, any, {}>>;
+declare const getJSONApiCall: <T>(endpoint: string, config: GetConfig, privateLogging?: boolean) => Promise<T>;
+declare const getDownloadFileApiCall: <T>(endpoint: string, config: GetConfig, output: string, privateLogging?: boolean) => Promise<{
+    outputDir: string;
+}>;
+interface PostConfig {
+    qs?: {
+        [key: string]: any;
+    };
+    headers?: AxiosRequestHeaders;
+    retryCount?: number;
 }
-interface DirectionalRelationships {
-    left: DirectionalRelationship[];
-    right: DirectionalRelationship[];
+declare const postJSONApiCall: <T>(endpoint: string, config: PostConfig, payload: any) => Promise<T>;
+declare const postFormDataApiCall: <T>(endpoint: string, config: PostConfig, payload: any) => Promise<T>;
+declare const postUrlEncodedApiCall: (endpoint: string, config: PostConfig, payload: any) => Promise<any>;
+
+declare const BATCH_INTERVAL_MS = 100;
+declare const logger: {
+    info: (message: string | any, ...args: any[]) => void;
+    error: (message: string | any, ...args: any[]) => void;
+    debug: (message: string | any, ...args: any[]) => void;
+    warn: (message: string | any, ...args: any[]) => void;
+    closeWinston: (cb: () => void) => void;
+};
+
+declare const ALL_STREAM_ID_KEYWORD = "all";
+declare const ROWS_SYNCED_METRIC_NAME = "rows_synced_count";
+declare const API_CALLS_METRIC_NAME = "api_calls_count";
+declare const EXECUTION_TIME_METRIC_NAME = "execution_time_ms";
+interface MetricConfiguration {
+    isEnabled: () => boolean;
+    getStreamIds?: () => string[];
 }
+declare const getCounterMetrics: (name: string, streamIds: string[]) => CounterMetric[];
+declare const getCounterMetric: (name: string, streamId?: string) => CounterMetric;
+declare const getAllMetrics: () => CounterMetric[];
+declare class CounterMetric {
+    private value;
+    name: string;
+    streamId: string;
+    constructor(name: string, streamId: string);
+    increment(inc?: number): void;
+    getStreamId(): string;
+    getName(): string;
+    getValue(): number;
+}
+
+declare const printMemoryFootprint: (prefix: string) => void;
+
+type ErrorType = "unauthorized" | "unknown";
+
+declare const haltAndCatchFire: (errorType: ErrorType, errorText: string, errorDebugText: string) => Promise<void>;
+
+declare function gracefulExit(exitCode: number): Promise<void>;
+
+declare enum ReplicationMethod {
+    INCREMENTAL = "INCREMENTAL",
+    FULL_TABLE = "FULL_TABLE"
+}
+
+interface HTTPHeaders {
+    [headerName: string]: any;
+}
+interface URLParams {
+    [paramName: string]: any;
+}
+type HTTPMethod = "GET" | "POST";
+
+type StreamId = string;
 
 interface PropertiesMetadata {
     [propName: string]: {
@@ -70,7 +134,6 @@ declare class SchemaMessage implements Message {
     displayLabel: string | undefined;
     description: string | undefined;
     propertiesMetadata: PropertiesMetadata | undefined;
-    relationships: DirectionalRelationships;
     constructor(opts: {
         stream: StreamId;
         schema: any;
@@ -78,7 +141,6 @@ declare class SchemaMessage implements Message {
         bookmarkProperties?: string[];
         displayLabel?: string;
         description?: string;
-        relationships: DirectionalRelationships;
         propertiesMetadata?: PropertiesMetadata | undefined;
     });
     toString(): string;
@@ -94,26 +156,6 @@ interface TargetSchemaHook {
     writeSchema: (input: TargetSchemaHookInput) => Promise<void>;
 }
 
-type ErrorType = "unauthorized" | "unknown";
-
-declare abstract class Resolver {
-    constructor();
-    abstract checkIfCanSync(): Promise<boolean>;
-    abstract markSyncStarted(): Promise<void>;
-    abstract startVPNIfNeeded(): Promise<void>;
-    abstract getConfig(command: ShoreCommand): Promise<ShoreConfig>;
-    abstract writeCatalog(catalog: CatalogFile): Promise<void>;
-    abstract writeState(state: string): Promise<void>;
-    abstract markSyncFailed(): Promise<void>;
-    abstract markSyncComplete(): Promise<void>;
-    abstract markDiscoveryComplete(): Promise<void>;
-    abstract flushMetrics(): Promise<void>;
-    abstract onError(errorType: ErrorType, errorText: string, errorDebugText: string): Promise<void>;
-    abstract getSchemaHooks(): TargetSchemaHook[];
-    abstract updateSourceValue(optionKey: string, optionValue: string): Promise<void>;
-    abstract getLogFormat(): winston.Logform.Format;
-}
-
 interface FlattenedSchema {
     [unsafePropertyName: string]: JSONSchemaFieldDefinition;
 }
@@ -140,8 +182,9 @@ interface StreamWithTempFile {
     tempFile: TempFile;
 }
 interface BaseConfig {
-    schema: string;
+    connector_id: string;
     database: string;
+    schema: string;
 }
 
 interface ColumnMappingStoreUnsafeToSafe {
@@ -216,7 +259,7 @@ declare class StreamState$1 {
     streamId: StreamId;
     schema?: FlattenedSchema;
     dbSync: StreamWarehouseSyncService;
-    batchDate: moment.Moment;
+    batchDate: Dayjs;
     batchedRowCount: number;
     fileToLoad: TempFile;
     syncedRowCount: number;
@@ -239,11 +282,19 @@ declare class StreamState$1 {
     setHasBeenLoadedYet(): void;
 }
 
+interface StateHolder {
+    state?: any;
+}
+interface StateProvider {
+    getState(): Promise<StateHolder>;
+    writeState(state: string): Promise<void>;
+}
+
 declare abstract class ITarget<C extends BaseConfig = BaseConfig> {
     config: C;
     schemaHooks: TargetSchemaHook[];
-    resolver: Resolver;
-    syncTime: moment.Moment;
+    syncTime: Dayjs;
+    stateProvider: StateProvider;
     batchedState: any;
     flushedState: any;
     renameColumnStore: RenameColumnStore;
@@ -251,7 +302,7 @@ declare abstract class ITarget<C extends BaseConfig = BaseConfig> {
         [streamName: string]: StreamState$1;
     };
     validator: Validator;
-    constructor(config: C, resolver: Resolver);
+    constructor(config: C, stateProvider: StateProvider);
     static requiredConfigKeys: string[];
     abstract newDBSyncInstance: (streamId: StreamId, database: string, schema: string, table: string) => StreamWarehouseSyncService;
     abstract genTableName: (streamId: string) => string;
@@ -273,165 +324,6 @@ declare abstract class ITarget<C extends BaseConfig = BaseConfig> {
     static uploadSingleStreamToWarehouse: (streamState: StreamState$1) => Promise<void>;
 }
 
-type ReplicationMethod = "INCREMENTAL" | "FULL_TABLE" | "LOG_BASED";
-type SyncFunction<C> = (target: ITarget, catalogStream: CatalogStream, config: C, catalog: Catalog) => Promise<void>;
-type SourceType = `HUBSPOT` | `LAGROWTHMACHINE` | `SEGMENT` | `COHORT` | `AIRTABLE` | `AIRTABLE_OAUTH` | `BUBBLE` | `PIPEDRIVE` | `LINKEDIN_ADS` | `FACEBOOK_ADS` | `SEGMENT` | `SALESFORCE` | `SALESFORCE_SANDBOX` | `RECRUITCRM` | `GOOGLE_SHEETS` | `GOOGLE_ADS` | `GOOGLE_ANALYTICS` | `SLACK` | `STRIPE` | `ORBIT` | `GITHUB_STARS` | `JAFFLE_SHOP` | `POSTGRES` | `PENNYLANE` | `PENNYLANE_REDSHIFT` | `AIRCALL` | `AIRCALL_OAUTH` | `FAST_POSTGRES_FULL_TABLE` | `WOOCOMMERCE`;
-type DestinationType = `GOOGLE_BIGQUERY` | `SNOWFLAKE`;
-interface ShoreConfig {
-    command: ShoreCommand;
-    destination: DestinationType;
-    config: any;
-    targetConfig: any;
-    state?: any;
-    catalog?: CatalogFile;
-}
-type ShoreCommand = "READ" | "DISCOVER";
-type ResolverType = "WHALY" | "LOCAL";
-
-type InclusionMode = "available" | "automatic" | "unsupported";
-interface MetadataEntry {
-    selected?: boolean;
-    "replication-method"?: ReplicationMethod;
-    "replication-key"?: string;
-    "view-key-properties"?: string[];
-    "inclusion"?: InclusionMode;
-    "selected-by-default"?: boolean;
-    "valid-replication-keys"?: string[];
-    "forced-replication-method"?: ReplicationMethod;
-    "forced-replication-key"?: string;
-    "table-key-properties"?: string[];
-    "schema-name"?: string;
-    "is-view"?: boolean;
-    "row-count"?: number;
-    "database-name"?: string;
-    "sql-datatype"?: string;
-    "whaly-display-label"?: string;
-    "whaly-description"?: string;
-    "whaly-parent-stream"?: StreamId;
-}
-interface CatalogMetadata {
-    breadcrumb: string[];
-    metadata: MetadataEntry;
-}
-declare class MetadataMap extends Map<string[], MetadataEntry> {
-    private areEquals;
-    private findExistingEntry;
-    get(key: string[]): MetadataEntry | undefined;
-    set(key: string[], value: MetadataEntry): this;
-}
-declare class StreamMetadata {
-    metadataByBreadcrumb: MetadataMap;
-    constructor();
-    fromCatalogStreamMetadata(metadataArr: CatalogMetadata[]): this;
-    toString(): MetadataMap;
-    get(breadcrumb: string[]): MetadataEntry | undefined;
-    write(breadcrumb: string[], key: string, value: any): this;
-    private getRootBreadcrumbMetadata;
-    isStreamSelected(): boolean;
-    getKeyProperties(): string[];
-    getReplicationKey(): string | undefined;
-    getReplicationMethod(): ReplicationMethod | undefined;
-    getForcedReplicationMethod(): ReplicationMethod | undefined;
-    toList(): CatalogMetadata[];
-}
-
-type StreamId = string;
-type StreamName = string;
-interface CatalogStream {
-    tap_stream_id: StreamId;
-    stream: StreamName;
-    schema: any;
-    metadata: CatalogMetadata[];
-}
-interface CatalogFile {
-    streams: CatalogStream[];
-}
-declare class Catalog {
-    streams?: CatalogStream[];
-    static fromFile(fileName: string): Catalog;
-    static fromCatalogFile(catalogFile: CatalogFile): Catalog;
-    getSelectedStreams(): CatalogStream[];
-}
-
-declare const writeStateFile: (resolver: Resolver, state: string) => Promise<void>;
-declare const writeCatalogFile: (resolver: Resolver, catalog: CatalogFile) => Promise<void>;
-declare const loadShoreConfig: (resolver: Resolver) => Promise<ShoreConfig>;
-declare const loadJson: (fileName: string) => any;
-declare const loadSchema: (streamId: string) => any;
-declare const checkRequiredConfigKeys: (args: any, requiredConfigKeys: string[]) => void;
-
-declare const getAxiosInstance: (retryCount?: number) => AxiosInstance;
-interface GetConfig {
-    qs?: {
-        [key: string]: any;
-    };
-    headers?: AxiosRequestHeaders;
-    retryCount?: number;
-}
-declare const getRawJSONApiCall: <T>(endpoint: string, config: GetConfig, privateLogging?: boolean) => Promise<AxiosResponse<T>>;
-declare const getJSONApiCallWithFullResponse: <T>(endpoint: string, config: GetConfig, privateLogging?: boolean) => Promise<AxiosResponse<any, any, {}>>;
-declare const getJSONApiCall: <T>(endpoint: string, config: GetConfig, privateLogging?: boolean) => Promise<T>;
-declare const getDownloadFileApiCall: <T>(endpoint: string, config: GetConfig, output: string, privateLogging?: boolean) => Promise<{
-    outputDir: string;
-}>;
-interface PostConfig {
-    qs?: {
-        [key: string]: any;
-    };
-    headers?: AxiosRequestHeaders;
-    retryCount?: number;
-}
-declare const postJSONApiCall: <T>(endpoint: string, config: PostConfig, payload: any) => Promise<T>;
-declare const postFormDataApiCall: <T>(endpoint: string, config: PostConfig, payload: any) => Promise<T>;
-declare const postUrlEncodedApiCall: (endpoint: string, config: PostConfig, payload: any) => Promise<any>;
-
-declare const BATCH_INTERVAL_MS = 100;
-declare const logger: {
-    info: (message: string | any, ...args: any[]) => void;
-    error: (message: string | any, ...args: any[]) => void;
-    debug: (message: string | any, ...args: any[]) => void;
-    warn: (message: string | any, ...args: any[]) => void;
-    closeWinston: (cb: () => void) => void;
-};
-
-declare const ALL_STREAM_ID_KEYWORD = "all";
-declare const ROWS_SYNCED_METRIC_NAME = "rows_synced_count";
-declare const API_CALLS_METRIC_NAME = "api_calls_count";
-declare const EXECUTION_TIME_METRIC_NAME = "execution_time_ms";
-interface MetricConfiguration {
-    isEnabled: () => boolean;
-    getStreamIds?: () => string[];
-}
-declare const getCounterMetrics: (name: string, streamIds: string[]) => CounterMetric[];
-declare const getCounterMetric: (name: string, streamId?: string) => CounterMetric;
-declare const getAllMetrics: () => CounterMetric[];
-declare class CounterMetric {
-    private value;
-    name: string;
-    streamId: string;
-    constructor(name: string, streamId: string);
-    increment(inc?: number): void;
-    getStreamId(): string;
-    getName(): string;
-    getValue(): number;
-}
-
-declare const printMemoryFootprint: (prefix: string) => void;
-
-declare const haltAndCatchFire: (errorType: ErrorType, errorText: string, errorDebugText: string) => Promise<void>;
-
-declare function gracefulExit(exitCode: number): Promise<void>;
-
-declare const findRelatedRelationships: (streamId: StreamId, allRels: StreamRelationship[]) => DirectionalRelationships;
-
-interface HTTPHeaders {
-    [headerName: string]: any;
-}
-interface URLParams {
-    [paramName: string]: any;
-}
-type HTTPMethod = "GET" | "POST";
-
 interface InputTapState {
     bookmarks?: {
         [streamId: string]: StreamState;
@@ -474,11 +366,9 @@ declare class StateService {
     bookmarks: Bookmarks;
     constructor();
     static getInstance(): StateService;
-    setBookmark(streamId: StreamId, ts: moment.Moment): void;
-    setBookmarkV2(streamId: StreamId, streamState: StreamState): void;
-    setBookmarkSignpostV2(streamId: StreamId, signpostValue: string): void;
-    getBookmark(streamId: StreamId): moment.Moment;
-    getBookmarkV2(streamId: StreamId): StreamState;
+    setBookmark(streamId: StreamId, streamState: StreamState): void;
+    setBookmarkSignpost(streamId: StreamId, signpostValue: string): void;
+    getBookmark(streamId: StreamId): StreamState;
     clearState(): void;
     get(): State;
     forwardStateToTarget(target: ITarget): Promise<void>;
@@ -499,13 +389,11 @@ declare class Authenticator<C> {
  * C: Type of the tap configuration
  * P: For children Streams, this is the type of the parent
  */
-declare abstract class StreamV2<O = any, C = any, P = undefined> {
-    _config: C;
-    _tapName: string;
-    _tapState: InputTapState;
-    _target: ITarget;
-    rootMetadataEntry: MetadataEntry | undefined;
-    forcedReplicationMethod: ReplicationMethod | undefined;
+declare abstract class Stream<O = any, C = any, P = undefined> {
+    config: C;
+    tapState: InputTapState;
+    target: ITarget;
+    replicationMethod: ReplicationMethod | undefined;
     selectedByDefault: boolean;
     STATE_MSG_FREQUENCY: number;
     streamId: StreamId;
@@ -513,17 +401,15 @@ declare abstract class StreamV2<O = any, C = any, P = undefined> {
     displayLabel: string | undefined;
     description: string | undefined;
     primaryKey: string[];
-    forcedReplicationKey: string | undefined;
+    replicationKey: string | undefined;
     useStateFromStreamId: string | undefined;
-    relationships: StreamRelationship[];
-    children: StreamV2<any, any, O | O[]>[];
+    children: Stream<any, any, O | O[]>[];
     isSorted: boolean;
     isSilent: boolean;
     childConcurrency: number;
     rowsSyncedMetricsConf: MetricConfiguration;
     executionTimeMetricsConf: MetricConfiguration;
-    constructor(tapName: string, config: C, state: InputTapState, target: ITarget, childConcurrency?: number);
-    setRootMetadataEntry: (rootMetadataEntry: MetadataEntry) => void;
+    constructor(config: C, tapState: InputTapState, target: ITarget, childConcurrency?: number);
     /**
      * Sync this stream
      * Called only for streams with no parents, aka. "root tap streams"
@@ -560,17 +446,17 @@ declare abstract class StreamV2<O = any, C = any, P = undefined> {
      * or `start_date` config if set,
      * or the UNIX Epoch
      **/
-    getStartingTimestamp(): moment.Moment;
+    getStartingTimestamp(): Dayjs;
     /**
      * Return the max allowable bookmark value for this stream's replication key.
      *
-     * For timestamp-based replication keys, this defaults to `moment()`. For
+     * For timestamp-based replication keys, this defaults to `dayjs()`. For
      * non-timestamp replication keys, default to `undefined`.
      *
      * Override this value to prevent bookmarks from being advanced in cases where we
      * may only have a partial set of records.
      */
-    getReplicationKeySignpost: () => Promise<moment.Moment | undefined>;
+    getReplicationKeySignpost: () => Promise<Dayjs | undefined>;
     /**
      * Return the default replication method for the stream that will be used to write the catalog.
      */
@@ -615,9 +501,9 @@ declare abstract class StreamV2<O = any, C = any, P = undefined> {
  * R: Type of the API result
  * NPT: Type of the "nextPageToken"
  */
-declare abstract class RESTStreamV2<R, O, NPT, C, P = undefined> extends StreamV2<O, C, P> {
+declare abstract class RESTStream<R, O, NPT, C, P = undefined> extends Stream<O, C, P> {
     axiosInstance: AxiosInstance | undefined;
-    constructor(tapName: string, config: C, state: InputTapState, target: ITarget, childConcurrency?: number);
+    constructor(config: C, state: InputTapState, target: ITarget, childConcurrency?: number);
     /**
      * Prepare an Axios request object.
      * Pagination information can be parsed from `nextPageToken` if `nextPageToken` is not undefined.
@@ -672,7 +558,7 @@ declare abstract class RESTStreamV2<R, O, NPT, C, P = undefined> extends StreamV
      * TODO: Make URL + Path templatisable and replace those with Extractor (or Tap) config
      * @returns
      */
-    getUrl(parent?: P): string;
+    getNextUrl(previousToken?: NPT, parent?: P): string;
     /**
      * Parse the response and return an iterator of result rows.
      *
@@ -683,20 +569,20 @@ declare abstract class RESTStreamV2<R, O, NPT, C, P = undefined> extends StreamV
 }
 
 declare const DEFAULT_MAX_CONCURRENT_STREAMS = 5;
+type SyncOptions = {
+    include?: string[];
+    exclude?: string[];
+};
 declare abstract class Tap<C> {
-    _target: ITarget;
-    _config: C;
-    _streams: StreamV2[];
-    _concurrency: number;
+    target: ITarget;
+    config: C;
+    streams: Stream[];
+    concurrency: number;
+    stateProvider: StateProvider;
+    tapState: InputTapState;
     abstract init(): Promise<void>;
-    abstract requiredConfigKeys(): string[];
-    constructor(target: ITarget, config: C);
-    discover: (existingCatalog: CatalogFile | undefined) => Promise<CatalogFile>;
-    sync: (catalog: Catalog) => Promise<void>;
-    generateMetadataFromStream(stream: StreamV2, parentStreamId: string | undefined): Promise<{
-        metadata: CatalogMetadata[];
-    }>;
-    initiateSync(catalog: Catalog): Promise<void>;
+    constructor(target: ITarget, config: C, stateProvider: StateProvider);
+    sync: (options?: SyncOptions) => Promise<void>;
     end(): Promise<void>;
 }
 
@@ -722,4 +608,71 @@ declare const flattenSchema: <T>(streamId: StreamId, schema: T) => FlattenedSche
 declare const createTemporaryFileStream: (streamId: StreamId) => TempFile;
 declare function safePath(path: string): string;
 
-export { ALL_STREAM_ID_KEYWORD, API_CALLS_METRIC_NAME, Authenticator, BATCH_INTERVAL_MS, type BaseConfig, type Bookmark, type Bookmarks, Catalog, type CatalogFile, type CatalogMetadata, type CatalogStream, type ColumnMappingStoreUnsafeToSafe, CounterMetric, DEFAULT_MAX_CONCURRENT_STREAMS, type DestinationType, type DirectionalRelationship, type DirectionalRelationships, EXECUTION_TIME_METRIC_NAME, type ErrorType, type FlattenedSchema, type GetConfig, type HTTPHeaders, type HTTPMethod, ITarget, type InclusionMode, type InputTapState, type InternalTableField, type JSONSchemaFieldDefinition, type Message, type MessageType, type MetadataEntry, type MetricConfiguration, MissingFieldInSchemaError, MissingSchemaError, type PostConfig, type ProgressMarkers, type PropertiesMetadata, RESTStreamV2, ROWS_SYNCED_METRIC_NAME, RecordMessage, RenameColumnStore, type ReplicationMethod, ReplicationMethodMessage, Resolver, type ResolverType, type SafeColumnNameConverterFn, type Schema, SchemaMessage, SchemaValidationError, type ShoreCommand, type ShoreConfig, type SourceType, type State, StateMessage, StateService, type StreamId, StreamMetadata, type StreamName, type StreamRelationship, type StreamState, StreamV2, StreamWarehouseSyncService, type StreamWithTempFile, type SyncFunction, Tap, type TapState, type TargetSchemaHook, type TargetSchemaHookInput, type TempFile, type URLParams, type WarehouseTableField, _greaterThan, addWhalyFields, checkRequiredConfigKeys, createTemporaryFileStream, extractStateForStream, finalizeStateProgressMarkers, findRelatedRelationships, flattenSchema, getAllMetrics, getAxiosInstance, getCounterMetric, getCounterMetrics, getDownloadFileApiCall, getJSONApiCall, getJSONApiCallWithFullResponse, getRawJSONApiCall, gracefulExit, haltAndCatchFire, incrementStreamState, loadJson, loadSchema, loadShoreConfig, logger, postFormDataApiCall, postJSONApiCall, postUrlEncodedApiCall, printMemoryFootprint, removeParasiteProperties, safePath, writeCatalogFile, writeStateFile };
+interface BigQueryConfig extends BaseConfig {
+    schema: string;
+    project_id: string;
+    loading_deck_gcs_bucket_name: string;
+    location?: BigqueryLocation;
+}
+type BigqueryLocation = "US" | "EU";
+
+declare class BigQueryDBSync extends StreamWarehouseSyncService {
+    config: BigQueryConfig;
+    bqClient: BigQuery;
+    bqDataset: Dataset;
+    bqTable: Table;
+    bqStagingTable: Table;
+    tableName: string;
+    stagingTableName: string;
+    sqlTableId: string;
+    sqlStagingTableId: string;
+    constructor(config: BigQueryConfig, streamId: StreamId, database: string, schema: string, table: string, renameColumnStore: RenameColumnStore);
+    getSerializedRecord: (record: any) => string;
+    safeColumnName: (key: string) => string;
+    getWarehouseTypeFromJSONSchema: (definition: JSONSchemaFieldDefinition) => string;
+    getMergeQueries: () => string[];
+    getReplaceQueries: () => string[];
+    createDatabaseAndSchemaIfNotExists: (retryCount: number) => Promise<void>;
+    createTable: () => Promise<void>;
+    addColumns: (tableFields: InternalTableField[]) => Promise<void>;
+    createStagingArea: () => Promise<void>;
+    loadStreamInStagingArea: (localFilePath: string) => Promise<void>;
+    deleteStagingArea: () => Promise<void>;
+    getTablesInSchema: () => Promise<{
+        tableName: string;
+    }[]>;
+    getTableColumnsFromWarehouse: () => Promise<WarehouseTableField[]>;
+    runQueries: (queries: string[]) => Promise<void>;
+    /**
+     *
+     * Generate a TableField definition object that can be passed to BigQuery APIs to define a column on a table
+     *
+     * @param colName
+     * @param schemaFieldDefinition
+     * @returns
+     */
+    private getBigqueryTableFieldFromInternalSchemaFieldDefinition;
+    private generateBigquerySchema;
+    private loadGCSFileInTable;
+}
+
+declare class BigQueryTarget extends ITarget<BigQueryConfig> {
+    constructor(config: BigQueryConfig, stateProvider: StateProvider);
+    static requiredConfigKeys: string[];
+    newDBSyncInstance: (streamId: StreamId, database: string, schema: string, table: string) => BigQueryDBSync;
+    genTableName: (key: string) => string;
+    safeColumnNameConverter: (key: string) => string;
+    validateDateRange: (record: any, schema: FlattenedSchema) => any;
+    convertNumberIntoDecimal: (record: any, schema: FlattenedSchema) => any;
+}
+
+declare class GCSStateProvider implements StateProvider {
+    private bucketName;
+    private connectorId;
+    constructor(connectorId: string, bucketName: string);
+    private getObjectPath;
+    getState(): Promise<StateHolder>;
+    writeState(state: string): Promise<void>;
+}
+
+export { ALL_STREAM_ID_KEYWORD, API_CALLS_METRIC_NAME, Authenticator, BATCH_INTERVAL_MS, type BaseConfig, type BigQueryConfig, BigQueryTarget, type BigqueryLocation, type Bookmark, type Bookmarks, type ColumnMappingStoreUnsafeToSafe, CounterMetric, DEFAULT_MAX_CONCURRENT_STREAMS, EXECUTION_TIME_METRIC_NAME, type ErrorType, type FlattenedSchema, GCSStateProvider, type GetConfig, type HTTPHeaders, type HTTPMethod, ITarget, type InputTapState, type InternalTableField, type JSONSchemaFieldDefinition, type Message, type MessageType, type MetricConfiguration, MissingFieldInSchemaError, MissingSchemaError, type PostConfig, type ProgressMarkers, type PropertiesMetadata, RESTStream, ROWS_SYNCED_METRIC_NAME, RecordMessage, RenameColumnStore, ReplicationMethod, ReplicationMethodMessage, type SafeColumnNameConverterFn, type Schema, SchemaMessage, SchemaValidationError, type State, type StateHolder, StateMessage, type StateProvider, StateService, Stream, type StreamId, type StreamState, StreamWarehouseSyncService, type StreamWithTempFile, type SyncOptions, Tap, type TapState, type TargetSchemaHook, type TargetSchemaHookInput, type TempFile, type URLParams, type WarehouseTableField, _greaterThan, addWhalyFields, createTemporaryFileStream, extractStateForStream, finalizeStateProgressMarkers, flattenSchema, getAllMetrics, getAxiosInstance, getCounterMetric, getCounterMetrics, getDownloadFileApiCall, getJSONApiCall, getJSONApiCallWithFullResponse, getRawJSONApiCall, gracefulExit, haltAndCatchFire, incrementStreamState, loadJson, loadSchema, logger, postFormDataApiCall, postJSONApiCall, postUrlEncodedApiCall, printMemoryFootprint, removeParasiteProperties, safePath };