@whaly/connector-sdk 0.1.0 → 0.1.1

package/dist/index.d.mts

@@ -1,8 +1,31 @@
 import winston from 'winston';
-import { Validator, ValidationError } from 'jsonschema';
+import { AxiosInstance, AxiosRequestHeaders, AxiosResponse, AxiosRequestConfig, AxiosError } from 'axios';
 import moment from 'moment';
+import { Validator, ValidationError } from 'jsonschema';
 import { WriteStream } from 'fs';
-import { AxiosInstance, AxiosRequestHeaders, AxiosResponse, AxiosRequestConfig, AxiosError } from 'axios';
+import { BigQuery, Dataset, Table } from '@google-cloud/bigquery';
+
+type StreamId = string;
+type StreamName = string;
+interface CatalogStreamMetadata {
+    breadcrumb: string[];
+    metadata: Record<string, any>;
+}
+interface CatalogStream {
+    tap_stream_id: StreamId;
+    stream: StreamName;
+    schema: any;
+    metadata: CatalogStreamMetadata[];
+}
+interface CatalogFile {
+    streams: CatalogStream[];
+}
+
+type ReplicationMethod = "INCREMENTAL" | "FULL_TABLE";
+type DestinationType = `GOOGLE_BIGQUERY` | `SNOWFLAKE`;
+interface StateHolder {
+    state?: any;
+}
 
 interface StreamRelationship {
     left: StreamId;
@@ -100,8 +123,7 @@ declare abstract class Resolver {
     constructor();
     abstract checkIfCanSync(): Promise<boolean>;
     abstract markSyncStarted(): Promise<void>;
-    abstract startVPNIfNeeded(): Promise<void>;
-    abstract getConfig(command: ShoreCommand): Promise<ShoreConfig>;
+    abstract getState(): Promise<StateHolder>;
     abstract writeCatalog(catalog: CatalogFile): Promise<void>;
     abstract writeState(state: string): Promise<void>;
     abstract markSyncFailed(): Promise<void>;
@@ -114,6 +136,131 @@ declare abstract class Resolver {
     abstract getLogFormat(): winston.Logform.Format;
 }
 
+declare const writeStateFile: (resolver: Resolver, state: string) => Promise<void>;
+declare const loadJson: (fileName: string) => any;
+declare const loadSchema: (streamId: string) => any;
+
+declare const getAxiosInstance: (retryCount?: number) => AxiosInstance;
+interface GetConfig {
+    qs?: {
+        [key: string]: any;
+    };
+    headers?: AxiosRequestHeaders;
+    retryCount?: number;
+}
+declare const getRawJSONApiCall: <T>(endpoint: string, config: GetConfig, privateLogging?: boolean) => Promise<AxiosResponse<T>>;
+declare const getJSONApiCallWithFullResponse: <T>(endpoint: string, config: GetConfig, privateLogging?: boolean) => Promise<AxiosResponse<any, any, {}>>;
+declare const getJSONApiCall: <T>(endpoint: string, config: GetConfig, privateLogging?: boolean) => Promise<T>;
+declare const getDownloadFileApiCall: <T>(endpoint: string, config: GetConfig, output: string, privateLogging?: boolean) => Promise<{
+    outputDir: string;
+}>;
+interface PostConfig {
+    qs?: {
+        [key: string]: any;
+    };
+    headers?: AxiosRequestHeaders;
+    retryCount?: number;
+}
+declare const postJSONApiCall: <T>(endpoint: string, config: PostConfig, payload: any) => Promise<T>;
+declare const postFormDataApiCall: <T>(endpoint: string, config: PostConfig, payload: any) => Promise<T>;
+declare const postUrlEncodedApiCall: (endpoint: string, config: PostConfig, payload: any) => Promise<any>;
+
+declare const BATCH_INTERVAL_MS = 100;
+declare const logger: {
+    info: (message: string | any, ...args: any[]) => void;
+    error: (message: string | any, ...args: any[]) => void;
+    debug: (message: string | any, ...args: any[]) => void;
+    warn: (message: string | any, ...args: any[]) => void;
+    closeWinston: (cb: () => void) => void;
+};
+
+declare const ALL_STREAM_ID_KEYWORD = "all";
+declare const ROWS_SYNCED_METRIC_NAME = "rows_synced_count";
+declare const API_CALLS_METRIC_NAME = "api_calls_count";
+declare const EXECUTION_TIME_METRIC_NAME = "execution_time_ms";
+interface MetricConfiguration {
+    isEnabled: () => boolean;
+    getStreamIds?: () => string[];
+}
+declare const getCounterMetrics: (name: string, streamIds: string[]) => CounterMetric[];
+declare const getCounterMetric: (name: string, streamId?: string) => CounterMetric;
+declare const getAllMetrics: () => CounterMetric[];
+declare class CounterMetric {
+    private value;
+    name: string;
+    streamId: string;
+    constructor(name: string, streamId: string);
+    increment(inc?: number): void;
+    getStreamId(): string;
+    getName(): string;
+    getValue(): number;
+}
+
+declare const printMemoryFootprint: (prefix: string) => void;
+
+declare const haltAndCatchFire: (errorType: ErrorType, errorText: string, errorDebugText: string) => Promise<void>;
+
+declare function gracefulExit(exitCode: number): Promise<void>;
+
+declare const findRelatedRelationships: (streamId: StreamId, allRels: StreamRelationship[]) => DirectionalRelationships;
+
+declare const loadResolver: () => Resolver;
+
+interface HTTPHeaders {
+    [headerName: string]: any;
+}
+interface URLParams {
+    [paramName: string]: any;
+}
+type HTTPMethod = "GET" | "POST";
+
+type InclusionMode = "available" | "automatic" | "unsupported";
+interface MetadataEntry {
+    selected?: boolean;
+    "replication-method"?: ReplicationMethod;
+    "replication-key"?: string;
+    "view-key-properties"?: string[];
+    "inclusion"?: InclusionMode;
+    "selected-by-default"?: boolean;
+    "valid-replication-keys"?: string[];
+    "forced-replication-method"?: ReplicationMethod;
+    "forced-replication-key"?: string;
+    "table-key-properties"?: string[];
+    "schema-name"?: string;
+    "is-view"?: boolean;
+    "row-count"?: number;
+    "database-name"?: string;
+    "sql-datatype"?: string;
+    "whaly-display-label"?: string;
+    "whaly-description"?: string;
+    "whaly-parent-stream"?: StreamId;
+}
+interface CatalogMetadata {
+    breadcrumb: string[];
+    metadata: MetadataEntry;
+}
+declare class MetadataMap extends Map<string[], MetadataEntry> {
+    private areEquals;
+    private findExistingEntry;
+    get(key: string[]): MetadataEntry | undefined;
+    set(key: string[], value: MetadataEntry): this;
+}
+declare class StreamMetadata {
+    metadataByBreadcrumb: MetadataMap;
+    constructor();
+    fromCatalogStreamMetadata(metadataArr: CatalogMetadata[]): this;
+    toString(): MetadataMap;
+    get(breadcrumb: string[]): MetadataEntry | undefined;
+    write(breadcrumb: string[], key: string, value: any): this;
+    private getRootBreadcrumbMetadata;
+    isStreamSelected(): boolean;
+    getKeyProperties(): string[];
+    getReplicationKey(): string | undefined;
+    getReplicationMethod(): ReplicationMethod | undefined;
+    getForcedReplicationMethod(): ReplicationMethod | undefined;
+    toList(): CatalogMetadata[];
+}
+
 interface FlattenedSchema {
     [unsafePropertyName: string]: JSONSchemaFieldDefinition;
 }
@@ -273,165 +420,6 @@ declare abstract class ITarget<C extends BaseConfig = BaseConfig> {
     static uploadSingleStreamToWarehouse: (streamState: StreamState$1) => Promise<void>;
 }
 
-type ReplicationMethod = "INCREMENTAL" | "FULL_TABLE" | "LOG_BASED";
-type SyncFunction<C> = (target: ITarget, catalogStream: CatalogStream, config: C, catalog: Catalog) => Promise<void>;
-type SourceType = `HUBSPOT` | `LAGROWTHMACHINE` | `SEGMENT` | `COHORT` | `AIRTABLE` | `AIRTABLE_OAUTH` | `BUBBLE` | `PIPEDRIVE` | `LINKEDIN_ADS` | `FACEBOOK_ADS` | `SEGMENT` | `SALESFORCE` | `SALESFORCE_SANDBOX` | `RECRUITCRM` | `GOOGLE_SHEETS` | `GOOGLE_ADS` | `GOOGLE_ANALYTICS` | `SLACK` | `STRIPE` | `ORBIT` | `GITHUB_STARS` | `JAFFLE_SHOP` | `POSTGRES` | `PENNYLANE` | `PENNYLANE_REDSHIFT` | `AIRCALL` | `AIRCALL_OAUTH` | `FAST_POSTGRES_FULL_TABLE` | `WOOCOMMERCE`;
-type DestinationType = `GOOGLE_BIGQUERY` | `SNOWFLAKE`;
-interface ShoreConfig {
-    command: ShoreCommand;
-    destination: DestinationType;
-    config: any;
-    targetConfig: any;
-    state?: any;
-    catalog?: CatalogFile;
-}
-type ShoreCommand = "READ" | "DISCOVER";
-type ResolverType = "WHALY" | "LOCAL";
-
-type InclusionMode = "available" | "automatic" | "unsupported";
-interface MetadataEntry {
-    selected?: boolean;
-    "replication-method"?: ReplicationMethod;
-    "replication-key"?: string;
-    "view-key-properties"?: string[];
-    "inclusion"?: InclusionMode;
-    "selected-by-default"?: boolean;
-    "valid-replication-keys"?: string[];
-    "forced-replication-method"?: ReplicationMethod;
-    "forced-replication-key"?: string;
-    "table-key-properties"?: string[];
-    "schema-name"?: string;
-    "is-view"?: boolean;
-    "row-count"?: number;
-    "database-name"?: string;
-    "sql-datatype"?: string;
-    "whaly-display-label"?: string;
-    "whaly-description"?: string;
-    "whaly-parent-stream"?: StreamId;
-}
-interface CatalogMetadata {
-    breadcrumb: string[];
-    metadata: MetadataEntry;
-}
-declare class MetadataMap extends Map<string[], MetadataEntry> {
-    private areEquals;
-    private findExistingEntry;
-    get(key: string[]): MetadataEntry | undefined;
-    set(key: string[], value: MetadataEntry): this;
-}
-declare class StreamMetadata {
-    metadataByBreadcrumb: MetadataMap;
-    constructor();
-    fromCatalogStreamMetadata(metadataArr: CatalogMetadata[]): this;
-    toString(): MetadataMap;
-    get(breadcrumb: string[]): MetadataEntry | undefined;
-    write(breadcrumb: string[], key: string, value: any): this;
-    private getRootBreadcrumbMetadata;
-    isStreamSelected(): boolean;
-    getKeyProperties(): string[];
-    getReplicationKey(): string | undefined;
-    getReplicationMethod(): ReplicationMethod | undefined;
-    getForcedReplicationMethod(): ReplicationMethod | undefined;
-    toList(): CatalogMetadata[];
-}
-
-type StreamId = string;
-type StreamName = string;
-interface CatalogStream {
-    tap_stream_id: StreamId;
-    stream: StreamName;
-    schema: any;
-    metadata: CatalogMetadata[];
-}
-interface CatalogFile {
-    streams: CatalogStream[];
-}
-declare class Catalog {
-    streams?: CatalogStream[];
-    static fromFile(fileName: string): Catalog;
-    static fromCatalogFile(catalogFile: CatalogFile): Catalog;
-    getSelectedStreams(): CatalogStream[];
-}
-
-declare const writeStateFile: (resolver: Resolver, state: string) => Promise<void>;
-declare const writeCatalogFile: (resolver: Resolver, catalog: CatalogFile) => Promise<void>;
-declare const loadShoreConfig: (resolver: Resolver) => Promise<ShoreConfig>;
-declare const loadJson: (fileName: string) => any;
-declare const loadSchema: (streamId: string) => any;
-declare const checkRequiredConfigKeys: (args: any, requiredConfigKeys: string[]) => void;
-
-declare const getAxiosInstance: (retryCount?: number) => AxiosInstance;
-interface GetConfig {
-    qs?: {
-        [key: string]: any;
-    };
-    headers?: AxiosRequestHeaders;
-    retryCount?: number;
-}
-declare const getRawJSONApiCall: <T>(endpoint: string, config: GetConfig, privateLogging?: boolean) => Promise<AxiosResponse<T>>;
-declare const getJSONApiCallWithFullResponse: <T>(endpoint: string, config: GetConfig, privateLogging?: boolean) => Promise<AxiosResponse<any, any, {}>>;
-declare const getJSONApiCall: <T>(endpoint: string, config: GetConfig, privateLogging?: boolean) => Promise<T>;
-declare const getDownloadFileApiCall: <T>(endpoint: string, config: GetConfig, output: string, privateLogging?: boolean) => Promise<{
-    outputDir: string;
-}>;
-interface PostConfig {
-    qs?: {
-        [key: string]: any;
-    };
-    headers?: AxiosRequestHeaders;
-    retryCount?: number;
-}
-declare const postJSONApiCall: <T>(endpoint: string, config: PostConfig, payload: any) => Promise<T>;
-declare const postFormDataApiCall: <T>(endpoint: string, config: PostConfig, payload: any) => Promise<T>;
-declare const postUrlEncodedApiCall: (endpoint: string, config: PostConfig, payload: any) => Promise<any>;
-
-declare const BATCH_INTERVAL_MS = 100;
-declare const logger: {
-    info: (message: string | any, ...args: any[]) => void;
-    error: (message: string | any, ...args: any[]) => void;
-    debug: (message: string | any, ...args: any[]) => void;
-    warn: (message: string | any, ...args: any[]) => void;
-    closeWinston: (cb: () => void) => void;
-};
-
-declare const ALL_STREAM_ID_KEYWORD = "all";
-declare const ROWS_SYNCED_METRIC_NAME = "rows_synced_count";
-declare const API_CALLS_METRIC_NAME = "api_calls_count";
-declare const EXECUTION_TIME_METRIC_NAME = "execution_time_ms";
-interface MetricConfiguration {
-    isEnabled: () => boolean;
-    getStreamIds?: () => string[];
-}
-declare const getCounterMetrics: (name: string, streamIds: string[]) => CounterMetric[];
-declare const getCounterMetric: (name: string, streamId?: string) => CounterMetric;
-declare const getAllMetrics: () => CounterMetric[];
-declare class CounterMetric {
-    private value;
-    name: string;
-    streamId: string;
-    constructor(name: string, streamId: string);
-    increment(inc?: number): void;
-    getStreamId(): string;
-    getName(): string;
-    getValue(): number;
-}
-
-declare const printMemoryFootprint: (prefix: string) => void;
-
-declare const haltAndCatchFire: (errorType: ErrorType, errorText: string, errorDebugText: string) => Promise<void>;
-
-declare function gracefulExit(exitCode: number): Promise<void>;
-
-declare const findRelatedRelationships: (streamId: StreamId, allRels: StreamRelationship[]) => DirectionalRelationships;
-
-interface HTTPHeaders {
-    [headerName: string]: any;
-}
-interface URLParams {
-    [paramName: string]: any;
-}
-type HTTPMethod = "GET" | "POST";
-
 interface InputTapState {
     bookmarks?: {
         [streamId: string]: StreamState;
@@ -499,12 +487,10 @@ declare class Authenticator<C> {
  * C: Type of the tap configuration
  * P: For children Streams, this is the type of the parent
  */
-declare abstract class StreamV2<O = any, C = any, P = undefined> {
-    _config: C;
-    _tapName: string;
-    _tapState: InputTapState;
-    _target: ITarget;
-    rootMetadataEntry: MetadataEntry | undefined;
+declare abstract class Stream<O = any, C = any, P = undefined> {
+    config: C;
+    tapState: InputTapState;
+    target: ITarget;
     forcedReplicationMethod: ReplicationMethod | undefined;
     selectedByDefault: boolean;
     STATE_MSG_FREQUENCY: number;
@@ -516,14 +502,13 @@ declare abstract class StreamV2<O = any, C = any, P = undefined> {
     forcedReplicationKey: string | undefined;
     useStateFromStreamId: string | undefined;
     relationships: StreamRelationship[];
-    children: StreamV2<any, any, O | O[]>[];
+    children: Stream<any, any, O | O[]>[];
     isSorted: boolean;
     isSilent: boolean;
     childConcurrency: number;
     rowsSyncedMetricsConf: MetricConfiguration;
     executionTimeMetricsConf: MetricConfiguration;
-    constructor(tapName: string, config: C, state: InputTapState, target: ITarget, childConcurrency?: number);
-    setRootMetadataEntry: (rootMetadataEntry: MetadataEntry) => void;
+    constructor(config: C, state: InputTapState, target: ITarget, childConcurrency?: number);
     /**
      * Sync this stream
      * Called only for streams with no parents, aka. "root tap streams"
@@ -615,9 +600,9 @@ declare abstract class StreamV2<O = any, C = any, P = undefined> {
  * R: Type of the API result
  * NPT: Type of the "nextPageToken"
  */
-declare abstract class RESTStreamV2<R, O, NPT, C, P = undefined> extends StreamV2<O, C, P> {
+declare abstract class RESTStream<R, O, NPT, C, P = undefined> extends Stream<O, C, P> {
     axiosInstance: AxiosInstance | undefined;
-    constructor(tapName: string, config: C, state: InputTapState, target: ITarget, childConcurrency?: number);
+    constructor(config: C, state: InputTapState, target: ITarget, childConcurrency?: number);
     /**
      * Prepare an Axios request object.
      * Pagination information can be parsed from `nextPageToken` if `nextPageToken` is not undefined.
@@ -683,20 +668,18 @@ declare abstract class RESTStreamV2<R, O, NPT, C, P = undefined> extends StreamV
 }
 
 declare const DEFAULT_MAX_CONCURRENT_STREAMS = 5;
+type SyncOptions = {
+    include?: string[];
+    exclude?: string[];
+};
 declare abstract class Tap<C> {
-    _target: ITarget;
-    _config: C;
-    _streams: StreamV2[];
-    _concurrency: number;
+    target: ITarget;
+    config: C;
+    streams: Stream[];
+    concurrency: number;
     abstract init(): Promise<void>;
-    abstract requiredConfigKeys(): string[];
     constructor(target: ITarget, config: C);
-    discover: (existingCatalog: CatalogFile | undefined) => Promise<CatalogFile>;
-    sync: (catalog: Catalog) => Promise<void>;
-    generateMetadataFromStream(stream: StreamV2, parentStreamId: string | undefined): Promise<{
-        metadata: CatalogMetadata[];
-    }>;
-    initiateSync(catalog: Catalog): Promise<void>;
+    sync: (options?: SyncOptions) => Promise<void>;
     end(): Promise<void>;
 }
 
@@ -722,4 +705,62 @@ declare const flattenSchema: <T>(streamId: StreamId, schema: T) => FlattenedSche
 declare const createTemporaryFileStream: (streamId: StreamId) => TempFile;
 declare function safePath(path: string): string;
 
-export { ALL_STREAM_ID_KEYWORD, API_CALLS_METRIC_NAME, Authenticator, BATCH_INTERVAL_MS, type BaseConfig, type Bookmark, type Bookmarks, Catalog, type CatalogFile, type CatalogMetadata, type CatalogStream, type ColumnMappingStoreUnsafeToSafe, CounterMetric, DEFAULT_MAX_CONCURRENT_STREAMS, type DestinationType, type DirectionalRelationship, type DirectionalRelationships, EXECUTION_TIME_METRIC_NAME, type ErrorType, type FlattenedSchema, type GetConfig, type HTTPHeaders, type HTTPMethod, ITarget, type InclusionMode, type InputTapState, type InternalTableField, type JSONSchemaFieldDefinition, type Message, type MessageType, type MetadataEntry, type MetricConfiguration, MissingFieldInSchemaError, MissingSchemaError, type PostConfig, type ProgressMarkers, type PropertiesMetadata, RESTStreamV2, ROWS_SYNCED_METRIC_NAME, RecordMessage, RenameColumnStore, type ReplicationMethod, ReplicationMethodMessage, Resolver, type ResolverType, type SafeColumnNameConverterFn, type Schema, SchemaMessage, SchemaValidationError, type ShoreCommand, type ShoreConfig, type SourceType, type State, StateMessage, StateService, type StreamId, StreamMetadata, type StreamName, type StreamRelationship, type StreamState, StreamV2, StreamWarehouseSyncService, type StreamWithTempFile, type SyncFunction, Tap, type TapState, type TargetSchemaHook, type TargetSchemaHookInput, type TempFile, type URLParams, type WarehouseTableField, _greaterThan, addWhalyFields, checkRequiredConfigKeys, createTemporaryFileStream, extractStateForStream, finalizeStateProgressMarkers, findRelatedRelationships, flattenSchema, getAllMetrics, getAxiosInstance, getCounterMetric, getCounterMetrics, getDownloadFileApiCall, getJSONApiCall, getJSONApiCallWithFullResponse, getRawJSONApiCall, gracefulExit, haltAndCatchFire, incrementStreamState, loadJson, loadSchema, loadShoreConfig, logger, postFormDataApiCall, postJSONApiCall, postUrlEncodedApiCall, printMemoryFootprint, removeParasiteProperties, safePath, writeCatalogFile, writeStateFile };
+interface BigQueryConfig extends BaseConfig {
+    schema: string;
+    project_id: string;
+    loading_deck_gcs_bucket_name: string;
+    location?: BigqueryLocation;
+}
+type BigqueryLocation = "US" | "EU";
+
+declare class BigQueryDBSync extends StreamWarehouseSyncService {
+    config: BigQueryConfig;
+    bqClient: BigQuery;
+    bqDataset: Dataset;
+    bqTable: Table;
+    bqStagingTable: Table;
+    tableName: string;
+    stagingTableName: string;
+    sqlTableId: string;
+    sqlStagingTableId: string;
+    constructor(config: BigQueryConfig, streamId: StreamId, database: string, schema: string, table: string, renameColumnStore: RenameColumnStore);
+    getSerializedRecord: (record: any) => string;
+    safeColumnName: (key: string) => string;
+    getWarehouseTypeFromJSONSchema: (definition: JSONSchemaFieldDefinition) => string;
+    getMergeQueries: () => string[];
+    getReplaceQueries: () => string[];
+    createDatabaseAndSchemaIfNotExists: (retryCount: number) => Promise<void>;
+    createTable: () => Promise<void>;
+    addColumns: (tableFields: InternalTableField[]) => Promise<void>;
+    createStagingArea: () => Promise<void>;
+    loadStreamInStagingArea: (localFilePath: string) => Promise<void>;
+    deleteStagingArea: () => Promise<void>;
+    getTablesInSchema: () => Promise<{
+        tableName: string;
+    }[]>;
+    getTableColumnsFromWarehouse: () => Promise<WarehouseTableField[]>;
+    runQueries: (queries: string[]) => Promise<void>;
+    /**
+     *
+     * Generate a TableField definition object that can be passed to BigQuery APIs to define a column on a table
+     *
+     * @param colName
+     * @param schemaFieldDefinition
+     * @returns
+     */
+    private getBigqueryTableFieldFromInternalSchemaFieldDefinition;
+    private generateBigquerySchema;
+    private loadGCSFileInTable;
+}
+
+declare class BigQueryTarget extends ITarget<BigQueryConfig> {
+    constructor(config: BigQueryConfig, resolver: Resolver);
+    static requiredConfigKeys: string[];
+    newDBSyncInstance: (streamId: StreamId, database: string, schema: string, table: string) => BigQueryDBSync;
+    genTableName: (key: string) => string;
+    safeColumnNameConverter: (key: string) => string;
+    validateDateRange: (record: any, schema: FlattenedSchema) => any;
+    convertNumberIntoDecimal: (record: any, schema: FlattenedSchema) => any;
+}
+
+export { ALL_STREAM_ID_KEYWORD, API_CALLS_METRIC_NAME, Authenticator, BATCH_INTERVAL_MS, type BaseConfig, type BigQueryConfig, BigQueryTarget, type BigqueryLocation, type Bookmark, type Bookmarks, type CatalogFile, type CatalogMetadata, type CatalogStream, type CatalogStreamMetadata, type ColumnMappingStoreUnsafeToSafe, CounterMetric, DEFAULT_MAX_CONCURRENT_STREAMS, type DestinationType, type DirectionalRelationship, type DirectionalRelationships, EXECUTION_TIME_METRIC_NAME, type ErrorType, type FlattenedSchema, type GetConfig, type HTTPHeaders, type HTTPMethod, ITarget, type InclusionMode, type InputTapState, type InternalTableField, type JSONSchemaFieldDefinition, type Message, type MessageType, type MetadataEntry, type MetricConfiguration, MissingFieldInSchemaError, MissingSchemaError, type PostConfig, type ProgressMarkers, type PropertiesMetadata, RESTStream, ROWS_SYNCED_METRIC_NAME, RecordMessage, RenameColumnStore, type ReplicationMethod, ReplicationMethodMessage, Resolver, type SafeColumnNameConverterFn, type Schema, SchemaMessage, SchemaValidationError, type State, type StateHolder, StateMessage, StateService, Stream, type StreamId, StreamMetadata, type StreamName, type StreamRelationship, type StreamState, StreamWarehouseSyncService, type StreamWithTempFile, type SyncOptions, Tap, type TapState, type TargetSchemaHook, type TargetSchemaHookInput, type TempFile, type URLParams, type WarehouseTableField, _greaterThan, addWhalyFields, createTemporaryFileStream, extractStateForStream, finalizeStateProgressMarkers, findRelatedRelationships, flattenSchema, getAllMetrics, getAxiosInstance, getCounterMetric, getCounterMetrics, getDownloadFileApiCall, getJSONApiCall, getJSONApiCallWithFullResponse, getRawJSONApiCall, gracefulExit, haltAndCatchFire, incrementStreamState, loadJson, loadResolver, loadSchema, logger, postFormDataApiCall, postJSONApiCall, postUrlEncodedApiCall, printMemoryFootprint, removeParasiteProperties, safePath, writeStateFile };