@whaly/connector-sdk 0.1.0

package/dist/index.d.mts

@@ -0,0 +1,725 @@
+import winston from 'winston';
+import { Validator, ValidationError } from 'jsonschema';
+import moment from 'moment';
+import { WriteStream } from 'fs';
+import { AxiosInstance, AxiosRequestHeaders, AxiosResponse, AxiosRequestConfig, AxiosError } from 'axios';
+
+interface StreamRelationship {
+    left: StreamId;
+    right: StreamId;
+    from: string;
+    to: string;
+    type: "1-1" | "1-N" | "N-1";
+}
+interface DirectionalRelationship {
+    type: "1-N" | "N-1" | "1-1";
+    streamId: StreamId;
+    from: string;
+    to: string;
+}
+interface DirectionalRelationships {
+    left: DirectionalRelationship[];
+    right: DirectionalRelationship[];
+}
+
+interface PropertiesMetadata {
+    [propName: string]: {
+        label: string;
+        description?: string;
+    };
+}
+interface Schema {
+    "jsonSchema": {
+        "type": "object";
+        "properties": any;
+    };
+    "propertiesMetadata"?: PropertiesMetadata;
+}
+
+type MessageType = "RECORD" | "STATE" | "SCHEMA" | "ACTIVATE_VERSION" | "REPLICATION_METHOD";
+interface Message {
+    type: MessageType;
+    toString(): string;
+}
+declare class RecordMessage implements Message {
+    type: MessageType;
+    stream: StreamId;
+    record: any;
+    constructor(record: any, stream: StreamId);
+    toString(): string;
+}
+declare class StateMessage implements Message {
+    type: MessageType;
+    value: any;
+    constructor(value: any);
+    toString(): string;
+}
+declare class ReplicationMethodMessage implements Message {
+    type: MessageType;
+    stream: string;
+    replication: ReplicationMethod;
+    constructor(stream: string, version: ReplicationMethod);
+    toString(): string;
+}
+declare class SchemaMessage implements Message {
+    type: MessageType;
+    stream: StreamId;
+    schema: any;
+    keyProperties: string[];
+    bookmarkProperties: string[] | undefined;
+    displayLabel: string | undefined;
+    description: string | undefined;
+    propertiesMetadata: PropertiesMetadata | undefined;
+    relationships: DirectionalRelationships;
+    constructor(opts: {
+        stream: StreamId;
+        schema: any;
+        keyProperties: string[];
+        bookmarkProperties?: string[];
+        displayLabel?: string;
+        description?: string;
+        relationships: DirectionalRelationships;
+        propertiesMetadata?: PropertiesMetadata | undefined;
+    });
+    toString(): string;
+}
+
+interface TargetSchemaHookInput {
+    message: SchemaMessage;
+    databaseName: string;
+    schemaName: string;
+    tableName: string;
+}
+interface TargetSchemaHook {
+    writeSchema: (input: TargetSchemaHookInput) => Promise<void>;
+}
+
+type ErrorType = "unauthorized" | "unknown";
+
+declare abstract class Resolver {
+    constructor();
+    abstract checkIfCanSync(): Promise<boolean>;
+    abstract markSyncStarted(): Promise<void>;
+    abstract startVPNIfNeeded(): Promise<void>;
+    abstract getConfig(command: ShoreCommand): Promise<ShoreConfig>;
+    abstract writeCatalog(catalog: CatalogFile): Promise<void>;
+    abstract writeState(state: string): Promise<void>;
+    abstract markSyncFailed(): Promise<void>;
+    abstract markSyncComplete(): Promise<void>;
+    abstract markDiscoveryComplete(): Promise<void>;
+    abstract flushMetrics(): Promise<void>;
+    abstract onError(errorType: ErrorType, errorText: string, errorDebugText: string): Promise<void>;
+    abstract getSchemaHooks(): TargetSchemaHook[];
+    abstract updateSourceValue(optionKey: string, optionValue: string): Promise<void>;
+    abstract getLogFormat(): winston.Logform.Format;
+}
+
+interface FlattenedSchema {
+    [unsafePropertyName: string]: JSONSchemaFieldDefinition;
+}
+interface JSONSchemaFieldDefinition {
+    type: string[] | string;
+    format?: "date-time" | "json";
+    items?: JSONSchemaFieldDefinition;
+}
+interface WarehouseTableField {
+    name: string;
+    type: string;
+}
+interface InternalTableField {
+    unsafeName: string;
+    name: string;
+    definition: JSONSchemaFieldDefinition;
+}
+interface TempFile {
+    stream: WriteStream;
+    path: string;
+}
+interface StreamWithTempFile {
+    streamId: StreamId;
+    tempFile: TempFile;
+}
+interface BaseConfig {
+    schema: string;
+    database: string;
+}
+
+interface ColumnMappingStoreUnsafeToSafe {
+    [originalColumnName: string]: string;
+}
+type SafeColumnNameConverterFn = (unsafe: string) => string;
+/**
+ * This stateful class is to be used to store all the columns names mapping between the "raw" column name
+ * found in the records and the destination column in the data warehouse
+ *
+ * A mapping exists between the fields declared by the Tap and the reality in the Warehouse for those reasons:
+ * - Columns that are versionned due to type changes since the last load
+ * - Columns that have a name conflict after "Warehouse" sanitization
+ */
+declare class RenameColumnStore {
+    private renamedColumns;
+    private safeColumnNameConverter?;
+    constructor();
+    setSafeColumnNameConverter: (fn: SafeColumnNameConverterFn) => void;
+    computeColumnNameForStream: (streamId: string, flattenedSchema: FlattenedSchema) => void;
+    getColumnTranslation: (streamId: string, originalColumnName: string) => string;
+    getUnsafeToSafeColumnMapping: (streamId: string) => ColumnMappingStoreUnsafeToSafe;
+    isReady: (streamId: string) => boolean;
+}
+
+declare abstract class StreamWarehouseSyncService {
+    streamId: StreamId;
+    database: string;
+    schema: string;
+    table: string;
+    flattenedSchema: FlattenedSchema;
+    primaryKeys: string[];
+    maxRetryCount: number;
+    renamedColumnStore: RenameColumnStore;
+    constructor(streamId: StreamId, database: string, schema: string, table: string, renameColumnStore: RenameColumnStore);
+    abstract safeColumnName: (unsafe: string) => string;
+    abstract getWarehouseTypeFromJSONSchema: (definition: JSONSchemaFieldDefinition) => string | undefined;
+    getcolumnNameWithTypeSuffix(colName: string, warehouseType: string | undefined): string;
+    abstract getMergeQueries: () => string[];
+    abstract getReplaceQueries: () => string[];
+    abstract createDatabaseAndSchemaIfNotExists: (retryCount: number) => Promise<void>;
+    abstract createTable: () => Promise<void>;
+    abstract addColumns: (fields: InternalTableField[]) => Promise<void>;
+    abstract createStagingArea: () => Promise<void>;
+    abstract loadStreamInStagingArea: (localFilePath: string) => Promise<void>;
+    abstract deleteStagingArea: () => Promise<void>;
+    abstract getTablesInSchema: () => Promise<{
+        tableName: string;
+    }[]>;
+    abstract getTableColumnsFromWarehouse: () => Promise<WarehouseTableField[]>;
+    abstract runQueries: (queries: string[]) => Promise<void>;
+    abstract getSerializedRecord: (record: any) => string;
+    updateSchemaInWarehouse: (flattenedSchema: FlattenedSchema, primaryKeys: string[]) => Promise<void>;
+    renameColumns(record: any): any;
+    protected runQueriesWithRetry: (queries: string[]) => Promise<void>;
+    private syncTableSchemaWithWarehouse;
+    /**
+     * When there is a conflict between the synced data type and the data type of an
+     * already existing column in the Warehouse, this create a new column in Warehouse with the new
+     * datatype as suffix.
+     *
+     * This refer the fact that data should now be synced with the new column.
+     *
+     * @param tableFields
+     */
+    private addVersionedColumns;
+    private updateTableColumns;
+    mergeTempTableWithDestination: (replicationMethod: ReplicationMethod) => Promise<void>;
+}
+
+declare class StreamState$1 {
+    streamId: StreamId;
+    schema?: FlattenedSchema;
+    dbSync: StreamWarehouseSyncService;
+    batchDate: moment.Moment;
+    batchedRowCount: number;
+    fileToLoad: TempFile;
+    syncedRowCount: number;
+    keyProperties: string[];
+    _hasBeenLoadedYetDuringThisSync: boolean;
+    replicationMethod: ReplicationMethod;
+    constructor(streamId: StreamId, dbSync: StreamWarehouseSyncService, replicationMethod: ReplicationMethod);
+    setSchema(schema: FlattenedSchema): void;
+    getSchema(): FlattenedSchema | undefined;
+    getBatchDate(): string;
+    getBatchedRowCount(): number;
+    incrementBatchedRowCount(): void;
+    getFileToLoad(): TempFile;
+    resetFileToLoad(): void;
+    getKeyProperties(): string[];
+    setKeyProperties(keyProperties: string[]): void;
+    getDbSync(): StreamWarehouseSyncService;
+    getReplicationMethod(): ReplicationMethod;
+    setReplicationMethod(r: ReplicationMethod): void;
+    setHasBeenLoadedYet(): void;
+}
+
+declare abstract class ITarget<C extends BaseConfig = BaseConfig> {
+    config: C;
+    schemaHooks: TargetSchemaHook[];
+    resolver: Resolver;
+    syncTime: moment.Moment;
+    batchedState: any;
+    flushedState: any;
+    renameColumnStore: RenameColumnStore;
+    streams: {
+        [streamName: string]: StreamState$1;
+    };
+    validator: Validator;
+    constructor(config: C, resolver: Resolver);
+    static requiredConfigKeys: string[];
+    abstract newDBSyncInstance: (streamId: StreamId, database: string, schema: string, table: string) => StreamWarehouseSyncService;
+    abstract genTableName: (streamId: string) => string;
+    abstract safeColumnNameConverter: SafeColumnNameConverterFn;
+    abstract validateDateRange: (reacord: any, schema: FlattenedSchema) => any;
+    abstract convertNumberIntoDecimal: (record: any, schema: FlattenedSchema) => any;
+    private analyzeSchemaChanges;
+    getSerializedRecord: (streamId: string, record: any) => string;
+    record: (message: RecordMessage) => Promise<void>;
+    private initStreamStateAndSyncService;
+    replicationMethod: (message: ReplicationMethodMessage) => void;
+    schema: (message: SchemaMessage) => Promise<void>;
+    state: (message: StateMessage) => Promise<void>;
+    complete: () => Promise<void>;
+    private shouldUploadIntermediateBatch;
+    private loadAllStreamsInWarehouse;
+    private emitState;
+    private uploadStreamsToWarehouse;
+    static uploadSingleStreamToWarehouse: (streamState: StreamState$1) => Promise<void>;
+}
+
+type ReplicationMethod = "INCREMENTAL" | "FULL_TABLE" | "LOG_BASED";
+type SyncFunction<C> = (target: ITarget, catalogStream: CatalogStream, config: C, catalog: Catalog) => Promise<void>;
+type SourceType = `HUBSPOT` | `LAGROWTHMACHINE` | `SEGMENT` | `COHORT` | `AIRTABLE` | `AIRTABLE_OAUTH` | `BUBBLE` | `PIPEDRIVE` | `LINKEDIN_ADS` | `FACEBOOK_ADS` | `SEGMENT` | `SALESFORCE` | `SALESFORCE_SANDBOX` | `RECRUITCRM` | `GOOGLE_SHEETS` | `GOOGLE_ADS` | `GOOGLE_ANALYTICS` | `SLACK` | `STRIPE` | `ORBIT` | `GITHUB_STARS` | `JAFFLE_SHOP` | `POSTGRES` | `PENNYLANE` | `PENNYLANE_REDSHIFT` | `AIRCALL` | `AIRCALL_OAUTH` | `FAST_POSTGRES_FULL_TABLE` | `WOOCOMMERCE`;
+type DestinationType = `GOOGLE_BIGQUERY` | `SNOWFLAKE`;
+interface ShoreConfig {
+    command: ShoreCommand;
+    destination: DestinationType;
+    config: any;
+    targetConfig: any;
+    state?: any;
+    catalog?: CatalogFile;
+}
+type ShoreCommand = "READ" | "DISCOVER";
+type ResolverType = "WHALY" | "LOCAL";
+
+type InclusionMode = "available" | "automatic" | "unsupported";
+interface MetadataEntry {
+    selected?: boolean;
+    "replication-method"?: ReplicationMethod;
+    "replication-key"?: string;
+    "view-key-properties"?: string[];
+    "inclusion"?: InclusionMode;
+    "selected-by-default"?: boolean;
+    "valid-replication-keys"?: string[];
+    "forced-replication-method"?: ReplicationMethod;
+    "forced-replication-key"?: string;
+    "table-key-properties"?: string[];
+    "schema-name"?: string;
+    "is-view"?: boolean;
+    "row-count"?: number;
+    "database-name"?: string;
+    "sql-datatype"?: string;
+    "whaly-display-label"?: string;
+    "whaly-description"?: string;
+    "whaly-parent-stream"?: StreamId;
+}
+interface CatalogMetadata {
+    breadcrumb: string[];
+    metadata: MetadataEntry;
+}
+declare class MetadataMap extends Map<string[], MetadataEntry> {
+    private areEquals;
+    private findExistingEntry;
+    get(key: string[]): MetadataEntry | undefined;
+    set(key: string[], value: MetadataEntry): this;
+}
+declare class StreamMetadata {
+    metadataByBreadcrumb: MetadataMap;
+    constructor();
+    fromCatalogStreamMetadata(metadataArr: CatalogMetadata[]): this;
+    toString(): MetadataMap;
+    get(breadcrumb: string[]): MetadataEntry | undefined;
+    write(breadcrumb: string[], key: string, value: any): this;
+    private getRootBreadcrumbMetadata;
+    isStreamSelected(): boolean;
+    getKeyProperties(): string[];
+    getReplicationKey(): string | undefined;
+    getReplicationMethod(): ReplicationMethod | undefined;
+    getForcedReplicationMethod(): ReplicationMethod | undefined;
+    toList(): CatalogMetadata[];
+}
+
+type StreamId = string;
+type StreamName = string;
+interface CatalogStream {
+    tap_stream_id: StreamId;
+    stream: StreamName;
+    schema: any;
+    metadata: CatalogMetadata[];
+}
+interface CatalogFile {
+    streams: CatalogStream[];
+}
+declare class Catalog {
+    streams?: CatalogStream[];
+    static fromFile(fileName: string): Catalog;
+    static fromCatalogFile(catalogFile: CatalogFile): Catalog;
+    getSelectedStreams(): CatalogStream[];
+}
+
+declare const writeStateFile: (resolver: Resolver, state: string) => Promise<void>;
+declare const writeCatalogFile: (resolver: Resolver, catalog: CatalogFile) => Promise<void>;
+declare const loadShoreConfig: (resolver: Resolver) => Promise<ShoreConfig>;
+declare const loadJson: (fileName: string) => any;
+declare const loadSchema: (streamId: string) => any;
+declare const checkRequiredConfigKeys: (args: any, requiredConfigKeys: string[]) => void;
+
+declare const getAxiosInstance: (retryCount?: number) => AxiosInstance;
+interface GetConfig {
+    qs?: {
+        [key: string]: any;
+    };
+    headers?: AxiosRequestHeaders;
+    retryCount?: number;
+}
+declare const getRawJSONApiCall: <T>(endpoint: string, config: GetConfig, privateLogging?: boolean) => Promise<AxiosResponse<T>>;
+declare const getJSONApiCallWithFullResponse: <T>(endpoint: string, config: GetConfig, privateLogging?: boolean) => Promise<AxiosResponse<any, any, {}>>;
+declare const getJSONApiCall: <T>(endpoint: string, config: GetConfig, privateLogging?: boolean) => Promise<T>;
+declare const getDownloadFileApiCall: <T>(endpoint: string, config: GetConfig, output: string, privateLogging?: boolean) => Promise<{
+    outputDir: string;
+}>;
+interface PostConfig {
+    qs?: {
+        [key: string]: any;
+    };
+    headers?: AxiosRequestHeaders;
+    retryCount?: number;
+}
+declare const postJSONApiCall: <T>(endpoint: string, config: PostConfig, payload: any) => Promise<T>;
+declare const postFormDataApiCall: <T>(endpoint: string, config: PostConfig, payload: any) => Promise<T>;
+declare const postUrlEncodedApiCall: (endpoint: string, config: PostConfig, payload: any) => Promise<any>;
+
+declare const BATCH_INTERVAL_MS = 100;
+declare const logger: {
+    info: (message: string | any, ...args: any[]) => void;
+    error: (message: string | any, ...args: any[]) => void;
+    debug: (message: string | any, ...args: any[]) => void;
+    warn: (message: string | any, ...args: any[]) => void;
+    closeWinston: (cb: () => void) => void;
+};
+
+declare const ALL_STREAM_ID_KEYWORD = "all";
+declare const ROWS_SYNCED_METRIC_NAME = "rows_synced_count";
+declare const API_CALLS_METRIC_NAME = "api_calls_count";
+declare const EXECUTION_TIME_METRIC_NAME = "execution_time_ms";
+interface MetricConfiguration {
+    isEnabled: () => boolean;
+    getStreamIds?: () => string[];
+}
+declare const getCounterMetrics: (name: string, streamIds: string[]) => CounterMetric[];
+declare const getCounterMetric: (name: string, streamId?: string) => CounterMetric;
+declare const getAllMetrics: () => CounterMetric[];
+declare class CounterMetric {
+    private value;
+    name: string;
+    streamId: string;
+    constructor(name: string, streamId: string);
+    increment(inc?: number): void;
+    getStreamId(): string;
+    getName(): string;
+    getValue(): number;
+}
+
+declare const printMemoryFootprint: (prefix: string) => void;
+
+declare const haltAndCatchFire: (errorType: ErrorType, errorText: string, errorDebugText: string) => Promise<void>;
+
+declare function gracefulExit(exitCode: number): Promise<void>;
+
+declare const findRelatedRelationships: (streamId: StreamId, allRels: StreamRelationship[]) => DirectionalRelationships;
+
+interface HTTPHeaders {
+    [headerName: string]: any;
+}
+interface URLParams {
+    [paramName: string]: any;
+}
+type HTTPMethod = "GET" | "POST";
+
+interface InputTapState {
+    bookmarks?: {
+        [streamId: string]: StreamState;
+    };
+}
+interface TapState {
+    bookmarks: {
+        [streamId: string]: StreamState;
+    };
+}
+interface Bookmark {
+    replicationKey?: string;
+    replicationKeyValue?: string;
+}
+interface StreamState extends Bookmark {
+    progressMarkers?: ProgressMarkers;
+    replicationKeySignpost?: string;
+}
+interface ProgressMarkers extends Bookmark {
+    Note?: string;
+}
+interface State {
+    bookmarks: Bookmarks;
+}
+interface Bookmarks {
+    [streamId: string]: any;
+}
+declare const incrementStreamState: (state: StreamState, replicationKey: string, latestRecord: any, isSorted: boolean) => StreamState;
+/**
+ * Promote or wipe progress markers once sync is complete."""
+
+ * @param state
+ * @returns
+ */
+declare const finalizeStateProgressMarkers: (state: StreamState) => StreamState;
+declare const _greaterThan: (aValue: string, bValue: string) => boolean;
+declare const extractStateForStream: (state: InputTapState, streamId: StreamId) => StreamState;
+declare class StateService {
+    private static instance;
+    bookmarks: Bookmarks;
+    constructor();
+    static getInstance(): StateService;
+    setBookmark(streamId: StreamId, ts: moment.Moment): void;
+    setBookmarkV2(streamId: StreamId, streamState: StreamState): void;
+    setBookmarkSignpostV2(streamId: StreamId, signpostValue: string): void;
+    getBookmark(streamId: StreamId): moment.Moment;
+    getBookmarkV2(streamId: StreamId): StreamState;
+    clearState(): void;
+    get(): State;
+    forwardStateToTarget(target: ITarget): Promise<void>;
+}
+
+/**
+ * C: The type of the tap config
+ */
+declare class Authenticator<C> {
+    getAuthHeaders(config: C): Promise<HTTPHeaders>;
+    getAuthQS(config: C): Promise<URLParams>;
+}
+
+/**
+ * TODO: Handle non timestamp based replication key
+ *
+ * O: Type of a resulted row after post processing
+ * C: Type of the tap configuration
+ * P: For children Streams, this is the type of the parent
+ */
+declare abstract class StreamV2<O = any, C = any, P = undefined> {
+    _config: C;
+    _tapName: string;
+    _tapState: InputTapState;
+    _target: ITarget;
+    rootMetadataEntry: MetadataEntry | undefined;
+    forcedReplicationMethod: ReplicationMethod | undefined;
+    selectedByDefault: boolean;
+    STATE_MSG_FREQUENCY: number;
+    streamId: StreamId;
+    schemaPath: string | undefined;
+    displayLabel: string | undefined;
+    description: string | undefined;
+    primaryKey: string[];
+    forcedReplicationKey: string | undefined;
+    useStateFromStreamId: string | undefined;
+    relationships: StreamRelationship[];
+    children: StreamV2<any, any, O | O[]>[];
+    isSorted: boolean;
+    isSilent: boolean;
+    childConcurrency: number;
+    rowsSyncedMetricsConf: MetricConfiguration;
+    executionTimeMetricsConf: MetricConfiguration;
+    constructor(tapName: string, config: C, state: InputTapState, target: ITarget, childConcurrency?: number);
+    setRootMetadataEntry: (rootMetadataEntry: MetadataEntry) => void;
+    /**
+     * Sync this stream
+     * Called only for streams with no parents, aka. "root tap streams"
+     * @returns
+     */
+    sync: () => Promise<void>;
+    flush: () => Promise<void>;
+    /**
+     * Write out a STATE message with the latest state.
+     */
+    _writeStateMessage: () => Promise<void>;
+    /**
+     * Write out a ACTIVATE_VERSION message.
+     */
+    _writeReplicationMethodMessage: () => Promise<void>;
+    /**
+     * Write out a SCHEMA message with the stream schema.
+     */
+    _writeSchemaMessage: (skipChildrenSchema?: boolean) => Promise<void>;
+    _writeSignpostInState: () => Promise<void>;
+    /**
+     * Update state of stream with data from the provided record.
+     */
+    _incrementStreamState: (latestRecord: O) => void;
+    /**
+     * Return True if the stream uses a timestamp-based replication key.
+     *
+     * Developers can override with `is_timestamp_replication_key = True` in
+        order to force this value.
+     */
+    isTimestampReplicationKey: () => Promise<boolean>;
+    /**
+     * Return state timestamp if set for the stream,
+     * or `start_date` config if set,
+     * or the UNIX Epoch
+     **/
+    getStartingTimestamp(): moment.Moment;
+    /**
+     * Return the max allowable bookmark value for this stream's replication key.
+     *
+     * For timestamp-based replication keys, this defaults to `moment()`. For
+     * non-timestamp replication keys, default to `undefined`.
+     *
+     * Override this value to prevent bookmarks from being advanced in cases where we
+     * may only have a partial set of records.
+     */
+    getReplicationKeySignpost: () => Promise<moment.Moment | undefined>;
+    /**
+     * Return the default replication method for the stream that will be used to write the catalog.
+     */
+    defaultReplicationConfig: () => {
+        replicationMethod: ReplicationMethod;
+        replicationKey: string | undefined;
+        isForced: boolean;
+    };
+    /**
+     * Return the configured replication method that will be used for the sync.
+     */
+    configuredReplicationConfig: () => {
+        replicationMethod: ReplicationMethod;
+        replicationKey: string | undefined;
+    };
+    _syncRecords(parent?: P): Promise<void>;
+    asyncInit(parent?: P): Promise<void>;
+    /**
+     * Return the stream schema.
+     *
+     * Can be overriden to return a dynamic schema
+     */
+    getSchema(): Promise<Schema | undefined>;
+    getRowCount(): Promise<number | undefined>;
+    /**
+     * Abstract row generator function. Must be overridden by the child class.
+     */
+    _getRecords(parent?: P): AsyncIterable<O>;
+    /**
+     * Return the list of keys that can be configured in the UI by the end user as the replication key
+     * @returns
+     */
+    getValidReplicationKeys(): Promise<string[]>;
+    /**
+     * Called at the end of the sync. Use to flush all records before closing the Stream.
+     * @returns
+     */
+    _flush(): Promise<void>;
+}
+
+/**
+ * R: Type of the API result
+ * NPT: Type of the "nextPageToken"
+ */
+declare abstract class RESTStreamV2<R, O, NPT, C, P = undefined> extends StreamV2<O, C, P> {
+    axiosInstance: AxiosInstance | undefined;
+    constructor(tapName: string, config: C, state: InputTapState, target: ITarget, childConcurrency?: number);
+    /**
+     * Prepare an Axios request object.
+     * Pagination information can be parsed from `nextPageToken` if `nextPageToken` is not undefined.
+     * @param nextPageToken
+     * @returns
+     */
+    _prepareRequest: (nextPageToken: NPT | undefined, parent?: P) => Promise<AxiosRequestConfig>;
+    _requestWithRetry(request: AxiosRequestConfig, privateLogging?: boolean): Promise<R | "STOP_SYNC">;
+    _requestRecords(parent?: P): AsyncIterable<O>;
+    _getRecords(parent?: P): AsyncIterable<O>;
+    httpMethod: HTTPMethod;
+    baseUrl: string;
+    path: string;
+    httpRetryCount: number;
+    authenticator?: Authenticator<C>;
+    apiCallsMetricsConf: MetricConfiguration;
+    httpErrorHandler(url: string | undefined, err: AxiosError): Promise<R | undefined>;
+    /**
+     * Return a dictionary of values to be used in URL parameterization.
+     * If paging is supported, developers may override with specific paging logic.
+     *
+     * By default, no params are passed.
+    */
+    getNextUrlParams(nextPageToken: NPT | undefined, parent?: P): URLParams;
+    /**
+     * Prepare the data payload for the REST API request.
+     * Useful when working with POST APIs.
+     *
+     * By default, no payload will be sent (return undefined).
+    */
+    getRequestBodyForNextCall(nextPageToken: NPT | undefined, parent?: P): any;
+    /**
+     * Return token identifying next page or undefined if all records have been read.
+     *
+     * By default, no token will be returned and a single API call will be made.
+     *
+     * @param response
+     * @param previousToken
+     * @returns
+     */
+    getNextPageToken(response: R, previousToken?: NPT): NPT | undefined;
+    /**
+     * Return custom headers to be used for HTTP requests.
+     *
+     * By default, no additional header will be added
+     */
+    getCustomHTTPHeaders(parent?: P): HTTPHeaders | undefined;
+    /**
+     * Return a URL, optionally targeted to a specific partition.
+     * This can be overriden to perform dynamic URL generation.
+     *
+     * TODO: Make URL + Path templatisable and replace those with Extractor (or Tap) config
+     * @returns
+     */
+    getUrl(parent?: P): string;
+    /**
+     * Parse the response and return an iterator of result rows.
+     *
+     * Default impl assume that R = O or R = O[]
+     * @param response
+     */
+    parseResponse(response: R, parent?: P, nextPageToken?: NPT): AsyncIterable<O>;
+}
+
+declare const DEFAULT_MAX_CONCURRENT_STREAMS = 5;
+declare abstract class Tap<C> {
+    _target: ITarget;
+    _config: C;
+    _streams: StreamV2[];
+    _concurrency: number;
+    abstract init(): Promise<void>;
+    abstract requiredConfigKeys(): string[];
+    constructor(target: ITarget, config: C);
+    discover: (existingCatalog: CatalogFile | undefined) => Promise<CatalogFile>;
+    sync: (catalog: Catalog) => Promise<void>;
+    generateMetadataFromStream(stream: StreamV2, parentStreamId: string | undefined): Promise<{
+        metadata: CatalogMetadata[];
+    }>;
+    initiateSync(catalog: Catalog): Promise<void>;
+    end(): Promise<void>;
+}
+
+declare class SchemaValidationError extends Error {
+    streamId: StreamId;
+    validationErrors: ValidationError[];
+    constructor(message: string, streamId: StreamId, errors: ValidationError[]);
+}
+declare class MissingFieldInSchemaError extends Error {
+    streamId: StreamId;
+    constructor(message: string, streamId: StreamId);
+}
+declare class MissingSchemaError extends Error {
+    streamId: StreamId;
+    constructor(message: string, streamId: StreamId);
+}
+
+declare const removeParasiteProperties: (record: any, schema: FlattenedSchema) => any;
+declare const addWhalyFields: (record: any, batchDate: string) => any;
+
+declare const flattenSchema: <T>(streamId: StreamId, schema: T) => FlattenedSchema;
+
+declare const createTemporaryFileStream: (streamId: StreamId) => TempFile;
+declare function safePath(path: string): string;
+
+export { ALL_STREAM_ID_KEYWORD, API_CALLS_METRIC_NAME, Authenticator, BATCH_INTERVAL_MS, type BaseConfig, type Bookmark, type Bookmarks, Catalog, type CatalogFile, type CatalogMetadata, type CatalogStream, type ColumnMappingStoreUnsafeToSafe, CounterMetric, DEFAULT_MAX_CONCURRENT_STREAMS, type DestinationType, type DirectionalRelationship, type DirectionalRelationships, EXECUTION_TIME_METRIC_NAME, type ErrorType, type FlattenedSchema, type GetConfig, type HTTPHeaders, type HTTPMethod, ITarget, type InclusionMode, type InputTapState, type InternalTableField, type JSONSchemaFieldDefinition, type Message, type MessageType, type MetadataEntry, type MetricConfiguration, MissingFieldInSchemaError, MissingSchemaError, type PostConfig, type ProgressMarkers, type PropertiesMetadata, RESTStreamV2, ROWS_SYNCED_METRIC_NAME, RecordMessage, RenameColumnStore, type ReplicationMethod, ReplicationMethodMessage, Resolver, type ResolverType, type SafeColumnNameConverterFn, type Schema, SchemaMessage, SchemaValidationError, type ShoreCommand, type ShoreConfig, type SourceType, type State, StateMessage, StateService, type StreamId, StreamMetadata, type StreamName, type StreamRelationship, type StreamState, StreamV2, StreamWarehouseSyncService, type StreamWithTempFile, type SyncFunction, Tap, type TapState, type TargetSchemaHook, type TargetSchemaHookInput, type TempFile, type URLParams, type WarehouseTableField, _greaterThan, addWhalyFields, checkRequiredConfigKeys, createTemporaryFileStream, extractStateForStream, finalizeStateProgressMarkers, findRelatedRelationships, flattenSchema, getAllMetrics, getAxiosInstance, getCounterMetric, getCounterMetrics, getDownloadFileApiCall, getJSONApiCall, getJSONApiCallWithFullResponse, getRawJSONApiCall, gracefulExit, haltAndCatchFire, incrementStreamState, loadJson, loadSchema, loadShoreConfig, logger, postFormDataApiCall, postJSONApiCall, postUrlEncodedApiCall, printMemoryFootprint, removeParasiteProperties, safePath, writeCatalogFile, writeStateFile };