@whaly/connector-sdk 0.1.1 → 0.2.0

package/dist/index.d.mts

@@ -1,49 +1,90 @@
-import winston from 'winston';
 import { AxiosInstance, AxiosRequestHeaders, AxiosResponse, AxiosRequestConfig, AxiosError } from 'axios';
-import moment from 'moment';
 import { Validator, ValidationError } from 'jsonschema';
+import { Dayjs } from 'dayjs';
 import { WriteStream } from 'fs';
 import { BigQuery, Dataset, Table } from '@google-cloud/bigquery';
 
-type StreamId = string;
-type StreamName = string;
-interface CatalogStreamMetadata {
-    breadcrumb: string[];
-    metadata: Record<string, any>;
-}
-interface CatalogStream {
-    tap_stream_id: StreamId;
-    stream: StreamName;
-    schema: any;
-    metadata: CatalogStreamMetadata[];
+declare const loadJson: (fileName: string) => any;
+declare const loadSchema: (streamId: string) => any;
+
+declare const getAxiosInstance: (retryCount?: number) => AxiosInstance;
+interface GetConfig {
+    qs?: {
+        [key: string]: any;
+    };
+    headers?: AxiosRequestHeaders;
+    retryCount?: number;
 }
-interface CatalogFile {
-    streams: CatalogStream[];
+declare const getRawJSONApiCall: <T>(endpoint: string, config: GetConfig, privateLogging?: boolean) => Promise<AxiosResponse<T>>;
+declare const getJSONApiCallWithFullResponse: <T>(endpoint: string, config: GetConfig, privateLogging?: boolean) => Promise<AxiosResponse<any, any, {}>>;
+declare const getJSONApiCall: <T>(endpoint: string, config: GetConfig, privateLogging?: boolean) => Promise<T>;
+declare const getDownloadFileApiCall: <T>(endpoint: string, config: GetConfig, output: string, privateLogging?: boolean) => Promise<{
+    outputDir: string;
+}>;
+interface PostConfig {
+    qs?: {
+        [key: string]: any;
+    };
+    headers?: AxiosRequestHeaders;
+    retryCount?: number;
 }
+declare const postJSONApiCall: <T>(endpoint: string, config: PostConfig, payload: any) => Promise<T>;
+declare const postFormDataApiCall: <T>(endpoint: string, config: PostConfig, payload: any) => Promise<T>;
+declare const postUrlEncodedApiCall: (endpoint: string, config: PostConfig, payload: any) => Promise<any>;
 
-type ReplicationMethod = "INCREMENTAL" | "FULL_TABLE";
-type DestinationType = `GOOGLE_BIGQUERY` | `SNOWFLAKE`;
-interface StateHolder {
-    state?: any;
+declare const BATCH_INTERVAL_MS = 100;
+declare const logger: {
+    info: (message: string | any, ...args: any[]) => void;
+    error: (message: string | any, ...args: any[]) => void;
+    debug: (message: string | any, ...args: any[]) => void;
+    warn: (message: string | any, ...args: any[]) => void;
+    closeWinston: (cb: () => void) => void;
+};
+
+declare const ALL_STREAM_ID_KEYWORD = "all";
+declare const ROWS_SYNCED_METRIC_NAME = "rows_synced_count";
+declare const API_CALLS_METRIC_NAME = "api_calls_count";
+declare const EXECUTION_TIME_METRIC_NAME = "execution_time_ms";
+interface MetricConfiguration {
+    isEnabled: () => boolean;
+    getStreamIds?: () => string[];
 }
+declare const getCounterMetrics: (name: string, streamIds: string[]) => CounterMetric[];
+declare const getCounterMetric: (name: string, streamId?: string) => CounterMetric;
+declare const getAllMetrics: () => CounterMetric[];
+declare class CounterMetric {
+    private value;
+    name: string;
+    streamId: string;
+    constructor(name: string, streamId: string);
+    increment(inc?: number): void;
+    getStreamId(): string;
+    getName(): string;
+    getValue(): number;
+}
+
+declare const printMemoryFootprint: (prefix: string) => void;
+
+type ErrorType = "unauthorized" | "unknown";
+
+declare const haltAndCatchFire: (errorType: ErrorType, errorText: string, errorDebugText: string) => Promise<void>;
+
+declare function gracefulExit(exitCode: number): Promise<void>;
 
-interface StreamRelationship {
-    left: StreamId;
-    right: StreamId;
-    from: string;
-    to: string;
-    type: "1-1" | "1-N" | "N-1";
+declare enum ReplicationMethod {
+    INCREMENTAL = "INCREMENTAL",
+    FULL_TABLE = "FULL_TABLE"
 }
-interface DirectionalRelationship {
-    type: "1-N" | "N-1" | "1-1";
-    streamId: StreamId;
-    from: string;
-    to: string;
+
+interface HTTPHeaders {
+    [headerName: string]: any;
 }
-interface DirectionalRelationships {
-    left: DirectionalRelationship[];
-    right: DirectionalRelationship[];
+interface URLParams {
+    [paramName: string]: any;
 }
+type HTTPMethod = "GET" | "POST";
+
+type StreamId = string;
 
 interface PropertiesMetadata {
     [propName: string]: {
@@ -93,7 +134,6 @@ declare class SchemaMessage implements Message {
     displayLabel: string | undefined;
     description: string | undefined;
     propertiesMetadata: PropertiesMetadata | undefined;
-    relationships: DirectionalRelationships;
     constructor(opts: {
         stream: StreamId;
         schema: any;
@@ -101,7 +141,6 @@ declare class SchemaMessage implements Message {
         bookmarkProperties?: string[];
         displayLabel?: string;
         description?: string;
-        relationships: DirectionalRelationships;
         propertiesMetadata?: PropertiesMetadata | undefined;
     });
     toString(): string;
@@ -117,150 +156,6 @@ interface TargetSchemaHook {
     writeSchema: (input: TargetSchemaHookInput) => Promise<void>;
 }
 
-type ErrorType = "unauthorized" | "unknown";
-
-declare abstract class Resolver {
-    constructor();
-    abstract checkIfCanSync(): Promise<boolean>;
-    abstract markSyncStarted(): Promise<void>;
-    abstract getState(): Promise<StateHolder>;
-    abstract writeCatalog(catalog: CatalogFile): Promise<void>;
-    abstract writeState(state: string): Promise<void>;
-    abstract markSyncFailed(): Promise<void>;
-    abstract markSyncComplete(): Promise<void>;
-    abstract markDiscoveryComplete(): Promise<void>;
-    abstract flushMetrics(): Promise<void>;
-    abstract onError(errorType: ErrorType, errorText: string, errorDebugText: string): Promise<void>;
-    abstract getSchemaHooks(): TargetSchemaHook[];
-    abstract updateSourceValue(optionKey: string, optionValue: string): Promise<void>;
-    abstract getLogFormat(): winston.Logform.Format;
-}
-
-declare const writeStateFile: (resolver: Resolver, state: string) => Promise<void>;
-declare const loadJson: (fileName: string) => any;
-declare const loadSchema: (streamId: string) => any;
-
-declare const getAxiosInstance: (retryCount?: number) => AxiosInstance;
-interface GetConfig {
-    qs?: {
-        [key: string]: any;
-    };
-    headers?: AxiosRequestHeaders;
-    retryCount?: number;
-}
-declare const getRawJSONApiCall: <T>(endpoint: string, config: GetConfig, privateLogging?: boolean) => Promise<AxiosResponse<T>>;
-declare const getJSONApiCallWithFullResponse: <T>(endpoint: string, config: GetConfig, privateLogging?: boolean) => Promise<AxiosResponse<any, any, {}>>;
-declare const getJSONApiCall: <T>(endpoint: string, config: GetConfig, privateLogging?: boolean) => Promise<T>;
-declare const getDownloadFileApiCall: <T>(endpoint: string, config: GetConfig, output: string, privateLogging?: boolean) => Promise<{
-    outputDir: string;
-}>;
-interface PostConfig {
-    qs?: {
-        [key: string]: any;
-    };
-    headers?: AxiosRequestHeaders;
-    retryCount?: number;
-}
-declare const postJSONApiCall: <T>(endpoint: string, config: PostConfig, payload: any) => Promise<T>;
-declare const postFormDataApiCall: <T>(endpoint: string, config: PostConfig, payload: any) => Promise<T>;
-declare const postUrlEncodedApiCall: (endpoint: string, config: PostConfig, payload: any) => Promise<any>;
-
-declare const BATCH_INTERVAL_MS = 100;
-declare const logger: {
-    info: (message: string | any, ...args: any[]) => void;
-    error: (message: string | any, ...args: any[]) => void;
-    debug: (message: string | any, ...args: any[]) => void;
-    warn: (message: string | any, ...args: any[]) => void;
-    closeWinston: (cb: () => void) => void;
-};
-
-declare const ALL_STREAM_ID_KEYWORD = "all";
-declare const ROWS_SYNCED_METRIC_NAME = "rows_synced_count";
-declare const API_CALLS_METRIC_NAME = "api_calls_count";
-declare const EXECUTION_TIME_METRIC_NAME = "execution_time_ms";
-interface MetricConfiguration {
-    isEnabled: () => boolean;
-    getStreamIds?: () => string[];
-}
-declare const getCounterMetrics: (name: string, streamIds: string[]) => CounterMetric[];
-declare const getCounterMetric: (name: string, streamId?: string) => CounterMetric;
-declare const getAllMetrics: () => CounterMetric[];
-declare class CounterMetric {
-    private value;
-    name: string;
-    streamId: string;
-    constructor(name: string, streamId: string);
-    increment(inc?: number): void;
-    getStreamId(): string;
-    getName(): string;
-    getValue(): number;
-}
-
-declare const printMemoryFootprint: (prefix: string) => void;
-
-declare const haltAndCatchFire: (errorType: ErrorType, errorText: string, errorDebugText: string) => Promise<void>;
-
-declare function gracefulExit(exitCode: number): Promise<void>;
-
-declare const findRelatedRelationships: (streamId: StreamId, allRels: StreamRelationship[]) => DirectionalRelationships;
-
-declare const loadResolver: () => Resolver;
-
-interface HTTPHeaders {
-    [headerName: string]: any;
-}
-interface URLParams {
-    [paramName: string]: any;
-}
-type HTTPMethod = "GET" | "POST";
-
-type InclusionMode = "available" | "automatic" | "unsupported";
-interface MetadataEntry {
-    selected?: boolean;
-    "replication-method"?: ReplicationMethod;
-    "replication-key"?: string;
-    "view-key-properties"?: string[];
-    "inclusion"?: InclusionMode;
-    "selected-by-default"?: boolean;
-    "valid-replication-keys"?: string[];
-    "forced-replication-method"?: ReplicationMethod;
-    "forced-replication-key"?: string;
-    "table-key-properties"?: string[];
-    "schema-name"?: string;
-    "is-view"?: boolean;
-    "row-count"?: number;
-    "database-name"?: string;
-    "sql-datatype"?: string;
-    "whaly-display-label"?: string;
-    "whaly-description"?: string;
-    "whaly-parent-stream"?: StreamId;
-}
-interface CatalogMetadata {
-    breadcrumb: string[];
-    metadata: MetadataEntry;
-}
-declare class MetadataMap extends Map<string[], MetadataEntry> {
-    private areEquals;
-    private findExistingEntry;
-    get(key: string[]): MetadataEntry | undefined;
-    set(key: string[], value: MetadataEntry): this;
-}
-declare class StreamMetadata {
-    metadataByBreadcrumb: MetadataMap;
-    constructor();
-    fromCatalogStreamMetadata(metadataArr: CatalogMetadata[]): this;
-    toString(): MetadataMap;
-    get(breadcrumb: string[]): MetadataEntry | undefined;
-    write(breadcrumb: string[], key: string, value: any): this;
-    private getRootBreadcrumbMetadata;
-    isStreamSelected(): boolean;
-    getKeyProperties(): string[];
-    getReplicationKey(): string | undefined;
-    getReplicationMethod(): ReplicationMethod | undefined;
-    getForcedReplicationMethod(): ReplicationMethod | undefined;
-    toList(): CatalogMetadata[];
-}
-
 interface FlattenedSchema {
     [unsafePropertyName: string]: JSONSchemaFieldDefinition;
 }
@@ -287,8 +182,9 @@ interface StreamWithTempFile {
     tempFile: TempFile;
 }
 interface BaseConfig {
-    schema: string;
+    connector_id: string;
     database: string;
+    schema: string;
 }
 
 interface ColumnMappingStoreUnsafeToSafe {
@@ -363,7 +259,7 @@ declare class StreamState$1 {
     streamId: StreamId;
     schema?: FlattenedSchema;
     dbSync: StreamWarehouseSyncService;
-    batchDate: moment.Moment;
+    batchDate: Dayjs;
     batchedRowCount: number;
     fileToLoad: TempFile;
     syncedRowCount: number;
@@ -386,11 +282,19 @@ declare class StreamState$1 {
     setHasBeenLoadedYet(): void;
 }
 
+interface StateHolder {
+    state?: any;
+}
+interface StateProvider {
+    getState(): Promise<StateHolder>;
+    writeState(state: string): Promise<void>;
+}
+
 declare abstract class ITarget<C extends BaseConfig = BaseConfig> {
     config: C;
     schemaHooks: TargetSchemaHook[];
-    resolver: Resolver;
-    syncTime: moment.Moment;
+    syncTime: Dayjs;
+    stateProvider: StateProvider;
     batchedState: any;
     flushedState: any;
     renameColumnStore: RenameColumnStore;
@@ -398,7 +302,7 @@ declare abstract class ITarget<C extends BaseConfig = BaseConfig> {
         [streamName: string]: StreamState$1;
     };
     validator: Validator;
-    constructor(config: C, resolver: Resolver);
+    constructor(config: C, stateProvider: StateProvider);
     static requiredConfigKeys: string[];
     abstract newDBSyncInstance: (streamId: StreamId, database: string, schema: string, table: string) => StreamWarehouseSyncService;
     abstract genTableName: (streamId: string) => string;
@@ -462,11 +366,9 @@ declare class StateService {
     bookmarks: Bookmarks;
     constructor();
     static getInstance(): StateService;
-    setBookmark(streamId: StreamId, ts: moment.Moment): void;
-    setBookmarkV2(streamId: StreamId, streamState: StreamState): void;
-    setBookmarkSignpostV2(streamId: StreamId, signpostValue: string): void;
-    getBookmark(streamId: StreamId): moment.Moment;
-    getBookmarkV2(streamId: StreamId): StreamState;
+    setBookmark(streamId: StreamId, streamState: StreamState): void;
+    setBookmarkSignpost(streamId: StreamId, signpostValue: string): void;
+    getBookmark(streamId: StreamId): StreamState;
     clearState(): void;
     get(): State;
     forwardStateToTarget(target: ITarget): Promise<void>;
@@ -491,7 +393,7 @@ declare abstract class Stream<O = any, C = any, P = undefined> {
     config: C;
     tapState: InputTapState;
     target: ITarget;
-    forcedReplicationMethod: ReplicationMethod | undefined;
+    replicationMethod: ReplicationMethod | undefined;
     selectedByDefault: boolean;
     STATE_MSG_FREQUENCY: number;
     streamId: StreamId;
@@ -499,16 +401,15 @@ declare abstract class Stream<O = any, C = any, P = undefined> {
     displayLabel: string | undefined;
     description: string | undefined;
     primaryKey: string[];
-    forcedReplicationKey: string | undefined;
+    replicationKey: string | undefined;
     useStateFromStreamId: string | undefined;
-    relationships: StreamRelationship[];
     children: Stream<any, any, O | O[]>[];
     isSorted: boolean;
     isSilent: boolean;
     childConcurrency: number;
     rowsSyncedMetricsConf: MetricConfiguration;
     executionTimeMetricsConf: MetricConfiguration;
-    constructor(config: C, state: InputTapState, target: ITarget, childConcurrency?: number);
+    constructor(config: C, tapState: InputTapState, target: ITarget, childConcurrency?: number);
     /**
      * Sync this stream
      * Called only for streams with no parents, aka. "root tap streams"
@@ -545,17 +446,17 @@ declare abstract class Stream<O = any, C = any, P = undefined> {
      * or `start_date` config if set,
      * or the UNIX Epoch
      **/
-    getStartingTimestamp(): moment.Moment;
+    getStartingTimestamp(): Dayjs;
     /**
      * Return the max allowable bookmark value for this stream's replication key.
      *
-     * For timestamp-based replication keys, this defaults to `moment()`. For
+     * For timestamp-based replication keys, this defaults to `dayjs()`. For
      * non-timestamp replication keys, default to `undefined`.
      *
      * Override this value to prevent bookmarks from being advanced in cases where we
      * may only have a partial set of records.
      */
-    getReplicationKeySignpost: () => Promise<moment.Moment | undefined>;
+    getReplicationKeySignpost: () => Promise<Dayjs | undefined>;
     /**
      * Return the default replication method for the stream that will be used to write the catalog.
      */
@@ -657,7 +558,7 @@ declare abstract class RESTStream<R, O, NPT, C, P = undefined> extends Stream<O,
      * TODO: Make URL + Path templatisable and replace those with Extractor (or Tap) config
      * @returns
      */
-    getUrl(parent?: P): string;
+    getNextUrl(previousToken?: NPT, parent?: P): string;
     /**
      * Parse the response and return an iterator of result rows.
      *
@@ -677,8 +578,10 @@ declare abstract class Tap<C> {
     config: C;
     streams: Stream[];
     concurrency: number;
+    stateProvider: StateProvider;
+    tapState: InputTapState;
     abstract init(): Promise<void>;
-    constructor(target: ITarget, config: C);
+    constructor(target: ITarget, config: C, stateProvider: StateProvider);
     sync: (options?: SyncOptions) => Promise<void>;
     end(): Promise<void>;
 }
@@ -754,7 +657,7 @@ declare class BigQueryDBSync extends StreamWarehouseSyncService {
 }
 
 declare class BigQueryTarget extends ITarget<BigQueryConfig> {
-    constructor(config: BigQueryConfig, resolver: Resolver);
+    constructor(config: BigQueryConfig, stateProvider: StateProvider);
     static requiredConfigKeys: string[];
     newDBSyncInstance: (streamId: StreamId, database: string, schema: string, table: string) => BigQueryDBSync;
     genTableName: (key: string) => string;
@@ -763,4 +666,13 @@ declare class BigQueryTarget extends ITarget<BigQueryConfig> {
     convertNumberIntoDecimal: (record: any, schema: FlattenedSchema) => any;
 }
 
-export { ALL_STREAM_ID_KEYWORD, API_CALLS_METRIC_NAME, Authenticator, BATCH_INTERVAL_MS, type BaseConfig, type BigQueryConfig, BigQueryTarget, type BigqueryLocation, type Bookmark, type Bookmarks, type CatalogFile, type CatalogMetadata, type CatalogStream, type CatalogStreamMetadata, type ColumnMappingStoreUnsafeToSafe, CounterMetric, DEFAULT_MAX_CONCURRENT_STREAMS, type DestinationType, type DirectionalRelationship, type DirectionalRelationships, EXECUTION_TIME_METRIC_NAME, type ErrorType, type FlattenedSchema, type GetConfig, type HTTPHeaders, type HTTPMethod, ITarget, type InclusionMode, type InputTapState, type InternalTableField, type JSONSchemaFieldDefinition, type Message, type MessageType, type MetadataEntry, type MetricConfiguration, MissingFieldInSchemaError, MissingSchemaError, type PostConfig, type ProgressMarkers, type PropertiesMetadata, RESTStream, ROWS_SYNCED_METRIC_NAME, RecordMessage, RenameColumnStore, type ReplicationMethod, ReplicationMethodMessage, Resolver, type SafeColumnNameConverterFn, type Schema, SchemaMessage, SchemaValidationError, type State, type StateHolder, StateMessage, StateService, Stream, type StreamId, StreamMetadata, type StreamName, type StreamRelationship, type StreamState, StreamWarehouseSyncService, type StreamWithTempFile, type SyncOptions, Tap, type TapState, type TargetSchemaHook, type TargetSchemaHookInput, type TempFile, type URLParams, type WarehouseTableField, _greaterThan, addWhalyFields, createTemporaryFileStream, extractStateForStream, finalizeStateProgressMarkers, findRelatedRelationships, flattenSchema, getAllMetrics, getAxiosInstance, getCounterMetric, getCounterMetrics, getDownloadFileApiCall, getJSONApiCall, getJSONApiCallWithFullResponse, getRawJSONApiCall, gracefulExit, haltAndCatchFire, incrementStreamState, loadJson, loadResolver, loadSchema, logger, postFormDataApiCall, postJSONApiCall, postUrlEncodedApiCall, printMemoryFootprint, removeParasiteProperties, safePath, writeStateFile };
+declare class GCSStateProvider implements StateProvider {
+    private bucketName;
+    private connectorId;
+    constructor(connectorId: string, bucketName: string);
+    private getObjectPath;
+    getState(): Promise<StateHolder>;
+    writeState(state: string): Promise<void>;
+}
+
+export { ALL_STREAM_ID_KEYWORD, API_CALLS_METRIC_NAME, Authenticator, BATCH_INTERVAL_MS, type BaseConfig, type BigQueryConfig, BigQueryTarget, type BigqueryLocation, type Bookmark, type Bookmarks, type ColumnMappingStoreUnsafeToSafe, CounterMetric, DEFAULT_MAX_CONCURRENT_STREAMS, EXECUTION_TIME_METRIC_NAME, type ErrorType, type FlattenedSchema, GCSStateProvider, type GetConfig, type HTTPHeaders, type HTTPMethod, ITarget, type InputTapState, type InternalTableField, type JSONSchemaFieldDefinition, type Message, type MessageType, type MetricConfiguration, MissingFieldInSchemaError, MissingSchemaError, type PostConfig, type ProgressMarkers, type PropertiesMetadata, RESTStream, ROWS_SYNCED_METRIC_NAME, RecordMessage, RenameColumnStore, ReplicationMethod, ReplicationMethodMessage, type SafeColumnNameConverterFn, type Schema, SchemaMessage, SchemaValidationError, type State, type StateHolder, StateMessage, type StateProvider, StateService, Stream, type StreamId, type StreamState, StreamWarehouseSyncService, type StreamWithTempFile, type SyncOptions, Tap, type TapState, type TargetSchemaHook, type TargetSchemaHookInput, type TempFile, type URLParams, type WarehouseTableField, _greaterThan, addWhalyFields, createTemporaryFileStream, extractStateForStream, finalizeStateProgressMarkers, flattenSchema, getAllMetrics, getAxiosInstance, getCounterMetric, getCounterMetrics, getDownloadFileApiCall, getJSONApiCall, getJSONApiCallWithFullResponse, getRawJSONApiCall, gracefulExit, haltAndCatchFire, incrementStreamState, loadJson, loadSchema, logger, postFormDataApiCall, postJSONApiCall, postUrlEncodedApiCall, printMemoryFootprint, removeParasiteProperties, safePath };