@whaly/connector-sdk 0.3.3 → 0.3.5

package/dist/index.d.mts

@@ -5,7 +5,7 @@ import { WriteStream } from 'fs';
 import { WorkBook } from 'xlsx';
 export { WorkBook } from 'xlsx';
 import Client from 'ssh2-sftp-client';
-export { default as SftpClient, ConnectOptions as SftpConnectOptions } from 'ssh2-sftp-client';
+export { default as SftpClient, ConnectOptions as SftpConnectOptions, FileInfo as SftpFileInfo } from 'ssh2-sftp-client';
 import { File } from '@google-cloud/storage';
 import { BigQuery, Dataset, Table } from '@google-cloud/bigquery';
 
@@ -36,6 +36,33 @@ interface PostConfig {
 declare const postJSONApiCall: <T>(endpoint: string, config: PostConfig, payload: any) => Promise<T>;
 declare const postFormDataApiCall: <T>(endpoint: string, config: PostConfig, payload: any) => Promise<T>;
 declare const postUrlEncodedApiCall: (endpoint: string, config: PostConfig, payload: any) => Promise<any>;
+/**
+ * Collect all records from a paginated HTTP endpoint.
+ *
+ * Eliminates the need for manual `while (!done)` loops in connector code,
+ * and avoids the TypeScript TS7022 circular-inference error that occurs when
+ * a `string | null` loop variable is reassigned from the response body.
+ *
+ * @param options.initialUrl  - First page URL (always a `string`, never null)
+ * @param options.fetchPage   - Fetches one page; receives the current URL, returns parsed response
+ * @param options.getRecords  - Extracts the array of records from a page response
+ * @param options.getNextUrl  - Extracts the next-page URL, or `undefined` when done
+ *
+ * @example
+ * // Salesforce SOQL pagination
+ * const records = await collectPaginated({
+ *   initialUrl: `/services/data/v61.0/queryAll?q=${encodeURIComponent(soql)}`,
+ *   fetchPage:  (url) => http.get(url, { headers }).then(r => r.data),
+ *   getRecords: (res) => res.records,
+ *   getNextUrl: (res) => res.nextRecordsUrl,
+ * });
+ */
+declare function collectPaginated<TResponse, TRecord>(options: {
+    initialUrl: string;
+    fetchPage: (url: string) => Promise<TResponse>;
+    getRecords: (response: TResponse) => TRecord[];
+    getNextUrl: (response: TResponse) => string | undefined;
+}): Promise<TRecord[]>;
 
 declare const BATCH_INTERVAL_MS = 100;
 declare const logger: {
@@ -76,6 +103,51 @@ declare const haltAndCatchFire: (errorType: ErrorType, errorText: string, errorD
 
 declare function gracefulExit(exitCode: number): Promise<void>;
 
+declare function isDryRun(): boolean;
+declare function getDryRunLimit(): number | undefined;
+
+declare function getApiEndpoint(explicit?: string): string;
+
+declare function getServiceAccountKey(explicit?: string): string;
+
+declare function getCdnId(explicit?: string): string;
+
+/**
+ * Read a required environment variable. Throws if not set or empty.
+ * Automatically loads .env file on first call.
+ */
+declare function requireEnv(name: string): string;
+/**
+ * Read an optional environment variable, returning a default if not set.
+ * Automatically loads .env file on first call.
+ */
+declare function optionalEnv(name: string, defaultValue: string): string;
+
+/**
+ * Run an array of async task factories with a bounded concurrency limit.
+ * Results are returned in the same order as the input tasks.
+ */
+declare function runWithConcurrency<T>(tasks: (() => Promise<T>)[], concurrency: number): Promise<T[]>;
+/**
+ * Process items from an async iterable with bounded concurrency.
+ *
+ * Spawns `concurrency` workers that each pull from the shared iterator.
+ * - If `processor` returns `"stop"`, no new items are pulled (in-flight items finish).
+ * - If `processor` throws and no `onError` is provided, the error propagates and stops all workers.
+ * - If `onError` is provided, errors are swallowed (handled by callback) and processing continues.
+ */
+declare function processFromAsyncIterable<T>(iterable: AsyncIterable<T>, processor: (item: T) => Promise<void | "continue" | "stop">, concurrency: number, onError?: (item: T, error: unknown) => void): Promise<void>;
+
+/**
+ * Known MIME types indexed by lowercase extension (with leading dot).
+ */
+declare const MIME_TYPES: Record<string, string>;
+/**
+ * Look up the MIME type for a file path or extension.
+ * Returns `application/octet-stream` when the extension is unknown.
+ */
+declare function getMimeType(filePathOrExt: string): string;
+
 declare enum ReplicationMethod {
     INCREMENTAL = "INCREMENTAL",
     FULL_TABLE = "FULL_TABLE",
@@ -193,6 +265,7 @@ interface BaseConfig {
     database: string;
     schema: string;
     syncedAtColumnName?: string;
+    syncedAtColumnUseLegacyStringType?: boolean;
 }
 
 interface ColumnMappingStoreUnsafeToSafe {
@@ -305,6 +378,7 @@ interface StateProvider {
 declare abstract class ITarget<C extends BaseConfig = BaseConfig> {
     config: C;
     syncedAtColumnName: string;
+    syncedAtColumnUseLegacyStringType: boolean;
     schemaHooks: TargetSchemaHook[];
     syncTime: Dayjs;
     stateProvider: StateProvider;
@@ -828,10 +902,17 @@ declare const checkCsvHeaderRow: (path: string, fileConfig: CsvFileConfig) => Pr
  * Writes an array of objects directly to a CSV file.
  */
 declare const writeDataToCsv: (fileName: string, data: any[]) => Promise<void>;
+interface WriteGeneratorToCsvOptions {
+    /** Number of rows per write batch. Defaults to 10 000. */
+    batchSize?: number;
+    /** Wrap every field in quotes. Defaults to true. */
+    alwaysQuote?: boolean;
+}
 /**
- * Writes data from an async generator to CSV with batch writing (10k row batches).
+ * Writes data from an async generator to CSV with batch writing.
+ * Headers are inferred from the first yielded record.
  */
-declare const writeGeneratorToCSV: (generator: AsyncGenerator, outputFileName: string) => Promise<number>;
+declare const writeGeneratorToCSV: (generator: AsyncGenerator, outputFileName: string, options?: WriteGeneratorToCsvOptions) => Promise<number>;
 
 /**
  * Builder pattern for creating CSV configurations.
@@ -1008,6 +1089,27 @@ declare class FilePatterns {
  * @deprecated Prefer creating your own SftpClient instance for better lifecycle control.
  */
 declare const SftpService: Client;
+interface ListFilesOptions {
+    /** Recurse into subdirectories. Defaults to true. */
+    recursive?: boolean;
+    /** Only include files whose extension (lowercased, with dot) is in this set. */
+    extensions?: string[];
+    /** Safety limit on the number of directories traversed. Defaults to 1000. */
+    maxIterations?: number;
+}
+/**
+ * List files on an SFTP server, optionally recursing into subdirectories.
+ */
+declare function listFilesRecursively(client: Client, remotePath: string, options?: ListFilesOptions): Promise<Client.FileInfo[]>;
+interface DownloadFilesOptions {
+    /** Max concurrent downloads. Defaults to 5. */
+    concurrency?: number;
+}
+/**
+ * Download a list of remote files to a local directory.
+ * Returns the array of local file paths written.
+ */
+declare function downloadFiles(client: Client, files: Client.FileInfo[], localDir: string, options?: DownloadFilesOptions): Promise<string[]>;
 
 interface CloudStorageServiceOptions {
     processedSuffix?: string;
@@ -1073,6 +1175,187 @@ declare class CloudStorageService {
  */
 declare const unzip: (zipFilePath: string, extractedPath: string) => Promise<string>;
 
+interface CdnServiceConfig {
+    /** e.g. "https://org.my.whaly.io" */
+    apiEndpoint?: string;
+    /** e.g. "sk:xxxx" */
+    serviceAccountKey?: string;
+}
+interface CdnUploadResult {
+    /** "/org/{cdnId}/file/{fileName}" */
+    filePath: string;
+}
+interface CdnFileMetadata {
+    exists: boolean;
+    /** GCS `updated` timestamp, null if file doesn't exist or header absent. */
+    lastModified: Date | null;
+    contentType: string | null;
+}
+declare class CdnService {
+    private axiosClient;
+    constructor(config: CdnServiceConfig);
+    /**
+     * Returns metadata for a CDN file via a HEAD request.
+     * If the file does not exist, returns `{ exists: false, lastModified: null, contentType: null }`.
+     * Falls back to not-existing if HEAD is not supported (405).
+     */
+    getFileMetadata(cdnId: string, fileName: string): Promise<CdnFileMetadata>;
+    /**
+     * Checks whether a file already exists in the CDN via a HEAD request.
+     * Returns true if the server responds 2xx, false on 404.
+     * Falls back to false (treat as not existing) if HEAD is not supported (405).
+     * No body is transferred.
+     */
+    fileExists(cdnId: string, fileName: string): Promise<boolean>;
+    /**
+     * Uploads a file buffer to the Whaly CDN.
+     * The PUT is idempotent — uploading the same fileName overwrites the existing file.
+     */
+    uploadFile(cdnId: string, fileName: string, fileBuffer: Buffer): Promise<CdnUploadResult>;
+    private throwMeaningfulError;
+}
+
+type AssetReplicationMode = "FULL" | "INCREMENTAL";
+interface AssetEntry {
+    /** Path/identifier in the source system (e.g. /remote/images/logo.png) */
+    sourcePath: string;
+    /** Path in the destination (e.g. logos/logo.webp) */
+    destinationPath: string;
+    /** Source last-modified timestamp; undefined if unavailable */
+    lastModified: Date | undefined;
+    /** MIME content type */
+    contentType: string;
+}
+interface ProcessedAsset {
+    entry: AssetEntry;
+    /** Where the original source file was downloaded (e.g. out/tmp/logo.jpg) */
+    downloadedPath: string;
+    /**
+     * File path ready for upload.
+     * - No transform: same reference as downloadedPath
+     * - Transformed: a new path produced by transformFile()
+     */
+    uploadPath: string;
+    /** True when a transform was applied (downloadedPath !== uploadPath) */
+    wasTransformed: boolean;
+    size: number;
+    contentType: string;
+}
+interface AssetManifestEntry {
+    sourcePath: string;
+    destinationPath: string;
+    /** Path in the downloaded/ folder */
+    downloadedPath: string;
+    /** Path in the transformed/ folder (empty string if not transformed) */
+    transformedPath: string;
+    size: number;
+    contentType: string;
+    status: "uploaded" | "skipped" | "error";
+    transformed: boolean;
+    error?: string;
+}
+interface StreamManifest {
+    streamId: string;
+    mode: AssetReplicationMode;
+    syncedAt: string;
+    assets: AssetManifestEntry[];
+    summary: {
+        total: number;
+        uploaded: number;
+        skipped: number;
+        errors: number;
+    };
+}
+interface AssetManifest {
+    /** ISO 8601 timestamp of when the sync completed */
+    syncedAt: string;
+    mode: AssetReplicationMode;
+    streams: StreamManifest[];
+    summary: {
+        total: number;
+        uploaded: number;
+        skipped: number;
+        errors: number;
+    };
+}
+
+declare abstract class AssetStream<C> {
+    readonly streamId: string;
+    readonly config: C;
+    replicationMode: AssetReplicationMode;
+    constructor(streamId: string, config: C, replicationMode?: AssetReplicationMode);
+    abstract listAssets(): AsyncIterable<AssetEntry>;
+    /**
+     * Download a single source entry to `destPath` on local disk.
+     * Override this in concrete streams to pull from SFTP, API, etc.
+     */
+    abstract downloadEntry(entry: AssetEntry, destPath: string): Promise<void>;
+    transformFile(downloadedPath: string, _entry: AssetEntry): Promise<string>;
+}
+
+declare abstract class AssetTarget<C> {
+    readonly config: C;
+    constructor(config: C);
+    abstract shouldSync(entry: AssetEntry): Promise<boolean>;
+    abstract uploadAsset(asset: ProcessedAsset): Promise<void>;
+    complete(): Promise<void>;
+}
+
+declare abstract class AssetTap<C> {
+    readonly config: C;
+    readonly outputDir: string;
+    readonly streams: AssetStream<unknown>[];
+    readonly concurrency: number;
+    private readonly target;
+    constructor(target: AssetTarget<unknown>, config: C, outputDir?: string, concurrency?: number);
+    /** Register streams. Called once at the start of sync(). */
+    abstract init(): Promise<void>;
+    sync(): Promise<AssetManifest>;
+}
+
+interface WebpTransformOptions {
+    /** Target width in pixels */
+    width?: number;
+    /** Target height in pixels */
+    height?: number;
+    /**
+     * Background color for padding when using `extent`.
+     * - "none" or "transparent" → fully transparent
+     * - "white", "black", etc. → named CSS color
+     * - "#rrggbb" or "#rrggbbaa" → hex color
+     */
+    background?: string;
+    /**
+     * How to position the image within the extent canvas.
+     * Maps to sharp's `position` option on `resize`.
+     * Common values: "center" (default), "top", "bottom", "left", "right",
+     * "top left", "top right", "bottom left", "bottom right".
+     */
+    gravity?: string;
+    /**
+     * If true and width/height are set, the output will be padded to
+     * exactly width×height (like ImageMagick's `-extent`).
+     * The image is resized to fit inside the dimensions first,
+     * then padded with the background color to fill the canvas.
+     */
+    extent?: boolean;
+    /** WebP quality (1-100). Defaults to sharp's default (80). */
+    quality?: number;
+}
+declare class ImageTransform {
+    /**
+     * Converts an image to WebP format using sharp (libvips).
+     *
+     * The output file is written to the same directory as the input,
+     * with the extension replaced by `.webp`.
+     *
+     * @param inputPath  Path to the source image file.
+     * @param options    Optional resize / padding parameters.
+     * @returns          Path to the produced `.webp` file.
+     */
+    static toWebp(inputPath: string, options: WebpTransformOptions): Promise<string>;
+}
+
 interface BigQueryConfig extends BaseConfig {
     schema: string;
     project_id: string;
@@ -1132,6 +1415,18 @@ declare class BigQueryTarget extends ITarget<BigQueryConfig> {
     convertNumberIntoDecimal: (record: any, schema: FlattenedSchema) => any;
 }
 
+interface CdnAssetTargetConfig extends CdnServiceConfig {
+    /** The CDN ID (organization CDN identifier) */
+    cdnId?: string;
+}
+
+declare class CdnAssetTarget extends AssetTarget<CdnAssetTargetConfig> {
+    private readonly cdnService;
+    constructor(config: CdnAssetTargetConfig);
+    shouldSync(entry: AssetEntry): Promise<boolean>;
+    uploadAsset(asset: ProcessedAsset): Promise<void>;
+}
+
 declare class GCSStateProvider implements StateProvider {
     private bucketName;
     private connectorId;
@@ -1142,4 +1437,4 @@ declare class GCSStateProvider implements StateProvider {
     writeState(state: string): Promise<void>;
 }
 
-export { ALL_STREAM_ID_KEYWORD, API_CALLS_METRIC_NAME, Authenticator, BATCH_INTERVAL_MS, type BaseConfig, type BigQueryConfig, BigQueryTarget, type BigqueryLocation, type Bookmark, type Bookmarks, CloudStorageService, type CloudStorageServiceOptions, type ColumnMappingStoreUnsafeToSafe, CounterMetric, CsvExtractionConfigBuilder, type CsvFieldsArrayConfig, type CsvFieldsConfig, type CsvFieldsDictConfig, type CsvFileConfig, type CsvStreamConfig, DEFAULT_MAX_CONCURRENT_STREAMS, DEFAULT_SYNCED_AT_COLUMN, EXECUTION_TIME_METRIC_NAME, type ErrorType, type ExcelCustomExtractorConfig, type ExcelDerivedField, type ExcelExtractionBaseConfig, type ExcelExtractionConfig, ExcelExtractionConfigBuilder, type ExcelFieldMapping, type ExcelFieldSpec, type ExcelSingleSheetExtractionConfig, type ExcelSourceColumn, type ExcelStreamConfig, type FieldType, FileFormat, FilePatterns, FileStream, type FileStreamConfig, type FileStreamEntry, FileTap, type FileTapConfig, type FlattenedSchema, GCSStateProvider, type GetConfig, type HTTPHeaders, type HTTPMethod, ITarget, type InputTapState, type InternalTableField, type JSONSchemaFieldDefinition, type Message, type MessageType, type MetricConfiguration, MissingFieldInSchemaError, MissingSchemaError, type PostConfig, type ProgressMarkers, type PropertiesMetadata, RESTStream, ROWS_SYNCED_METRIC_NAME, RecordMessage, RenameColumnStore, ReplicationMethod, ReplicationMethodMessage, type SafeColumnNameConverterFn, type Schema, SchemaMessage, SchemaValidationError, SftpService, type State, type StateHolder, StateMessage, type StateProvider, StateService, Stream, type StreamId, type StreamState, StreamWarehouseSyncService, type StreamWithTempFile, type SyncOptions, Tap, type TapState, type TargetSchemaHook, type TargetSchemaHookInput, type TempFile, type URLParams, VariableExtractors, type WarehouseTableField, _greaterThan, addWhalyFields, checkCsvHeaderRow, countCsvLines, createCellReference, createCsvStreamConfig, createExcelGenerator, createExcelStreamConfig, createTemporaryFileStream, csvFieldsToJsonSchema, excelColumnToIndex, excelFieldsToJsonSchema, excelRowToIndex, extractExcelRows, extractPrimaryKeysFromCsvConfig, extractPrimaryKeysFromExcelFields, extractStateForStream, fieldTypeToJsonSchema, finalizeStateProgressMarkers, findAllMatchingConfigs, flattenSchema, getAllMetrics, getAxiosInstance, getCounterMetric, getCounterMetrics, getDownloadFileApiCall, getJSONApiCall, getJSONApiCallWithFullResponse, getRawJSONApiCall, gracefulExit, haltAndCatchFire, incrementStreamState, indexToExcelColumn, indexToExcelRow, loadJson, loadSchema, logger, parseCellReference, postFormDataApiCall, postJSONApiCall, postUrlEncodedApiCall, printMemoryFootprint, processFileStreams, removeParasiteProperties, rowGeneratorFromCsv, safePath, unzip, validateExcelConfig, writeDataToCsv, writeGeneratorToCSV };
+export { ALL_STREAM_ID_KEYWORD, API_CALLS_METRIC_NAME, type AssetEntry, type AssetManifest, type AssetManifestEntry, type AssetReplicationMode, AssetStream, AssetTap, AssetTarget, Authenticator, BATCH_INTERVAL_MS, type BaseConfig, type BigQueryConfig, BigQueryTarget, type BigqueryLocation, type Bookmark, type Bookmarks, CdnAssetTarget, type CdnAssetTargetConfig, type CdnFileMetadata, CdnService, type CdnServiceConfig, type CdnUploadResult, CloudStorageService, type CloudStorageServiceOptions, type ColumnMappingStoreUnsafeToSafe, CounterMetric, CsvExtractionConfigBuilder, type CsvFieldsArrayConfig, type CsvFieldsConfig, type CsvFieldsDictConfig, type CsvFileConfig, type CsvStreamConfig, DEFAULT_MAX_CONCURRENT_STREAMS, DEFAULT_SYNCED_AT_COLUMN, type DownloadFilesOptions, EXECUTION_TIME_METRIC_NAME, type ErrorType, type ExcelCustomExtractorConfig, type ExcelDerivedField, type ExcelExtractionBaseConfig, type ExcelExtractionConfig, ExcelExtractionConfigBuilder, type ExcelFieldMapping, type ExcelFieldSpec, type ExcelSingleSheetExtractionConfig, type ExcelSourceColumn, type ExcelStreamConfig, type FieldType, FileFormat, FilePatterns, FileStream, type FileStreamConfig, type FileStreamEntry, FileTap, type FileTapConfig, type FlattenedSchema, GCSStateProvider, type GetConfig, type HTTPHeaders, type HTTPMethod, ITarget, ImageTransform, type InputTapState, type InternalTableField, type JSONSchemaFieldDefinition, type ListFilesOptions, MIME_TYPES, type Message, type MessageType, type MetricConfiguration, MissingFieldInSchemaError, MissingSchemaError, type PostConfig, type ProcessedAsset, type ProgressMarkers, type PropertiesMetadata, RESTStream, ROWS_SYNCED_METRIC_NAME, RecordMessage, RenameColumnStore, ReplicationMethod, ReplicationMethodMessage, type SafeColumnNameConverterFn, type Schema, SchemaMessage, SchemaValidationError, SftpService, type State, type StateHolder, StateMessage, type StateProvider, StateService, Stream, type StreamId, type StreamManifest, type StreamState, StreamWarehouseSyncService, type StreamWithTempFile, type SyncOptions, Tap, type TapState, type TargetSchemaHook, type TargetSchemaHookInput, type TempFile, type URLParams, VariableExtractors, type WarehouseTableField, type WebpTransformOptions, type WriteGeneratorToCsvOptions, _greaterThan, addWhalyFields, checkCsvHeaderRow, collectPaginated, countCsvLines, createCellReference, createCsvStreamConfig, createExcelGenerator, createExcelStreamConfig, createTemporaryFileStream, csvFieldsToJsonSchema, downloadFiles, excelColumnToIndex, excelFieldsToJsonSchema, excelRowToIndex, extractExcelRows, extractPrimaryKeysFromCsvConfig, extractPrimaryKeysFromExcelFields, extractStateForStream, fieldTypeToJsonSchema, finalizeStateProgressMarkers, findAllMatchingConfigs, flattenSchema, getAllMetrics, getApiEndpoint, getAxiosInstance, getCdnId, getCounterMetric, getCounterMetrics, getDownloadFileApiCall, getDryRunLimit, getJSONApiCall, getJSONApiCallWithFullResponse, getMimeType, getRawJSONApiCall, getServiceAccountKey, gracefulExit, haltAndCatchFire, incrementStreamState, indexToExcelColumn, indexToExcelRow, isDryRun, listFilesRecursively, loadJson, loadSchema, logger, optionalEnv, parseCellReference, postFormDataApiCall, postJSONApiCall, postUrlEncodedApiCall, printMemoryFootprint, processFileStreams, processFromAsyncIterable, removeParasiteProperties, requireEnv, rowGeneratorFromCsv, runWithConcurrency, safePath, unzip, validateExcelConfig, writeDataToCsv, writeGeneratorToCSV };