@whaly/connector-sdk 0.3.10 → 0.3.11

package/dist/index.d.mts

@@ -1125,15 +1125,59 @@ interface DownloadFilesOptions {
  */
 declare function downloadFiles(client: Client, files: Client.FileInfo[], localDir: string, options?: DownloadFilesOptions): Promise<string[]>;
 
+/**
+ * Common interface for storage services (cloud or local).
+ * Implemented by CloudStorageService and LocalStorageService.
+ */
+interface StorageService {
+    /**
+     * Lists all files with an optional prefix/path filter.
+     */
+    listFiles(prefix?: string): Promise<string[]>;
+    /**
+     * Returns files that haven't been marked as processed.
+     */
+    getUnprocessedFiles(): Promise<string[]>;
+    /**
+     * Creates a marker file to indicate a file has been processed.
+     */
+    createMarkerFile(fileName: string): Promise<void>;
+    /**
+     * Downloads/resolves a file and returns a local file path.
+     */
+    downloadFile(filePath: string, fileName: string): Promise<string>;
+    /**
+     * Resolves a file URI to a local file path.
+     * Cloud: downloads from bucket. Local: resolves and validates the local path.
+     */
+    resolveFileUri(fileUri: string): Promise<string>;
+    /**
+     * Uploads a local file to the storage destination.
+     */
+    uploadFile(localPath: string, destPath: string): Promise<File>;
+    /**
+     * Reads a storage object and returns its contents as a UTF-8 string.
+     */
+    readObjectAsString(objectPath: string): Promise<string>;
+    /**
+     * Writes a string to a storage object with JSON content type.
+     */
+    writeStringObject(objectPath: string, contents: string): Promise<void>;
+    /**
+     * Uploads a local file with a unique generated name.
+     */
+    uploadFileWithUniqueName(filePath: string, prefix: string, streamId: string): Promise<File>;
+}
+
 interface CloudStorageServiceOptions {
     processedSuffix?: string;
     supportedExtensions?: string[];
 }
 /**
- * Service for interacting with Google Cloud Storage.
+ * Google Cloud Storage implementation of StorageService.
  * Supports file download, upload, listing, and marker-file-based processing tracking.
  */
-declare class CloudStorageService {
+declare class CloudStorageService implements StorageService {
     private storage;
     private bucket;
     private processedSuffix;
@@ -1170,6 +1214,14 @@ declare class CloudStorageService {
      * Writes a string to a GCS object with JSON content type.
      */
     writeStringObject(objectPath: string, contents: string): Promise<void>;
+    /**
+     * Resolves a file URI to a local file path by downloading it from the configured GCS bucket.
+     *
+     * @example
+     * const storage = new CloudStorageService("my-bucket", "my-folder");
+     * const localPath = await storage.resolveFileUri("folder/file.xlsx");
+     */
+    resolveFileUri(fileUri: string): Promise<string>;
     /**
      * Uploads a local file to the bucket with a unique name based on prefix, streamId, and UUID.
      * Files are stored under `<prefix>/<run-id>/` when the RUN_ID env var is set,
@@ -1179,6 +1231,30 @@ declare class CloudStorageService {
     uploadFileWithUniqueName(filePath: string, prefix: string, streamId: string): Promise<File>;
 }
 
+interface LocalStorageServiceOptions {
+    processedSuffix?: string;
+    supportedExtensions?: string[];
+}
+/**
+ * Local filesystem implementation of StorageService.
+ * Operates entirely on the local filesystem — no cloud interaction.
+ */
+declare class LocalStorageService implements StorageService {
+    private basePath;
+    private processedSuffix;
+    private supportedExtensions;
+    constructor(basePath: string, opts?: LocalStorageServiceOptions);
+    listFiles(prefix?: string): Promise<string[]>;
+    getUnprocessedFiles(): Promise<string[]>;
+    createMarkerFile(fileName: string): Promise<void>;
+    downloadFile(filePath: string, _fileName: string): Promise<string>;
+    resolveFileUri(fileUri: string): Promise<string>;
+    uploadFile(localPath: string, destPath: string): Promise<File>;
+    readObjectAsString(objectPath: string): Promise<string>;
+    writeStringObject(objectPath: string, contents: string): Promise<void>;
+    uploadFileWithUniqueName(filePath: string, prefix: string, streamId: string): Promise<File>;
+}
+
 /**
  * Extract a ZIP (or other archive) file to a target directory.
  * Uses the `decompress` library which supports zip, tar, gz, bz2, etc.
@@ -1644,4 +1720,4 @@ declare class GCSStateProvider implements StateProvider {
     writeState(state: string): Promise<void>;
 }
 
-export { ALL_STREAM_ID_KEYWORD, API_CALLS_METRIC_NAME, type AssetEntry, type AssetManifest, type AssetManifestEntry, type AssetReplicationMode, AssetStream, AssetTap, AssetTarget, Authenticator, BATCH_INTERVAL_MS, type BaseConfig, type BigQueryConfig, BigQueryTarget, type BigqueryLocation, type Bookmark, type Bookmarks, CdnAssetTarget, type CdnAssetTargetConfig, type CdnFileMetadata, CdnService, type CdnServiceConfig, type CdnUploadResult, CloudStorageService, type CloudStorageServiceOptions, type ColumnMappingStoreUnsafeToSafe, CounterMetric, CsvExtractionConfigBuilder, type CsvFieldsArrayConfig, type CsvFieldsConfig, type CsvFieldsDictConfig, type CsvFileConfig, type CsvStreamConfig, DEFAULT_MAX_CONCURRENT_STREAMS, DEFAULT_SYNCED_AT_COLUMN, DocumentDownloadSkipError, type DocumentEntry, type DocumentManifest, type DocumentManifestEntry, type DocumentStatus, DocumentStream, type DocumentStreamManifest, type DocumentSummary, DocumentTap, type DownloadFilesOptions, EXECUTION_TIME_METRIC_NAME, type ErrorType, type ExcelCustomExtractorConfig, type ExcelDerivedField, type ExcelExtractionBaseConfig, type ExcelExtractionConfig, ExcelExtractionConfigBuilder, type ExcelFieldMapping, type ExcelFieldSpec, type ExcelSingleSheetExtractionConfig, type ExcelSourceColumn, type ExcelStreamConfig, type FieldType, FileFormat, FilePatterns, FileStream, type FileStreamConfig, type FileStreamEntry, FileTap, type FileTapConfig, type FlattenedSchema, GCSStateProvider, type GetConfig, type HTTPHeaders, type HTTPMethod, ITarget, ImageTransform, type InputTapState, type InternalTableField, type JSONSchemaFieldDefinition, type ListFilesOptions, MIME_TYPES, type Message, type MessageType, type MetricConfiguration, MissingFieldInSchemaError, MissingSchemaError, type PostConfig, type ProcessedAsset, type ProgressMarkers, type PropertiesMetadata, RESTStream, ROWS_SYNCED_METRIC_NAME, RecordMessage, RenameColumnStore, ReplicationMethod, ReplicationMethodMessage, type SafeColumnNameConverterFn, type Schema, SchemaMessage, SchemaValidationError, SftpService, type State, type StateHolder, StateMessage, type StateProvider, StateService, Stream, type StreamId, type StreamManifest, type StreamState, StreamWarehouseSyncService, type StreamWithTempFile, type SyncOptions, Tap, type TapState, type TargetSchemaHook, type TargetSchemaHookInput, type TempFile, type URLParams, VariableExtractors, type WarehouseTableField, type WebpTransformOptions, type WhalyDocument, WhalyDocumentService, type WhalyDocumentServiceConfig, WhalyDocumentTarget, type WhalyPaginatedResponse, type WhalyUploadResult, type WriteGeneratorToCsvOptions, _greaterThan, addDocumentSummaries, addWhalyFields, checkCsvHeaderRow, coerceCellValue, collectPaginated, countCsvLines, createCellReference, createCsvStreamConfig, createExcelGenerator, createExcelStreamConfig, createTemporaryFileStream, csvFieldsToJsonSchema, downloadFiles, emptyDocumentSummary, excelColumnToIndex, excelFieldsToJsonSchema, excelRowToIndex, extractExcelRows, extractPrimaryKeysFromCsvConfig, extractPrimaryKeysFromExcelFields, extractStateForStream, fieldTypeToJsonSchema, finalizeStateProgressMarkers, findAllMatchingConfigs, flattenSchema, getAllMetrics, getApiEndpoint, getAxiosInstance, getCdnId, getCounterMetric, getCounterMetrics, getDownloadFileApiCall, getDryRunLimit, getExtension, getJSONApiCall, getJSONApiCallWithFullResponse, getMimeType, getRawJSONApiCall, getServiceAccountKey, gracefulExit, haltAndCatchFire, incrementStreamState, indexToExcelColumn, indexToExcelRow, isDryRun, listFilesRecursively, loadJson, loadSchema, logger, optionalEnv, parseCellReference, postFormDataApiCall, postJSONApiCall, postUrlEncodedApiCall, printMemoryFootprint, processFileStreams, processFromAsyncIterable, removeParasiteProperties, requireEnv, rowGeneratorFromCsv, runWithConcurrency, safePath, unzip, validateExcelConfig, writeDataToCsv, writeGeneratorToCSV };
+export { ALL_STREAM_ID_KEYWORD, API_CALLS_METRIC_NAME, type AssetEntry, type AssetManifest, type AssetManifestEntry, type AssetReplicationMode, AssetStream, AssetTap, AssetTarget, Authenticator, BATCH_INTERVAL_MS, type BaseConfig, type BigQueryConfig, BigQueryTarget, type BigqueryLocation, type Bookmark, type Bookmarks, CdnAssetTarget, type CdnAssetTargetConfig, type CdnFileMetadata, CdnService, type CdnServiceConfig, type CdnUploadResult, CloudStorageService, type CloudStorageServiceOptions, type ColumnMappingStoreUnsafeToSafe, CounterMetric, CsvExtractionConfigBuilder, type CsvFieldsArrayConfig, type CsvFieldsConfig, type CsvFieldsDictConfig, type CsvFileConfig, type CsvStreamConfig, DEFAULT_MAX_CONCURRENT_STREAMS, DEFAULT_SYNCED_AT_COLUMN, DocumentDownloadSkipError, type DocumentEntry, type DocumentManifest, type DocumentManifestEntry, type DocumentStatus, DocumentStream, type DocumentStreamManifest, type DocumentSummary, DocumentTap, type DownloadFilesOptions, EXECUTION_TIME_METRIC_NAME, type ErrorType, type ExcelCustomExtractorConfig, type ExcelDerivedField, type ExcelExtractionBaseConfig, type ExcelExtractionConfig, ExcelExtractionConfigBuilder, type ExcelFieldMapping, type ExcelFieldSpec, type ExcelSingleSheetExtractionConfig, type ExcelSourceColumn, type ExcelStreamConfig, type FieldType, FileFormat, FilePatterns, FileStream, type FileStreamConfig, type FileStreamEntry, FileTap, type FileTapConfig, type FlattenedSchema, GCSStateProvider, type GetConfig, type HTTPHeaders, type HTTPMethod, ITarget, ImageTransform, type InputTapState, type InternalTableField, type JSONSchemaFieldDefinition, type ListFilesOptions, LocalStorageService, type LocalStorageServiceOptions, MIME_TYPES, type Message, type MessageType, type MetricConfiguration, MissingFieldInSchemaError, MissingSchemaError, type PostConfig, type ProcessedAsset, type ProgressMarkers, type PropertiesMetadata, RESTStream, ROWS_SYNCED_METRIC_NAME, RecordMessage, RenameColumnStore, ReplicationMethod, ReplicationMethodMessage, type SafeColumnNameConverterFn, type Schema, SchemaMessage, SchemaValidationError, SftpService, type State, type StateHolder, StateMessage, type StateProvider, StateService, type StorageService, Stream, type StreamId, type StreamManifest, type StreamState, StreamWarehouseSyncService, type StreamWithTempFile, type SyncOptions, Tap, type TapState, type TargetSchemaHook, type TargetSchemaHookInput, type TempFile, type URLParams, VariableExtractors, type WarehouseTableField, type WebpTransformOptions, type WhalyDocument, WhalyDocumentService, type WhalyDocumentServiceConfig, WhalyDocumentTarget, type WhalyPaginatedResponse, type WhalyUploadResult, type WriteGeneratorToCsvOptions, _greaterThan, addDocumentSummaries, addWhalyFields, checkCsvHeaderRow, coerceCellValue, collectPaginated, countCsvLines, createCellReference, createCsvStreamConfig, createExcelGenerator, createExcelStreamConfig, createTemporaryFileStream, csvFieldsToJsonSchema, downloadFiles, emptyDocumentSummary, excelColumnToIndex, excelFieldsToJsonSchema, excelRowToIndex, extractExcelRows, extractPrimaryKeysFromCsvConfig, extractPrimaryKeysFromExcelFields, extractStateForStream, fieldTypeToJsonSchema, finalizeStateProgressMarkers, findAllMatchingConfigs, flattenSchema, getAllMetrics, getApiEndpoint, getAxiosInstance, getCdnId, getCounterMetric, getCounterMetrics, getDownloadFileApiCall, getDryRunLimit, getExtension, getJSONApiCall, getJSONApiCallWithFullResponse, getMimeType, getRawJSONApiCall, getServiceAccountKey, gracefulExit, haltAndCatchFire, incrementStreamState, indexToExcelColumn, indexToExcelRow, isDryRun, listFilesRecursively, loadJson, loadSchema, logger, optionalEnv, parseCellReference, postFormDataApiCall, postJSONApiCall, postUrlEncodedApiCall, printMemoryFootprint, processFileStreams, processFromAsyncIterable, removeParasiteProperties, requireEnv, rowGeneratorFromCsv, runWithConcurrency, safePath, unzip, validateExcelConfig, writeDataToCsv, writeGeneratorToCSV };

package/dist/index.d.ts

@@ -1125,15 +1125,59 @@ interface DownloadFilesOptions {
  */
 declare function downloadFiles(client: Client, files: Client.FileInfo[], localDir: string, options?: DownloadFilesOptions): Promise<string[]>;
 
+/**
+ * Common interface for storage services (cloud or local).
+ * Implemented by CloudStorageService and LocalStorageService.
+ */
+interface StorageService {
+    /**
+     * Lists all files with an optional prefix/path filter.
+     */
+    listFiles(prefix?: string): Promise<string[]>;
+    /**
+     * Returns files that haven't been marked as processed.
+     */
+    getUnprocessedFiles(): Promise<string[]>;
+    /**
+     * Creates a marker file to indicate a file has been processed.
+     */
+    createMarkerFile(fileName: string): Promise<void>;
+    /**
+     * Downloads/resolves a file and returns a local file path.
+     */
+    downloadFile(filePath: string, fileName: string): Promise<string>;
+    /**
+     * Resolves a file URI to a local file path.
+     * Cloud: downloads from bucket. Local: resolves and validates the local path.
+     */
+    resolveFileUri(fileUri: string): Promise<string>;
+    /**
+     * Uploads a local file to the storage destination.
+     */
+    uploadFile(localPath: string, destPath: string): Promise<File>;
+    /**
+     * Reads a storage object and returns its contents as a UTF-8 string.
+     */
+    readObjectAsString(objectPath: string): Promise<string>;
+    /**
+     * Writes a string to a storage object with JSON content type.
+     */
+    writeStringObject(objectPath: string, contents: string): Promise<void>;
+    /**
+     * Uploads a local file with a unique generated name.
+     */
+    uploadFileWithUniqueName(filePath: string, prefix: string, streamId: string): Promise<File>;
+}
+
 interface CloudStorageServiceOptions {
     processedSuffix?: string;
     supportedExtensions?: string[];
 }
 /**
- * Service for interacting with Google Cloud Storage.
+ * Google Cloud Storage implementation of StorageService.
  * Supports file download, upload, listing, and marker-file-based processing tracking.
  */
-declare class CloudStorageService {
+declare class CloudStorageService implements StorageService {
     private storage;
     private bucket;
     private processedSuffix;
@@ -1170,6 +1214,14 @@ declare class CloudStorageService {
      * Writes a string to a GCS object with JSON content type.
      */
     writeStringObject(objectPath: string, contents: string): Promise<void>;
+    /**
+     * Resolves a file URI to a local file path by downloading it from the configured GCS bucket.
+     *
+     * @example
+     * const storage = new CloudStorageService("my-bucket", "my-folder");
+     * const localPath = await storage.resolveFileUri("folder/file.xlsx");
+     */
+    resolveFileUri(fileUri: string): Promise<string>;
     /**
      * Uploads a local file to the bucket with a unique name based on prefix, streamId, and UUID.
      * Files are stored under `<prefix>/<run-id>/` when the RUN_ID env var is set,
@@ -1179,6 +1231,30 @@ declare class CloudStorageService {
     uploadFileWithUniqueName(filePath: string, prefix: string, streamId: string): Promise<File>;
 }
 
+interface LocalStorageServiceOptions {
+    processedSuffix?: string;
+    supportedExtensions?: string[];
+}
+/**
+ * Local filesystem implementation of StorageService.
+ * Operates entirely on the local filesystem — no cloud interaction.
+ */
+declare class LocalStorageService implements StorageService {
+    private basePath;
+    private processedSuffix;
+    private supportedExtensions;
+    constructor(basePath: string, opts?: LocalStorageServiceOptions);
+    listFiles(prefix?: string): Promise<string[]>;
+    getUnprocessedFiles(): Promise<string[]>;
+    createMarkerFile(fileName: string): Promise<void>;
+    downloadFile(filePath: string, _fileName: string): Promise<string>;
+    resolveFileUri(fileUri: string): Promise<string>;
+    uploadFile(localPath: string, destPath: string): Promise<File>;
+    readObjectAsString(objectPath: string): Promise<string>;
+    writeStringObject(objectPath: string, contents: string): Promise<void>;
+    uploadFileWithUniqueName(filePath: string, prefix: string, streamId: string): Promise<File>;
+}
+
 /**
  * Extract a ZIP (or other archive) file to a target directory.
  * Uses the `decompress` library which supports zip, tar, gz, bz2, etc.
@@ -1644,4 +1720,4 @@ declare class GCSStateProvider implements StateProvider {
     writeState(state: string): Promise<void>;
 }
 
-export { ALL_STREAM_ID_KEYWORD, API_CALLS_METRIC_NAME, type AssetEntry, type AssetManifest, type AssetManifestEntry, type AssetReplicationMode, AssetStream, AssetTap, AssetTarget, Authenticator, BATCH_INTERVAL_MS, type BaseConfig, type BigQueryConfig, BigQueryTarget, type BigqueryLocation, type Bookmark, type Bookmarks, CdnAssetTarget, type CdnAssetTargetConfig, type CdnFileMetadata, CdnService, type CdnServiceConfig, type CdnUploadResult, CloudStorageService, type CloudStorageServiceOptions, type ColumnMappingStoreUnsafeToSafe, CounterMetric, CsvExtractionConfigBuilder, type CsvFieldsArrayConfig, type CsvFieldsConfig, type CsvFieldsDictConfig, type CsvFileConfig, type CsvStreamConfig, DEFAULT_MAX_CONCURRENT_STREAMS, DEFAULT_SYNCED_AT_COLUMN, DocumentDownloadSkipError, type DocumentEntry, type DocumentManifest, type DocumentManifestEntry, type DocumentStatus, DocumentStream, type DocumentStreamManifest, type DocumentSummary, DocumentTap, type DownloadFilesOptions, EXECUTION_TIME_METRIC_NAME, type ErrorType, type ExcelCustomExtractorConfig, type ExcelDerivedField, type ExcelExtractionBaseConfig, type ExcelExtractionConfig, ExcelExtractionConfigBuilder, type ExcelFieldMapping, type ExcelFieldSpec, type ExcelSingleSheetExtractionConfig, type ExcelSourceColumn, type ExcelStreamConfig, type FieldType, FileFormat, FilePatterns, FileStream, type FileStreamConfig, type FileStreamEntry, FileTap, type FileTapConfig, type FlattenedSchema, GCSStateProvider, type GetConfig, type HTTPHeaders, type HTTPMethod, ITarget, ImageTransform, type InputTapState, type InternalTableField, type JSONSchemaFieldDefinition, type ListFilesOptions, MIME_TYPES, type Message, type MessageType, type MetricConfiguration, MissingFieldInSchemaError, MissingSchemaError, type PostConfig, type ProcessedAsset, type ProgressMarkers, type PropertiesMetadata, RESTStream, ROWS_SYNCED_METRIC_NAME, RecordMessage, RenameColumnStore, ReplicationMethod, ReplicationMethodMessage, type SafeColumnNameConverterFn, type Schema, SchemaMessage, SchemaValidationError, SftpService, type State, type StateHolder, StateMessage, type StateProvider, StateService, Stream, type StreamId, type StreamManifest, type StreamState, StreamWarehouseSyncService, type StreamWithTempFile, type SyncOptions, Tap, type TapState, type TargetSchemaHook, type TargetSchemaHookInput, type TempFile, type URLParams, VariableExtractors, type WarehouseTableField, type WebpTransformOptions, type WhalyDocument, WhalyDocumentService, type WhalyDocumentServiceConfig, WhalyDocumentTarget, type WhalyPaginatedResponse, type WhalyUploadResult, type WriteGeneratorToCsvOptions, _greaterThan, addDocumentSummaries, addWhalyFields, checkCsvHeaderRow, coerceCellValue, collectPaginated, countCsvLines, createCellReference, createCsvStreamConfig, createExcelGenerator, createExcelStreamConfig, createTemporaryFileStream, csvFieldsToJsonSchema, downloadFiles, emptyDocumentSummary, excelColumnToIndex, excelFieldsToJsonSchema, excelRowToIndex, extractExcelRows, extractPrimaryKeysFromCsvConfig, extractPrimaryKeysFromExcelFields, extractStateForStream, fieldTypeToJsonSchema, finalizeStateProgressMarkers, findAllMatchingConfigs, flattenSchema, getAllMetrics, getApiEndpoint, getAxiosInstance, getCdnId, getCounterMetric, getCounterMetrics, getDownloadFileApiCall, getDryRunLimit, getExtension, getJSONApiCall, getJSONApiCallWithFullResponse, getMimeType, getRawJSONApiCall, getServiceAccountKey, gracefulExit, haltAndCatchFire, incrementStreamState, indexToExcelColumn, indexToExcelRow, isDryRun, listFilesRecursively, loadJson, loadSchema, logger, optionalEnv, parseCellReference, postFormDataApiCall, postJSONApiCall, postUrlEncodedApiCall, printMemoryFootprint, processFileStreams, processFromAsyncIterable, removeParasiteProperties, requireEnv, rowGeneratorFromCsv, runWithConcurrency, safePath, unzip, validateExcelConfig, writeDataToCsv, writeGeneratorToCSV };
+export { ALL_STREAM_ID_KEYWORD, API_CALLS_METRIC_NAME, type AssetEntry, type AssetManifest, type AssetManifestEntry, type AssetReplicationMode, AssetStream, AssetTap, AssetTarget, Authenticator, BATCH_INTERVAL_MS, type BaseConfig, type BigQueryConfig, BigQueryTarget, type BigqueryLocation, type Bookmark, type Bookmarks, CdnAssetTarget, type CdnAssetTargetConfig, type CdnFileMetadata, CdnService, type CdnServiceConfig, type CdnUploadResult, CloudStorageService, type CloudStorageServiceOptions, type ColumnMappingStoreUnsafeToSafe, CounterMetric, CsvExtractionConfigBuilder, type CsvFieldsArrayConfig, type CsvFieldsConfig, type CsvFieldsDictConfig, type CsvFileConfig, type CsvStreamConfig, DEFAULT_MAX_CONCURRENT_STREAMS, DEFAULT_SYNCED_AT_COLUMN, DocumentDownloadSkipError, type DocumentEntry, type DocumentManifest, type DocumentManifestEntry, type DocumentStatus, DocumentStream, type DocumentStreamManifest, type DocumentSummary, DocumentTap, type DownloadFilesOptions, EXECUTION_TIME_METRIC_NAME, type ErrorType, type ExcelCustomExtractorConfig, type ExcelDerivedField, type ExcelExtractionBaseConfig, type ExcelExtractionConfig, ExcelExtractionConfigBuilder, type ExcelFieldMapping, type ExcelFieldSpec, type ExcelSingleSheetExtractionConfig, type ExcelSourceColumn, type ExcelStreamConfig, type FieldType, FileFormat, FilePatterns, FileStream, type FileStreamConfig, type FileStreamEntry, FileTap, type FileTapConfig, type FlattenedSchema, GCSStateProvider, type GetConfig, type HTTPHeaders, type HTTPMethod, ITarget, ImageTransform, type InputTapState, type InternalTableField, type JSONSchemaFieldDefinition, type ListFilesOptions, LocalStorageService, type LocalStorageServiceOptions, MIME_TYPES, type Message, type MessageType, type MetricConfiguration, MissingFieldInSchemaError, MissingSchemaError, type PostConfig, type ProcessedAsset, type ProgressMarkers, type PropertiesMetadata, RESTStream, ROWS_SYNCED_METRIC_NAME, RecordMessage, RenameColumnStore, ReplicationMethod, ReplicationMethodMessage, type SafeColumnNameConverterFn, type Schema, SchemaMessage, SchemaValidationError, SftpService, type State, type StateHolder, StateMessage, type StateProvider, StateService, type StorageService, Stream, type StreamId, type StreamManifest, type StreamState, StreamWarehouseSyncService, type StreamWithTempFile, type SyncOptions, Tap, type TapState, type TargetSchemaHook, type TargetSchemaHookInput, type TempFile, type URLParams, VariableExtractors, type WarehouseTableField, type WebpTransformOptions, type WhalyDocument, WhalyDocumentService, type WhalyDocumentServiceConfig, WhalyDocumentTarget, type WhalyPaginatedResponse, type WhalyUploadResult, type WriteGeneratorToCsvOptions, _greaterThan, addDocumentSummaries, addWhalyFields, checkCsvHeaderRow, coerceCellValue, collectPaginated, countCsvLines, createCellReference, createCsvStreamConfig, createExcelGenerator, createExcelStreamConfig, createTemporaryFileStream, csvFieldsToJsonSchema, downloadFiles, emptyDocumentSummary, excelColumnToIndex, excelFieldsToJsonSchema, excelRowToIndex, extractExcelRows, extractPrimaryKeysFromCsvConfig, extractPrimaryKeysFromExcelFields, extractStateForStream, fieldTypeToJsonSchema, finalizeStateProgressMarkers, findAllMatchingConfigs, flattenSchema, getAllMetrics, getApiEndpoint, getAxiosInstance, getCdnId, getCounterMetric, getCounterMetrics, getDownloadFileApiCall, getDryRunLimit, getExtension, getJSONApiCall, getJSONApiCallWithFullResponse, getMimeType, getRawJSONApiCall, getServiceAccountKey, gracefulExit, haltAndCatchFire, incrementStreamState, indexToExcelColumn, indexToExcelRow, isDryRun, listFilesRecursively, loadJson, loadSchema, logger, optionalEnv, parseCellReference, postFormDataApiCall, postJSONApiCall, postUrlEncodedApiCall, printMemoryFootprint, processFileStreams, processFromAsyncIterable, removeParasiteProperties, requireEnv, rowGeneratorFromCsv, runWithConcurrency, safePath, unzip, validateExcelConfig, writeDataToCsv, writeGeneratorToCSV };