@whaly/connector-sdk 0.3.10 → 0.3.12

package/dist/index.d.mts

@@ -1125,15 +1125,59 @@ interface DownloadFilesOptions {
  */
 declare function downloadFiles(client: Client, files: Client.FileInfo[], localDir: string, options?: DownloadFilesOptions): Promise<string[]>;
 
+/**
+ * Common interface for storage services (cloud or local).
+ * Implemented by CloudStorageService and LocalStorageService.
+ */
+interface StorageService {
+    /**
+     * Lists all files with an optional prefix/path filter.
+     */
+    listFiles(prefix?: string): Promise<string[]>;
+    /**
+     * Returns files that haven't been marked as processed.
+     */
+    getUnprocessedFiles(): Promise<string[]>;
+    /**
+     * Creates a marker file to indicate a file has been processed.
+     */
+    createMarkerFile(fileName: string): Promise<void>;
+    /**
+     * Downloads/resolves a file and returns a local file path.
+     */
+    downloadFile(filePath: string, fileName: string): Promise<string>;
+    /**
+     * Resolves a file URI to a local file path.
+     * Cloud: downloads from bucket. Local: resolves and validates the local path.
+     */
+    resolveFileUri(fileUri: string): Promise<string>;
+    /**
+     * Uploads a local file to the storage destination.
+     */
+    uploadFile(localPath: string, destPath: string): Promise<File>;
+    /**
+     * Reads a storage object and returns its contents as a UTF-8 string.
+     */
+    readObjectAsString(objectPath: string): Promise<string>;
+    /**
+     * Writes a string to a storage object with JSON content type.
+     */
+    writeStringObject(objectPath: string, contents: string): Promise<void>;
+    /**
+     * Uploads a local file with a unique generated name.
+     */
+    uploadFileWithUniqueName(filePath: string, prefix: string, streamId: string): Promise<File>;
+}
+
 interface CloudStorageServiceOptions {
     processedSuffix?: string;
     supportedExtensions?: string[];
 }
 /**
- * Service for interacting with Google Cloud Storage.
+ * Google Cloud Storage implementation of StorageService.
  * Supports file download, upload, listing, and marker-file-based processing tracking.
  */
-declare class CloudStorageService {
+declare class CloudStorageService implements StorageService {
     private storage;
     private bucket;
     private processedSuffix;
@@ -1170,6 +1214,14 @@ declare class CloudStorageService {
      * Writes a string to a GCS object with JSON content type.
      */
     writeStringObject(objectPath: string, contents: string): Promise<void>;
+    /**
+     * Resolves a file URI to a local file path by downloading it from the configured GCS bucket.
+     *
+     * @example
+     * const storage = new CloudStorageService("my-bucket", "my-folder");
+     * const localPath = await storage.resolveFileUri("folder/file.xlsx");
+     */
+    resolveFileUri(fileUri: string): Promise<string>;
     /**
      * Uploads a local file to the bucket with a unique name based on prefix, streamId, and UUID.
      * Files are stored under `<prefix>/<run-id>/` when the RUN_ID env var is set,
@@ -1179,6 +1231,30 @@ declare class CloudStorageService {
     uploadFileWithUniqueName(filePath: string, prefix: string, streamId: string): Promise<File>;
 }
 
+interface LocalStorageServiceOptions {
+    processedSuffix?: string;
+    supportedExtensions?: string[];
+}
+/**
+ * Local filesystem implementation of StorageService.
+ * Operates entirely on the local filesystem — no cloud interaction.
+ */
+declare class LocalStorageService implements StorageService {
+    private basePath;
+    private processedSuffix;
+    private supportedExtensions;
+    constructor(basePath: string, opts?: LocalStorageServiceOptions);
+    listFiles(prefix?: string): Promise<string[]>;
+    getUnprocessedFiles(): Promise<string[]>;
+    createMarkerFile(fileName: string): Promise<void>;
+    downloadFile(filePath: string, _fileName: string): Promise<string>;
+    resolveFileUri(fileUri: string): Promise<string>;
+    uploadFile(localPath: string, destPath: string): Promise<File>;
+    readObjectAsString(objectPath: string): Promise<string>;
+    writeStringObject(objectPath: string, contents: string): Promise<void>;
+    uploadFileWithUniqueName(filePath: string, prefix: string, streamId: string): Promise<File>;
+}
+
 /**
  * Extract a ZIP (or other archive) file to a target directory.
  * Uses the `decompress` library which supports zip, tar, gz, bz2, etc.
@@ -1413,6 +1489,12 @@ interface WhalyDocument {
     size_kb: number;
     storage: string;
     metadata: Record<string, string> | null;
+    /**
+     * The document source this document belongs to.
+     * `null` means the org's default source. On create this field is optional —
+     * when omitted the document is attached to the default source.
+     */
+    document_source_id: string | null;
 }
 /** Paginated response from the Whaly Document API. */
 interface WhalyPaginatedResponse<T> {
@@ -1490,6 +1572,16 @@ interface WhalyDocumentServiceConfig {
     serviceAccountKey?: string;
     /** Object storage ID for file uploads. */
     objectStorageId: string;
+    /**
+     * The document source this connector manages. One connector maps to exactly
+     * one source. All operations are scoped to it:
+     *  - `listAllDocuments()` only returns documents belonging to this source,
+     *    so the reconciliation (create / update / **delete**) never touches
+     *    documents from other sources — important because the DocumentTap
+     *    deletes any listed document not present in the source.
+     *  - Created and updated documents are always attached to this source.
+     */
+    documentSourceId: string;
 }
 interface WhalyUploadResult {
     storage: string;
@@ -1499,17 +1591,22 @@ interface WhalyUploadResult {
 declare class WhalyDocumentService {
     private axiosClient;
     readonly objectStorageId: string;
+    readonly documentSourceId: string;
     private gcsBucket;
     constructor(config: WhalyDocumentServiceConfig);
-    /** Fetch all documents from the API, handling cursor-based pagination. */
+    /**
+     * Fetch all documents belonging to the configured document source,
+     * handling cursor-based pagination. The `document_source_id` server-side
+     * filter scopes the result to this connector's source only.
+     */
     listAllDocuments(): Promise<WhalyDocument[]>;
     /** Upload a file to object storage. Returns storage path and size. */
     uploadFile(destinationPath: string, localFilePath: string, fileName: string): Promise<WhalyUploadResult>;
     private uploadFileViaGCS;
     private uploadFileViaAPI;
-    /** Create a new document record. */
-    createDocument(payload: Omit<WhalyDocument, "id">): Promise<WhalyDocument>;
-    /** Update an existing document record. */
+    /** Create a new document record, attached to the configured document source. */
+    createDocument(payload: Omit<WhalyDocument, "id" | "document_source_id">): Promise<WhalyDocument>;
+    /** Update an existing document record, keeping it in the configured document source. */
     updateDocument(id: string, payload: Partial<WhalyDocument>): Promise<WhalyDocument>;
     /** Delete a document record. */
     deleteDocument(id: string): Promise<void>;
@@ -1644,4 +1741,4 @@ declare class GCSStateProvider implements StateProvider {
     writeState(state: string): Promise<void>;
 }
 
-export { ALL_STREAM_ID_KEYWORD, API_CALLS_METRIC_NAME, type AssetEntry, type AssetManifest, type AssetManifestEntry, type AssetReplicationMode, AssetStream, AssetTap, AssetTarget, Authenticator, BATCH_INTERVAL_MS, type BaseConfig, type BigQueryConfig, BigQueryTarget, type BigqueryLocation, type Bookmark, type Bookmarks, CdnAssetTarget, type CdnAssetTargetConfig, type CdnFileMetadata, CdnService, type CdnServiceConfig, type CdnUploadResult, CloudStorageService, type CloudStorageServiceOptions, type ColumnMappingStoreUnsafeToSafe, CounterMetric, CsvExtractionConfigBuilder, type CsvFieldsArrayConfig, type CsvFieldsConfig, type CsvFieldsDictConfig, type CsvFileConfig, type CsvStreamConfig, DEFAULT_MAX_CONCURRENT_STREAMS, DEFAULT_SYNCED_AT_COLUMN, DocumentDownloadSkipError, type DocumentEntry, type DocumentManifest, type DocumentManifestEntry, type DocumentStatus, DocumentStream, type DocumentStreamManifest, type DocumentSummary, DocumentTap, type DownloadFilesOptions, EXECUTION_TIME_METRIC_NAME, type ErrorType, type ExcelCustomExtractorConfig, type ExcelDerivedField, type ExcelExtractionBaseConfig, type ExcelExtractionConfig, ExcelExtractionConfigBuilder, type ExcelFieldMapping, type ExcelFieldSpec, type ExcelSingleSheetExtractionConfig, type ExcelSourceColumn, type ExcelStreamConfig, type FieldType, FileFormat, FilePatterns, FileStream, type FileStreamConfig, type FileStreamEntry, FileTap, type FileTapConfig, type FlattenedSchema, GCSStateProvider, type GetConfig, type HTTPHeaders, type HTTPMethod, ITarget, ImageTransform, type InputTapState, type InternalTableField, type JSONSchemaFieldDefinition, type ListFilesOptions, MIME_TYPES, type Message, type MessageType, type MetricConfiguration, MissingFieldInSchemaError, MissingSchemaError, type PostConfig, type ProcessedAsset, type ProgressMarkers, type PropertiesMetadata, RESTStream, ROWS_SYNCED_METRIC_NAME, RecordMessage, RenameColumnStore, ReplicationMethod, ReplicationMethodMessage, type SafeColumnNameConverterFn, type Schema, SchemaMessage, SchemaValidationError, SftpService, type State, type StateHolder, StateMessage, type StateProvider, StateService, Stream, type StreamId, type StreamManifest, type StreamState, StreamWarehouseSyncService, type StreamWithTempFile, type SyncOptions, Tap, type TapState, type TargetSchemaHook, type TargetSchemaHookInput, type TempFile, type URLParams, VariableExtractors, type WarehouseTableField, type WebpTransformOptions, type WhalyDocument, WhalyDocumentService, type WhalyDocumentServiceConfig, WhalyDocumentTarget, type WhalyPaginatedResponse, type WhalyUploadResult, type WriteGeneratorToCsvOptions, _greaterThan, addDocumentSummaries, addWhalyFields, checkCsvHeaderRow, coerceCellValue, collectPaginated, countCsvLines, createCellReference, createCsvStreamConfig, createExcelGenerator, createExcelStreamConfig, createTemporaryFileStream, csvFieldsToJsonSchema, downloadFiles, emptyDocumentSummary, excelColumnToIndex, excelFieldsToJsonSchema, excelRowToIndex, extractExcelRows, extractPrimaryKeysFromCsvConfig, extractPrimaryKeysFromExcelFields, extractStateForStream, fieldTypeToJsonSchema, finalizeStateProgressMarkers, findAllMatchingConfigs, flattenSchema, getAllMetrics, getApiEndpoint, getAxiosInstance, getCdnId, getCounterMetric, getCounterMetrics, getDownloadFileApiCall, getDryRunLimit, getExtension, getJSONApiCall, getJSONApiCallWithFullResponse, getMimeType, getRawJSONApiCall, getServiceAccountKey, gracefulExit, haltAndCatchFire, incrementStreamState, indexToExcelColumn, indexToExcelRow, isDryRun, listFilesRecursively, loadJson, loadSchema, logger, optionalEnv, parseCellReference, postFormDataApiCall, postJSONApiCall, postUrlEncodedApiCall, printMemoryFootprint, processFileStreams, processFromAsyncIterable, removeParasiteProperties, requireEnv, rowGeneratorFromCsv, runWithConcurrency, safePath, unzip, validateExcelConfig, writeDataToCsv, writeGeneratorToCSV };
+export { ALL_STREAM_ID_KEYWORD, API_CALLS_METRIC_NAME, type AssetEntry, type AssetManifest, type AssetManifestEntry, type AssetReplicationMode, AssetStream, AssetTap, AssetTarget, Authenticator, BATCH_INTERVAL_MS, type BaseConfig, type BigQueryConfig, BigQueryTarget, type BigqueryLocation, type Bookmark, type Bookmarks, CdnAssetTarget, type CdnAssetTargetConfig, type CdnFileMetadata, CdnService, type CdnServiceConfig, type CdnUploadResult, CloudStorageService, type CloudStorageServiceOptions, type ColumnMappingStoreUnsafeToSafe, CounterMetric, CsvExtractionConfigBuilder, type CsvFieldsArrayConfig, type CsvFieldsConfig, type CsvFieldsDictConfig, type CsvFileConfig, type CsvStreamConfig, DEFAULT_MAX_CONCURRENT_STREAMS, DEFAULT_SYNCED_AT_COLUMN, DocumentDownloadSkipError, type DocumentEntry, type DocumentManifest, type DocumentManifestEntry, type DocumentStatus, DocumentStream, type DocumentStreamManifest, type DocumentSummary, DocumentTap, type DownloadFilesOptions, EXECUTION_TIME_METRIC_NAME, type ErrorType, type ExcelCustomExtractorConfig, type ExcelDerivedField, type ExcelExtractionBaseConfig, type ExcelExtractionConfig, ExcelExtractionConfigBuilder, type ExcelFieldMapping, type ExcelFieldSpec, type ExcelSingleSheetExtractionConfig, type ExcelSourceColumn, type ExcelStreamConfig, type FieldType, FileFormat, FilePatterns, FileStream, type FileStreamConfig, type FileStreamEntry, FileTap, type FileTapConfig, type FlattenedSchema, GCSStateProvider, type GetConfig, type HTTPHeaders, type HTTPMethod, ITarget, ImageTransform, type InputTapState, type InternalTableField, type JSONSchemaFieldDefinition, type ListFilesOptions, LocalStorageService, type LocalStorageServiceOptions, MIME_TYPES, type Message, type MessageType, type MetricConfiguration, MissingFieldInSchemaError, MissingSchemaError, type PostConfig, type ProcessedAsset, type ProgressMarkers, type PropertiesMetadata, RESTStream, ROWS_SYNCED_METRIC_NAME, RecordMessage, RenameColumnStore, ReplicationMethod, ReplicationMethodMessage, type SafeColumnNameConverterFn, type Schema, SchemaMessage, SchemaValidationError, SftpService, type State, type StateHolder, StateMessage, type StateProvider, StateService, type StorageService, Stream, type StreamId, type StreamManifest, type StreamState, StreamWarehouseSyncService, type StreamWithTempFile, type SyncOptions, Tap, type TapState, type TargetSchemaHook, type TargetSchemaHookInput, type TempFile, type URLParams, VariableExtractors, type WarehouseTableField, type WebpTransformOptions, type WhalyDocument, WhalyDocumentService, type WhalyDocumentServiceConfig, WhalyDocumentTarget, type WhalyPaginatedResponse, type WhalyUploadResult, type WriteGeneratorToCsvOptions, _greaterThan, addDocumentSummaries, addWhalyFields, checkCsvHeaderRow, coerceCellValue, collectPaginated, countCsvLines, createCellReference, createCsvStreamConfig, createExcelGenerator, createExcelStreamConfig, createTemporaryFileStream, csvFieldsToJsonSchema, downloadFiles, emptyDocumentSummary, excelColumnToIndex, excelFieldsToJsonSchema, excelRowToIndex, extractExcelRows, extractPrimaryKeysFromCsvConfig, extractPrimaryKeysFromExcelFields, extractStateForStream, fieldTypeToJsonSchema, finalizeStateProgressMarkers, findAllMatchingConfigs, flattenSchema, getAllMetrics, getApiEndpoint, getAxiosInstance, getCdnId, getCounterMetric, getCounterMetrics, getDownloadFileApiCall, getDryRunLimit, getExtension, getJSONApiCall, getJSONApiCallWithFullResponse, getMimeType, getRawJSONApiCall, getServiceAccountKey, gracefulExit, haltAndCatchFire, incrementStreamState, indexToExcelColumn, indexToExcelRow, isDryRun, listFilesRecursively, loadJson, loadSchema, logger, optionalEnv, parseCellReference, postFormDataApiCall, postJSONApiCall, postUrlEncodedApiCall, printMemoryFootprint, processFileStreams, processFromAsyncIterable, removeParasiteProperties, requireEnv, rowGeneratorFromCsv, runWithConcurrency, safePath, unzip, validateExcelConfig, writeDataToCsv, writeGeneratorToCSV };

package/dist/index.d.ts

@@ -1125,15 +1125,59 @@ interface DownloadFilesOptions {
  */
 declare function downloadFiles(client: Client, files: Client.FileInfo[], localDir: string, options?: DownloadFilesOptions): Promise<string[]>;
 
+/**
+ * Common interface for storage services (cloud or local).
+ * Implemented by CloudStorageService and LocalStorageService.
+ */
+interface StorageService {
+    /**
+     * Lists all files with an optional prefix/path filter.
+     */
+    listFiles(prefix?: string): Promise<string[]>;
+    /**
+     * Returns files that haven't been marked as processed.
+     */
+    getUnprocessedFiles(): Promise<string[]>;
+    /**
+     * Creates a marker file to indicate a file has been processed.
+     */
+    createMarkerFile(fileName: string): Promise<void>;
+    /**
+     * Downloads/resolves a file and returns a local file path.
+     */
+    downloadFile(filePath: string, fileName: string): Promise<string>;
+    /**
+     * Resolves a file URI to a local file path.
+     * Cloud: downloads from bucket. Local: resolves and validates the local path.
+     */
+    resolveFileUri(fileUri: string): Promise<string>;
+    /**
+     * Uploads a local file to the storage destination.
+     */
+    uploadFile(localPath: string, destPath: string): Promise<File>;
+    /**
+     * Reads a storage object and returns its contents as a UTF-8 string.
+     */
+    readObjectAsString(objectPath: string): Promise<string>;
+    /**
+     * Writes a string to a storage object with JSON content type.
+     */
+    writeStringObject(objectPath: string, contents: string): Promise<void>;
+    /**
+     * Uploads a local file with a unique generated name.
+     */
+    uploadFileWithUniqueName(filePath: string, prefix: string, streamId: string): Promise<File>;
+}
+
 interface CloudStorageServiceOptions {
     processedSuffix?: string;
     supportedExtensions?: string[];
 }
 /**
- * Service for interacting with Google Cloud Storage.
+ * Google Cloud Storage implementation of StorageService.
  * Supports file download, upload, listing, and marker-file-based processing tracking.
  */
-declare class CloudStorageService {
+declare class CloudStorageService implements StorageService {
     private storage;
     private bucket;
     private processedSuffix;
@@ -1170,6 +1214,14 @@ declare class CloudStorageService {
      * Writes a string to a GCS object with JSON content type.
      */
     writeStringObject(objectPath: string, contents: string): Promise<void>;
+    /**
+     * Resolves a file URI to a local file path by downloading it from the configured GCS bucket.
+     *
+     * @example
+     * const storage = new CloudStorageService("my-bucket", "my-folder");
+     * const localPath = await storage.resolveFileUri("folder/file.xlsx");
+     */
+    resolveFileUri(fileUri: string): Promise<string>;
     /**
      * Uploads a local file to the bucket with a unique name based on prefix, streamId, and UUID.
      * Files are stored under `<prefix>/<run-id>/` when the RUN_ID env var is set,
@@ -1179,6 +1231,30 @@ declare class CloudStorageService {
     uploadFileWithUniqueName(filePath: string, prefix: string, streamId: string): Promise<File>;
 }
 
+interface LocalStorageServiceOptions {
+    processedSuffix?: string;
+    supportedExtensions?: string[];
+}
+/**
+ * Local filesystem implementation of StorageService.
+ * Operates entirely on the local filesystem — no cloud interaction.
+ */
+declare class LocalStorageService implements StorageService {
+    private basePath;
+    private processedSuffix;
+    private supportedExtensions;
+    constructor(basePath: string, opts?: LocalStorageServiceOptions);
+    listFiles(prefix?: string): Promise<string[]>;
+    getUnprocessedFiles(): Promise<string[]>;
+    createMarkerFile(fileName: string): Promise<void>;
+    downloadFile(filePath: string, _fileName: string): Promise<string>;
+    resolveFileUri(fileUri: string): Promise<string>;
+    uploadFile(localPath: string, destPath: string): Promise<File>;
+    readObjectAsString(objectPath: string): Promise<string>;
+    writeStringObject(objectPath: string, contents: string): Promise<void>;
+    uploadFileWithUniqueName(filePath: string, prefix: string, streamId: string): Promise<File>;
+}
+
 /**
  * Extract a ZIP (or other archive) file to a target directory.
  * Uses the `decompress` library which supports zip, tar, gz, bz2, etc.
@@ -1413,6 +1489,12 @@ interface WhalyDocument {
     size_kb: number;
     storage: string;
     metadata: Record<string, string> | null;
+    /**
+     * The document source this document belongs to.
+     * `null` means the org's default source. On create this field is optional —
+     * when omitted the document is attached to the default source.
+     */
+    document_source_id: string | null;
 }
 /** Paginated response from the Whaly Document API. */
 interface WhalyPaginatedResponse<T> {
@@ -1490,6 +1572,16 @@ interface WhalyDocumentServiceConfig {
     serviceAccountKey?: string;
     /** Object storage ID for file uploads. */
     objectStorageId: string;
+    /**
+     * The document source this connector manages. One connector maps to exactly
+     * one source. All operations are scoped to it:
+     *  - `listAllDocuments()` only returns documents belonging to this source,
+     *    so the reconciliation (create / update / **delete**) never touches
+     *    documents from other sources — important because the DocumentTap
+     *    deletes any listed document not present in the source.
+     *  - Created and updated documents are always attached to this source.
+     */
+    documentSourceId: string;
 }
 interface WhalyUploadResult {
     storage: string;
@@ -1499,17 +1591,22 @@ interface WhalyUploadResult {
 declare class WhalyDocumentService {
     private axiosClient;
     readonly objectStorageId: string;
+    readonly documentSourceId: string;
     private gcsBucket;
     constructor(config: WhalyDocumentServiceConfig);
-    /** Fetch all documents from the API, handling cursor-based pagination. */
+    /**
+     * Fetch all documents belonging to the configured document source,
+     * handling cursor-based pagination. The `document_source_id` server-side
+     * filter scopes the result to this connector's source only.
+     */
     listAllDocuments(): Promise<WhalyDocument[]>;
     /** Upload a file to object storage. Returns storage path and size. */
     uploadFile(destinationPath: string, localFilePath: string, fileName: string): Promise<WhalyUploadResult>;
     private uploadFileViaGCS;
     private uploadFileViaAPI;
-    /** Create a new document record. */
-    createDocument(payload: Omit<WhalyDocument, "id">): Promise<WhalyDocument>;
-    /** Update an existing document record. */
+    /** Create a new document record, attached to the configured document source. */
+    createDocument(payload: Omit<WhalyDocument, "id" | "document_source_id">): Promise<WhalyDocument>;
+    /** Update an existing document record, keeping it in the configured document source. */
     updateDocument(id: string, payload: Partial<WhalyDocument>): Promise<WhalyDocument>;
     /** Delete a document record. */
     deleteDocument(id: string): Promise<void>;
@@ -1644,4 +1741,4 @@ declare class GCSStateProvider implements StateProvider {
     writeState(state: string): Promise<void>;
 }
 
-export { ALL_STREAM_ID_KEYWORD, API_CALLS_METRIC_NAME, type AssetEntry, type AssetManifest, type AssetManifestEntry, type AssetReplicationMode, AssetStream, AssetTap, AssetTarget, Authenticator, BATCH_INTERVAL_MS, type BaseConfig, type BigQueryConfig, BigQueryTarget, type BigqueryLocation, type Bookmark, type Bookmarks, CdnAssetTarget, type CdnAssetTargetConfig, type CdnFileMetadata, CdnService, type CdnServiceConfig, type CdnUploadResult, CloudStorageService, type CloudStorageServiceOptions, type ColumnMappingStoreUnsafeToSafe, CounterMetric, CsvExtractionConfigBuilder, type CsvFieldsArrayConfig, type CsvFieldsConfig, type CsvFieldsDictConfig, type CsvFileConfig, type CsvStreamConfig, DEFAULT_MAX_CONCURRENT_STREAMS, DEFAULT_SYNCED_AT_COLUMN, DocumentDownloadSkipError, type DocumentEntry, type DocumentManifest, type DocumentManifestEntry, type DocumentStatus, DocumentStream, type DocumentStreamManifest, type DocumentSummary, DocumentTap, type DownloadFilesOptions, EXECUTION_TIME_METRIC_NAME, type ErrorType, type ExcelCustomExtractorConfig, type ExcelDerivedField, type ExcelExtractionBaseConfig, type ExcelExtractionConfig, ExcelExtractionConfigBuilder, type ExcelFieldMapping, type ExcelFieldSpec, type ExcelSingleSheetExtractionConfig, type ExcelSourceColumn, type ExcelStreamConfig, type FieldType, FileFormat, FilePatterns, FileStream, type FileStreamConfig, type FileStreamEntry, FileTap, type FileTapConfig, type FlattenedSchema, GCSStateProvider, type GetConfig, type HTTPHeaders, type HTTPMethod, ITarget, ImageTransform, type InputTapState, type InternalTableField, type JSONSchemaFieldDefinition, type ListFilesOptions, MIME_TYPES, type Message, type MessageType, type MetricConfiguration, MissingFieldInSchemaError, MissingSchemaError, type PostConfig, type ProcessedAsset, type ProgressMarkers, type PropertiesMetadata, RESTStream, ROWS_SYNCED_METRIC_NAME, RecordMessage, RenameColumnStore, ReplicationMethod, ReplicationMethodMessage, type SafeColumnNameConverterFn, type Schema, SchemaMessage, SchemaValidationError, SftpService, type State, type StateHolder, StateMessage, type StateProvider, StateService, Stream, type StreamId, type StreamManifest, type StreamState, StreamWarehouseSyncService, type StreamWithTempFile, type SyncOptions, Tap, type TapState, type TargetSchemaHook, type TargetSchemaHookInput, type TempFile, type URLParams, VariableExtractors, type WarehouseTableField, type WebpTransformOptions, type WhalyDocument, WhalyDocumentService, type WhalyDocumentServiceConfig, WhalyDocumentTarget, type WhalyPaginatedResponse, type WhalyUploadResult, type WriteGeneratorToCsvOptions, _greaterThan, addDocumentSummaries, addWhalyFields, checkCsvHeaderRow, coerceCellValue, collectPaginated, countCsvLines, createCellReference, createCsvStreamConfig, createExcelGenerator, createExcelStreamConfig, createTemporaryFileStream, csvFieldsToJsonSchema, downloadFiles, emptyDocumentSummary, excelColumnToIndex, excelFieldsToJsonSchema, excelRowToIndex, extractExcelRows, extractPrimaryKeysFromCsvConfig, extractPrimaryKeysFromExcelFields, extractStateForStream, fieldTypeToJsonSchema, finalizeStateProgressMarkers, findAllMatchingConfigs, flattenSchema, getAllMetrics, getApiEndpoint, getAxiosInstance, getCdnId, getCounterMetric, getCounterMetrics, getDownloadFileApiCall, getDryRunLimit, getExtension, getJSONApiCall, getJSONApiCallWithFullResponse, getMimeType, getRawJSONApiCall, getServiceAccountKey, gracefulExit, haltAndCatchFire, incrementStreamState, indexToExcelColumn, indexToExcelRow, isDryRun, listFilesRecursively, loadJson, loadSchema, logger, optionalEnv, parseCellReference, postFormDataApiCall, postJSONApiCall, postUrlEncodedApiCall, printMemoryFootprint, processFileStreams, processFromAsyncIterable, removeParasiteProperties, requireEnv, rowGeneratorFromCsv, runWithConcurrency, safePath, unzip, validateExcelConfig, writeDataToCsv, writeGeneratorToCSV };
+export { ALL_STREAM_ID_KEYWORD, API_CALLS_METRIC_NAME, type AssetEntry, type AssetManifest, type AssetManifestEntry, type AssetReplicationMode, AssetStream, AssetTap, AssetTarget, Authenticator, BATCH_INTERVAL_MS, type BaseConfig, type BigQueryConfig, BigQueryTarget, type BigqueryLocation, type Bookmark, type Bookmarks, CdnAssetTarget, type CdnAssetTargetConfig, type CdnFileMetadata, CdnService, type CdnServiceConfig, type CdnUploadResult, CloudStorageService, type CloudStorageServiceOptions, type ColumnMappingStoreUnsafeToSafe, CounterMetric, CsvExtractionConfigBuilder, type CsvFieldsArrayConfig, type CsvFieldsConfig, type CsvFieldsDictConfig, type CsvFileConfig, type CsvStreamConfig, DEFAULT_MAX_CONCURRENT_STREAMS, DEFAULT_SYNCED_AT_COLUMN, DocumentDownloadSkipError, type DocumentEntry, type DocumentManifest, type DocumentManifestEntry, type DocumentStatus, DocumentStream, type DocumentStreamManifest, type DocumentSummary, DocumentTap, type DownloadFilesOptions, EXECUTION_TIME_METRIC_NAME, type ErrorType, type ExcelCustomExtractorConfig, type ExcelDerivedField, type ExcelExtractionBaseConfig, type ExcelExtractionConfig, ExcelExtractionConfigBuilder, type ExcelFieldMapping, type ExcelFieldSpec, type ExcelSingleSheetExtractionConfig, type ExcelSourceColumn, type ExcelStreamConfig, type FieldType, FileFormat, FilePatterns, FileStream, type FileStreamConfig, type FileStreamEntry, FileTap, type FileTapConfig, type FlattenedSchema, GCSStateProvider, type GetConfig, type HTTPHeaders, type HTTPMethod, ITarget, ImageTransform, type InputTapState, type InternalTableField, type JSONSchemaFieldDefinition, type ListFilesOptions, LocalStorageService, type LocalStorageServiceOptions, MIME_TYPES, type Message, type MessageType, type MetricConfiguration, MissingFieldInSchemaError, MissingSchemaError, type PostConfig, type ProcessedAsset, type ProgressMarkers, type PropertiesMetadata, RESTStream, ROWS_SYNCED_METRIC_NAME, RecordMessage, RenameColumnStore, ReplicationMethod, ReplicationMethodMessage, type SafeColumnNameConverterFn, type Schema, SchemaMessage, SchemaValidationError, SftpService, type State, type StateHolder, StateMessage, type StateProvider, StateService, type StorageService, Stream, type StreamId, type StreamManifest, type StreamState, StreamWarehouseSyncService, type StreamWithTempFile, type SyncOptions, Tap, type TapState, type TargetSchemaHook, type TargetSchemaHookInput, type TempFile, type URLParams, VariableExtractors, type WarehouseTableField, type WebpTransformOptions, type WhalyDocument, WhalyDocumentService, type WhalyDocumentServiceConfig, WhalyDocumentTarget, type WhalyPaginatedResponse, type WhalyUploadResult, type WriteGeneratorToCsvOptions, _greaterThan, addDocumentSummaries, addWhalyFields, checkCsvHeaderRow, coerceCellValue, collectPaginated, countCsvLines, createCellReference, createCsvStreamConfig, createExcelGenerator, createExcelStreamConfig, createTemporaryFileStream, csvFieldsToJsonSchema, downloadFiles, emptyDocumentSummary, excelColumnToIndex, excelFieldsToJsonSchema, excelRowToIndex, extractExcelRows, extractPrimaryKeysFromCsvConfig, extractPrimaryKeysFromExcelFields, extractStateForStream, fieldTypeToJsonSchema, finalizeStateProgressMarkers, findAllMatchingConfigs, flattenSchema, getAllMetrics, getApiEndpoint, getAxiosInstance, getCdnId, getCounterMetric, getCounterMetrics, getDownloadFileApiCall, getDryRunLimit, getExtension, getJSONApiCall, getJSONApiCallWithFullResponse, getMimeType, getRawJSONApiCall, getServiceAccountKey, gracefulExit, haltAndCatchFire, incrementStreamState, indexToExcelColumn, indexToExcelRow, isDryRun, listFilesRecursively, loadJson, loadSchema, logger, optionalEnv, parseCellReference, postFormDataApiCall, postJSONApiCall, postUrlEncodedApiCall, printMemoryFootprint, processFileStreams, processFromAsyncIterable, removeParasiteProperties, requireEnv, rowGeneratorFromCsv, runWithConcurrency, safePath, unzip, validateExcelConfig, writeDataToCsv, writeGeneratorToCSV };