@whaly/connector-sdk 0.3.12 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +38 -1
- package/dist/index.d.ts +38 -1
- package/dist/index.js +84 -4
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +80 -4
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/index.d.mts
CHANGED
|
@@ -153,6 +153,26 @@ declare function getMimeType(filePathOrExt: string): string;
|
|
|
153
153
|
*/
|
|
154
154
|
declare function getExtension(mimeType: string): string;
|
|
155
155
|
|
|
156
|
+
/**
|
|
157
|
+
* What a Whaly FileUpload run hands its transformer: the files to process and, for runs started
|
|
158
|
+
* by an inbound email, who sent it and when.
|
|
159
|
+
*/
|
|
160
|
+
interface EmailMetadata {
|
|
161
|
+
from?: string | null;
|
|
162
|
+
subject?: string | null;
|
|
163
|
+
receivedAt?: string | null;
|
|
164
|
+
}
|
|
165
|
+
type RunTrigger = "MANUAL" | "EMAIL" | "REPLAY";
|
|
166
|
+
/**
|
|
167
|
+
* The run's input files, as object keys in the job's storage (pass each to
|
|
168
|
+
* `storage.resolveFileUri`). Email runs list every kept attachment in `WLY_FILE_URIS`; manual
|
|
169
|
+
* uploads only set `FILE_URI`, so this works for both.
|
|
170
|
+
*/
|
|
171
|
+
declare function getInputFiles(env?: NodeJS.ProcessEnv): string[];
|
|
172
|
+
/** Sender, subject and receive time of the email that started the run; undefined otherwise. */
|
|
173
|
+
declare function getEmailMetadata(env?: NodeJS.ProcessEnv): EmailMetadata | undefined;
|
|
174
|
+
declare function getRunTrigger(env?: NodeJS.ProcessEnv): RunTrigger;
|
|
175
|
+
|
|
156
176
|
declare enum ReplicationMethod {
|
|
157
177
|
INCREMENTAL = "INCREMENTAL",
|
|
158
178
|
FULL_TABLE = "FULL_TABLE",
|
|
@@ -1029,6 +1049,16 @@ declare function createCsvStreamConfig(streamId: string, csvConfig: CsvFileConfi
|
|
|
1029
1049
|
* @param target - The target (e.g. BigQueryTarget)
|
|
1030
1050
|
*/
|
|
1031
1051
|
declare function processFileStreams(entries: FileStreamEntry[], tapState: InputTapState, target: ITarget): Promise<void>;
|
|
1052
|
+
/**
|
|
1053
|
+
* A run can process several files (one email, several attachments): says which one failed,
|
|
1054
|
+
* so the run's error is not just "the run failed".
|
|
1055
|
+
*/
|
|
1056
|
+
declare class FileProcessingError extends Error {
|
|
1057
|
+
readonly filePath: string | string[];
|
|
1058
|
+
readonly streamId: string;
|
|
1059
|
+
readonly cause: unknown;
|
|
1060
|
+
constructor(filePath: string | string[], streamId: string, cause: unknown);
|
|
1061
|
+
}
|
|
1032
1062
|
|
|
1033
1063
|
interface FileTapConfig {
|
|
1034
1064
|
[key: string]: any;
|
|
@@ -1067,6 +1097,13 @@ declare class VariableExtractors {
|
|
|
1067
1097
|
static regex(pattern: RegExp): (filename: string) => {
|
|
1068
1098
|
[key: string]: string | undefined;
|
|
1069
1099
|
};
|
|
1100
|
+
/**
|
|
1101
|
+
* Extracts named capture groups from the subject of the email that started the run, for
|
|
1102
|
+
* scheduled reports whose date only appears in the subject. Empty for manual uploads.
|
|
1103
|
+
*/
|
|
1104
|
+
static emailSubject(pattern: RegExp): (filename: string) => {
|
|
1105
|
+
[key: string]: string | undefined;
|
|
1106
|
+
};
|
|
1070
1107
|
/**
|
|
1071
1108
|
* Combines multiple extraction functions.
|
|
1072
1109
|
*/
|
|
@@ -1741,4 +1778,4 @@ declare class GCSStateProvider implements StateProvider {
|
|
|
1741
1778
|
writeState(state: string): Promise<void>;
|
|
1742
1779
|
}
|
|
1743
1780
|
|
|
1744
|
-
export { ALL_STREAM_ID_KEYWORD, API_CALLS_METRIC_NAME, type AssetEntry, type AssetManifest, type AssetManifestEntry, type AssetReplicationMode, AssetStream, AssetTap, AssetTarget, Authenticator, BATCH_INTERVAL_MS, type BaseConfig, type BigQueryConfig, BigQueryTarget, type BigqueryLocation, type Bookmark, type Bookmarks, CdnAssetTarget, type CdnAssetTargetConfig, type CdnFileMetadata, CdnService, type CdnServiceConfig, type CdnUploadResult, CloudStorageService, type CloudStorageServiceOptions, type ColumnMappingStoreUnsafeToSafe, CounterMetric, CsvExtractionConfigBuilder, type CsvFieldsArrayConfig, type CsvFieldsConfig, type CsvFieldsDictConfig, type CsvFileConfig, type CsvStreamConfig, DEFAULT_MAX_CONCURRENT_STREAMS, DEFAULT_SYNCED_AT_COLUMN, DocumentDownloadSkipError, type DocumentEntry, type DocumentManifest, type DocumentManifestEntry, type DocumentStatus, DocumentStream, type DocumentStreamManifest, type DocumentSummary, DocumentTap, type DownloadFilesOptions, EXECUTION_TIME_METRIC_NAME, type ErrorType, type ExcelCustomExtractorConfig, type ExcelDerivedField, type ExcelExtractionBaseConfig, type ExcelExtractionConfig, ExcelExtractionConfigBuilder, type ExcelFieldMapping, type ExcelFieldSpec, type ExcelSingleSheetExtractionConfig, type ExcelSourceColumn, type ExcelStreamConfig, type FieldType, FileFormat, FilePatterns, FileStream, type FileStreamConfig, type FileStreamEntry, FileTap, type FileTapConfig, type FlattenedSchema, GCSStateProvider, type GetConfig, type HTTPHeaders, type HTTPMethod, ITarget, ImageTransform, type InputTapState, type InternalTableField, type JSONSchemaFieldDefinition, type ListFilesOptions, LocalStorageService, type LocalStorageServiceOptions, MIME_TYPES, type Message, type MessageType, type MetricConfiguration, MissingFieldInSchemaError, MissingSchemaError, type PostConfig, type ProcessedAsset, type ProgressMarkers, type PropertiesMetadata, RESTStream, ROWS_SYNCED_METRIC_NAME, RecordMessage, RenameColumnStore, ReplicationMethod, ReplicationMethodMessage, type SafeColumnNameConverterFn, type Schema, SchemaMessage, SchemaValidationError, SftpService, type State, type StateHolder, StateMessage, type StateProvider, StateService, type StorageService, Stream, type StreamId, type StreamManifest, type StreamState, StreamWarehouseSyncService, type StreamWithTempFile, type SyncOptions, Tap, type TapState, type TargetSchemaHook, type TargetSchemaHookInput, type TempFile, type URLParams, VariableExtractors, type WarehouseTableField, type WebpTransformOptions, type WhalyDocument, WhalyDocumentService, type WhalyDocumentServiceConfig, WhalyDocumentTarget, type WhalyPaginatedResponse, type WhalyUploadResult, type WriteGeneratorToCsvOptions, _greaterThan, addDocumentSummaries, addWhalyFields, checkCsvHeaderRow, coerceCellValue, collectPaginated, countCsvLines, createCellReference, createCsvStreamConfig, createExcelGenerator, createExcelStreamConfig, createTemporaryFileStream, csvFieldsToJsonSchema, downloadFiles, emptyDocumentSummary, excelColumnToIndex, excelFieldsToJsonSchema, excelRowToIndex, extractExcelRows, extractPrimaryKeysFromCsvConfig, extractPrimaryKeysFromExcelFields, extractStateForStream, fieldTypeToJsonSchema, finalizeStateProgressMarkers, findAllMatchingConfigs, flattenSchema, getAllMetrics, getApiEndpoint, getAxiosInstance, getCdnId, getCounterMetric, getCounterMetrics, getDownloadFileApiCall, getDryRunLimit, getExtension, getJSONApiCall, getJSONApiCallWithFullResponse, getMimeType, getRawJSONApiCall, getServiceAccountKey, gracefulExit, haltAndCatchFire, incrementStreamState, indexToExcelColumn, indexToExcelRow, isDryRun, listFilesRecursively, loadJson, loadSchema, logger, optionalEnv, parseCellReference, postFormDataApiCall, postJSONApiCall, postUrlEncodedApiCall, printMemoryFootprint, processFileStreams, processFromAsyncIterable, removeParasiteProperties, requireEnv, rowGeneratorFromCsv, runWithConcurrency, safePath, unzip, validateExcelConfig, writeDataToCsv, writeGeneratorToCSV };
|
|
1781
|
+
export { ALL_STREAM_ID_KEYWORD, API_CALLS_METRIC_NAME, type AssetEntry, type AssetManifest, type AssetManifestEntry, type AssetReplicationMode, AssetStream, AssetTap, AssetTarget, Authenticator, BATCH_INTERVAL_MS, type BaseConfig, type BigQueryConfig, BigQueryTarget, type BigqueryLocation, type Bookmark, type Bookmarks, CdnAssetTarget, type CdnAssetTargetConfig, type CdnFileMetadata, CdnService, type CdnServiceConfig, type CdnUploadResult, CloudStorageService, type CloudStorageServiceOptions, type ColumnMappingStoreUnsafeToSafe, CounterMetric, CsvExtractionConfigBuilder, type CsvFieldsArrayConfig, type CsvFieldsConfig, type CsvFieldsDictConfig, type CsvFileConfig, type CsvStreamConfig, DEFAULT_MAX_CONCURRENT_STREAMS, DEFAULT_SYNCED_AT_COLUMN, DocumentDownloadSkipError, type DocumentEntry, type DocumentManifest, type DocumentManifestEntry, type DocumentStatus, DocumentStream, type DocumentStreamManifest, type DocumentSummary, DocumentTap, type DownloadFilesOptions, EXECUTION_TIME_METRIC_NAME, type EmailMetadata, type ErrorType, type ExcelCustomExtractorConfig, type ExcelDerivedField, type ExcelExtractionBaseConfig, type ExcelExtractionConfig, ExcelExtractionConfigBuilder, type ExcelFieldMapping, type ExcelFieldSpec, type ExcelSingleSheetExtractionConfig, type ExcelSourceColumn, type ExcelStreamConfig, type FieldType, FileFormat, FilePatterns, FileProcessingError, FileStream, type FileStreamConfig, type FileStreamEntry, FileTap, type FileTapConfig, type FlattenedSchema, GCSStateProvider, type GetConfig, type HTTPHeaders, type HTTPMethod, ITarget, ImageTransform, type InputTapState, type InternalTableField, type JSONSchemaFieldDefinition, type ListFilesOptions, LocalStorageService, type LocalStorageServiceOptions, MIME_TYPES, type Message, type MessageType, type MetricConfiguration, MissingFieldInSchemaError, MissingSchemaError, type PostConfig, type ProcessedAsset, type ProgressMarkers, type PropertiesMetadata, RESTStream, ROWS_SYNCED_METRIC_NAME, RecordMessage, RenameColumnStore, ReplicationMethod, ReplicationMethodMessage, type RunTrigger, type SafeColumnNameConverterFn, type Schema, SchemaMessage, SchemaValidationError, SftpService, type State, type StateHolder, StateMessage, type StateProvider, StateService, type StorageService, Stream, type StreamId, type StreamManifest, type StreamState, StreamWarehouseSyncService, type StreamWithTempFile, type SyncOptions, Tap, type TapState, type TargetSchemaHook, type TargetSchemaHookInput, type TempFile, type URLParams, VariableExtractors, type WarehouseTableField, type WebpTransformOptions, type WhalyDocument, WhalyDocumentService, type WhalyDocumentServiceConfig, WhalyDocumentTarget, type WhalyPaginatedResponse, type WhalyUploadResult, type WriteGeneratorToCsvOptions, _greaterThan, addDocumentSummaries, addWhalyFields, checkCsvHeaderRow, coerceCellValue, collectPaginated, countCsvLines, createCellReference, createCsvStreamConfig, createExcelGenerator, createExcelStreamConfig, createTemporaryFileStream, csvFieldsToJsonSchema, downloadFiles, emptyDocumentSummary, excelColumnToIndex, excelFieldsToJsonSchema, excelRowToIndex, extractExcelRows, extractPrimaryKeysFromCsvConfig, extractPrimaryKeysFromExcelFields, extractStateForStream, fieldTypeToJsonSchema, finalizeStateProgressMarkers, findAllMatchingConfigs, flattenSchema, getAllMetrics, getApiEndpoint, getAxiosInstance, getCdnId, getCounterMetric, getCounterMetrics, getDownloadFileApiCall, getDryRunLimit, getEmailMetadata, getExtension, getInputFiles, getJSONApiCall, getJSONApiCallWithFullResponse, getMimeType, getRawJSONApiCall, getRunTrigger, getServiceAccountKey, gracefulExit, haltAndCatchFire, incrementStreamState, indexToExcelColumn, indexToExcelRow, isDryRun, listFilesRecursively, loadJson, loadSchema, logger, optionalEnv, parseCellReference, postFormDataApiCall, postJSONApiCall, postUrlEncodedApiCall, printMemoryFootprint, processFileStreams, processFromAsyncIterable, removeParasiteProperties, requireEnv, rowGeneratorFromCsv, runWithConcurrency, safePath, unzip, validateExcelConfig, writeDataToCsv, writeGeneratorToCSV };
|
package/dist/index.d.ts
CHANGED
|
@@ -153,6 +153,26 @@ declare function getMimeType(filePathOrExt: string): string;
|
|
|
153
153
|
*/
|
|
154
154
|
declare function getExtension(mimeType: string): string;
|
|
155
155
|
|
|
156
|
+
/**
|
|
157
|
+
* What a Whaly FileUpload run hands its transformer: the files to process and, for runs started
|
|
158
|
+
* by an inbound email, who sent it and when.
|
|
159
|
+
*/
|
|
160
|
+
interface EmailMetadata {
|
|
161
|
+
from?: string | null;
|
|
162
|
+
subject?: string | null;
|
|
163
|
+
receivedAt?: string | null;
|
|
164
|
+
}
|
|
165
|
+
type RunTrigger = "MANUAL" | "EMAIL" | "REPLAY";
|
|
166
|
+
/**
|
|
167
|
+
* The run's input files, as object keys in the job's storage (pass each to
|
|
168
|
+
* `storage.resolveFileUri`). Email runs list every kept attachment in `WLY_FILE_URIS`; manual
|
|
169
|
+
* uploads only set `FILE_URI`, so this works for both.
|
|
170
|
+
*/
|
|
171
|
+
declare function getInputFiles(env?: NodeJS.ProcessEnv): string[];
|
|
172
|
+
/** Sender, subject and receive time of the email that started the run; undefined otherwise. */
|
|
173
|
+
declare function getEmailMetadata(env?: NodeJS.ProcessEnv): EmailMetadata | undefined;
|
|
174
|
+
declare function getRunTrigger(env?: NodeJS.ProcessEnv): RunTrigger;
|
|
175
|
+
|
|
156
176
|
declare enum ReplicationMethod {
|
|
157
177
|
INCREMENTAL = "INCREMENTAL",
|
|
158
178
|
FULL_TABLE = "FULL_TABLE",
|
|
@@ -1029,6 +1049,16 @@ declare function createCsvStreamConfig(streamId: string, csvConfig: CsvFileConfi
|
|
|
1029
1049
|
* @param target - The target (e.g. BigQueryTarget)
|
|
1030
1050
|
*/
|
|
1031
1051
|
declare function processFileStreams(entries: FileStreamEntry[], tapState: InputTapState, target: ITarget): Promise<void>;
|
|
1052
|
+
/**
|
|
1053
|
+
* A run can process several files (one email, several attachments): says which one failed,
|
|
1054
|
+
* so the run's error is not just "the run failed".
|
|
1055
|
+
*/
|
|
1056
|
+
declare class FileProcessingError extends Error {
|
|
1057
|
+
readonly filePath: string | string[];
|
|
1058
|
+
readonly streamId: string;
|
|
1059
|
+
readonly cause: unknown;
|
|
1060
|
+
constructor(filePath: string | string[], streamId: string, cause: unknown);
|
|
1061
|
+
}
|
|
1032
1062
|
|
|
1033
1063
|
interface FileTapConfig {
|
|
1034
1064
|
[key: string]: any;
|
|
@@ -1067,6 +1097,13 @@ declare class VariableExtractors {
|
|
|
1067
1097
|
static regex(pattern: RegExp): (filename: string) => {
|
|
1068
1098
|
[key: string]: string | undefined;
|
|
1069
1099
|
};
|
|
1100
|
+
/**
|
|
1101
|
+
* Extracts named capture groups from the subject of the email that started the run, for
|
|
1102
|
+
* scheduled reports whose date only appears in the subject. Empty for manual uploads.
|
|
1103
|
+
*/
|
|
1104
|
+
static emailSubject(pattern: RegExp): (filename: string) => {
|
|
1105
|
+
[key: string]: string | undefined;
|
|
1106
|
+
};
|
|
1070
1107
|
/**
|
|
1071
1108
|
* Combines multiple extraction functions.
|
|
1072
1109
|
*/
|
|
@@ -1741,4 +1778,4 @@ declare class GCSStateProvider implements StateProvider {
|
|
|
1741
1778
|
writeState(state: string): Promise<void>;
|
|
1742
1779
|
}
|
|
1743
1780
|
|
|
1744
|
-
export { ALL_STREAM_ID_KEYWORD, API_CALLS_METRIC_NAME, type AssetEntry, type AssetManifest, type AssetManifestEntry, type AssetReplicationMode, AssetStream, AssetTap, AssetTarget, Authenticator, BATCH_INTERVAL_MS, type BaseConfig, type BigQueryConfig, BigQueryTarget, type BigqueryLocation, type Bookmark, type Bookmarks, CdnAssetTarget, type CdnAssetTargetConfig, type CdnFileMetadata, CdnService, type CdnServiceConfig, type CdnUploadResult, CloudStorageService, type CloudStorageServiceOptions, type ColumnMappingStoreUnsafeToSafe, CounterMetric, CsvExtractionConfigBuilder, type CsvFieldsArrayConfig, type CsvFieldsConfig, type CsvFieldsDictConfig, type CsvFileConfig, type CsvStreamConfig, DEFAULT_MAX_CONCURRENT_STREAMS, DEFAULT_SYNCED_AT_COLUMN, DocumentDownloadSkipError, type DocumentEntry, type DocumentManifest, type DocumentManifestEntry, type DocumentStatus, DocumentStream, type DocumentStreamManifest, type DocumentSummary, DocumentTap, type DownloadFilesOptions, EXECUTION_TIME_METRIC_NAME, type ErrorType, type ExcelCustomExtractorConfig, type ExcelDerivedField, type ExcelExtractionBaseConfig, type ExcelExtractionConfig, ExcelExtractionConfigBuilder, type ExcelFieldMapping, type ExcelFieldSpec, type ExcelSingleSheetExtractionConfig, type ExcelSourceColumn, type ExcelStreamConfig, type FieldType, FileFormat, FilePatterns, FileStream, type FileStreamConfig, type FileStreamEntry, FileTap, type FileTapConfig, type FlattenedSchema, GCSStateProvider, type GetConfig, type HTTPHeaders, type HTTPMethod, ITarget, ImageTransform, type InputTapState, type InternalTableField, type JSONSchemaFieldDefinition, type ListFilesOptions, LocalStorageService, type LocalStorageServiceOptions, MIME_TYPES, type Message, type MessageType, type MetricConfiguration, MissingFieldInSchemaError, MissingSchemaError, type PostConfig, type ProcessedAsset, type ProgressMarkers, type PropertiesMetadata, RESTStream, ROWS_SYNCED_METRIC_NAME, RecordMessage, RenameColumnStore, ReplicationMethod, ReplicationMethodMessage, type SafeColumnNameConverterFn, type Schema, SchemaMessage, SchemaValidationError, SftpService, type State, type StateHolder, StateMessage, type StateProvider, StateService, type StorageService, Stream, type StreamId, type StreamManifest, type StreamState, StreamWarehouseSyncService, type StreamWithTempFile, type SyncOptions, Tap, type TapState, type TargetSchemaHook, type TargetSchemaHookInput, type TempFile, type URLParams, VariableExtractors, type WarehouseTableField, type WebpTransformOptions, type WhalyDocument, WhalyDocumentService, type WhalyDocumentServiceConfig, WhalyDocumentTarget, type WhalyPaginatedResponse, type WhalyUploadResult, type WriteGeneratorToCsvOptions, _greaterThan, addDocumentSummaries, addWhalyFields, checkCsvHeaderRow, coerceCellValue, collectPaginated, countCsvLines, createCellReference, createCsvStreamConfig, createExcelGenerator, createExcelStreamConfig, createTemporaryFileStream, csvFieldsToJsonSchema, downloadFiles, emptyDocumentSummary, excelColumnToIndex, excelFieldsToJsonSchema, excelRowToIndex, extractExcelRows, extractPrimaryKeysFromCsvConfig, extractPrimaryKeysFromExcelFields, extractStateForStream, fieldTypeToJsonSchema, finalizeStateProgressMarkers, findAllMatchingConfigs, flattenSchema, getAllMetrics, getApiEndpoint, getAxiosInstance, getCdnId, getCounterMetric, getCounterMetrics, getDownloadFileApiCall, getDryRunLimit, getExtension, getJSONApiCall, getJSONApiCallWithFullResponse, getMimeType, getRawJSONApiCall, getServiceAccountKey, gracefulExit, haltAndCatchFire, incrementStreamState, indexToExcelColumn, indexToExcelRow, isDryRun, listFilesRecursively, loadJson, loadSchema, logger, optionalEnv, parseCellReference, postFormDataApiCall, postJSONApiCall, postUrlEncodedApiCall, printMemoryFootprint, processFileStreams, processFromAsyncIterable, removeParasiteProperties, requireEnv, rowGeneratorFromCsv, runWithConcurrency, safePath, unzip, validateExcelConfig, writeDataToCsv, writeGeneratorToCSV };
|
|
1781
|
+
export { ALL_STREAM_ID_KEYWORD, API_CALLS_METRIC_NAME, type AssetEntry, type AssetManifest, type AssetManifestEntry, type AssetReplicationMode, AssetStream, AssetTap, AssetTarget, Authenticator, BATCH_INTERVAL_MS, type BaseConfig, type BigQueryConfig, BigQueryTarget, type BigqueryLocation, type Bookmark, type Bookmarks, CdnAssetTarget, type CdnAssetTargetConfig, type CdnFileMetadata, CdnService, type CdnServiceConfig, type CdnUploadResult, CloudStorageService, type CloudStorageServiceOptions, type ColumnMappingStoreUnsafeToSafe, CounterMetric, CsvExtractionConfigBuilder, type CsvFieldsArrayConfig, type CsvFieldsConfig, type CsvFieldsDictConfig, type CsvFileConfig, type CsvStreamConfig, DEFAULT_MAX_CONCURRENT_STREAMS, DEFAULT_SYNCED_AT_COLUMN, DocumentDownloadSkipError, type DocumentEntry, type DocumentManifest, type DocumentManifestEntry, type DocumentStatus, DocumentStream, type DocumentStreamManifest, type DocumentSummary, DocumentTap, type DownloadFilesOptions, EXECUTION_TIME_METRIC_NAME, type EmailMetadata, type ErrorType, type ExcelCustomExtractorConfig, type ExcelDerivedField, type ExcelExtractionBaseConfig, type ExcelExtractionConfig, ExcelExtractionConfigBuilder, type ExcelFieldMapping, type ExcelFieldSpec, type ExcelSingleSheetExtractionConfig, type ExcelSourceColumn, type ExcelStreamConfig, type FieldType, FileFormat, FilePatterns, FileProcessingError, FileStream, type FileStreamConfig, type FileStreamEntry, FileTap, type FileTapConfig, type FlattenedSchema, GCSStateProvider, type GetConfig, type HTTPHeaders, type HTTPMethod, ITarget, ImageTransform, type InputTapState, type InternalTableField, type JSONSchemaFieldDefinition, type ListFilesOptions, LocalStorageService, type LocalStorageServiceOptions, MIME_TYPES, type Message, type MessageType, type MetricConfiguration, MissingFieldInSchemaError, MissingSchemaError, type PostConfig, type ProcessedAsset, type ProgressMarkers, type PropertiesMetadata, RESTStream, ROWS_SYNCED_METRIC_NAME, RecordMessage, RenameColumnStore, ReplicationMethod, ReplicationMethodMessage, type RunTrigger, type SafeColumnNameConverterFn, type Schema, SchemaMessage, SchemaValidationError, SftpService, type State, type StateHolder, StateMessage, type StateProvider, StateService, type StorageService, Stream, type StreamId, type StreamManifest, type StreamState, StreamWarehouseSyncService, type StreamWithTempFile, type SyncOptions, Tap, type TapState, type TargetSchemaHook, type TargetSchemaHookInput, type TempFile, type URLParams, VariableExtractors, type WarehouseTableField, type WebpTransformOptions, type WhalyDocument, WhalyDocumentService, type WhalyDocumentServiceConfig, WhalyDocumentTarget, type WhalyPaginatedResponse, type WhalyUploadResult, type WriteGeneratorToCsvOptions, _greaterThan, addDocumentSummaries, addWhalyFields, checkCsvHeaderRow, coerceCellValue, collectPaginated, countCsvLines, createCellReference, createCsvStreamConfig, createExcelGenerator, createExcelStreamConfig, createTemporaryFileStream, csvFieldsToJsonSchema, downloadFiles, emptyDocumentSummary, excelColumnToIndex, excelFieldsToJsonSchema, excelRowToIndex, extractExcelRows, extractPrimaryKeysFromCsvConfig, extractPrimaryKeysFromExcelFields, extractStateForStream, fieldTypeToJsonSchema, finalizeStateProgressMarkers, findAllMatchingConfigs, flattenSchema, getAllMetrics, getApiEndpoint, getAxiosInstance, getCdnId, getCounterMetric, getCounterMetrics, getDownloadFileApiCall, getDryRunLimit, getEmailMetadata, getExtension, getInputFiles, getJSONApiCall, getJSONApiCallWithFullResponse, getMimeType, getRawJSONApiCall, getRunTrigger, getServiceAccountKey, gracefulExit, haltAndCatchFire, incrementStreamState, indexToExcelColumn, indexToExcelRow, isDryRun, listFilesRecursively, loadJson, loadSchema, logger, optionalEnv, parseCellReference, postFormDataApiCall, postJSONApiCall, postUrlEncodedApiCall, printMemoryFootprint, processFileStreams, processFromAsyncIterable, removeParasiteProperties, requireEnv, rowGeneratorFromCsv, runWithConcurrency, safePath, unzip, validateExcelConfig, writeDataToCsv, writeGeneratorToCSV };
|
package/dist/index.js
CHANGED
|
@@ -10337,6 +10337,7 @@ __export(index_exports, {
|
|
|
10337
10337
|
ExcelExtractionConfigBuilder: () => ExcelExtractionConfigBuilder,
|
|
10338
10338
|
FileFormat: () => FileFormat,
|
|
10339
10339
|
FilePatterns: () => FilePatterns,
|
|
10340
|
+
FileProcessingError: () => FileProcessingError,
|
|
10340
10341
|
FileStream: () => FileStream,
|
|
10341
10342
|
FileTap: () => FileTap,
|
|
10342
10343
|
GCSStateProvider: () => GCSStateProvider,
|
|
@@ -10398,11 +10399,14 @@ __export(index_exports, {
|
|
|
10398
10399
|
getCounterMetrics: () => getCounterMetrics,
|
|
10399
10400
|
getDownloadFileApiCall: () => getDownloadFileApiCall,
|
|
10400
10401
|
getDryRunLimit: () => getDryRunLimit,
|
|
10402
|
+
getEmailMetadata: () => getEmailMetadata,
|
|
10401
10403
|
getExtension: () => getExtension,
|
|
10404
|
+
getInputFiles: () => getInputFiles,
|
|
10402
10405
|
getJSONApiCall: () => getJSONApiCall,
|
|
10403
10406
|
getJSONApiCallWithFullResponse: () => getJSONApiCallWithFullResponse,
|
|
10404
10407
|
getMimeType: () => getMimeType,
|
|
10405
10408
|
getRawJSONApiCall: () => getRawJSONApiCall,
|
|
10409
|
+
getRunTrigger: () => getRunTrigger,
|
|
10406
10410
|
getServiceAccountKey: () => getServiceAccountKey,
|
|
10407
10411
|
gracefulExit: () => gracefulExit,
|
|
10408
10412
|
haltAndCatchFire: () => haltAndCatchFire,
|
|
@@ -10986,6 +10990,32 @@ function getExtension(mimeType) {
|
|
|
10986
10990
|
return EXTENSION_BY_MIME[mimeType] ?? ".bin";
|
|
10987
10991
|
}
|
|
10988
10992
|
|
|
10993
|
+
// src/sdk/service/runInput.ts
|
|
10994
|
+
function getInputFiles(env = process.env) {
|
|
10995
|
+
const list = env.WLY_FILE_URIS;
|
|
10996
|
+
if (list) {
|
|
10997
|
+
const parsed = JSON.parse(list);
|
|
10998
|
+
if (!Array.isArray(parsed) || !parsed.every((item) => typeof item === "string")) {
|
|
10999
|
+
throw new Error("WLY_FILE_URIS must be a JSON array of strings");
|
|
11000
|
+
}
|
|
11001
|
+
if (parsed.length > 0) {
|
|
11002
|
+
return parsed;
|
|
11003
|
+
}
|
|
11004
|
+
}
|
|
11005
|
+
return env.FILE_URI ? [env.FILE_URI] : [];
|
|
11006
|
+
}
|
|
11007
|
+
function getEmailMetadata(env = process.env) {
|
|
11008
|
+
const raw = env.WLY_EMAIL_METADATA;
|
|
11009
|
+
if (!raw) {
|
|
11010
|
+
return void 0;
|
|
11011
|
+
}
|
|
11012
|
+
return JSON.parse(raw);
|
|
11013
|
+
}
|
|
11014
|
+
function getRunTrigger(env = process.env) {
|
|
11015
|
+
const trigger = env.WLY_RUN_TRIGGER;
|
|
11016
|
+
return trigger === "EMAIL" || trigger === "REPLAY" ? trigger : "MANUAL";
|
|
11017
|
+
}
|
|
11018
|
+
|
|
10989
11019
|
// src/sdk/models/replication.ts
|
|
10990
11020
|
var ReplicationMethod = /* @__PURE__ */ ((ReplicationMethod2) => {
|
|
10991
11021
|
ReplicationMethod2["INCREMENTAL"] = "INCREMENTAL";
|
|
@@ -11317,6 +11347,20 @@ var Tap = class {
|
|
|
11317
11347
|
|
|
11318
11348
|
// src/sdk/models/tap/stream.ts
|
|
11319
11349
|
var import_util2 = require("util");
|
|
11350
|
+
var DEFAULT_PROGRESS_LOG_PERCENT_STEP = 10;
|
|
11351
|
+
var DEFAULT_PROGRESS_LOG_FALLBACK_INTERVAL = 1e5;
|
|
11352
|
+
function getProgressLogPercentStep() {
|
|
11353
|
+
const raw = optionalEnv("PROGRESS_LOG_PERCENT_STEP", String(DEFAULT_PROGRESS_LOG_PERCENT_STEP));
|
|
11354
|
+
const n = parseInt(raw, 10);
|
|
11355
|
+
if (isNaN(n) || n <= 0) return DEFAULT_PROGRESS_LOG_PERCENT_STEP;
|
|
11356
|
+
return Math.min(n, 100);
|
|
11357
|
+
}
|
|
11358
|
+
function getProgressLogFallbackInterval() {
|
|
11359
|
+
const raw = optionalEnv("PROGRESS_LOG_FALLBACK_INTERVAL", String(DEFAULT_PROGRESS_LOG_FALLBACK_INTERVAL));
|
|
11360
|
+
const n = parseInt(raw, 10);
|
|
11361
|
+
if (isNaN(n) || n <= 0) return DEFAULT_PROGRESS_LOG_FALLBACK_INTERVAL;
|
|
11362
|
+
return n;
|
|
11363
|
+
}
|
|
11320
11364
|
var Stream = class {
|
|
11321
11365
|
// Runtime values, can't be overriden
|
|
11322
11366
|
config;
|
|
@@ -11579,6 +11623,9 @@ var Stream = class {
|
|
|
11579
11623
|
// Private sync methods:
|
|
11580
11624
|
async _syncRecords(parent) {
|
|
11581
11625
|
let rowsSent = 0;
|
|
11626
|
+
let lastLoggedPercent = 0;
|
|
11627
|
+
const progressPercentStep = getProgressLogPercentStep();
|
|
11628
|
+
const progressFallbackInterval = getProgressLogFallbackInterval();
|
|
11582
11629
|
const dryRunLimit = isDryRun() ? getDryRunLimit() : void 0;
|
|
11583
11630
|
let recordSyncedMetrics = [];
|
|
11584
11631
|
if (this.rowsSyncedMetricsConf.isEnabled() === true) {
|
|
@@ -11625,9 +11672,14 @@ var Stream = class {
|
|
|
11625
11672
|
logger.info(`\u{1F6D1} Stream: ${this.streamId} - DRY_RUN_LIMIT reached (${dryRunLimit} records), stopping.`);
|
|
11626
11673
|
break;
|
|
11627
11674
|
}
|
|
11628
|
-
if (
|
|
11629
|
-
const
|
|
11630
|
-
|
|
11675
|
+
if (this.totalRows) {
|
|
11676
|
+
const percent = Math.floor(rowsSent / this.totalRows * 100);
|
|
11677
|
+
if (percent >= lastLoggedPercent + progressPercentStep) {
|
|
11678
|
+
lastLoggedPercent = percent - percent % progressPercentStep;
|
|
11679
|
+
logger.info(`\u23F3 Stream: ${this.streamId} - ${rowsSent} records synced so far... (${percent}%)`);
|
|
11680
|
+
}
|
|
11681
|
+
} else if (rowsSent % progressFallbackInterval === 0) {
|
|
11682
|
+
logger.info(`\u23F3 Stream: ${this.streamId} - ${rowsSent} records synced so far...`);
|
|
11631
11683
|
}
|
|
11632
11684
|
}
|
|
11633
11685
|
const stateServiceInst = StateService.getInstance();
|
|
@@ -13554,10 +13606,24 @@ function createCsvStreamConfig(streamId, csvConfig, options) {
|
|
|
13554
13606
|
async function processFileStreams(entries, tapState, target) {
|
|
13555
13607
|
for (const entry of entries) {
|
|
13556
13608
|
const stream = new FileStream(entry.config, entry.filePath, tapState, target);
|
|
13557
|
-
|
|
13609
|
+
try {
|
|
13610
|
+
await stream.sync();
|
|
13611
|
+
} catch (err) {
|
|
13612
|
+
throw new FileProcessingError(entry.filePath, entry.config.streamId, err);
|
|
13613
|
+
}
|
|
13558
13614
|
}
|
|
13559
13615
|
await target.complete();
|
|
13560
13616
|
}
|
|
13617
|
+
var FileProcessingError = class extends Error {
|
|
13618
|
+
constructor(filePath, streamId, cause) {
|
|
13619
|
+
const files = Array.isArray(filePath) ? filePath.join(", ") : filePath;
|
|
13620
|
+
super(`Processing ${files} (stream ${streamId}) failed: ${cause instanceof Error ? cause.message : String(cause)}`);
|
|
13621
|
+
this.filePath = filePath;
|
|
13622
|
+
this.streamId = streamId;
|
|
13623
|
+
this.cause = cause;
|
|
13624
|
+
this.name = "FileProcessingError";
|
|
13625
|
+
}
|
|
13626
|
+
};
|
|
13561
13627
|
|
|
13562
13628
|
// src/sdk/file-processing/csv/config-builder.ts
|
|
13563
13629
|
var CsvExtractionConfigBuilder = class _CsvExtractionConfigBuilder {
|
|
@@ -13683,6 +13749,16 @@ var VariableExtractors = class {
|
|
|
13683
13749
|
return match?.groups ?? {};
|
|
13684
13750
|
};
|
|
13685
13751
|
}
|
|
13752
|
+
/**
|
|
13753
|
+
* Extracts named capture groups from the subject of the email that started the run, for
|
|
13754
|
+
* scheduled reports whose date only appears in the subject. Empty for manual uploads.
|
|
13755
|
+
*/
|
|
13756
|
+
static emailSubject(pattern) {
|
|
13757
|
+
return () => {
|
|
13758
|
+
const subject = getEmailMetadata()?.subject;
|
|
13759
|
+
return subject ? subject.match(pattern)?.groups ?? {} : {};
|
|
13760
|
+
};
|
|
13761
|
+
}
|
|
13686
13762
|
/**
|
|
13687
13763
|
* Combines multiple extraction functions.
|
|
13688
13764
|
*/
|
|
@@ -15678,6 +15754,7 @@ var GCSStateProvider = class {
|
|
|
15678
15754
|
ExcelExtractionConfigBuilder,
|
|
15679
15755
|
FileFormat,
|
|
15680
15756
|
FilePatterns,
|
|
15757
|
+
FileProcessingError,
|
|
15681
15758
|
FileStream,
|
|
15682
15759
|
FileTap,
|
|
15683
15760
|
GCSStateProvider,
|
|
@@ -15739,11 +15816,14 @@ var GCSStateProvider = class {
|
|
|
15739
15816
|
getCounterMetrics,
|
|
15740
15817
|
getDownloadFileApiCall,
|
|
15741
15818
|
getDryRunLimit,
|
|
15819
|
+
getEmailMetadata,
|
|
15742
15820
|
getExtension,
|
|
15821
|
+
getInputFiles,
|
|
15743
15822
|
getJSONApiCall,
|
|
15744
15823
|
getJSONApiCallWithFullResponse,
|
|
15745
15824
|
getMimeType,
|
|
15746
15825
|
getRawJSONApiCall,
|
|
15826
|
+
getRunTrigger,
|
|
15747
15827
|
getServiceAccountKey,
|
|
15748
15828
|
gracefulExit,
|
|
15749
15829
|
haltAndCatchFire,
|