@whaly/connector-sdk 0.3.13 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +38 -1
- package/dist/index.d.ts +38 -1
- package/dist/index.js +59 -1
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +55 -1
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/index.d.mts
CHANGED
|
@@ -153,6 +153,26 @@ declare function getMimeType(filePathOrExt: string): string;
|
|
|
153
153
|
*/
|
|
154
154
|
declare function getExtension(mimeType: string): string;
|
|
155
155
|
|
|
156
|
+
/**
|
|
157
|
+
* What a Whaly FileUpload run hands its transformer: the files to process and, for runs started
|
|
158
|
+
* by an inbound email, who sent it and when.
|
|
159
|
+
*/
|
|
160
|
+
interface EmailMetadata {
|
|
161
|
+
from?: string | null;
|
|
162
|
+
subject?: string | null;
|
|
163
|
+
receivedAt?: string | null;
|
|
164
|
+
}
|
|
165
|
+
type RunTrigger = "MANUAL" | "EMAIL" | "REPLAY";
|
|
166
|
+
/**
|
|
167
|
+
* The run's input files, as object keys in the job's storage (pass each to
|
|
168
|
+
* `storage.resolveFileUri`). Email runs list every kept attachment in `WLY_FILE_URIS`; manual
|
|
169
|
+
* uploads only set `FILE_URI`, so this works for both.
|
|
170
|
+
*/
|
|
171
|
+
declare function getInputFiles(env?: NodeJS.ProcessEnv): string[];
|
|
172
|
+
/** Sender, subject and receive time of the email that started the run; undefined otherwise. */
|
|
173
|
+
declare function getEmailMetadata(env?: NodeJS.ProcessEnv): EmailMetadata | undefined;
|
|
174
|
+
declare function getRunTrigger(env?: NodeJS.ProcessEnv): RunTrigger;
|
|
175
|
+
|
|
156
176
|
declare enum ReplicationMethod {
|
|
157
177
|
INCREMENTAL = "INCREMENTAL",
|
|
158
178
|
FULL_TABLE = "FULL_TABLE",
|
|
@@ -1029,6 +1049,16 @@ declare function createCsvStreamConfig(streamId: string, csvConfig: CsvFileConfi
|
|
|
1029
1049
|
* @param target - The target (e.g. BigQueryTarget)
|
|
1030
1050
|
*/
|
|
1031
1051
|
declare function processFileStreams(entries: FileStreamEntry[], tapState: InputTapState, target: ITarget): Promise<void>;
|
|
1052
|
+
/**
|
|
1053
|
+
* A run can process several files (one email, several attachments): says which one failed,
|
|
1054
|
+
* so the run's error is not just "the run failed".
|
|
1055
|
+
*/
|
|
1056
|
+
declare class FileProcessingError extends Error {
|
|
1057
|
+
readonly filePath: string | string[];
|
|
1058
|
+
readonly streamId: string;
|
|
1059
|
+
readonly cause: unknown;
|
|
1060
|
+
constructor(filePath: string | string[], streamId: string, cause: unknown);
|
|
1061
|
+
}
|
|
1032
1062
|
|
|
1033
1063
|
interface FileTapConfig {
|
|
1034
1064
|
[key: string]: any;
|
|
@@ -1067,6 +1097,13 @@ declare class VariableExtractors {
|
|
|
1067
1097
|
static regex(pattern: RegExp): (filename: string) => {
|
|
1068
1098
|
[key: string]: string | undefined;
|
|
1069
1099
|
};
|
|
1100
|
+
/**
|
|
1101
|
+
* Extracts named capture groups from the subject of the email that started the run, for
|
|
1102
|
+
* scheduled reports whose date only appears in the subject. Empty for manual uploads.
|
|
1103
|
+
*/
|
|
1104
|
+
static emailSubject(pattern: RegExp): (filename: string) => {
|
|
1105
|
+
[key: string]: string | undefined;
|
|
1106
|
+
};
|
|
1070
1107
|
/**
|
|
1071
1108
|
* Combines multiple extraction functions.
|
|
1072
1109
|
*/
|
|
@@ -1741,4 +1778,4 @@ declare class GCSStateProvider implements StateProvider {
|
|
|
1741
1778
|
writeState(state: string): Promise<void>;
|
|
1742
1779
|
}
|
|
1743
1780
|
|
|
1744
|
-
export { ALL_STREAM_ID_KEYWORD, API_CALLS_METRIC_NAME, type AssetEntry, type AssetManifest, type AssetManifestEntry, type AssetReplicationMode, AssetStream, AssetTap, AssetTarget, Authenticator, BATCH_INTERVAL_MS, type BaseConfig, type BigQueryConfig, BigQueryTarget, type BigqueryLocation, type Bookmark, type Bookmarks, CdnAssetTarget, type CdnAssetTargetConfig, type CdnFileMetadata, CdnService, type CdnServiceConfig, type CdnUploadResult, CloudStorageService, type CloudStorageServiceOptions, type ColumnMappingStoreUnsafeToSafe, CounterMetric, CsvExtractionConfigBuilder, type CsvFieldsArrayConfig, type CsvFieldsConfig, type CsvFieldsDictConfig, type CsvFileConfig, type CsvStreamConfig, DEFAULT_MAX_CONCURRENT_STREAMS, DEFAULT_SYNCED_AT_COLUMN, DocumentDownloadSkipError, type DocumentEntry, type DocumentManifest, type DocumentManifestEntry, type DocumentStatus, DocumentStream, type DocumentStreamManifest, type DocumentSummary, DocumentTap, type DownloadFilesOptions, EXECUTION_TIME_METRIC_NAME, type ErrorType, type ExcelCustomExtractorConfig, type ExcelDerivedField, type ExcelExtractionBaseConfig, type ExcelExtractionConfig, ExcelExtractionConfigBuilder, type ExcelFieldMapping, type ExcelFieldSpec, type ExcelSingleSheetExtractionConfig, type ExcelSourceColumn, type ExcelStreamConfig, type FieldType, FileFormat, FilePatterns, FileStream, type FileStreamConfig, type FileStreamEntry, FileTap, type FileTapConfig, type FlattenedSchema, GCSStateProvider, type GetConfig, type HTTPHeaders, type HTTPMethod, ITarget, ImageTransform, type InputTapState, type InternalTableField, type JSONSchemaFieldDefinition, type ListFilesOptions, LocalStorageService, type LocalStorageServiceOptions, MIME_TYPES, type Message, type MessageType, type MetricConfiguration, MissingFieldInSchemaError, MissingSchemaError, type PostConfig, type ProcessedAsset, type ProgressMarkers, type PropertiesMetadata, RESTStream, ROWS_SYNCED_METRIC_NAME, RecordMessage, RenameColumnStore, ReplicationMethod, ReplicationMethodMessage, type SafeColumnNameConverterFn, type Schema, SchemaMessage, SchemaValidationError, SftpService, type State, type StateHolder, StateMessage, type StateProvider, StateService, type StorageService, Stream, type StreamId, type StreamManifest, type StreamState, StreamWarehouseSyncService, type StreamWithTempFile, type SyncOptions, Tap, type TapState, type TargetSchemaHook, type TargetSchemaHookInput, type TempFile, type URLParams, VariableExtractors, type WarehouseTableField, type WebpTransformOptions, type WhalyDocument, WhalyDocumentService, type WhalyDocumentServiceConfig, WhalyDocumentTarget, type WhalyPaginatedResponse, type WhalyUploadResult, type WriteGeneratorToCsvOptions, _greaterThan, addDocumentSummaries, addWhalyFields, checkCsvHeaderRow, coerceCellValue, collectPaginated, countCsvLines, createCellReference, createCsvStreamConfig, createExcelGenerator, createExcelStreamConfig, createTemporaryFileStream, csvFieldsToJsonSchema, downloadFiles, emptyDocumentSummary, excelColumnToIndex, excelFieldsToJsonSchema, excelRowToIndex, extractExcelRows, extractPrimaryKeysFromCsvConfig, extractPrimaryKeysFromExcelFields, extractStateForStream, fieldTypeToJsonSchema, finalizeStateProgressMarkers, findAllMatchingConfigs, flattenSchema, getAllMetrics, getApiEndpoint, getAxiosInstance, getCdnId, getCounterMetric, getCounterMetrics, getDownloadFileApiCall, getDryRunLimit, getExtension, getJSONApiCall, getJSONApiCallWithFullResponse, getMimeType, getRawJSONApiCall, getServiceAccountKey, gracefulExit, haltAndCatchFire, incrementStreamState, indexToExcelColumn, indexToExcelRow, isDryRun, listFilesRecursively, loadJson, loadSchema, logger, optionalEnv, parseCellReference, postFormDataApiCall, postJSONApiCall, postUrlEncodedApiCall, printMemoryFootprint, processFileStreams, processFromAsyncIterable, removeParasiteProperties, requireEnv, rowGeneratorFromCsv, runWithConcurrency, safePath, unzip, validateExcelConfig, writeDataToCsv, writeGeneratorToCSV };
|
|
1781
|
+
export { ALL_STREAM_ID_KEYWORD, API_CALLS_METRIC_NAME, type AssetEntry, type AssetManifest, type AssetManifestEntry, type AssetReplicationMode, AssetStream, AssetTap, AssetTarget, Authenticator, BATCH_INTERVAL_MS, type BaseConfig, type BigQueryConfig, BigQueryTarget, type BigqueryLocation, type Bookmark, type Bookmarks, CdnAssetTarget, type CdnAssetTargetConfig, type CdnFileMetadata, CdnService, type CdnServiceConfig, type CdnUploadResult, CloudStorageService, type CloudStorageServiceOptions, type ColumnMappingStoreUnsafeToSafe, CounterMetric, CsvExtractionConfigBuilder, type CsvFieldsArrayConfig, type CsvFieldsConfig, type CsvFieldsDictConfig, type CsvFileConfig, type CsvStreamConfig, DEFAULT_MAX_CONCURRENT_STREAMS, DEFAULT_SYNCED_AT_COLUMN, DocumentDownloadSkipError, type DocumentEntry, type DocumentManifest, type DocumentManifestEntry, type DocumentStatus, DocumentStream, type DocumentStreamManifest, type DocumentSummary, DocumentTap, type DownloadFilesOptions, EXECUTION_TIME_METRIC_NAME, type EmailMetadata, type ErrorType, type ExcelCustomExtractorConfig, type ExcelDerivedField, type ExcelExtractionBaseConfig, type ExcelExtractionConfig, ExcelExtractionConfigBuilder, type ExcelFieldMapping, type ExcelFieldSpec, type ExcelSingleSheetExtractionConfig, type ExcelSourceColumn, type ExcelStreamConfig, type FieldType, FileFormat, FilePatterns, FileProcessingError, FileStream, type FileStreamConfig, type FileStreamEntry, FileTap, type FileTapConfig, type FlattenedSchema, GCSStateProvider, type GetConfig, type HTTPHeaders, type HTTPMethod, ITarget, ImageTransform, type InputTapState, type InternalTableField, type JSONSchemaFieldDefinition, type ListFilesOptions, LocalStorageService, type LocalStorageServiceOptions, MIME_TYPES, type Message, type MessageType, type MetricConfiguration, MissingFieldInSchemaError, MissingSchemaError, type PostConfig, type ProcessedAsset, type ProgressMarkers, type PropertiesMetadata, RESTStream, ROWS_SYNCED_METRIC_NAME, RecordMessage, RenameColumnStore, ReplicationMethod, ReplicationMethodMessage, type RunTrigger, type SafeColumnNameConverterFn, type Schema, SchemaMessage, SchemaValidationError, SftpService, type State, type StateHolder, StateMessage, type StateProvider, StateService, type StorageService, Stream, type StreamId, type StreamManifest, type StreamState, StreamWarehouseSyncService, type StreamWithTempFile, type SyncOptions, Tap, type TapState, type TargetSchemaHook, type TargetSchemaHookInput, type TempFile, type URLParams, VariableExtractors, type WarehouseTableField, type WebpTransformOptions, type WhalyDocument, WhalyDocumentService, type WhalyDocumentServiceConfig, WhalyDocumentTarget, type WhalyPaginatedResponse, type WhalyUploadResult, type WriteGeneratorToCsvOptions, _greaterThan, addDocumentSummaries, addWhalyFields, checkCsvHeaderRow, coerceCellValue, collectPaginated, countCsvLines, createCellReference, createCsvStreamConfig, createExcelGenerator, createExcelStreamConfig, createTemporaryFileStream, csvFieldsToJsonSchema, downloadFiles, emptyDocumentSummary, excelColumnToIndex, excelFieldsToJsonSchema, excelRowToIndex, extractExcelRows, extractPrimaryKeysFromCsvConfig, extractPrimaryKeysFromExcelFields, extractStateForStream, fieldTypeToJsonSchema, finalizeStateProgressMarkers, findAllMatchingConfigs, flattenSchema, getAllMetrics, getApiEndpoint, getAxiosInstance, getCdnId, getCounterMetric, getCounterMetrics, getDownloadFileApiCall, getDryRunLimit, getEmailMetadata, getExtension, getInputFiles, getJSONApiCall, getJSONApiCallWithFullResponse, getMimeType, getRawJSONApiCall, getRunTrigger, getServiceAccountKey, gracefulExit, haltAndCatchFire, incrementStreamState, indexToExcelColumn, indexToExcelRow, isDryRun, listFilesRecursively, loadJson, loadSchema, logger, optionalEnv, parseCellReference, postFormDataApiCall, postJSONApiCall, postUrlEncodedApiCall, printMemoryFootprint, processFileStreams, processFromAsyncIterable, removeParasiteProperties, requireEnv, rowGeneratorFromCsv, runWithConcurrency, safePath, unzip, validateExcelConfig, writeDataToCsv, writeGeneratorToCSV };
|
package/dist/index.d.ts
CHANGED
|
@@ -153,6 +153,26 @@ declare function getMimeType(filePathOrExt: string): string;
|
|
|
153
153
|
*/
|
|
154
154
|
declare function getExtension(mimeType: string): string;
|
|
155
155
|
|
|
156
|
+
/**
|
|
157
|
+
* What a Whaly FileUpload run hands its transformer: the files to process and, for runs started
|
|
158
|
+
* by an inbound email, who sent it and when.
|
|
159
|
+
*/
|
|
160
|
+
interface EmailMetadata {
|
|
161
|
+
from?: string | null;
|
|
162
|
+
subject?: string | null;
|
|
163
|
+
receivedAt?: string | null;
|
|
164
|
+
}
|
|
165
|
+
type RunTrigger = "MANUAL" | "EMAIL" | "REPLAY";
|
|
166
|
+
/**
|
|
167
|
+
* The run's input files, as object keys in the job's storage (pass each to
|
|
168
|
+
* `storage.resolveFileUri`). Email runs list every kept attachment in `WLY_FILE_URIS`; manual
|
|
169
|
+
* uploads only set `FILE_URI`, so this works for both.
|
|
170
|
+
*/
|
|
171
|
+
declare function getInputFiles(env?: NodeJS.ProcessEnv): string[];
|
|
172
|
+
/** Sender, subject and receive time of the email that started the run; undefined otherwise. */
|
|
173
|
+
declare function getEmailMetadata(env?: NodeJS.ProcessEnv): EmailMetadata | undefined;
|
|
174
|
+
declare function getRunTrigger(env?: NodeJS.ProcessEnv): RunTrigger;
|
|
175
|
+
|
|
156
176
|
declare enum ReplicationMethod {
|
|
157
177
|
INCREMENTAL = "INCREMENTAL",
|
|
158
178
|
FULL_TABLE = "FULL_TABLE",
|
|
@@ -1029,6 +1049,16 @@ declare function createCsvStreamConfig(streamId: string, csvConfig: CsvFileConfi
|
|
|
1029
1049
|
* @param target - The target (e.g. BigQueryTarget)
|
|
1030
1050
|
*/
|
|
1031
1051
|
declare function processFileStreams(entries: FileStreamEntry[], tapState: InputTapState, target: ITarget): Promise<void>;
|
|
1052
|
+
/**
|
|
1053
|
+
* A run can process several files (one email, several attachments): says which one failed,
|
|
1054
|
+
* so the run's error is not just "the run failed".
|
|
1055
|
+
*/
|
|
1056
|
+
declare class FileProcessingError extends Error {
|
|
1057
|
+
readonly filePath: string | string[];
|
|
1058
|
+
readonly streamId: string;
|
|
1059
|
+
readonly cause: unknown;
|
|
1060
|
+
constructor(filePath: string | string[], streamId: string, cause: unknown);
|
|
1061
|
+
}
|
|
1032
1062
|
|
|
1033
1063
|
interface FileTapConfig {
|
|
1034
1064
|
[key: string]: any;
|
|
@@ -1067,6 +1097,13 @@ declare class VariableExtractors {
|
|
|
1067
1097
|
static regex(pattern: RegExp): (filename: string) => {
|
|
1068
1098
|
[key: string]: string | undefined;
|
|
1069
1099
|
};
|
|
1100
|
+
/**
|
|
1101
|
+
* Extracts named capture groups from the subject of the email that started the run, for
|
|
1102
|
+
* scheduled reports whose date only appears in the subject. Empty for manual uploads.
|
|
1103
|
+
*/
|
|
1104
|
+
static emailSubject(pattern: RegExp): (filename: string) => {
|
|
1105
|
+
[key: string]: string | undefined;
|
|
1106
|
+
};
|
|
1070
1107
|
/**
|
|
1071
1108
|
* Combines multiple extraction functions.
|
|
1072
1109
|
*/
|
|
@@ -1741,4 +1778,4 @@ declare class GCSStateProvider implements StateProvider {
|
|
|
1741
1778
|
writeState(state: string): Promise<void>;
|
|
1742
1779
|
}
|
|
1743
1780
|
|
|
1744
|
-
export { ALL_STREAM_ID_KEYWORD, API_CALLS_METRIC_NAME, type AssetEntry, type AssetManifest, type AssetManifestEntry, type AssetReplicationMode, AssetStream, AssetTap, AssetTarget, Authenticator, BATCH_INTERVAL_MS, type BaseConfig, type BigQueryConfig, BigQueryTarget, type BigqueryLocation, type Bookmark, type Bookmarks, CdnAssetTarget, type CdnAssetTargetConfig, type CdnFileMetadata, CdnService, type CdnServiceConfig, type CdnUploadResult, CloudStorageService, type CloudStorageServiceOptions, type ColumnMappingStoreUnsafeToSafe, CounterMetric, CsvExtractionConfigBuilder, type CsvFieldsArrayConfig, type CsvFieldsConfig, type CsvFieldsDictConfig, type CsvFileConfig, type CsvStreamConfig, DEFAULT_MAX_CONCURRENT_STREAMS, DEFAULT_SYNCED_AT_COLUMN, DocumentDownloadSkipError, type DocumentEntry, type DocumentManifest, type DocumentManifestEntry, type DocumentStatus, DocumentStream, type DocumentStreamManifest, type DocumentSummary, DocumentTap, type DownloadFilesOptions, EXECUTION_TIME_METRIC_NAME, type ErrorType, type ExcelCustomExtractorConfig, type ExcelDerivedField, type ExcelExtractionBaseConfig, type ExcelExtractionConfig, ExcelExtractionConfigBuilder, type ExcelFieldMapping, type ExcelFieldSpec, type ExcelSingleSheetExtractionConfig, type ExcelSourceColumn, type ExcelStreamConfig, type FieldType, FileFormat, FilePatterns, FileStream, type FileStreamConfig, type FileStreamEntry, FileTap, type FileTapConfig, type FlattenedSchema, GCSStateProvider, type GetConfig, type HTTPHeaders, type HTTPMethod, ITarget, ImageTransform, type InputTapState, type InternalTableField, type JSONSchemaFieldDefinition, type ListFilesOptions, LocalStorageService, type LocalStorageServiceOptions, MIME_TYPES, type Message, type MessageType, type MetricConfiguration, MissingFieldInSchemaError, MissingSchemaError, type PostConfig, type ProcessedAsset, type ProgressMarkers, type PropertiesMetadata, RESTStream, ROWS_SYNCED_METRIC_NAME, RecordMessage, RenameColumnStore, ReplicationMethod, ReplicationMethodMessage, type SafeColumnNameConverterFn, type Schema, SchemaMessage, SchemaValidationError, SftpService, type State, type StateHolder, StateMessage, type StateProvider, StateService, type StorageService, Stream, type StreamId, type StreamManifest, type StreamState, StreamWarehouseSyncService, type StreamWithTempFile, type SyncOptions, Tap, type TapState, type TargetSchemaHook, type TargetSchemaHookInput, type TempFile, type URLParams, VariableExtractors, type WarehouseTableField, type WebpTransformOptions, type WhalyDocument, WhalyDocumentService, type WhalyDocumentServiceConfig, WhalyDocumentTarget, type WhalyPaginatedResponse, type WhalyUploadResult, type WriteGeneratorToCsvOptions, _greaterThan, addDocumentSummaries, addWhalyFields, checkCsvHeaderRow, coerceCellValue, collectPaginated, countCsvLines, createCellReference, createCsvStreamConfig, createExcelGenerator, createExcelStreamConfig, createTemporaryFileStream, csvFieldsToJsonSchema, downloadFiles, emptyDocumentSummary, excelColumnToIndex, excelFieldsToJsonSchema, excelRowToIndex, extractExcelRows, extractPrimaryKeysFromCsvConfig, extractPrimaryKeysFromExcelFields, extractStateForStream, fieldTypeToJsonSchema, finalizeStateProgressMarkers, findAllMatchingConfigs, flattenSchema, getAllMetrics, getApiEndpoint, getAxiosInstance, getCdnId, getCounterMetric, getCounterMetrics, getDownloadFileApiCall, getDryRunLimit, getExtension, getJSONApiCall, getJSONApiCallWithFullResponse, getMimeType, getRawJSONApiCall, getServiceAccountKey, gracefulExit, haltAndCatchFire, incrementStreamState, indexToExcelColumn, indexToExcelRow, isDryRun, listFilesRecursively, loadJson, loadSchema, logger, optionalEnv, parseCellReference, postFormDataApiCall, postJSONApiCall, postUrlEncodedApiCall, printMemoryFootprint, processFileStreams, processFromAsyncIterable, removeParasiteProperties, requireEnv, rowGeneratorFromCsv, runWithConcurrency, safePath, unzip, validateExcelConfig, writeDataToCsv, writeGeneratorToCSV };
|
|
1781
|
+
export { ALL_STREAM_ID_KEYWORD, API_CALLS_METRIC_NAME, type AssetEntry, type AssetManifest, type AssetManifestEntry, type AssetReplicationMode, AssetStream, AssetTap, AssetTarget, Authenticator, BATCH_INTERVAL_MS, type BaseConfig, type BigQueryConfig, BigQueryTarget, type BigqueryLocation, type Bookmark, type Bookmarks, CdnAssetTarget, type CdnAssetTargetConfig, type CdnFileMetadata, CdnService, type CdnServiceConfig, type CdnUploadResult, CloudStorageService, type CloudStorageServiceOptions, type ColumnMappingStoreUnsafeToSafe, CounterMetric, CsvExtractionConfigBuilder, type CsvFieldsArrayConfig, type CsvFieldsConfig, type CsvFieldsDictConfig, type CsvFileConfig, type CsvStreamConfig, DEFAULT_MAX_CONCURRENT_STREAMS, DEFAULT_SYNCED_AT_COLUMN, DocumentDownloadSkipError, type DocumentEntry, type DocumentManifest, type DocumentManifestEntry, type DocumentStatus, DocumentStream, type DocumentStreamManifest, type DocumentSummary, DocumentTap, type DownloadFilesOptions, EXECUTION_TIME_METRIC_NAME, type EmailMetadata, type ErrorType, type ExcelCustomExtractorConfig, type ExcelDerivedField, type ExcelExtractionBaseConfig, type ExcelExtractionConfig, ExcelExtractionConfigBuilder, type ExcelFieldMapping, type ExcelFieldSpec, type ExcelSingleSheetExtractionConfig, type ExcelSourceColumn, type ExcelStreamConfig, type FieldType, FileFormat, FilePatterns, FileProcessingError, FileStream, type FileStreamConfig, type FileStreamEntry, FileTap, type FileTapConfig, type FlattenedSchema, GCSStateProvider, type GetConfig, type HTTPHeaders, type HTTPMethod, ITarget, ImageTransform, type InputTapState, type InternalTableField, type JSONSchemaFieldDefinition, type ListFilesOptions, LocalStorageService, type LocalStorageServiceOptions, MIME_TYPES, type Message, type MessageType, type MetricConfiguration, MissingFieldInSchemaError, MissingSchemaError, type PostConfig, type ProcessedAsset, type ProgressMarkers, type PropertiesMetadata, RESTStream, ROWS_SYNCED_METRIC_NAME, RecordMessage, RenameColumnStore, ReplicationMethod, ReplicationMethodMessage, type RunTrigger, type SafeColumnNameConverterFn, type Schema, SchemaMessage, SchemaValidationError, SftpService, type State, type StateHolder, StateMessage, type StateProvider, StateService, type StorageService, Stream, type StreamId, type StreamManifest, type StreamState, StreamWarehouseSyncService, type StreamWithTempFile, type SyncOptions, Tap, type TapState, type TargetSchemaHook, type TargetSchemaHookInput, type TempFile, type URLParams, VariableExtractors, type WarehouseTableField, type WebpTransformOptions, type WhalyDocument, WhalyDocumentService, type WhalyDocumentServiceConfig, WhalyDocumentTarget, type WhalyPaginatedResponse, type WhalyUploadResult, type WriteGeneratorToCsvOptions, _greaterThan, addDocumentSummaries, addWhalyFields, checkCsvHeaderRow, coerceCellValue, collectPaginated, countCsvLines, createCellReference, createCsvStreamConfig, createExcelGenerator, createExcelStreamConfig, createTemporaryFileStream, csvFieldsToJsonSchema, downloadFiles, emptyDocumentSummary, excelColumnToIndex, excelFieldsToJsonSchema, excelRowToIndex, extractExcelRows, extractPrimaryKeysFromCsvConfig, extractPrimaryKeysFromExcelFields, extractStateForStream, fieldTypeToJsonSchema, finalizeStateProgressMarkers, findAllMatchingConfigs, flattenSchema, getAllMetrics, getApiEndpoint, getAxiosInstance, getCdnId, getCounterMetric, getCounterMetrics, getDownloadFileApiCall, getDryRunLimit, getEmailMetadata, getExtension, getInputFiles, getJSONApiCall, getJSONApiCallWithFullResponse, getMimeType, getRawJSONApiCall, getRunTrigger, getServiceAccountKey, gracefulExit, haltAndCatchFire, incrementStreamState, indexToExcelColumn, indexToExcelRow, isDryRun, listFilesRecursively, loadJson, loadSchema, logger, optionalEnv, parseCellReference, postFormDataApiCall, postJSONApiCall, postUrlEncodedApiCall, printMemoryFootprint, processFileStreams, processFromAsyncIterable, removeParasiteProperties, requireEnv, rowGeneratorFromCsv, runWithConcurrency, safePath, unzip, validateExcelConfig, writeDataToCsv, writeGeneratorToCSV };
|
package/dist/index.js
CHANGED
|
@@ -10337,6 +10337,7 @@ __export(index_exports, {
|
|
|
10337
10337
|
ExcelExtractionConfigBuilder: () => ExcelExtractionConfigBuilder,
|
|
10338
10338
|
FileFormat: () => FileFormat,
|
|
10339
10339
|
FilePatterns: () => FilePatterns,
|
|
10340
|
+
FileProcessingError: () => FileProcessingError,
|
|
10340
10341
|
FileStream: () => FileStream,
|
|
10341
10342
|
FileTap: () => FileTap,
|
|
10342
10343
|
GCSStateProvider: () => GCSStateProvider,
|
|
@@ -10398,11 +10399,14 @@ __export(index_exports, {
|
|
|
10398
10399
|
getCounterMetrics: () => getCounterMetrics,
|
|
10399
10400
|
getDownloadFileApiCall: () => getDownloadFileApiCall,
|
|
10400
10401
|
getDryRunLimit: () => getDryRunLimit,
|
|
10402
|
+
getEmailMetadata: () => getEmailMetadata,
|
|
10401
10403
|
getExtension: () => getExtension,
|
|
10404
|
+
getInputFiles: () => getInputFiles,
|
|
10402
10405
|
getJSONApiCall: () => getJSONApiCall,
|
|
10403
10406
|
getJSONApiCallWithFullResponse: () => getJSONApiCallWithFullResponse,
|
|
10404
10407
|
getMimeType: () => getMimeType,
|
|
10405
10408
|
getRawJSONApiCall: () => getRawJSONApiCall,
|
|
10409
|
+
getRunTrigger: () => getRunTrigger,
|
|
10406
10410
|
getServiceAccountKey: () => getServiceAccountKey,
|
|
10407
10411
|
gracefulExit: () => gracefulExit,
|
|
10408
10412
|
haltAndCatchFire: () => haltAndCatchFire,
|
|
@@ -10986,6 +10990,32 @@ function getExtension(mimeType) {
|
|
|
10986
10990
|
return EXTENSION_BY_MIME[mimeType] ?? ".bin";
|
|
10987
10991
|
}
|
|
10988
10992
|
|
|
10993
|
+
// src/sdk/service/runInput.ts
|
|
10994
|
+
function getInputFiles(env = process.env) {
|
|
10995
|
+
const list = env.WLY_FILE_URIS;
|
|
10996
|
+
if (list) {
|
|
10997
|
+
const parsed = JSON.parse(list);
|
|
10998
|
+
if (!Array.isArray(parsed) || !parsed.every((item) => typeof item === "string")) {
|
|
10999
|
+
throw new Error("WLY_FILE_URIS must be a JSON array of strings");
|
|
11000
|
+
}
|
|
11001
|
+
if (parsed.length > 0) {
|
|
11002
|
+
return parsed;
|
|
11003
|
+
}
|
|
11004
|
+
}
|
|
11005
|
+
return env.FILE_URI ? [env.FILE_URI] : [];
|
|
11006
|
+
}
|
|
11007
|
+
function getEmailMetadata(env = process.env) {
|
|
11008
|
+
const raw = env.WLY_EMAIL_METADATA;
|
|
11009
|
+
if (!raw) {
|
|
11010
|
+
return void 0;
|
|
11011
|
+
}
|
|
11012
|
+
return JSON.parse(raw);
|
|
11013
|
+
}
|
|
11014
|
+
function getRunTrigger(env = process.env) {
|
|
11015
|
+
const trigger = env.WLY_RUN_TRIGGER;
|
|
11016
|
+
return trigger === "EMAIL" || trigger === "REPLAY" ? trigger : "MANUAL";
|
|
11017
|
+
}
|
|
11018
|
+
|
|
10989
11019
|
// src/sdk/models/replication.ts
|
|
10990
11020
|
var ReplicationMethod = /* @__PURE__ */ ((ReplicationMethod2) => {
|
|
10991
11021
|
ReplicationMethod2["INCREMENTAL"] = "INCREMENTAL";
|
|
@@ -13576,10 +13606,24 @@ function createCsvStreamConfig(streamId, csvConfig, options) {
|
|
|
13576
13606
|
async function processFileStreams(entries, tapState, target) {
|
|
13577
13607
|
for (const entry of entries) {
|
|
13578
13608
|
const stream = new FileStream(entry.config, entry.filePath, tapState, target);
|
|
13579
|
-
|
|
13609
|
+
try {
|
|
13610
|
+
await stream.sync();
|
|
13611
|
+
} catch (err) {
|
|
13612
|
+
throw new FileProcessingError(entry.filePath, entry.config.streamId, err);
|
|
13613
|
+
}
|
|
13580
13614
|
}
|
|
13581
13615
|
await target.complete();
|
|
13582
13616
|
}
|
|
13617
|
+
var FileProcessingError = class extends Error {
|
|
13618
|
+
constructor(filePath, streamId, cause) {
|
|
13619
|
+
const files = Array.isArray(filePath) ? filePath.join(", ") : filePath;
|
|
13620
|
+
super(`Processing ${files} (stream ${streamId}) failed: ${cause instanceof Error ? cause.message : String(cause)}`);
|
|
13621
|
+
this.filePath = filePath;
|
|
13622
|
+
this.streamId = streamId;
|
|
13623
|
+
this.cause = cause;
|
|
13624
|
+
this.name = "FileProcessingError";
|
|
13625
|
+
}
|
|
13626
|
+
};
|
|
13583
13627
|
|
|
13584
13628
|
// src/sdk/file-processing/csv/config-builder.ts
|
|
13585
13629
|
var CsvExtractionConfigBuilder = class _CsvExtractionConfigBuilder {
|
|
@@ -13705,6 +13749,16 @@ var VariableExtractors = class {
|
|
|
13705
13749
|
return match?.groups ?? {};
|
|
13706
13750
|
};
|
|
13707
13751
|
}
|
|
13752
|
+
/**
|
|
13753
|
+
* Extracts named capture groups from the subject of the email that started the run, for
|
|
13754
|
+
* scheduled reports whose date only appears in the subject. Empty for manual uploads.
|
|
13755
|
+
*/
|
|
13756
|
+
static emailSubject(pattern) {
|
|
13757
|
+
return () => {
|
|
13758
|
+
const subject = getEmailMetadata()?.subject;
|
|
13759
|
+
return subject ? subject.match(pattern)?.groups ?? {} : {};
|
|
13760
|
+
};
|
|
13761
|
+
}
|
|
13708
13762
|
/**
|
|
13709
13763
|
* Combines multiple extraction functions.
|
|
13710
13764
|
*/
|
|
@@ -15700,6 +15754,7 @@ var GCSStateProvider = class {
|
|
|
15700
15754
|
ExcelExtractionConfigBuilder,
|
|
15701
15755
|
FileFormat,
|
|
15702
15756
|
FilePatterns,
|
|
15757
|
+
FileProcessingError,
|
|
15703
15758
|
FileStream,
|
|
15704
15759
|
FileTap,
|
|
15705
15760
|
GCSStateProvider,
|
|
@@ -15761,11 +15816,14 @@ var GCSStateProvider = class {
|
|
|
15761
15816
|
getCounterMetrics,
|
|
15762
15817
|
getDownloadFileApiCall,
|
|
15763
15818
|
getDryRunLimit,
|
|
15819
|
+
getEmailMetadata,
|
|
15764
15820
|
getExtension,
|
|
15821
|
+
getInputFiles,
|
|
15765
15822
|
getJSONApiCall,
|
|
15766
15823
|
getJSONApiCallWithFullResponse,
|
|
15767
15824
|
getMimeType,
|
|
15768
15825
|
getRawJSONApiCall,
|
|
15826
|
+
getRunTrigger,
|
|
15769
15827
|
getServiceAccountKey,
|
|
15770
15828
|
gracefulExit,
|
|
15771
15829
|
haltAndCatchFire,
|