@whaly/connector-sdk 0.3.12 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.mts CHANGED
@@ -153,6 +153,26 @@ declare function getMimeType(filePathOrExt: string): string;
153
153
  */
154
154
  declare function getExtension(mimeType: string): string;
155
155
 
156
+ /**
157
+ * What a Whaly FileUpload run hands its transformer: the files to process and, for runs started
158
+ * by an inbound email, who sent it and when.
159
+ */
160
+ interface EmailMetadata {
161
+ from?: string | null;
162
+ subject?: string | null;
163
+ receivedAt?: string | null;
164
+ }
165
+ type RunTrigger = "MANUAL" | "EMAIL" | "REPLAY";
166
+ /**
167
+ * The run's input files, as object keys in the job's storage (pass each to
168
+ * `storage.resolveFileUri`). Email runs list every kept attachment in `WLY_FILE_URIS`; manual
169
+ * uploads only set `FILE_URI`, so this works for both.
170
+ */
171
+ declare function getInputFiles(env?: NodeJS.ProcessEnv): string[];
172
+ /** Sender, subject and receive time of the email that started the run; undefined otherwise. */
173
+ declare function getEmailMetadata(env?: NodeJS.ProcessEnv): EmailMetadata | undefined;
174
+ declare function getRunTrigger(env?: NodeJS.ProcessEnv): RunTrigger;
175
+
156
176
  declare enum ReplicationMethod {
157
177
  INCREMENTAL = "INCREMENTAL",
158
178
  FULL_TABLE = "FULL_TABLE",
@@ -1029,6 +1049,16 @@ declare function createCsvStreamConfig(streamId: string, csvConfig: CsvFileConfi
1029
1049
  * @param target - The target (e.g. BigQueryTarget)
1030
1050
  */
1031
1051
  declare function processFileStreams(entries: FileStreamEntry[], tapState: InputTapState, target: ITarget): Promise<void>;
1052
+ /**
1053
+ * A run can process several files (one email, several attachments): says which one failed,
1054
+ * so the run's error is not just "the run failed".
1055
+ */
1056
+ declare class FileProcessingError extends Error {
1057
+ readonly filePath: string | string[];
1058
+ readonly streamId: string;
1059
+ readonly cause: unknown;
1060
+ constructor(filePath: string | string[], streamId: string, cause: unknown);
1061
+ }
1032
1062
 
1033
1063
  interface FileTapConfig {
1034
1064
  [key: string]: any;
@@ -1067,6 +1097,13 @@ declare class VariableExtractors {
1067
1097
  static regex(pattern: RegExp): (filename: string) => {
1068
1098
  [key: string]: string | undefined;
1069
1099
  };
1100
+ /**
1101
+ * Extracts named capture groups from the subject of the email that started the run, for
1102
+ * scheduled reports whose date only appears in the subject. Empty for manual uploads.
1103
+ */
1104
+ static emailSubject(pattern: RegExp): (filename: string) => {
1105
+ [key: string]: string | undefined;
1106
+ };
1070
1107
  /**
1071
1108
  * Combines multiple extraction functions.
1072
1109
  */
@@ -1741,4 +1778,4 @@ declare class GCSStateProvider implements StateProvider {
1741
1778
  writeState(state: string): Promise<void>;
1742
1779
  }
1743
1780
 
1744
- export { ALL_STREAM_ID_KEYWORD, API_CALLS_METRIC_NAME, type AssetEntry, type AssetManifest, type AssetManifestEntry, type AssetReplicationMode, AssetStream, AssetTap, AssetTarget, Authenticator, BATCH_INTERVAL_MS, type BaseConfig, type BigQueryConfig, BigQueryTarget, type BigqueryLocation, type Bookmark, type Bookmarks, CdnAssetTarget, type CdnAssetTargetConfig, type CdnFileMetadata, CdnService, type CdnServiceConfig, type CdnUploadResult, CloudStorageService, type CloudStorageServiceOptions, type ColumnMappingStoreUnsafeToSafe, CounterMetric, CsvExtractionConfigBuilder, type CsvFieldsArrayConfig, type CsvFieldsConfig, type CsvFieldsDictConfig, type CsvFileConfig, type CsvStreamConfig, DEFAULT_MAX_CONCURRENT_STREAMS, DEFAULT_SYNCED_AT_COLUMN, DocumentDownloadSkipError, type DocumentEntry, type DocumentManifest, type DocumentManifestEntry, type DocumentStatus, DocumentStream, type DocumentStreamManifest, type DocumentSummary, DocumentTap, type DownloadFilesOptions, EXECUTION_TIME_METRIC_NAME, type ErrorType, type ExcelCustomExtractorConfig, type ExcelDerivedField, type ExcelExtractionBaseConfig, type ExcelExtractionConfig, ExcelExtractionConfigBuilder, type ExcelFieldMapping, type ExcelFieldSpec, type ExcelSingleSheetExtractionConfig, type ExcelSourceColumn, type ExcelStreamConfig, type FieldType, FileFormat, FilePatterns, FileStream, type FileStreamConfig, type FileStreamEntry, FileTap, type FileTapConfig, type FlattenedSchema, GCSStateProvider, type GetConfig, type HTTPHeaders, type HTTPMethod, ITarget, ImageTransform, type InputTapState, type InternalTableField, type JSONSchemaFieldDefinition, type ListFilesOptions, LocalStorageService, type LocalStorageServiceOptions, MIME_TYPES, type Message, type MessageType, type MetricConfiguration, MissingFieldInSchemaError, MissingSchemaError, type PostConfig, type ProcessedAsset, type ProgressMarkers, type PropertiesMetadata, RESTStream, ROWS_SYNCED_METRIC_NAME, RecordMessage, RenameColumnStore, ReplicationMethod, ReplicationMethodMessage, type SafeColumnNameConverterFn, type Schema, SchemaMessage, SchemaValidationError, SftpService, type State, type StateHolder, StateMessage, type StateProvider, StateService, type StorageService, Stream, type StreamId, type StreamManifest, type StreamState, StreamWarehouseSyncService, type StreamWithTempFile, type SyncOptions, Tap, type TapState, type TargetSchemaHook, type TargetSchemaHookInput, type TempFile, type URLParams, VariableExtractors, type WarehouseTableField, type WebpTransformOptions, type WhalyDocument, WhalyDocumentService, type WhalyDocumentServiceConfig, WhalyDocumentTarget, type WhalyPaginatedResponse, type WhalyUploadResult, type WriteGeneratorToCsvOptions, _greaterThan, addDocumentSummaries, addWhalyFields, checkCsvHeaderRow, coerceCellValue, collectPaginated, countCsvLines, createCellReference, createCsvStreamConfig, createExcelGenerator, createExcelStreamConfig, createTemporaryFileStream, csvFieldsToJsonSchema, downloadFiles, emptyDocumentSummary, excelColumnToIndex, excelFieldsToJsonSchema, excelRowToIndex, extractExcelRows, extractPrimaryKeysFromCsvConfig, extractPrimaryKeysFromExcelFields, extractStateForStream, fieldTypeToJsonSchema, finalizeStateProgressMarkers, findAllMatchingConfigs, flattenSchema, getAllMetrics, getApiEndpoint, getAxiosInstance, getCdnId, getCounterMetric, getCounterMetrics, getDownloadFileApiCall, getDryRunLimit, getExtension, getJSONApiCall, getJSONApiCallWithFullResponse, getMimeType, getRawJSONApiCall, getServiceAccountKey, gracefulExit, haltAndCatchFire, incrementStreamState, indexToExcelColumn, indexToExcelRow, isDryRun, listFilesRecursively, loadJson, loadSchema, logger, optionalEnv, parseCellReference, postFormDataApiCall, postJSONApiCall, postUrlEncodedApiCall, printMemoryFootprint, processFileStreams, processFromAsyncIterable, removeParasiteProperties, requireEnv, rowGeneratorFromCsv, runWithConcurrency, safePath, unzip, validateExcelConfig, writeDataToCsv, writeGeneratorToCSV };
1781
+ export { ALL_STREAM_ID_KEYWORD, API_CALLS_METRIC_NAME, type AssetEntry, type AssetManifest, type AssetManifestEntry, type AssetReplicationMode, AssetStream, AssetTap, AssetTarget, Authenticator, BATCH_INTERVAL_MS, type BaseConfig, type BigQueryConfig, BigQueryTarget, type BigqueryLocation, type Bookmark, type Bookmarks, CdnAssetTarget, type CdnAssetTargetConfig, type CdnFileMetadata, CdnService, type CdnServiceConfig, type CdnUploadResult, CloudStorageService, type CloudStorageServiceOptions, type ColumnMappingStoreUnsafeToSafe, CounterMetric, CsvExtractionConfigBuilder, type CsvFieldsArrayConfig, type CsvFieldsConfig, type CsvFieldsDictConfig, type CsvFileConfig, type CsvStreamConfig, DEFAULT_MAX_CONCURRENT_STREAMS, DEFAULT_SYNCED_AT_COLUMN, DocumentDownloadSkipError, type DocumentEntry, type DocumentManifest, type DocumentManifestEntry, type DocumentStatus, DocumentStream, type DocumentStreamManifest, type DocumentSummary, DocumentTap, type DownloadFilesOptions, EXECUTION_TIME_METRIC_NAME, type EmailMetadata, type ErrorType, type ExcelCustomExtractorConfig, type ExcelDerivedField, type ExcelExtractionBaseConfig, type ExcelExtractionConfig, ExcelExtractionConfigBuilder, type ExcelFieldMapping, type ExcelFieldSpec, type ExcelSingleSheetExtractionConfig, type ExcelSourceColumn, type ExcelStreamConfig, type FieldType, FileFormat, FilePatterns, FileProcessingError, FileStream, type FileStreamConfig, type FileStreamEntry, FileTap, type FileTapConfig, type FlattenedSchema, GCSStateProvider, type GetConfig, type HTTPHeaders, type HTTPMethod, ITarget, ImageTransform, type InputTapState, type InternalTableField, type JSONSchemaFieldDefinition, type ListFilesOptions, LocalStorageService, type LocalStorageServiceOptions, MIME_TYPES, type Message, type MessageType, type MetricConfiguration, MissingFieldInSchemaError, MissingSchemaError, type PostConfig, type ProcessedAsset, type ProgressMarkers, type PropertiesMetadata, RESTStream, ROWS_SYNCED_METRIC_NAME, RecordMessage, RenameColumnStore, ReplicationMethod, ReplicationMethodMessage, type RunTrigger, type SafeColumnNameConverterFn, type Schema, SchemaMessage, SchemaValidationError, SftpService, type State, type StateHolder, StateMessage, type StateProvider, StateService, type StorageService, Stream, type StreamId, type StreamManifest, type StreamState, StreamWarehouseSyncService, type StreamWithTempFile, type SyncOptions, Tap, type TapState, type TargetSchemaHook, type TargetSchemaHookInput, type TempFile, type URLParams, VariableExtractors, type WarehouseTableField, type WebpTransformOptions, type WhalyDocument, WhalyDocumentService, type WhalyDocumentServiceConfig, WhalyDocumentTarget, type WhalyPaginatedResponse, type WhalyUploadResult, type WriteGeneratorToCsvOptions, _greaterThan, addDocumentSummaries, addWhalyFields, checkCsvHeaderRow, coerceCellValue, collectPaginated, countCsvLines, createCellReference, createCsvStreamConfig, createExcelGenerator, createExcelStreamConfig, createTemporaryFileStream, csvFieldsToJsonSchema, downloadFiles, emptyDocumentSummary, excelColumnToIndex, excelFieldsToJsonSchema, excelRowToIndex, extractExcelRows, extractPrimaryKeysFromCsvConfig, extractPrimaryKeysFromExcelFields, extractStateForStream, fieldTypeToJsonSchema, finalizeStateProgressMarkers, findAllMatchingConfigs, flattenSchema, getAllMetrics, getApiEndpoint, getAxiosInstance, getCdnId, getCounterMetric, getCounterMetrics, getDownloadFileApiCall, getDryRunLimit, getEmailMetadata, getExtension, getInputFiles, getJSONApiCall, getJSONApiCallWithFullResponse, getMimeType, getRawJSONApiCall, getRunTrigger, getServiceAccountKey, gracefulExit, haltAndCatchFire, incrementStreamState, indexToExcelColumn, indexToExcelRow, isDryRun, listFilesRecursively, loadJson, loadSchema, logger, optionalEnv, parseCellReference, postFormDataApiCall, postJSONApiCall, postUrlEncodedApiCall, printMemoryFootprint, processFileStreams, processFromAsyncIterable, removeParasiteProperties, requireEnv, rowGeneratorFromCsv, runWithConcurrency, safePath, unzip, validateExcelConfig, writeDataToCsv, writeGeneratorToCSV };
package/dist/index.d.ts CHANGED
@@ -153,6 +153,26 @@ declare function getMimeType(filePathOrExt: string): string;
153
153
  */
154
154
  declare function getExtension(mimeType: string): string;
155
155
 
156
+ /**
157
+ * What a Whaly FileUpload run hands its transformer: the files to process and, for runs started
158
+ * by an inbound email, who sent it and when.
159
+ */
160
+ interface EmailMetadata {
161
+ from?: string | null;
162
+ subject?: string | null;
163
+ receivedAt?: string | null;
164
+ }
165
+ type RunTrigger = "MANUAL" | "EMAIL" | "REPLAY";
166
+ /**
167
+ * The run's input files, as object keys in the job's storage (pass each to
168
+ * `storage.resolveFileUri`). Email runs list every kept attachment in `WLY_FILE_URIS`; manual
169
+ * uploads only set `FILE_URI`, so this works for both.
170
+ */
171
+ declare function getInputFiles(env?: NodeJS.ProcessEnv): string[];
172
+ /** Sender, subject and receive time of the email that started the run; undefined otherwise. */
173
+ declare function getEmailMetadata(env?: NodeJS.ProcessEnv): EmailMetadata | undefined;
174
+ declare function getRunTrigger(env?: NodeJS.ProcessEnv): RunTrigger;
175
+
156
176
  declare enum ReplicationMethod {
157
177
  INCREMENTAL = "INCREMENTAL",
158
178
  FULL_TABLE = "FULL_TABLE",
@@ -1029,6 +1049,16 @@ declare function createCsvStreamConfig(streamId: string, csvConfig: CsvFileConfi
1029
1049
  * @param target - The target (e.g. BigQueryTarget)
1030
1050
  */
1031
1051
  declare function processFileStreams(entries: FileStreamEntry[], tapState: InputTapState, target: ITarget): Promise<void>;
1052
+ /**
1053
+ * A run can process several files (one email, several attachments): says which one failed,
1054
+ * so the run's error is not just "the run failed".
1055
+ */
1056
+ declare class FileProcessingError extends Error {
1057
+ readonly filePath: string | string[];
1058
+ readonly streamId: string;
1059
+ readonly cause: unknown;
1060
+ constructor(filePath: string | string[], streamId: string, cause: unknown);
1061
+ }
1032
1062
 
1033
1063
  interface FileTapConfig {
1034
1064
  [key: string]: any;
@@ -1067,6 +1097,13 @@ declare class VariableExtractors {
1067
1097
  static regex(pattern: RegExp): (filename: string) => {
1068
1098
  [key: string]: string | undefined;
1069
1099
  };
1100
+ /**
1101
+ * Extracts named capture groups from the subject of the email that started the run, for
1102
+ * scheduled reports whose date only appears in the subject. Empty for manual uploads.
1103
+ */
1104
+ static emailSubject(pattern: RegExp): (filename: string) => {
1105
+ [key: string]: string | undefined;
1106
+ };
1070
1107
  /**
1071
1108
  * Combines multiple extraction functions.
1072
1109
  */
@@ -1741,4 +1778,4 @@ declare class GCSStateProvider implements StateProvider {
1741
1778
  writeState(state: string): Promise<void>;
1742
1779
  }
1743
1780
 
1744
- export { ALL_STREAM_ID_KEYWORD, API_CALLS_METRIC_NAME, type AssetEntry, type AssetManifest, type AssetManifestEntry, type AssetReplicationMode, AssetStream, AssetTap, AssetTarget, Authenticator, BATCH_INTERVAL_MS, type BaseConfig, type BigQueryConfig, BigQueryTarget, type BigqueryLocation, type Bookmark, type Bookmarks, CdnAssetTarget, type CdnAssetTargetConfig, type CdnFileMetadata, CdnService, type CdnServiceConfig, type CdnUploadResult, CloudStorageService, type CloudStorageServiceOptions, type ColumnMappingStoreUnsafeToSafe, CounterMetric, CsvExtractionConfigBuilder, type CsvFieldsArrayConfig, type CsvFieldsConfig, type CsvFieldsDictConfig, type CsvFileConfig, type CsvStreamConfig, DEFAULT_MAX_CONCURRENT_STREAMS, DEFAULT_SYNCED_AT_COLUMN, DocumentDownloadSkipError, type DocumentEntry, type DocumentManifest, type DocumentManifestEntry, type DocumentStatus, DocumentStream, type DocumentStreamManifest, type DocumentSummary, DocumentTap, type DownloadFilesOptions, EXECUTION_TIME_METRIC_NAME, type ErrorType, type ExcelCustomExtractorConfig, type ExcelDerivedField, type ExcelExtractionBaseConfig, type ExcelExtractionConfig, ExcelExtractionConfigBuilder, type ExcelFieldMapping, type ExcelFieldSpec, type ExcelSingleSheetExtractionConfig, type ExcelSourceColumn, type ExcelStreamConfig, type FieldType, FileFormat, FilePatterns, FileStream, type FileStreamConfig, type FileStreamEntry, FileTap, type FileTapConfig, type FlattenedSchema, GCSStateProvider, type GetConfig, type HTTPHeaders, type HTTPMethod, ITarget, ImageTransform, type InputTapState, type InternalTableField, type JSONSchemaFieldDefinition, type ListFilesOptions, LocalStorageService, type LocalStorageServiceOptions, MIME_TYPES, type Message, type MessageType, type MetricConfiguration, MissingFieldInSchemaError, MissingSchemaError, type PostConfig, type ProcessedAsset, type ProgressMarkers, type PropertiesMetadata, RESTStream, ROWS_SYNCED_METRIC_NAME, RecordMessage, RenameColumnStore, ReplicationMethod, ReplicationMethodMessage, type SafeColumnNameConverterFn, type Schema, SchemaMessage, SchemaValidationError, SftpService, type State, type StateHolder, StateMessage, type StateProvider, StateService, type StorageService, Stream, type StreamId, type StreamManifest, type StreamState, StreamWarehouseSyncService, type StreamWithTempFile, type SyncOptions, Tap, type TapState, type TargetSchemaHook, type TargetSchemaHookInput, type TempFile, type URLParams, VariableExtractors, type WarehouseTableField, type WebpTransformOptions, type WhalyDocument, WhalyDocumentService, type WhalyDocumentServiceConfig, WhalyDocumentTarget, type WhalyPaginatedResponse, type WhalyUploadResult, type WriteGeneratorToCsvOptions, _greaterThan, addDocumentSummaries, addWhalyFields, checkCsvHeaderRow, coerceCellValue, collectPaginated, countCsvLines, createCellReference, createCsvStreamConfig, createExcelGenerator, createExcelStreamConfig, createTemporaryFileStream, csvFieldsToJsonSchema, downloadFiles, emptyDocumentSummary, excelColumnToIndex, excelFieldsToJsonSchema, excelRowToIndex, extractExcelRows, extractPrimaryKeysFromCsvConfig, extractPrimaryKeysFromExcelFields, extractStateForStream, fieldTypeToJsonSchema, finalizeStateProgressMarkers, findAllMatchingConfigs, flattenSchema, getAllMetrics, getApiEndpoint, getAxiosInstance, getCdnId, getCounterMetric, getCounterMetrics, getDownloadFileApiCall, getDryRunLimit, getExtension, getJSONApiCall, getJSONApiCallWithFullResponse, getMimeType, getRawJSONApiCall, getServiceAccountKey, gracefulExit, haltAndCatchFire, incrementStreamState, indexToExcelColumn, indexToExcelRow, isDryRun, listFilesRecursively, loadJson, loadSchema, logger, optionalEnv, parseCellReference, postFormDataApiCall, postJSONApiCall, postUrlEncodedApiCall, printMemoryFootprint, processFileStreams, processFromAsyncIterable, removeParasiteProperties, requireEnv, rowGeneratorFromCsv, runWithConcurrency, safePath, unzip, validateExcelConfig, writeDataToCsv, writeGeneratorToCSV };
1781
+ export { ALL_STREAM_ID_KEYWORD, API_CALLS_METRIC_NAME, type AssetEntry, type AssetManifest, type AssetManifestEntry, type AssetReplicationMode, AssetStream, AssetTap, AssetTarget, Authenticator, BATCH_INTERVAL_MS, type BaseConfig, type BigQueryConfig, BigQueryTarget, type BigqueryLocation, type Bookmark, type Bookmarks, CdnAssetTarget, type CdnAssetTargetConfig, type CdnFileMetadata, CdnService, type CdnServiceConfig, type CdnUploadResult, CloudStorageService, type CloudStorageServiceOptions, type ColumnMappingStoreUnsafeToSafe, CounterMetric, CsvExtractionConfigBuilder, type CsvFieldsArrayConfig, type CsvFieldsConfig, type CsvFieldsDictConfig, type CsvFileConfig, type CsvStreamConfig, DEFAULT_MAX_CONCURRENT_STREAMS, DEFAULT_SYNCED_AT_COLUMN, DocumentDownloadSkipError, type DocumentEntry, type DocumentManifest, type DocumentManifestEntry, type DocumentStatus, DocumentStream, type DocumentStreamManifest, type DocumentSummary, DocumentTap, type DownloadFilesOptions, EXECUTION_TIME_METRIC_NAME, type EmailMetadata, type ErrorType, type ExcelCustomExtractorConfig, type ExcelDerivedField, type ExcelExtractionBaseConfig, type ExcelExtractionConfig, ExcelExtractionConfigBuilder, type ExcelFieldMapping, type ExcelFieldSpec, type ExcelSingleSheetExtractionConfig, type ExcelSourceColumn, type ExcelStreamConfig, type FieldType, FileFormat, FilePatterns, FileProcessingError, FileStream, type FileStreamConfig, type FileStreamEntry, FileTap, type FileTapConfig, type FlattenedSchema, GCSStateProvider, type GetConfig, type HTTPHeaders, type HTTPMethod, ITarget, ImageTransform, type InputTapState, type InternalTableField, type JSONSchemaFieldDefinition, type ListFilesOptions, LocalStorageService, type LocalStorageServiceOptions, MIME_TYPES, type Message, type MessageType, type MetricConfiguration, MissingFieldInSchemaError, MissingSchemaError, type PostConfig, type ProcessedAsset, type ProgressMarkers, type PropertiesMetadata, RESTStream, ROWS_SYNCED_METRIC_NAME, RecordMessage, RenameColumnStore, ReplicationMethod, ReplicationMethodMessage, type RunTrigger, type SafeColumnNameConverterFn, type Schema, SchemaMessage, SchemaValidationError, SftpService, type State, type StateHolder, StateMessage, type StateProvider, StateService, type StorageService, Stream, type StreamId, type StreamManifest, type StreamState, StreamWarehouseSyncService, type StreamWithTempFile, type SyncOptions, Tap, type TapState, type TargetSchemaHook, type TargetSchemaHookInput, type TempFile, type URLParams, VariableExtractors, type WarehouseTableField, type WebpTransformOptions, type WhalyDocument, WhalyDocumentService, type WhalyDocumentServiceConfig, WhalyDocumentTarget, type WhalyPaginatedResponse, type WhalyUploadResult, type WriteGeneratorToCsvOptions, _greaterThan, addDocumentSummaries, addWhalyFields, checkCsvHeaderRow, coerceCellValue, collectPaginated, countCsvLines, createCellReference, createCsvStreamConfig, createExcelGenerator, createExcelStreamConfig, createTemporaryFileStream, csvFieldsToJsonSchema, downloadFiles, emptyDocumentSummary, excelColumnToIndex, excelFieldsToJsonSchema, excelRowToIndex, extractExcelRows, extractPrimaryKeysFromCsvConfig, extractPrimaryKeysFromExcelFields, extractStateForStream, fieldTypeToJsonSchema, finalizeStateProgressMarkers, findAllMatchingConfigs, flattenSchema, getAllMetrics, getApiEndpoint, getAxiosInstance, getCdnId, getCounterMetric, getCounterMetrics, getDownloadFileApiCall, getDryRunLimit, getEmailMetadata, getExtension, getInputFiles, getJSONApiCall, getJSONApiCallWithFullResponse, getMimeType, getRawJSONApiCall, getRunTrigger, getServiceAccountKey, gracefulExit, haltAndCatchFire, incrementStreamState, indexToExcelColumn, indexToExcelRow, isDryRun, listFilesRecursively, loadJson, loadSchema, logger, optionalEnv, parseCellReference, postFormDataApiCall, postJSONApiCall, postUrlEncodedApiCall, printMemoryFootprint, processFileStreams, processFromAsyncIterable, removeParasiteProperties, requireEnv, rowGeneratorFromCsv, runWithConcurrency, safePath, unzip, validateExcelConfig, writeDataToCsv, writeGeneratorToCSV };
package/dist/index.js CHANGED
@@ -10337,6 +10337,7 @@ __export(index_exports, {
10337
10337
  ExcelExtractionConfigBuilder: () => ExcelExtractionConfigBuilder,
10338
10338
  FileFormat: () => FileFormat,
10339
10339
  FilePatterns: () => FilePatterns,
10340
+ FileProcessingError: () => FileProcessingError,
10340
10341
  FileStream: () => FileStream,
10341
10342
  FileTap: () => FileTap,
10342
10343
  GCSStateProvider: () => GCSStateProvider,
@@ -10398,11 +10399,14 @@ __export(index_exports, {
10398
10399
  getCounterMetrics: () => getCounterMetrics,
10399
10400
  getDownloadFileApiCall: () => getDownloadFileApiCall,
10400
10401
  getDryRunLimit: () => getDryRunLimit,
10402
+ getEmailMetadata: () => getEmailMetadata,
10401
10403
  getExtension: () => getExtension,
10404
+ getInputFiles: () => getInputFiles,
10402
10405
  getJSONApiCall: () => getJSONApiCall,
10403
10406
  getJSONApiCallWithFullResponse: () => getJSONApiCallWithFullResponse,
10404
10407
  getMimeType: () => getMimeType,
10405
10408
  getRawJSONApiCall: () => getRawJSONApiCall,
10409
+ getRunTrigger: () => getRunTrigger,
10406
10410
  getServiceAccountKey: () => getServiceAccountKey,
10407
10411
  gracefulExit: () => gracefulExit,
10408
10412
  haltAndCatchFire: () => haltAndCatchFire,
@@ -10986,6 +10990,32 @@ function getExtension(mimeType) {
10986
10990
  return EXTENSION_BY_MIME[mimeType] ?? ".bin";
10987
10991
  }
10988
10992
 
10993
+ // src/sdk/service/runInput.ts
10994
+ function getInputFiles(env = process.env) {
10995
+ const list = env.WLY_FILE_URIS;
10996
+ if (list) {
10997
+ const parsed = JSON.parse(list);
10998
+ if (!Array.isArray(parsed) || !parsed.every((item) => typeof item === "string")) {
10999
+ throw new Error("WLY_FILE_URIS must be a JSON array of strings");
11000
+ }
11001
+ if (parsed.length > 0) {
11002
+ return parsed;
11003
+ }
11004
+ }
11005
+ return env.FILE_URI ? [env.FILE_URI] : [];
11006
+ }
11007
+ function getEmailMetadata(env = process.env) {
11008
+ const raw = env.WLY_EMAIL_METADATA;
11009
+ if (!raw) {
11010
+ return void 0;
11011
+ }
11012
+ return JSON.parse(raw);
11013
+ }
11014
+ function getRunTrigger(env = process.env) {
11015
+ const trigger = env.WLY_RUN_TRIGGER;
11016
+ return trigger === "EMAIL" || trigger === "REPLAY" ? trigger : "MANUAL";
11017
+ }
11018
+
10989
11019
  // src/sdk/models/replication.ts
10990
11020
  var ReplicationMethod = /* @__PURE__ */ ((ReplicationMethod2) => {
10991
11021
  ReplicationMethod2["INCREMENTAL"] = "INCREMENTAL";
@@ -11317,6 +11347,20 @@ var Tap = class {
11317
11347
 
11318
11348
  // src/sdk/models/tap/stream.ts
11319
11349
  var import_util2 = require("util");
11350
+ var DEFAULT_PROGRESS_LOG_PERCENT_STEP = 10;
11351
+ var DEFAULT_PROGRESS_LOG_FALLBACK_INTERVAL = 1e5;
11352
+ function getProgressLogPercentStep() {
11353
+ const raw = optionalEnv("PROGRESS_LOG_PERCENT_STEP", String(DEFAULT_PROGRESS_LOG_PERCENT_STEP));
11354
+ const n = parseInt(raw, 10);
11355
+ if (isNaN(n) || n <= 0) return DEFAULT_PROGRESS_LOG_PERCENT_STEP;
11356
+ return Math.min(n, 100);
11357
+ }
11358
+ function getProgressLogFallbackInterval() {
11359
+ const raw = optionalEnv("PROGRESS_LOG_FALLBACK_INTERVAL", String(DEFAULT_PROGRESS_LOG_FALLBACK_INTERVAL));
11360
+ const n = parseInt(raw, 10);
11361
+ if (isNaN(n) || n <= 0) return DEFAULT_PROGRESS_LOG_FALLBACK_INTERVAL;
11362
+ return n;
11363
+ }
11320
11364
  var Stream = class {
11321
11365
  // Runtime values, can't be overriden
11322
11366
  config;
@@ -11579,6 +11623,9 @@ var Stream = class {
11579
11623
  // Private sync methods:
11580
11624
  async _syncRecords(parent) {
11581
11625
  let rowsSent = 0;
11626
+ let lastLoggedPercent = 0;
11627
+ const progressPercentStep = getProgressLogPercentStep();
11628
+ const progressFallbackInterval = getProgressLogFallbackInterval();
11582
11629
  const dryRunLimit = isDryRun() ? getDryRunLimit() : void 0;
11583
11630
  let recordSyncedMetrics = [];
11584
11631
  if (this.rowsSyncedMetricsConf.isEnabled() === true) {
@@ -11625,9 +11672,14 @@ var Stream = class {
11625
11672
  logger.info(`\u{1F6D1} Stream: ${this.streamId} - DRY_RUN_LIMIT reached (${dryRunLimit} records), stopping.`);
11626
11673
  break;
11627
11674
  }
11628
- if (rowsSent % 1e3 === 0) {
11629
- const percentStr = this.totalRows ? ` (${Math.round(rowsSent / this.totalRows * 100)}%)` : "";
11630
- logger.info(`\u23F3 Stream: ${this.streamId} - ${rowsSent} records synced so far...${percentStr}`);
11675
+ if (this.totalRows) {
11676
+ const percent = Math.floor(rowsSent / this.totalRows * 100);
11677
+ if (percent >= lastLoggedPercent + progressPercentStep) {
11678
+ lastLoggedPercent = percent - percent % progressPercentStep;
11679
+ logger.info(`\u23F3 Stream: ${this.streamId} - ${rowsSent} records synced so far... (${percent}%)`);
11680
+ }
11681
+ } else if (rowsSent % progressFallbackInterval === 0) {
11682
+ logger.info(`\u23F3 Stream: ${this.streamId} - ${rowsSent} records synced so far...`);
11631
11683
  }
11632
11684
  }
11633
11685
  const stateServiceInst = StateService.getInstance();
@@ -13554,10 +13606,24 @@ function createCsvStreamConfig(streamId, csvConfig, options) {
13554
13606
  async function processFileStreams(entries, tapState, target) {
13555
13607
  for (const entry of entries) {
13556
13608
  const stream = new FileStream(entry.config, entry.filePath, tapState, target);
13557
- await stream.sync();
13609
+ try {
13610
+ await stream.sync();
13611
+ } catch (err) {
13612
+ throw new FileProcessingError(entry.filePath, entry.config.streamId, err);
13613
+ }
13558
13614
  }
13559
13615
  await target.complete();
13560
13616
  }
13617
+ var FileProcessingError = class extends Error {
13618
+ constructor(filePath, streamId, cause) {
13619
+ const files = Array.isArray(filePath) ? filePath.join(", ") : filePath;
13620
+ super(`Processing ${files} (stream ${streamId}) failed: ${cause instanceof Error ? cause.message : String(cause)}`);
13621
+ this.filePath = filePath;
13622
+ this.streamId = streamId;
13623
+ this.cause = cause;
13624
+ this.name = "FileProcessingError";
13625
+ }
13626
+ };
13561
13627
 
13562
13628
  // src/sdk/file-processing/csv/config-builder.ts
13563
13629
  var CsvExtractionConfigBuilder = class _CsvExtractionConfigBuilder {
@@ -13683,6 +13749,16 @@ var VariableExtractors = class {
13683
13749
  return match?.groups ?? {};
13684
13750
  };
13685
13751
  }
13752
+ /**
13753
+ * Extracts named capture groups from the subject of the email that started the run, for
13754
+ * scheduled reports whose date only appears in the subject. Empty for manual uploads.
13755
+ */
13756
+ static emailSubject(pattern) {
13757
+ return () => {
13758
+ const subject = getEmailMetadata()?.subject;
13759
+ return subject ? subject.match(pattern)?.groups ?? {} : {};
13760
+ };
13761
+ }
13686
13762
  /**
13687
13763
  * Combines multiple extraction functions.
13688
13764
  */
@@ -15678,6 +15754,7 @@ var GCSStateProvider = class {
15678
15754
  ExcelExtractionConfigBuilder,
15679
15755
  FileFormat,
15680
15756
  FilePatterns,
15757
+ FileProcessingError,
15681
15758
  FileStream,
15682
15759
  FileTap,
15683
15760
  GCSStateProvider,
@@ -15739,11 +15816,14 @@ var GCSStateProvider = class {
15739
15816
  getCounterMetrics,
15740
15817
  getDownloadFileApiCall,
15741
15818
  getDryRunLimit,
15819
+ getEmailMetadata,
15742
15820
  getExtension,
15821
+ getInputFiles,
15743
15822
  getJSONApiCall,
15744
15823
  getJSONApiCallWithFullResponse,
15745
15824
  getMimeType,
15746
15825
  getRawJSONApiCall,
15826
+ getRunTrigger,
15747
15827
  getServiceAccountKey,
15748
15828
  gracefulExit,
15749
15829
  haltAndCatchFire,