@whaly/connector-sdk 0.3.12 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.mjs CHANGED
@@ -10864,6 +10864,32 @@ function getExtension(mimeType) {
10864
10864
  return EXTENSION_BY_MIME[mimeType] ?? ".bin";
10865
10865
  }
10866
10866
 
10867
+ // src/sdk/service/runInput.ts
10868
+ function getInputFiles(env = process.env) {
10869
+ const list = env.WLY_FILE_URIS;
10870
+ if (list) {
10871
+ const parsed = JSON.parse(list);
10872
+ if (!Array.isArray(parsed) || !parsed.every((item) => typeof item === "string")) {
10873
+ throw new Error("WLY_FILE_URIS must be a JSON array of strings");
10874
+ }
10875
+ if (parsed.length > 0) {
10876
+ return parsed;
10877
+ }
10878
+ }
10879
+ return env.FILE_URI ? [env.FILE_URI] : [];
10880
+ }
10881
+ function getEmailMetadata(env = process.env) {
10882
+ const raw = env.WLY_EMAIL_METADATA;
10883
+ if (!raw) {
10884
+ return void 0;
10885
+ }
10886
+ return JSON.parse(raw);
10887
+ }
10888
+ function getRunTrigger(env = process.env) {
10889
+ const trigger = env.WLY_RUN_TRIGGER;
10890
+ return trigger === "EMAIL" || trigger === "REPLAY" ? trigger : "MANUAL";
10891
+ }
10892
+
10867
10893
  // src/sdk/models/replication.ts
10868
10894
  var ReplicationMethod = /* @__PURE__ */ ((ReplicationMethod2) => {
10869
10895
  ReplicationMethod2["INCREMENTAL"] = "INCREMENTAL";
@@ -11195,6 +11221,20 @@ var Tap = class {
11195
11221
 
11196
11222
  // src/sdk/models/tap/stream.ts
11197
11223
  import { format as format3 } from "util";
11224
+ var DEFAULT_PROGRESS_LOG_PERCENT_STEP = 10;
11225
+ var DEFAULT_PROGRESS_LOG_FALLBACK_INTERVAL = 1e5;
11226
+ function getProgressLogPercentStep() {
11227
+ const raw = optionalEnv("PROGRESS_LOG_PERCENT_STEP", String(DEFAULT_PROGRESS_LOG_PERCENT_STEP));
11228
+ const n = parseInt(raw, 10);
11229
+ if (isNaN(n) || n <= 0) return DEFAULT_PROGRESS_LOG_PERCENT_STEP;
11230
+ return Math.min(n, 100);
11231
+ }
11232
+ function getProgressLogFallbackInterval() {
11233
+ const raw = optionalEnv("PROGRESS_LOG_FALLBACK_INTERVAL", String(DEFAULT_PROGRESS_LOG_FALLBACK_INTERVAL));
11234
+ const n = parseInt(raw, 10);
11235
+ if (isNaN(n) || n <= 0) return DEFAULT_PROGRESS_LOG_FALLBACK_INTERVAL;
11236
+ return n;
11237
+ }
11198
11238
  var Stream = class {
11199
11239
  // Runtime values, can't be overriden
11200
11240
  config;
@@ -11457,6 +11497,9 @@ var Stream = class {
11457
11497
  // Private sync methods:
11458
11498
  async _syncRecords(parent) {
11459
11499
  let rowsSent = 0;
11500
+ let lastLoggedPercent = 0;
11501
+ const progressPercentStep = getProgressLogPercentStep();
11502
+ const progressFallbackInterval = getProgressLogFallbackInterval();
11460
11503
  const dryRunLimit = isDryRun() ? getDryRunLimit() : void 0;
11461
11504
  let recordSyncedMetrics = [];
11462
11505
  if (this.rowsSyncedMetricsConf.isEnabled() === true) {
@@ -11503,9 +11546,14 @@ var Stream = class {
11503
11546
  logger.info(`\u{1F6D1} Stream: ${this.streamId} - DRY_RUN_LIMIT reached (${dryRunLimit} records), stopping.`);
11504
11547
  break;
11505
11548
  }
11506
- if (rowsSent % 1e3 === 0) {
11507
- const percentStr = this.totalRows ? ` (${Math.round(rowsSent / this.totalRows * 100)}%)` : "";
11508
- logger.info(`\u23F3 Stream: ${this.streamId} - ${rowsSent} records synced so far...${percentStr}`);
11549
+ if (this.totalRows) {
11550
+ const percent = Math.floor(rowsSent / this.totalRows * 100);
11551
+ if (percent >= lastLoggedPercent + progressPercentStep) {
11552
+ lastLoggedPercent = percent - percent % progressPercentStep;
11553
+ logger.info(`\u23F3 Stream: ${this.streamId} - ${rowsSent} records synced so far... (${percent}%)`);
11554
+ }
11555
+ } else if (rowsSent % progressFallbackInterval === 0) {
11556
+ logger.info(`\u23F3 Stream: ${this.streamId} - ${rowsSent} records synced so far...`);
11509
11557
  }
11510
11558
  }
11511
11559
  const stateServiceInst = StateService.getInstance();
@@ -13432,10 +13480,24 @@ function createCsvStreamConfig(streamId, csvConfig, options) {
13432
13480
  async function processFileStreams(entries, tapState, target) {
13433
13481
  for (const entry of entries) {
13434
13482
  const stream = new FileStream(entry.config, entry.filePath, tapState, target);
13435
- await stream.sync();
13483
+ try {
13484
+ await stream.sync();
13485
+ } catch (err) {
13486
+ throw new FileProcessingError(entry.filePath, entry.config.streamId, err);
13487
+ }
13436
13488
  }
13437
13489
  await target.complete();
13438
13490
  }
13491
+ var FileProcessingError = class extends Error {
13492
+ constructor(filePath, streamId, cause) {
13493
+ const files = Array.isArray(filePath) ? filePath.join(", ") : filePath;
13494
+ super(`Processing ${files} (stream ${streamId}) failed: ${cause instanceof Error ? cause.message : String(cause)}`);
13495
+ this.filePath = filePath;
13496
+ this.streamId = streamId;
13497
+ this.cause = cause;
13498
+ this.name = "FileProcessingError";
13499
+ }
13500
+ };
13439
13501
 
13440
13502
  // src/sdk/file-processing/csv/config-builder.ts
13441
13503
  var CsvExtractionConfigBuilder = class _CsvExtractionConfigBuilder {
@@ -13561,6 +13623,16 @@ var VariableExtractors = class {
13561
13623
  return match?.groups ?? {};
13562
13624
  };
13563
13625
  }
13626
+ /**
13627
+ * Extracts named capture groups from the subject of the email that started the run, for
13628
+ * scheduled reports whose date only appears in the subject. Empty for manual uploads.
13629
+ */
13630
+ static emailSubject(pattern) {
13631
+ return () => {
13632
+ const subject = getEmailMetadata()?.subject;
13633
+ return subject ? subject.match(pattern)?.groups ?? {} : {};
13634
+ };
13635
+ }
13564
13636
  /**
13565
13637
  * Combines multiple extraction functions.
13566
13638
  */
@@ -15555,6 +15627,7 @@ export {
15555
15627
  ExcelExtractionConfigBuilder,
15556
15628
  FileFormat,
15557
15629
  FilePatterns,
15630
+ FileProcessingError,
15558
15631
  FileStream,
15559
15632
  FileTap,
15560
15633
  GCSStateProvider,
@@ -15616,11 +15689,14 @@ export {
15616
15689
  getCounterMetrics,
15617
15690
  getDownloadFileApiCall,
15618
15691
  getDryRunLimit,
15692
+ getEmailMetadata,
15619
15693
  getExtension,
15694
+ getInputFiles,
15620
15695
  getJSONApiCall,
15621
15696
  getJSONApiCallWithFullResponse,
15622
15697
  getMimeType,
15623
15698
  getRawJSONApiCall,
15699
+ getRunTrigger,
15624
15700
  getServiceAccountKey,
15625
15701
  gracefulExit,
15626
15702
  haltAndCatchFire,