@whaly/connector-sdk 0.3.12 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +38 -1
- package/dist/index.d.ts +38 -1
- package/dist/index.js +84 -4
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +80 -4
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/index.mjs
CHANGED
|
@@ -10864,6 +10864,32 @@ function getExtension(mimeType) {
|
|
|
10864
10864
|
return EXTENSION_BY_MIME[mimeType] ?? ".bin";
|
|
10865
10865
|
}
|
|
10866
10866
|
|
|
10867
|
+
// src/sdk/service/runInput.ts
|
|
10868
|
+
function getInputFiles(env = process.env) {
|
|
10869
|
+
const list = env.WLY_FILE_URIS;
|
|
10870
|
+
if (list) {
|
|
10871
|
+
const parsed = JSON.parse(list);
|
|
10872
|
+
if (!Array.isArray(parsed) || !parsed.every((item) => typeof item === "string")) {
|
|
10873
|
+
throw new Error("WLY_FILE_URIS must be a JSON array of strings");
|
|
10874
|
+
}
|
|
10875
|
+
if (parsed.length > 0) {
|
|
10876
|
+
return parsed;
|
|
10877
|
+
}
|
|
10878
|
+
}
|
|
10879
|
+
return env.FILE_URI ? [env.FILE_URI] : [];
|
|
10880
|
+
}
|
|
10881
|
+
function getEmailMetadata(env = process.env) {
|
|
10882
|
+
const raw = env.WLY_EMAIL_METADATA;
|
|
10883
|
+
if (!raw) {
|
|
10884
|
+
return void 0;
|
|
10885
|
+
}
|
|
10886
|
+
return JSON.parse(raw);
|
|
10887
|
+
}
|
|
10888
|
+
function getRunTrigger(env = process.env) {
|
|
10889
|
+
const trigger = env.WLY_RUN_TRIGGER;
|
|
10890
|
+
return trigger === "EMAIL" || trigger === "REPLAY" ? trigger : "MANUAL";
|
|
10891
|
+
}
|
|
10892
|
+
|
|
10867
10893
|
// src/sdk/models/replication.ts
|
|
10868
10894
|
var ReplicationMethod = /* @__PURE__ */ ((ReplicationMethod2) => {
|
|
10869
10895
|
ReplicationMethod2["INCREMENTAL"] = "INCREMENTAL";
|
|
@@ -11195,6 +11221,20 @@ var Tap = class {
|
|
|
11195
11221
|
|
|
11196
11222
|
// src/sdk/models/tap/stream.ts
|
|
11197
11223
|
import { format as format3 } from "util";
|
|
11224
|
+
var DEFAULT_PROGRESS_LOG_PERCENT_STEP = 10;
|
|
11225
|
+
var DEFAULT_PROGRESS_LOG_FALLBACK_INTERVAL = 1e5;
|
|
11226
|
+
function getProgressLogPercentStep() {
|
|
11227
|
+
const raw = optionalEnv("PROGRESS_LOG_PERCENT_STEP", String(DEFAULT_PROGRESS_LOG_PERCENT_STEP));
|
|
11228
|
+
const n = parseInt(raw, 10);
|
|
11229
|
+
if (isNaN(n) || n <= 0) return DEFAULT_PROGRESS_LOG_PERCENT_STEP;
|
|
11230
|
+
return Math.min(n, 100);
|
|
11231
|
+
}
|
|
11232
|
+
function getProgressLogFallbackInterval() {
|
|
11233
|
+
const raw = optionalEnv("PROGRESS_LOG_FALLBACK_INTERVAL", String(DEFAULT_PROGRESS_LOG_FALLBACK_INTERVAL));
|
|
11234
|
+
const n = parseInt(raw, 10);
|
|
11235
|
+
if (isNaN(n) || n <= 0) return DEFAULT_PROGRESS_LOG_FALLBACK_INTERVAL;
|
|
11236
|
+
return n;
|
|
11237
|
+
}
|
|
11198
11238
|
var Stream = class {
|
|
11199
11239
|
// Runtime values, can't be overriden
|
|
11200
11240
|
config;
|
|
@@ -11457,6 +11497,9 @@ var Stream = class {
|
|
|
11457
11497
|
// Private sync methods:
|
|
11458
11498
|
async _syncRecords(parent) {
|
|
11459
11499
|
let rowsSent = 0;
|
|
11500
|
+
let lastLoggedPercent = 0;
|
|
11501
|
+
const progressPercentStep = getProgressLogPercentStep();
|
|
11502
|
+
const progressFallbackInterval = getProgressLogFallbackInterval();
|
|
11460
11503
|
const dryRunLimit = isDryRun() ? getDryRunLimit() : void 0;
|
|
11461
11504
|
let recordSyncedMetrics = [];
|
|
11462
11505
|
if (this.rowsSyncedMetricsConf.isEnabled() === true) {
|
|
@@ -11503,9 +11546,14 @@ var Stream = class {
|
|
|
11503
11546
|
logger.info(`\u{1F6D1} Stream: ${this.streamId} - DRY_RUN_LIMIT reached (${dryRunLimit} records), stopping.`);
|
|
11504
11547
|
break;
|
|
11505
11548
|
}
|
|
11506
|
-
if (
|
|
11507
|
-
const
|
|
11508
|
-
|
|
11549
|
+
if (this.totalRows) {
|
|
11550
|
+
const percent = Math.floor(rowsSent / this.totalRows * 100);
|
|
11551
|
+
if (percent >= lastLoggedPercent + progressPercentStep) {
|
|
11552
|
+
lastLoggedPercent = percent - percent % progressPercentStep;
|
|
11553
|
+
logger.info(`\u23F3 Stream: ${this.streamId} - ${rowsSent} records synced so far... (${percent}%)`);
|
|
11554
|
+
}
|
|
11555
|
+
} else if (rowsSent % progressFallbackInterval === 0) {
|
|
11556
|
+
logger.info(`\u23F3 Stream: ${this.streamId} - ${rowsSent} records synced so far...`);
|
|
11509
11557
|
}
|
|
11510
11558
|
}
|
|
11511
11559
|
const stateServiceInst = StateService.getInstance();
|
|
@@ -13432,10 +13480,24 @@ function createCsvStreamConfig(streamId, csvConfig, options) {
|
|
|
13432
13480
|
async function processFileStreams(entries, tapState, target) {
|
|
13433
13481
|
for (const entry of entries) {
|
|
13434
13482
|
const stream = new FileStream(entry.config, entry.filePath, tapState, target);
|
|
13435
|
-
|
|
13483
|
+
try {
|
|
13484
|
+
await stream.sync();
|
|
13485
|
+
} catch (err) {
|
|
13486
|
+
throw new FileProcessingError(entry.filePath, entry.config.streamId, err);
|
|
13487
|
+
}
|
|
13436
13488
|
}
|
|
13437
13489
|
await target.complete();
|
|
13438
13490
|
}
|
|
13491
|
+
var FileProcessingError = class extends Error {
|
|
13492
|
+
constructor(filePath, streamId, cause) {
|
|
13493
|
+
const files = Array.isArray(filePath) ? filePath.join(", ") : filePath;
|
|
13494
|
+
super(`Processing ${files} (stream ${streamId}) failed: ${cause instanceof Error ? cause.message : String(cause)}`);
|
|
13495
|
+
this.filePath = filePath;
|
|
13496
|
+
this.streamId = streamId;
|
|
13497
|
+
this.cause = cause;
|
|
13498
|
+
this.name = "FileProcessingError";
|
|
13499
|
+
}
|
|
13500
|
+
};
|
|
13439
13501
|
|
|
13440
13502
|
// src/sdk/file-processing/csv/config-builder.ts
|
|
13441
13503
|
var CsvExtractionConfigBuilder = class _CsvExtractionConfigBuilder {
|
|
@@ -13561,6 +13623,16 @@ var VariableExtractors = class {
|
|
|
13561
13623
|
return match?.groups ?? {};
|
|
13562
13624
|
};
|
|
13563
13625
|
}
|
|
13626
|
+
/**
|
|
13627
|
+
* Extracts named capture groups from the subject of the email that started the run, for
|
|
13628
|
+
* scheduled reports whose date only appears in the subject. Empty for manual uploads.
|
|
13629
|
+
*/
|
|
13630
|
+
static emailSubject(pattern) {
|
|
13631
|
+
return () => {
|
|
13632
|
+
const subject = getEmailMetadata()?.subject;
|
|
13633
|
+
return subject ? subject.match(pattern)?.groups ?? {} : {};
|
|
13634
|
+
};
|
|
13635
|
+
}
|
|
13564
13636
|
/**
|
|
13565
13637
|
* Combines multiple extraction functions.
|
|
13566
13638
|
*/
|
|
@@ -15555,6 +15627,7 @@ export {
|
|
|
15555
15627
|
ExcelExtractionConfigBuilder,
|
|
15556
15628
|
FileFormat,
|
|
15557
15629
|
FilePatterns,
|
|
15630
|
+
FileProcessingError,
|
|
15558
15631
|
FileStream,
|
|
15559
15632
|
FileTap,
|
|
15560
15633
|
GCSStateProvider,
|
|
@@ -15616,11 +15689,14 @@ export {
|
|
|
15616
15689
|
getCounterMetrics,
|
|
15617
15690
|
getDownloadFileApiCall,
|
|
15618
15691
|
getDryRunLimit,
|
|
15692
|
+
getEmailMetadata,
|
|
15619
15693
|
getExtension,
|
|
15694
|
+
getInputFiles,
|
|
15620
15695
|
getJSONApiCall,
|
|
15621
15696
|
getJSONApiCallWithFullResponse,
|
|
15622
15697
|
getMimeType,
|
|
15623
15698
|
getRawJSONApiCall,
|
|
15699
|
+
getRunTrigger,
|
|
15624
15700
|
getServiceAccountKey,
|
|
15625
15701
|
gracefulExit,
|
|
15626
15702
|
haltAndCatchFire,
|