@walkeros/cli 4.2.0 → 4.2.1-next-1781538735002
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +39 -0
- package/dist/cli.js +822 -381
- package/dist/index.d.ts +879 -114
- package/dist/index.js +844 -399
- package/dist/index.js.map +1 -1
- package/openapi/spec.json +2543 -1442
- package/package.json +8 -8
package/dist/cli.js
CHANGED
|
@@ -31,25 +31,62 @@ var init_client_context = __esm({
|
|
|
31
31
|
});
|
|
32
32
|
|
|
33
33
|
// src/core/api-error.ts
|
|
34
|
-
function
|
|
35
|
-
if (error
|
|
36
|
-
|
|
37
|
-
const message = inner.message || fallbackMessage;
|
|
38
|
-
const code = inner.code;
|
|
39
|
-
const details = inner.details?.errors;
|
|
40
|
-
const options = { code, details };
|
|
41
|
-
if (code === "CLIENT_OUTDATED") {
|
|
42
|
-
options.minVersion = inner.minVersion;
|
|
43
|
-
options.clientVersion = inner.clientVersion;
|
|
44
|
-
options.client = inner.client;
|
|
45
|
-
options.upgrade = inner.upgrade;
|
|
46
|
-
options.docs = inner.docs;
|
|
47
|
-
}
|
|
48
|
-
throw new ApiError(message, options);
|
|
34
|
+
function extractApiErrorOptions(error, fallbackMessage) {
|
|
35
|
+
if (!error || typeof error !== "object" || !("error" in error) || typeof error.error !== "object") {
|
|
36
|
+
return null;
|
|
49
37
|
}
|
|
38
|
+
const inner = error.error;
|
|
39
|
+
const message = inner.message || fallbackMessage;
|
|
40
|
+
const code = inner.code;
|
|
41
|
+
const details = inner.details?.errors;
|
|
42
|
+
const options = { code, details };
|
|
43
|
+
if (code === "CLIENT_OUTDATED") {
|
|
44
|
+
options.minVersion = inner.minVersion;
|
|
45
|
+
options.clientVersion = inner.clientVersion;
|
|
46
|
+
options.client = inner.client;
|
|
47
|
+
options.upgrade = inner.upgrade;
|
|
48
|
+
options.docs = inner.docs;
|
|
49
|
+
}
|
|
50
|
+
return { message, options };
|
|
51
|
+
}
|
|
52
|
+
function throwApiError(error, fallbackMessage) {
|
|
53
|
+
const extracted = extractApiErrorOptions(error, fallbackMessage);
|
|
54
|
+
if (extracted) throw new ApiError(extracted.message, extracted.options);
|
|
50
55
|
throw new ApiError(fallbackMessage);
|
|
51
56
|
}
|
|
57
|
+
function parseRetryAfter(value) {
|
|
58
|
+
if (!value) return void 0;
|
|
59
|
+
const trimmed = value.trim();
|
|
60
|
+
if (/^\d+$/.test(trimmed)) return parseInt(trimmed, 10);
|
|
61
|
+
const when = Date.parse(trimmed);
|
|
62
|
+
if (Number.isNaN(when)) return void 0;
|
|
63
|
+
return Math.max(0, Math.round((when - Date.now()) / 1e3));
|
|
64
|
+
}
|
|
65
|
+
function throwApiResponseError(response, body, fallbackMessage) {
|
|
66
|
+
const status = response.status;
|
|
67
|
+
const retryAfterSeconds = parseRetryAfter(
|
|
68
|
+
response.headers.get("retry-after")
|
|
69
|
+
);
|
|
70
|
+
const retryable = status === 429 || status === 503 && retryAfterSeconds !== void 0;
|
|
71
|
+
const extracted = extractApiErrorOptions(body, fallbackMessage);
|
|
72
|
+
const message = extracted?.message ?? fallbackMessage;
|
|
73
|
+
const options = {
|
|
74
|
+
...extracted?.options ?? {},
|
|
75
|
+
status,
|
|
76
|
+
retryable,
|
|
77
|
+
retryAfterSeconds
|
|
78
|
+
};
|
|
79
|
+
throw new ApiError(message, options);
|
|
80
|
+
}
|
|
81
|
+
function machineReadableErrorLine(err) {
|
|
82
|
+
const code = err instanceof ApiError ? err.code ?? "UNKNOWN" : "UNKNOWN";
|
|
83
|
+
const retryable = err instanceof ApiError ? err.retryable === true : false;
|
|
84
|
+
const retryAfter = err instanceof ApiError && err.retryAfterSeconds !== void 0 ? ` retryAfter=${err.retryAfterSeconds}` : "";
|
|
85
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
86
|
+
return `error: code=${code} retryable=${retryable}${retryAfter} message=${message}`;
|
|
87
|
+
}
|
|
52
88
|
function handleCliError(err) {
|
|
89
|
+
console.error(machineReadableErrorLine(err));
|
|
53
90
|
if (err instanceof ApiError && err.code === "CLIENT_OUTDATED") {
|
|
54
91
|
console.error(`
|
|
55
92
|
${err.message}
|
|
@@ -59,17 +96,25 @@ ${err.message}
|
|
|
59
96
|
`);
|
|
60
97
|
process.exit(2);
|
|
61
98
|
}
|
|
62
|
-
|
|
63
|
-
|
|
99
|
+
if (err instanceof ApiError && err.retryable) {
|
|
100
|
+
const hint = err.retryAfterSeconds !== void 0 ? ` Retry after ${err.retryAfterSeconds}s.` : " This is temporary, retry shortly.";
|
|
101
|
+
console.error(`${err.message}${hint}`);
|
|
102
|
+
process.exit(EXIT_RETRYABLE);
|
|
103
|
+
}
|
|
104
|
+
console.error(err instanceof Error ? err.message : String(err));
|
|
64
105
|
process.exit(1);
|
|
65
106
|
}
|
|
66
|
-
var ApiError;
|
|
107
|
+
var EXIT_RETRYABLE, ApiError;
|
|
67
108
|
var init_api_error = __esm({
|
|
68
109
|
"src/core/api-error.ts"() {
|
|
69
110
|
"use strict";
|
|
111
|
+
EXIT_RETRYABLE = 75;
|
|
70
112
|
ApiError = class extends Error {
|
|
71
113
|
code;
|
|
72
114
|
details;
|
|
115
|
+
status;
|
|
116
|
+
retryable;
|
|
117
|
+
retryAfterSeconds;
|
|
73
118
|
// Populated only for CLIENT_OUTDATED responses (HTTP 426).
|
|
74
119
|
minVersion;
|
|
75
120
|
clientVersion;
|
|
@@ -81,6 +126,9 @@ var init_api_error = __esm({
|
|
|
81
126
|
this.name = "ApiError";
|
|
82
127
|
this.code = options?.code;
|
|
83
128
|
this.details = options?.details;
|
|
129
|
+
this.status = options?.status;
|
|
130
|
+
this.retryable = options?.retryable;
|
|
131
|
+
this.retryAfterSeconds = options?.retryAfterSeconds;
|
|
84
132
|
this.minVersion = options?.minVersion;
|
|
85
133
|
this.clientVersion = options?.clientVersion;
|
|
86
134
|
this.client = options?.client;
|
|
@@ -1229,6 +1277,10 @@ function createCLILogger(options = {}) {
|
|
|
1229
1277
|
handler: (level, message, _context, scope) => {
|
|
1230
1278
|
const scopePath = scope.length > 0 ? `[${scope.join(":")}] ` : "";
|
|
1231
1279
|
const fullMessage = `${scopePath}${message}`;
|
|
1280
|
+
try {
|
|
1281
|
+
options.onLine?.(level, fullMessage);
|
|
1282
|
+
} catch {
|
|
1283
|
+
}
|
|
1232
1284
|
if (level === l.ERROR) {
|
|
1233
1285
|
if (!json) console.error(chalk2.red(fullMessage));
|
|
1234
1286
|
return;
|
|
@@ -4570,7 +4622,7 @@ ${destinationsEntries.join(",\n")}
|
|
|
4570
4622
|
stores${collectorStr}
|
|
4571
4623
|
}`;
|
|
4572
4624
|
const dataPayload = JSON.stringify(dataPayloadObj, null, 2);
|
|
4573
|
-
return { storesDeclaration, codeConfigObject, dataPayload };
|
|
4625
|
+
return { storesDeclaration, codeConfigObject, dataPayloadObj, dataPayload };
|
|
4574
4626
|
}
|
|
4575
4627
|
function generateSplitWireConfigModule(storesDeclaration, codeConfigObject, userCode) {
|
|
4576
4628
|
const codeSection = userCode ? `
|
|
@@ -5314,7 +5366,7 @@ var init_contract = __esm({
|
|
|
5314
5366
|
"src/core/contract.ts"() {
|
|
5315
5367
|
"use strict";
|
|
5316
5368
|
init_config_file();
|
|
5317
|
-
bakedContractVersion = true ? "
|
|
5369
|
+
bakedContractVersion = true ? "2.1.0" : PLACEHOLDER;
|
|
5318
5370
|
}
|
|
5319
5371
|
});
|
|
5320
5372
|
|
|
@@ -6387,9 +6439,10 @@ async function nt(e4, t4, n5 = {}, o4) {
|
|
|
6387
6439
|
const v4 = n5.ingest ? { ...n5.ingest, _meta: { ...n5.ingest._meta, path: [...n5.ingest._meta.path] } } : E("unknown");
|
|
6388
6440
|
if (!w4.length && !s7.queueOn?.length) return { id: o5, destination: s7, skipped: true };
|
|
6389
6441
|
if (!w4.length && s7.queueOn?.length) {
|
|
6442
|
+
if (!je(s7.config.consent, r5)) return { id: o5, destination: s7, skipped: true };
|
|
6390
6443
|
let t5 = false;
|
|
6391
6444
|
try {
|
|
6392
|
-
t5 = await ot2(e4, s7, o5);
|
|
6445
|
+
t5 = await ot2(e4, s7, o5, true);
|
|
6393
6446
|
} catch (t6) {
|
|
6394
6447
|
e4.status.failed++;
|
|
6395
6448
|
const n6 = s7.type || "unknown";
|
|
@@ -6418,7 +6471,7 @@ async function nt(e4, t4, n5 = {}, o4) {
|
|
|
6418
6471
|
if (!b4.length) return { id: o5, destination: s7, queue: w4 };
|
|
6419
6472
|
let C2, S2, j2 = false;
|
|
6420
6473
|
try {
|
|
6421
|
-
j2 = await ot2(e4, s7, o5);
|
|
6474
|
+
j2 = await ot2(e4, s7, o5, true);
|
|
6422
6475
|
} catch (t5) {
|
|
6423
6476
|
e4.status.failed++;
|
|
6424
6477
|
const n6 = s7.type || "unknown";
|
|
@@ -6484,26 +6537,30 @@ async function nt(e4, t4, n5 = {}, o4) {
|
|
|
6484
6537
|
}
|
|
6485
6538
|
return rt({ event: t4, ...Object.keys(v3).length && { done: v3 }, ...Object.keys(b3).length && { queued: b3 }, ...Object.keys(k3).length && { failed: k3 } });
|
|
6486
6539
|
}
|
|
6487
|
-
async function ot2(e4, t4, n5) {
|
|
6540
|
+
async function ot2(e4, t4, n5, o4 = false) {
|
|
6488
6541
|
if (t4.init && !t4.config.init) {
|
|
6489
|
-
|
|
6490
|
-
|
|
6491
|
-
|
|
6492
|
-
|
|
6493
|
-
|
|
6542
|
+
if (!o4 && (function(e5) {
|
|
6543
|
+
const t5 = e5.config.consent;
|
|
6544
|
+
return !!t5 && Object.keys(t5).length > 0;
|
|
6545
|
+
})(t4)) return e4.logger.scope(t4.type || "unknown").debug("init blocked: consent gate not cleared"), false;
|
|
6546
|
+
const s6 = t4.type || "unknown", r5 = e4.logger.scope(s6), a4 = { collector: e4, logger: r5, id: n5, config: t4.config, env: ct2(t4.env, t4.config.env) };
|
|
6547
|
+
r5.debug("init");
|
|
6548
|
+
const i4 = Date.now();
|
|
6549
|
+
let c2;
|
|
6550
|
+
yt(e4, D2(e4, { stepId: o("destination", n5), stepType: "destination", phase: "init", eventId: "", now: i4 }));
|
|
6494
6551
|
try {
|
|
6495
|
-
|
|
6552
|
+
c2 = await ct(t4.init, "DestinationInit", e4.hooks, e4.logger)(a4);
|
|
6496
6553
|
} catch (t5) {
|
|
6497
6554
|
const o5 = Date.now(), s7 = D2(e4, { stepId: o("destination", n5), stepType: "destination", phase: "error", eventId: "", now: o5 });
|
|
6498
|
-
throw s7.durationMs = o5 -
|
|
6555
|
+
throw s7.durationMs = o5 - i4, s7.error = t5 instanceof Error ? { name: t5.name, message: t5.message } : { message: String(t5) }, yt(e4, s7), t5;
|
|
6499
6556
|
}
|
|
6500
|
-
if (false ===
|
|
6501
|
-
if (t4.config = { ...
|
|
6557
|
+
if (false === c2) return c2;
|
|
6558
|
+
if (t4.config = { ...c2 || t4.config, init: true }, t4.queueOn?.length) {
|
|
6502
6559
|
const o5 = t4.queueOn;
|
|
6503
6560
|
t4.queueOn = [];
|
|
6504
6561
|
for (const { type: s7, data: r6 } of o5) He2(e4, t4, n5, s7, r6);
|
|
6505
6562
|
}
|
|
6506
|
-
|
|
6563
|
+
r5.debug("init done");
|
|
6507
6564
|
}
|
|
6508
6565
|
return true;
|
|
6509
6566
|
}
|
|
@@ -6698,7 +6755,7 @@ async function wt(e4, t4, o4) {
|
|
|
6698
6755
|
}
|
|
6699
6756
|
}
|
|
6700
6757
|
function vt(e4, t4) {
|
|
6701
|
-
return { timing: Math.round((Date.now() - e4.timing) / 10) / 100, source: { type: "collector", schema: "4", version: "4.2.
|
|
6758
|
+
return { timing: Math.round((Date.now() - e4.timing) / 10) / 100, source: { type: "collector", schema: "4", version: "4.2.1-next-1781538735002" }, ...t4 };
|
|
6702
6759
|
}
|
|
6703
6760
|
function bt(e4, t4) {
|
|
6704
6761
|
if (!t4.name) throw new Error("Event name is required");
|
|
@@ -7451,13 +7508,13 @@ function installTimerInterception(options = {}) {
|
|
|
7451
7508
|
setInterval: Reflect.get(target, "setInterval"),
|
|
7452
7509
|
clearInterval: Reflect.get(target, "clearInterval")
|
|
7453
7510
|
});
|
|
7454
|
-
const trackedSetTimeout = (callback,
|
|
7511
|
+
const trackedSetTimeout = (callback, delay2, ...args) => {
|
|
7455
7512
|
if (typeof callback !== "function") return 0;
|
|
7456
7513
|
const id = nextId++;
|
|
7457
7514
|
pending.set(id, {
|
|
7458
7515
|
id,
|
|
7459
7516
|
callback,
|
|
7460
|
-
delay:
|
|
7517
|
+
delay: delay2 ?? 0,
|
|
7461
7518
|
type: "timeout",
|
|
7462
7519
|
args,
|
|
7463
7520
|
cleared: false
|
|
@@ -7470,13 +7527,13 @@ function installTimerInterception(options = {}) {
|
|
|
7470
7527
|
const entry = pending.get(numId);
|
|
7471
7528
|
if (entry) entry.cleared = true;
|
|
7472
7529
|
};
|
|
7473
|
-
const trackedSetInterval = (callback,
|
|
7530
|
+
const trackedSetInterval = (callback, delay2, ...args) => {
|
|
7474
7531
|
if (typeof callback !== "function") return 0;
|
|
7475
7532
|
const id = nextId++;
|
|
7476
7533
|
pending.set(id, {
|
|
7477
7534
|
id,
|
|
7478
7535
|
callback,
|
|
7479
|
-
delay:
|
|
7536
|
+
delay: delay2 ?? 0,
|
|
7480
7537
|
type: "interval",
|
|
7481
7538
|
args,
|
|
7482
7539
|
cleared: false
|
|
@@ -7753,7 +7810,7 @@ function toError(error) {
|
|
|
7753
7810
|
return error instanceof Error ? error : new Error(getErrorMessage(error));
|
|
7754
7811
|
}
|
|
7755
7812
|
function buildSimulationResult(args) {
|
|
7756
|
-
const { step, name, startTime, captured, usage, error } = args;
|
|
7813
|
+
const { step, name, startTime, captured, usage, mappingKey, error } = args;
|
|
7757
7814
|
const events = (captured ?? []).filter(hasEvent).map((entry) => entry.event);
|
|
7758
7815
|
const calls = usage ? Object.values(usage).flat().map((call) => ({ fn: call.fn, args: call.args, ts: call.ts })) : [];
|
|
7759
7816
|
return {
|
|
@@ -7762,6 +7819,7 @@ function buildSimulationResult(args) {
|
|
|
7762
7819
|
events,
|
|
7763
7820
|
calls,
|
|
7764
7821
|
duration: Date.now() - startTime,
|
|
7822
|
+
...mappingKey !== void 0 ? { mappingKey } : {},
|
|
7765
7823
|
...error !== void 0 ? { error: toError(error) } : {}
|
|
7766
7824
|
};
|
|
7767
7825
|
}
|
|
@@ -8355,7 +8413,9 @@ async function simulateSource(configOrPath, input, options) {
|
|
|
8355
8413
|
`Source package "${sourceConfig.package}" has no createTrigger in /dev export`
|
|
8356
8414
|
);
|
|
8357
8415
|
}
|
|
8358
|
-
const flowConfig = module.wireConfig(
|
|
8416
|
+
const flowConfig = module.wireConfig(
|
|
8417
|
+
options.data ?? module.__configData ?? void 0
|
|
8418
|
+
);
|
|
8359
8419
|
applyOverrides(flowConfig, prepared.overrides);
|
|
8360
8420
|
const captured = [];
|
|
8361
8421
|
flowConfig.hooks = {
|
|
@@ -8460,7 +8520,9 @@ async function simulateTransformer(configOrPath, event, options) {
|
|
|
8460
8520
|
networkCalls
|
|
8461
8521
|
},
|
|
8462
8522
|
async (module) => {
|
|
8463
|
-
const flowConfig = module.wireConfig(
|
|
8523
|
+
const flowConfig = module.wireConfig(
|
|
8524
|
+
options.data ?? module.__configData ?? void 0
|
|
8525
|
+
);
|
|
8464
8526
|
applyOverrides(flowConfig, prepared.overrides);
|
|
8465
8527
|
if (flowConfig.sources) flowConfig.sources = {};
|
|
8466
8528
|
if (flowConfig.destinations) flowConfig.destinations = {};
|
|
@@ -8626,7 +8688,9 @@ async function simulateCollector(configOrPath, event, options) {
|
|
|
8626
8688
|
networkCalls
|
|
8627
8689
|
},
|
|
8628
8690
|
async (module) => {
|
|
8629
|
-
const flowConfig = module.wireConfig(
|
|
8691
|
+
const flowConfig = module.wireConfig(
|
|
8692
|
+
options.data ?? module.__configData ?? void 0
|
|
8693
|
+
);
|
|
8630
8694
|
applyOverrides(flowConfig, prepared.overrides);
|
|
8631
8695
|
if (flowConfig.sources) flowConfig.sources = {};
|
|
8632
8696
|
if (flowConfig.destinations) flowConfig.destinations = {};
|
|
@@ -8729,7 +8793,9 @@ async function simulateDestination(configOrPath, event, options) {
|
|
|
8729
8793
|
networkCalls
|
|
8730
8794
|
},
|
|
8731
8795
|
async (module) => {
|
|
8732
|
-
const flowConfig = module.wireConfig(
|
|
8796
|
+
const flowConfig = module.wireConfig(
|
|
8797
|
+
options.data ?? module.__configData ?? void 0
|
|
8798
|
+
);
|
|
8733
8799
|
applyOverrides(flowConfig, prepared.overrides);
|
|
8734
8800
|
const destPkg = (prepared.flowSettings.destinations ?? {})[options.destinationId];
|
|
8735
8801
|
let trackedCalls = [];
|
|
@@ -8765,6 +8831,16 @@ async function simulateDestination(configOrPath, event, options) {
|
|
|
8765
8831
|
);
|
|
8766
8832
|
}
|
|
8767
8833
|
logger.info(`Simulating destination: ${options.destinationId}`);
|
|
8834
|
+
let mappingKey;
|
|
8835
|
+
const targetStepId = o("destination", options.destinationId);
|
|
8836
|
+
const captureMappingKey = (state) => {
|
|
8837
|
+
if (state.stepId === targetStepId && state.mappingKey) {
|
|
8838
|
+
mappingKey = state.mappingKey;
|
|
8839
|
+
}
|
|
8840
|
+
};
|
|
8841
|
+
if (collector.observers instanceof Set) {
|
|
8842
|
+
collector.observers.add(captureMappingKey);
|
|
8843
|
+
}
|
|
8768
8844
|
await collector.push(event, {
|
|
8769
8845
|
include: [options.destinationId]
|
|
8770
8846
|
});
|
|
@@ -8773,7 +8849,8 @@ async function simulateDestination(configOrPath, event, options) {
|
|
|
8773
8849
|
step: "destination",
|
|
8774
8850
|
name: options.destinationId,
|
|
8775
8851
|
startTime,
|
|
8776
|
-
usage: trackedCalls.length ? { [options.destinationId]: trackedCalls } : void 0
|
|
8852
|
+
usage: trackedCalls.length ? { [options.destinationId]: trackedCalls } : void 0,
|
|
8853
|
+
mappingKey
|
|
8777
8854
|
});
|
|
8778
8855
|
},
|
|
8779
8856
|
(error) => buildSimulationResult({
|
|
@@ -8796,16 +8873,91 @@ async function simulateDestination(configOrPath, event, options) {
|
|
|
8796
8873
|
}
|
|
8797
8874
|
|
|
8798
8875
|
// src/commands/run/index.ts
|
|
8876
|
+
init_dist();
|
|
8799
8877
|
init_cli_logger();
|
|
8800
8878
|
init_core();
|
|
8801
|
-
init_tmp();
|
|
8802
|
-
init_config_file();
|
|
8803
|
-
init_auth();
|
|
8804
8879
|
import path18 from "path";
|
|
8805
8880
|
import { writeFileSync as writeFileSync5 } from "fs";
|
|
8806
8881
|
import { homedir as homedir2 } from "os";
|
|
8807
8882
|
import { join as join7 } from "path";
|
|
8808
8883
|
|
|
8884
|
+
// src/runtime/runner.ts
|
|
8885
|
+
import { resolve as resolve3, dirname as dirname4 } from "path";
|
|
8886
|
+
|
|
8887
|
+
// src/runtime/load-bundle.ts
|
|
8888
|
+
import { resolve as resolve2 } from "path";
|
|
8889
|
+
import { pathToFileURL as pathToFileURL2 } from "url";
|
|
8890
|
+
async function loadBundle(file, context2, logger) {
|
|
8891
|
+
const absolutePath = resolve2(file);
|
|
8892
|
+
const fileUrl = pathToFileURL2(absolutePath).href;
|
|
8893
|
+
logger?.debug?.(`Importing bundle: ${absolutePath}`);
|
|
8894
|
+
const module = await import(`${fileUrl}?t=${Date.now()}`);
|
|
8895
|
+
if (!module.default || typeof module.default !== "function") {
|
|
8896
|
+
throw new Error(
|
|
8897
|
+
`Invalid bundle: ${file} must export a default factory function`
|
|
8898
|
+
);
|
|
8899
|
+
}
|
|
8900
|
+
logger?.debug?.("Calling factory function...");
|
|
8901
|
+
const result = await module.default(context2 ?? {});
|
|
8902
|
+
if (!result || !result.collector || typeof result.collector.push !== "function") {
|
|
8903
|
+
throw new Error(
|
|
8904
|
+
`Invalid bundle: factory must return { collector } with a push function`
|
|
8905
|
+
);
|
|
8906
|
+
}
|
|
8907
|
+
return {
|
|
8908
|
+
collector: result.collector,
|
|
8909
|
+
...typeof result.httpHandler === "function" ? { httpHandler: result.httpHandler } : {}
|
|
8910
|
+
};
|
|
8911
|
+
}
|
|
8912
|
+
|
|
8913
|
+
// src/runtime/runner.ts
|
|
8914
|
+
async function loadFlow(file, config, logger, loggerConfig, healthServer, observers) {
|
|
8915
|
+
const absolutePath = resolve3(file);
|
|
8916
|
+
const flowDir = dirname4(absolutePath);
|
|
8917
|
+
process.chdir(flowDir);
|
|
8918
|
+
const flowContext = {
|
|
8919
|
+
...config,
|
|
8920
|
+
...loggerConfig ? { logger: loggerConfig } : {},
|
|
8921
|
+
...healthServer ? { sourceSettings: { port: void 0 } } : {},
|
|
8922
|
+
...observers ? { observers } : {}
|
|
8923
|
+
};
|
|
8924
|
+
const result = await loadBundle(absolutePath, flowContext, logger);
|
|
8925
|
+
if (healthServer && typeof result.httpHandler === "function") {
|
|
8926
|
+
healthServer.setFlowHandler(result.httpHandler);
|
|
8927
|
+
}
|
|
8928
|
+
return {
|
|
8929
|
+
collector: {
|
|
8930
|
+
command: result.collector.command,
|
|
8931
|
+
status: result.collector.status
|
|
8932
|
+
},
|
|
8933
|
+
file,
|
|
8934
|
+
httpHandler: result.httpHandler
|
|
8935
|
+
};
|
|
8936
|
+
}
|
|
8937
|
+
async function swapFlow(currentHandle, newFile, config, logger, loggerConfig, healthServer, observers) {
|
|
8938
|
+
logger.info("Shutting down current flow for hot-swap...");
|
|
8939
|
+
if (healthServer) {
|
|
8940
|
+
healthServer.setFlowHandler(null);
|
|
8941
|
+
}
|
|
8942
|
+
try {
|
|
8943
|
+
if (currentHandle.collector.command) {
|
|
8944
|
+
await currentHandle.collector.command("shutdown");
|
|
8945
|
+
}
|
|
8946
|
+
} catch (error) {
|
|
8947
|
+
logger.debug(`Shutdown warning: ${error}`);
|
|
8948
|
+
}
|
|
8949
|
+
const newHandle = await loadFlow(
|
|
8950
|
+
newFile,
|
|
8951
|
+
config,
|
|
8952
|
+
logger,
|
|
8953
|
+
loggerConfig,
|
|
8954
|
+
healthServer,
|
|
8955
|
+
observers
|
|
8956
|
+
);
|
|
8957
|
+
logger.info("Flow swapped successfully");
|
|
8958
|
+
return newHandle;
|
|
8959
|
+
}
|
|
8960
|
+
|
|
8809
8961
|
// src/runtime/resolve-bundle.ts
|
|
8810
8962
|
init_stdin();
|
|
8811
8963
|
import {
|
|
@@ -8815,11 +8967,127 @@ import {
|
|
|
8815
8967
|
rmSync,
|
|
8816
8968
|
writeFileSync as writeFileSync2
|
|
8817
8969
|
} from "fs";
|
|
8818
|
-
import { dirname as
|
|
8970
|
+
import { dirname as dirname5, join as join5 } from "path";
|
|
8819
8971
|
import { Readable } from "stream";
|
|
8820
8972
|
import { x as tarExtract } from "tar";
|
|
8973
|
+
|
|
8974
|
+
// src/runtime/fetch-retry.ts
|
|
8975
|
+
var DEFAULT_PER_ATTEMPT_TIMEOUT_MS = 3e4;
|
|
8976
|
+
var DEFAULT_MAX_TOTAL_MS = 6e4;
|
|
8977
|
+
var DEFAULT_ATTEMPTS = 3;
|
|
8978
|
+
var MIN_ATTEMPT_BUDGET_MS = 1e3;
|
|
8979
|
+
var BASE_BACKOFF_MS = [2e3, 5e3];
|
|
8980
|
+
var JITTER = 0.2;
|
|
8981
|
+
var RETRYABLE_NETWORK_CODES = /* @__PURE__ */ new Set([
|
|
8982
|
+
"ECONNRESET",
|
|
8983
|
+
"ECONNREFUSED",
|
|
8984
|
+
"ETIMEDOUT",
|
|
8985
|
+
"EAI_AGAIN",
|
|
8986
|
+
"ENOTFOUND"
|
|
8987
|
+
]);
|
|
8988
|
+
function readErrorCode(value) {
|
|
8989
|
+
if (typeof value !== "object" || value === null) return void 0;
|
|
8990
|
+
const code = Reflect.get(value, "code");
|
|
8991
|
+
return typeof code === "string" ? code : void 0;
|
|
8992
|
+
}
|
|
8993
|
+
function readErrorName(value) {
|
|
8994
|
+
if (typeof value !== "object" || value === null) return void 0;
|
|
8995
|
+
const name = Reflect.get(value, "name");
|
|
8996
|
+
return typeof name === "string" ? name : void 0;
|
|
8997
|
+
}
|
|
8998
|
+
function isTransientThrow(error) {
|
|
8999
|
+
const name = readErrorName(error);
|
|
9000
|
+
if (name === "TimeoutError" || name === "AbortError") return true;
|
|
9001
|
+
const directCode = readErrorCode(error);
|
|
9002
|
+
if (directCode && RETRYABLE_NETWORK_CODES.has(directCode)) return true;
|
|
9003
|
+
if (typeof error === "object" && error !== null) {
|
|
9004
|
+
const causeCode = readErrorCode(Reflect.get(error, "cause"));
|
|
9005
|
+
if (causeCode && RETRYABLE_NETWORK_CODES.has(causeCode)) return true;
|
|
9006
|
+
}
|
|
9007
|
+
return error instanceof TypeError;
|
|
9008
|
+
}
|
|
9009
|
+
function isTransientStatus(status) {
|
|
9010
|
+
return status >= 500 || status === 429;
|
|
9011
|
+
}
|
|
9012
|
+
function readErrorMessage(value) {
|
|
9013
|
+
if (typeof value !== "object" || value === null) return void 0;
|
|
9014
|
+
const message = Reflect.get(value, "message");
|
|
9015
|
+
return typeof message === "string" ? message : void 0;
|
|
9016
|
+
}
|
|
9017
|
+
function describeReason(reason) {
|
|
9018
|
+
if (reason.kind === "status") return `HTTP ${reason.status}`;
|
|
9019
|
+
const name = readErrorName(reason.error);
|
|
9020
|
+
const message = reason.error instanceof Error ? reason.error.message : String(reason.error);
|
|
9021
|
+
let detail = name ? `${name}: ${message}` : message;
|
|
9022
|
+
if (typeof reason.error === "object" && reason.error !== null) {
|
|
9023
|
+
const cause = Reflect.get(reason.error, "cause");
|
|
9024
|
+
const causeCode = readErrorCode(cause);
|
|
9025
|
+
const causeMessage = readErrorMessage(cause);
|
|
9026
|
+
const causeDetail = causeCode ?? causeMessage;
|
|
9027
|
+
if (causeDetail) detail = `${detail} (${causeDetail})`;
|
|
9028
|
+
}
|
|
9029
|
+
return detail;
|
|
9030
|
+
}
|
|
9031
|
+
function backoffForAttempt(index) {
|
|
9032
|
+
const base = BASE_BACKOFF_MS[Math.min(index, BASE_BACKOFF_MS.length - 1)] ?? 0;
|
|
9033
|
+
const spread = base * JITTER;
|
|
9034
|
+
return base + (Math.random() * 2 - 1) * spread;
|
|
9035
|
+
}
|
|
9036
|
+
function delay(ms) {
|
|
9037
|
+
return new Promise((resolve4) => {
|
|
9038
|
+
setTimeout(resolve4, ms);
|
|
9039
|
+
});
|
|
9040
|
+
}
|
|
9041
|
+
async function fetchWithRetry(url, options = {}) {
|
|
9042
|
+
const attempts = options.attempts ?? DEFAULT_ATTEMPTS;
|
|
9043
|
+
const perAttemptTimeoutMs = options.perAttemptTimeoutMs ?? DEFAULT_PER_ATTEMPT_TIMEOUT_MS;
|
|
9044
|
+
const maxTotalMs = options.maxTotalMs ?? DEFAULT_MAX_TOTAL_MS;
|
|
9045
|
+
const init = options.init;
|
|
9046
|
+
const start = Date.now();
|
|
9047
|
+
let lastReason;
|
|
9048
|
+
let made = 0;
|
|
9049
|
+
for (let attempt = 0; attempt < attempts; attempt++) {
|
|
9050
|
+
const remaining = maxTotalMs - (Date.now() - start);
|
|
9051
|
+
if (remaining <= MIN_ATTEMPT_BUDGET_MS) break;
|
|
9052
|
+
const attemptTimeoutMs = Math.min(perAttemptTimeoutMs, remaining);
|
|
9053
|
+
made = attempt + 1;
|
|
9054
|
+
let reason;
|
|
9055
|
+
try {
|
|
9056
|
+
const controller = new AbortController();
|
|
9057
|
+
const timeoutId = setTimeout(() => controller.abort(), attemptTimeoutMs);
|
|
9058
|
+
let response;
|
|
9059
|
+
try {
|
|
9060
|
+
response = await fetch(url, {
|
|
9061
|
+
...init,
|
|
9062
|
+
signal: controller.signal
|
|
9063
|
+
});
|
|
9064
|
+
} finally {
|
|
9065
|
+
clearTimeout(timeoutId);
|
|
9066
|
+
}
|
|
9067
|
+
if (!isTransientStatus(response.status)) return response;
|
|
9068
|
+
await response.body?.cancel();
|
|
9069
|
+
reason = { kind: "status", status: response.status };
|
|
9070
|
+
} catch (error) {
|
|
9071
|
+
if (!isTransientThrow(error)) throw error;
|
|
9072
|
+
reason = { kind: "throw", error };
|
|
9073
|
+
}
|
|
9074
|
+
lastReason = reason;
|
|
9075
|
+
const isLastAttempt = attempt === attempts - 1;
|
|
9076
|
+
const budgetSpent = Date.now() - start >= maxTotalMs;
|
|
9077
|
+
if (isLastAttempt || budgetSpent) break;
|
|
9078
|
+
const sleepMs = Math.min(
|
|
9079
|
+
backoffForAttempt(attempt),
|
|
9080
|
+
maxTotalMs - (Date.now() - start)
|
|
9081
|
+
);
|
|
9082
|
+
if (sleepMs <= 0) break;
|
|
9083
|
+
await delay(sleepMs);
|
|
9084
|
+
}
|
|
9085
|
+
const cause = lastReason ? describeReason(lastReason) : "no attempts made";
|
|
9086
|
+
throw new Error(`Fetch failed after ${made} attempts: ${cause}`);
|
|
9087
|
+
}
|
|
9088
|
+
|
|
9089
|
+
// src/runtime/resolve-bundle.ts
|
|
8821
9090
|
var ARCHIVE_ENTRY = "flow.mjs";
|
|
8822
|
-
var FETCH_TIMEOUT_MS = 3e4;
|
|
8823
9091
|
function getDefaultWritePath() {
|
|
8824
9092
|
if (existsSync2("/app/flow")) return "/app/flow/flow.mjs";
|
|
8825
9093
|
return "/tmp/walkeros-flow.mjs";
|
|
@@ -8837,16 +9105,14 @@ function isArchive(value, contentType) {
|
|
|
8837
9105
|
return false;
|
|
8838
9106
|
}
|
|
8839
9107
|
function writeBundleToDisk(writePath, content) {
|
|
8840
|
-
const dir =
|
|
9108
|
+
const dir = dirname5(writePath);
|
|
8841
9109
|
if (!existsSync2(dir)) {
|
|
8842
9110
|
mkdirSync2(dir, { recursive: true });
|
|
8843
9111
|
}
|
|
8844
9112
|
writeFileSync2(writePath, content, "utf-8");
|
|
8845
9113
|
}
|
|
8846
9114
|
async function fetchOk(url) {
|
|
8847
|
-
const response = await
|
|
8848
|
-
signal: AbortSignal.timeout(FETCH_TIMEOUT_MS)
|
|
8849
|
-
});
|
|
9115
|
+
const response = await fetchWithRetry(url);
|
|
8850
9116
|
if (!response.ok) {
|
|
8851
9117
|
throw new Error(
|
|
8852
9118
|
`Failed to fetch bundle from ${url}: ${response.status} ${response.statusText}`
|
|
@@ -8898,7 +9164,7 @@ async function readBundleFromStdin(writePath) {
|
|
|
8898
9164
|
}
|
|
8899
9165
|
async function resolveBundle(bundleEnv) {
|
|
8900
9166
|
const writePath = getDefaultWritePath();
|
|
8901
|
-
const archiveDestDir =
|
|
9167
|
+
const archiveDestDir = dirname5(writePath);
|
|
8902
9168
|
if (!isUrl2(bundleEnv) && existsSync2(bundleEnv)) {
|
|
8903
9169
|
if (isArchive(bundleEnv)) {
|
|
8904
9170
|
const path19 = await extractToDir(
|
|
@@ -8967,10 +9233,7 @@ async function fetchConfig(options) {
|
|
|
8967
9233
|
options.token,
|
|
8968
9234
|
options.lastEtag ? { "If-None-Match": options.lastEtag } : void 0
|
|
8969
9235
|
);
|
|
8970
|
-
const response = await
|
|
8971
|
-
headers,
|
|
8972
|
-
signal: AbortSignal.timeout(3e4)
|
|
8973
|
-
});
|
|
9236
|
+
const response = await fetchWithRetry(url, { init: { headers } });
|
|
8974
9237
|
if (response.status === 304) {
|
|
8975
9238
|
return { changed: false };
|
|
8976
9239
|
}
|
|
@@ -8991,271 +9254,126 @@ async function fetchConfig(options) {
|
|
|
8991
9254
|
};
|
|
8992
9255
|
}
|
|
8993
9256
|
|
|
8994
|
-
// src/
|
|
9257
|
+
// src/runtime/index.ts
|
|
8995
9258
|
init_cache();
|
|
8996
9259
|
|
|
8997
|
-
// src/
|
|
8998
|
-
|
|
8999
|
-
|
|
9000
|
-
|
|
9001
|
-
// src/schemas/primitives.ts
|
|
9002
|
-
init_dev();
|
|
9003
|
-
var PortSchema = n3.number().int("Port must be an integer").min(1, "Port must be at least 1").max(65535, "Port must be at most 65535").describe("HTTP server port number");
|
|
9004
|
-
var FilePathSchema = n3.string().min(1, "File path cannot be empty").describe("Path to configuration file");
|
|
9005
|
-
|
|
9006
|
-
// src/schemas/run.ts
|
|
9007
|
-
init_dev();
|
|
9008
|
-
var RunOptionsSchema = n3.object({
|
|
9009
|
-
flow: FilePathSchema,
|
|
9010
|
-
port: PortSchema.default(8080),
|
|
9011
|
-
flowName: n3.string().optional().describe("Specific flow name to run")
|
|
9012
|
-
});
|
|
9013
|
-
|
|
9014
|
-
// src/schemas/validate.ts
|
|
9015
|
-
init_dev();
|
|
9016
|
-
var ValidationTypeSchema = n3.enum(["contract", "event", "flow", "mapping"]).describe('Validation type: "event", "flow", "mapping", or "contract"');
|
|
9017
|
-
var ValidateOptionsSchema = n3.object({
|
|
9018
|
-
flow: n3.string().optional().describe("Flow name for multi-flow configs"),
|
|
9019
|
-
path: n3.string().optional().describe(
|
|
9020
|
-
'Entry path for package schema validation (e.g., "destinations.snowplow", "sources.browser")'
|
|
9021
|
-
)
|
|
9022
|
-
});
|
|
9023
|
-
var ValidateInputShape = {
|
|
9024
|
-
type: ValidationTypeSchema,
|
|
9025
|
-
input: n3.string().min(1).describe("JSON string, file path, or URL to validate"),
|
|
9026
|
-
flow: n3.string().optional().describe("Flow name for multi-flow configs"),
|
|
9027
|
-
path: n3.string().optional().describe(
|
|
9028
|
-
'Entry path for package schema validation (e.g., "destinations.snowplow"). When provided, validates the entry against its package JSON Schema instead of using --type.'
|
|
9029
|
-
)
|
|
9030
|
-
};
|
|
9031
|
-
var ValidateInputSchema = n3.object(ValidateInputShape);
|
|
9032
|
-
|
|
9033
|
-
// src/schemas/bundle.ts
|
|
9034
|
-
init_dev();
|
|
9035
|
-
var BundleOptionsSchema = n3.object({
|
|
9036
|
-
silent: n3.boolean().optional().describe("Suppress all output"),
|
|
9037
|
-
verbose: n3.boolean().optional().describe("Enable verbose logging"),
|
|
9038
|
-
stats: n3.boolean().optional().default(true).describe("Return bundle statistics"),
|
|
9039
|
-
cache: n3.boolean().optional().default(true).describe("Enable package caching"),
|
|
9040
|
-
flowName: n3.string().optional().describe("Flow name for multi-flow configs")
|
|
9041
|
-
});
|
|
9042
|
-
var BundleInputShape = {
|
|
9043
|
-
configPath: FilePathSchema.describe(
|
|
9044
|
-
"Path to flow configuration file (JSON or JavaScript), URL, or inline JSON string"
|
|
9045
|
-
),
|
|
9046
|
-
flow: n3.string().optional().describe("Flow name for multi-flow configs"),
|
|
9047
|
-
stats: n3.boolean().optional().default(true).describe("Return bundle statistics"),
|
|
9048
|
-
output: n3.string().optional().describe("Output file path (defaults to config-defined)")
|
|
9049
|
-
};
|
|
9050
|
-
var BundleInputSchema = n3.object(BundleInputShape);
|
|
9051
|
-
|
|
9052
|
-
// src/schemas/simulate.ts
|
|
9053
|
-
init_dev();
|
|
9054
|
-
var PlatformSchema = n3.enum(["web", "server"]).describe("Platform type for event processing");
|
|
9055
|
-
var SimulateOptionsSchema = n3.object({
|
|
9056
|
-
silent: n3.boolean().optional().describe("Suppress all output"),
|
|
9057
|
-
verbose: n3.boolean().optional().describe("Enable verbose logging"),
|
|
9058
|
-
json: n3.boolean().optional().describe("Format output as JSON")
|
|
9059
|
-
});
|
|
9060
|
-
var SimulateInputShape = {
|
|
9061
|
-
configPath: FilePathSchema.describe(
|
|
9062
|
-
"Path to flow configuration file, URL, or inline JSON string"
|
|
9063
|
-
),
|
|
9064
|
-
event: n3.string().min(1).optional().describe(
|
|
9065
|
-
"Event as JSON string, file path, or URL. For sources: { content, trigger?, env? }."
|
|
9066
|
-
),
|
|
9067
|
-
flow: n3.string().optional().describe("Flow name for multi-flow configs"),
|
|
9068
|
-
platform: PlatformSchema.optional().describe("Override platform detection"),
|
|
9069
|
-
step: n3.string().optional().describe(
|
|
9070
|
-
'Step target in type.name format (e.g. "source.browser", "destination.gtag")'
|
|
9071
|
-
)
|
|
9072
|
-
};
|
|
9073
|
-
var SimulateInputSchema = n3.object(SimulateInputShape);
|
|
9074
|
-
|
|
9075
|
-
// src/schemas/push.ts
|
|
9076
|
-
init_dev();
|
|
9077
|
-
var PushOptionsSchema = n3.object({
|
|
9078
|
-
silent: n3.boolean().optional().describe("Suppress all output"),
|
|
9079
|
-
verbose: n3.boolean().optional().describe("Enable verbose logging"),
|
|
9080
|
-
json: n3.boolean().optional().describe("Format output as JSON")
|
|
9081
|
-
});
|
|
9082
|
-
var PushInputShape = {
|
|
9083
|
-
configPath: FilePathSchema.describe("Path to flow configuration file"),
|
|
9084
|
-
event: n3.string().min(1).describe("Event as JSON string, file path, or URL"),
|
|
9085
|
-
flow: n3.string().optional().describe("Flow name for multi-flow configs"),
|
|
9086
|
-
platform: PlatformSchema.optional().describe("Override platform detection")
|
|
9087
|
-
};
|
|
9088
|
-
var PushInputSchema = n3.object(PushInputShape);
|
|
9260
|
+
// src/runtime/heartbeat.ts
|
|
9261
|
+
import { randomBytes } from "crypto";
|
|
9262
|
+
init_http();
|
|
9089
9263
|
|
|
9090
|
-
// src/
|
|
9091
|
-
|
|
9092
|
-
|
|
9093
|
-
|
|
9094
|
-
|
|
9095
|
-
|
|
9096
|
-
|
|
9097
|
-
|
|
9098
|
-
|
|
9099
|
-
|
|
9100
|
-
|
|
9264
|
+
// src/runtime/redact.ts
|
|
9265
|
+
var MAX_LENGTH = 256;
|
|
9266
|
+
var MIN_TOKEN_LEN = 20;
|
|
9267
|
+
var ENTROPY_THRESHOLD = 4;
|
|
9268
|
+
var FORCE_MASK_PREFIXES = [
|
|
9269
|
+
"sk-",
|
|
9270
|
+
"sk_",
|
|
9271
|
+
"pk_",
|
|
9272
|
+
"ghp_",
|
|
9273
|
+
"gho_",
|
|
9274
|
+
"xoxb-",
|
|
9275
|
+
"xoxp-",
|
|
9276
|
+
"AKIA"
|
|
9277
|
+
];
|
|
9278
|
+
var RE_URL_CREDS = /(:\/\/[^/:@\s]+:)[^@\s]+(@)/g;
|
|
9279
|
+
var RE_JSON_PRIVATE_KEY = /("private_key"\s*:\s*)"[^"]*"/g;
|
|
9280
|
+
var RE_PEM_BEGIN = /^\s*-----BEGIN [A-Z ]*PRIVATE KEY-----/i;
|
|
9281
|
+
var RE_PEM_END = /-----END[A-Z -]*-----/i;
|
|
9282
|
+
var TOKEN_VALUE_CLASS = "[A-Za-z0-9+/._-]";
|
|
9283
|
+
var RE_KV_SECRET = new RegExp(
|
|
9284
|
+
`(^|\\s)([A-Za-z_][A-Za-z0-9_.]{0,63}\\s*[=:]\\s*)(${TOKEN_VALUE_CLASS}{12,}={0,2})`,
|
|
9285
|
+
"g"
|
|
9286
|
+
);
|
|
9287
|
+
var RE_TOKEN_RUN = new RegExp(`${TOKEN_VALUE_CLASS}{${MIN_TOKEN_LEN},}`, "g");
|
|
9288
|
+
function escapeRegex(s6) {
|
|
9289
|
+
return s6.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
9290
|
+
}
|
|
9291
|
+
var PREFIX_ALTERNATION = [...FORCE_MASK_PREFIXES].sort((a4, b3) => b3.length - a4.length).map(escapeRegex).join("|");
|
|
9292
|
+
var RE_PREFIXED_TOKEN = new RegExp(
|
|
9293
|
+
`(?:${PREFIX_ALTERNATION})${TOKEN_VALUE_CLASS}*`,
|
|
9294
|
+
"g"
|
|
9295
|
+
);
|
|
9296
|
+
function shannonEntropy(s6) {
|
|
9297
|
+
if (s6.length === 0) return 0;
|
|
9298
|
+
const counts = /* @__PURE__ */ new Map();
|
|
9299
|
+
for (const ch of s6) counts.set(ch, (counts.get(ch) ?? 0) + 1);
|
|
9300
|
+
let entropy = 0;
|
|
9301
|
+
for (const count of counts.values()) {
|
|
9302
|
+
const p3 = count / s6.length;
|
|
9303
|
+
entropy -= p3 * Math.log2(p3);
|
|
9304
|
+
}
|
|
9305
|
+
return entropy;
|
|
9306
|
+
}
|
|
9307
|
+
var RE_ALL_HEX = /^[0-9a-fA-F]+$/;
|
|
9308
|
+
var RE_ALL_DIGIT = /^[0-9]+$/;
|
|
9309
|
+
var RE_BASE64_SPECIAL = /[+/=]/;
|
|
9310
|
+
var RE_HAS_DIGIT = /[0-9]/;
|
|
9311
|
+
var RE_HAS_LETTER = /[A-Za-z]/;
|
|
9312
|
+
function shouldMaskToken(run) {
|
|
9313
|
+
if (RE_ALL_HEX.test(run)) return true;
|
|
9314
|
+
if (RE_ALL_DIGIT.test(run)) return true;
|
|
9315
|
+
if (RE_BASE64_SPECIAL.test(run)) return true;
|
|
9316
|
+
if (RE_HAS_DIGIT.test(run) && RE_HAS_LETTER.test(run)) return true;
|
|
9317
|
+
if (shannonEntropy(run) >= ENTROPY_THRESHOLD) return true;
|
|
9318
|
+
return false;
|
|
9101
9319
|
}
|
|
9102
|
-
function
|
|
9103
|
-
const
|
|
9104
|
-
|
|
9105
|
-
|
|
9106
|
-
|
|
9107
|
-
|
|
9108
|
-
|
|
9109
|
-
|
|
9320
|
+
function removePemBlocks(lines) {
|
|
9321
|
+
const out = [];
|
|
9322
|
+
let inBlock = false;
|
|
9323
|
+
for (const line of lines) {
|
|
9324
|
+
if (!inBlock) {
|
|
9325
|
+
if (RE_PEM_BEGIN.test(line)) {
|
|
9326
|
+
inBlock = true;
|
|
9327
|
+
continue;
|
|
9328
|
+
}
|
|
9329
|
+
out.push(line);
|
|
9330
|
+
} else {
|
|
9331
|
+
if (RE_PEM_END.test(line)) {
|
|
9332
|
+
inBlock = false;
|
|
9333
|
+
}
|
|
9334
|
+
}
|
|
9110
9335
|
}
|
|
9111
|
-
|
|
9112
|
-
|
|
9113
|
-
|
|
9114
|
-
|
|
9115
|
-
|
|
9116
|
-
|
|
9117
|
-
|
|
9118
|
-
|
|
9119
|
-
|
|
9120
|
-
|
|
9121
|
-
|
|
9122
|
-
|
|
9123
|
-
|
|
9124
|
-
function
|
|
9125
|
-
|
|
9126
|
-
|
|
9127
|
-
|
|
9128
|
-
|
|
9129
|
-
|
|
9130
|
-
|
|
9131
|
-
|
|
9132
|
-
|
|
9133
|
-
|
|
9134
|
-
|
|
9135
|
-
|
|
9136
|
-
|
|
9137
|
-
|
|
9138
|
-
|
|
9139
|
-
|
|
9140
|
-
|
|
9141
|
-
|
|
9142
|
-
|
|
9143
|
-
|
|
9144
|
-
|
|
9145
|
-
|
|
9146
|
-
|
|
9147
|
-
|
|
9148
|
-
|
|
9149
|
-
}
|
|
9150
|
-
res.writeHead(503, { "Content-Type": "application/json" });
|
|
9151
|
-
res.end(JSON.stringify({ error: "No flow loaded" }));
|
|
9152
|
-
});
|
|
9153
|
-
server.keepAliveTimeout = 5e3;
|
|
9154
|
-
server.headersTimeout = 1e4;
|
|
9155
|
-
server.listen(port, "0.0.0.0", () => {
|
|
9156
|
-
logger.info(`Health server listening on port ${port}`);
|
|
9157
|
-
resolve4({
|
|
9158
|
-
server,
|
|
9159
|
-
setFlowHandler(handler) {
|
|
9160
|
-
flowHandler = handler;
|
|
9161
|
-
},
|
|
9162
|
-
setReady(value) {
|
|
9163
|
-
ready = value;
|
|
9164
|
-
if (value) failureReason = null;
|
|
9165
|
-
},
|
|
9166
|
-
setFailed(reason) {
|
|
9167
|
-
ready = false;
|
|
9168
|
-
failureReason = reason;
|
|
9169
|
-
},
|
|
9170
|
-
close: () => new Promise((res, rej) => {
|
|
9171
|
-
server.close((err) => err ? rej(err) : res());
|
|
9172
|
-
})
|
|
9173
|
-
});
|
|
9174
|
-
});
|
|
9175
|
-
server.on("error", reject);
|
|
9176
|
-
});
|
|
9177
|
-
}
|
|
9178
|
-
|
|
9179
|
-
// src/runtime/runner.ts
|
|
9180
|
-
import { resolve as resolve3, dirname as dirname5 } from "path";
|
|
9181
|
-
|
|
9182
|
-
// src/runtime/load-bundle.ts
|
|
9183
|
-
import { resolve as resolve2 } from "path";
|
|
9184
|
-
import { pathToFileURL as pathToFileURL2 } from "url";
|
|
9185
|
-
async function loadBundle(file, context2, logger) {
|
|
9186
|
-
const absolutePath = resolve2(file);
|
|
9187
|
-
const fileUrl = pathToFileURL2(absolutePath).href;
|
|
9188
|
-
logger?.debug?.(`Importing bundle: ${absolutePath}`);
|
|
9189
|
-
const module = await import(`${fileUrl}?t=${Date.now()}`);
|
|
9190
|
-
if (!module.default || typeof module.default !== "function") {
|
|
9191
|
-
throw new Error(
|
|
9192
|
-
`Invalid bundle: ${file} must export a default factory function`
|
|
9193
|
-
);
|
|
9194
|
-
}
|
|
9195
|
-
logger?.debug?.("Calling factory function...");
|
|
9196
|
-
const result = await module.default(context2 ?? {});
|
|
9197
|
-
if (!result || !result.collector || typeof result.collector.push !== "function") {
|
|
9198
|
-
throw new Error(
|
|
9199
|
-
`Invalid bundle: factory must return { collector } with a push function`
|
|
9200
|
-
);
|
|
9201
|
-
}
|
|
9202
|
-
return {
|
|
9203
|
-
collector: result.collector,
|
|
9204
|
-
...typeof result.httpHandler === "function" ? { httpHandler: result.httpHandler } : {}
|
|
9205
|
-
};
|
|
9206
|
-
}
|
|
9207
|
-
|
|
9208
|
-
// src/runtime/runner.ts
|
|
9209
|
-
async function loadFlow(file, config, logger, loggerConfig, healthServer, observers) {
|
|
9210
|
-
const absolutePath = resolve3(file);
|
|
9211
|
-
const flowDir = dirname5(absolutePath);
|
|
9212
|
-
process.chdir(flowDir);
|
|
9213
|
-
const flowContext = {
|
|
9214
|
-
...config,
|
|
9215
|
-
...loggerConfig ? { logger: loggerConfig } : {},
|
|
9216
|
-
...healthServer ? { sourceSettings: { port: void 0 } } : {},
|
|
9217
|
-
...observers ? { observers } : {}
|
|
9218
|
-
};
|
|
9219
|
-
const result = await loadBundle(absolutePath, flowContext, logger);
|
|
9220
|
-
if (healthServer && typeof result.httpHandler === "function") {
|
|
9221
|
-
healthServer.setFlowHandler(result.httpHandler);
|
|
9222
|
-
}
|
|
9223
|
-
return {
|
|
9224
|
-
collector: {
|
|
9225
|
-
command: result.collector.command,
|
|
9226
|
-
status: result.collector.status
|
|
9227
|
-
},
|
|
9228
|
-
file,
|
|
9229
|
-
httpHandler: result.httpHandler
|
|
9230
|
-
};
|
|
9231
|
-
}
|
|
9232
|
-
async function swapFlow(currentHandle, newFile, config, logger, loggerConfig, healthServer, observers) {
|
|
9233
|
-
logger.info("Shutting down current flow for hot-swap...");
|
|
9234
|
-
if (healthServer) {
|
|
9235
|
-
healthServer.setFlowHandler(null);
|
|
9236
|
-
}
|
|
9237
|
-
try {
|
|
9238
|
-
if (currentHandle.collector.command) {
|
|
9239
|
-
await currentHandle.collector.command("shutdown");
|
|
9240
|
-
}
|
|
9241
|
-
} catch (error) {
|
|
9242
|
-
logger.debug(`Shutdown warning: ${error}`);
|
|
9243
|
-
}
|
|
9244
|
-
const newHandle = await loadFlow(
|
|
9245
|
-
newFile,
|
|
9246
|
-
config,
|
|
9247
|
-
logger,
|
|
9248
|
-
loggerConfig,
|
|
9249
|
-
healthServer,
|
|
9250
|
-
observers
|
|
9251
|
-
);
|
|
9252
|
-
logger.info("Flow swapped successfully");
|
|
9253
|
-
return newHandle;
|
|
9336
|
+
return out;
|
|
9337
|
+
}
|
|
9338
|
+
function maskLine(line) {
|
|
9339
|
+
let s6 = line;
|
|
9340
|
+
s6 = s6.replace(RE_URL_CREDS, "$1***$2");
|
|
9341
|
+
s6 = s6.replace(RE_KV_SECRET, "$1$2***");
|
|
9342
|
+
s6 = s6.replace(RE_PREFIXED_TOKEN, "***");
|
|
9343
|
+
s6 = s6.replace(
|
|
9344
|
+
RE_TOKEN_RUN,
|
|
9345
|
+
(match) => shouldMaskToken(match) ? "***" : match
|
|
9346
|
+
);
|
|
9347
|
+
return s6;
|
|
9348
|
+
}
|
|
9349
|
+
function redactLine(line) {
|
|
9350
|
+
const withoutJsonKey = line.replace(RE_JSON_PRIVATE_KEY, '$1"***"');
|
|
9351
|
+
const rawLines = withoutJsonKey.split("\n");
|
|
9352
|
+
const cleanLines = removePemBlocks(rawLines);
|
|
9353
|
+
const maskedLines = cleanLines.map(maskLine);
|
|
9354
|
+
const joined = maskedLines.join("\n");
|
|
9355
|
+
if (joined.length > MAX_LENGTH) {
|
|
9356
|
+
return joined.slice(0, MAX_LENGTH - 1) + "\u2026";
|
|
9357
|
+
}
|
|
9358
|
+
return joined;
|
|
9359
|
+
}
|
|
9360
|
+
function redactErrors(errors) {
|
|
9361
|
+
return errors.map((e4) => ({
|
|
9362
|
+
message: redactLine(e4.message),
|
|
9363
|
+
count: e4.count,
|
|
9364
|
+
firstSeen: new Date(e4.firstSeen).toISOString(),
|
|
9365
|
+
lastSeen: new Date(e4.lastSeen).toISOString()
|
|
9366
|
+
}));
|
|
9367
|
+
}
|
|
9368
|
+
function redactLogs(entries) {
|
|
9369
|
+
return entries.map((e4) => ({
|
|
9370
|
+
time: new Date(e4.time).toISOString(),
|
|
9371
|
+
level: e4.level,
|
|
9372
|
+
message: redactLine(e4.message)
|
|
9373
|
+
}));
|
|
9254
9374
|
}
|
|
9255
9375
|
|
|
9256
9376
|
// src/runtime/heartbeat.ts
|
|
9257
|
-
import { randomBytes } from "crypto";
|
|
9258
|
-
init_http();
|
|
9259
9377
|
function computeCounterDelta(current, last) {
|
|
9260
9378
|
const destinations = {};
|
|
9261
9379
|
for (const [name, dest] of Object.entries(current.destinations)) {
|
|
@@ -9316,6 +9434,8 @@ function createHeartbeat(config, logger) {
|
|
|
9316
9434
|
};
|
|
9317
9435
|
counters = computeCounterDelta(current, lastReported);
|
|
9318
9436
|
}
|
|
9437
|
+
const errors = config.getErrors ? redactErrors(config.getErrors()) : [];
|
|
9438
|
+
const logs = config.getLogs ? redactLogs(config.getLogs().slice(-50)) : [];
|
|
9319
9439
|
const response = await fetch(
|
|
9320
9440
|
`${config.appUrl}/api/projects/${config.projectId}/runners/heartbeat`,
|
|
9321
9441
|
{
|
|
@@ -9332,7 +9452,9 @@ function createHeartbeat(config, logger) {
|
|
|
9332
9452
|
configVersion,
|
|
9333
9453
|
cliVersion: VERSION,
|
|
9334
9454
|
uptime: Math.floor((Date.now() - startTime) / 1e3),
|
|
9335
|
-
...counters && { counters }
|
|
9455
|
+
...counters && { counters },
|
|
9456
|
+
...errors.length && { recentErrors: errors },
|
|
9457
|
+
...logs.length && { recentLogs: logs }
|
|
9336
9458
|
}),
|
|
9337
9459
|
signal: AbortSignal.timeout(1e4)
|
|
9338
9460
|
}
|
|
@@ -9407,6 +9529,270 @@ function createPoller(config, logger) {
|
|
|
9407
9529
|
return { start, stop, pollOnce };
|
|
9408
9530
|
}
|
|
9409
9531
|
|
|
9532
|
+
// src/runtime/health-server.ts
|
|
9533
|
+
import http from "http";
|
|
9534
|
+
function createHealthServer(port, logger) {
|
|
9535
|
+
return new Promise((resolve4, reject) => {
|
|
9536
|
+
let flowHandler = null;
|
|
9537
|
+
let ready = false;
|
|
9538
|
+
let failureReason = null;
|
|
9539
|
+
const server = http.createServer((req, res) => {
|
|
9540
|
+
if (req.url === "/health" && req.method === "GET") {
|
|
9541
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
9542
|
+
res.end(JSON.stringify({ status: "ok" }));
|
|
9543
|
+
return;
|
|
9544
|
+
}
|
|
9545
|
+
if (req.url === "/ready" && req.method === "GET") {
|
|
9546
|
+
const code = ready ? 200 : 503;
|
|
9547
|
+
const status = ready ? "ready" : failureReason ? "failed" : "not_ready";
|
|
9548
|
+
res.writeHead(code, { "Content-Type": "application/json" });
|
|
9549
|
+
res.end(
|
|
9550
|
+
JSON.stringify(
|
|
9551
|
+
failureReason && !ready ? { status, reason: failureReason } : { status }
|
|
9552
|
+
)
|
|
9553
|
+
);
|
|
9554
|
+
return;
|
|
9555
|
+
}
|
|
9556
|
+
if (flowHandler) {
|
|
9557
|
+
flowHandler(req, res);
|
|
9558
|
+
return;
|
|
9559
|
+
}
|
|
9560
|
+
res.writeHead(503, { "Content-Type": "application/json" });
|
|
9561
|
+
res.end(JSON.stringify({ error: "No flow loaded" }));
|
|
9562
|
+
});
|
|
9563
|
+
server.keepAliveTimeout = 5e3;
|
|
9564
|
+
server.headersTimeout = 1e4;
|
|
9565
|
+
server.listen(port, "0.0.0.0", () => {
|
|
9566
|
+
logger.info(`Health server listening on port ${port}`);
|
|
9567
|
+
resolve4({
|
|
9568
|
+
server,
|
|
9569
|
+
setFlowHandler(handler) {
|
|
9570
|
+
flowHandler = handler;
|
|
9571
|
+
},
|
|
9572
|
+
setReady(value) {
|
|
9573
|
+
ready = value;
|
|
9574
|
+
if (value) failureReason = null;
|
|
9575
|
+
},
|
|
9576
|
+
setFailed(reason) {
|
|
9577
|
+
ready = false;
|
|
9578
|
+
failureReason = reason;
|
|
9579
|
+
},
|
|
9580
|
+
close: () => new Promise((res, rej) => {
|
|
9581
|
+
server.close((err) => err ? rej(err) : res());
|
|
9582
|
+
})
|
|
9583
|
+
});
|
|
9584
|
+
});
|
|
9585
|
+
server.on("error", reject);
|
|
9586
|
+
});
|
|
9587
|
+
}
|
|
9588
|
+
|
|
9589
|
+
// src/runtime/secrets-fetcher.ts
|
|
9590
|
+
init_http();
|
|
9591
|
+
var SecretsHttpError = class extends Error {
|
|
9592
|
+
constructor(status, statusText) {
|
|
9593
|
+
super(`Failed to fetch secrets: ${status} ${statusText}`);
|
|
9594
|
+
this.status = status;
|
|
9595
|
+
this.name = "SecretsHttpError";
|
|
9596
|
+
}
|
|
9597
|
+
status;
|
|
9598
|
+
};
|
|
9599
|
+
async function fetchSecrets(options) {
|
|
9600
|
+
const { appUrl, token, projectId, flowId } = options;
|
|
9601
|
+
const url = `${appUrl}/api/projects/${encodeURIComponent(projectId)}/flows/${encodeURIComponent(flowId)}/secrets/values`;
|
|
9602
|
+
const res = await fetchWithRetry(url, {
|
|
9603
|
+
maxTotalMs: 2e4,
|
|
9604
|
+
init: {
|
|
9605
|
+
headers: mergeAuthHeaders(token, { "Content-Type": "application/json" })
|
|
9606
|
+
}
|
|
9607
|
+
});
|
|
9608
|
+
await throwIfRunnerAuthFailure(res);
|
|
9609
|
+
if (!res.ok) {
|
|
9610
|
+
throw new SecretsHttpError(res.status, res.statusText);
|
|
9611
|
+
}
|
|
9612
|
+
const json = await res.json();
|
|
9613
|
+
return json.values;
|
|
9614
|
+
}
|
|
9615
|
+
|
|
9616
|
+
// src/runtime/log-ring.ts
|
|
9617
|
+
var LogRing = class {
|
|
9618
|
+
constructor(max) {
|
|
9619
|
+
this.max = max;
|
|
9620
|
+
}
|
|
9621
|
+
max;
|
|
9622
|
+
entries = [];
|
|
9623
|
+
add(entry) {
|
|
9624
|
+
this.entries.push(entry);
|
|
9625
|
+
if (this.entries.length > this.max) this.entries.shift();
|
|
9626
|
+
}
|
|
9627
|
+
snapshot(limit = this.max) {
|
|
9628
|
+
return this.entries.slice(Math.max(0, this.entries.length - limit));
|
|
9629
|
+
}
|
|
9630
|
+
};
|
|
9631
|
+
var ErrorRing = class {
|
|
9632
|
+
constructor(maxUnique, now = () => Date.now()) {
|
|
9633
|
+
this.maxUnique = maxUnique;
|
|
9634
|
+
this.now = now;
|
|
9635
|
+
}
|
|
9636
|
+
maxUnique;
|
|
9637
|
+
now;
|
|
9638
|
+
map = /* @__PURE__ */ new Map();
|
|
9639
|
+
add(message) {
|
|
9640
|
+
const ts = this.now();
|
|
9641
|
+
const existing = this.map.get(message);
|
|
9642
|
+
if (existing) {
|
|
9643
|
+
existing.count += 1;
|
|
9644
|
+
existing.lastSeen = ts;
|
|
9645
|
+
return;
|
|
9646
|
+
}
|
|
9647
|
+
if (this.map.size >= this.maxUnique) {
|
|
9648
|
+
let oldestKey;
|
|
9649
|
+
let oldest = Infinity;
|
|
9650
|
+
for (const [k3, v3] of this.map) {
|
|
9651
|
+
if (v3.lastSeen < oldest) {
|
|
9652
|
+
oldest = v3.lastSeen;
|
|
9653
|
+
oldestKey = k3;
|
|
9654
|
+
}
|
|
9655
|
+
}
|
|
9656
|
+
if (oldestKey !== void 0) this.map.delete(oldestKey);
|
|
9657
|
+
}
|
|
9658
|
+
this.map.set(message, { message, count: 1, firstSeen: ts, lastSeen: ts });
|
|
9659
|
+
}
|
|
9660
|
+
snapshot() {
|
|
9661
|
+
return [...this.map.values()].sort((a4, b3) => b3.lastSeen - a4.lastSeen);
|
|
9662
|
+
}
|
|
9663
|
+
};
|
|
9664
|
+
|
|
9665
|
+
// src/commands/run/index.ts
|
|
9666
|
+
init_tmp();
|
|
9667
|
+
init_config_file();
|
|
9668
|
+
init_auth();
|
|
9669
|
+
init_cache();
|
|
9670
|
+
|
|
9671
|
+
// src/commands/run/validators.ts
|
|
9672
|
+
init_asset_resolver();
|
|
9673
|
+
import { existsSync as existsSync4 } from "fs";
|
|
9674
|
+
|
|
9675
|
+
// src/schemas/primitives.ts
|
|
9676
|
+
init_dev();
|
|
9677
|
+
var PortSchema = n3.number().int("Port must be an integer").min(1, "Port must be at least 1").max(65535, "Port must be at most 65535").describe("HTTP server port number");
|
|
9678
|
+
var FilePathSchema = n3.string().min(1, "File path cannot be empty").describe("Path to configuration file");
|
|
9679
|
+
|
|
9680
|
+
// src/schemas/run.ts
|
|
9681
|
+
init_dev();
|
|
9682
|
+
var RunOptionsSchema = n3.object({
|
|
9683
|
+
flow: FilePathSchema,
|
|
9684
|
+
port: PortSchema.default(8080),
|
|
9685
|
+
flowName: n3.string().optional().describe("Specific flow name to run")
|
|
9686
|
+
});
|
|
9687
|
+
|
|
9688
|
+
// src/schemas/validate.ts
|
|
9689
|
+
init_dev();
|
|
9690
|
+
var ValidationTypeSchema = n3.enum(["contract", "event", "flow", "mapping"]).describe('Validation type: "event", "flow", "mapping", or "contract"');
|
|
9691
|
+
var ValidateOptionsSchema = n3.object({
|
|
9692
|
+
flow: n3.string().optional().describe("Flow name for multi-flow configs"),
|
|
9693
|
+
path: n3.string().optional().describe(
|
|
9694
|
+
'Entry path for package schema validation (e.g., "destinations.snowplow", "sources.browser")'
|
|
9695
|
+
)
|
|
9696
|
+
});
|
|
9697
|
+
var ValidateInputShape = {
|
|
9698
|
+
type: ValidationTypeSchema,
|
|
9699
|
+
input: n3.string().min(1).describe("JSON string, file path, or URL to validate"),
|
|
9700
|
+
flow: n3.string().optional().describe("Flow name for multi-flow configs"),
|
|
9701
|
+
path: n3.string().optional().describe(
|
|
9702
|
+
'Entry path for package schema validation (e.g., "destinations.snowplow"). When provided, validates the entry against its package JSON Schema instead of using --type.'
|
|
9703
|
+
)
|
|
9704
|
+
};
|
|
9705
|
+
var ValidateInputSchema = n3.object(ValidateInputShape);
|
|
9706
|
+
|
|
9707
|
+
// src/schemas/bundle.ts
|
|
9708
|
+
init_dev();
|
|
9709
|
+
var BundleOptionsSchema = n3.object({
|
|
9710
|
+
silent: n3.boolean().optional().describe("Suppress all output"),
|
|
9711
|
+
verbose: n3.boolean().optional().describe("Enable verbose logging"),
|
|
9712
|
+
stats: n3.boolean().optional().default(true).describe("Return bundle statistics"),
|
|
9713
|
+
cache: n3.boolean().optional().default(true).describe("Enable package caching"),
|
|
9714
|
+
flowName: n3.string().optional().describe("Flow name for multi-flow configs")
|
|
9715
|
+
});
|
|
9716
|
+
var BundleInputShape = {
|
|
9717
|
+
configPath: FilePathSchema.describe(
|
|
9718
|
+
"Path to flow configuration file (JSON or JavaScript), URL, or inline JSON string"
|
|
9719
|
+
),
|
|
9720
|
+
flow: n3.string().optional().describe("Flow name for multi-flow configs"),
|
|
9721
|
+
stats: n3.boolean().optional().default(true).describe("Return bundle statistics"),
|
|
9722
|
+
output: n3.string().optional().describe("Output file path (defaults to config-defined)")
|
|
9723
|
+
};
|
|
9724
|
+
var BundleInputSchema = n3.object(BundleInputShape);
|
|
9725
|
+
|
|
9726
|
+
// src/schemas/simulate.ts
|
|
9727
|
+
init_dev();
|
|
9728
|
+
var PlatformSchema = n3.enum(["web", "server"]).describe("Platform type for event processing");
|
|
9729
|
+
var SimulateOptionsSchema = n3.object({
|
|
9730
|
+
silent: n3.boolean().optional().describe("Suppress all output"),
|
|
9731
|
+
verbose: n3.boolean().optional().describe("Enable verbose logging"),
|
|
9732
|
+
json: n3.boolean().optional().describe("Format output as JSON")
|
|
9733
|
+
});
|
|
9734
|
+
var SimulateInputShape = {
|
|
9735
|
+
configPath: FilePathSchema.describe(
|
|
9736
|
+
"Path to flow configuration file, URL, or inline JSON string"
|
|
9737
|
+
),
|
|
9738
|
+
event: n3.string().min(1).optional().describe(
|
|
9739
|
+
"Event as JSON string, file path, or URL. For sources: { content, trigger?, env? }."
|
|
9740
|
+
),
|
|
9741
|
+
flow: n3.string().optional().describe("Flow name for multi-flow configs"),
|
|
9742
|
+
platform: PlatformSchema.optional().describe("Override platform detection"),
|
|
9743
|
+
step: n3.string().optional().describe(
|
|
9744
|
+
'Step target in type.name format (e.g. "source.browser", "destination.gtag")'
|
|
9745
|
+
)
|
|
9746
|
+
};
|
|
9747
|
+
var SimulateInputSchema = n3.object(SimulateInputShape);
|
|
9748
|
+
|
|
9749
|
+
// src/schemas/push.ts
|
|
9750
|
+
init_dev();
|
|
9751
|
+
var PushOptionsSchema = n3.object({
|
|
9752
|
+
silent: n3.boolean().optional().describe("Suppress all output"),
|
|
9753
|
+
verbose: n3.boolean().optional().describe("Enable verbose logging"),
|
|
9754
|
+
json: n3.boolean().optional().describe("Format output as JSON")
|
|
9755
|
+
});
|
|
9756
|
+
var PushInputShape = {
|
|
9757
|
+
configPath: FilePathSchema.describe("Path to flow configuration file"),
|
|
9758
|
+
event: n3.string().min(1).describe("Event as JSON string, file path, or URL"),
|
|
9759
|
+
flow: n3.string().optional().describe("Flow name for multi-flow configs"),
|
|
9760
|
+
platform: PlatformSchema.optional().describe("Override platform detection")
|
|
9761
|
+
};
|
|
9762
|
+
var PushInputSchema = n3.object(PushInputShape);
|
|
9763
|
+
|
|
9764
|
+
// src/commands/run/validators.ts
|
|
9765
|
+
function validateFlowFile(filePath) {
|
|
9766
|
+
const absolutePath = resolveAsset(filePath, "bundle");
|
|
9767
|
+
if (!existsSync4(absolutePath)) {
|
|
9768
|
+
throw new Error(
|
|
9769
|
+
`Flow file not found: ${filePath}
|
|
9770
|
+
Resolved path: ${absolutePath}
|
|
9771
|
+
Make sure the file exists and the path is correct`
|
|
9772
|
+
);
|
|
9773
|
+
}
|
|
9774
|
+
return absolutePath;
|
|
9775
|
+
}
|
|
9776
|
+
function validatePort(port) {
|
|
9777
|
+
const result = PortSchema.safeParse(port);
|
|
9778
|
+
if (!result.success) {
|
|
9779
|
+
throw new Error(
|
|
9780
|
+
`Invalid port: ${port}
|
|
9781
|
+
Port must be an integer between 1 and 65535
|
|
9782
|
+
Example: --port 8080`
|
|
9783
|
+
);
|
|
9784
|
+
}
|
|
9785
|
+
}
|
|
9786
|
+
|
|
9787
|
+
// src/commands/run/index.ts
|
|
9788
|
+
init_utils3();
|
|
9789
|
+
|
|
9790
|
+
// src/commands/run/pipeline.ts
|
|
9791
|
+
init_dist();
|
|
9792
|
+
init_tmp();
|
|
9793
|
+
import { writeFileSync as writeFileSync4 } from "fs";
|
|
9794
|
+
import fs17 from "fs-extra";
|
|
9795
|
+
|
|
9410
9796
|
// src/runtime/trace-poller.ts
|
|
9411
9797
|
init_dist();
|
|
9412
9798
|
init_http();
|
|
@@ -9469,42 +9855,26 @@ function createTracePoller(config, logger) {
|
|
|
9469
9855
|
}
|
|
9470
9856
|
var defaultFetch = (url, init) => fetch(url, init);
|
|
9471
9857
|
|
|
9472
|
-
// src/runtime/secrets-fetcher.ts
|
|
9473
|
-
init_http();
|
|
9474
|
-
var SecretsHttpError = class extends Error {
|
|
9475
|
-
constructor(status, statusText) {
|
|
9476
|
-
super(`Failed to fetch secrets: ${status} ${statusText}`);
|
|
9477
|
-
this.status = status;
|
|
9478
|
-
this.name = "SecretsHttpError";
|
|
9479
|
-
}
|
|
9480
|
-
status;
|
|
9481
|
-
};
|
|
9482
|
-
async function fetchSecrets(options) {
|
|
9483
|
-
const { appUrl, token, projectId, flowId } = options;
|
|
9484
|
-
const url = `${appUrl}/api/projects/${encodeURIComponent(projectId)}/flows/${encodeURIComponent(flowId)}/secrets/values`;
|
|
9485
|
-
const res = await fetch(url, {
|
|
9486
|
-
headers: mergeAuthHeaders(token, { "Content-Type": "application/json" })
|
|
9487
|
-
});
|
|
9488
|
-
await throwIfRunnerAuthFailure(res);
|
|
9489
|
-
if (!res.ok) {
|
|
9490
|
-
throw new SecretsHttpError(res.status, res.statusText);
|
|
9491
|
-
}
|
|
9492
|
-
const json = await res.json();
|
|
9493
|
-
return json.values;
|
|
9494
|
-
}
|
|
9495
|
-
|
|
9496
9858
|
// src/commands/run/pipeline.ts
|
|
9497
9859
|
init_cache();
|
|
9498
9860
|
async function runPipeline(options) {
|
|
9499
9861
|
const { bundlePath, port, logger, loggerConfig, api } = options;
|
|
9500
9862
|
let configVersion;
|
|
9863
|
+
const configFrozen = readConfigFrozen();
|
|
9501
9864
|
if (api) {
|
|
9502
9865
|
await injectSecrets(api, logger);
|
|
9503
9866
|
}
|
|
9504
9867
|
logger.info(`walkeros/flow v${VERSION}`);
|
|
9505
9868
|
logger.info(`Instance: ${getInstanceId()}`);
|
|
9869
|
+
if (configFrozen) {
|
|
9870
|
+
logger.info("Config frozen: hot-swap and heartbeat disabled");
|
|
9871
|
+
}
|
|
9506
9872
|
const healthServer = await createHealthServer(port, logger);
|
|
9507
|
-
const
|
|
9873
|
+
const observeLevel = readObserveLevel(logger);
|
|
9874
|
+
const telemetryObservers = buildTelemetryObservers(
|
|
9875
|
+
api?.flowId ?? "flow",
|
|
9876
|
+
observeLevel
|
|
9877
|
+
);
|
|
9508
9878
|
const runtimeConfig = { port };
|
|
9509
9879
|
let handle;
|
|
9510
9880
|
try {
|
|
@@ -9533,20 +9903,24 @@ async function runPipeline(options) {
|
|
|
9533
9903
|
const ingestToken = process.env.WALKEROS_INGEST_TOKEN;
|
|
9534
9904
|
const deploymentId = process.env.WALKEROS_DEPLOYMENT_ID;
|
|
9535
9905
|
if (observerBase && ingestToken && deploymentId) {
|
|
9536
|
-
|
|
9537
|
-
|
|
9538
|
-
|
|
9539
|
-
|
|
9540
|
-
|
|
9541
|
-
|
|
9542
|
-
|
|
9543
|
-
|
|
9544
|
-
|
|
9545
|
-
|
|
9906
|
+
if (observeLevel === "trace") {
|
|
9907
|
+
logger.info("Trace poller: skipped (observe level is trace)");
|
|
9908
|
+
} else {
|
|
9909
|
+
tracePoller = createTracePoller(
|
|
9910
|
+
{
|
|
9911
|
+
url: `${observerBase}/trace/${deploymentId}`,
|
|
9912
|
+
token: ingestToken,
|
|
9913
|
+
intervalMs: 15e3
|
|
9914
|
+
},
|
|
9915
|
+
logger
|
|
9916
|
+
);
|
|
9917
|
+
tracePoller.start();
|
|
9918
|
+
logger.info("Trace poller: active (every 15s)");
|
|
9919
|
+
}
|
|
9546
9920
|
}
|
|
9547
9921
|
let currentBundleCleanup;
|
|
9548
9922
|
let currentConfigPath;
|
|
9549
|
-
if (api) {
|
|
9923
|
+
if (api && !configFrozen) {
|
|
9550
9924
|
heartbeat = createHeartbeat(
|
|
9551
9925
|
{
|
|
9552
9926
|
appUrl: api.appUrl,
|
|
@@ -9556,7 +9930,9 @@ async function runPipeline(options) {
|
|
|
9556
9930
|
deploymentId: api.deploymentId,
|
|
9557
9931
|
configVersion,
|
|
9558
9932
|
intervalMs: api.heartbeatIntervalMs,
|
|
9559
|
-
getCounters: () => handle.collector.status
|
|
9933
|
+
getCounters: () => handle.collector.status,
|
|
9934
|
+
getErrors: () => options.errorRing?.snapshot() ?? [],
|
|
9935
|
+
getLogs: () => options.logRing?.snapshot() ?? []
|
|
9560
9936
|
},
|
|
9561
9937
|
logger
|
|
9562
9938
|
);
|
|
@@ -9663,17 +10039,39 @@ async function runPipeline(options) {
|
|
|
9663
10039
|
await new Promise(() => {
|
|
9664
10040
|
});
|
|
9665
10041
|
}
|
|
9666
|
-
|
|
10042
|
+
var OBSERVE_LEVELS = [
|
|
10043
|
+
"off",
|
|
10044
|
+
"standard",
|
|
10045
|
+
"trace"
|
|
10046
|
+
];
|
|
10047
|
+
function readConfigFrozen() {
|
|
10048
|
+
const raw = process.env.WALKEROS_CONFIG_FROZEN;
|
|
10049
|
+
return raw === "1" || raw === "true";
|
|
10050
|
+
}
|
|
10051
|
+
function readObserveLevel(logger) {
|
|
10052
|
+
const raw = process.env.WALKEROS_OBSERVE_LEVEL;
|
|
10053
|
+
if (raw === void 0 || raw === "") return void 0;
|
|
10054
|
+
const level = OBSERVE_LEVELS.find((candidate) => candidate === raw);
|
|
10055
|
+
if (!level) {
|
|
10056
|
+
logger.warn(
|
|
10057
|
+
`Ignoring invalid WALKEROS_OBSERVE_LEVEL "${raw}" (expected off, standard, or trace)`
|
|
10058
|
+
);
|
|
10059
|
+
return void 0;
|
|
10060
|
+
}
|
|
10061
|
+
return level;
|
|
10062
|
+
}
|
|
10063
|
+
function buildTelemetryObservers(flowId, observeLevel) {
|
|
9667
10064
|
const base = process.env.WALKEROS_OBSERVER_URL;
|
|
9668
10065
|
const token = process.env.WALKEROS_INGEST_TOKEN;
|
|
9669
10066
|
const deploymentId = process.env.WALKEROS_DEPLOYMENT_ID;
|
|
9670
10067
|
if (!base || !token || !deploymentId) return void 0;
|
|
9671
10068
|
const url = `${base}/ingest/${deploymentId}`;
|
|
9672
10069
|
const emit = ht({ url, token });
|
|
10070
|
+
const observe = observeLevel !== void 0 ? { level: observeLevel } : void 0;
|
|
9673
10071
|
return [
|
|
9674
10072
|
ft(
|
|
9675
10073
|
emit,
|
|
9676
|
-
() => pt({ flowId, traceUntil: mt() })
|
|
10074
|
+
() => pt({ flowId, observe, traceUntil: mt() })
|
|
9677
10075
|
)
|
|
9678
10076
|
];
|
|
9679
10077
|
}
|
|
@@ -9716,7 +10114,21 @@ async function lazyPrepareBundleForRun(configPath, options) {
|
|
|
9716
10114
|
async function runCommand(options) {
|
|
9717
10115
|
const timer = createTimer();
|
|
9718
10116
|
timer.start();
|
|
9719
|
-
const
|
|
10117
|
+
const errorRing = new ErrorRing(20);
|
|
10118
|
+
const logRing = new LogRing(100);
|
|
10119
|
+
const LEVEL_NAME = {
|
|
10120
|
+
[l.ERROR]: "error",
|
|
10121
|
+
[l.WARN]: "warn",
|
|
10122
|
+
[l.INFO]: "info",
|
|
10123
|
+
[l.DEBUG]: "debug"
|
|
10124
|
+
};
|
|
10125
|
+
const logger = createCLILogger({
|
|
10126
|
+
...options,
|
|
10127
|
+
onLine: (level, message) => {
|
|
10128
|
+
if (level === l.ERROR) errorRing.add(message);
|
|
10129
|
+
logRing.add({ time: Date.now(), level: LEVEL_NAME[level], message });
|
|
10130
|
+
}
|
|
10131
|
+
});
|
|
9720
10132
|
try {
|
|
9721
10133
|
if (options.envFile) {
|
|
9722
10134
|
const { loadEnvFile: loadEnvFile2 } = await Promise.resolve().then(() => (init_env_file(), env_file_exports));
|
|
@@ -9786,7 +10198,9 @@ async function runCommand(options) {
|
|
|
9786
10198
|
port,
|
|
9787
10199
|
logger: logger.scope("runner"),
|
|
9788
10200
|
loggerConfig: options.verbose ? { level: 0 } : void 0,
|
|
9789
|
-
api: apiConfig
|
|
10201
|
+
api: apiConfig,
|
|
10202
|
+
errorRing,
|
|
10203
|
+
logRing
|
|
9790
10204
|
});
|
|
9791
10205
|
} catch (error) {
|
|
9792
10206
|
const duration = timer.getElapsed() / 1e3;
|
|
@@ -11940,7 +12354,7 @@ function validateMapping(input) {
|
|
|
11940
12354
|
// src/commands/validate/validators/entry.ts
|
|
11941
12355
|
init_dist();
|
|
11942
12356
|
import Ajv from "ajv";
|
|
11943
|
-
var CLIENT_HEADER = "walkeros-cli/4.2.
|
|
12357
|
+
var CLIENT_HEADER = "walkeros-cli/4.2.1-next-1781538735002";
|
|
11944
12358
|
var SECTIONS = ["destinations", "sources", "transformers"];
|
|
11945
12359
|
function resolveEntry(path19, flowConfig) {
|
|
11946
12360
|
const flows = flowConfig.flows;
|
|
@@ -12575,6 +12989,7 @@ init_sse();
|
|
|
12575
12989
|
init_cli_logger();
|
|
12576
12990
|
init_output();
|
|
12577
12991
|
init_flows();
|
|
12992
|
+
import { randomUUID as randomUUID2 } from "crypto";
|
|
12578
12993
|
async function resolveSettingsId(options) {
|
|
12579
12994
|
const flow = await getFlow({
|
|
12580
12995
|
flowId: options.flowId,
|
|
@@ -12600,8 +13015,9 @@ async function getAvailableFlowNames(options) {
|
|
|
12600
13015
|
const settings = flow.settings;
|
|
12601
13016
|
return settings?.map((c2) => c2.name) ?? [];
|
|
12602
13017
|
}
|
|
13018
|
+
var DEFAULT_DEPLOY_WAIT_MS = 12 * 60 * 1e3;
|
|
12603
13019
|
async function streamDeploymentStatus(projectId, deploymentId, options) {
|
|
12604
|
-
const timeoutMs = options.timeout ??
|
|
13020
|
+
const timeoutMs = options.timeout ?? DEFAULT_DEPLOY_WAIT_MS;
|
|
12605
13021
|
const response = await apiFetch(
|
|
12606
13022
|
`/api/projects/${projectId}/deployments/${deploymentId}/stream`,
|
|
12607
13023
|
{
|
|
@@ -12660,7 +13076,10 @@ async function deploy(options) {
|
|
|
12660
13076
|
}
|
|
12661
13077
|
const { data, error } = await client.POST(
|
|
12662
13078
|
"/api/projects/{projectId}/flows/{flowId}/deploy",
|
|
12663
|
-
{
|
|
13079
|
+
{
|
|
13080
|
+
params: { path: { projectId, flowId: options.flowId } },
|
|
13081
|
+
headers: { "Idempotency-Key": randomUUID2() }
|
|
13082
|
+
}
|
|
12664
13083
|
);
|
|
12665
13084
|
if (error) {
|
|
12666
13085
|
try {
|
|
@@ -12690,13 +13109,38 @@ Available: ${names.join(", ")}`,
|
|
|
12690
13109
|
}
|
|
12691
13110
|
async function deploySettings(options) {
|
|
12692
13111
|
const { flowId, projectId, settingsId } = options;
|
|
12693
|
-
const
|
|
13112
|
+
const triggerDeploy = () => apiFetch(
|
|
12694
13113
|
`/api/projects/${projectId}/flows/${flowId}/settings/${settingsId}/deploy`,
|
|
12695
|
-
{ method: "POST" }
|
|
13114
|
+
{ method: "POST", headers: { "Idempotency-Key": randomUUID2() } }
|
|
12696
13115
|
);
|
|
13116
|
+
let response = await triggerDeploy();
|
|
13117
|
+
if (!response.ok && options.wait) {
|
|
13118
|
+
let retryBody = {};
|
|
13119
|
+
try {
|
|
13120
|
+
retryBody = await response.clone().json();
|
|
13121
|
+
} catch {
|
|
13122
|
+
retryBody = {};
|
|
13123
|
+
}
|
|
13124
|
+
try {
|
|
13125
|
+
throwApiResponseError(
|
|
13126
|
+
response,
|
|
13127
|
+
retryBody,
|
|
13128
|
+
`Deploy failed (${response.status})`
|
|
13129
|
+
);
|
|
13130
|
+
} catch (e4) {
|
|
13131
|
+
if (e4 instanceof ApiError && e4.retryable) {
|
|
13132
|
+
const waitSeconds = Math.min(e4.retryAfterSeconds ?? 5, 60);
|
|
13133
|
+
options.onStatus?.("rate_limited", `retrying in ${waitSeconds}s`);
|
|
13134
|
+
await new Promise((resolve4) => setTimeout(resolve4, waitSeconds * 1e3));
|
|
13135
|
+
response = await triggerDeploy();
|
|
13136
|
+
} else {
|
|
13137
|
+
throw e4;
|
|
13138
|
+
}
|
|
13139
|
+
}
|
|
13140
|
+
}
|
|
12697
13141
|
if (!response.ok) {
|
|
12698
13142
|
const body = await response.json().catch(() => ({}));
|
|
12699
|
-
|
|
13143
|
+
throwApiResponseError(response, body, `Deploy failed (${response.status})`);
|
|
12700
13144
|
}
|
|
12701
13145
|
const data = await response.json();
|
|
12702
13146
|
if (!options.wait) return data;
|
|
@@ -12708,16 +13152,20 @@ async function deploySettings(options) {
|
|
|
12708
13152
|
return { ...data, ...result };
|
|
12709
13153
|
}
|
|
12710
13154
|
var statusLabels = {
|
|
12711
|
-
|
|
12712
|
-
"
|
|
12713
|
-
"bundling:publishing": "Publishing to web...",
|
|
12714
|
-
deploying: "Deploying container...",
|
|
13155
|
+
"deploying:building": "Building bundle...",
|
|
13156
|
+
"deploying:publishing": "Publishing to web...",
|
|
12715
13157
|
"deploying:provisioning": "Provisioning container...",
|
|
12716
13158
|
"deploying:starting": "Starting container...",
|
|
13159
|
+
deploying: "Deploying...",
|
|
12717
13160
|
active: "Container is live",
|
|
12718
13161
|
published: "Published",
|
|
13162
|
+
stopped: "Stopped",
|
|
12719
13163
|
failed: "Deployment failed"
|
|
12720
13164
|
};
|
|
13165
|
+
function renderStatusLabel(status, substatus) {
|
|
13166
|
+
const key = substatus ? `${status}:${substatus}` : status;
|
|
13167
|
+
return statusLabels[key] || statusLabels[status] || `Status: ${status}`;
|
|
13168
|
+
}
|
|
12721
13169
|
async function deployCommand(flowId, options) {
|
|
12722
13170
|
const log = createCLILogger(options);
|
|
12723
13171
|
const timeoutMs = options.timeout ? parseInt(options.timeout, 10) * 1e3 : void 0;
|
|
@@ -12729,10 +13177,7 @@ async function deployCommand(flowId, options) {
|
|
|
12729
13177
|
wait: options.wait !== false,
|
|
12730
13178
|
timeout: timeoutMs,
|
|
12731
13179
|
onStatus: options.json ? void 0 : (status, substatus) => {
|
|
12732
|
-
|
|
12733
|
-
log.info(
|
|
12734
|
-
statusLabels[key] || statusLabels[status] || `Status: ${status}`
|
|
12735
|
-
);
|
|
13180
|
+
log.info(renderStatusLabel(status, substatus));
|
|
12736
13181
|
}
|
|
12737
13182
|
});
|
|
12738
13183
|
if (options.json) {
|
|
@@ -12747,7 +13192,7 @@ async function deployCommand(flowId, options) {
|
|
|
12747
13192
|
} else if (r5.status === "failed") {
|
|
12748
13193
|
log.error(`Failed: ${r5.errorMessage || "Unknown error"}`);
|
|
12749
13194
|
process.exit(1);
|
|
12750
|
-
} else if (r5.status === "
|
|
13195
|
+
} else if (r5.status === "deploying") {
|
|
12751
13196
|
log.info(`Deployment started: ${r5.deploymentId} (${r5.type})`);
|
|
12752
13197
|
} else {
|
|
12753
13198
|
log.info(`Status: ${r5.status}`);
|
|
@@ -12902,20 +13347,16 @@ async function createDeployCommand(config, options) {
|
|
|
12902
13347
|
log.info(`Deployment created: ${result.id}`);
|
|
12903
13348
|
log.info(` Slug: ${result.slug}`);
|
|
12904
13349
|
log.info(` Type: ${result.type}`);
|
|
12905
|
-
if (result.deployToken) {
|
|
12906
|
-
log.info(` Token: ${result.deployToken}`);
|
|
12907
|
-
log.warn(" Save this token \u2014 it will not be shown again.");
|
|
12908
|
-
}
|
|
12909
13350
|
log.info("");
|
|
12910
13351
|
log.info("Run locally:");
|
|
12911
13352
|
log.info(
|
|
12912
13353
|
` walkeros run ${isRemoteFlow ? "flow.json" : config} --deploy ${result.id}`
|
|
12913
13354
|
);
|
|
12914
13355
|
log.info("");
|
|
12915
|
-
log.info("
|
|
12916
|
-
log.info(
|
|
12917
|
-
|
|
12918
|
-
);
|
|
13356
|
+
log.info("Create a deploy token for this flow in the app");
|
|
13357
|
+
log.info("(Settings, Self-hosted deploy token) and set it as");
|
|
13358
|
+
log.info("WALKEROS_DEPLOY_TOKEN, then run with Docker:");
|
|
13359
|
+
log.info(" docker run -e WALKEROS_DEPLOY_TOKEN \\");
|
|
12919
13360
|
log.info(" -e WALKEROS_APP_URL=https://app.walkeros.io \\");
|
|
12920
13361
|
log.info(" walkeros/flow:latest");
|
|
12921
13362
|
} catch (err) {
|
|
@@ -13231,7 +13672,7 @@ deployCmd.command("create [config]").description(
|
|
|
13231
13672
|
});
|
|
13232
13673
|
deployCmd.command("start <flowId>").description("Deploy a flow to walkerOS cloud (auto-detects web or server)").option("--project <id>", "project ID (defaults to WALKEROS_PROJECT_ID)").option("-f, --flow <name>", "flow name for multi-config flows").option("--no-wait", "do not wait for deployment to complete").option(
|
|
13233
13674
|
"--timeout <seconds>",
|
|
13234
|
-
"timeout for deployment polling (default:
|
|
13675
|
+
"timeout in seconds for deployment polling (default: 720, the 12-minute server deploy budget)"
|
|
13235
13676
|
).option("-o, --output <path>", "output file path").option("--json", "output as JSON").option("-v, --verbose", "verbose output").option("-s, --silent", "suppress output").action(async (flowId, options) => {
|
|
13236
13677
|
await deployCommand(flowId, options);
|
|
13237
13678
|
});
|