@datarails-ai/dr-cli 1.6.153 → 1.6.155
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +1762 -1476
- package/dist/scopes-map.json +25 -1
- package/package.json +1 -1
package/dist/cli.js
CHANGED
|
@@ -8,7 +8,7 @@ import { fileURLToPath as fileURLToPath3 } from "url";
|
|
|
8
8
|
// package.json
|
|
9
9
|
var package_default = {
|
|
10
10
|
name: "@datarails-ai/dr-cli",
|
|
11
|
-
version: "1.6.
|
|
11
|
+
version: "1.6.155",
|
|
12
12
|
description: "Datarails CLI",
|
|
13
13
|
type: "module",
|
|
14
14
|
engines: {
|
|
@@ -69,7 +69,7 @@ var package_default = {
|
|
|
69
69
|
};
|
|
70
70
|
|
|
71
71
|
// src/buildInfo.ts
|
|
72
|
-
var BUILD_TIME = "2026-10-
|
|
72
|
+
var BUILD_TIME = "2026-10-23T09:17:56.480Z";
|
|
73
73
|
function formatBuildTime(iso) {
|
|
74
74
|
if (iso === "") return "";
|
|
75
75
|
const d = new Date(iso);
|
|
@@ -4317,6 +4317,13 @@ var PROD_GATE_TIER_A = /* @__PURE__ */ new Set([
|
|
|
4317
4317
|
// (`connector-poll|seed` / `|set-seed`, DR-55448).
|
|
4318
4318
|
"connector-poll|set-land",
|
|
4319
4319
|
"connector-poll|resume-land",
|
|
4320
|
+
// connectors AWS role (dr connectors aws delete / rotate-external-id, DR-56412). A delete
|
|
4321
|
+
// drops the role and its stored credentials; re-creating it mints a new external ID. A
|
|
4322
|
+
// rotation discards the old external ID for good. Either way AWS refuses the connection
|
|
4323
|
+
// until the customer edits their role's trust policy, the same outside-Datarails step as
|
|
4324
|
+
// `integration|revoke`.
|
|
4325
|
+
"aws-connection|delete",
|
|
4326
|
+
"aws-connection|rotate-external-id",
|
|
4320
4327
|
// Agent Studio (dr agents …, DR-55958). An agent's reach is not per-tenant: listing
|
|
4321
4328
|
// offers it to every consumer, global makes it exist in every environment, and
|
|
4322
4329
|
// visibility decides who sees it across this one, so each flip moves access for every
|
|
@@ -46478,12 +46485,15 @@ async function listPollItems(client, queryId, query = {}, reqOptions) {
|
|
|
46478
46485
|
}
|
|
46479
46486
|
return out;
|
|
46480
46487
|
}
|
|
46481
|
-
|
|
46482
|
-
|
|
46488
|
+
function outputQuery(outputId) {
|
|
46489
|
+
return outputId === void 0 ? "" : `?output=${outputId}`;
|
|
46490
|
+
}
|
|
46491
|
+
async function retryPollItem(client, queryId, itemId, reqOptions, outputId) {
|
|
46492
|
+
const path13 = `/connector-queries/${queryId}/poll/items/${itemId}/retry${outputQuery(outputId)}`;
|
|
46483
46493
|
return postPollJSON(client, path13, {}, "poll item retry", reqOptions);
|
|
46484
46494
|
}
|
|
46485
|
-
async function skipPollItem(client, queryId, itemId, reqOptions) {
|
|
46486
|
-
const path13 = `/connector-queries/${queryId}/poll/items/${itemId}/skip`;
|
|
46495
|
+
async function skipPollItem(client, queryId, itemId, reqOptions, outputId) {
|
|
46496
|
+
const path13 = `/connector-queries/${queryId}/poll/items/${itemId}/skip${outputQuery(outputId)}`;
|
|
46487
46497
|
return postPollJSON(client, path13, {}, "poll item skip", reqOptions);
|
|
46488
46498
|
}
|
|
46489
46499
|
function toControlPlaneError(method, path13, err) {
|
|
@@ -46546,1542 +46556,792 @@ async function updateAwsConnection(client, id, fields, reqOptions) {
|
|
|
46546
46556
|
"aws connection update"
|
|
46547
46557
|
);
|
|
46548
46558
|
}
|
|
46549
|
-
|
|
46550
|
-
|
|
46551
|
-
|
|
46552
|
-
|
|
46553
|
-
// src/api/scheduledTasks.ts
|
|
46554
|
-
var BASE2 = "/scheduled-tasks";
|
|
46555
|
-
async function call2(method, path13, send, what) {
|
|
46556
|
-
let out;
|
|
46557
|
-
try {
|
|
46558
|
-
out = await send();
|
|
46559
|
-
} catch (err) {
|
|
46560
|
-
throw toControlPlaneError(method, path13, err);
|
|
46561
|
-
}
|
|
46562
|
-
if (out === null) throw new Error(`${method} ${path13}: empty response from control-plane ${what}`);
|
|
46563
|
-
return out;
|
|
46564
|
-
}
|
|
46565
|
-
async function createTask(client, fields, reqOptions) {
|
|
46566
|
-
return call2(
|
|
46559
|
+
async function rotateAwsExternalId(client, id, reqOptions) {
|
|
46560
|
+
const path13 = `/aws-connections/${id}/rotate-external-id`;
|
|
46561
|
+
return call(
|
|
46567
46562
|
"POST",
|
|
46568
|
-
BASE2,
|
|
46569
|
-
() => client.postJSON(BASE2, fields, reqOptions),
|
|
46570
|
-
"task create"
|
|
46571
|
-
);
|
|
46572
|
-
}
|
|
46573
|
-
async function listTasks(client, orgId, reqOptions) {
|
|
46574
|
-
const path13 = orgId === void 0 ? BASE2 : `${BASE2}?org_id=${orgId}`;
|
|
46575
|
-
return call2("GET", path13, () => client.getJSON(path13, reqOptions), "task list");
|
|
46576
|
-
}
|
|
46577
|
-
async function getTask(client, taskId, reqOptions) {
|
|
46578
|
-
const path13 = `/scheduled-tasks/${encodeURIComponent(taskId)}`;
|
|
46579
|
-
return call2("GET", path13, () => client.getJSON(path13, reqOptions), "task get");
|
|
46580
|
-
}
|
|
46581
|
-
async function updateTask(client, taskId, fields, reqOptions) {
|
|
46582
|
-
const path13 = `/scheduled-tasks/${encodeURIComponent(taskId)}`;
|
|
46583
|
-
return call2(
|
|
46584
|
-
"PATCH",
|
|
46585
46563
|
path13,
|
|
46586
|
-
() => client.
|
|
46587
|
-
"
|
|
46564
|
+
() => client.postJSON(path13, {}, reqOptions),
|
|
46565
|
+
"aws connection rotate-external-id"
|
|
46588
46566
|
);
|
|
46589
46567
|
}
|
|
46590
|
-
async function
|
|
46591
|
-
const path13 = `/
|
|
46568
|
+
async function deleteAwsConnection(client, id, reqOptions) {
|
|
46569
|
+
const path13 = `/aws-connections/${id}`;
|
|
46592
46570
|
try {
|
|
46593
46571
|
await client.deleteJSON(path13, reqOptions);
|
|
46594
46572
|
} catch (err) {
|
|
46595
46573
|
throw toControlPlaneError("DELETE", path13, err);
|
|
46596
46574
|
}
|
|
46597
46575
|
}
|
|
46598
|
-
async function runTask(client, taskId, reqOptions) {
|
|
46599
|
-
const path13 = `/scheduled-tasks/${encodeURIComponent(taskId)}/run`;
|
|
46600
|
-
return call2(
|
|
46601
|
-
"POST",
|
|
46602
|
-
path13,
|
|
46603
|
-
() => client.postJSON(path13, {}, reqOptions),
|
|
46604
|
-
"task run"
|
|
46605
|
-
);
|
|
46606
|
-
}
|
|
46607
|
-
async function pauseTask(client, taskId, reqOptions) {
|
|
46608
|
-
const path13 = `/scheduled-tasks/${encodeURIComponent(taskId)}/pause`;
|
|
46609
|
-
return call2(
|
|
46610
|
-
"POST",
|
|
46611
|
-
path13,
|
|
46612
|
-
() => client.postJSON(path13, {}, reqOptions),
|
|
46613
|
-
"task pause"
|
|
46614
|
-
);
|
|
46615
|
-
}
|
|
46616
|
-
async function resumeTask(client, taskId, reqOptions) {
|
|
46617
|
-
const path13 = `/scheduled-tasks/${encodeURIComponent(taskId)}/resume`;
|
|
46618
|
-
return call2(
|
|
46619
|
-
"POST",
|
|
46620
|
-
path13,
|
|
46621
|
-
() => client.postJSON(path13, {}, reqOptions),
|
|
46622
|
-
"task resume"
|
|
46623
|
-
);
|
|
46624
|
-
}
|
|
46625
|
-
async function listRuns(client, taskId, limit, reqOptions) {
|
|
46626
|
-
const path13 = `/scheduled-tasks/${encodeURIComponent(taskId)}/runs?limit=${limit}`;
|
|
46627
|
-
return call2(
|
|
46628
|
-
"GET",
|
|
46629
|
-
path13,
|
|
46630
|
-
() => client.getJSON(path13, reqOptions),
|
|
46631
|
-
"task runs"
|
|
46632
|
-
);
|
|
46633
|
-
}
|
|
46634
46576
|
|
|
46635
|
-
// src/commands/
|
|
46577
|
+
// src/commands/connectorsPoll.ts
|
|
46578
|
+
import * as readline2 from "readline";
|
|
46579
|
+
import { createHash as createHash12 } from "crypto";
|
|
46580
|
+
|
|
46581
|
+
// src/types/connectors.ts
|
|
46582
|
+
var POLL_ITEM_STATES = [
|
|
46583
|
+
"seen",
|
|
46584
|
+
"new",
|
|
46585
|
+
"dispatched",
|
|
46586
|
+
"running",
|
|
46587
|
+
"retry_wait",
|
|
46588
|
+
"landed",
|
|
46589
|
+
"failed",
|
|
46590
|
+
"skipped",
|
|
46591
|
+
"seeded",
|
|
46592
|
+
"superseded"
|
|
46593
|
+
];
|
|
46594
|
+
|
|
46595
|
+
// src/commands/connectorsPoll.ts
|
|
46636
46596
|
var DURATION_UNIT_SECONDS = { s: 1, m: 60, h: 3600, d: 86400 };
|
|
46637
|
-
var
|
|
46638
|
-
|
|
46639
|
-
|
|
46597
|
+
var DURATION_BOUNDS = {
|
|
46598
|
+
"--interval": [300, 7 * 86400],
|
|
46599
|
+
"--settle": [0, 86400]
|
|
46600
|
+
};
|
|
46601
|
+
var LOOKBACK_MAX_DAYS = 3650;
|
|
46602
|
+
function parseDurationSeconds(raw2, flag) {
|
|
46640
46603
|
const m = /^(\d+)([smhd])?$/.exec(raw2.trim());
|
|
46641
46604
|
const seconds = m === null ? NaN : Number(m[1]) * (DURATION_UNIT_SECONDS[m[2] ?? "s"] ?? 1);
|
|
46642
|
-
|
|
46643
|
-
|
|
46644
|
-
|
|
46645
|
-
|
|
46605
|
+
const [min, max] = DURATION_BOUNDS[flag] ?? [0, Number.MAX_SAFE_INTEGER];
|
|
46606
|
+
if (m === null) {
|
|
46607
|
+
throw new Error(`invalid ${flag} "${raw2}": expected a duration like 90s, 5m or 1h`);
|
|
46608
|
+
}
|
|
46609
|
+
if (!Number.isSafeInteger(seconds) || seconds < min || seconds > max) {
|
|
46610
|
+
throw new Error(
|
|
46611
|
+
`invalid ${flag} "${raw2}": must be between ${formatSeconds(min)} and ${formatSeconds(max)}`
|
|
46612
|
+
);
|
|
46646
46613
|
}
|
|
46647
46614
|
return seconds;
|
|
46648
46615
|
}
|
|
46649
|
-
function
|
|
46650
|
-
const
|
|
46651
|
-
|
|
46652
|
-
|
|
46616
|
+
function parseLookbackDays(raw2) {
|
|
46617
|
+
const m = /^(\d+)d?$/.exec(raw2.trim());
|
|
46618
|
+
const days = m === null ? 0 : Number(m[1]);
|
|
46619
|
+
if (days < 1 || days > LOOKBACK_MAX_DAYS) {
|
|
46620
|
+
throw new Error(
|
|
46621
|
+
`invalid --lookback "${raw2}": must be between 1 and ${LOOKBACK_MAX_DAYS} days, e.g. 180d`
|
|
46622
|
+
);
|
|
46653
46623
|
}
|
|
46654
|
-
return
|
|
46624
|
+
return days;
|
|
46655
46625
|
}
|
|
46656
|
-
var
|
|
46657
|
-
var
|
|
46658
|
-
function
|
|
46659
|
-
const
|
|
46660
|
-
|
|
46661
|
-
const eq = raw2.indexOf("=");
|
|
46662
|
-
const role = eq < 0 ? "" : raw2.slice(0, eq).trim();
|
|
46663
|
-
const id = eq < 0 ? "" : raw2.slice(eq + 1).trim();
|
|
46664
|
-
if (!ROLE_RE.test(role) || !DATASOURCE_ID_RE.test(id)) {
|
|
46665
|
-
throw new Error(
|
|
46666
|
-
`invalid --datasource "${raw2}": expected <role>=<datasource id>, e.g. target=4242 (role: lowercase letters and _; id: digits)`
|
|
46667
|
-
);
|
|
46668
|
-
}
|
|
46669
|
-
if (Object.hasOwn(out, role)) throw new Error(`--datasource role "${role}" given twice`);
|
|
46670
|
-
out[role] = id;
|
|
46671
|
-
}
|
|
46672
|
-
const count = Object.keys(out).length;
|
|
46673
|
-
if (count < 1 || count > 4) throw new Error("give between 1 and 4 --datasource roles");
|
|
46674
|
-
return out;
|
|
46626
|
+
var DATE_ONLY_RE = /^(\d{4})-(\d{2})-(\d{2})$/;
|
|
46627
|
+
var DATE_TIME_RE = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2})(?::(\d{2})(?:\.\d+)?)?(?:Z|[+-](\d{2}):(\d{2}))$/;
|
|
46628
|
+
function inRange(value, max, min = 0) {
|
|
46629
|
+
const n = Number(value ?? "0");
|
|
46630
|
+
return n >= min && n <= max;
|
|
46675
46631
|
}
|
|
46676
|
-
function
|
|
46677
|
-
|
|
46678
|
-
|
|
46679
|
-
|
|
46680
|
-
parsed2 = JSON.parse(text2);
|
|
46681
|
-
} catch (err) {
|
|
46682
|
-
const msg = err instanceof Error ? err.message : String(err);
|
|
46683
|
-
throw new Error(`invalid --config JSON: ${msg}`);
|
|
46684
|
-
}
|
|
46685
|
-
if (typeof parsed2 !== "object" || parsed2 === null || Array.isArray(parsed2)) {
|
|
46686
|
-
throw new Error("invalid --config: expected a JSON object");
|
|
46687
|
-
}
|
|
46688
|
-
return parsed2;
|
|
46632
|
+
function isCalendarDay(y, m, d) {
|
|
46633
|
+
if (!inRange(m, 12, 1)) return false;
|
|
46634
|
+
const daysInMonth3 = new Date(Date.UTC(Number(y), Number(m), 0)).getUTCDate();
|
|
46635
|
+
return inRange(d, daysInMonth3, 1);
|
|
46689
46636
|
}
|
|
46690
|
-
function
|
|
46691
|
-
|
|
46637
|
+
function parseTimestampFlag(raw2, flag) {
|
|
46638
|
+
const invalid = new Error(
|
|
46639
|
+
`invalid ${flag} "${raw2}": expected YYYY-MM-DD or a timestamp with a zone (e.g. 2026-10-08T17:30:00+03:00)`
|
|
46640
|
+
);
|
|
46641
|
+
const value = raw2.trim();
|
|
46642
|
+
const dateOnly = DATE_ONLY_RE.exec(value);
|
|
46643
|
+
if (dateOnly !== null) {
|
|
46644
|
+
if (!isCalendarDay(dateOnly[1], dateOnly[2], dateOnly[3])) throw invalid;
|
|
46645
|
+
return (/* @__PURE__ */ new Date(`${value}T00:00:00Z`)).toISOString();
|
|
46646
|
+
}
|
|
46647
|
+
const dt = DATE_TIME_RE.exec(value);
|
|
46648
|
+
const valid = dt !== null && isCalendarDay(dt[1], dt[2], dt[3]) && inRange(dt[4], 23) && inRange(dt[5], 59) && inRange(dt[6], 59) && inRange(dt[7], 23) && inRange(dt[8], 59);
|
|
46649
|
+
const ms = valid ? Date.parse(value) : NaN;
|
|
46650
|
+
if (Number.isNaN(ms)) throw invalid;
|
|
46651
|
+
return new Date(ms).toISOString();
|
|
46692
46652
|
}
|
|
46693
|
-
function
|
|
46694
|
-
return
|
|
46653
|
+
function parseSameName(raw2) {
|
|
46654
|
+
if (raw2 === "new-version") return "new_version";
|
|
46655
|
+
if (raw2 === "ignore") return "ignore";
|
|
46656
|
+
throw new Error(`--same-name must be new-version or ignore (got "${raw2}")`);
|
|
46695
46657
|
}
|
|
46696
|
-
function
|
|
46697
|
-
|
|
46698
|
-
|
|
46658
|
+
function parseChoice(raw2, flag, choices) {
|
|
46659
|
+
const match = choices.find((c) => c === raw2);
|
|
46660
|
+
if (match === void 0) {
|
|
46661
|
+
throw new Error(`${flag} must be ${choices.join(" or ")} (got "${raw2}")`);
|
|
46699
46662
|
}
|
|
46663
|
+
return match;
|
|
46664
|
+
}
|
|
46665
|
+
var WIDEN_CHOICES = ["seed", "land"];
|
|
46666
|
+
var POLICY_CHOICES = ["rekey", "seed", "land"];
|
|
46667
|
+
function parseSettings(opts) {
|
|
46700
46668
|
const fields = {};
|
|
46701
|
-
if (opts.
|
|
46702
|
-
|
|
46703
|
-
|
|
46704
|
-
if (opts.
|
|
46705
|
-
if (opts.
|
|
46706
|
-
if (opts.
|
|
46707
|
-
|
|
46708
|
-
if (opts.tz !== void 0) fields.timezone = opts.tz;
|
|
46709
|
-
if (fields.primary_role !== void 0 && fields.datasources !== void 0) {
|
|
46710
|
-
if (!Object.hasOwn(fields.datasources, fields.primary_role)) {
|
|
46711
|
-
throw new Error(
|
|
46712
|
-
`--primary-role "${fields.primary_role}" is not one of the --datasource roles`
|
|
46713
|
-
);
|
|
46714
|
-
}
|
|
46669
|
+
if (opts.interval !== void 0) {
|
|
46670
|
+
fields.interval_seconds = parseDurationSeconds(opts.interval, "--interval");
|
|
46671
|
+
}
|
|
46672
|
+
if (opts.since !== void 0) fields.since = parseTimestampFlag(opts.since, "--since");
|
|
46673
|
+
if (opts.lookback !== void 0) fields.lookback_days = parseLookbackDays(opts.lookback);
|
|
46674
|
+
if (opts.settle !== void 0) {
|
|
46675
|
+
fields.settle_seconds = parseDurationSeconds(opts.settle, "--settle");
|
|
46715
46676
|
}
|
|
46677
|
+
if (opts.sameName !== void 0) fields.change_policy = parseSameName(opts.sameName);
|
|
46716
46678
|
return fields;
|
|
46717
46679
|
}
|
|
46718
|
-
function
|
|
46719
|
-
const
|
|
46720
|
-
["--
|
|
46721
|
-
["--
|
|
46722
|
-
["--
|
|
46723
|
-
["--
|
|
46724
|
-
|
|
46725
|
-
|
|
46726
|
-
|
|
46727
|
-
const fields = parseScheduleFlags(opts);
|
|
46728
|
-
return {
|
|
46729
|
-
org_id: orgId,
|
|
46730
|
-
name,
|
|
46731
|
-
artifact_name: fields.artifact_name,
|
|
46732
|
-
runtime_flavor: fields.runtime_flavor,
|
|
46733
|
-
datasources: fields.datasources,
|
|
46734
|
-
config: fields.config,
|
|
46735
|
-
...fields.primary_role !== void 0 ? { primary_role: fields.primary_role } : {},
|
|
46736
|
-
...fields.every_seconds !== void 0 ? { every_seconds: fields.every_seconds } : {},
|
|
46737
|
-
...fields.cron !== void 0 ? { cron: fields.cron } : {},
|
|
46738
|
-
...fields.timezone !== void 0 ? { timezone: fields.timezone } : {},
|
|
46739
|
-
...opts.disabled === true ? { enabled: false } : {}
|
|
46740
|
-
};
|
|
46680
|
+
function settingsArgs(opts) {
|
|
46681
|
+
const pairs = [
|
|
46682
|
+
["--interval", opts.interval],
|
|
46683
|
+
["--since", opts.since],
|
|
46684
|
+
["--lookback", opts.lookback],
|
|
46685
|
+
["--settle", opts.settle],
|
|
46686
|
+
["--same-name", opts.sameName]
|
|
46687
|
+
];
|
|
46688
|
+
return pairs.filter((p) => p[1] !== void 0).map(([flag, value]) => ` ${flag} ${value.trim()}`).join("");
|
|
46741
46689
|
}
|
|
46742
|
-
|
|
46743
|
-
|
|
46744
|
-
|
|
46745
|
-
const
|
|
46746
|
-
const
|
|
46747
|
-
|
|
46690
|
+
async function resolvePollBinding(client, filebox, signal) {
|
|
46691
|
+
const wanted = filebox.trim();
|
|
46692
|
+
const byId = /^\d+$/.test(wanted);
|
|
46693
|
+
const rows = await listQueries(client, { signal });
|
|
46694
|
+
const matches = rows.filter(
|
|
46695
|
+
(q) => byId ? q.filebox_id === Number(wanted) : q.filebox_uuid.toLowerCase() === wanted.toLowerCase()
|
|
46696
|
+
);
|
|
46697
|
+
const [first] = matches;
|
|
46698
|
+
if (first === void 0) {
|
|
46748
46699
|
throw new Error(
|
|
46749
|
-
`no
|
|
46750
|
-
);
|
|
46751
|
-
}
|
|
46752
|
-
return match.id;
|
|
46753
|
-
}
|
|
46754
|
-
function explainTaskError(err) {
|
|
46755
|
-
const forbidden = err instanceof ControlPlaneError && err.status === 403 || err instanceof Error && /\bHTTP 403\b/.test(err.message);
|
|
46756
|
-
if (forbidden) {
|
|
46757
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
46758
|
-
const body = err instanceof ControlPlaneError ? err.body : (/HTTP 403[^:]*(?:\([^)]*\))?:\s*([\s\S]*)$/.exec(message)?.[1] ?? "").trim();
|
|
46759
|
-
return withCause(
|
|
46760
|
-
new ControlPlaneError(
|
|
46761
|
-
403,
|
|
46762
|
-
body,
|
|
46763
|
-
`${message}
|
|
46764
|
-
Scheduled tasks need a support user or an org admin: log in as one (--account) \u2014 e.g. dr login-as the customer's support user.`
|
|
46765
|
-
),
|
|
46766
|
-
err
|
|
46700
|
+
`no connector binding for filebox ${wanted} in this org (see dr connectors query list)`
|
|
46767
46701
|
);
|
|
46768
46702
|
}
|
|
46769
|
-
if (
|
|
46770
|
-
|
|
46771
|
-
|
|
46772
|
-
const message = `scheduled task not found (${detail || err.message})`;
|
|
46773
|
-
return withCause(new ControlPlaneError(404, err.body, message), err);
|
|
46774
|
-
}
|
|
46775
|
-
if (err.status === 409 || err.status === 422) {
|
|
46776
|
-
return withCause(new ControlPlaneError(err.status, err.body, detail || err.message), err);
|
|
46703
|
+
if (matches.length > 1) {
|
|
46704
|
+
const ids = matches.map((q) => `#${q.query_id}`).join(", ");
|
|
46705
|
+
throw new Error(`filebox ${wanted} has several connector bindings (${ids})`);
|
|
46777
46706
|
}
|
|
46778
|
-
return
|
|
46707
|
+
return first;
|
|
46779
46708
|
}
|
|
46780
|
-
function
|
|
46781
|
-
|
|
46782
|
-
return error;
|
|
46709
|
+
function isNotPolled(err) {
|
|
46710
|
+
return err instanceof ControlPlaneError && err.status === 404 && err.body.includes("not polled");
|
|
46783
46711
|
}
|
|
46784
|
-
function
|
|
46712
|
+
function isPollThrottled(err) {
|
|
46713
|
+
return err instanceof ControlPlaneError && err.status === 409 && (err.body.includes("discover is in flight") || err.body.includes("polled less than"));
|
|
46714
|
+
}
|
|
46715
|
+
async function explainNotPolled(filebox, call3) {
|
|
46785
46716
|
try {
|
|
46786
|
-
|
|
46787
|
-
|
|
46788
|
-
if (
|
|
46789
|
-
|
|
46790
|
-
|
|
46791
|
-
|
|
46792
|
-
)
|
|
46717
|
+
return await call3;
|
|
46718
|
+
} catch (err) {
|
|
46719
|
+
if (isNotPolled(err)) {
|
|
46720
|
+
throw new Error(
|
|
46721
|
+
`filebox ${filebox} is not set to polled ingestion; run dr connectors poll enable ${filebox}`,
|
|
46722
|
+
{ cause: err }
|
|
46723
|
+
);
|
|
46793
46724
|
}
|
|
46794
|
-
|
|
46725
|
+
throw err;
|
|
46795
46726
|
}
|
|
46796
|
-
return body;
|
|
46797
46727
|
}
|
|
46798
|
-
function
|
|
46799
|
-
|
|
46800
|
-
const
|
|
46801
|
-
const
|
|
46802
|
-
return
|
|
46728
|
+
async function readFileboxDateTag(monolith, fileboxId, signal) {
|
|
46729
|
+
const details = await getDetails(monolith, fileboxId, { signal });
|
|
46730
|
+
const dateTags = (details.tags_config ?? []).filter((t) => t.type === "DATE");
|
|
46731
|
+
const [only] = dateTags;
|
|
46732
|
+
if (only === void 0) return {};
|
|
46733
|
+
if (dateTags.length > 1) {
|
|
46734
|
+
const names = dateTags.map((t) => t.name).join(", ");
|
|
46735
|
+
throw new Error(
|
|
46736
|
+
`filebox ${fileboxId} has several DATE tags (${names}); a polled filebox needs exactly one`
|
|
46737
|
+
);
|
|
46738
|
+
}
|
|
46739
|
+
return { id: only.id, name: only.name };
|
|
46803
46740
|
}
|
|
46804
|
-
function
|
|
46805
|
-
if (
|
|
46806
|
-
if (
|
|
46807
|
-
|
|
46741
|
+
function formatSeconds(sec) {
|
|
46742
|
+
if (sec > 0 && sec % 86400 === 0) return `${sec / 86400}d`;
|
|
46743
|
+
if (sec > 0 && sec % 3600 === 0) return `${sec / 3600}h`;
|
|
46744
|
+
if (sec > 0 && sec % 60 === 0) return `${sec / 60}m`;
|
|
46745
|
+
return `${sec}s`;
|
|
46808
46746
|
}
|
|
46809
|
-
function
|
|
46810
|
-
|
|
46811
|
-
|
|
46812
|
-
|
|
46813
|
-
|
|
46814
|
-
|
|
46815
|
-
|
|
46816
|
-
|
|
46817
|
-
|
|
46818
|
-
|
|
46819
|
-
|
|
46820
|
-
`)
|
|
46821
|
-
|
|
46822
|
-
|
|
46823
|
-
|
|
46824
|
-
`)
|
|
46825
|
-
if (t.consecutive_failures > 0) out.write(` Failures: ${t.consecutive_failures} in a row
|
|
46826
|
-
`);
|
|
46827
|
-
out.write(` Datasources: ${roles}
|
|
46747
|
+
function oneLine3(s) {
|
|
46748
|
+
return s.replace(/\s+/g, " ").trim();
|
|
46749
|
+
}
|
|
46750
|
+
function sameNameLabel(policy) {
|
|
46751
|
+
return policy === "new_version" ? "new-version" : String(policy);
|
|
46752
|
+
}
|
|
46753
|
+
function formatDateTag(tag) {
|
|
46754
|
+
if (tag?.id === void 0) return "none (undated filebox)";
|
|
46755
|
+
return `${tag.name ?? "?"} (id ${tag.id})`;
|
|
46756
|
+
}
|
|
46757
|
+
function bindingTarget(binding) {
|
|
46758
|
+
return `filebox ${binding.filebox_uuid} (query #${binding.query_id})`;
|
|
46759
|
+
}
|
|
46760
|
+
function renderPoll(out, headline, binding, p) {
|
|
46761
|
+
const row = (label, value) => {
|
|
46762
|
+
out.write(` ${`${label}:`.padEnd(15)}${value}
|
|
46828
46763
|
`);
|
|
46829
|
-
|
|
46764
|
+
};
|
|
46765
|
+
const status2 = `${p.enabled ? "enabled" : "disabled"}${p.paused_reason === null ? "" : `, paused: ${oneLine3(p.paused_reason)}`}`;
|
|
46766
|
+
const counts = Object.entries(p.counts).sort(([a], [b]) => a.localeCompare(b)).map(([state, n]) => `${state}=${n}`).join(" ");
|
|
46767
|
+
out.write(`${headline} for ${bindingTarget(binding)}
|
|
46830
46768
|
`);
|
|
46769
|
+
row("Status", status2);
|
|
46770
|
+
row("Interval", formatSeconds(p.interval_seconds));
|
|
46771
|
+
row("Date tag", formatDateTag(p.date_tag));
|
|
46772
|
+
row("Since", p.since ?? "-");
|
|
46773
|
+
row("Lookback", `${p.lookback_days}d`);
|
|
46774
|
+
row("Settle", formatSeconds(p.settle_seconds));
|
|
46775
|
+
row("Same name", sameNameLabel(p.change_policy));
|
|
46776
|
+
row("Max attempts", String(p.max_attempts));
|
|
46777
|
+
row("Last poll", `${p.last_poll_at ?? "-"} ${oneLine3(p.last_poll_status ?? "")}`.trimEnd());
|
|
46778
|
+
row("Next poll", p.next_poll_at);
|
|
46779
|
+
row("Failures", String(p.consecutive_failures));
|
|
46780
|
+
row("Discover", p.discover_in_flight ? `in flight since ${p.discover_sent_at ?? "?"}` : "-");
|
|
46781
|
+
row("Last digest", p.last_digest ?? "-");
|
|
46782
|
+
row("Pending seed", p.poll_run_seed_until ?? "-");
|
|
46783
|
+
row(
|
|
46784
|
+
"Pending rekey",
|
|
46785
|
+
p.poll_run_rekey_to === null ? "-" : `to ${sameNameLabel(p.poll_run_rekey_to)}`
|
|
46786
|
+
);
|
|
46787
|
+
row("Items", counts === "" ? "(none)" : counts);
|
|
46788
|
+
const outputs = p.outputs ?? [];
|
|
46789
|
+
row(
|
|
46790
|
+
"Outputs",
|
|
46791
|
+
outputs.length === 0 ? "-" : outputs.map((o) => `filebox ${o.filebox_uuid} (query #${o.connector_query_id})`).join(", ")
|
|
46792
|
+
);
|
|
46831
46793
|
}
|
|
46832
|
-
function
|
|
46833
|
-
|
|
46834
|
-
|
|
46835
|
-
|
|
46794
|
+
function writeTable26(out, rows) {
|
|
46795
|
+
const widths = [];
|
|
46796
|
+
for (const r of rows) {
|
|
46797
|
+
r.forEach((cell, i) => {
|
|
46798
|
+
widths[i] = Math.max(widths[i] ?? 0, cell.length);
|
|
46799
|
+
});
|
|
46836
46800
|
}
|
|
46837
|
-
for (const
|
|
46838
|
-
|
|
46839
|
-
|
|
46840
|
-
`
|
|
46801
|
+
for (const r of rows) {
|
|
46802
|
+
const cells = r.map(
|
|
46803
|
+
(cell, i) => i === r.length - 1 ? cell : cell.padEnd((widths[i] ?? 0) + 2)
|
|
46841
46804
|
);
|
|
46805
|
+
out.write(`${cells.join("").trimEnd()}
|
|
46806
|
+
`);
|
|
46842
46807
|
}
|
|
46843
46808
|
}
|
|
46844
|
-
function
|
|
46845
|
-
|
|
46846
|
-
|
|
46847
|
-
|
|
46809
|
+
function formatItemOutputs(it) {
|
|
46810
|
+
const outputs = it.outputs ?? [];
|
|
46811
|
+
if (outputs.length === 0) return "-";
|
|
46812
|
+
return outputs.map((o) => {
|
|
46813
|
+
const error = o.last_error ?? (o.skip_reason === null ? null : `skipped: ${o.skip_reason}`);
|
|
46814
|
+
return `#${o.connector_query_id}=${o.state}${error === null ? "" : ` (${truncate3(oneLine3(error), 40)})`}`;
|
|
46815
|
+
}).join(" ");
|
|
46816
|
+
}
|
|
46817
|
+
function renderItemTable(out, items) {
|
|
46818
|
+
const multi = items.some((it) => (it.outputs ?? []).length > 0);
|
|
46819
|
+
const header = ["ID", "LABEL", "DATE", "STATE", "ATTEMPTS", "VERSION"];
|
|
46820
|
+
const rows = [[...header, ...multi ? ["OUTPUTS"] : [], "ERROR"]];
|
|
46821
|
+
for (const it of items) {
|
|
46822
|
+
const error = it.last_error ?? (it.skip_reason === null ? null : `skipped: ${it.skip_reason}`);
|
|
46823
|
+
rows.push([
|
|
46824
|
+
String(it.id),
|
|
46825
|
+
oneLine3(it.label),
|
|
46826
|
+
it.item_date ?? "-",
|
|
46827
|
+
it.state,
|
|
46828
|
+
String(it.attempts),
|
|
46829
|
+
it.version_id === null ? "-" : String(it.version_id),
|
|
46830
|
+
...multi ? [formatItemOutputs(it)] : [],
|
|
46831
|
+
error === null ? "-" : truncate3(oneLine3(error), 80)
|
|
46832
|
+
]);
|
|
46848
46833
|
}
|
|
46849
|
-
|
|
46850
|
-
|
|
46851
|
-
|
|
46852
|
-
|
|
46853
|
-
|
|
46834
|
+
writeTable26(out, rows);
|
|
46835
|
+
}
|
|
46836
|
+
async function runPollEnable(cfg) {
|
|
46837
|
+
const binding = await resolvePollBinding(cfg.client, cfg.filebox, cfg.signal);
|
|
46838
|
+
let existing;
|
|
46839
|
+
try {
|
|
46840
|
+
existing = await getPoll(cfg.client, binding.query_id, { signal: cfg.signal });
|
|
46841
|
+
} catch (err) {
|
|
46842
|
+
if (!isNotPolled(err)) throw err;
|
|
46843
|
+
existing = null;
|
|
46844
|
+
}
|
|
46845
|
+
const s = cfg.settings;
|
|
46846
|
+
if (existing !== null && (s.since !== void 0 || s.lookback_days !== void 0 || s.change_policy !== void 0)) {
|
|
46847
|
+
throw new Error(
|
|
46848
|
+
`filebox ${cfg.filebox} is already polled; change --since, --lookback or --same-name with dr connectors poll set ${cfg.filebox}, which asks what to do with a wider window or a policy switch`
|
|
46854
46849
|
);
|
|
46855
46850
|
}
|
|
46851
|
+
if (binding.filebox_id <= 0) {
|
|
46852
|
+
throw new Error(
|
|
46853
|
+
`connector-query #${binding.query_id} has no filebox id recorded, so its date tag cannot be read; record it with dr connectors query repoint ${binding.query_id} --filebox-id <id>`
|
|
46854
|
+
);
|
|
46855
|
+
}
|
|
46856
|
+
const dateTag = await readFileboxDateTag(cfg.monolith, binding.filebox_id, cfg.signal);
|
|
46857
|
+
const stored = await putPoll(
|
|
46858
|
+
cfg.client,
|
|
46859
|
+
binding.query_id,
|
|
46860
|
+
{ enabled: true, date_tag: dateTag, ...s },
|
|
46861
|
+
{ signal: cfg.signal }
|
|
46862
|
+
);
|
|
46863
|
+
if (cfg.asJson) {
|
|
46864
|
+
writeJSONIndented(cfg.out, stored);
|
|
46865
|
+
return;
|
|
46866
|
+
}
|
|
46867
|
+
renderPoll(cfg.out, "Polling enabled", binding, stored);
|
|
46856
46868
|
}
|
|
46857
|
-
function
|
|
46858
|
-
|
|
46859
|
-
|
|
46860
|
-
addAccountFlag(cmd);
|
|
46861
|
-
addConnectorsUrlFlag(cmd);
|
|
46862
|
-
return cmd;
|
|
46863
|
-
}
|
|
46864
|
-
function addScheduleFlags(cmd) {
|
|
46865
|
-
return cmd.option("--artifact <name>", "published artifact name, e.g. sftp_export__daily_sales").option("--flavor <flavor>", "runner flavor the artifact was published for, e.g. sftp").option("--datasource <role=id>", "monolith datasource for a role (repeat per role)", collect).option(
|
|
46866
|
-
"--primary-role <role>",
|
|
46867
|
-
"the role passed as the primary datasource (default: the first)"
|
|
46868
|
-
).option("--config <json|@file>", "task config: a JSON object, or @path to a JSON file").option("--every <duration>", "run on an interval, e.g. 5m, 1h (at least 1m)").option("--cron <expr>", 'run on a five-field cron line, e.g. "15 * * * *"').option("--tz <iana>", "timezone the cron line is read in (default UTC)");
|
|
46869
|
+
function windowStart(since, lookbackDays, now) {
|
|
46870
|
+
const start = now - lookbackDays * 864e5;
|
|
46871
|
+
return since === null ? start : Math.max(start, Date.parse(since));
|
|
46869
46872
|
}
|
|
46870
|
-
|
|
46871
|
-
|
|
46872
|
-
|
|
46873
|
-
|
|
46874
|
-
|
|
46875
|
-
|
|
46876
|
-
|
|
46877
|
-
} catch (err) {
|
|
46878
|
-
throw explainTaskError(err);
|
|
46879
|
-
}
|
|
46880
|
-
});
|
|
46881
|
-
}
|
|
46882
|
-
function registerConnectorsTasks(connectors) {
|
|
46883
|
-
const tasks = connectors.command("tasks").description("DRIP scheduled tasks: run an artifact on a schedule (support user or org admin)");
|
|
46884
|
-
withFlags(
|
|
46885
|
-
addScheduleFlags(
|
|
46886
|
-
tasks.command("create").description("Create a scheduled task").argument("<name>", "task name, unique in the org (a-z, 0-9, _ and -)").option("--org-id <id>", "org id (defaults to the JWT org)").option("--disabled", "create it disabled", false)
|
|
46887
|
-
)
|
|
46888
|
-
).action(
|
|
46889
|
-
async (name, opts) => {
|
|
46890
|
-
const explicitOrg = opts.orgId !== void 0 ? parsePositiveIntArg2(opts.orgId, "--org-id") : void 0;
|
|
46891
|
-
const parsed2 = buildCreateFields(name, explicitOrg ?? 0, opts);
|
|
46892
|
-
await withConnectorsSetup(opts, async ({ client, signal }) => {
|
|
46893
|
-
const fields = { ...parsed2, org_id: explicitOrg ?? client.getCurrentOrgId() };
|
|
46894
|
-
try {
|
|
46895
|
-
const t = await createTask(client, fields, { signal });
|
|
46896
|
-
if (opts.json === true) writeJSONIndented(process.stdout, t);
|
|
46897
|
-
else writeTask(process.stdout, t);
|
|
46898
|
-
} catch (err) {
|
|
46899
|
-
throw explainTaskError(err);
|
|
46900
|
-
}
|
|
46901
|
-
});
|
|
46902
|
-
}
|
|
46903
|
-
);
|
|
46904
|
-
withFlags(
|
|
46905
|
-
tasks.command("list").description("List the org's scheduled tasks").option("--org-id <id>", "org id (defaults to the JWT org)")
|
|
46906
|
-
).action(async (opts) => {
|
|
46907
|
-
await withConnectorsSetup(opts, async ({ client, signal }) => {
|
|
46908
|
-
const orgId = opts.orgId !== void 0 ? parsePositiveIntArg2(opts.orgId, "--org-id") : void 0;
|
|
46909
|
-
try {
|
|
46910
|
-
const list28 = await listTasks(client, orgId, { signal });
|
|
46911
|
-
if (opts.json === true) writeJSONIndented(process.stdout, list28);
|
|
46912
|
-
else writeTaskList(process.stdout, list28);
|
|
46913
|
-
} catch (err) {
|
|
46914
|
-
throw explainTaskError(err);
|
|
46915
|
-
}
|
|
46916
|
-
});
|
|
46917
|
-
});
|
|
46918
|
-
withFlags(
|
|
46919
|
-
tasks.command("get").description("Show a scheduled task").argument("<task>", "task id or name")
|
|
46920
|
-
).action(
|
|
46921
|
-
async (task, opts) => onTask(
|
|
46922
|
-
task,
|
|
46923
|
-
opts,
|
|
46924
|
-
(c, id, signal) => getTask(c, id, { signal }),
|
|
46925
|
-
(o, r) => writeTask(o, r)
|
|
46926
|
-
)
|
|
46927
|
-
);
|
|
46928
|
-
withFlags(
|
|
46929
|
-
addScheduleFlags(
|
|
46930
|
-
tasks.command("update").description("Change a scheduled task (only the flags given change)").argument("<task>", "task id or name").option("--enable", "enable the task").option("--disable", "disable the task")
|
|
46931
|
-
)
|
|
46932
|
-
).action(
|
|
46933
|
-
async (task, opts) => {
|
|
46934
|
-
if (opts.enable === true && opts.disable === true)
|
|
46935
|
-
throw new Error("give --enable or --disable, not both");
|
|
46936
|
-
const fields = parseScheduleFlags(opts);
|
|
46937
|
-
if (opts.enable === true) fields.enabled = true;
|
|
46938
|
-
if (opts.disable === true) fields.enabled = false;
|
|
46939
|
-
if (Object.keys(fields).length === 0)
|
|
46940
|
-
throw new Error("nothing to change: give at least one flag");
|
|
46941
|
-
await onTask(
|
|
46942
|
-
task,
|
|
46943
|
-
opts,
|
|
46944
|
-
(c, id, signal) => updateTask(c, id, fields, { signal }),
|
|
46945
|
-
(o, r) => writeTask(o, r)
|
|
46946
|
-
);
|
|
46947
|
-
}
|
|
46948
|
-
);
|
|
46949
|
-
withFlags(
|
|
46950
|
-
tasks.command("delete").description("Delete a scheduled task and its run history").argument("<task>", "task id or name")
|
|
46951
|
-
).action(
|
|
46952
|
-
async (task, opts) => onTask(
|
|
46953
|
-
task,
|
|
46954
|
-
opts,
|
|
46955
|
-
async (c, id, signal) => {
|
|
46956
|
-
await deleteTask(c, id, { signal });
|
|
46957
|
-
return { deleted: id };
|
|
46958
|
-
},
|
|
46959
|
-
(o, r) => o.write(`Deleted scheduled task ${r.deleted}
|
|
46960
|
-
`)
|
|
46961
|
-
)
|
|
46962
|
-
);
|
|
46963
|
-
withFlags(
|
|
46964
|
-
tasks.command("run").description("Run a task now (does not move its schedule)").argument("<task>", "task id or name")
|
|
46965
|
-
).action(
|
|
46966
|
-
async (task, opts) => onTask(
|
|
46967
|
-
task,
|
|
46968
|
-
opts,
|
|
46969
|
-
(c, id, signal) => runTask(c, id, { signal }),
|
|
46970
|
-
(o, r) => {
|
|
46971
|
-
const run2 = r;
|
|
46972
|
-
o.write(`Queued run ${run2.id} (${run2.status}); see: dr connectors tasks runs ${task}
|
|
46973
|
-
`);
|
|
46974
|
-
}
|
|
46975
|
-
)
|
|
46976
|
-
);
|
|
46977
|
-
withFlags(
|
|
46978
|
-
tasks.command("pause").description("Pause a scheduled task").argument("<task>", "task id or name")
|
|
46979
|
-
).action(
|
|
46980
|
-
async (task, opts) => onTask(
|
|
46981
|
-
task,
|
|
46982
|
-
opts,
|
|
46983
|
-
(c, id, signal) => pauseTask(c, id, { signal }),
|
|
46984
|
-
(o, r) => writeTask(o, r)
|
|
46985
|
-
)
|
|
46986
|
-
);
|
|
46987
|
-
withFlags(
|
|
46988
|
-
tasks.command("resume").description("Resume a paused task (clears its failure count, schedules the next run)").argument("<task>", "task id or name")
|
|
46989
|
-
).action(
|
|
46990
|
-
async (task, opts) => onTask(
|
|
46991
|
-
task,
|
|
46992
|
-
opts,
|
|
46993
|
-
(c, id, signal) => resumeTask(c, id, { signal }),
|
|
46994
|
-
(o, r) => writeTask(o, r)
|
|
46995
|
-
)
|
|
46873
|
+
function widening(current, changes, now) {
|
|
46874
|
+
if (changes.since === void 0 && changes.lookback_days === void 0) return null;
|
|
46875
|
+
const from = windowStart(current.since, current.lookback_days, now);
|
|
46876
|
+
const to = windowStart(
|
|
46877
|
+
changes.since ?? current.since,
|
|
46878
|
+
changes.lookback_days ?? current.lookback_days,
|
|
46879
|
+
now
|
|
46996
46880
|
);
|
|
46997
|
-
|
|
46998
|
-
|
|
46999
|
-
).action(async (task, opts) => {
|
|
47000
|
-
const limit = parsePositiveIntArg2(opts.limit, "--limit");
|
|
47001
|
-
if (limit > 200) throw new Error("invalid --limit: at most 200");
|
|
47002
|
-
await onTask(
|
|
47003
|
-
task,
|
|
47004
|
-
opts,
|
|
47005
|
-
(c, id, signal) => listRuns(c, id, limit, { signal }),
|
|
47006
|
-
(o, r) => writeRuns(o, r)
|
|
47007
|
-
);
|
|
47008
|
-
});
|
|
47009
|
-
}
|
|
47010
|
-
|
|
47011
|
-
// src/commands/connectorsAws.ts
|
|
47012
|
-
var ROLE_ARN_RE = /^arn:aws[a-z-]*:iam::\d{12}:role\/\S+$/;
|
|
47013
|
-
var OTHER_ORG_HINT = "if your session is valid, the connection or --org-id may belong to another org than the --account you are logged in as";
|
|
47014
|
-
function explainAwsError(err, id) {
|
|
47015
|
-
if (!(err instanceof ControlPlaneError)) {
|
|
47016
|
-
if (err instanceof Error && /\bHTTP 403\b/.test(err.message)) {
|
|
47017
|
-
return withCause(
|
|
47018
|
-
new ControlPlaneError(
|
|
47019
|
-
403,
|
|
47020
|
-
"",
|
|
47021
|
-
`${err.message}
|
|
47022
|
-
${OTHER_ORG_HINT}`
|
|
47023
|
-
),
|
|
47024
|
-
err
|
|
47025
|
-
);
|
|
47026
|
-
}
|
|
47027
|
-
return err;
|
|
47028
|
-
}
|
|
47029
|
-
if (!hasDetail(err.body)) return err;
|
|
47030
|
-
const detail = detailOf(err.body) || err.message;
|
|
47031
|
-
const messages = {
|
|
47032
|
-
403: `forbidden: ${detail} (${OTHER_ORG_HINT})`,
|
|
47033
|
-
404: `AWS connection${id === void 0 ? "" : ` ${id}`} not found: ${detail}`,
|
|
47034
|
-
409: `conflict: ${detail}`,
|
|
47035
|
-
422: `rejected: ${detail}`,
|
|
47036
|
-
503: `the control plane cannot broker AWS roles on this site: ${detail}`
|
|
47037
|
-
};
|
|
47038
|
-
const message = messages[err.status];
|
|
47039
|
-
if (message === void 0) return err;
|
|
47040
|
-
return withCause(new ControlPlaneError(err.status, err.body, message), err);
|
|
46881
|
+
if (to >= from) return null;
|
|
46882
|
+
return { from: new Date(from).toISOString(), to: new Date(to).toISOString() };
|
|
47041
46883
|
}
|
|
47042
|
-
function
|
|
47043
|
-
|
|
47044
|
-
|
|
47045
|
-
|
|
47046
|
-
} catch {
|
|
47047
|
-
return false;
|
|
47048
|
-
}
|
|
46884
|
+
async function ask(cfg, message, choices, defaultChoice) {
|
|
46885
|
+
const answer = await cfg.prompt(message, choices, defaultChoice);
|
|
46886
|
+
if (answer === null) throw new Error("poll set cancelled; nothing was changed");
|
|
46887
|
+
return parseChoice(answer, "the answer", choices);
|
|
47049
46888
|
}
|
|
47050
|
-
|
|
47051
|
-
|
|
46889
|
+
var WIDEN_HELP = "seed = mark the newly exposed items as already landed, land = land them all";
|
|
46890
|
+
var POLICY_HELP = "rekey = re-key the ledger from a fresh listing (nothing lands again), seed = mark everything in the window as landed, land = land everything in the window";
|
|
46891
|
+
function pollSetConfirmVerb(widen, onPolicyChange) {
|
|
46892
|
+
return widen === "land" || onPolicyChange === "land" ? "set-land" : "set-seed";
|
|
47052
46893
|
}
|
|
47053
|
-
function
|
|
47054
|
-
|
|
47055
|
-
|
|
47056
|
-
|
|
47057
|
-
|
|
47058
|
-
out.write(`AWS connection #${c.id} (org ${c.org_id})
|
|
47059
|
-
`);
|
|
47060
|
-
row("Datasource", c.secret_ref);
|
|
47061
|
-
row("Role ARN", c.role_arn);
|
|
47062
|
-
row("Status", `${c.status}${c.enabled ? "" : " (disabled)"}`);
|
|
47063
|
-
row("Broker ARN", c.broker_arn ?? "-");
|
|
47064
|
-
row("External ID", c.external_id);
|
|
47065
|
-
row("Expires", c.expires_at ?? "-");
|
|
47066
|
-
row("Last assumed", c.last_assumed_at ?? "-");
|
|
47067
|
-
row("Last error", c.last_error === null ? "-" : oneLine3(c.last_error));
|
|
47068
|
-
row("Failures", String(c.consecutive_failures));
|
|
47069
|
-
if (c.trust_statement === null) {
|
|
47070
|
-
out.write("\nTrust statement: none returned by the control plane.\n");
|
|
47071
|
-
} else {
|
|
47072
|
-
out.write("\nTrust statement (add it to the customer role's trust policy):\n");
|
|
47073
|
-
out.write(`${JSON.stringify(c.trust_statement, null, 2)}
|
|
47074
|
-
`);
|
|
46894
|
+
async function runPollSet(cfg) {
|
|
46895
|
+
if (Object.keys(cfg.changes).length === 0) {
|
|
46896
|
+
throw new Error(
|
|
46897
|
+
"poll set: pass at least one of --interval, --since, --lookback, --settle, --same-name"
|
|
46898
|
+
);
|
|
47075
46899
|
}
|
|
47076
|
-
|
|
47077
|
-
|
|
46900
|
+
const binding = await resolvePollBinding(cfg.client, cfg.filebox, cfg.signal);
|
|
46901
|
+
const current = await explainNotPolled(
|
|
46902
|
+
cfg.filebox,
|
|
46903
|
+
getPoll(cfg.client, binding.query_id, { signal: cfg.signal })
|
|
47078
46904
|
);
|
|
47079
|
-
}
|
|
47080
|
-
|
|
47081
|
-
|
|
47082
|
-
|
|
47083
|
-
|
|
46905
|
+
const body = { ...cfg.changes };
|
|
46906
|
+
const detail = [`target: ${bindingTarget(binding)}`];
|
|
46907
|
+
const wider = widening(current, cfg.changes, cfg.now ?? Date.now());
|
|
46908
|
+
if (wider !== null) {
|
|
46909
|
+
const what = `the polling window widens (start ${wider.from} -> ${wider.to})`;
|
|
46910
|
+
if (cfg.widen === void 0 && !cfg.interactive) {
|
|
46911
|
+
throw new Error(`${what}; pass --widen seed or --widen land (${WIDEN_HELP})`);
|
|
46912
|
+
}
|
|
46913
|
+
body.widen = cfg.widen ?? await ask(cfg, `${what}. ${WIDEN_HELP}.`, WIDEN_CHOICES);
|
|
46914
|
+
detail.push(`window: start ${wider.from} -> ${wider.to}, widen ${body.widen}`);
|
|
46915
|
+
} else if (cfg.widen !== void 0) {
|
|
46916
|
+
body.widen = cfg.widen;
|
|
47084
46917
|
}
|
|
47085
|
-
const
|
|
47086
|
-
|
|
47087
|
-
|
|
47088
|
-
|
|
47089
|
-
|
|
47090
|
-
|
|
47091
|
-
|
|
47092
|
-
|
|
47093
|
-
|
|
47094
|
-
|
|
46918
|
+
const newPolicy = cfg.changes.change_policy;
|
|
46919
|
+
const policySwitch = newPolicy !== void 0 && newPolicy !== current.change_policy;
|
|
46920
|
+
if (policySwitch) {
|
|
46921
|
+
const change = `${sameNameLabel(current.change_policy)} -> ${sameNameLabel(newPolicy)}`;
|
|
46922
|
+
const what = `--same-name changes ${change}`;
|
|
46923
|
+
if (cfg.onPolicyChange === void 0 && !cfg.interactive) {
|
|
46924
|
+
throw new Error(`${what}; pass --on-policy-change rekey, seed or land (${POLICY_HELP})`);
|
|
46925
|
+
}
|
|
46926
|
+
body.on_policy_change = cfg.onPolicyChange ?? await ask(cfg, `${what}. ${POLICY_HELP}.`, POLICY_CHOICES, "rekey");
|
|
46927
|
+
detail.push(`same name: ${change}, on policy change ${body.on_policy_change}`);
|
|
46928
|
+
} else if (cfg.onPolicyChange !== void 0) {
|
|
46929
|
+
body.on_policy_change = cfg.onPolicyChange;
|
|
47095
46930
|
}
|
|
47096
|
-
const
|
|
47097
|
-
|
|
47098
|
-
|
|
47099
|
-
|
|
47100
|
-
|
|
47101
|
-
|
|
46931
|
+
const put = async () => {
|
|
46932
|
+
const stored = await putPoll(cfg.client, binding.query_id, body, {
|
|
46933
|
+
signal: cfg.signal
|
|
46934
|
+
});
|
|
46935
|
+
if (cfg.asJson) {
|
|
46936
|
+
writeJSONIndented(cfg.out, stored);
|
|
46937
|
+
return;
|
|
46938
|
+
}
|
|
46939
|
+
renderPoll(cfg.out, "Polling settings updated", binding, stored);
|
|
46940
|
+
};
|
|
46941
|
+
const landsOrHides = wider !== null && body.widen === "land" || policySwitch && (body.on_policy_change === "seed" || body.on_policy_change === "land");
|
|
46942
|
+
if (!landsOrHides) {
|
|
46943
|
+
await put();
|
|
46944
|
+
return;
|
|
47102
46945
|
}
|
|
47103
|
-
}
|
|
47104
|
-
|
|
47105
|
-
|
|
47106
|
-
|
|
47107
|
-
|
|
47108
|
-
|
|
47109
|
-
|
|
47110
|
-
|
|
47111
|
-
|
|
47112
|
-
|
|
47113
|
-
|
|
47114
|
-
|
|
47115
|
-
|
|
47116
|
-
|
|
47117
|
-
|
|
46946
|
+
const flags = `${body.widen === void 0 ? "" : ` --widen ${body.widen}`}${body.on_policy_change === void 0 ? "" : ` --on-policy-change ${body.on_policy_change}`}`;
|
|
46947
|
+
cfg.deadline?.pause();
|
|
46948
|
+
await twoStepConfirm({
|
|
46949
|
+
out: cfg.out,
|
|
46950
|
+
action: pollConfirmAction(
|
|
46951
|
+
cfg.filebox,
|
|
46952
|
+
pollSetConfirmVerb(body.widen, body.on_policy_change),
|
|
46953
|
+
cfg.scope,
|
|
46954
|
+
{ ...body }
|
|
46955
|
+
),
|
|
46956
|
+
token: cfg.token,
|
|
46957
|
+
server: cfg.scope.serverKey,
|
|
46958
|
+
confirmCmd: `dr connectors poll set ${cfg.filebox}${cfg.settingsArgs}${flags}${cfg.targetArgs}`,
|
|
46959
|
+
describe: async () => detail,
|
|
46960
|
+
execute: async () => {
|
|
46961
|
+
cfg.deadline?.resume();
|
|
46962
|
+
await put();
|
|
47118
46963
|
}
|
|
47119
|
-
if (opts.json === true) writeJSONIndented(process.stdout, result);
|
|
47120
|
-
else print(process.stdout, result);
|
|
47121
46964
|
});
|
|
47122
46965
|
}
|
|
47123
|
-
function
|
|
47124
|
-
const
|
|
47125
|
-
|
|
47126
|
-
|
|
47127
|
-
|
|
47128
|
-
|
|
47129
|
-
if (secretRef === "" || secretRef.length > 200) {
|
|
47130
|
-
throw new Error(`invalid --secret-ref "${opts.secretRef}": expected 1-200 characters`);
|
|
47131
|
-
}
|
|
47132
|
-
if (!ROLE_ARN_RE.test(opts.roleArn)) {
|
|
47133
|
-
throw new Error(
|
|
47134
|
-
`invalid --role-arn "${opts.roleArn}": expected arn:aws:iam::<12-digit account>:role/<name>`
|
|
47135
|
-
);
|
|
47136
|
-
}
|
|
47137
|
-
const orgId = opts.orgId === void 0 ? void 0 : parsePositiveIntArg2(opts.orgId, "--org-id");
|
|
47138
|
-
const fields = {
|
|
47139
|
-
secret_ref: secretRef,
|
|
47140
|
-
role_arn: opts.roleArn,
|
|
47141
|
-
...orgId !== void 0 ? { org_id: orgId } : {}
|
|
47142
|
-
};
|
|
47143
|
-
await run(
|
|
47144
|
-
opts,
|
|
47145
|
-
({ client, signal }) => createAwsConnection(client, fields, { signal }),
|
|
47146
|
-
writeConnection
|
|
47147
|
-
);
|
|
47148
|
-
});
|
|
47149
|
-
withFlags2(
|
|
47150
|
-
aws.command("list").description("List the org's AWS connections").option("--secret-ref <datasourceId>", "only the connection of this datasource")
|
|
47151
|
-
).action(async (opts) => {
|
|
47152
|
-
await run(
|
|
47153
|
-
opts,
|
|
47154
|
-
({ client, signal }) => listAwsConnections(client, opts.secretRef, { signal }),
|
|
47155
|
-
writeConnectionTable
|
|
47156
|
-
);
|
|
47157
|
-
});
|
|
47158
|
-
withFlags2(
|
|
47159
|
-
aws.command("get").description("Show an AWS connection and its trust statement").argument("<id>", "connection id")
|
|
47160
|
-
).action(async (idArg, opts) => {
|
|
47161
|
-
const id = parsePositiveIntArg2(idArg, "connection id");
|
|
47162
|
-
await run(
|
|
47163
|
-
opts,
|
|
47164
|
-
({ client, signal }) => getAwsConnection(client, id, { signal }),
|
|
47165
|
-
writeConnection,
|
|
47166
|
-
id
|
|
47167
|
-
);
|
|
46966
|
+
function confirmDigest(scope, payload) {
|
|
46967
|
+
const canonical = JSON.stringify({
|
|
46968
|
+
server: scope.serverKey,
|
|
46969
|
+
account: scope.account ?? "",
|
|
46970
|
+
controlPlane: scope.controlPlane,
|
|
46971
|
+
payload: Object.fromEntries(Object.entries(payload).sort(([a], [b]) => a.localeCompare(b)))
|
|
47168
46972
|
});
|
|
47169
|
-
|
|
47170
|
-
["enable", true, "Enable an AWS connection; also retries now (stored credentials are dropped)"],
|
|
47171
|
-
["disable", false, "Disable an AWS connection (stored credentials are dropped)"]
|
|
47172
|
-
]) {
|
|
47173
|
-
withFlags2(aws.command(verb).description(help).argument("<id>", "connection id")).action(
|
|
47174
|
-
async (idArg, opts) => {
|
|
47175
|
-
const id = parsePositiveIntArg2(idArg, "connection id");
|
|
47176
|
-
await run(
|
|
47177
|
-
opts,
|
|
47178
|
-
({ client, signal }) => updateAwsConnection(client, id, { enabled }, { signal }),
|
|
47179
|
-
(out, c) => {
|
|
47180
|
-
writeConnection(out, c);
|
|
47181
|
-
if (enabled)
|
|
47182
|
-
out.write(
|
|
47183
|
-
"\nStored credentials were dropped; the poller assumes the role again now.\n"
|
|
47184
|
-
);
|
|
47185
|
-
},
|
|
47186
|
-
id
|
|
47187
|
-
);
|
|
47188
|
-
}
|
|
47189
|
-
);
|
|
47190
|
-
}
|
|
46973
|
+
return createHash12("sha256").update(canonical).digest("hex").slice(0, 12);
|
|
47191
46974
|
}
|
|
47192
|
-
|
|
47193
|
-
|
|
47194
|
-
|
|
47195
|
-
|
|
47196
|
-
|
|
47197
|
-
function buildDevEnvelope(i) {
|
|
47198
|
-
const config = {
|
|
47199
|
-
...i.publicConfig,
|
|
47200
|
-
...i.queryConfig,
|
|
47201
|
-
// Coerce absent values to null (not undefined) so JSON.stringify keeps the
|
|
47202
|
-
// key — matches build_hook_job's dq.get(...) -> None; a dropped objectType
|
|
47203
|
-
// becomes a cryptic KeyError in the runner's dispatch.
|
|
47204
|
-
objectType: i.dataQuery.objectType ?? null,
|
|
47205
|
-
type: i.dataQuery.type ?? null,
|
|
47206
|
-
query: i.dataQuery.query
|
|
46975
|
+
function pollConfirmAction(filebox, verb, scope, payload = {}) {
|
|
46976
|
+
return {
|
|
46977
|
+
resource: "connector-poll",
|
|
46978
|
+
id: `${filebox.trim()}#${confirmDigest(scope, payload)}`,
|
|
46979
|
+
verb
|
|
47207
46980
|
};
|
|
47208
|
-
|
|
46981
|
+
}
|
|
46982
|
+
function pollItemConfirmAction(filebox, item, scope, output) {
|
|
47209
46983
|
return {
|
|
47210
|
-
|
|
47211
|
-
|
|
47212
|
-
|
|
47213
|
-
cred_token: "dev-local",
|
|
47214
|
-
output_sas: "file:///out/output.csv",
|
|
47215
|
-
input_sas: null,
|
|
47216
|
-
cursor: null,
|
|
47217
|
-
contract_version: i.contractVersion,
|
|
47218
|
-
metadata: {}
|
|
46984
|
+
resource: "connector-poll-item",
|
|
46985
|
+
id: `${filebox.trim()}/${item}#${confirmDigest(scope, output === void 0 ? {} : { output: output.trim() })}`,
|
|
46986
|
+
verb: "skip"
|
|
47219
46987
|
};
|
|
47220
46988
|
}
|
|
47221
|
-
function
|
|
47222
|
-
|
|
47223
|
-
|
|
46989
|
+
async function runPollDisable(cfg) {
|
|
46990
|
+
const binding = await resolvePollBinding(cfg.client, cfg.filebox, cfg.signal);
|
|
46991
|
+
await explainNotPolled(
|
|
46992
|
+
cfg.filebox,
|
|
46993
|
+
getPoll(cfg.client, binding.query_id, { signal: cfg.signal })
|
|
46994
|
+
);
|
|
46995
|
+
const stored = await putPoll(
|
|
46996
|
+
cfg.client,
|
|
46997
|
+
binding.query_id,
|
|
46998
|
+
{ enabled: false },
|
|
46999
|
+
{ signal: cfg.signal }
|
|
47000
|
+
);
|
|
47001
|
+
if (cfg.asJson) {
|
|
47002
|
+
writeJSONIndented(cfg.out, stored);
|
|
47003
|
+
return;
|
|
47224
47004
|
}
|
|
47225
|
-
|
|
47005
|
+
renderPoll(cfg.out, "Polling disabled", binding, stored);
|
|
47226
47006
|
}
|
|
47227
|
-
function
|
|
47228
|
-
const
|
|
47229
|
-
|
|
47230
|
-
|
|
47231
|
-
|
|
47232
|
-
|
|
47007
|
+
async function runPollGet(cfg) {
|
|
47008
|
+
const binding = await resolvePollBinding(cfg.client, cfg.filebox, cfg.signal);
|
|
47009
|
+
const p = await explainNotPolled(
|
|
47010
|
+
cfg.filebox,
|
|
47011
|
+
getPoll(cfg.client, binding.query_id, { signal: cfg.signal })
|
|
47012
|
+
);
|
|
47013
|
+
if (cfg.asJson) {
|
|
47014
|
+
writeJSONIndented(cfg.out, p);
|
|
47015
|
+
return;
|
|
47233
47016
|
}
|
|
47234
|
-
|
|
47235
|
-
|
|
47236
|
-
|
|
47237
|
-
|
|
47238
|
-
|
|
47239
|
-
|
|
47240
|
-
|
|
47241
|
-
|
|
47242
|
-
|
|
47017
|
+
renderPoll(cfg.out, "Polling", binding, p);
|
|
47018
|
+
}
|
|
47019
|
+
async function runPollRun(cfg) {
|
|
47020
|
+
const binding = await resolvePollBinding(cfg.client, cfg.filebox, cfg.signal);
|
|
47021
|
+
let p;
|
|
47022
|
+
try {
|
|
47023
|
+
p = await explainNotPolled(
|
|
47024
|
+
cfg.filebox,
|
|
47025
|
+
runPoll(cfg.client, binding.query_id, { signal: cfg.signal })
|
|
47026
|
+
);
|
|
47027
|
+
} catch (err) {
|
|
47028
|
+
if (isPollThrottled(err)) {
|
|
47243
47029
|
throw new Error(
|
|
47244
|
-
`
|
|
47030
|
+
`a poll of filebox ${cfg.filebox} is running or ran less than a minute ago; try again shortly`,
|
|
47031
|
+
{ cause: err }
|
|
47245
47032
|
);
|
|
47246
47033
|
}
|
|
47247
|
-
|
|
47034
|
+
throw err;
|
|
47248
47035
|
}
|
|
47249
|
-
|
|
47036
|
+
if (cfg.asJson) {
|
|
47037
|
+
writeJSONIndented(cfg.out, p);
|
|
47038
|
+
return;
|
|
47039
|
+
}
|
|
47040
|
+
renderPoll(cfg.out, "Poll requested", binding, p);
|
|
47250
47041
|
}
|
|
47251
|
-
function
|
|
47252
|
-
const
|
|
47253
|
-
|
|
47254
|
-
|
|
47255
|
-
|
|
47042
|
+
async function runPollResume(cfg) {
|
|
47043
|
+
const binding = await resolvePollBinding(cfg.client, cfg.filebox, cfg.signal);
|
|
47044
|
+
const resume = async () => {
|
|
47045
|
+
const p = await explainNotPolled(
|
|
47046
|
+
cfg.filebox,
|
|
47047
|
+
resumePoll(
|
|
47048
|
+
cfg.client,
|
|
47049
|
+
binding.query_id,
|
|
47050
|
+
cfg.source === void 0 ? {} : { source: cfg.source },
|
|
47051
|
+
{ signal: cfg.signal }
|
|
47052
|
+
)
|
|
47256
47053
|
);
|
|
47054
|
+
if (cfg.asJson) {
|
|
47055
|
+
writeJSONIndented(cfg.out, p);
|
|
47056
|
+
return;
|
|
47057
|
+
}
|
|
47058
|
+
renderPoll(cfg.out, "Polling resumed", binding, p);
|
|
47059
|
+
};
|
|
47060
|
+
if (cfg.source !== "land") {
|
|
47061
|
+
await resume();
|
|
47062
|
+
return;
|
|
47257
47063
|
}
|
|
47258
|
-
|
|
47259
|
-
|
|
47260
|
-
|
|
47261
|
-
|
|
47262
|
-
|
|
47263
|
-
|
|
47264
|
-
|
|
47265
|
-
|
|
47266
|
-
|
|
47267
|
-
|
|
47268
|
-
|
|
47269
|
-
|
|
47270
|
-
|
|
47271
|
-
|
|
47272
|
-
|
|
47273
|
-
|
|
47274
|
-
} catch {
|
|
47275
|
-
throw new Error(`${input.queryPath}: dataquery source not found or unreadable`);
|
|
47276
|
-
}
|
|
47277
|
-
await fs19.mkdir(input.outDir, { recursive: true });
|
|
47278
|
-
await fs19.writeFile(
|
|
47279
|
-
path9.join(input.outDir, ENV_FILE),
|
|
47280
|
-
JSON.stringify(input.env, null, 2) + "\n",
|
|
47281
|
-
"utf-8"
|
|
47282
|
-
);
|
|
47283
|
-
const secretsPath = path9.join(input.outDir, SECRETS_FILE);
|
|
47284
|
-
await fs19.writeFile(
|
|
47285
|
-
secretsPath,
|
|
47286
|
-
JSON.stringify({ [input.secretRef]: input.credentials }, null, 2) + "\n",
|
|
47287
|
-
"utf-8"
|
|
47288
|
-
);
|
|
47289
|
-
if (process.platform !== "win32") {
|
|
47290
|
-
await fs19.chmod(secretsPath, 384);
|
|
47291
|
-
}
|
|
47292
|
-
await fs19.writeFile(path9.join(input.outDir, CONNECTION_FILE), connectionSource, "utf-8");
|
|
47293
|
-
await fs19.writeFile(path9.join(input.outDir, QUERY_FILE), querySource, "utf-8");
|
|
47294
|
-
}
|
|
47295
|
-
async function ensureGitignored(entry) {
|
|
47296
|
-
const gitignorePath = path9.join(process.cwd(), ".gitignore");
|
|
47297
|
-
let current = "";
|
|
47298
|
-
try {
|
|
47299
|
-
current = await fs19.readFile(gitignorePath, "utf-8");
|
|
47300
|
-
} catch {
|
|
47301
|
-
}
|
|
47302
|
-
const normalized = entry.split("\\").join("/").replace(/\/+$/, "") + "/";
|
|
47303
|
-
const lines = current.split(/\r?\n/).map((l) => l.trim());
|
|
47304
|
-
if (lines.includes(normalized) || lines.includes(entry)) return;
|
|
47305
|
-
const sep = current === "" || current.endsWith("\n") ? "" : "\n";
|
|
47306
|
-
await fs19.writeFile(gitignorePath, `${current}${sep}${normalized}
|
|
47307
|
-
`, "utf-8");
|
|
47064
|
+
cfg.deadline?.pause();
|
|
47065
|
+
await twoStepConfirm({
|
|
47066
|
+
out: cfg.out,
|
|
47067
|
+
action: pollConfirmAction(cfg.filebox, "resume-land", cfg.scope),
|
|
47068
|
+
token: cfg.token,
|
|
47069
|
+
server: cfg.scope.serverKey,
|
|
47070
|
+
confirmCmd: `dr connectors poll resume ${cfg.filebox} --source land${cfg.targetArgs}`,
|
|
47071
|
+
describe: async () => [
|
|
47072
|
+
`target: ${bindingTarget(binding)}`,
|
|
47073
|
+
"source: land every item the changed source holds now"
|
|
47074
|
+
],
|
|
47075
|
+
execute: async () => {
|
|
47076
|
+
cfg.deadline?.resume();
|
|
47077
|
+
await resume();
|
|
47078
|
+
}
|
|
47079
|
+
});
|
|
47308
47080
|
}
|
|
47309
|
-
function
|
|
47310
|
-
const
|
|
47311
|
-
|
|
47312
|
-
|
|
47081
|
+
async function countSeedable(cfg, binding) {
|
|
47082
|
+
const untilMs = Date.parse(cfg.until);
|
|
47083
|
+
let count = 0;
|
|
47084
|
+
for (const state of ["seen", "new"]) {
|
|
47085
|
+
let after = null;
|
|
47086
|
+
do {
|
|
47087
|
+
cfg.deadline?.resume();
|
|
47088
|
+
const page = await explainNotPolled(
|
|
47089
|
+
cfg.filebox,
|
|
47090
|
+
listPollItems(
|
|
47091
|
+
cfg.client,
|
|
47092
|
+
binding.query_id,
|
|
47093
|
+
{ state, ...after === null ? {} : { after } },
|
|
47094
|
+
{ signal: cfg.signal }
|
|
47095
|
+
)
|
|
47096
|
+
);
|
|
47097
|
+
count += page.items.filter(
|
|
47098
|
+
(it) => it.source_modified_at !== null && Date.parse(it.source_modified_at) <= untilMs
|
|
47099
|
+
).length;
|
|
47100
|
+
after = page.next_after;
|
|
47101
|
+
} while (after !== null);
|
|
47313
47102
|
}
|
|
47314
|
-
return
|
|
47103
|
+
return count;
|
|
47315
47104
|
}
|
|
47316
|
-
function
|
|
47317
|
-
const
|
|
47318
|
-
|
|
47319
|
-
|
|
47320
|
-
|
|
47321
|
-
|
|
47322
|
-
|
|
47323
|
-
"--document-id <n>",
|
|
47324
|
-
"numeric filebox document id (no uuid->id lookup exists; required)"
|
|
47325
|
-
).requiredOption("--connection <path>", "path to the connector's connection .py source").requiredOption("--query <path>", "path to the connector's dataquery .py source").option("--from <date>", "period start (paired with --to)").option("--to <date>", "period end (paired with --from)").option("-o, --output <dir>", "output run dir (default ./dev-run/<fileboxUuid>/)").action(async (fileboxUuid, opts) => {
|
|
47326
|
-
const datasourceId = parsePositiveInt3(opts.datasource, "datasource id");
|
|
47327
|
-
const documentId = parsePositiveInt3(opts.documentId, "document-id");
|
|
47328
|
-
if (opts.from === void 0 !== (opts.to === void 0)) {
|
|
47329
|
-
throw new Error("--from and --to must be provided together");
|
|
47105
|
+
async function runPollSeed(cfg) {
|
|
47106
|
+
const binding = await resolvePollBinding(cfg.client, cfg.filebox, cfg.signal);
|
|
47107
|
+
if (cfg.dryRun) {
|
|
47108
|
+
const wouldSeed = await countSeedable(cfg, binding);
|
|
47109
|
+
if (cfg.asJson) {
|
|
47110
|
+
writeJSONIndented(cfg.out, { dry_run: true, until: cfg.until, would_seed: wouldSeed });
|
|
47111
|
+
return;
|
|
47330
47112
|
}
|
|
47331
|
-
|
|
47332
|
-
const outDir = opts.output ?? defaultRoot;
|
|
47333
|
-
const gitignoreEntry = opts.output ?? "dev-run";
|
|
47334
|
-
const serverKey = resolveServer(opts.server);
|
|
47335
|
-
const host = resolveConnectorsUrl(serverKey, opts.connectorsUrl);
|
|
47336
|
-
const monolithHost = resolve(serverKey).host;
|
|
47337
|
-
const cpSetup = await setupCall(opts.server, opts.account, { host, authHost: monolithHost });
|
|
47338
|
-
try {
|
|
47339
|
-
const binding = await getBinding(cpSetup.client, fileboxUuid, {
|
|
47340
|
-
signal: cpSetup.signal
|
|
47341
|
-
});
|
|
47342
|
-
const monoSetup = await setupCall(opts.server, opts.account, {
|
|
47343
|
-
host: monolithHost,
|
|
47344
|
-
authHost: monolithHost
|
|
47345
|
-
});
|
|
47346
|
-
try {
|
|
47347
|
-
const rawDataQuery = await getDataQueryRaw(monoSetup.client, documentId, {
|
|
47348
|
-
signal: monoSetup.signal
|
|
47349
|
-
});
|
|
47350
|
-
const dataQuery = extractDataQueryFields(rawDataQuery);
|
|
47351
|
-
if (dataQuery.objectType === void 0 || dataQuery.objectType === null || dataQuery.objectType === "") {
|
|
47352
|
-
process.stderr.write(
|
|
47353
|
-
'WARNING: objectType is not set inside the Sync-query (config.query.objectType). get_query_class will receive None \u2014 fine only if the connector has a single default dataset. Add it to the data-query, e.g. config.query.objectType = "<dataset>".\n'
|
|
47354
|
-
);
|
|
47355
|
-
}
|
|
47356
|
-
const integration = await itGet(monoSetup.client, datasourceId, {
|
|
47357
|
-
signal: monoSetup.signal
|
|
47358
|
-
});
|
|
47359
|
-
const credentials = extractCredentials(integration);
|
|
47360
|
-
let period;
|
|
47361
|
-
if (opts.from !== void 0 && opts.to !== void 0) {
|
|
47362
|
-
period = { from: opts.from, to: opts.to };
|
|
47363
|
-
}
|
|
47364
|
-
const env = buildDevEnvelope({
|
|
47365
|
-
correlationId: randomUUID2(),
|
|
47366
|
-
secretRef: binding.secret_ref,
|
|
47367
|
-
publicConfig: binding.public_config,
|
|
47368
|
-
queryConfig: binding.query_config,
|
|
47369
|
-
dataQuery,
|
|
47370
|
-
...period !== void 0 ? { period } : {},
|
|
47371
|
-
// Deliberate sandbox stub: the fetch/dispatch path doesn't branch on
|
|
47372
|
-
// contract_version, so a fixed "1" is sufficient for a local run.
|
|
47373
|
-
contractVersion: "1"
|
|
47374
|
-
});
|
|
47375
|
-
await writeRunDir({
|
|
47376
|
-
outDir,
|
|
47377
|
-
env,
|
|
47378
|
-
secretRef: binding.secret_ref,
|
|
47379
|
-
credentials,
|
|
47380
|
-
connectionPath: opts.connection,
|
|
47381
|
-
queryPath: opts.query
|
|
47382
|
-
});
|
|
47383
|
-
await ensureGitignored(gitignoreEntry);
|
|
47384
|
-
process.stdout.write(`Wrote run dir: ${outDir}
|
|
47113
|
+
cfg.out.write(`Seed dry run for ${bindingTarget(binding)}: nothing was changed.
|
|
47385
47114
|
`);
|
|
47386
|
-
|
|
47387
|
-
|
|
47115
|
+
cfg.out.write(
|
|
47116
|
+
`${wouldSeed} already-discovered items would be seeded (items not yet discovered are not counted).
|
|
47388
47117
|
`
|
|
47389
|
-
|
|
47390
|
-
|
|
47391
|
-
|
|
47392
|
-
|
|
47393
|
-
|
|
47118
|
+
);
|
|
47119
|
+
return;
|
|
47120
|
+
}
|
|
47121
|
+
cfg.deadline?.pause();
|
|
47122
|
+
await twoStepConfirm({
|
|
47123
|
+
out: cfg.out,
|
|
47124
|
+
action: pollConfirmAction(cfg.filebox, "seed", cfg.scope, { until: cfg.until }),
|
|
47125
|
+
token: cfg.token,
|
|
47126
|
+
server: cfg.scope.serverKey,
|
|
47127
|
+
confirmCmd: `dr connectors poll seed ${cfg.filebox} --until ${cfg.untilArg}${cfg.targetArgs}`,
|
|
47128
|
+
describe: async () => [
|
|
47129
|
+
`target: ${bindingTarget(binding)}`,
|
|
47130
|
+
`until: ${cfg.until}`,
|
|
47131
|
+
"effect: items modified at or before it are marked landed and never land"
|
|
47132
|
+
],
|
|
47133
|
+
execute: async () => {
|
|
47134
|
+
cfg.deadline?.resume();
|
|
47135
|
+
const p = await explainNotPolled(
|
|
47136
|
+
cfg.filebox,
|
|
47137
|
+
seedPoll(
|
|
47138
|
+
cfg.client,
|
|
47139
|
+
binding.query_id,
|
|
47140
|
+
{ until: cfg.until },
|
|
47141
|
+
{
|
|
47142
|
+
signal: cfg.signal
|
|
47143
|
+
}
|
|
47144
|
+
)
|
|
47145
|
+
);
|
|
47146
|
+
if (cfg.asJson) {
|
|
47147
|
+
writeJSONIndented(cfg.out, p);
|
|
47148
|
+
return;
|
|
47394
47149
|
}
|
|
47395
|
-
|
|
47396
|
-
throw augmentTlsCertError(err);
|
|
47397
|
-
} finally {
|
|
47398
|
-
cpSetup.cleanup();
|
|
47150
|
+
renderPoll(cfg.out, "Seed requested", binding, p);
|
|
47399
47151
|
}
|
|
47400
47152
|
});
|
|
47401
|
-
addServerFlag(cmd);
|
|
47402
|
-
addAccountFlag(cmd);
|
|
47403
|
-
cmd.option(
|
|
47404
|
-
"--connectors-url <url>",
|
|
47405
|
-
"Override the control-plane base URL (e.g. a kubectl port-forward http://localhost:8000)"
|
|
47406
|
-
);
|
|
47407
47153
|
}
|
|
47408
|
-
|
|
47409
|
-
|
|
47410
|
-
|
|
47411
|
-
|
|
47412
|
-
|
|
47413
|
-
|
|
47414
|
-
|
|
47415
|
-
|
|
47416
|
-
|
|
47417
|
-
|
|
47418
|
-
|
|
47419
|
-
|
|
47420
|
-
|
|
47421
|
-
|
|
47422
|
-
|
|
47423
|
-
|
|
47424
|
-
|
|
47425
|
-
|
|
47426
|
-
|
|
47427
|
-
// src/commands/connectorsPoll.ts
|
|
47428
|
-
var DURATION_UNIT_SECONDS2 = { s: 1, m: 60, h: 3600, d: 86400 };
|
|
47429
|
-
var DURATION_BOUNDS = {
|
|
47430
|
-
"--interval": [300, 7 * 86400],
|
|
47431
|
-
"--settle": [0, 86400]
|
|
47432
|
-
};
|
|
47433
|
-
var LOOKBACK_MAX_DAYS = 3650;
|
|
47434
|
-
function parseDurationSeconds(raw2, flag) {
|
|
47435
|
-
const m = /^(\d+)([smhd])?$/.exec(raw2.trim());
|
|
47436
|
-
const seconds = m === null ? NaN : Number(m[1]) * (DURATION_UNIT_SECONDS2[m[2] ?? "s"] ?? 1);
|
|
47437
|
-
const [min, max] = DURATION_BOUNDS[flag] ?? [0, Number.MAX_SAFE_INTEGER];
|
|
47438
|
-
if (m === null) {
|
|
47439
|
-
throw new Error(`invalid ${flag} "${raw2}": expected a duration like 90s, 5m or 1h`);
|
|
47154
|
+
async function runPollItems(cfg) {
|
|
47155
|
+
const binding = await resolvePollBinding(cfg.client, cfg.filebox, cfg.signal);
|
|
47156
|
+
const page = await explainNotPolled(
|
|
47157
|
+
cfg.filebox,
|
|
47158
|
+
listPollItems(
|
|
47159
|
+
cfg.client,
|
|
47160
|
+
binding.query_id,
|
|
47161
|
+
{
|
|
47162
|
+
...cfg.state !== void 0 ? { state: cfg.state } : {},
|
|
47163
|
+
...cfg.limit !== void 0 ? { limit: cfg.limit } : {},
|
|
47164
|
+
...cfg.after !== void 0 ? { after: cfg.after } : {}
|
|
47165
|
+
},
|
|
47166
|
+
{ signal: cfg.signal }
|
|
47167
|
+
)
|
|
47168
|
+
);
|
|
47169
|
+
if (cfg.asJson) {
|
|
47170
|
+
writeJSONIndented(cfg.out, page);
|
|
47171
|
+
return;
|
|
47440
47172
|
}
|
|
47441
|
-
if (
|
|
47442
|
-
|
|
47443
|
-
|
|
47444
|
-
|
|
47173
|
+
if (page.items.length === 0) {
|
|
47174
|
+
cfg.out.write(`No items${cfg.state === void 0 ? "" : ` in state ${cfg.state}`}.
|
|
47175
|
+
`);
|
|
47176
|
+
return;
|
|
47445
47177
|
}
|
|
47446
|
-
|
|
47447
|
-
|
|
47448
|
-
|
|
47449
|
-
|
|
47450
|
-
|
|
47451
|
-
|
|
47452
|
-
|
|
47453
|
-
|
|
47178
|
+
renderItemTable(cfg.out, page.items);
|
|
47179
|
+
if (page.next_after !== null) {
|
|
47180
|
+
const state = cfg.state === void 0 ? "" : ` --state ${cfg.state}`;
|
|
47181
|
+
const limit = cfg.limit === void 0 ? "" : ` --limit ${cfg.limit}`;
|
|
47182
|
+
cfg.out.write(
|
|
47183
|
+
`
|
|
47184
|
+
More: dr connectors poll items ${cfg.filebox}${state}${limit} --after ${page.next_after}
|
|
47185
|
+
`
|
|
47454
47186
|
);
|
|
47455
47187
|
}
|
|
47456
|
-
return days;
|
|
47457
|
-
}
|
|
47458
|
-
var DATE_ONLY_RE = /^(\d{4})-(\d{2})-(\d{2})$/;
|
|
47459
|
-
var DATE_TIME_RE = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2})(?::(\d{2})(?:\.\d+)?)?(?:Z|[+-](\d{2}):(\d{2}))$/;
|
|
47460
|
-
function inRange(value, max, min = 0) {
|
|
47461
|
-
const n = Number(value ?? "0");
|
|
47462
|
-
return n >= min && n <= max;
|
|
47463
47188
|
}
|
|
47464
|
-
function
|
|
47465
|
-
if (
|
|
47466
|
-
|
|
47467
|
-
return inRange(d, daysInMonth3, 1);
|
|
47189
|
+
async function resolveItemOutput(cfg) {
|
|
47190
|
+
if (cfg.output === void 0) return void 0;
|
|
47191
|
+
return (await resolvePollBinding(cfg.client, cfg.output, cfg.signal)).query_id;
|
|
47468
47192
|
}
|
|
47469
|
-
function
|
|
47470
|
-
|
|
47471
|
-
|
|
47472
|
-
|
|
47473
|
-
const value = raw2.trim();
|
|
47474
|
-
const dateOnly = DATE_ONLY_RE.exec(value);
|
|
47475
|
-
if (dateOnly !== null) {
|
|
47476
|
-
if (!isCalendarDay(dateOnly[1], dateOnly[2], dateOnly[3])) throw invalid;
|
|
47477
|
-
return (/* @__PURE__ */ new Date(`${value}T00:00:00Z`)).toISOString();
|
|
47193
|
+
function renderItemResult(cfg, binding, headline, item) {
|
|
47194
|
+
if (cfg.asJson) {
|
|
47195
|
+
writeJSONIndented(cfg.out, item);
|
|
47196
|
+
return;
|
|
47478
47197
|
}
|
|
47479
|
-
|
|
47480
|
-
|
|
47481
|
-
|
|
47482
|
-
if (Number.isNaN(ms)) throw invalid;
|
|
47483
|
-
return new Date(ms).toISOString();
|
|
47198
|
+
cfg.out.write(`${headline} (${bindingTarget(binding)})
|
|
47199
|
+
`);
|
|
47200
|
+
renderItemTable(cfg.out, [item]);
|
|
47484
47201
|
}
|
|
47485
|
-
function
|
|
47486
|
-
|
|
47487
|
-
|
|
47488
|
-
|
|
47202
|
+
async function runPollRetry(cfg) {
|
|
47203
|
+
const binding = await resolvePollBinding(cfg.client, cfg.filebox, cfg.signal);
|
|
47204
|
+
const outputId = await resolveItemOutput(cfg);
|
|
47205
|
+
const item = await explainNotPolled(
|
|
47206
|
+
cfg.filebox,
|
|
47207
|
+
retryPollItem(
|
|
47208
|
+
cfg.client,
|
|
47209
|
+
binding.query_id,
|
|
47210
|
+
cfg.itemId,
|
|
47211
|
+
{ signal: cfg.signal },
|
|
47212
|
+
outputId
|
|
47213
|
+
)
|
|
47214
|
+
);
|
|
47215
|
+
const what = outputId === void 0 ? "" : ` (output #${outputId})`;
|
|
47216
|
+
renderItemResult(cfg, binding, `Item ${cfg.itemId}${what} queued for retry`, item);
|
|
47489
47217
|
}
|
|
47490
|
-
function
|
|
47491
|
-
const
|
|
47492
|
-
|
|
47493
|
-
|
|
47494
|
-
|
|
47495
|
-
|
|
47218
|
+
async function runPollSkip(cfg) {
|
|
47219
|
+
const binding = await resolvePollBinding(cfg.client, cfg.filebox, cfg.signal);
|
|
47220
|
+
const outputId = await resolveItemOutput(cfg);
|
|
47221
|
+
const outputArg = cfg.output === void 0 ? "" : ` --output ${cfg.output.trim()}`;
|
|
47222
|
+
cfg.deadline?.pause();
|
|
47223
|
+
await twoStepConfirm({
|
|
47224
|
+
out: cfg.out,
|
|
47225
|
+
action: pollItemConfirmAction(cfg.filebox, cfg.itemId, cfg.scope, cfg.output),
|
|
47226
|
+
token: cfg.token,
|
|
47227
|
+
server: cfg.scope.serverKey,
|
|
47228
|
+
confirmCmd: `dr connectors poll skip ${cfg.filebox} ${cfg.itemId}${outputArg}${cfg.targetArgs}`,
|
|
47229
|
+
describe: async () => [
|
|
47230
|
+
`item: ${cfg.itemId} of ${bindingTarget(binding)}`,
|
|
47231
|
+
...outputId === void 0 ? ["effect: the item is never landed (dr connectors poll retry queues it again)"] : [
|
|
47232
|
+
`output: query #${outputId} only`,
|
|
47233
|
+
"effect: the item is never landed in that output; its other outputs still land (dr connectors poll retry --output queues it again)"
|
|
47234
|
+
]
|
|
47235
|
+
],
|
|
47236
|
+
execute: async () => {
|
|
47237
|
+
cfg.deadline?.resume();
|
|
47238
|
+
const item = await explainNotPolled(
|
|
47239
|
+
cfg.filebox,
|
|
47240
|
+
skipPollItem(
|
|
47241
|
+
cfg.client,
|
|
47242
|
+
binding.query_id,
|
|
47243
|
+
cfg.itemId,
|
|
47244
|
+
{ signal: cfg.signal },
|
|
47245
|
+
outputId
|
|
47246
|
+
)
|
|
47247
|
+
);
|
|
47248
|
+
const what = outputId === void 0 ? "" : ` (output #${outputId})`;
|
|
47249
|
+
renderItemResult(cfg, binding, `Item ${cfg.itemId}${what} skipped`, item);
|
|
47250
|
+
}
|
|
47251
|
+
});
|
|
47496
47252
|
}
|
|
47497
|
-
|
|
47498
|
-
|
|
47499
|
-
|
|
47500
|
-
|
|
47501
|
-
|
|
47502
|
-
|
|
47503
|
-
|
|
47504
|
-
|
|
47505
|
-
|
|
47506
|
-
|
|
47507
|
-
|
|
47253
|
+
async function runPollOutputs(cfg) {
|
|
47254
|
+
const binding = await resolvePollBinding(cfg.client, cfg.filebox, cfg.signal);
|
|
47255
|
+
const current = await explainNotPolled(
|
|
47256
|
+
cfg.filebox,
|
|
47257
|
+
getPoll(cfg.client, binding.query_id, { signal: cfg.signal })
|
|
47258
|
+
);
|
|
47259
|
+
if (cfg.add.length === 0 && cfg.remove.length === 0 && !cfg.clear) {
|
|
47260
|
+
if (cfg.asJson) {
|
|
47261
|
+
writeJSONIndented(cfg.out, current.outputs ?? []);
|
|
47262
|
+
return;
|
|
47263
|
+
}
|
|
47264
|
+
renderPoll(cfg.out, "Polling", binding, current);
|
|
47265
|
+
return;
|
|
47508
47266
|
}
|
|
47509
|
-
|
|
47510
|
-
|
|
47511
|
-
}
|
|
47512
|
-
function settingsArgs(opts) {
|
|
47513
|
-
const pairs = [
|
|
47514
|
-
["--interval", opts.interval],
|
|
47515
|
-
["--since", opts.since],
|
|
47516
|
-
["--lookback", opts.lookback],
|
|
47517
|
-
["--settle", opts.settle],
|
|
47518
|
-
["--same-name", opts.sameName]
|
|
47519
|
-
];
|
|
47520
|
-
return pairs.filter((p) => p[1] !== void 0).map(([flag, value]) => ` ${flag} ${value.trim()}`).join("");
|
|
47521
|
-
}
|
|
47522
|
-
async function resolvePollBinding(client, filebox, signal) {
|
|
47523
|
-
const wanted = filebox.trim();
|
|
47524
|
-
const byId = /^\d+$/.test(wanted);
|
|
47525
|
-
const rows = await listQueries(client, { signal });
|
|
47526
|
-
const matches = rows.filter(
|
|
47527
|
-
(q) => byId ? q.filebox_id === Number(wanted) : q.filebox_uuid.toLowerCase() === wanted.toLowerCase()
|
|
47267
|
+
const kept = new Map(
|
|
47268
|
+
cfg.clear ? [] : (current.outputs ?? []).map((o) => [o.connector_query_id, o.date_tag])
|
|
47528
47269
|
);
|
|
47529
|
-
const
|
|
47530
|
-
|
|
47531
|
-
|
|
47532
|
-
`no connector binding for filebox ${wanted} in this org (see dr connectors query list)`
|
|
47270
|
+
for (const filebox of cfg.remove) {
|
|
47271
|
+
const stored2 = (current.outputs ?? []).find(
|
|
47272
|
+
(o) => o.filebox_uuid.toLowerCase() === filebox.trim().toLowerCase()
|
|
47533
47273
|
);
|
|
47274
|
+
const queryId = stored2?.connector_query_id ?? (await resolvePollBinding(cfg.client, filebox, cfg.signal)).query_id;
|
|
47275
|
+
if (!kept.delete(queryId)) {
|
|
47276
|
+
throw new Error(`filebox ${filebox} is not an output of ${bindingTarget(binding)}`);
|
|
47277
|
+
}
|
|
47534
47278
|
}
|
|
47535
|
-
|
|
47536
|
-
const
|
|
47537
|
-
|
|
47538
|
-
|
|
47539
|
-
|
|
47540
|
-
|
|
47541
|
-
function isNotPolled(err) {
|
|
47542
|
-
return err instanceof ControlPlaneError && err.status === 404 && err.body.includes("not polled");
|
|
47543
|
-
}
|
|
47544
|
-
function isPollThrottled(err) {
|
|
47545
|
-
return err instanceof ControlPlaneError && err.status === 409 && (err.body.includes("discover is in flight") || err.body.includes("polled less than"));
|
|
47546
|
-
}
|
|
47547
|
-
async function explainNotPolled(filebox, call3) {
|
|
47548
|
-
try {
|
|
47549
|
-
return await call3;
|
|
47550
|
-
} catch (err) {
|
|
47551
|
-
if (isNotPolled(err)) {
|
|
47279
|
+
for (const filebox of cfg.add) {
|
|
47280
|
+
const output = await resolvePollBinding(cfg.client, filebox, cfg.signal);
|
|
47281
|
+
if (output.query_id === binding.query_id) {
|
|
47282
|
+
throw new Error(`filebox ${filebox} is the binding's own filebox, always its first output`);
|
|
47283
|
+
}
|
|
47284
|
+
if (output.filebox_id <= 0) {
|
|
47552
47285
|
throw new Error(
|
|
47553
|
-
`
|
|
47554
|
-
{ cause: err }
|
|
47286
|
+
`connector-query #${output.query_id} has no filebox id recorded, so its date tag cannot be read; record it with dr connectors query repoint ${output.query_id} --filebox-id <id>`
|
|
47555
47287
|
);
|
|
47556
47288
|
}
|
|
47557
|
-
|
|
47289
|
+
kept.set(
|
|
47290
|
+
output.query_id,
|
|
47291
|
+
await readFileboxDateTag(cfg.monolith, output.filebox_id, cfg.signal)
|
|
47292
|
+
);
|
|
47293
|
+
}
|
|
47294
|
+
const outputs = [...kept].map(([id, tag]) => ({ connector_query_id: id, date_tag: tag }));
|
|
47295
|
+
const stored = await putPoll(
|
|
47296
|
+
cfg.client,
|
|
47297
|
+
binding.query_id,
|
|
47298
|
+
{ outputs },
|
|
47299
|
+
{ signal: cfg.signal }
|
|
47300
|
+
);
|
|
47301
|
+
if (cfg.asJson) {
|
|
47302
|
+
writeJSONIndented(cfg.out, stored);
|
|
47303
|
+
return;
|
|
47558
47304
|
}
|
|
47305
|
+
renderPoll(cfg.out, "Polling outputs updated", binding, stored);
|
|
47559
47306
|
}
|
|
47560
|
-
async function
|
|
47561
|
-
const
|
|
47562
|
-
const
|
|
47563
|
-
|
|
47564
|
-
|
|
47565
|
-
|
|
47566
|
-
|
|
47567
|
-
|
|
47568
|
-
|
|
47569
|
-
);
|
|
47307
|
+
async function withMonolith(opts, action) {
|
|
47308
|
+
const monolithHost = resolve(resolveServer(opts.server)).host;
|
|
47309
|
+
const setup = await setupCall(opts.server, opts.account, {
|
|
47310
|
+
host: monolithHost,
|
|
47311
|
+
authHost: monolithHost
|
|
47312
|
+
});
|
|
47313
|
+
try {
|
|
47314
|
+
await action(setup.client);
|
|
47315
|
+
} finally {
|
|
47316
|
+
setup.cleanup();
|
|
47570
47317
|
}
|
|
47571
|
-
return { id: only.id, name: only.name };
|
|
47572
47318
|
}
|
|
47573
|
-
function
|
|
47574
|
-
|
|
47575
|
-
|
|
47576
|
-
|
|
47577
|
-
|
|
47319
|
+
function scopeOf(opts) {
|
|
47320
|
+
const serverKey = resolveServer(opts.server);
|
|
47321
|
+
return {
|
|
47322
|
+
serverKey,
|
|
47323
|
+
account: normaliseAccount(opts.account),
|
|
47324
|
+
controlPlane: resolveConnectorsUrl(serverKey, opts.connectorsUrl)
|
|
47325
|
+
};
|
|
47578
47326
|
}
|
|
47579
|
-
function
|
|
47580
|
-
|
|
47327
|
+
function targetArgsOf(opts) {
|
|
47328
|
+
const url = (opts.connectorsUrl ?? "").trim();
|
|
47329
|
+
return ` --server ${resolveServer(opts.server)}${url === "" ? "" : ` --connectors-url ${url}`}`;
|
|
47581
47330
|
}
|
|
47582
|
-
function
|
|
47583
|
-
|
|
47331
|
+
function addCommonFlags(cmd, jsonHelp) {
|
|
47332
|
+
cmd.option("--json", jsonHelp, false);
|
|
47333
|
+
addServerFlag(cmd);
|
|
47334
|
+
addAccountFlag(cmd);
|
|
47335
|
+
addConnectorsUrlFlag(cmd);
|
|
47584
47336
|
}
|
|
47585
|
-
function
|
|
47586
|
-
|
|
47587
|
-
return `${tag.name ?? "?"} (id ${tag.id})`;
|
|
47337
|
+
function addConfirmTokenFlag(cmd) {
|
|
47338
|
+
return cmd.option("--confirm-token <token>", "Token from a prior plan run \u2014 required to proceed");
|
|
47588
47339
|
}
|
|
47589
|
-
function
|
|
47590
|
-
return
|
|
47591
|
-
|
|
47592
|
-
|
|
47593
|
-
|
|
47594
|
-
out.write(` ${`${label}:`.padEnd(15)}${value}
|
|
47595
|
-
`);
|
|
47596
|
-
};
|
|
47597
|
-
const status2 = `${p.enabled ? "enabled" : "disabled"}${p.paused_reason === null ? "" : `, paused: ${oneLine4(p.paused_reason)}`}`;
|
|
47598
|
-
const counts = Object.entries(p.counts).sort(([a], [b]) => a.localeCompare(b)).map(([state, n]) => `${state}=${n}`).join(" ");
|
|
47599
|
-
out.write(`${headline} for ${bindingTarget(binding)}
|
|
47600
|
-
`);
|
|
47601
|
-
row("Status", status2);
|
|
47602
|
-
row("Interval", formatSeconds(p.interval_seconds));
|
|
47603
|
-
row("Date tag", formatDateTag(p.date_tag));
|
|
47604
|
-
row("Since", p.since ?? "-");
|
|
47605
|
-
row("Lookback", `${p.lookback_days}d`);
|
|
47606
|
-
row("Settle", formatSeconds(p.settle_seconds));
|
|
47607
|
-
row("Same name", sameNameLabel(p.change_policy));
|
|
47608
|
-
row("Max attempts", String(p.max_attempts));
|
|
47609
|
-
row("Last poll", `${p.last_poll_at ?? "-"} ${oneLine4(p.last_poll_status ?? "")}`.trimEnd());
|
|
47610
|
-
row("Next poll", p.next_poll_at);
|
|
47611
|
-
row("Failures", String(p.consecutive_failures));
|
|
47612
|
-
row("Discover", p.discover_in_flight ? `in flight since ${p.discover_sent_at ?? "?"}` : "-");
|
|
47613
|
-
row("Last digest", p.last_digest ?? "-");
|
|
47614
|
-
row("Pending seed", p.poll_run_seed_until ?? "-");
|
|
47615
|
-
row(
|
|
47616
|
-
"Pending rekey",
|
|
47617
|
-
p.poll_run_rekey_to === null ? "-" : `to ${sameNameLabel(p.poll_run_rekey_to)}`
|
|
47618
|
-
);
|
|
47619
|
-
row("Items", counts === "" ? "(none)" : counts);
|
|
47620
|
-
}
|
|
47621
|
-
function writeTable26(out, rows) {
|
|
47622
|
-
const widths = [];
|
|
47623
|
-
for (const r of rows) {
|
|
47624
|
-
r.forEach((cell, i) => {
|
|
47625
|
-
widths[i] = Math.max(widths[i] ?? 0, cell.length);
|
|
47626
|
-
});
|
|
47627
|
-
}
|
|
47628
|
-
for (const r of rows) {
|
|
47629
|
-
const cells = r.map(
|
|
47630
|
-
(cell, i) => i === r.length - 1 ? cell : cell.padEnd((widths[i] ?? 0) + 2)
|
|
47631
|
-
);
|
|
47632
|
-
out.write(`${cells.join("").trimEnd()}
|
|
47633
|
-
`);
|
|
47634
|
-
}
|
|
47635
|
-
}
|
|
47636
|
-
function renderItemTable(out, items) {
|
|
47637
|
-
const rows = [["ID", "LABEL", "DATE", "STATE", "ATTEMPTS", "VERSION", "ERROR"]];
|
|
47638
|
-
for (const it of items) {
|
|
47639
|
-
const error = it.last_error ?? (it.skip_reason === null ? null : `skipped: ${it.skip_reason}`);
|
|
47640
|
-
rows.push([
|
|
47641
|
-
String(it.id),
|
|
47642
|
-
oneLine4(it.label),
|
|
47643
|
-
it.item_date ?? "-",
|
|
47644
|
-
it.state,
|
|
47645
|
-
String(it.attempts),
|
|
47646
|
-
it.version_id === null ? "-" : String(it.version_id),
|
|
47647
|
-
error === null ? "-" : truncate3(oneLine4(error), 80)
|
|
47648
|
-
]);
|
|
47649
|
-
}
|
|
47650
|
-
writeTable26(out, rows);
|
|
47651
|
-
}
|
|
47652
|
-
async function runPollEnable(cfg) {
|
|
47653
|
-
const binding = await resolvePollBinding(cfg.client, cfg.filebox, cfg.signal);
|
|
47654
|
-
let existing;
|
|
47655
|
-
try {
|
|
47656
|
-
existing = await getPoll(cfg.client, binding.query_id, { signal: cfg.signal });
|
|
47657
|
-
} catch (err) {
|
|
47658
|
-
if (!isNotPolled(err)) throw err;
|
|
47659
|
-
existing = null;
|
|
47660
|
-
}
|
|
47661
|
-
const s = cfg.settings;
|
|
47662
|
-
if (existing !== null && (s.since !== void 0 || s.lookback_days !== void 0 || s.change_policy !== void 0)) {
|
|
47663
|
-
throw new Error(
|
|
47664
|
-
`filebox ${cfg.filebox} is already polled; change --since, --lookback or --same-name with dr connectors poll set ${cfg.filebox}, which asks what to do with a wider window or a policy switch`
|
|
47665
|
-
);
|
|
47666
|
-
}
|
|
47667
|
-
if (binding.filebox_id <= 0) {
|
|
47668
|
-
throw new Error(
|
|
47669
|
-
`connector-query #${binding.query_id} has no filebox id recorded, so its date tag cannot be read; record it with dr connectors query repoint ${binding.query_id} --filebox-id <id>`
|
|
47670
|
-
);
|
|
47671
|
-
}
|
|
47672
|
-
const dateTag = await readFileboxDateTag(cfg.monolith, binding.filebox_id, cfg.signal);
|
|
47673
|
-
const stored = await putPoll(
|
|
47674
|
-
cfg.client,
|
|
47675
|
-
binding.query_id,
|
|
47676
|
-
{ enabled: true, date_tag: dateTag, ...s },
|
|
47677
|
-
{ signal: cfg.signal }
|
|
47678
|
-
);
|
|
47679
|
-
if (cfg.asJson) {
|
|
47680
|
-
writeJSONIndented(cfg.out, stored);
|
|
47681
|
-
return;
|
|
47682
|
-
}
|
|
47683
|
-
renderPoll(cfg.out, "Polling enabled", binding, stored);
|
|
47684
|
-
}
|
|
47685
|
-
function windowStart(since, lookbackDays, now) {
|
|
47686
|
-
const start = now - lookbackDays * 864e5;
|
|
47687
|
-
return since === null ? start : Math.max(start, Date.parse(since));
|
|
47688
|
-
}
|
|
47689
|
-
function widening(current, changes, now) {
|
|
47690
|
-
if (changes.since === void 0 && changes.lookback_days === void 0) return null;
|
|
47691
|
-
const from = windowStart(current.since, current.lookback_days, now);
|
|
47692
|
-
const to = windowStart(
|
|
47693
|
-
changes.since ?? current.since,
|
|
47694
|
-
changes.lookback_days ?? current.lookback_days,
|
|
47695
|
-
now
|
|
47696
|
-
);
|
|
47697
|
-
if (to >= from) return null;
|
|
47698
|
-
return { from: new Date(from).toISOString(), to: new Date(to).toISOString() };
|
|
47699
|
-
}
|
|
47700
|
-
async function ask(cfg, message, choices, defaultChoice) {
|
|
47701
|
-
const answer = await cfg.prompt(message, choices, defaultChoice);
|
|
47702
|
-
if (answer === null) throw new Error("poll set cancelled; nothing was changed");
|
|
47703
|
-
return parseChoice(answer, "the answer", choices);
|
|
47704
|
-
}
|
|
47705
|
-
var WIDEN_HELP = "seed = mark the newly exposed items as already landed, land = land them all";
|
|
47706
|
-
var POLICY_HELP = "rekey = re-key the ledger from a fresh listing (nothing lands again), seed = mark everything in the window as landed, land = land everything in the window";
|
|
47707
|
-
function pollSetConfirmVerb(widen, onPolicyChange) {
|
|
47708
|
-
return widen === "land" || onPolicyChange === "land" ? "set-land" : "set-seed";
|
|
47709
|
-
}
|
|
47710
|
-
async function runPollSet(cfg) {
|
|
47711
|
-
if (Object.keys(cfg.changes).length === 0) {
|
|
47712
|
-
throw new Error(
|
|
47713
|
-
"poll set: pass at least one of --interval, --since, --lookback, --settle, --same-name"
|
|
47714
|
-
);
|
|
47715
|
-
}
|
|
47716
|
-
const binding = await resolvePollBinding(cfg.client, cfg.filebox, cfg.signal);
|
|
47717
|
-
const current = await explainNotPolled(
|
|
47718
|
-
cfg.filebox,
|
|
47719
|
-
getPoll(cfg.client, binding.query_id, { signal: cfg.signal })
|
|
47720
|
-
);
|
|
47721
|
-
const body = { ...cfg.changes };
|
|
47722
|
-
const detail = [`target: ${bindingTarget(binding)}`];
|
|
47723
|
-
const wider = widening(current, cfg.changes, cfg.now ?? Date.now());
|
|
47724
|
-
if (wider !== null) {
|
|
47725
|
-
const what = `the polling window widens (start ${wider.from} -> ${wider.to})`;
|
|
47726
|
-
if (cfg.widen === void 0 && !cfg.interactive) {
|
|
47727
|
-
throw new Error(`${what}; pass --widen seed or --widen land (${WIDEN_HELP})`);
|
|
47728
|
-
}
|
|
47729
|
-
body.widen = cfg.widen ?? await ask(cfg, `${what}. ${WIDEN_HELP}.`, WIDEN_CHOICES);
|
|
47730
|
-
detail.push(`window: start ${wider.from} -> ${wider.to}, widen ${body.widen}`);
|
|
47731
|
-
} else if (cfg.widen !== void 0) {
|
|
47732
|
-
body.widen = cfg.widen;
|
|
47733
|
-
}
|
|
47734
|
-
const newPolicy = cfg.changes.change_policy;
|
|
47735
|
-
const policySwitch = newPolicy !== void 0 && newPolicy !== current.change_policy;
|
|
47736
|
-
if (policySwitch) {
|
|
47737
|
-
const change = `${sameNameLabel(current.change_policy)} -> ${sameNameLabel(newPolicy)}`;
|
|
47738
|
-
const what = `--same-name changes ${change}`;
|
|
47739
|
-
if (cfg.onPolicyChange === void 0 && !cfg.interactive) {
|
|
47740
|
-
throw new Error(`${what}; pass --on-policy-change rekey, seed or land (${POLICY_HELP})`);
|
|
47741
|
-
}
|
|
47742
|
-
body.on_policy_change = cfg.onPolicyChange ?? await ask(cfg, `${what}. ${POLICY_HELP}.`, POLICY_CHOICES, "rekey");
|
|
47743
|
-
detail.push(`same name: ${change}, on policy change ${body.on_policy_change}`);
|
|
47744
|
-
} else if (cfg.onPolicyChange !== void 0) {
|
|
47745
|
-
body.on_policy_change = cfg.onPolicyChange;
|
|
47746
|
-
}
|
|
47747
|
-
const put = async () => {
|
|
47748
|
-
const stored = await putPoll(cfg.client, binding.query_id, body, {
|
|
47749
|
-
signal: cfg.signal
|
|
47750
|
-
});
|
|
47751
|
-
if (cfg.asJson) {
|
|
47752
|
-
writeJSONIndented(cfg.out, stored);
|
|
47753
|
-
return;
|
|
47754
|
-
}
|
|
47755
|
-
renderPoll(cfg.out, "Polling settings updated", binding, stored);
|
|
47756
|
-
};
|
|
47757
|
-
const landsOrHides = wider !== null && body.widen === "land" || policySwitch && (body.on_policy_change === "seed" || body.on_policy_change === "land");
|
|
47758
|
-
if (!landsOrHides) {
|
|
47759
|
-
await put();
|
|
47760
|
-
return;
|
|
47761
|
-
}
|
|
47762
|
-
const flags = `${body.widen === void 0 ? "" : ` --widen ${body.widen}`}${body.on_policy_change === void 0 ? "" : ` --on-policy-change ${body.on_policy_change}`}`;
|
|
47763
|
-
cfg.deadline?.pause();
|
|
47764
|
-
await twoStepConfirm({
|
|
47765
|
-
out: cfg.out,
|
|
47766
|
-
action: pollConfirmAction(
|
|
47767
|
-
cfg.filebox,
|
|
47768
|
-
pollSetConfirmVerb(body.widen, body.on_policy_change),
|
|
47769
|
-
cfg.scope,
|
|
47770
|
-
{ ...body }
|
|
47771
|
-
),
|
|
47772
|
-
token: cfg.token,
|
|
47773
|
-
server: cfg.scope.serverKey,
|
|
47774
|
-
confirmCmd: `dr connectors poll set ${cfg.filebox}${cfg.settingsArgs}${flags}${cfg.targetArgs}`,
|
|
47775
|
-
describe: async () => detail,
|
|
47776
|
-
execute: async () => {
|
|
47777
|
-
cfg.deadline?.resume();
|
|
47778
|
-
await put();
|
|
47779
|
-
}
|
|
47780
|
-
});
|
|
47781
|
-
}
|
|
47782
|
-
function confirmDigest(scope, payload) {
|
|
47783
|
-
const canonical = JSON.stringify({
|
|
47784
|
-
server: scope.serverKey,
|
|
47785
|
-
account: scope.account ?? "",
|
|
47786
|
-
controlPlane: scope.controlPlane,
|
|
47787
|
-
payload: Object.fromEntries(Object.entries(payload).sort(([a], [b]) => a.localeCompare(b)))
|
|
47788
|
-
});
|
|
47789
|
-
return createHash12("sha256").update(canonical).digest("hex").slice(0, 12);
|
|
47790
|
-
}
|
|
47791
|
-
function pollConfirmAction(filebox, verb, scope, payload = {}) {
|
|
47792
|
-
return {
|
|
47793
|
-
resource: "connector-poll",
|
|
47794
|
-
id: `${filebox.trim()}#${confirmDigest(scope, payload)}`,
|
|
47795
|
-
verb
|
|
47796
|
-
};
|
|
47797
|
-
}
|
|
47798
|
-
function pollItemConfirmAction(filebox, item, scope) {
|
|
47799
|
-
return {
|
|
47800
|
-
resource: "connector-poll-item",
|
|
47801
|
-
id: `${filebox.trim()}/${item}#${confirmDigest(scope, {})}`,
|
|
47802
|
-
verb: "skip"
|
|
47803
|
-
};
|
|
47804
|
-
}
|
|
47805
|
-
async function runPollDisable(cfg) {
|
|
47806
|
-
const binding = await resolvePollBinding(cfg.client, cfg.filebox, cfg.signal);
|
|
47807
|
-
await explainNotPolled(
|
|
47808
|
-
cfg.filebox,
|
|
47809
|
-
getPoll(cfg.client, binding.query_id, { signal: cfg.signal })
|
|
47810
|
-
);
|
|
47811
|
-
const stored = await putPoll(
|
|
47812
|
-
cfg.client,
|
|
47813
|
-
binding.query_id,
|
|
47814
|
-
{ enabled: false },
|
|
47815
|
-
{ signal: cfg.signal }
|
|
47816
|
-
);
|
|
47817
|
-
if (cfg.asJson) {
|
|
47818
|
-
writeJSONIndented(cfg.out, stored);
|
|
47819
|
-
return;
|
|
47820
|
-
}
|
|
47821
|
-
renderPoll(cfg.out, "Polling disabled", binding, stored);
|
|
47822
|
-
}
|
|
47823
|
-
async function runPollGet(cfg) {
|
|
47824
|
-
const binding = await resolvePollBinding(cfg.client, cfg.filebox, cfg.signal);
|
|
47825
|
-
const p = await explainNotPolled(
|
|
47826
|
-
cfg.filebox,
|
|
47827
|
-
getPoll(cfg.client, binding.query_id, { signal: cfg.signal })
|
|
47828
|
-
);
|
|
47829
|
-
if (cfg.asJson) {
|
|
47830
|
-
writeJSONIndented(cfg.out, p);
|
|
47831
|
-
return;
|
|
47832
|
-
}
|
|
47833
|
-
renderPoll(cfg.out, "Polling", binding, p);
|
|
47834
|
-
}
|
|
47835
|
-
async function runPollRun(cfg) {
|
|
47836
|
-
const binding = await resolvePollBinding(cfg.client, cfg.filebox, cfg.signal);
|
|
47837
|
-
let p;
|
|
47838
|
-
try {
|
|
47839
|
-
p = await explainNotPolled(
|
|
47840
|
-
cfg.filebox,
|
|
47841
|
-
runPoll(cfg.client, binding.query_id, { signal: cfg.signal })
|
|
47842
|
-
);
|
|
47843
|
-
} catch (err) {
|
|
47844
|
-
if (isPollThrottled(err)) {
|
|
47845
|
-
throw new Error(
|
|
47846
|
-
`a poll of filebox ${cfg.filebox} is running or ran less than a minute ago; try again shortly`,
|
|
47847
|
-
{ cause: err }
|
|
47848
|
-
);
|
|
47849
|
-
}
|
|
47850
|
-
throw err;
|
|
47851
|
-
}
|
|
47852
|
-
if (cfg.asJson) {
|
|
47853
|
-
writeJSONIndented(cfg.out, p);
|
|
47854
|
-
return;
|
|
47855
|
-
}
|
|
47856
|
-
renderPoll(cfg.out, "Poll requested", binding, p);
|
|
47857
|
-
}
|
|
47858
|
-
async function runPollResume(cfg) {
|
|
47859
|
-
const binding = await resolvePollBinding(cfg.client, cfg.filebox, cfg.signal);
|
|
47860
|
-
const resume = async () => {
|
|
47861
|
-
const p = await explainNotPolled(
|
|
47862
|
-
cfg.filebox,
|
|
47863
|
-
resumePoll(
|
|
47864
|
-
cfg.client,
|
|
47865
|
-
binding.query_id,
|
|
47866
|
-
cfg.source === void 0 ? {} : { source: cfg.source },
|
|
47867
|
-
{ signal: cfg.signal }
|
|
47868
|
-
)
|
|
47869
|
-
);
|
|
47870
|
-
if (cfg.asJson) {
|
|
47871
|
-
writeJSONIndented(cfg.out, p);
|
|
47872
|
-
return;
|
|
47873
|
-
}
|
|
47874
|
-
renderPoll(cfg.out, "Polling resumed", binding, p);
|
|
47875
|
-
};
|
|
47876
|
-
if (cfg.source !== "land") {
|
|
47877
|
-
await resume();
|
|
47878
|
-
return;
|
|
47879
|
-
}
|
|
47880
|
-
cfg.deadline?.pause();
|
|
47881
|
-
await twoStepConfirm({
|
|
47882
|
-
out: cfg.out,
|
|
47883
|
-
action: pollConfirmAction(cfg.filebox, "resume-land", cfg.scope),
|
|
47884
|
-
token: cfg.token,
|
|
47885
|
-
server: cfg.scope.serverKey,
|
|
47886
|
-
confirmCmd: `dr connectors poll resume ${cfg.filebox} --source land${cfg.targetArgs}`,
|
|
47887
|
-
describe: async () => [
|
|
47888
|
-
`target: ${bindingTarget(binding)}`,
|
|
47889
|
-
"source: land every item the changed source holds now"
|
|
47890
|
-
],
|
|
47891
|
-
execute: async () => {
|
|
47892
|
-
cfg.deadline?.resume();
|
|
47893
|
-
await resume();
|
|
47894
|
-
}
|
|
47895
|
-
});
|
|
47896
|
-
}
|
|
47897
|
-
async function countSeedable(cfg, binding) {
|
|
47898
|
-
const untilMs = Date.parse(cfg.until);
|
|
47899
|
-
let count = 0;
|
|
47900
|
-
for (const state of ["seen", "new"]) {
|
|
47901
|
-
let after = null;
|
|
47902
|
-
do {
|
|
47903
|
-
cfg.deadline?.resume();
|
|
47904
|
-
const page = await explainNotPolled(
|
|
47905
|
-
cfg.filebox,
|
|
47906
|
-
listPollItems(
|
|
47907
|
-
cfg.client,
|
|
47908
|
-
binding.query_id,
|
|
47909
|
-
{ state, ...after === null ? {} : { after } },
|
|
47910
|
-
{ signal: cfg.signal }
|
|
47911
|
-
)
|
|
47912
|
-
);
|
|
47913
|
-
count += page.items.filter(
|
|
47914
|
-
(it) => it.source_modified_at !== null && Date.parse(it.source_modified_at) <= untilMs
|
|
47915
|
-
).length;
|
|
47916
|
-
after = page.next_after;
|
|
47917
|
-
} while (after !== null);
|
|
47918
|
-
}
|
|
47919
|
-
return count;
|
|
47920
|
-
}
|
|
47921
|
-
async function runPollSeed(cfg) {
|
|
47922
|
-
const binding = await resolvePollBinding(cfg.client, cfg.filebox, cfg.signal);
|
|
47923
|
-
if (cfg.dryRun) {
|
|
47924
|
-
const wouldSeed = await countSeedable(cfg, binding);
|
|
47925
|
-
if (cfg.asJson) {
|
|
47926
|
-
writeJSONIndented(cfg.out, { dry_run: true, until: cfg.until, would_seed: wouldSeed });
|
|
47927
|
-
return;
|
|
47928
|
-
}
|
|
47929
|
-
cfg.out.write(`Seed dry run for ${bindingTarget(binding)}: nothing was changed.
|
|
47930
|
-
`);
|
|
47931
|
-
cfg.out.write(
|
|
47932
|
-
`${wouldSeed} already-discovered items would be seeded (items not yet discovered are not counted).
|
|
47933
|
-
`
|
|
47934
|
-
);
|
|
47935
|
-
return;
|
|
47936
|
-
}
|
|
47937
|
-
cfg.deadline?.pause();
|
|
47938
|
-
await twoStepConfirm({
|
|
47939
|
-
out: cfg.out,
|
|
47940
|
-
action: pollConfirmAction(cfg.filebox, "seed", cfg.scope, { until: cfg.until }),
|
|
47941
|
-
token: cfg.token,
|
|
47942
|
-
server: cfg.scope.serverKey,
|
|
47943
|
-
confirmCmd: `dr connectors poll seed ${cfg.filebox} --until ${cfg.untilArg}${cfg.targetArgs}`,
|
|
47944
|
-
describe: async () => [
|
|
47945
|
-
`target: ${bindingTarget(binding)}`,
|
|
47946
|
-
`until: ${cfg.until}`,
|
|
47947
|
-
"effect: items modified at or before it are marked landed and never land"
|
|
47948
|
-
],
|
|
47949
|
-
execute: async () => {
|
|
47950
|
-
cfg.deadline?.resume();
|
|
47951
|
-
const p = await explainNotPolled(
|
|
47952
|
-
cfg.filebox,
|
|
47953
|
-
seedPoll(
|
|
47954
|
-
cfg.client,
|
|
47955
|
-
binding.query_id,
|
|
47956
|
-
{ until: cfg.until },
|
|
47957
|
-
{
|
|
47958
|
-
signal: cfg.signal
|
|
47959
|
-
}
|
|
47960
|
-
)
|
|
47961
|
-
);
|
|
47962
|
-
if (cfg.asJson) {
|
|
47963
|
-
writeJSONIndented(cfg.out, p);
|
|
47964
|
-
return;
|
|
47965
|
-
}
|
|
47966
|
-
renderPoll(cfg.out, "Seed requested", binding, p);
|
|
47967
|
-
}
|
|
47968
|
-
});
|
|
47969
|
-
}
|
|
47970
|
-
async function runPollItems(cfg) {
|
|
47971
|
-
const binding = await resolvePollBinding(cfg.client, cfg.filebox, cfg.signal);
|
|
47972
|
-
const page = await explainNotPolled(
|
|
47973
|
-
cfg.filebox,
|
|
47974
|
-
listPollItems(
|
|
47975
|
-
cfg.client,
|
|
47976
|
-
binding.query_id,
|
|
47977
|
-
{
|
|
47978
|
-
...cfg.state !== void 0 ? { state: cfg.state } : {},
|
|
47979
|
-
...cfg.limit !== void 0 ? { limit: cfg.limit } : {},
|
|
47980
|
-
...cfg.after !== void 0 ? { after: cfg.after } : {}
|
|
47981
|
-
},
|
|
47982
|
-
{ signal: cfg.signal }
|
|
47983
|
-
)
|
|
47984
|
-
);
|
|
47985
|
-
if (cfg.asJson) {
|
|
47986
|
-
writeJSONIndented(cfg.out, page);
|
|
47987
|
-
return;
|
|
47988
|
-
}
|
|
47989
|
-
if (page.items.length === 0) {
|
|
47990
|
-
cfg.out.write(`No items${cfg.state === void 0 ? "" : ` in state ${cfg.state}`}.
|
|
47991
|
-
`);
|
|
47992
|
-
return;
|
|
47993
|
-
}
|
|
47994
|
-
renderItemTable(cfg.out, page.items);
|
|
47995
|
-
if (page.next_after !== null) {
|
|
47996
|
-
const state = cfg.state === void 0 ? "" : ` --state ${cfg.state}`;
|
|
47997
|
-
const limit = cfg.limit === void 0 ? "" : ` --limit ${cfg.limit}`;
|
|
47998
|
-
cfg.out.write(
|
|
47999
|
-
`
|
|
48000
|
-
More: dr connectors poll items ${cfg.filebox}${state}${limit} --after ${page.next_after}
|
|
48001
|
-
`
|
|
48002
|
-
);
|
|
48003
|
-
}
|
|
48004
|
-
}
|
|
48005
|
-
function renderItemResult(cfg, binding, headline, item) {
|
|
48006
|
-
if (cfg.asJson) {
|
|
48007
|
-
writeJSONIndented(cfg.out, item);
|
|
48008
|
-
return;
|
|
48009
|
-
}
|
|
48010
|
-
cfg.out.write(`${headline} (${bindingTarget(binding)})
|
|
48011
|
-
`);
|
|
48012
|
-
renderItemTable(cfg.out, [item]);
|
|
48013
|
-
}
|
|
48014
|
-
async function runPollRetry(cfg) {
|
|
48015
|
-
const binding = await resolvePollBinding(cfg.client, cfg.filebox, cfg.signal);
|
|
48016
|
-
const item = await explainNotPolled(
|
|
48017
|
-
cfg.filebox,
|
|
48018
|
-
retryPollItem(cfg.client, binding.query_id, cfg.itemId, { signal: cfg.signal })
|
|
48019
|
-
);
|
|
48020
|
-
renderItemResult(cfg, binding, `Item ${cfg.itemId} queued for retry`, item);
|
|
48021
|
-
}
|
|
48022
|
-
async function runPollSkip(cfg) {
|
|
48023
|
-
const binding = await resolvePollBinding(cfg.client, cfg.filebox, cfg.signal);
|
|
48024
|
-
cfg.deadline?.pause();
|
|
48025
|
-
await twoStepConfirm({
|
|
48026
|
-
out: cfg.out,
|
|
48027
|
-
action: pollItemConfirmAction(cfg.filebox, cfg.itemId, cfg.scope),
|
|
48028
|
-
token: cfg.token,
|
|
48029
|
-
server: cfg.scope.serverKey,
|
|
48030
|
-
confirmCmd: `dr connectors poll skip ${cfg.filebox} ${cfg.itemId}${cfg.targetArgs}`,
|
|
48031
|
-
describe: async () => [
|
|
48032
|
-
`item: ${cfg.itemId} of ${bindingTarget(binding)}`,
|
|
48033
|
-
"effect: the item is never landed (dr connectors poll retry queues it again)"
|
|
48034
|
-
],
|
|
48035
|
-
execute: async () => {
|
|
48036
|
-
cfg.deadline?.resume();
|
|
48037
|
-
const item = await explainNotPolled(
|
|
48038
|
-
cfg.filebox,
|
|
48039
|
-
skipPollItem(cfg.client, binding.query_id, cfg.itemId, {
|
|
48040
|
-
signal: cfg.signal
|
|
48041
|
-
})
|
|
48042
|
-
);
|
|
48043
|
-
renderItemResult(cfg, binding, `Item ${cfg.itemId} skipped`, item);
|
|
48044
|
-
}
|
|
48045
|
-
});
|
|
48046
|
-
}
|
|
48047
|
-
async function withMonolith(opts, action) {
|
|
48048
|
-
const monolithHost = resolve(resolveServer(opts.server)).host;
|
|
48049
|
-
const setup = await setupCall(opts.server, opts.account, {
|
|
48050
|
-
host: monolithHost,
|
|
48051
|
-
authHost: monolithHost
|
|
48052
|
-
});
|
|
48053
|
-
try {
|
|
48054
|
-
await action(setup.client);
|
|
48055
|
-
} finally {
|
|
48056
|
-
setup.cleanup();
|
|
48057
|
-
}
|
|
48058
|
-
}
|
|
48059
|
-
function scopeOf(opts) {
|
|
48060
|
-
const serverKey = resolveServer(opts.server);
|
|
48061
|
-
return {
|
|
48062
|
-
serverKey,
|
|
48063
|
-
account: normaliseAccount(opts.account),
|
|
48064
|
-
controlPlane: resolveConnectorsUrl(serverKey, opts.connectorsUrl)
|
|
48065
|
-
};
|
|
48066
|
-
}
|
|
48067
|
-
function targetArgsOf(opts) {
|
|
48068
|
-
const url = (opts.connectorsUrl ?? "").trim();
|
|
48069
|
-
return ` --server ${resolveServer(opts.server)}${url === "" ? "" : ` --connectors-url ${url}`}`;
|
|
48070
|
-
}
|
|
48071
|
-
function addCommonFlags(cmd, jsonHelp) {
|
|
48072
|
-
cmd.option("--json", jsonHelp, false);
|
|
48073
|
-
addServerFlag(cmd);
|
|
48074
|
-
addAccountFlag(cmd);
|
|
48075
|
-
addConnectorsUrlFlag(cmd);
|
|
48076
|
-
}
|
|
48077
|
-
function addConfirmTokenFlag(cmd) {
|
|
48078
|
-
return cmd.option("--confirm-token <token>", "Token from a prior plan run \u2014 required to proceed");
|
|
48079
|
-
}
|
|
48080
|
-
function addSettingsFlags(cmd) {
|
|
48081
|
-
return cmd.option("--interval <duration>", "how often to poll, 5m to 7d (default: per source)").option("--since <date>", "never pick up items modified before this date or timestamp").option("--lookback <days>", "window every poll lists, 1d to 3650d (default: per source)").option("--settle <duration>", "how long an item must stay unchanged before landing, 0s to 1d").option(
|
|
48082
|
-
"--same-name <policy>",
|
|
48083
|
-
"a changed file under the same name: new-version (lands again) or ignore"
|
|
48084
|
-
);
|
|
47340
|
+
function addSettingsFlags(cmd) {
|
|
47341
|
+
return cmd.option("--interval <duration>", "how often to poll, 5m to 7d (default: per source)").option("--since <date>", "never pick up items modified before this date or timestamp").option("--lookback <days>", "window every poll lists, 1d to 3650d (default: per source)").option("--settle <duration>", "how long an item must stay unchanged before landing, 0s to 1d").option(
|
|
47342
|
+
"--same-name <policy>",
|
|
47343
|
+
"a changed file under the same name: new-version (lands again) or ignore"
|
|
47344
|
+
);
|
|
48085
47345
|
}
|
|
48086
47346
|
function ttyChoicePrompt(message, choices, defaultChoice, io = { input: process.stdin, output: process.stderr }) {
|
|
48087
47347
|
return new Promise((resolvePrompt) => {
|
|
@@ -48300,13 +47560,14 @@ ${POLL_CONFIRM_HELP}`)
|
|
|
48300
47560
|
}
|
|
48301
47561
|
);
|
|
48302
47562
|
addCommonFlags(itemsCmd, "Output the raw page as JSON");
|
|
48303
|
-
const retryCmd = poll2.command("retry").description("Queue a failed, waiting or skipped item again").argument("<filebox>", "filebox id or uuid").argument("<item>", "item id (from poll items)").action(async (filebox, item, opts) => {
|
|
47563
|
+
const retryCmd = poll2.command("retry").description("Queue a failed, waiting or skipped item again").argument("<filebox>", "filebox id or uuid").argument("<item>", "item id (from poll items)").option("--output <filebox>", "a multi-output item: retry only its landing in this filebox").action(async (filebox, item, opts) => {
|
|
48304
47564
|
const itemId = parsePositiveIntArg2(item, "item id");
|
|
48305
47565
|
await withConnectorsSetup(opts, async ({ client, signal }) => {
|
|
48306
47566
|
await runPollRetry({
|
|
48307
47567
|
client,
|
|
48308
47568
|
filebox,
|
|
48309
47569
|
itemId,
|
|
47570
|
+
...opts.output !== void 0 ? { output: opts.output } : {},
|
|
48310
47571
|
asJson: opts.json === true,
|
|
48311
47572
|
signal,
|
|
48312
47573
|
out: process.stdout
|
|
@@ -48315,59 +47576,94 @@ ${POLL_CONFIRM_HELP}`)
|
|
|
48315
47576
|
});
|
|
48316
47577
|
addCommonFlags(retryCmd, "Output the item as JSON");
|
|
48317
47578
|
const skipCmd = addConfirmTokenFlag(
|
|
48318
|
-
poll2.command("skip").description("Skip an item so it is never landed (two-step confirmation)").argument("<filebox>", "filebox id or uuid").argument("<item>", "item id (from poll items)")
|
|
48319
|
-
).action(async (filebox, item, opts) => {
|
|
48320
|
-
const itemId = parsePositiveIntArg2(item, "item id");
|
|
48321
|
-
await preflightConfirmToken(
|
|
48322
|
-
pollItemConfirmAction(filebox, itemId, scopeOf(opts)),
|
|
48323
|
-
opts.confirmToken
|
|
48324
|
-
);
|
|
48325
|
-
await withConnectorsSetup(opts, async (setup) => {
|
|
48326
|
-
await runPollSkip({
|
|
48327
|
-
client: setup.client,
|
|
48328
|
-
filebox,
|
|
48329
|
-
itemId,
|
|
48330
|
-
deadline: {
|
|
48331
|
-
pause: () => setup.pauseTimeout(),
|
|
48332
|
-
resume: () => setup.extendTimeout(DEFAULT_CALL_TIMEOUT_MS)
|
|
48333
|
-
},
|
|
48334
|
-
...opts.confirmToken !== void 0 ? { token: opts.confirmToken } : {},
|
|
48335
|
-
scope: scopeOf(opts),
|
|
48336
|
-
targetArgs: targetArgsOf(opts),
|
|
48337
|
-
asJson: opts.json === true,
|
|
48338
|
-
signal: setup.signal,
|
|
48339
|
-
out: process.stdout
|
|
48340
|
-
});
|
|
48341
|
-
});
|
|
48342
|
-
});
|
|
48343
|
-
addCommonFlags(skipCmd, "Output the item as JSON");
|
|
48344
|
-
const seedCmd = addConfirmTokenFlag(
|
|
48345
|
-
poll2.command("seed").description(
|
|
48346
|
-
"Mark items modified at or before --until as already landed (cut-over; token, no dialog)"
|
|
48347
|
-
).argument("<filebox>", "filebox id or uuid").requiredOption(
|
|
48348
|
-
"--until <timestamp>",
|
|
48349
|
-
"date or timestamp with a zone, e.g. 2026-10-08T17:30:00Z"
|
|
48350
|
-
).option(
|
|
48351
|
-
"--dry-run",
|
|
48352
|
-
"count the already-discovered items it would mark, without changing anything",
|
|
48353
|
-
false
|
|
48354
|
-
).addHelpText("after", `
|
|
48355
|
-
${POLL_CONFIRM_HELP}`)
|
|
47579
|
+
poll2.command("skip").description("Skip an item so it is never landed (two-step confirmation)").argument("<filebox>", "filebox id or uuid").argument("<item>", "item id (from poll items)").option("--output <filebox>", "a multi-output item: skip only its landing in this filebox")
|
|
48356
47580
|
).action(
|
|
48357
|
-
async (filebox, opts) => {
|
|
48358
|
-
const
|
|
48359
|
-
|
|
48360
|
-
|
|
48361
|
-
|
|
48362
|
-
|
|
48363
|
-
opts.confirmToken
|
|
48364
|
-
);
|
|
48365
|
-
}
|
|
47581
|
+
async (filebox, item, opts) => {
|
|
47582
|
+
const itemId = parsePositiveIntArg2(item, "item id");
|
|
47583
|
+
await preflightConfirmToken(
|
|
47584
|
+
pollItemConfirmAction(filebox, itemId, scopeOf(opts), opts.output),
|
|
47585
|
+
opts.confirmToken
|
|
47586
|
+
);
|
|
48366
47587
|
await withConnectorsSetup(opts, async (setup) => {
|
|
48367
|
-
await
|
|
47588
|
+
await runPollSkip({
|
|
48368
47589
|
client: setup.client,
|
|
48369
47590
|
filebox,
|
|
48370
|
-
|
|
47591
|
+
itemId,
|
|
47592
|
+
...opts.output !== void 0 ? { output: opts.output } : {},
|
|
47593
|
+
deadline: {
|
|
47594
|
+
pause: () => setup.pauseTimeout(),
|
|
47595
|
+
resume: () => setup.extendTimeout(DEFAULT_CALL_TIMEOUT_MS)
|
|
47596
|
+
},
|
|
47597
|
+
...opts.confirmToken !== void 0 ? { token: opts.confirmToken } : {},
|
|
47598
|
+
scope: scopeOf(opts),
|
|
47599
|
+
targetArgs: targetArgsOf(opts),
|
|
47600
|
+
asJson: opts.json === true,
|
|
47601
|
+
signal: setup.signal,
|
|
47602
|
+
out: process.stdout
|
|
47603
|
+
});
|
|
47604
|
+
});
|
|
47605
|
+
}
|
|
47606
|
+
);
|
|
47607
|
+
addCommonFlags(skipCmd, "Output the item as JSON");
|
|
47608
|
+
const collect2 = (value, previous) => [...previous, value];
|
|
47609
|
+
const outputsCmd = poll2.command("outputs").description(
|
|
47610
|
+
"Show or change the other fileboxes each file lands in, from one read of the source"
|
|
47611
|
+
).argument("<filebox>", "the binding's filebox id or uuid").option("--add <filebox>", "land each file in this filebox too (repeatable)", collect2, []).option("--remove <filebox>", "stop landing in this filebox (repeatable)", collect2, []).option("--clear", "remove every extra output", false).addHelpText(
|
|
47612
|
+
"after",
|
|
47613
|
+
[
|
|
47614
|
+
"",
|
|
47615
|
+
"For a source that deletes a file once it is read (a bank SFTP): one binding reads each",
|
|
47616
|
+
"file once and lands it in its own filebox and in every output, each through its own",
|
|
47617
|
+
"Sync-query (e.g. a BAI2 feed). The outputs must use the same connection and code.",
|
|
47618
|
+
"Set the Sync-query `dated_by` to `fetch` so discover never opens a file."
|
|
47619
|
+
].join("\n")
|
|
47620
|
+
).action(
|
|
47621
|
+
async (filebox, opts) => {
|
|
47622
|
+
await withConnectorsSetup(opts, async ({ client, signal }) => {
|
|
47623
|
+
await withMonolith(opts, async (monolith) => {
|
|
47624
|
+
await runPollOutputs({
|
|
47625
|
+
client,
|
|
47626
|
+
monolith,
|
|
47627
|
+
filebox,
|
|
47628
|
+
add: opts.add,
|
|
47629
|
+
remove: opts.remove,
|
|
47630
|
+
clear: opts.clear === true,
|
|
47631
|
+
asJson: opts.json === true,
|
|
47632
|
+
signal,
|
|
47633
|
+
out: process.stdout
|
|
47634
|
+
});
|
|
47635
|
+
});
|
|
47636
|
+
});
|
|
47637
|
+
}
|
|
47638
|
+
);
|
|
47639
|
+
addCommonFlags(outputsCmd, "Output the outputs (or the stored binding) as JSON");
|
|
47640
|
+
const seedCmd = addConfirmTokenFlag(
|
|
47641
|
+
poll2.command("seed").description(
|
|
47642
|
+
"Mark items modified at or before --until as already landed (cut-over; token, no dialog)"
|
|
47643
|
+
).argument("<filebox>", "filebox id or uuid").requiredOption(
|
|
47644
|
+
"--until <timestamp>",
|
|
47645
|
+
"date or timestamp with a zone, e.g. 2026-10-08T17:30:00Z"
|
|
47646
|
+
).option(
|
|
47647
|
+
"--dry-run",
|
|
47648
|
+
"count the already-discovered items it would mark, without changing anything",
|
|
47649
|
+
false
|
|
47650
|
+
).addHelpText("after", `
|
|
47651
|
+
${POLL_CONFIRM_HELP}`)
|
|
47652
|
+
).action(
|
|
47653
|
+
async (filebox, opts) => {
|
|
47654
|
+
const until = parseTimestampFlag(opts.until, "--until");
|
|
47655
|
+
const dryRun = opts.dryRun === true;
|
|
47656
|
+
if (!dryRun) {
|
|
47657
|
+
await preflightConfirmToken(
|
|
47658
|
+
pollConfirmAction(filebox, "seed", scopeOf(opts), { until }),
|
|
47659
|
+
opts.confirmToken
|
|
47660
|
+
);
|
|
47661
|
+
}
|
|
47662
|
+
await withConnectorsSetup(opts, async (setup) => {
|
|
47663
|
+
await runPollSeed({
|
|
47664
|
+
client: setup.client,
|
|
47665
|
+
filebox,
|
|
47666
|
+
until,
|
|
48371
47667
|
untilArg: opts.until.trim(),
|
|
48372
47668
|
dryRun,
|
|
48373
47669
|
deadline: {
|
|
@@ -48387,6 +47683,996 @@ ${POLL_CONFIRM_HELP}`)
|
|
|
48387
47683
|
addCommonFlags(seedCmd, "Output the binding (or the dry-run count) as JSON");
|
|
48388
47684
|
}
|
|
48389
47685
|
|
|
47686
|
+
// src/commands/connectorsTasks.ts
|
|
47687
|
+
import { readFileSync as readFileSync3 } from "fs";
|
|
47688
|
+
|
|
47689
|
+
// src/api/scheduledTasks.ts
|
|
47690
|
+
var BASE2 = "/scheduled-tasks";
|
|
47691
|
+
async function call2(method, path13, send, what) {
|
|
47692
|
+
let out;
|
|
47693
|
+
try {
|
|
47694
|
+
out = await send();
|
|
47695
|
+
} catch (err) {
|
|
47696
|
+
throw toControlPlaneError(method, path13, err);
|
|
47697
|
+
}
|
|
47698
|
+
if (out === null) throw new Error(`${method} ${path13}: empty response from control-plane ${what}`);
|
|
47699
|
+
return out;
|
|
47700
|
+
}
|
|
47701
|
+
async function createTask(client, fields, reqOptions) {
|
|
47702
|
+
return call2(
|
|
47703
|
+
"POST",
|
|
47704
|
+
BASE2,
|
|
47705
|
+
() => client.postJSON(BASE2, fields, reqOptions),
|
|
47706
|
+
"task create"
|
|
47707
|
+
);
|
|
47708
|
+
}
|
|
47709
|
+
async function listTasks(client, orgId, reqOptions) {
|
|
47710
|
+
const path13 = orgId === void 0 ? BASE2 : `${BASE2}?org_id=${orgId}`;
|
|
47711
|
+
return call2("GET", path13, () => client.getJSON(path13, reqOptions), "task list");
|
|
47712
|
+
}
|
|
47713
|
+
async function getTask(client, taskId, reqOptions) {
|
|
47714
|
+
const path13 = `/scheduled-tasks/${encodeURIComponent(taskId)}`;
|
|
47715
|
+
return call2("GET", path13, () => client.getJSON(path13, reqOptions), "task get");
|
|
47716
|
+
}
|
|
47717
|
+
async function updateTask(client, taskId, fields, reqOptions) {
|
|
47718
|
+
const path13 = `/scheduled-tasks/${encodeURIComponent(taskId)}`;
|
|
47719
|
+
return call2(
|
|
47720
|
+
"PATCH",
|
|
47721
|
+
path13,
|
|
47722
|
+
() => client.patchJSON(path13, fields, reqOptions),
|
|
47723
|
+
"task update"
|
|
47724
|
+
);
|
|
47725
|
+
}
|
|
47726
|
+
async function deleteTask(client, taskId, reqOptions) {
|
|
47727
|
+
const path13 = `/scheduled-tasks/${encodeURIComponent(taskId)}`;
|
|
47728
|
+
try {
|
|
47729
|
+
await client.deleteJSON(path13, reqOptions);
|
|
47730
|
+
} catch (err) {
|
|
47731
|
+
throw toControlPlaneError("DELETE", path13, err);
|
|
47732
|
+
}
|
|
47733
|
+
}
|
|
47734
|
+
async function runTask(client, taskId, reqOptions) {
|
|
47735
|
+
const path13 = `/scheduled-tasks/${encodeURIComponent(taskId)}/run`;
|
|
47736
|
+
return call2(
|
|
47737
|
+
"POST",
|
|
47738
|
+
path13,
|
|
47739
|
+
() => client.postJSON(path13, {}, reqOptions),
|
|
47740
|
+
"task run"
|
|
47741
|
+
);
|
|
47742
|
+
}
|
|
47743
|
+
async function pauseTask(client, taskId, reqOptions) {
|
|
47744
|
+
const path13 = `/scheduled-tasks/${encodeURIComponent(taskId)}/pause`;
|
|
47745
|
+
return call2(
|
|
47746
|
+
"POST",
|
|
47747
|
+
path13,
|
|
47748
|
+
() => client.postJSON(path13, {}, reqOptions),
|
|
47749
|
+
"task pause"
|
|
47750
|
+
);
|
|
47751
|
+
}
|
|
47752
|
+
async function resumeTask(client, taskId, reqOptions) {
|
|
47753
|
+
const path13 = `/scheduled-tasks/${encodeURIComponent(taskId)}/resume`;
|
|
47754
|
+
return call2(
|
|
47755
|
+
"POST",
|
|
47756
|
+
path13,
|
|
47757
|
+
() => client.postJSON(path13, {}, reqOptions),
|
|
47758
|
+
"task resume"
|
|
47759
|
+
);
|
|
47760
|
+
}
|
|
47761
|
+
async function listRuns(client, taskId, limit, reqOptions) {
|
|
47762
|
+
const path13 = `/scheduled-tasks/${encodeURIComponent(taskId)}/runs?limit=${limit}`;
|
|
47763
|
+
return call2(
|
|
47764
|
+
"GET",
|
|
47765
|
+
path13,
|
|
47766
|
+
() => client.getJSON(path13, reqOptions),
|
|
47767
|
+
"task runs"
|
|
47768
|
+
);
|
|
47769
|
+
}
|
|
47770
|
+
|
|
47771
|
+
// src/commands/connectorsTasks.ts
|
|
47772
|
+
var DURATION_UNIT_SECONDS2 = { s: 1, m: 60, h: 3600, d: 86400 };
|
|
47773
|
+
var MIN_EVERY_SECONDS = 60;
|
|
47774
|
+
var MAX_EVERY_SECONDS = 2 ** 31 - 1;
|
|
47775
|
+
function parseEvery(raw2) {
|
|
47776
|
+
const m = /^(\d+)([smhd])?$/.exec(raw2.trim());
|
|
47777
|
+
const seconds = m === null ? NaN : Number(m[1]) * (DURATION_UNIT_SECONDS2[m[2] ?? "s"] ?? 1);
|
|
47778
|
+
if (m === null)
|
|
47779
|
+
throw new Error(`invalid --every "${raw2}": expected a duration like 5m, 1h or 1d`);
|
|
47780
|
+
if (!Number.isSafeInteger(seconds) || seconds < MIN_EVERY_SECONDS || seconds > MAX_EVERY_SECONDS) {
|
|
47781
|
+
throw new Error(`invalid --every "${raw2}": must be at least 1m`);
|
|
47782
|
+
}
|
|
47783
|
+
return seconds;
|
|
47784
|
+
}
|
|
47785
|
+
function parseCron(raw2) {
|
|
47786
|
+
const fields = raw2.trim().split(/\s+/);
|
|
47787
|
+
if (fields.length !== 5 || fields.some((f) => f === "")) {
|
|
47788
|
+
throw new Error(`invalid --cron "${raw2}": expected five fields, e.g. "15 * * * *"`);
|
|
47789
|
+
}
|
|
47790
|
+
return fields.join(" ");
|
|
47791
|
+
}
|
|
47792
|
+
var ROLE_RE = /^[a-z][a-z_]*$/;
|
|
47793
|
+
var DATASOURCE_ID_RE = /^[0-9]{1,18}$/;
|
|
47794
|
+
function parseDatasources(values) {
|
|
47795
|
+
const out = {};
|
|
47796
|
+
for (const raw2 of values) {
|
|
47797
|
+
const eq = raw2.indexOf("=");
|
|
47798
|
+
const role = eq < 0 ? "" : raw2.slice(0, eq).trim();
|
|
47799
|
+
const id = eq < 0 ? "" : raw2.slice(eq + 1).trim();
|
|
47800
|
+
if (!ROLE_RE.test(role) || !DATASOURCE_ID_RE.test(id)) {
|
|
47801
|
+
throw new Error(
|
|
47802
|
+
`invalid --datasource "${raw2}": expected <role>=<datasource id>, e.g. target=4242 (role: lowercase letters and _; id: digits)`
|
|
47803
|
+
);
|
|
47804
|
+
}
|
|
47805
|
+
if (Object.hasOwn(out, role)) throw new Error(`--datasource role "${role}" given twice`);
|
|
47806
|
+
out[role] = id;
|
|
47807
|
+
}
|
|
47808
|
+
const count = Object.keys(out).length;
|
|
47809
|
+
if (count < 1 || count > 4) throw new Error("give between 1 and 4 --datasource roles");
|
|
47810
|
+
return out;
|
|
47811
|
+
}
|
|
47812
|
+
function parseConfig(raw2, readFile10 = readUtf8) {
|
|
47813
|
+
const text2 = raw2.startsWith("@") ? readFile10(raw2.slice(1)) : raw2;
|
|
47814
|
+
let parsed2;
|
|
47815
|
+
try {
|
|
47816
|
+
parsed2 = JSON.parse(text2);
|
|
47817
|
+
} catch (err) {
|
|
47818
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
47819
|
+
throw new Error(`invalid --config JSON: ${msg}`);
|
|
47820
|
+
}
|
|
47821
|
+
if (typeof parsed2 !== "object" || parsed2 === null || Array.isArray(parsed2)) {
|
|
47822
|
+
throw new Error("invalid --config: expected a JSON object");
|
|
47823
|
+
}
|
|
47824
|
+
return parsed2;
|
|
47825
|
+
}
|
|
47826
|
+
function readUtf8(p) {
|
|
47827
|
+
return readFileSync3(p, "utf-8");
|
|
47828
|
+
}
|
|
47829
|
+
function collect(value, previous = []) {
|
|
47830
|
+
return [...previous, value];
|
|
47831
|
+
}
|
|
47832
|
+
function parseScheduleFlags(opts) {
|
|
47833
|
+
if (opts.every !== void 0 && opts.cron !== void 0) {
|
|
47834
|
+
throw new Error("give --every or --cron, not both");
|
|
47835
|
+
}
|
|
47836
|
+
const fields = {};
|
|
47837
|
+
if (opts.artifact !== void 0) fields.artifact_name = opts.artifact;
|
|
47838
|
+
if (opts.flavor !== void 0) fields.runtime_flavor = opts.flavor;
|
|
47839
|
+
if (opts.datasource !== void 0) fields.datasources = parseDatasources(opts.datasource);
|
|
47840
|
+
if (opts.primaryRole !== void 0) fields.primary_role = opts.primaryRole;
|
|
47841
|
+
if (opts.config !== void 0) fields.config = parseConfig(opts.config);
|
|
47842
|
+
if (opts.every !== void 0) fields.every_seconds = parseEvery(opts.every);
|
|
47843
|
+
if (opts.cron !== void 0) fields.cron = parseCron(opts.cron);
|
|
47844
|
+
if (opts.tz !== void 0) fields.timezone = opts.tz;
|
|
47845
|
+
if (fields.primary_role !== void 0 && fields.datasources !== void 0) {
|
|
47846
|
+
if (!Object.hasOwn(fields.datasources, fields.primary_role)) {
|
|
47847
|
+
throw new Error(
|
|
47848
|
+
`--primary-role "${fields.primary_role}" is not one of the --datasource roles`
|
|
47849
|
+
);
|
|
47850
|
+
}
|
|
47851
|
+
}
|
|
47852
|
+
return fields;
|
|
47853
|
+
}
|
|
47854
|
+
function buildCreateFields(name, orgId, opts) {
|
|
47855
|
+
const missing = [
|
|
47856
|
+
["--artifact", opts.artifact],
|
|
47857
|
+
["--flavor", opts.flavor],
|
|
47858
|
+
["--datasource", opts.datasource],
|
|
47859
|
+
["--config", opts.config]
|
|
47860
|
+
].filter(([, v]) => v === void 0).map(([flag]) => flag);
|
|
47861
|
+
if (opts.every === void 0 && opts.cron === void 0) missing.push("--every or --cron");
|
|
47862
|
+
if (missing.length > 0) throw new Error(`missing ${missing.join(", ")}`);
|
|
47863
|
+
const fields = parseScheduleFlags(opts);
|
|
47864
|
+
return {
|
|
47865
|
+
org_id: orgId,
|
|
47866
|
+
name,
|
|
47867
|
+
artifact_name: fields.artifact_name,
|
|
47868
|
+
runtime_flavor: fields.runtime_flavor,
|
|
47869
|
+
datasources: fields.datasources,
|
|
47870
|
+
config: fields.config,
|
|
47871
|
+
...fields.primary_role !== void 0 ? { primary_role: fields.primary_role } : {},
|
|
47872
|
+
...fields.every_seconds !== void 0 ? { every_seconds: fields.every_seconds } : {},
|
|
47873
|
+
...fields.cron !== void 0 ? { cron: fields.cron } : {},
|
|
47874
|
+
...fields.timezone !== void 0 ? { timezone: fields.timezone } : {},
|
|
47875
|
+
...opts.disabled === true ? { enabled: false } : {}
|
|
47876
|
+
};
|
|
47877
|
+
}
|
|
47878
|
+
var UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
47879
|
+
async function resolveTaskId(client, task, signal) {
|
|
47880
|
+
if (UUID_RE.test(task)) return task;
|
|
47881
|
+
const tasks = await listTasks(client, void 0, { signal });
|
|
47882
|
+
const match = tasks.find((t) => t.name === task);
|
|
47883
|
+
if (match === void 0) {
|
|
47884
|
+
throw new Error(
|
|
47885
|
+
`no scheduled task named "${task}" in this org (see: dr connectors tasks list)`
|
|
47886
|
+
);
|
|
47887
|
+
}
|
|
47888
|
+
return match.id;
|
|
47889
|
+
}
|
|
47890
|
+
function explainTaskError(err) {
|
|
47891
|
+
const forbidden = err instanceof ControlPlaneError && err.status === 403 || err instanceof Error && /\bHTTP 403\b/.test(err.message);
|
|
47892
|
+
if (forbidden) {
|
|
47893
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
47894
|
+
const body = err instanceof ControlPlaneError ? err.body : (/HTTP 403[^:]*(?:\([^)]*\))?:\s*([\s\S]*)$/.exec(message)?.[1] ?? "").trim();
|
|
47895
|
+
return withCause(
|
|
47896
|
+
new ControlPlaneError(
|
|
47897
|
+
403,
|
|
47898
|
+
body,
|
|
47899
|
+
`${message}
|
|
47900
|
+
Scheduled tasks need a support user or an org admin: log in as one (--account) \u2014 e.g. dr login-as the customer's support user.`
|
|
47901
|
+
),
|
|
47902
|
+
err
|
|
47903
|
+
);
|
|
47904
|
+
}
|
|
47905
|
+
if (!(err instanceof ControlPlaneError)) return err;
|
|
47906
|
+
const detail = detailOf(err.body);
|
|
47907
|
+
if (err.status === 404) {
|
|
47908
|
+
const message = `scheduled task not found (${detail || err.message})`;
|
|
47909
|
+
return withCause(new ControlPlaneError(404, err.body, message), err);
|
|
47910
|
+
}
|
|
47911
|
+
if (err.status === 409 || err.status === 422) {
|
|
47912
|
+
return withCause(new ControlPlaneError(err.status, err.body, detail || err.message), err);
|
|
47913
|
+
}
|
|
47914
|
+
return err;
|
|
47915
|
+
}
|
|
47916
|
+
function withCause(error, cause) {
|
|
47917
|
+
Object.defineProperty(error, "cause", { value: cause, enumerable: false, configurable: true });
|
|
47918
|
+
return error;
|
|
47919
|
+
}
|
|
47920
|
+
function detailOf(body) {
|
|
47921
|
+
try {
|
|
47922
|
+
const parsed2 = JSON.parse(body);
|
|
47923
|
+
const d = parsed2.detail;
|
|
47924
|
+
if (typeof d === "string") return d;
|
|
47925
|
+
if (Array.isArray(d)) {
|
|
47926
|
+
return d.map(
|
|
47927
|
+
(e) => `${Array.isArray(e.loc) ? e.loc.slice(1).join(".") : ""}: ${e.msg ?? ""}`.trim()
|
|
47928
|
+
).join("; ");
|
|
47929
|
+
}
|
|
47930
|
+
} catch {
|
|
47931
|
+
}
|
|
47932
|
+
return body;
|
|
47933
|
+
}
|
|
47934
|
+
function scheduleText(t) {
|
|
47935
|
+
if (t.schedule_kind === "cron") return `cron "${t.cron ?? ""}" ${t.timezone}`;
|
|
47936
|
+
const s = t.every_seconds ?? 0;
|
|
47937
|
+
const unit = s % 86400 === 0 ? `${s / 86400}d` : s % 3600 === 0 ? `${s / 3600}h` : s % 60 === 0 ? `${s / 60}m` : `${s}s`;
|
|
47938
|
+
return `every ${unit}`;
|
|
47939
|
+
}
|
|
47940
|
+
function stateText(t) {
|
|
47941
|
+
if (!t.enabled) return "disabled";
|
|
47942
|
+
if (t.paused_reason !== null) return `paused (${t.paused_reason})`;
|
|
47943
|
+
return t.running ? "running" : "enabled";
|
|
47944
|
+
}
|
|
47945
|
+
function writeTask(out, t) {
|
|
47946
|
+
const roles = Object.entries(t.datasources).map(([role, id]) => `${role}=${id}${role === t.primary_role ? " (primary)" : ""}`).join(", ");
|
|
47947
|
+
out.write(`Task: ${t.name} (${t.id})
|
|
47948
|
+
`);
|
|
47949
|
+
out.write(` Org: ${t.org_id}
|
|
47950
|
+
`);
|
|
47951
|
+
out.write(` State: ${stateText(t)}
|
|
47952
|
+
`);
|
|
47953
|
+
out.write(` Artifact: ${t.artifact_name} [${t.runtime_flavor}]
|
|
47954
|
+
`);
|
|
47955
|
+
out.write(` Schedule: ${scheduleText(t)}
|
|
47956
|
+
`);
|
|
47957
|
+
out.write(` Next run: ${t.next_run_at}
|
|
47958
|
+
`);
|
|
47959
|
+
out.write(` Last run: ${t.last_run_at ?? "-"} ${t.last_run_status ?? ""}
|
|
47960
|
+
`);
|
|
47961
|
+
if (t.consecutive_failures > 0) out.write(` Failures: ${t.consecutive_failures} in a row
|
|
47962
|
+
`);
|
|
47963
|
+
out.write(` Datasources: ${roles}
|
|
47964
|
+
`);
|
|
47965
|
+
out.write(` Config: ${truncate3(JSON.stringify(t.config), 200)}
|
|
47966
|
+
`);
|
|
47967
|
+
}
|
|
47968
|
+
function writeTaskList(out, tasks) {
|
|
47969
|
+
if (tasks.length === 0) {
|
|
47970
|
+
out.write("No scheduled tasks.\n");
|
|
47971
|
+
return;
|
|
47972
|
+
}
|
|
47973
|
+
for (const t of tasks) {
|
|
47974
|
+
out.write(
|
|
47975
|
+
`${t.name.padEnd(28)} ${stateText(t).padEnd(20)} ${scheduleText(t).padEnd(26)} next ${t.next_run_at} last ${t.last_run_status ?? "-"} ${t.id}
|
|
47976
|
+
`
|
|
47977
|
+
);
|
|
47978
|
+
}
|
|
47979
|
+
}
|
|
47980
|
+
function writeRuns(out, runs) {
|
|
47981
|
+
if (runs.length === 0) {
|
|
47982
|
+
out.write("No runs yet.\n");
|
|
47983
|
+
return;
|
|
47984
|
+
}
|
|
47985
|
+
for (const r of runs) {
|
|
47986
|
+
const tail = r.error !== null ? ` error: ${truncate3(r.error, 160)}` : r.summary !== null ? ` ${truncate3(JSON.stringify(r.summary), 160)}` : "";
|
|
47987
|
+
out.write(
|
|
47988
|
+
`${r.started_at} ${r.status.padEnd(15)} ${r.trigger.padEnd(8)} attempt ${r.attempt} ${r.artifact_version ?? "-"}${tail}
|
|
47989
|
+
`
|
|
47990
|
+
);
|
|
47991
|
+
}
|
|
47992
|
+
}
|
|
47993
|
+
function withFlags(cmd) {
|
|
47994
|
+
cmd.option("--json", "Output raw JSON instead of a summary", false);
|
|
47995
|
+
addServerFlag(cmd);
|
|
47996
|
+
addAccountFlag(cmd);
|
|
47997
|
+
addConnectorsUrlFlag(cmd);
|
|
47998
|
+
return cmd;
|
|
47999
|
+
}
|
|
48000
|
+
function addScheduleFlags(cmd) {
|
|
48001
|
+
return cmd.option("--artifact <name>", "published artifact name, e.g. sftp_export__daily_sales").option("--flavor <flavor>", "runner flavor the artifact was published for, e.g. sftp").option("--datasource <role=id>", "monolith datasource for a role (repeat per role)", collect).option(
|
|
48002
|
+
"--primary-role <role>",
|
|
48003
|
+
"the role passed as the primary datasource (default: the first)"
|
|
48004
|
+
).option("--config <json|@file>", "task config: a JSON object, or @path to a JSON file").option("--every <duration>", "run on an interval, e.g. 5m, 1h (at least 1m)").option("--cron <expr>", 'run on a five-field cron line, e.g. "15 * * * *"').option("--tz <iana>", "timezone the cron line is read in (default UTC)");
|
|
48005
|
+
}
|
|
48006
|
+
async function onTask(task, opts, act, print) {
|
|
48007
|
+
await withConnectorsSetup(opts, async ({ client, signal }) => {
|
|
48008
|
+
try {
|
|
48009
|
+
const taskId = await resolveTaskId(client, task, signal);
|
|
48010
|
+
const result = await act(client, taskId, signal);
|
|
48011
|
+
if (opts.json === true) writeJSONIndented(process.stdout, result);
|
|
48012
|
+
else print(process.stdout, result);
|
|
48013
|
+
} catch (err) {
|
|
48014
|
+
throw explainTaskError(err);
|
|
48015
|
+
}
|
|
48016
|
+
});
|
|
48017
|
+
}
|
|
48018
|
+
function registerConnectorsTasks(connectors) {
|
|
48019
|
+
const tasks = connectors.command("tasks").description("DRIP scheduled tasks: run an artifact on a schedule (support user or org admin)");
|
|
48020
|
+
withFlags(
|
|
48021
|
+
addScheduleFlags(
|
|
48022
|
+
tasks.command("create").description("Create a scheduled task").argument("<name>", "task name, unique in the org (a-z, 0-9, _ and -)").option("--org-id <id>", "org id (defaults to the JWT org)").option("--disabled", "create it disabled", false)
|
|
48023
|
+
)
|
|
48024
|
+
).action(
|
|
48025
|
+
async (name, opts) => {
|
|
48026
|
+
const explicitOrg = opts.orgId !== void 0 ? parsePositiveIntArg2(opts.orgId, "--org-id") : void 0;
|
|
48027
|
+
const parsed2 = buildCreateFields(name, explicitOrg ?? 0, opts);
|
|
48028
|
+
await withConnectorsSetup(opts, async ({ client, signal }) => {
|
|
48029
|
+
const fields = { ...parsed2, org_id: explicitOrg ?? client.getCurrentOrgId() };
|
|
48030
|
+
try {
|
|
48031
|
+
const t = await createTask(client, fields, { signal });
|
|
48032
|
+
if (opts.json === true) writeJSONIndented(process.stdout, t);
|
|
48033
|
+
else writeTask(process.stdout, t);
|
|
48034
|
+
} catch (err) {
|
|
48035
|
+
throw explainTaskError(err);
|
|
48036
|
+
}
|
|
48037
|
+
});
|
|
48038
|
+
}
|
|
48039
|
+
);
|
|
48040
|
+
withFlags(
|
|
48041
|
+
tasks.command("list").description("List the org's scheduled tasks").option("--org-id <id>", "org id (defaults to the JWT org)")
|
|
48042
|
+
).action(async (opts) => {
|
|
48043
|
+
await withConnectorsSetup(opts, async ({ client, signal }) => {
|
|
48044
|
+
const orgId = opts.orgId !== void 0 ? parsePositiveIntArg2(opts.orgId, "--org-id") : void 0;
|
|
48045
|
+
try {
|
|
48046
|
+
const list28 = await listTasks(client, orgId, { signal });
|
|
48047
|
+
if (opts.json === true) writeJSONIndented(process.stdout, list28);
|
|
48048
|
+
else writeTaskList(process.stdout, list28);
|
|
48049
|
+
} catch (err) {
|
|
48050
|
+
throw explainTaskError(err);
|
|
48051
|
+
}
|
|
48052
|
+
});
|
|
48053
|
+
});
|
|
48054
|
+
withFlags(
|
|
48055
|
+
tasks.command("get").description("Show a scheduled task").argument("<task>", "task id or name")
|
|
48056
|
+
).action(
|
|
48057
|
+
async (task, opts) => onTask(
|
|
48058
|
+
task,
|
|
48059
|
+
opts,
|
|
48060
|
+
(c, id, signal) => getTask(c, id, { signal }),
|
|
48061
|
+
(o, r) => writeTask(o, r)
|
|
48062
|
+
)
|
|
48063
|
+
);
|
|
48064
|
+
withFlags(
|
|
48065
|
+
addScheduleFlags(
|
|
48066
|
+
tasks.command("update").description("Change a scheduled task (only the flags given change)").argument("<task>", "task id or name").option("--enable", "enable the task").option("--disable", "disable the task")
|
|
48067
|
+
)
|
|
48068
|
+
).action(
|
|
48069
|
+
async (task, opts) => {
|
|
48070
|
+
if (opts.enable === true && opts.disable === true)
|
|
48071
|
+
throw new Error("give --enable or --disable, not both");
|
|
48072
|
+
const fields = parseScheduleFlags(opts);
|
|
48073
|
+
if (opts.enable === true) fields.enabled = true;
|
|
48074
|
+
if (opts.disable === true) fields.enabled = false;
|
|
48075
|
+
if (Object.keys(fields).length === 0)
|
|
48076
|
+
throw new Error("nothing to change: give at least one flag");
|
|
48077
|
+
await onTask(
|
|
48078
|
+
task,
|
|
48079
|
+
opts,
|
|
48080
|
+
(c, id, signal) => updateTask(c, id, fields, { signal }),
|
|
48081
|
+
(o, r) => writeTask(o, r)
|
|
48082
|
+
);
|
|
48083
|
+
}
|
|
48084
|
+
);
|
|
48085
|
+
withFlags(
|
|
48086
|
+
tasks.command("delete").description("Delete a scheduled task and its run history").argument("<task>", "task id or name")
|
|
48087
|
+
).action(
|
|
48088
|
+
async (task, opts) => onTask(
|
|
48089
|
+
task,
|
|
48090
|
+
opts,
|
|
48091
|
+
async (c, id, signal) => {
|
|
48092
|
+
await deleteTask(c, id, { signal });
|
|
48093
|
+
return { deleted: id };
|
|
48094
|
+
},
|
|
48095
|
+
(o, r) => o.write(`Deleted scheduled task ${r.deleted}
|
|
48096
|
+
`)
|
|
48097
|
+
)
|
|
48098
|
+
);
|
|
48099
|
+
withFlags(
|
|
48100
|
+
tasks.command("run").description("Run a task now (does not move its schedule)").argument("<task>", "task id or name")
|
|
48101
|
+
).action(
|
|
48102
|
+
async (task, opts) => onTask(
|
|
48103
|
+
task,
|
|
48104
|
+
opts,
|
|
48105
|
+
(c, id, signal) => runTask(c, id, { signal }),
|
|
48106
|
+
(o, r) => {
|
|
48107
|
+
const run2 = r;
|
|
48108
|
+
o.write(`Queued run ${run2.id} (${run2.status}); see: dr connectors tasks runs ${task}
|
|
48109
|
+
`);
|
|
48110
|
+
}
|
|
48111
|
+
)
|
|
48112
|
+
);
|
|
48113
|
+
withFlags(
|
|
48114
|
+
tasks.command("pause").description("Pause a scheduled task").argument("<task>", "task id or name")
|
|
48115
|
+
).action(
|
|
48116
|
+
async (task, opts) => onTask(
|
|
48117
|
+
task,
|
|
48118
|
+
opts,
|
|
48119
|
+
(c, id, signal) => pauseTask(c, id, { signal }),
|
|
48120
|
+
(o, r) => writeTask(o, r)
|
|
48121
|
+
)
|
|
48122
|
+
);
|
|
48123
|
+
withFlags(
|
|
48124
|
+
tasks.command("resume").description("Resume a paused task (clears its failure count, schedules the next run)").argument("<task>", "task id or name")
|
|
48125
|
+
).action(
|
|
48126
|
+
async (task, opts) => onTask(
|
|
48127
|
+
task,
|
|
48128
|
+
opts,
|
|
48129
|
+
(c, id, signal) => resumeTask(c, id, { signal }),
|
|
48130
|
+
(o, r) => writeTask(o, r)
|
|
48131
|
+
)
|
|
48132
|
+
);
|
|
48133
|
+
withFlags(
|
|
48134
|
+
tasks.command("runs").description("A task's run history, newest first").argument("<task>", "task id or name").option("--limit <n>", "how many runs (1-200)", "20")
|
|
48135
|
+
).action(async (task, opts) => {
|
|
48136
|
+
const limit = parsePositiveIntArg2(opts.limit, "--limit");
|
|
48137
|
+
if (limit > 200) throw new Error("invalid --limit: at most 200");
|
|
48138
|
+
await onTask(
|
|
48139
|
+
task,
|
|
48140
|
+
opts,
|
|
48141
|
+
(c, id, signal) => listRuns(c, id, limit, { signal }),
|
|
48142
|
+
(o, r) => writeRuns(o, r)
|
|
48143
|
+
);
|
|
48144
|
+
});
|
|
48145
|
+
}
|
|
48146
|
+
|
|
48147
|
+
// src/commands/connectorsAws.ts
|
|
48148
|
+
var ROLE_ARN_RE = /^arn:aws[a-z-]*:iam::\d{12}:role\/\S+$/;
|
|
48149
|
+
var OTHER_ORG_HINT = "if your session is valid, the connection or --org-id may belong to another org than the --account you are logged in as";
|
|
48150
|
+
function explainAwsError(err, id) {
|
|
48151
|
+
if (!(err instanceof ControlPlaneError)) {
|
|
48152
|
+
if (err instanceof Error && /\bHTTP 403\b/.test(err.message)) {
|
|
48153
|
+
return withCause(
|
|
48154
|
+
new ControlPlaneError(
|
|
48155
|
+
403,
|
|
48156
|
+
"",
|
|
48157
|
+
`${err.message}
|
|
48158
|
+
${OTHER_ORG_HINT}`
|
|
48159
|
+
),
|
|
48160
|
+
err
|
|
48161
|
+
);
|
|
48162
|
+
}
|
|
48163
|
+
return err;
|
|
48164
|
+
}
|
|
48165
|
+
if (!hasDetail(err.body)) return err;
|
|
48166
|
+
const detail = detailOf(err.body) || err.message;
|
|
48167
|
+
const messages = {
|
|
48168
|
+
403: `forbidden: ${detail} (${OTHER_ORG_HINT})`,
|
|
48169
|
+
404: `AWS connection${id === void 0 ? "" : ` ${id}`} not found: ${detail}`,
|
|
48170
|
+
409: `conflict: ${detail}`,
|
|
48171
|
+
422: `rejected: ${detail}`,
|
|
48172
|
+
503: `the control plane cannot broker AWS roles on this site: ${detail}`
|
|
48173
|
+
};
|
|
48174
|
+
const message = messages[err.status];
|
|
48175
|
+
if (message === void 0) return err;
|
|
48176
|
+
return withCause(new ControlPlaneError(err.status, err.body, message), err);
|
|
48177
|
+
}
|
|
48178
|
+
function hasDetail(body) {
|
|
48179
|
+
try {
|
|
48180
|
+
const parsed2 = JSON.parse(body);
|
|
48181
|
+
return typeof parsed2 === "object" && parsed2 !== null && "detail" in parsed2;
|
|
48182
|
+
} catch {
|
|
48183
|
+
return false;
|
|
48184
|
+
}
|
|
48185
|
+
}
|
|
48186
|
+
function oneLine4(s) {
|
|
48187
|
+
return s.replace(/\s+/g, " ").trim();
|
|
48188
|
+
}
|
|
48189
|
+
function writeConnection(out, c) {
|
|
48190
|
+
const row = (label, value) => {
|
|
48191
|
+
out.write(` ${`${label}:`.padEnd(14)}${value}
|
|
48192
|
+
`);
|
|
48193
|
+
};
|
|
48194
|
+
out.write(`AWS connection #${c.id} (org ${c.org_id})
|
|
48195
|
+
`);
|
|
48196
|
+
row("Datasource", c.secret_ref);
|
|
48197
|
+
row("Role ARN", c.role_arn);
|
|
48198
|
+
row("Status", `${c.status}${c.enabled ? "" : " (disabled)"}`);
|
|
48199
|
+
row("Broker ARN", c.broker_arn ?? "-");
|
|
48200
|
+
row("External ID", c.external_id);
|
|
48201
|
+
row("Expires", c.expires_at ?? "-");
|
|
48202
|
+
row("Last assumed", c.last_assumed_at ?? "-");
|
|
48203
|
+
row("Last error", c.last_error === null ? "-" : oneLine4(c.last_error));
|
|
48204
|
+
row("Failures", String(c.consecutive_failures));
|
|
48205
|
+
if (c.trust_statement === null) {
|
|
48206
|
+
out.write("\nTrust statement: none returned by the control plane.\n");
|
|
48207
|
+
} else {
|
|
48208
|
+
out.write("\nTrust statement (add it to the customer role's trust policy):\n");
|
|
48209
|
+
out.write(`${JSON.stringify(c.trust_statement, null, 2)}
|
|
48210
|
+
`);
|
|
48211
|
+
}
|
|
48212
|
+
out.write(
|
|
48213
|
+
"\nThe customer role's permissions should be read-only on their bucket/prefix (plus kms:Decrypt for a customer-managed KMS key).\n"
|
|
48214
|
+
);
|
|
48215
|
+
}
|
|
48216
|
+
function writeConnectionTable(out, list28) {
|
|
48217
|
+
if (list28.length === 0) {
|
|
48218
|
+
out.write("No AWS connections for this org.\n");
|
|
48219
|
+
return;
|
|
48220
|
+
}
|
|
48221
|
+
const rows = [["ID", "SECRET_REF", "ROLE_ARN", "STATUS", "EXPIRES_AT", "LAST_ERROR"]];
|
|
48222
|
+
for (const c of list28) {
|
|
48223
|
+
rows.push([
|
|
48224
|
+
String(c.id),
|
|
48225
|
+
oneLine4(c.secret_ref),
|
|
48226
|
+
oneLine4(c.role_arn),
|
|
48227
|
+
c.status,
|
|
48228
|
+
c.expires_at ?? "-",
|
|
48229
|
+
c.last_error === null ? "-" : truncate3(oneLine4(c.last_error), 60)
|
|
48230
|
+
]);
|
|
48231
|
+
}
|
|
48232
|
+
const widths = rows[0].map((_, i) => Math.max(...rows.map((r) => r[i].length)));
|
|
48233
|
+
for (const r of rows) {
|
|
48234
|
+
out.write(
|
|
48235
|
+
`${r.map((cell, i) => cell.padEnd(widths[i] + 2)).join("").trimEnd()}
|
|
48236
|
+
`
|
|
48237
|
+
);
|
|
48238
|
+
}
|
|
48239
|
+
}
|
|
48240
|
+
function writeRotated(out, c) {
|
|
48241
|
+
out.write(`External ID rotated for AWS connection #${c.id} (org ${c.org_id}).
|
|
48242
|
+
`);
|
|
48243
|
+
out.write(`New external ID: ${c.external_id}
|
|
48244
|
+
`);
|
|
48245
|
+
out.write(
|
|
48246
|
+
`AWS refuses this connection until the customer replaces the trust statement in the trust policy of ${oneLine4(c.role_arn)} with the one below; the old external ID no longer works.
|
|
48247
|
+
|
|
48248
|
+
`
|
|
48249
|
+
);
|
|
48250
|
+
if (c.trust_statement === null) {
|
|
48251
|
+
out.write("New trust statement: none returned by the control plane.\n");
|
|
48252
|
+
} else {
|
|
48253
|
+
out.write(`New trust statement:
|
|
48254
|
+
${JSON.stringify(c.trust_statement, null, 2)}
|
|
48255
|
+
`);
|
|
48256
|
+
}
|
|
48257
|
+
}
|
|
48258
|
+
function withFlags2(cmd) {
|
|
48259
|
+
cmd.option("--json", "Output raw JSON instead of a summary", false);
|
|
48260
|
+
addServerFlag(cmd);
|
|
48261
|
+
addAccountFlag(cmd);
|
|
48262
|
+
addConnectorsUrlFlag(cmd);
|
|
48263
|
+
return cmd;
|
|
48264
|
+
}
|
|
48265
|
+
function parseRoleArn(arn) {
|
|
48266
|
+
if (!ROLE_ARN_RE.test(arn)) {
|
|
48267
|
+
throw new Error(
|
|
48268
|
+
`invalid --role-arn "${arn}": expected arn:aws:iam::<12-digit account>:role/<name>`
|
|
48269
|
+
);
|
|
48270
|
+
}
|
|
48271
|
+
return arn;
|
|
48272
|
+
}
|
|
48273
|
+
function awsConfirmAction(verb, id, opts, c) {
|
|
48274
|
+
const state = { role_arn: c.role_arn, external_id: c.external_id };
|
|
48275
|
+
return {
|
|
48276
|
+
resource: "aws-connection",
|
|
48277
|
+
id: `${id}#${confirmDigest(scopeOf(opts), state)}`,
|
|
48278
|
+
verb
|
|
48279
|
+
};
|
|
48280
|
+
}
|
|
48281
|
+
async function verifyAgainstCurrent(action, token, verb, id) {
|
|
48282
|
+
try {
|
|
48283
|
+
await preflightConfirmToken(action, token);
|
|
48284
|
+
} catch (err) {
|
|
48285
|
+
if (!(err instanceof Error) || err.message !== ERR_TOKEN_MISMATCH) throw err;
|
|
48286
|
+
throw new Error(
|
|
48287
|
+
`confirm token does not match ${verb} of AWS connection ${id} as it is now: the connection changed since the plan (role or external ID); plan again. (A token planned for another verb, connection, server, account or --connectors-url is refused the same way.)`,
|
|
48288
|
+
{ cause: err }
|
|
48289
|
+
);
|
|
48290
|
+
}
|
|
48291
|
+
}
|
|
48292
|
+
async function runGated(verb, id, opts, effect, act, print) {
|
|
48293
|
+
await withConnectorsSetup(opts, async (setup) => {
|
|
48294
|
+
try {
|
|
48295
|
+
const c = await getAwsConnection(setup.client, id, { signal: setup.signal });
|
|
48296
|
+
const action = awsConfirmAction(verb, id, opts, c);
|
|
48297
|
+
await verifyAgainstCurrent(action, opts.confirmToken, verb, id);
|
|
48298
|
+
const detail = [
|
|
48299
|
+
`connection: #${c.id} (org ${c.org_id})`,
|
|
48300
|
+
`datasource: ${oneLine4(c.secret_ref)}`,
|
|
48301
|
+
`role: ${oneLine4(c.role_arn)}`,
|
|
48302
|
+
`status: ${c.status}${c.enabled ? "" : " (disabled)"}`,
|
|
48303
|
+
`external ID: ${c.external_id}`,
|
|
48304
|
+
`effect: ${effect}`
|
|
48305
|
+
];
|
|
48306
|
+
setup.pauseTimeout();
|
|
48307
|
+
await twoStepConfirm({
|
|
48308
|
+
out: process.stdout,
|
|
48309
|
+
action,
|
|
48310
|
+
token: opts.confirmToken,
|
|
48311
|
+
server: scopeOf(opts).serverKey,
|
|
48312
|
+
confirmCmd: `dr connectors aws ${verb} ${id}${targetArgsOf(opts)}`,
|
|
48313
|
+
describe: async () => detail,
|
|
48314
|
+
execute: async () => {
|
|
48315
|
+
setup.extendTimeout(DEFAULT_CALL_TIMEOUT_MS);
|
|
48316
|
+
const now = await getAwsConnection(setup.client, id, { signal: setup.signal });
|
|
48317
|
+
if (now.role_arn !== c.role_arn || now.external_id !== c.external_id) {
|
|
48318
|
+
throw new Error(
|
|
48319
|
+
`AWS connection ${id} changed while waiting for confirmation (role or external ID); nothing was sent. Plan again.`
|
|
48320
|
+
);
|
|
48321
|
+
}
|
|
48322
|
+
const result = await act(setup);
|
|
48323
|
+
if (opts.json === true) writeJSONIndented(process.stdout, result);
|
|
48324
|
+
else print(process.stdout, result);
|
|
48325
|
+
}
|
|
48326
|
+
});
|
|
48327
|
+
} catch (err) {
|
|
48328
|
+
throw explainAwsError(err, id);
|
|
48329
|
+
}
|
|
48330
|
+
});
|
|
48331
|
+
}
|
|
48332
|
+
async function run(opts, act, print, id) {
|
|
48333
|
+
await withConnectorsSetup(opts, async (setup) => {
|
|
48334
|
+
let result;
|
|
48335
|
+
try {
|
|
48336
|
+
result = await act(setup);
|
|
48337
|
+
} catch (err) {
|
|
48338
|
+
throw explainAwsError(err, id);
|
|
48339
|
+
}
|
|
48340
|
+
if (opts.json === true) writeJSONIndented(process.stdout, result);
|
|
48341
|
+
else print(process.stdout, result);
|
|
48342
|
+
});
|
|
48343
|
+
}
|
|
48344
|
+
function registerConnectorsAws(connectors) {
|
|
48345
|
+
const aws = connectors.command("aws").description("Customer IAM roles DRIP assumes to read the customer's S3 (external ID, trust)");
|
|
48346
|
+
withFlags2(
|
|
48347
|
+
aws.command("create").description("Register a customer IAM role for a datasource; prints the trust statement").requiredOption("--secret-ref <datasourceId>", "monolith datasource id the role serves").requiredOption("--role-arn <arn>", "customer role, e.g. arn:aws:iam::123456789012:role/x").option("--org-id <id>", "org id (defaults to the JWT org)")
|
|
48348
|
+
).action(async (opts) => {
|
|
48349
|
+
const secretRef = opts.secretRef.trim();
|
|
48350
|
+
if (secretRef === "" || secretRef.length > 200) {
|
|
48351
|
+
throw new Error(`invalid --secret-ref "${opts.secretRef}": expected 1-200 characters`);
|
|
48352
|
+
}
|
|
48353
|
+
parseRoleArn(opts.roleArn);
|
|
48354
|
+
const orgId = opts.orgId === void 0 ? void 0 : parsePositiveIntArg2(opts.orgId, "--org-id");
|
|
48355
|
+
const fields = {
|
|
48356
|
+
secret_ref: secretRef,
|
|
48357
|
+
role_arn: opts.roleArn,
|
|
48358
|
+
...orgId !== void 0 ? { org_id: orgId } : {}
|
|
48359
|
+
};
|
|
48360
|
+
await run(
|
|
48361
|
+
opts,
|
|
48362
|
+
({ client, signal }) => createAwsConnection(client, fields, { signal }),
|
|
48363
|
+
writeConnection
|
|
48364
|
+
);
|
|
48365
|
+
});
|
|
48366
|
+
withFlags2(
|
|
48367
|
+
aws.command("list").description("List the org's AWS connections").option("--secret-ref <datasourceId>", "only the connection of this datasource")
|
|
48368
|
+
).action(async (opts) => {
|
|
48369
|
+
await run(
|
|
48370
|
+
opts,
|
|
48371
|
+
({ client, signal }) => listAwsConnections(client, opts.secretRef, { signal }),
|
|
48372
|
+
writeConnectionTable
|
|
48373
|
+
);
|
|
48374
|
+
});
|
|
48375
|
+
withFlags2(
|
|
48376
|
+
aws.command("get").description("Show an AWS connection and its trust statement").argument("<id>", "connection id")
|
|
48377
|
+
).action(async (idArg, opts) => {
|
|
48378
|
+
const id = parsePositiveIntArg2(idArg, "connection id");
|
|
48379
|
+
await run(
|
|
48380
|
+
opts,
|
|
48381
|
+
({ client, signal }) => getAwsConnection(client, id, { signal }),
|
|
48382
|
+
writeConnection,
|
|
48383
|
+
id
|
|
48384
|
+
);
|
|
48385
|
+
});
|
|
48386
|
+
for (const [verb, enabled, help] of [
|
|
48387
|
+
["enable", true, "Enable an AWS connection; also retries now (stored credentials are dropped)"],
|
|
48388
|
+
["disable", false, "Disable an AWS connection (stored credentials are dropped)"]
|
|
48389
|
+
]) {
|
|
48390
|
+
withFlags2(aws.command(verb).description(help).argument("<id>", "connection id")).action(
|
|
48391
|
+
async (idArg, opts) => {
|
|
48392
|
+
const id = parsePositiveIntArg2(idArg, "connection id");
|
|
48393
|
+
await run(
|
|
48394
|
+
opts,
|
|
48395
|
+
({ client, signal }) => updateAwsConnection(client, id, { enabled }, { signal }),
|
|
48396
|
+
(out, c) => {
|
|
48397
|
+
writeConnection(out, c);
|
|
48398
|
+
if (enabled)
|
|
48399
|
+
out.write(
|
|
48400
|
+
"\nStored credentials were dropped; the poller assumes the role again now.\n"
|
|
48401
|
+
);
|
|
48402
|
+
},
|
|
48403
|
+
id
|
|
48404
|
+
);
|
|
48405
|
+
}
|
|
48406
|
+
);
|
|
48407
|
+
}
|
|
48408
|
+
withFlags2(
|
|
48409
|
+
aws.command("set-role").description("Point an AWS connection at another customer role (stored credentials dropped)").argument("<id>", "connection id").requiredOption("--role-arn <arn>", "customer role, e.g. arn:aws:iam::123456789012:role/x")
|
|
48410
|
+
).action(async (idArg, opts) => {
|
|
48411
|
+
const id = parsePositiveIntArg2(idArg, "connection id");
|
|
48412
|
+
const roleArn = parseRoleArn(opts.roleArn);
|
|
48413
|
+
await run(
|
|
48414
|
+
opts,
|
|
48415
|
+
({ client, signal }) => updateAwsConnection(client, id, { role_arn: roleArn }, { signal }),
|
|
48416
|
+
(out, c) => {
|
|
48417
|
+
writeConnection(out, c);
|
|
48418
|
+
out.write(
|
|
48419
|
+
`
|
|
48420
|
+
The new role must trust the broker with the same external ID (${c.external_id}): add the trust statement above to its trust policy. Stored credentials were dropped; the poller assumes the new role next.
|
|
48421
|
+
`
|
|
48422
|
+
);
|
|
48423
|
+
},
|
|
48424
|
+
id
|
|
48425
|
+
);
|
|
48426
|
+
});
|
|
48427
|
+
const confirmHelp = "\nPrints a plan and a --confirm-token; re-run the printed command, token included, to apply it.\nOn production a person must also type a word in a dialog.";
|
|
48428
|
+
withFlags2(
|
|
48429
|
+
aws.command("rotate-external-id").description("Issue a new external ID; AWS refuses the role until the customer updates it").argument("<id>", "connection id").option("--confirm-token <token>", "Token from a prior plan run \u2014 required to proceed").addHelpText("after", confirmHelp)
|
|
48430
|
+
).action(async (idArg, opts) => {
|
|
48431
|
+
const id = parsePositiveIntArg2(idArg, "connection id");
|
|
48432
|
+
await runGated(
|
|
48433
|
+
"rotate-external-id",
|
|
48434
|
+
id,
|
|
48435
|
+
opts,
|
|
48436
|
+
"a new external ID; AWS refuses the role until the customer updates its trust policy",
|
|
48437
|
+
({ client, signal }) => rotateAwsExternalId(client, id, { signal }),
|
|
48438
|
+
writeRotated
|
|
48439
|
+
);
|
|
48440
|
+
});
|
|
48441
|
+
withFlags2(
|
|
48442
|
+
aws.command("delete").description("Delete an AWS connection: the role and its stored credentials").argument("<id>", "connection id").option("--confirm-token <token>", "Token from a prior plan run \u2014 required to proceed").addHelpText("after", confirmHelp)
|
|
48443
|
+
).action(async (idArg, opts) => {
|
|
48444
|
+
const id = parsePositiveIntArg2(idArg, "connection id");
|
|
48445
|
+
await runGated(
|
|
48446
|
+
"delete",
|
|
48447
|
+
id,
|
|
48448
|
+
opts,
|
|
48449
|
+
"removes the role and its stored credentials; the connector can no longer read S3",
|
|
48450
|
+
async ({ client, signal }) => {
|
|
48451
|
+
await deleteAwsConnection(client, id, { signal });
|
|
48452
|
+
return { deleted: id };
|
|
48453
|
+
},
|
|
48454
|
+
(out, r) => out.write(`Deleted AWS connection ${r.deleted}
|
|
48455
|
+
`)
|
|
48456
|
+
);
|
|
48457
|
+
});
|
|
48458
|
+
}
|
|
48459
|
+
|
|
48460
|
+
// src/commands/connectorsDevFetch.ts
|
|
48461
|
+
import { promises as fs19 } from "fs";
|
|
48462
|
+
import * as path9 from "path";
|
|
48463
|
+
import { randomUUID as randomUUID2 } from "crypto";
|
|
48464
|
+
function buildDevEnvelope(i) {
|
|
48465
|
+
const config = {
|
|
48466
|
+
...i.publicConfig,
|
|
48467
|
+
...i.queryConfig,
|
|
48468
|
+
// Coerce absent values to null (not undefined) so JSON.stringify keeps the
|
|
48469
|
+
// key — matches build_hook_job's dq.get(...) -> None; a dropped objectType
|
|
48470
|
+
// becomes a cryptic KeyError in the runner's dispatch.
|
|
48471
|
+
objectType: i.dataQuery.objectType ?? null,
|
|
48472
|
+
type: i.dataQuery.type ?? null,
|
|
48473
|
+
query: i.dataQuery.query
|
|
48474
|
+
};
|
|
48475
|
+
if (i.period) config.period = i.period;
|
|
48476
|
+
return {
|
|
48477
|
+
correlation_id: i.correlationId,
|
|
48478
|
+
config,
|
|
48479
|
+
secret_ref: i.secretRef,
|
|
48480
|
+
cred_token: "dev-local",
|
|
48481
|
+
output_sas: "file:///out/output.csv",
|
|
48482
|
+
input_sas: null,
|
|
48483
|
+
cursor: null,
|
|
48484
|
+
contract_version: i.contractVersion,
|
|
48485
|
+
metadata: {}
|
|
48486
|
+
};
|
|
48487
|
+
}
|
|
48488
|
+
function asObject(v) {
|
|
48489
|
+
if (typeof v === "object" && v !== null && !Array.isArray(v)) {
|
|
48490
|
+
return v;
|
|
48491
|
+
}
|
|
48492
|
+
return null;
|
|
48493
|
+
}
|
|
48494
|
+
function extractDataQueryFields(raw2) {
|
|
48495
|
+
const root = asObject(decodeShape(raw2));
|
|
48496
|
+
if (root === null || Object.keys(root).length === 0) {
|
|
48497
|
+
throw new Error(
|
|
48498
|
+
"no data-query attached to this filebox (empty data_query response) \u2014 attach one in the filebox UI first"
|
|
48499
|
+
);
|
|
48500
|
+
}
|
|
48501
|
+
const cfg = asObject(root.config);
|
|
48502
|
+
const src = cfg ?? root;
|
|
48503
|
+
const query = src.query;
|
|
48504
|
+
let queryObj;
|
|
48505
|
+
if (query === void 0 || query === null) {
|
|
48506
|
+
queryObj = {};
|
|
48507
|
+
} else {
|
|
48508
|
+
const asObj = asObject(query);
|
|
48509
|
+
if (asObj === null) {
|
|
48510
|
+
throw new Error(
|
|
48511
|
+
`data-query \`query\` is not an object (got ${typeof query}) \u2014 expected the Sync-query field`
|
|
48512
|
+
);
|
|
48513
|
+
}
|
|
48514
|
+
queryObj = asObj;
|
|
48515
|
+
}
|
|
48516
|
+
return { objectType: queryObj.objectType, type: src.type, query: queryObj };
|
|
48517
|
+
}
|
|
48518
|
+
function extractCredentials(integration) {
|
|
48519
|
+
const cfg = asObject(integration.config);
|
|
48520
|
+
if (cfg === null || Object.keys(cfg).length === 0) {
|
|
48521
|
+
throw new Error(
|
|
48522
|
+
`datasource ${integration.id} has an empty/missing \`config\` \u2014 nothing to use as connector credentials`
|
|
48523
|
+
);
|
|
48524
|
+
}
|
|
48525
|
+
return { ...cfg };
|
|
48526
|
+
}
|
|
48527
|
+
var ENV_FILE = "env.json";
|
|
48528
|
+
var SECRETS_FILE = "secrets.json";
|
|
48529
|
+
var CONNECTION_FILE = "connection.py";
|
|
48530
|
+
var QUERY_FILE = "query.py";
|
|
48531
|
+
async function writeRunDir(input) {
|
|
48532
|
+
let connectionSource;
|
|
48533
|
+
try {
|
|
48534
|
+
connectionSource = await fs19.readFile(input.connectionPath, "utf-8");
|
|
48535
|
+
} catch {
|
|
48536
|
+
throw new Error(`${input.connectionPath}: connection source not found or unreadable`);
|
|
48537
|
+
}
|
|
48538
|
+
let querySource;
|
|
48539
|
+
try {
|
|
48540
|
+
querySource = await fs19.readFile(input.queryPath, "utf-8");
|
|
48541
|
+
} catch {
|
|
48542
|
+
throw new Error(`${input.queryPath}: dataquery source not found or unreadable`);
|
|
48543
|
+
}
|
|
48544
|
+
await fs19.mkdir(input.outDir, { recursive: true });
|
|
48545
|
+
await fs19.writeFile(
|
|
48546
|
+
path9.join(input.outDir, ENV_FILE),
|
|
48547
|
+
JSON.stringify(input.env, null, 2) + "\n",
|
|
48548
|
+
"utf-8"
|
|
48549
|
+
);
|
|
48550
|
+
const secretsPath = path9.join(input.outDir, SECRETS_FILE);
|
|
48551
|
+
await fs19.writeFile(
|
|
48552
|
+
secretsPath,
|
|
48553
|
+
JSON.stringify({ [input.secretRef]: input.credentials }, null, 2) + "\n",
|
|
48554
|
+
"utf-8"
|
|
48555
|
+
);
|
|
48556
|
+
if (process.platform !== "win32") {
|
|
48557
|
+
await fs19.chmod(secretsPath, 384);
|
|
48558
|
+
}
|
|
48559
|
+
await fs19.writeFile(path9.join(input.outDir, CONNECTION_FILE), connectionSource, "utf-8");
|
|
48560
|
+
await fs19.writeFile(path9.join(input.outDir, QUERY_FILE), querySource, "utf-8");
|
|
48561
|
+
}
|
|
48562
|
+
async function ensureGitignored(entry) {
|
|
48563
|
+
const gitignorePath = path9.join(process.cwd(), ".gitignore");
|
|
48564
|
+
let current = "";
|
|
48565
|
+
try {
|
|
48566
|
+
current = await fs19.readFile(gitignorePath, "utf-8");
|
|
48567
|
+
} catch {
|
|
48568
|
+
}
|
|
48569
|
+
const normalized = entry.split("\\").join("/").replace(/\/+$/, "") + "/";
|
|
48570
|
+
const lines = current.split(/\r?\n/).map((l) => l.trim());
|
|
48571
|
+
if (lines.includes(normalized) || lines.includes(entry)) return;
|
|
48572
|
+
const sep = current === "" || current.endsWith("\n") ? "" : "\n";
|
|
48573
|
+
await fs19.writeFile(gitignorePath, `${current}${sep}${normalized}
|
|
48574
|
+
`, "utf-8");
|
|
48575
|
+
}
|
|
48576
|
+
function parsePositiveInt3(raw2, label) {
|
|
48577
|
+
const n = Number(raw2);
|
|
48578
|
+
if (!Number.isInteger(n) || String(n) !== raw2.trim() || n <= 0) {
|
|
48579
|
+
throw new Error(`invalid ${label}: "${raw2}"`);
|
|
48580
|
+
}
|
|
48581
|
+
return n;
|
|
48582
|
+
}
|
|
48583
|
+
function registerDevFetch(connectors) {
|
|
48584
|
+
const cmd = connectors.command("dev-fetch").description(
|
|
48585
|
+
"Assemble a local run dir (env.json + secrets.json + connection.py + query.py) for dev-run.sh"
|
|
48586
|
+
).argument("<fileboxUuid>", "filebox UUID of the connector binding").requiredOption(
|
|
48587
|
+
"--datasource <id>",
|
|
48588
|
+
"datasource (data-integration) id whose credentials to fetch"
|
|
48589
|
+
).requiredOption(
|
|
48590
|
+
"--document-id <n>",
|
|
48591
|
+
"numeric filebox document id (no uuid->id lookup exists; required)"
|
|
48592
|
+
).requiredOption("--connection <path>", "path to the connector's connection .py source").requiredOption("--query <path>", "path to the connector's dataquery .py source").option("--from <date>", "period start (paired with --to)").option("--to <date>", "period end (paired with --from)").option("-o, --output <dir>", "output run dir (default ./dev-run/<fileboxUuid>/)").action(async (fileboxUuid, opts) => {
|
|
48593
|
+
const datasourceId = parsePositiveInt3(opts.datasource, "datasource id");
|
|
48594
|
+
const documentId = parsePositiveInt3(opts.documentId, "document-id");
|
|
48595
|
+
if (opts.from === void 0 !== (opts.to === void 0)) {
|
|
48596
|
+
throw new Error("--from and --to must be provided together");
|
|
48597
|
+
}
|
|
48598
|
+
const defaultRoot = path9.join("dev-run", fileboxUuid);
|
|
48599
|
+
const outDir = opts.output ?? defaultRoot;
|
|
48600
|
+
const gitignoreEntry = opts.output ?? "dev-run";
|
|
48601
|
+
const serverKey = resolveServer(opts.server);
|
|
48602
|
+
const host = resolveConnectorsUrl(serverKey, opts.connectorsUrl);
|
|
48603
|
+
const monolithHost = resolve(serverKey).host;
|
|
48604
|
+
const cpSetup = await setupCall(opts.server, opts.account, { host, authHost: monolithHost });
|
|
48605
|
+
try {
|
|
48606
|
+
const binding = await getBinding(cpSetup.client, fileboxUuid, {
|
|
48607
|
+
signal: cpSetup.signal
|
|
48608
|
+
});
|
|
48609
|
+
const monoSetup = await setupCall(opts.server, opts.account, {
|
|
48610
|
+
host: monolithHost,
|
|
48611
|
+
authHost: monolithHost
|
|
48612
|
+
});
|
|
48613
|
+
try {
|
|
48614
|
+
const rawDataQuery = await getDataQueryRaw(monoSetup.client, documentId, {
|
|
48615
|
+
signal: monoSetup.signal
|
|
48616
|
+
});
|
|
48617
|
+
const dataQuery = extractDataQueryFields(rawDataQuery);
|
|
48618
|
+
if (dataQuery.objectType === void 0 || dataQuery.objectType === null || dataQuery.objectType === "") {
|
|
48619
|
+
process.stderr.write(
|
|
48620
|
+
'WARNING: objectType is not set inside the Sync-query (config.query.objectType). get_query_class will receive None \u2014 fine only if the connector has a single default dataset. Add it to the data-query, e.g. config.query.objectType = "<dataset>".\n'
|
|
48621
|
+
);
|
|
48622
|
+
}
|
|
48623
|
+
const integration = await itGet(monoSetup.client, datasourceId, {
|
|
48624
|
+
signal: monoSetup.signal
|
|
48625
|
+
});
|
|
48626
|
+
const credentials = extractCredentials(integration);
|
|
48627
|
+
let period;
|
|
48628
|
+
if (opts.from !== void 0 && opts.to !== void 0) {
|
|
48629
|
+
period = { from: opts.from, to: opts.to };
|
|
48630
|
+
}
|
|
48631
|
+
const env = buildDevEnvelope({
|
|
48632
|
+
correlationId: randomUUID2(),
|
|
48633
|
+
secretRef: binding.secret_ref,
|
|
48634
|
+
publicConfig: binding.public_config,
|
|
48635
|
+
queryConfig: binding.query_config,
|
|
48636
|
+
dataQuery,
|
|
48637
|
+
...period !== void 0 ? { period } : {},
|
|
48638
|
+
// Deliberate sandbox stub: the fetch/dispatch path doesn't branch on
|
|
48639
|
+
// contract_version, so a fixed "1" is sufficient for a local run.
|
|
48640
|
+
contractVersion: "1"
|
|
48641
|
+
});
|
|
48642
|
+
await writeRunDir({
|
|
48643
|
+
outDir,
|
|
48644
|
+
env,
|
|
48645
|
+
secretRef: binding.secret_ref,
|
|
48646
|
+
credentials,
|
|
48647
|
+
connectionPath: opts.connection,
|
|
48648
|
+
queryPath: opts.query
|
|
48649
|
+
});
|
|
48650
|
+
await ensureGitignored(gitignoreEntry);
|
|
48651
|
+
process.stdout.write(`Wrote run dir: ${outDir}
|
|
48652
|
+
`);
|
|
48653
|
+
process.stdout.write(
|
|
48654
|
+
` env.json (secret_ref=${binding.secret_ref}, flavor=${binding.runtime_flavor})
|
|
48655
|
+
`
|
|
48656
|
+
);
|
|
48657
|
+
process.stdout.write(" secrets.json (credentials NOT shown)\n");
|
|
48658
|
+
process.stdout.write(" connection.py, query.py\n");
|
|
48659
|
+
} finally {
|
|
48660
|
+
monoSetup.cleanup();
|
|
48661
|
+
}
|
|
48662
|
+
} catch (err) {
|
|
48663
|
+
throw augmentTlsCertError(err);
|
|
48664
|
+
} finally {
|
|
48665
|
+
cpSetup.cleanup();
|
|
48666
|
+
}
|
|
48667
|
+
});
|
|
48668
|
+
addServerFlag(cmd);
|
|
48669
|
+
addAccountFlag(cmd);
|
|
48670
|
+
cmd.option(
|
|
48671
|
+
"--connectors-url <url>",
|
|
48672
|
+
"Override the control-plane base URL (e.g. a kubectl port-forward http://localhost:8000)"
|
|
48673
|
+
);
|
|
48674
|
+
}
|
|
48675
|
+
|
|
48390
48676
|
// src/commands/connectors.ts
|
|
48391
48677
|
var CONTROL_PLANE_HOST = (subDomain) => `https://${subDomain}-connectors-control-plane.datarails.io`;
|
|
48392
48678
|
function resolveConnectorsUrl(serverKey, connectorsUrlFlag) {
|