@datarails-ai/dr-cli 1.6.153 → 1.6.154
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +1583 -1428
- package/dist/scopes-map.json +19 -1
- package/package.json +1 -1
package/dist/cli.js
CHANGED
|
@@ -8,7 +8,7 @@ import { fileURLToPath as fileURLToPath3 } from "url";
|
|
|
8
8
|
// package.json
|
|
9
9
|
var package_default = {
|
|
10
10
|
name: "@datarails-ai/dr-cli",
|
|
11
|
-
version: "1.6.
|
|
11
|
+
version: "1.6.154",
|
|
12
12
|
description: "Datarails CLI",
|
|
13
13
|
type: "module",
|
|
14
14
|
engines: {
|
|
@@ -69,7 +69,7 @@ var package_default = {
|
|
|
69
69
|
};
|
|
70
70
|
|
|
71
71
|
// src/buildInfo.ts
|
|
72
|
-
var BUILD_TIME = "2026-10-
|
|
72
|
+
var BUILD_TIME = "2026-10-22T15:15:19.668Z";
|
|
73
73
|
function formatBuildTime(iso) {
|
|
74
74
|
if (iso === "") return "";
|
|
75
75
|
const d = new Date(iso);
|
|
@@ -4317,6 +4317,13 @@ var PROD_GATE_TIER_A = /* @__PURE__ */ new Set([
|
|
|
4317
4317
|
// (`connector-poll|seed` / `|set-seed`, DR-55448).
|
|
4318
4318
|
"connector-poll|set-land",
|
|
4319
4319
|
"connector-poll|resume-land",
|
|
4320
|
+
// connectors AWS role (dr connectors aws delete / rotate-external-id, DR-56412). A delete
|
|
4321
|
+
// drops the role and its stored credentials; re-creating it mints a new external ID. A
|
|
4322
|
+
// rotation discards the old external ID for good. Either way AWS refuses the connection
|
|
4323
|
+
// until the customer edits their role's trust policy, the same outside-Datarails step as
|
|
4324
|
+
// `integration|revoke`.
|
|
4325
|
+
"aws-connection|delete",
|
|
4326
|
+
"aws-connection|rotate-external-id",
|
|
4320
4327
|
// Agent Studio (dr agents …, DR-55958). An agent's reach is not per-tenant: listing
|
|
4321
4328
|
// offers it to every consumer, global makes it exist in every environment, and
|
|
4322
4329
|
// visibility decides who sees it across this one, so each flip moves access for every
|
|
@@ -46546,1541 +46553,699 @@ async function updateAwsConnection(client, id, fields, reqOptions) {
|
|
|
46546
46553
|
"aws connection update"
|
|
46547
46554
|
);
|
|
46548
46555
|
}
|
|
46549
|
-
|
|
46550
|
-
|
|
46551
|
-
|
|
46552
|
-
|
|
46553
|
-
// src/api/scheduledTasks.ts
|
|
46554
|
-
var BASE2 = "/scheduled-tasks";
|
|
46555
|
-
async function call2(method, path13, send, what) {
|
|
46556
|
-
let out;
|
|
46557
|
-
try {
|
|
46558
|
-
out = await send();
|
|
46559
|
-
} catch (err) {
|
|
46560
|
-
throw toControlPlaneError(method, path13, err);
|
|
46561
|
-
}
|
|
46562
|
-
if (out === null) throw new Error(`${method} ${path13}: empty response from control-plane ${what}`);
|
|
46563
|
-
return out;
|
|
46564
|
-
}
|
|
46565
|
-
async function createTask(client, fields, reqOptions) {
|
|
46566
|
-
return call2(
|
|
46556
|
+
async function rotateAwsExternalId(client, id, reqOptions) {
|
|
46557
|
+
const path13 = `/aws-connections/${id}/rotate-external-id`;
|
|
46558
|
+
return call(
|
|
46567
46559
|
"POST",
|
|
46568
|
-
BASE2,
|
|
46569
|
-
() => client.postJSON(BASE2, fields, reqOptions),
|
|
46570
|
-
"task create"
|
|
46571
|
-
);
|
|
46572
|
-
}
|
|
46573
|
-
async function listTasks(client, orgId, reqOptions) {
|
|
46574
|
-
const path13 = orgId === void 0 ? BASE2 : `${BASE2}?org_id=${orgId}`;
|
|
46575
|
-
return call2("GET", path13, () => client.getJSON(path13, reqOptions), "task list");
|
|
46576
|
-
}
|
|
46577
|
-
async function getTask(client, taskId, reqOptions) {
|
|
46578
|
-
const path13 = `/scheduled-tasks/${encodeURIComponent(taskId)}`;
|
|
46579
|
-
return call2("GET", path13, () => client.getJSON(path13, reqOptions), "task get");
|
|
46580
|
-
}
|
|
46581
|
-
async function updateTask(client, taskId, fields, reqOptions) {
|
|
46582
|
-
const path13 = `/scheduled-tasks/${encodeURIComponent(taskId)}`;
|
|
46583
|
-
return call2(
|
|
46584
|
-
"PATCH",
|
|
46585
46560
|
path13,
|
|
46586
|
-
() => client.
|
|
46587
|
-
"
|
|
46561
|
+
() => client.postJSON(path13, {}, reqOptions),
|
|
46562
|
+
"aws connection rotate-external-id"
|
|
46588
46563
|
);
|
|
46589
46564
|
}
|
|
46590
|
-
async function
|
|
46591
|
-
const path13 = `/
|
|
46565
|
+
async function deleteAwsConnection(client, id, reqOptions) {
|
|
46566
|
+
const path13 = `/aws-connections/${id}`;
|
|
46592
46567
|
try {
|
|
46593
46568
|
await client.deleteJSON(path13, reqOptions);
|
|
46594
46569
|
} catch (err) {
|
|
46595
46570
|
throw toControlPlaneError("DELETE", path13, err);
|
|
46596
46571
|
}
|
|
46597
46572
|
}
|
|
46598
|
-
async function runTask(client, taskId, reqOptions) {
|
|
46599
|
-
const path13 = `/scheduled-tasks/${encodeURIComponent(taskId)}/run`;
|
|
46600
|
-
return call2(
|
|
46601
|
-
"POST",
|
|
46602
|
-
path13,
|
|
46603
|
-
() => client.postJSON(path13, {}, reqOptions),
|
|
46604
|
-
"task run"
|
|
46605
|
-
);
|
|
46606
|
-
}
|
|
46607
|
-
async function pauseTask(client, taskId, reqOptions) {
|
|
46608
|
-
const path13 = `/scheduled-tasks/${encodeURIComponent(taskId)}/pause`;
|
|
46609
|
-
return call2(
|
|
46610
|
-
"POST",
|
|
46611
|
-
path13,
|
|
46612
|
-
() => client.postJSON(path13, {}, reqOptions),
|
|
46613
|
-
"task pause"
|
|
46614
|
-
);
|
|
46615
|
-
}
|
|
46616
|
-
async function resumeTask(client, taskId, reqOptions) {
|
|
46617
|
-
const path13 = `/scheduled-tasks/${encodeURIComponent(taskId)}/resume`;
|
|
46618
|
-
return call2(
|
|
46619
|
-
"POST",
|
|
46620
|
-
path13,
|
|
46621
|
-
() => client.postJSON(path13, {}, reqOptions),
|
|
46622
|
-
"task resume"
|
|
46623
|
-
);
|
|
46624
|
-
}
|
|
46625
|
-
async function listRuns(client, taskId, limit, reqOptions) {
|
|
46626
|
-
const path13 = `/scheduled-tasks/${encodeURIComponent(taskId)}/runs?limit=${limit}`;
|
|
46627
|
-
return call2(
|
|
46628
|
-
"GET",
|
|
46629
|
-
path13,
|
|
46630
|
-
() => client.getJSON(path13, reqOptions),
|
|
46631
|
-
"task runs"
|
|
46632
|
-
);
|
|
46633
|
-
}
|
|
46634
46573
|
|
|
46635
|
-
// src/commands/
|
|
46574
|
+
// src/commands/connectorsPoll.ts
|
|
46575
|
+
import * as readline2 from "readline";
|
|
46576
|
+
import { createHash as createHash12 } from "crypto";
|
|
46577
|
+
|
|
46578
|
+
// src/types/connectors.ts
|
|
46579
|
+
var POLL_ITEM_STATES = [
|
|
46580
|
+
"seen",
|
|
46581
|
+
"new",
|
|
46582
|
+
"dispatched",
|
|
46583
|
+
"running",
|
|
46584
|
+
"retry_wait",
|
|
46585
|
+
"landed",
|
|
46586
|
+
"failed",
|
|
46587
|
+
"skipped",
|
|
46588
|
+
"seeded",
|
|
46589
|
+
"superseded"
|
|
46590
|
+
];
|
|
46591
|
+
|
|
46592
|
+
// src/commands/connectorsPoll.ts
|
|
46636
46593
|
var DURATION_UNIT_SECONDS = { s: 1, m: 60, h: 3600, d: 86400 };
|
|
46637
|
-
var
|
|
46638
|
-
|
|
46639
|
-
|
|
46594
|
+
var DURATION_BOUNDS = {
|
|
46595
|
+
"--interval": [300, 7 * 86400],
|
|
46596
|
+
"--settle": [0, 86400]
|
|
46597
|
+
};
|
|
46598
|
+
var LOOKBACK_MAX_DAYS = 3650;
|
|
46599
|
+
function parseDurationSeconds(raw2, flag) {
|
|
46640
46600
|
const m = /^(\d+)([smhd])?$/.exec(raw2.trim());
|
|
46641
46601
|
const seconds = m === null ? NaN : Number(m[1]) * (DURATION_UNIT_SECONDS[m[2] ?? "s"] ?? 1);
|
|
46642
|
-
|
|
46643
|
-
|
|
46644
|
-
|
|
46645
|
-
|
|
46602
|
+
const [min, max] = DURATION_BOUNDS[flag] ?? [0, Number.MAX_SAFE_INTEGER];
|
|
46603
|
+
if (m === null) {
|
|
46604
|
+
throw new Error(`invalid ${flag} "${raw2}": expected a duration like 90s, 5m or 1h`);
|
|
46605
|
+
}
|
|
46606
|
+
if (!Number.isSafeInteger(seconds) || seconds < min || seconds > max) {
|
|
46607
|
+
throw new Error(
|
|
46608
|
+
`invalid ${flag} "${raw2}": must be between ${formatSeconds(min)} and ${formatSeconds(max)}`
|
|
46609
|
+
);
|
|
46646
46610
|
}
|
|
46647
46611
|
return seconds;
|
|
46648
46612
|
}
|
|
46649
|
-
function
|
|
46650
|
-
const
|
|
46651
|
-
|
|
46652
|
-
|
|
46613
|
+
function parseLookbackDays(raw2) {
|
|
46614
|
+
const m = /^(\d+)d?$/.exec(raw2.trim());
|
|
46615
|
+
const days = m === null ? 0 : Number(m[1]);
|
|
46616
|
+
if (days < 1 || days > LOOKBACK_MAX_DAYS) {
|
|
46617
|
+
throw new Error(
|
|
46618
|
+
`invalid --lookback "${raw2}": must be between 1 and ${LOOKBACK_MAX_DAYS} days, e.g. 180d`
|
|
46619
|
+
);
|
|
46653
46620
|
}
|
|
46654
|
-
return
|
|
46621
|
+
return days;
|
|
46655
46622
|
}
|
|
46656
|
-
var
|
|
46657
|
-
var
|
|
46658
|
-
function
|
|
46659
|
-
const
|
|
46660
|
-
|
|
46661
|
-
const eq = raw2.indexOf("=");
|
|
46662
|
-
const role = eq < 0 ? "" : raw2.slice(0, eq).trim();
|
|
46663
|
-
const id = eq < 0 ? "" : raw2.slice(eq + 1).trim();
|
|
46664
|
-
if (!ROLE_RE.test(role) || !DATASOURCE_ID_RE.test(id)) {
|
|
46665
|
-
throw new Error(
|
|
46666
|
-
`invalid --datasource "${raw2}": expected <role>=<datasource id>, e.g. target=4242 (role: lowercase letters and _; id: digits)`
|
|
46667
|
-
);
|
|
46668
|
-
}
|
|
46669
|
-
if (Object.hasOwn(out, role)) throw new Error(`--datasource role "${role}" given twice`);
|
|
46670
|
-
out[role] = id;
|
|
46671
|
-
}
|
|
46672
|
-
const count = Object.keys(out).length;
|
|
46673
|
-
if (count < 1 || count > 4) throw new Error("give between 1 and 4 --datasource roles");
|
|
46674
|
-
return out;
|
|
46623
|
+
var DATE_ONLY_RE = /^(\d{4})-(\d{2})-(\d{2})$/;
|
|
46624
|
+
var DATE_TIME_RE = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2})(?::(\d{2})(?:\.\d+)?)?(?:Z|[+-](\d{2}):(\d{2}))$/;
|
|
46625
|
+
function inRange(value, max, min = 0) {
|
|
46626
|
+
const n = Number(value ?? "0");
|
|
46627
|
+
return n >= min && n <= max;
|
|
46675
46628
|
}
|
|
46676
|
-
function
|
|
46677
|
-
|
|
46678
|
-
|
|
46679
|
-
|
|
46680
|
-
parsed2 = JSON.parse(text2);
|
|
46681
|
-
} catch (err) {
|
|
46682
|
-
const msg = err instanceof Error ? err.message : String(err);
|
|
46683
|
-
throw new Error(`invalid --config JSON: ${msg}`);
|
|
46684
|
-
}
|
|
46685
|
-
if (typeof parsed2 !== "object" || parsed2 === null || Array.isArray(parsed2)) {
|
|
46686
|
-
throw new Error("invalid --config: expected a JSON object");
|
|
46687
|
-
}
|
|
46688
|
-
return parsed2;
|
|
46629
|
+
function isCalendarDay(y, m, d) {
|
|
46630
|
+
if (!inRange(m, 12, 1)) return false;
|
|
46631
|
+
const daysInMonth3 = new Date(Date.UTC(Number(y), Number(m), 0)).getUTCDate();
|
|
46632
|
+
return inRange(d, daysInMonth3, 1);
|
|
46689
46633
|
}
|
|
46690
|
-
function
|
|
46691
|
-
|
|
46634
|
+
function parseTimestampFlag(raw2, flag) {
|
|
46635
|
+
const invalid = new Error(
|
|
46636
|
+
`invalid ${flag} "${raw2}": expected YYYY-MM-DD or a timestamp with a zone (e.g. 2026-10-08T17:30:00+03:00)`
|
|
46637
|
+
);
|
|
46638
|
+
const value = raw2.trim();
|
|
46639
|
+
const dateOnly = DATE_ONLY_RE.exec(value);
|
|
46640
|
+
if (dateOnly !== null) {
|
|
46641
|
+
if (!isCalendarDay(dateOnly[1], dateOnly[2], dateOnly[3])) throw invalid;
|
|
46642
|
+
return (/* @__PURE__ */ new Date(`${value}T00:00:00Z`)).toISOString();
|
|
46643
|
+
}
|
|
46644
|
+
const dt = DATE_TIME_RE.exec(value);
|
|
46645
|
+
const valid = dt !== null && isCalendarDay(dt[1], dt[2], dt[3]) && inRange(dt[4], 23) && inRange(dt[5], 59) && inRange(dt[6], 59) && inRange(dt[7], 23) && inRange(dt[8], 59);
|
|
46646
|
+
const ms = valid ? Date.parse(value) : NaN;
|
|
46647
|
+
if (Number.isNaN(ms)) throw invalid;
|
|
46648
|
+
return new Date(ms).toISOString();
|
|
46692
46649
|
}
|
|
46693
|
-
function
|
|
46694
|
-
return
|
|
46650
|
+
function parseSameName(raw2) {
|
|
46651
|
+
if (raw2 === "new-version") return "new_version";
|
|
46652
|
+
if (raw2 === "ignore") return "ignore";
|
|
46653
|
+
throw new Error(`--same-name must be new-version or ignore (got "${raw2}")`);
|
|
46695
46654
|
}
|
|
46696
|
-
function
|
|
46697
|
-
|
|
46698
|
-
|
|
46655
|
+
function parseChoice(raw2, flag, choices) {
|
|
46656
|
+
const match = choices.find((c) => c === raw2);
|
|
46657
|
+
if (match === void 0) {
|
|
46658
|
+
throw new Error(`${flag} must be ${choices.join(" or ")} (got "${raw2}")`);
|
|
46699
46659
|
}
|
|
46660
|
+
return match;
|
|
46661
|
+
}
|
|
46662
|
+
var WIDEN_CHOICES = ["seed", "land"];
|
|
46663
|
+
var POLICY_CHOICES = ["rekey", "seed", "land"];
|
|
46664
|
+
function parseSettings(opts) {
|
|
46700
46665
|
const fields = {};
|
|
46701
|
-
if (opts.
|
|
46702
|
-
|
|
46703
|
-
if (opts.datasource !== void 0) fields.datasources = parseDatasources(opts.datasource);
|
|
46704
|
-
if (opts.primaryRole !== void 0) fields.primary_role = opts.primaryRole;
|
|
46705
|
-
if (opts.config !== void 0) fields.config = parseConfig(opts.config);
|
|
46706
|
-
if (opts.every !== void 0) fields.every_seconds = parseEvery(opts.every);
|
|
46707
|
-
if (opts.cron !== void 0) fields.cron = parseCron(opts.cron);
|
|
46708
|
-
if (opts.tz !== void 0) fields.timezone = opts.tz;
|
|
46709
|
-
if (fields.primary_role !== void 0 && fields.datasources !== void 0) {
|
|
46710
|
-
if (!Object.hasOwn(fields.datasources, fields.primary_role)) {
|
|
46711
|
-
throw new Error(
|
|
46712
|
-
`--primary-role "${fields.primary_role}" is not one of the --datasource roles`
|
|
46713
|
-
);
|
|
46714
|
-
}
|
|
46666
|
+
if (opts.interval !== void 0) {
|
|
46667
|
+
fields.interval_seconds = parseDurationSeconds(opts.interval, "--interval");
|
|
46715
46668
|
}
|
|
46669
|
+
if (opts.since !== void 0) fields.since = parseTimestampFlag(opts.since, "--since");
|
|
46670
|
+
if (opts.lookback !== void 0) fields.lookback_days = parseLookbackDays(opts.lookback);
|
|
46671
|
+
if (opts.settle !== void 0) {
|
|
46672
|
+
fields.settle_seconds = parseDurationSeconds(opts.settle, "--settle");
|
|
46673
|
+
}
|
|
46674
|
+
if (opts.sameName !== void 0) fields.change_policy = parseSameName(opts.sameName);
|
|
46716
46675
|
return fields;
|
|
46717
46676
|
}
|
|
46718
|
-
function
|
|
46719
|
-
const
|
|
46720
|
-
["--
|
|
46721
|
-
["--
|
|
46722
|
-
["--
|
|
46723
|
-
["--
|
|
46724
|
-
|
|
46725
|
-
|
|
46726
|
-
|
|
46727
|
-
const fields = parseScheduleFlags(opts);
|
|
46728
|
-
return {
|
|
46729
|
-
org_id: orgId,
|
|
46730
|
-
name,
|
|
46731
|
-
artifact_name: fields.artifact_name,
|
|
46732
|
-
runtime_flavor: fields.runtime_flavor,
|
|
46733
|
-
datasources: fields.datasources,
|
|
46734
|
-
config: fields.config,
|
|
46735
|
-
...fields.primary_role !== void 0 ? { primary_role: fields.primary_role } : {},
|
|
46736
|
-
...fields.every_seconds !== void 0 ? { every_seconds: fields.every_seconds } : {},
|
|
46737
|
-
...fields.cron !== void 0 ? { cron: fields.cron } : {},
|
|
46738
|
-
...fields.timezone !== void 0 ? { timezone: fields.timezone } : {},
|
|
46739
|
-
...opts.disabled === true ? { enabled: false } : {}
|
|
46740
|
-
};
|
|
46677
|
+
function settingsArgs(opts) {
|
|
46678
|
+
const pairs = [
|
|
46679
|
+
["--interval", opts.interval],
|
|
46680
|
+
["--since", opts.since],
|
|
46681
|
+
["--lookback", opts.lookback],
|
|
46682
|
+
["--settle", opts.settle],
|
|
46683
|
+
["--same-name", opts.sameName]
|
|
46684
|
+
];
|
|
46685
|
+
return pairs.filter((p) => p[1] !== void 0).map(([flag, value]) => ` ${flag} ${value.trim()}`).join("");
|
|
46741
46686
|
}
|
|
46742
|
-
|
|
46743
|
-
|
|
46744
|
-
|
|
46745
|
-
const
|
|
46746
|
-
const
|
|
46747
|
-
|
|
46687
|
+
async function resolvePollBinding(client, filebox, signal) {
|
|
46688
|
+
const wanted = filebox.trim();
|
|
46689
|
+
const byId = /^\d+$/.test(wanted);
|
|
46690
|
+
const rows = await listQueries(client, { signal });
|
|
46691
|
+
const matches = rows.filter(
|
|
46692
|
+
(q) => byId ? q.filebox_id === Number(wanted) : q.filebox_uuid.toLowerCase() === wanted.toLowerCase()
|
|
46693
|
+
);
|
|
46694
|
+
const [first] = matches;
|
|
46695
|
+
if (first === void 0) {
|
|
46748
46696
|
throw new Error(
|
|
46749
|
-
`no
|
|
46750
|
-
);
|
|
46751
|
-
}
|
|
46752
|
-
return match.id;
|
|
46753
|
-
}
|
|
46754
|
-
function explainTaskError(err) {
|
|
46755
|
-
const forbidden = err instanceof ControlPlaneError && err.status === 403 || err instanceof Error && /\bHTTP 403\b/.test(err.message);
|
|
46756
|
-
if (forbidden) {
|
|
46757
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
46758
|
-
const body = err instanceof ControlPlaneError ? err.body : (/HTTP 403[^:]*(?:\([^)]*\))?:\s*([\s\S]*)$/.exec(message)?.[1] ?? "").trim();
|
|
46759
|
-
return withCause(
|
|
46760
|
-
new ControlPlaneError(
|
|
46761
|
-
403,
|
|
46762
|
-
body,
|
|
46763
|
-
`${message}
|
|
46764
|
-
Scheduled tasks need a support user or an org admin: log in as one (--account) \u2014 e.g. dr login-as the customer's support user.`
|
|
46765
|
-
),
|
|
46766
|
-
err
|
|
46697
|
+
`no connector binding for filebox ${wanted} in this org (see dr connectors query list)`
|
|
46767
46698
|
);
|
|
46768
46699
|
}
|
|
46769
|
-
if (
|
|
46770
|
-
|
|
46771
|
-
|
|
46772
|
-
const message = `scheduled task not found (${detail || err.message})`;
|
|
46773
|
-
return withCause(new ControlPlaneError(404, err.body, message), err);
|
|
46774
|
-
}
|
|
46775
|
-
if (err.status === 409 || err.status === 422) {
|
|
46776
|
-
return withCause(new ControlPlaneError(err.status, err.body, detail || err.message), err);
|
|
46700
|
+
if (matches.length > 1) {
|
|
46701
|
+
const ids = matches.map((q) => `#${q.query_id}`).join(", ");
|
|
46702
|
+
throw new Error(`filebox ${wanted} has several connector bindings (${ids})`);
|
|
46777
46703
|
}
|
|
46778
|
-
return
|
|
46704
|
+
return first;
|
|
46779
46705
|
}
|
|
46780
|
-
function
|
|
46781
|
-
|
|
46782
|
-
return error;
|
|
46706
|
+
function isNotPolled(err) {
|
|
46707
|
+
return err instanceof ControlPlaneError && err.status === 404 && err.body.includes("not polled");
|
|
46783
46708
|
}
|
|
46784
|
-
function
|
|
46709
|
+
function isPollThrottled(err) {
|
|
46710
|
+
return err instanceof ControlPlaneError && err.status === 409 && (err.body.includes("discover is in flight") || err.body.includes("polled less than"));
|
|
46711
|
+
}
|
|
46712
|
+
async function explainNotPolled(filebox, call3) {
|
|
46785
46713
|
try {
|
|
46786
|
-
|
|
46787
|
-
|
|
46788
|
-
if (
|
|
46789
|
-
|
|
46790
|
-
|
|
46791
|
-
|
|
46792
|
-
)
|
|
46714
|
+
return await call3;
|
|
46715
|
+
} catch (err) {
|
|
46716
|
+
if (isNotPolled(err)) {
|
|
46717
|
+
throw new Error(
|
|
46718
|
+
`filebox ${filebox} is not set to polled ingestion; run dr connectors poll enable ${filebox}`,
|
|
46719
|
+
{ cause: err }
|
|
46720
|
+
);
|
|
46793
46721
|
}
|
|
46794
|
-
|
|
46722
|
+
throw err;
|
|
46795
46723
|
}
|
|
46796
|
-
return body;
|
|
46797
46724
|
}
|
|
46798
|
-
function
|
|
46799
|
-
|
|
46800
|
-
const
|
|
46801
|
-
const
|
|
46802
|
-
return
|
|
46725
|
+
async function readFileboxDateTag(monolith, fileboxId, signal) {
|
|
46726
|
+
const details = await getDetails(monolith, fileboxId, { signal });
|
|
46727
|
+
const dateTags = (details.tags_config ?? []).filter((t) => t.type === "DATE");
|
|
46728
|
+
const [only] = dateTags;
|
|
46729
|
+
if (only === void 0) return {};
|
|
46730
|
+
if (dateTags.length > 1) {
|
|
46731
|
+
const names = dateTags.map((t) => t.name).join(", ");
|
|
46732
|
+
throw new Error(
|
|
46733
|
+
`filebox ${fileboxId} has several DATE tags (${names}); a polled filebox needs exactly one`
|
|
46734
|
+
);
|
|
46735
|
+
}
|
|
46736
|
+
return { id: only.id, name: only.name };
|
|
46803
46737
|
}
|
|
46804
|
-
function
|
|
46805
|
-
if (
|
|
46806
|
-
if (
|
|
46807
|
-
|
|
46738
|
+
function formatSeconds(sec) {
|
|
46739
|
+
if (sec > 0 && sec % 86400 === 0) return `${sec / 86400}d`;
|
|
46740
|
+
if (sec > 0 && sec % 3600 === 0) return `${sec / 3600}h`;
|
|
46741
|
+
if (sec > 0 && sec % 60 === 0) return `${sec / 60}m`;
|
|
46742
|
+
return `${sec}s`;
|
|
46808
46743
|
}
|
|
46809
|
-
function
|
|
46810
|
-
|
|
46811
|
-
|
|
46812
|
-
|
|
46813
|
-
|
|
46814
|
-
|
|
46815
|
-
|
|
46816
|
-
|
|
46817
|
-
|
|
46818
|
-
|
|
46819
|
-
|
|
46820
|
-
`)
|
|
46821
|
-
|
|
46822
|
-
|
|
46823
|
-
|
|
46824
|
-
`)
|
|
46825
|
-
if (t.consecutive_failures > 0) out.write(` Failures: ${t.consecutive_failures} in a row
|
|
46826
|
-
`);
|
|
46827
|
-
out.write(` Datasources: ${roles}
|
|
46744
|
+
function oneLine3(s) {
|
|
46745
|
+
return s.replace(/\s+/g, " ").trim();
|
|
46746
|
+
}
|
|
46747
|
+
function sameNameLabel(policy) {
|
|
46748
|
+
return policy === "new_version" ? "new-version" : String(policy);
|
|
46749
|
+
}
|
|
46750
|
+
function formatDateTag(tag) {
|
|
46751
|
+
if (tag?.id === void 0) return "none (undated filebox)";
|
|
46752
|
+
return `${tag.name ?? "?"} (id ${tag.id})`;
|
|
46753
|
+
}
|
|
46754
|
+
function bindingTarget(binding) {
|
|
46755
|
+
return `filebox ${binding.filebox_uuid} (query #${binding.query_id})`;
|
|
46756
|
+
}
|
|
46757
|
+
function renderPoll(out, headline, binding, p) {
|
|
46758
|
+
const row = (label, value) => {
|
|
46759
|
+
out.write(` ${`${label}:`.padEnd(15)}${value}
|
|
46828
46760
|
`);
|
|
46829
|
-
|
|
46761
|
+
};
|
|
46762
|
+
const status2 = `${p.enabled ? "enabled" : "disabled"}${p.paused_reason === null ? "" : `, paused: ${oneLine3(p.paused_reason)}`}`;
|
|
46763
|
+
const counts = Object.entries(p.counts).sort(([a], [b]) => a.localeCompare(b)).map(([state, n]) => `${state}=${n}`).join(" ");
|
|
46764
|
+
out.write(`${headline} for ${bindingTarget(binding)}
|
|
46830
46765
|
`);
|
|
46766
|
+
row("Status", status2);
|
|
46767
|
+
row("Interval", formatSeconds(p.interval_seconds));
|
|
46768
|
+
row("Date tag", formatDateTag(p.date_tag));
|
|
46769
|
+
row("Since", p.since ?? "-");
|
|
46770
|
+
row("Lookback", `${p.lookback_days}d`);
|
|
46771
|
+
row("Settle", formatSeconds(p.settle_seconds));
|
|
46772
|
+
row("Same name", sameNameLabel(p.change_policy));
|
|
46773
|
+
row("Max attempts", String(p.max_attempts));
|
|
46774
|
+
row("Last poll", `${p.last_poll_at ?? "-"} ${oneLine3(p.last_poll_status ?? "")}`.trimEnd());
|
|
46775
|
+
row("Next poll", p.next_poll_at);
|
|
46776
|
+
row("Failures", String(p.consecutive_failures));
|
|
46777
|
+
row("Discover", p.discover_in_flight ? `in flight since ${p.discover_sent_at ?? "?"}` : "-");
|
|
46778
|
+
row("Last digest", p.last_digest ?? "-");
|
|
46779
|
+
row("Pending seed", p.poll_run_seed_until ?? "-");
|
|
46780
|
+
row(
|
|
46781
|
+
"Pending rekey",
|
|
46782
|
+
p.poll_run_rekey_to === null ? "-" : `to ${sameNameLabel(p.poll_run_rekey_to)}`
|
|
46783
|
+
);
|
|
46784
|
+
row("Items", counts === "" ? "(none)" : counts);
|
|
46831
46785
|
}
|
|
46832
|
-
function
|
|
46833
|
-
|
|
46834
|
-
|
|
46835
|
-
|
|
46786
|
+
function writeTable26(out, rows) {
|
|
46787
|
+
const widths = [];
|
|
46788
|
+
for (const r of rows) {
|
|
46789
|
+
r.forEach((cell, i) => {
|
|
46790
|
+
widths[i] = Math.max(widths[i] ?? 0, cell.length);
|
|
46791
|
+
});
|
|
46836
46792
|
}
|
|
46837
|
-
for (const
|
|
46838
|
-
|
|
46839
|
-
|
|
46840
|
-
`
|
|
46793
|
+
for (const r of rows) {
|
|
46794
|
+
const cells = r.map(
|
|
46795
|
+
(cell, i) => i === r.length - 1 ? cell : cell.padEnd((widths[i] ?? 0) + 2)
|
|
46841
46796
|
);
|
|
46797
|
+
out.write(`${cells.join("").trimEnd()}
|
|
46798
|
+
`);
|
|
46842
46799
|
}
|
|
46843
46800
|
}
|
|
46844
|
-
function
|
|
46845
|
-
|
|
46846
|
-
|
|
46847
|
-
|
|
46801
|
+
function renderItemTable(out, items) {
|
|
46802
|
+
const rows = [["ID", "LABEL", "DATE", "STATE", "ATTEMPTS", "VERSION", "ERROR"]];
|
|
46803
|
+
for (const it of items) {
|
|
46804
|
+
const error = it.last_error ?? (it.skip_reason === null ? null : `skipped: ${it.skip_reason}`);
|
|
46805
|
+
rows.push([
|
|
46806
|
+
String(it.id),
|
|
46807
|
+
oneLine3(it.label),
|
|
46808
|
+
it.item_date ?? "-",
|
|
46809
|
+
it.state,
|
|
46810
|
+
String(it.attempts),
|
|
46811
|
+
it.version_id === null ? "-" : String(it.version_id),
|
|
46812
|
+
error === null ? "-" : truncate3(oneLine3(error), 80)
|
|
46813
|
+
]);
|
|
46848
46814
|
}
|
|
46849
|
-
|
|
46850
|
-
|
|
46851
|
-
|
|
46852
|
-
|
|
46853
|
-
|
|
46815
|
+
writeTable26(out, rows);
|
|
46816
|
+
}
|
|
46817
|
+
async function runPollEnable(cfg) {
|
|
46818
|
+
const binding = await resolvePollBinding(cfg.client, cfg.filebox, cfg.signal);
|
|
46819
|
+
let existing;
|
|
46820
|
+
try {
|
|
46821
|
+
existing = await getPoll(cfg.client, binding.query_id, { signal: cfg.signal });
|
|
46822
|
+
} catch (err) {
|
|
46823
|
+
if (!isNotPolled(err)) throw err;
|
|
46824
|
+
existing = null;
|
|
46825
|
+
}
|
|
46826
|
+
const s = cfg.settings;
|
|
46827
|
+
if (existing !== null && (s.since !== void 0 || s.lookback_days !== void 0 || s.change_policy !== void 0)) {
|
|
46828
|
+
throw new Error(
|
|
46829
|
+
`filebox ${cfg.filebox} is already polled; change --since, --lookback or --same-name with dr connectors poll set ${cfg.filebox}, which asks what to do with a wider window or a policy switch`
|
|
46854
46830
|
);
|
|
46855
46831
|
}
|
|
46832
|
+
if (binding.filebox_id <= 0) {
|
|
46833
|
+
throw new Error(
|
|
46834
|
+
`connector-query #${binding.query_id} has no filebox id recorded, so its date tag cannot be read; record it with dr connectors query repoint ${binding.query_id} --filebox-id <id>`
|
|
46835
|
+
);
|
|
46836
|
+
}
|
|
46837
|
+
const dateTag = await readFileboxDateTag(cfg.monolith, binding.filebox_id, cfg.signal);
|
|
46838
|
+
const stored = await putPoll(
|
|
46839
|
+
cfg.client,
|
|
46840
|
+
binding.query_id,
|
|
46841
|
+
{ enabled: true, date_tag: dateTag, ...s },
|
|
46842
|
+
{ signal: cfg.signal }
|
|
46843
|
+
);
|
|
46844
|
+
if (cfg.asJson) {
|
|
46845
|
+
writeJSONIndented(cfg.out, stored);
|
|
46846
|
+
return;
|
|
46847
|
+
}
|
|
46848
|
+
renderPoll(cfg.out, "Polling enabled", binding, stored);
|
|
46856
46849
|
}
|
|
46857
|
-
function
|
|
46858
|
-
|
|
46859
|
-
|
|
46860
|
-
addAccountFlag(cmd);
|
|
46861
|
-
addConnectorsUrlFlag(cmd);
|
|
46862
|
-
return cmd;
|
|
46863
|
-
}
|
|
46864
|
-
function addScheduleFlags(cmd) {
|
|
46865
|
-
return cmd.option("--artifact <name>", "published artifact name, e.g. sftp_export__daily_sales").option("--flavor <flavor>", "runner flavor the artifact was published for, e.g. sftp").option("--datasource <role=id>", "monolith datasource for a role (repeat per role)", collect).option(
|
|
46866
|
-
"--primary-role <role>",
|
|
46867
|
-
"the role passed as the primary datasource (default: the first)"
|
|
46868
|
-
).option("--config <json|@file>", "task config: a JSON object, or @path to a JSON file").option("--every <duration>", "run on an interval, e.g. 5m, 1h (at least 1m)").option("--cron <expr>", 'run on a five-field cron line, e.g. "15 * * * *"').option("--tz <iana>", "timezone the cron line is read in (default UTC)");
|
|
46850
|
+
function windowStart(since, lookbackDays, now) {
|
|
46851
|
+
const start = now - lookbackDays * 864e5;
|
|
46852
|
+
return since === null ? start : Math.max(start, Date.parse(since));
|
|
46869
46853
|
}
|
|
46870
|
-
|
|
46871
|
-
|
|
46872
|
-
|
|
46873
|
-
|
|
46874
|
-
|
|
46875
|
-
|
|
46876
|
-
|
|
46877
|
-
} catch (err) {
|
|
46878
|
-
throw explainTaskError(err);
|
|
46879
|
-
}
|
|
46880
|
-
});
|
|
46881
|
-
}
|
|
46882
|
-
function registerConnectorsTasks(connectors) {
|
|
46883
|
-
const tasks = connectors.command("tasks").description("DRIP scheduled tasks: run an artifact on a schedule (support user or org admin)");
|
|
46884
|
-
withFlags(
|
|
46885
|
-
addScheduleFlags(
|
|
46886
|
-
tasks.command("create").description("Create a scheduled task").argument("<name>", "task name, unique in the org (a-z, 0-9, _ and -)").option("--org-id <id>", "org id (defaults to the JWT org)").option("--disabled", "create it disabled", false)
|
|
46887
|
-
)
|
|
46888
|
-
).action(
|
|
46889
|
-
async (name, opts) => {
|
|
46890
|
-
const explicitOrg = opts.orgId !== void 0 ? parsePositiveIntArg2(opts.orgId, "--org-id") : void 0;
|
|
46891
|
-
const parsed2 = buildCreateFields(name, explicitOrg ?? 0, opts);
|
|
46892
|
-
await withConnectorsSetup(opts, async ({ client, signal }) => {
|
|
46893
|
-
const fields = { ...parsed2, org_id: explicitOrg ?? client.getCurrentOrgId() };
|
|
46894
|
-
try {
|
|
46895
|
-
const t = await createTask(client, fields, { signal });
|
|
46896
|
-
if (opts.json === true) writeJSONIndented(process.stdout, t);
|
|
46897
|
-
else writeTask(process.stdout, t);
|
|
46898
|
-
} catch (err) {
|
|
46899
|
-
throw explainTaskError(err);
|
|
46900
|
-
}
|
|
46901
|
-
});
|
|
46902
|
-
}
|
|
46903
|
-
);
|
|
46904
|
-
withFlags(
|
|
46905
|
-
tasks.command("list").description("List the org's scheduled tasks").option("--org-id <id>", "org id (defaults to the JWT org)")
|
|
46906
|
-
).action(async (opts) => {
|
|
46907
|
-
await withConnectorsSetup(opts, async ({ client, signal }) => {
|
|
46908
|
-
const orgId = opts.orgId !== void 0 ? parsePositiveIntArg2(opts.orgId, "--org-id") : void 0;
|
|
46909
|
-
try {
|
|
46910
|
-
const list28 = await listTasks(client, orgId, { signal });
|
|
46911
|
-
if (opts.json === true) writeJSONIndented(process.stdout, list28);
|
|
46912
|
-
else writeTaskList(process.stdout, list28);
|
|
46913
|
-
} catch (err) {
|
|
46914
|
-
throw explainTaskError(err);
|
|
46915
|
-
}
|
|
46916
|
-
});
|
|
46917
|
-
});
|
|
46918
|
-
withFlags(
|
|
46919
|
-
tasks.command("get").description("Show a scheduled task").argument("<task>", "task id or name")
|
|
46920
|
-
).action(
|
|
46921
|
-
async (task, opts) => onTask(
|
|
46922
|
-
task,
|
|
46923
|
-
opts,
|
|
46924
|
-
(c, id, signal) => getTask(c, id, { signal }),
|
|
46925
|
-
(o, r) => writeTask(o, r)
|
|
46926
|
-
)
|
|
46927
|
-
);
|
|
46928
|
-
withFlags(
|
|
46929
|
-
addScheduleFlags(
|
|
46930
|
-
tasks.command("update").description("Change a scheduled task (only the flags given change)").argument("<task>", "task id or name").option("--enable", "enable the task").option("--disable", "disable the task")
|
|
46931
|
-
)
|
|
46932
|
-
).action(
|
|
46933
|
-
async (task, opts) => {
|
|
46934
|
-
if (opts.enable === true && opts.disable === true)
|
|
46935
|
-
throw new Error("give --enable or --disable, not both");
|
|
46936
|
-
const fields = parseScheduleFlags(opts);
|
|
46937
|
-
if (opts.enable === true) fields.enabled = true;
|
|
46938
|
-
if (opts.disable === true) fields.enabled = false;
|
|
46939
|
-
if (Object.keys(fields).length === 0)
|
|
46940
|
-
throw new Error("nothing to change: give at least one flag");
|
|
46941
|
-
await onTask(
|
|
46942
|
-
task,
|
|
46943
|
-
opts,
|
|
46944
|
-
(c, id, signal) => updateTask(c, id, fields, { signal }),
|
|
46945
|
-
(o, r) => writeTask(o, r)
|
|
46946
|
-
);
|
|
46947
|
-
}
|
|
46948
|
-
);
|
|
46949
|
-
withFlags(
|
|
46950
|
-
tasks.command("delete").description("Delete a scheduled task and its run history").argument("<task>", "task id or name")
|
|
46951
|
-
).action(
|
|
46952
|
-
async (task, opts) => onTask(
|
|
46953
|
-
task,
|
|
46954
|
-
opts,
|
|
46955
|
-
async (c, id, signal) => {
|
|
46956
|
-
await deleteTask(c, id, { signal });
|
|
46957
|
-
return { deleted: id };
|
|
46958
|
-
},
|
|
46959
|
-
(o, r) => o.write(`Deleted scheduled task ${r.deleted}
|
|
46960
|
-
`)
|
|
46961
|
-
)
|
|
46962
|
-
);
|
|
46963
|
-
withFlags(
|
|
46964
|
-
tasks.command("run").description("Run a task now (does not move its schedule)").argument("<task>", "task id or name")
|
|
46965
|
-
).action(
|
|
46966
|
-
async (task, opts) => onTask(
|
|
46967
|
-
task,
|
|
46968
|
-
opts,
|
|
46969
|
-
(c, id, signal) => runTask(c, id, { signal }),
|
|
46970
|
-
(o, r) => {
|
|
46971
|
-
const run2 = r;
|
|
46972
|
-
o.write(`Queued run ${run2.id} (${run2.status}); see: dr connectors tasks runs ${task}
|
|
46973
|
-
`);
|
|
46974
|
-
}
|
|
46975
|
-
)
|
|
46976
|
-
);
|
|
46977
|
-
withFlags(
|
|
46978
|
-
tasks.command("pause").description("Pause a scheduled task").argument("<task>", "task id or name")
|
|
46979
|
-
).action(
|
|
46980
|
-
async (task, opts) => onTask(
|
|
46981
|
-
task,
|
|
46982
|
-
opts,
|
|
46983
|
-
(c, id, signal) => pauseTask(c, id, { signal }),
|
|
46984
|
-
(o, r) => writeTask(o, r)
|
|
46985
|
-
)
|
|
46986
|
-
);
|
|
46987
|
-
withFlags(
|
|
46988
|
-
tasks.command("resume").description("Resume a paused task (clears its failure count, schedules the next run)").argument("<task>", "task id or name")
|
|
46989
|
-
).action(
|
|
46990
|
-
async (task, opts) => onTask(
|
|
46991
|
-
task,
|
|
46992
|
-
opts,
|
|
46993
|
-
(c, id, signal) => resumeTask(c, id, { signal }),
|
|
46994
|
-
(o, r) => writeTask(o, r)
|
|
46995
|
-
)
|
|
46854
|
+
function widening(current, changes, now) {
|
|
46855
|
+
if (changes.since === void 0 && changes.lookback_days === void 0) return null;
|
|
46856
|
+
const from = windowStart(current.since, current.lookback_days, now);
|
|
46857
|
+
const to = windowStart(
|
|
46858
|
+
changes.since ?? current.since,
|
|
46859
|
+
changes.lookback_days ?? current.lookback_days,
|
|
46860
|
+
now
|
|
46996
46861
|
);
|
|
46997
|
-
|
|
46998
|
-
|
|
46999
|
-
).action(async (task, opts) => {
|
|
47000
|
-
const limit = parsePositiveIntArg2(opts.limit, "--limit");
|
|
47001
|
-
if (limit > 200) throw new Error("invalid --limit: at most 200");
|
|
47002
|
-
await onTask(
|
|
47003
|
-
task,
|
|
47004
|
-
opts,
|
|
47005
|
-
(c, id, signal) => listRuns(c, id, limit, { signal }),
|
|
47006
|
-
(o, r) => writeRuns(o, r)
|
|
47007
|
-
);
|
|
47008
|
-
});
|
|
47009
|
-
}
|
|
47010
|
-
|
|
47011
|
-
// src/commands/connectorsAws.ts
|
|
47012
|
-
var ROLE_ARN_RE = /^arn:aws[a-z-]*:iam::\d{12}:role\/\S+$/;
|
|
47013
|
-
var OTHER_ORG_HINT = "if your session is valid, the connection or --org-id may belong to another org than the --account you are logged in as";
|
|
47014
|
-
function explainAwsError(err, id) {
|
|
47015
|
-
if (!(err instanceof ControlPlaneError)) {
|
|
47016
|
-
if (err instanceof Error && /\bHTTP 403\b/.test(err.message)) {
|
|
47017
|
-
return withCause(
|
|
47018
|
-
new ControlPlaneError(
|
|
47019
|
-
403,
|
|
47020
|
-
"",
|
|
47021
|
-
`${err.message}
|
|
47022
|
-
${OTHER_ORG_HINT}`
|
|
47023
|
-
),
|
|
47024
|
-
err
|
|
47025
|
-
);
|
|
47026
|
-
}
|
|
47027
|
-
return err;
|
|
47028
|
-
}
|
|
47029
|
-
if (!hasDetail(err.body)) return err;
|
|
47030
|
-
const detail = detailOf(err.body) || err.message;
|
|
47031
|
-
const messages = {
|
|
47032
|
-
403: `forbidden: ${detail} (${OTHER_ORG_HINT})`,
|
|
47033
|
-
404: `AWS connection${id === void 0 ? "" : ` ${id}`} not found: ${detail}`,
|
|
47034
|
-
409: `conflict: ${detail}`,
|
|
47035
|
-
422: `rejected: ${detail}`,
|
|
47036
|
-
503: `the control plane cannot broker AWS roles on this site: ${detail}`
|
|
47037
|
-
};
|
|
47038
|
-
const message = messages[err.status];
|
|
47039
|
-
if (message === void 0) return err;
|
|
47040
|
-
return withCause(new ControlPlaneError(err.status, err.body, message), err);
|
|
46862
|
+
if (to >= from) return null;
|
|
46863
|
+
return { from: new Date(from).toISOString(), to: new Date(to).toISOString() };
|
|
47041
46864
|
}
|
|
47042
|
-
function
|
|
47043
|
-
|
|
47044
|
-
|
|
47045
|
-
|
|
47046
|
-
} catch {
|
|
47047
|
-
return false;
|
|
47048
|
-
}
|
|
46865
|
+
async function ask(cfg, message, choices, defaultChoice) {
|
|
46866
|
+
const answer = await cfg.prompt(message, choices, defaultChoice);
|
|
46867
|
+
if (answer === null) throw new Error("poll set cancelled; nothing was changed");
|
|
46868
|
+
return parseChoice(answer, "the answer", choices);
|
|
47049
46869
|
}
|
|
47050
|
-
|
|
47051
|
-
|
|
46870
|
+
var WIDEN_HELP = "seed = mark the newly exposed items as already landed, land = land them all";
|
|
46871
|
+
var POLICY_HELP = "rekey = re-key the ledger from a fresh listing (nothing lands again), seed = mark everything in the window as landed, land = land everything in the window";
|
|
46872
|
+
function pollSetConfirmVerb(widen, onPolicyChange) {
|
|
46873
|
+
return widen === "land" || onPolicyChange === "land" ? "set-land" : "set-seed";
|
|
47052
46874
|
}
|
|
47053
|
-
function
|
|
47054
|
-
|
|
47055
|
-
|
|
47056
|
-
|
|
47057
|
-
|
|
47058
|
-
out.write(`AWS connection #${c.id} (org ${c.org_id})
|
|
47059
|
-
`);
|
|
47060
|
-
row("Datasource", c.secret_ref);
|
|
47061
|
-
row("Role ARN", c.role_arn);
|
|
47062
|
-
row("Status", `${c.status}${c.enabled ? "" : " (disabled)"}`);
|
|
47063
|
-
row("Broker ARN", c.broker_arn ?? "-");
|
|
47064
|
-
row("External ID", c.external_id);
|
|
47065
|
-
row("Expires", c.expires_at ?? "-");
|
|
47066
|
-
row("Last assumed", c.last_assumed_at ?? "-");
|
|
47067
|
-
row("Last error", c.last_error === null ? "-" : oneLine3(c.last_error));
|
|
47068
|
-
row("Failures", String(c.consecutive_failures));
|
|
47069
|
-
if (c.trust_statement === null) {
|
|
47070
|
-
out.write("\nTrust statement: none returned by the control plane.\n");
|
|
47071
|
-
} else {
|
|
47072
|
-
out.write("\nTrust statement (add it to the customer role's trust policy):\n");
|
|
47073
|
-
out.write(`${JSON.stringify(c.trust_statement, null, 2)}
|
|
47074
|
-
`);
|
|
46875
|
+
async function runPollSet(cfg) {
|
|
46876
|
+
if (Object.keys(cfg.changes).length === 0) {
|
|
46877
|
+
throw new Error(
|
|
46878
|
+
"poll set: pass at least one of --interval, --since, --lookback, --settle, --same-name"
|
|
46879
|
+
);
|
|
47075
46880
|
}
|
|
47076
|
-
|
|
47077
|
-
|
|
46881
|
+
const binding = await resolvePollBinding(cfg.client, cfg.filebox, cfg.signal);
|
|
46882
|
+
const current = await explainNotPolled(
|
|
46883
|
+
cfg.filebox,
|
|
46884
|
+
getPoll(cfg.client, binding.query_id, { signal: cfg.signal })
|
|
47078
46885
|
);
|
|
47079
|
-
}
|
|
47080
|
-
|
|
47081
|
-
|
|
47082
|
-
|
|
47083
|
-
|
|
46886
|
+
const body = { ...cfg.changes };
|
|
46887
|
+
const detail = [`target: ${bindingTarget(binding)}`];
|
|
46888
|
+
const wider = widening(current, cfg.changes, cfg.now ?? Date.now());
|
|
46889
|
+
if (wider !== null) {
|
|
46890
|
+
const what = `the polling window widens (start ${wider.from} -> ${wider.to})`;
|
|
46891
|
+
if (cfg.widen === void 0 && !cfg.interactive) {
|
|
46892
|
+
throw new Error(`${what}; pass --widen seed or --widen land (${WIDEN_HELP})`);
|
|
46893
|
+
}
|
|
46894
|
+
body.widen = cfg.widen ?? await ask(cfg, `${what}. ${WIDEN_HELP}.`, WIDEN_CHOICES);
|
|
46895
|
+
detail.push(`window: start ${wider.from} -> ${wider.to}, widen ${body.widen}`);
|
|
46896
|
+
} else if (cfg.widen !== void 0) {
|
|
46897
|
+
body.widen = cfg.widen;
|
|
47084
46898
|
}
|
|
47085
|
-
const
|
|
47086
|
-
|
|
47087
|
-
|
|
47088
|
-
|
|
47089
|
-
|
|
47090
|
-
|
|
47091
|
-
|
|
47092
|
-
|
|
47093
|
-
|
|
47094
|
-
|
|
46899
|
+
const newPolicy = cfg.changes.change_policy;
|
|
46900
|
+
const policySwitch = newPolicy !== void 0 && newPolicy !== current.change_policy;
|
|
46901
|
+
if (policySwitch) {
|
|
46902
|
+
const change = `${sameNameLabel(current.change_policy)} -> ${sameNameLabel(newPolicy)}`;
|
|
46903
|
+
const what = `--same-name changes ${change}`;
|
|
46904
|
+
if (cfg.onPolicyChange === void 0 && !cfg.interactive) {
|
|
46905
|
+
throw new Error(`${what}; pass --on-policy-change rekey, seed or land (${POLICY_HELP})`);
|
|
46906
|
+
}
|
|
46907
|
+
body.on_policy_change = cfg.onPolicyChange ?? await ask(cfg, `${what}. ${POLICY_HELP}.`, POLICY_CHOICES, "rekey");
|
|
46908
|
+
detail.push(`same name: ${change}, on policy change ${body.on_policy_change}`);
|
|
46909
|
+
} else if (cfg.onPolicyChange !== void 0) {
|
|
46910
|
+
body.on_policy_change = cfg.onPolicyChange;
|
|
47095
46911
|
}
|
|
47096
|
-
const
|
|
47097
|
-
|
|
47098
|
-
|
|
47099
|
-
|
|
47100
|
-
|
|
47101
|
-
|
|
46912
|
+
const put = async () => {
|
|
46913
|
+
const stored = await putPoll(cfg.client, binding.query_id, body, {
|
|
46914
|
+
signal: cfg.signal
|
|
46915
|
+
});
|
|
46916
|
+
if (cfg.asJson) {
|
|
46917
|
+
writeJSONIndented(cfg.out, stored);
|
|
46918
|
+
return;
|
|
46919
|
+
}
|
|
46920
|
+
renderPoll(cfg.out, "Polling settings updated", binding, stored);
|
|
46921
|
+
};
|
|
46922
|
+
const landsOrHides = wider !== null && body.widen === "land" || policySwitch && (body.on_policy_change === "seed" || body.on_policy_change === "land");
|
|
46923
|
+
if (!landsOrHides) {
|
|
46924
|
+
await put();
|
|
46925
|
+
return;
|
|
47102
46926
|
}
|
|
47103
|
-
}
|
|
47104
|
-
|
|
47105
|
-
|
|
47106
|
-
|
|
47107
|
-
|
|
47108
|
-
|
|
47109
|
-
|
|
47110
|
-
|
|
47111
|
-
|
|
47112
|
-
|
|
47113
|
-
|
|
47114
|
-
|
|
47115
|
-
|
|
47116
|
-
|
|
47117
|
-
|
|
46927
|
+
const flags = `${body.widen === void 0 ? "" : ` --widen ${body.widen}`}${body.on_policy_change === void 0 ? "" : ` --on-policy-change ${body.on_policy_change}`}`;
|
|
46928
|
+
cfg.deadline?.pause();
|
|
46929
|
+
await twoStepConfirm({
|
|
46930
|
+
out: cfg.out,
|
|
46931
|
+
action: pollConfirmAction(
|
|
46932
|
+
cfg.filebox,
|
|
46933
|
+
pollSetConfirmVerb(body.widen, body.on_policy_change),
|
|
46934
|
+
cfg.scope,
|
|
46935
|
+
{ ...body }
|
|
46936
|
+
),
|
|
46937
|
+
token: cfg.token,
|
|
46938
|
+
server: cfg.scope.serverKey,
|
|
46939
|
+
confirmCmd: `dr connectors poll set ${cfg.filebox}${cfg.settingsArgs}${flags}${cfg.targetArgs}`,
|
|
46940
|
+
describe: async () => detail,
|
|
46941
|
+
execute: async () => {
|
|
46942
|
+
cfg.deadline?.resume();
|
|
46943
|
+
await put();
|
|
47118
46944
|
}
|
|
47119
|
-
if (opts.json === true) writeJSONIndented(process.stdout, result);
|
|
47120
|
-
else print(process.stdout, result);
|
|
47121
46945
|
});
|
|
47122
46946
|
}
|
|
47123
|
-
function
|
|
47124
|
-
const
|
|
47125
|
-
|
|
47126
|
-
|
|
47127
|
-
|
|
47128
|
-
|
|
47129
|
-
if (secretRef === "" || secretRef.length > 200) {
|
|
47130
|
-
throw new Error(`invalid --secret-ref "${opts.secretRef}": expected 1-200 characters`);
|
|
47131
|
-
}
|
|
47132
|
-
if (!ROLE_ARN_RE.test(opts.roleArn)) {
|
|
47133
|
-
throw new Error(
|
|
47134
|
-
`invalid --role-arn "${opts.roleArn}": expected arn:aws:iam::<12-digit account>:role/<name>`
|
|
47135
|
-
);
|
|
47136
|
-
}
|
|
47137
|
-
const orgId = opts.orgId === void 0 ? void 0 : parsePositiveIntArg2(opts.orgId, "--org-id");
|
|
47138
|
-
const fields = {
|
|
47139
|
-
secret_ref: secretRef,
|
|
47140
|
-
role_arn: opts.roleArn,
|
|
47141
|
-
...orgId !== void 0 ? { org_id: orgId } : {}
|
|
47142
|
-
};
|
|
47143
|
-
await run(
|
|
47144
|
-
opts,
|
|
47145
|
-
({ client, signal }) => createAwsConnection(client, fields, { signal }),
|
|
47146
|
-
writeConnection
|
|
47147
|
-
);
|
|
47148
|
-
});
|
|
47149
|
-
withFlags2(
|
|
47150
|
-
aws.command("list").description("List the org's AWS connections").option("--secret-ref <datasourceId>", "only the connection of this datasource")
|
|
47151
|
-
).action(async (opts) => {
|
|
47152
|
-
await run(
|
|
47153
|
-
opts,
|
|
47154
|
-
({ client, signal }) => listAwsConnections(client, opts.secretRef, { signal }),
|
|
47155
|
-
writeConnectionTable
|
|
47156
|
-
);
|
|
47157
|
-
});
|
|
47158
|
-
withFlags2(
|
|
47159
|
-
aws.command("get").description("Show an AWS connection and its trust statement").argument("<id>", "connection id")
|
|
47160
|
-
).action(async (idArg, opts) => {
|
|
47161
|
-
const id = parsePositiveIntArg2(idArg, "connection id");
|
|
47162
|
-
await run(
|
|
47163
|
-
opts,
|
|
47164
|
-
({ client, signal }) => getAwsConnection(client, id, { signal }),
|
|
47165
|
-
writeConnection,
|
|
47166
|
-
id
|
|
47167
|
-
);
|
|
46947
|
+
function confirmDigest(scope, payload) {
|
|
46948
|
+
const canonical = JSON.stringify({
|
|
46949
|
+
server: scope.serverKey,
|
|
46950
|
+
account: scope.account ?? "",
|
|
46951
|
+
controlPlane: scope.controlPlane,
|
|
46952
|
+
payload: Object.fromEntries(Object.entries(payload).sort(([a], [b]) => a.localeCompare(b)))
|
|
47168
46953
|
});
|
|
47169
|
-
|
|
47170
|
-
["enable", true, "Enable an AWS connection; also retries now (stored credentials are dropped)"],
|
|
47171
|
-
["disable", false, "Disable an AWS connection (stored credentials are dropped)"]
|
|
47172
|
-
]) {
|
|
47173
|
-
withFlags2(aws.command(verb).description(help).argument("<id>", "connection id")).action(
|
|
47174
|
-
async (idArg, opts) => {
|
|
47175
|
-
const id = parsePositiveIntArg2(idArg, "connection id");
|
|
47176
|
-
await run(
|
|
47177
|
-
opts,
|
|
47178
|
-
({ client, signal }) => updateAwsConnection(client, id, { enabled }, { signal }),
|
|
47179
|
-
(out, c) => {
|
|
47180
|
-
writeConnection(out, c);
|
|
47181
|
-
if (enabled)
|
|
47182
|
-
out.write(
|
|
47183
|
-
"\nStored credentials were dropped; the poller assumes the role again now.\n"
|
|
47184
|
-
);
|
|
47185
|
-
},
|
|
47186
|
-
id
|
|
47187
|
-
);
|
|
47188
|
-
}
|
|
47189
|
-
);
|
|
47190
|
-
}
|
|
46954
|
+
return createHash12("sha256").update(canonical).digest("hex").slice(0, 12);
|
|
47191
46955
|
}
|
|
47192
|
-
|
|
47193
|
-
|
|
47194
|
-
|
|
47195
|
-
|
|
47196
|
-
|
|
47197
|
-
function buildDevEnvelope(i) {
|
|
47198
|
-
const config = {
|
|
47199
|
-
...i.publicConfig,
|
|
47200
|
-
...i.queryConfig,
|
|
47201
|
-
// Coerce absent values to null (not undefined) so JSON.stringify keeps the
|
|
47202
|
-
// key — matches build_hook_job's dq.get(...) -> None; a dropped objectType
|
|
47203
|
-
// becomes a cryptic KeyError in the runner's dispatch.
|
|
47204
|
-
objectType: i.dataQuery.objectType ?? null,
|
|
47205
|
-
type: i.dataQuery.type ?? null,
|
|
47206
|
-
query: i.dataQuery.query
|
|
46956
|
+
function pollConfirmAction(filebox, verb, scope, payload = {}) {
|
|
46957
|
+
return {
|
|
46958
|
+
resource: "connector-poll",
|
|
46959
|
+
id: `${filebox.trim()}#${confirmDigest(scope, payload)}`,
|
|
46960
|
+
verb
|
|
47207
46961
|
};
|
|
47208
|
-
|
|
46962
|
+
}
|
|
46963
|
+
function pollItemConfirmAction(filebox, item, scope) {
|
|
47209
46964
|
return {
|
|
47210
|
-
|
|
47211
|
-
|
|
47212
|
-
|
|
47213
|
-
cred_token: "dev-local",
|
|
47214
|
-
output_sas: "file:///out/output.csv",
|
|
47215
|
-
input_sas: null,
|
|
47216
|
-
cursor: null,
|
|
47217
|
-
contract_version: i.contractVersion,
|
|
47218
|
-
metadata: {}
|
|
46965
|
+
resource: "connector-poll-item",
|
|
46966
|
+
id: `${filebox.trim()}/${item}#${confirmDigest(scope, {})}`,
|
|
46967
|
+
verb: "skip"
|
|
47219
46968
|
};
|
|
47220
46969
|
}
|
|
47221
|
-
function
|
|
47222
|
-
|
|
47223
|
-
|
|
46970
|
+
async function runPollDisable(cfg) {
|
|
46971
|
+
const binding = await resolvePollBinding(cfg.client, cfg.filebox, cfg.signal);
|
|
46972
|
+
await explainNotPolled(
|
|
46973
|
+
cfg.filebox,
|
|
46974
|
+
getPoll(cfg.client, binding.query_id, { signal: cfg.signal })
|
|
46975
|
+
);
|
|
46976
|
+
const stored = await putPoll(
|
|
46977
|
+
cfg.client,
|
|
46978
|
+
binding.query_id,
|
|
46979
|
+
{ enabled: false },
|
|
46980
|
+
{ signal: cfg.signal }
|
|
46981
|
+
);
|
|
46982
|
+
if (cfg.asJson) {
|
|
46983
|
+
writeJSONIndented(cfg.out, stored);
|
|
46984
|
+
return;
|
|
47224
46985
|
}
|
|
47225
|
-
|
|
46986
|
+
renderPoll(cfg.out, "Polling disabled", binding, stored);
|
|
47226
46987
|
}
|
|
47227
|
-
function
|
|
47228
|
-
const
|
|
47229
|
-
|
|
47230
|
-
|
|
47231
|
-
|
|
47232
|
-
|
|
46988
|
+
async function runPollGet(cfg) {
|
|
46989
|
+
const binding = await resolvePollBinding(cfg.client, cfg.filebox, cfg.signal);
|
|
46990
|
+
const p = await explainNotPolled(
|
|
46991
|
+
cfg.filebox,
|
|
46992
|
+
getPoll(cfg.client, binding.query_id, { signal: cfg.signal })
|
|
46993
|
+
);
|
|
46994
|
+
if (cfg.asJson) {
|
|
46995
|
+
writeJSONIndented(cfg.out, p);
|
|
46996
|
+
return;
|
|
47233
46997
|
}
|
|
47234
|
-
|
|
47235
|
-
|
|
47236
|
-
|
|
47237
|
-
|
|
47238
|
-
|
|
47239
|
-
|
|
47240
|
-
|
|
47241
|
-
|
|
47242
|
-
|
|
46998
|
+
renderPoll(cfg.out, "Polling", binding, p);
|
|
46999
|
+
}
|
|
47000
|
+
async function runPollRun(cfg) {
|
|
47001
|
+
const binding = await resolvePollBinding(cfg.client, cfg.filebox, cfg.signal);
|
|
47002
|
+
let p;
|
|
47003
|
+
try {
|
|
47004
|
+
p = await explainNotPolled(
|
|
47005
|
+
cfg.filebox,
|
|
47006
|
+
runPoll(cfg.client, binding.query_id, { signal: cfg.signal })
|
|
47007
|
+
);
|
|
47008
|
+
} catch (err) {
|
|
47009
|
+
if (isPollThrottled(err)) {
|
|
47243
47010
|
throw new Error(
|
|
47244
|
-
`
|
|
47011
|
+
`a poll of filebox ${cfg.filebox} is running or ran less than a minute ago; try again shortly`,
|
|
47012
|
+
{ cause: err }
|
|
47245
47013
|
);
|
|
47246
47014
|
}
|
|
47247
|
-
|
|
47015
|
+
throw err;
|
|
47248
47016
|
}
|
|
47249
|
-
|
|
47017
|
+
if (cfg.asJson) {
|
|
47018
|
+
writeJSONIndented(cfg.out, p);
|
|
47019
|
+
return;
|
|
47020
|
+
}
|
|
47021
|
+
renderPoll(cfg.out, "Poll requested", binding, p);
|
|
47250
47022
|
}
|
|
47251
|
-
function
|
|
47252
|
-
const
|
|
47253
|
-
|
|
47254
|
-
|
|
47255
|
-
|
|
47023
|
+
async function runPollResume(cfg) {
|
|
47024
|
+
const binding = await resolvePollBinding(cfg.client, cfg.filebox, cfg.signal);
|
|
47025
|
+
const resume = async () => {
|
|
47026
|
+
const p = await explainNotPolled(
|
|
47027
|
+
cfg.filebox,
|
|
47028
|
+
resumePoll(
|
|
47029
|
+
cfg.client,
|
|
47030
|
+
binding.query_id,
|
|
47031
|
+
cfg.source === void 0 ? {} : { source: cfg.source },
|
|
47032
|
+
{ signal: cfg.signal }
|
|
47033
|
+
)
|
|
47256
47034
|
);
|
|
47035
|
+
if (cfg.asJson) {
|
|
47036
|
+
writeJSONIndented(cfg.out, p);
|
|
47037
|
+
return;
|
|
47038
|
+
}
|
|
47039
|
+
renderPoll(cfg.out, "Polling resumed", binding, p);
|
|
47040
|
+
};
|
|
47041
|
+
if (cfg.source !== "land") {
|
|
47042
|
+
await resume();
|
|
47043
|
+
return;
|
|
47257
47044
|
}
|
|
47258
|
-
|
|
47259
|
-
|
|
47260
|
-
|
|
47261
|
-
|
|
47262
|
-
|
|
47263
|
-
|
|
47264
|
-
|
|
47265
|
-
|
|
47266
|
-
|
|
47267
|
-
|
|
47268
|
-
|
|
47269
|
-
|
|
47270
|
-
|
|
47271
|
-
|
|
47272
|
-
|
|
47273
|
-
|
|
47274
|
-
} catch {
|
|
47275
|
-
throw new Error(`${input.queryPath}: dataquery source not found or unreadable`);
|
|
47276
|
-
}
|
|
47277
|
-
await fs19.mkdir(input.outDir, { recursive: true });
|
|
47278
|
-
await fs19.writeFile(
|
|
47279
|
-
path9.join(input.outDir, ENV_FILE),
|
|
47280
|
-
JSON.stringify(input.env, null, 2) + "\n",
|
|
47281
|
-
"utf-8"
|
|
47282
|
-
);
|
|
47283
|
-
const secretsPath = path9.join(input.outDir, SECRETS_FILE);
|
|
47284
|
-
await fs19.writeFile(
|
|
47285
|
-
secretsPath,
|
|
47286
|
-
JSON.stringify({ [input.secretRef]: input.credentials }, null, 2) + "\n",
|
|
47287
|
-
"utf-8"
|
|
47288
|
-
);
|
|
47289
|
-
if (process.platform !== "win32") {
|
|
47290
|
-
await fs19.chmod(secretsPath, 384);
|
|
47291
|
-
}
|
|
47292
|
-
await fs19.writeFile(path9.join(input.outDir, CONNECTION_FILE), connectionSource, "utf-8");
|
|
47293
|
-
await fs19.writeFile(path9.join(input.outDir, QUERY_FILE), querySource, "utf-8");
|
|
47294
|
-
}
|
|
47295
|
-
async function ensureGitignored(entry) {
|
|
47296
|
-
const gitignorePath = path9.join(process.cwd(), ".gitignore");
|
|
47297
|
-
let current = "";
|
|
47298
|
-
try {
|
|
47299
|
-
current = await fs19.readFile(gitignorePath, "utf-8");
|
|
47300
|
-
} catch {
|
|
47301
|
-
}
|
|
47302
|
-
const normalized = entry.split("\\").join("/").replace(/\/+$/, "") + "/";
|
|
47303
|
-
const lines = current.split(/\r?\n/).map((l) => l.trim());
|
|
47304
|
-
if (lines.includes(normalized) || lines.includes(entry)) return;
|
|
47305
|
-
const sep = current === "" || current.endsWith("\n") ? "" : "\n";
|
|
47306
|
-
await fs19.writeFile(gitignorePath, `${current}${sep}${normalized}
|
|
47307
|
-
`, "utf-8");
|
|
47045
|
+
cfg.deadline?.pause();
|
|
47046
|
+
await twoStepConfirm({
|
|
47047
|
+
out: cfg.out,
|
|
47048
|
+
action: pollConfirmAction(cfg.filebox, "resume-land", cfg.scope),
|
|
47049
|
+
token: cfg.token,
|
|
47050
|
+
server: cfg.scope.serverKey,
|
|
47051
|
+
confirmCmd: `dr connectors poll resume ${cfg.filebox} --source land${cfg.targetArgs}`,
|
|
47052
|
+
describe: async () => [
|
|
47053
|
+
`target: ${bindingTarget(binding)}`,
|
|
47054
|
+
"source: land every item the changed source holds now"
|
|
47055
|
+
],
|
|
47056
|
+
execute: async () => {
|
|
47057
|
+
cfg.deadline?.resume();
|
|
47058
|
+
await resume();
|
|
47059
|
+
}
|
|
47060
|
+
});
|
|
47308
47061
|
}
|
|
47309
|
-
function
|
|
47310
|
-
const
|
|
47311
|
-
|
|
47312
|
-
|
|
47062
|
+
async function countSeedable(cfg, binding) {
|
|
47063
|
+
const untilMs = Date.parse(cfg.until);
|
|
47064
|
+
let count = 0;
|
|
47065
|
+
for (const state of ["seen", "new"]) {
|
|
47066
|
+
let after = null;
|
|
47067
|
+
do {
|
|
47068
|
+
cfg.deadline?.resume();
|
|
47069
|
+
const page = await explainNotPolled(
|
|
47070
|
+
cfg.filebox,
|
|
47071
|
+
listPollItems(
|
|
47072
|
+
cfg.client,
|
|
47073
|
+
binding.query_id,
|
|
47074
|
+
{ state, ...after === null ? {} : { after } },
|
|
47075
|
+
{ signal: cfg.signal }
|
|
47076
|
+
)
|
|
47077
|
+
);
|
|
47078
|
+
count += page.items.filter(
|
|
47079
|
+
(it) => it.source_modified_at !== null && Date.parse(it.source_modified_at) <= untilMs
|
|
47080
|
+
).length;
|
|
47081
|
+
after = page.next_after;
|
|
47082
|
+
} while (after !== null);
|
|
47313
47083
|
}
|
|
47314
|
-
return
|
|
47084
|
+
return count;
|
|
47315
47085
|
}
|
|
47316
|
-
function
|
|
47317
|
-
const
|
|
47318
|
-
|
|
47319
|
-
|
|
47320
|
-
|
|
47321
|
-
|
|
47322
|
-
|
|
47323
|
-
"--document-id <n>",
|
|
47324
|
-
"numeric filebox document id (no uuid->id lookup exists; required)"
|
|
47325
|
-
).requiredOption("--connection <path>", "path to the connector's connection .py source").requiredOption("--query <path>", "path to the connector's dataquery .py source").option("--from <date>", "period start (paired with --to)").option("--to <date>", "period end (paired with --from)").option("-o, --output <dir>", "output run dir (default ./dev-run/<fileboxUuid>/)").action(async (fileboxUuid, opts) => {
|
|
47326
|
-
const datasourceId = parsePositiveInt3(opts.datasource, "datasource id");
|
|
47327
|
-
const documentId = parsePositiveInt3(opts.documentId, "document-id");
|
|
47328
|
-
if (opts.from === void 0 !== (opts.to === void 0)) {
|
|
47329
|
-
throw new Error("--from and --to must be provided together");
|
|
47086
|
+
async function runPollSeed(cfg) {
|
|
47087
|
+
const binding = await resolvePollBinding(cfg.client, cfg.filebox, cfg.signal);
|
|
47088
|
+
if (cfg.dryRun) {
|
|
47089
|
+
const wouldSeed = await countSeedable(cfg, binding);
|
|
47090
|
+
if (cfg.asJson) {
|
|
47091
|
+
writeJSONIndented(cfg.out, { dry_run: true, until: cfg.until, would_seed: wouldSeed });
|
|
47092
|
+
return;
|
|
47330
47093
|
}
|
|
47331
|
-
|
|
47332
|
-
const outDir = opts.output ?? defaultRoot;
|
|
47333
|
-
const gitignoreEntry = opts.output ?? "dev-run";
|
|
47334
|
-
const serverKey = resolveServer(opts.server);
|
|
47335
|
-
const host = resolveConnectorsUrl(serverKey, opts.connectorsUrl);
|
|
47336
|
-
const monolithHost = resolve(serverKey).host;
|
|
47337
|
-
const cpSetup = await setupCall(opts.server, opts.account, { host, authHost: monolithHost });
|
|
47338
|
-
try {
|
|
47339
|
-
const binding = await getBinding(cpSetup.client, fileboxUuid, {
|
|
47340
|
-
signal: cpSetup.signal
|
|
47341
|
-
});
|
|
47342
|
-
const monoSetup = await setupCall(opts.server, opts.account, {
|
|
47343
|
-
host: monolithHost,
|
|
47344
|
-
authHost: monolithHost
|
|
47345
|
-
});
|
|
47346
|
-
try {
|
|
47347
|
-
const rawDataQuery = await getDataQueryRaw(monoSetup.client, documentId, {
|
|
47348
|
-
signal: monoSetup.signal
|
|
47349
|
-
});
|
|
47350
|
-
const dataQuery = extractDataQueryFields(rawDataQuery);
|
|
47351
|
-
if (dataQuery.objectType === void 0 || dataQuery.objectType === null || dataQuery.objectType === "") {
|
|
47352
|
-
process.stderr.write(
|
|
47353
|
-
'WARNING: objectType is not set inside the Sync-query (config.query.objectType). get_query_class will receive None \u2014 fine only if the connector has a single default dataset. Add it to the data-query, e.g. config.query.objectType = "<dataset>".\n'
|
|
47354
|
-
);
|
|
47355
|
-
}
|
|
47356
|
-
const integration = await itGet(monoSetup.client, datasourceId, {
|
|
47357
|
-
signal: monoSetup.signal
|
|
47358
|
-
});
|
|
47359
|
-
const credentials = extractCredentials(integration);
|
|
47360
|
-
let period;
|
|
47361
|
-
if (opts.from !== void 0 && opts.to !== void 0) {
|
|
47362
|
-
period = { from: opts.from, to: opts.to };
|
|
47363
|
-
}
|
|
47364
|
-
const env = buildDevEnvelope({
|
|
47365
|
-
correlationId: randomUUID2(),
|
|
47366
|
-
secretRef: binding.secret_ref,
|
|
47367
|
-
publicConfig: binding.public_config,
|
|
47368
|
-
queryConfig: binding.query_config,
|
|
47369
|
-
dataQuery,
|
|
47370
|
-
...period !== void 0 ? { period } : {},
|
|
47371
|
-
// Deliberate sandbox stub: the fetch/dispatch path doesn't branch on
|
|
47372
|
-
// contract_version, so a fixed "1" is sufficient for a local run.
|
|
47373
|
-
contractVersion: "1"
|
|
47374
|
-
});
|
|
47375
|
-
await writeRunDir({
|
|
47376
|
-
outDir,
|
|
47377
|
-
env,
|
|
47378
|
-
secretRef: binding.secret_ref,
|
|
47379
|
-
credentials,
|
|
47380
|
-
connectionPath: opts.connection,
|
|
47381
|
-
queryPath: opts.query
|
|
47382
|
-
});
|
|
47383
|
-
await ensureGitignored(gitignoreEntry);
|
|
47384
|
-
process.stdout.write(`Wrote run dir: ${outDir}
|
|
47094
|
+
cfg.out.write(`Seed dry run for ${bindingTarget(binding)}: nothing was changed.
|
|
47385
47095
|
`);
|
|
47386
|
-
|
|
47387
|
-
|
|
47096
|
+
cfg.out.write(
|
|
47097
|
+
`${wouldSeed} already-discovered items would be seeded (items not yet discovered are not counted).
|
|
47388
47098
|
`
|
|
47389
|
-
|
|
47390
|
-
|
|
47391
|
-
|
|
47392
|
-
|
|
47393
|
-
|
|
47099
|
+
);
|
|
47100
|
+
return;
|
|
47101
|
+
}
|
|
47102
|
+
cfg.deadline?.pause();
|
|
47103
|
+
await twoStepConfirm({
|
|
47104
|
+
out: cfg.out,
|
|
47105
|
+
action: pollConfirmAction(cfg.filebox, "seed", cfg.scope, { until: cfg.until }),
|
|
47106
|
+
token: cfg.token,
|
|
47107
|
+
server: cfg.scope.serverKey,
|
|
47108
|
+
confirmCmd: `dr connectors poll seed ${cfg.filebox} --until ${cfg.untilArg}${cfg.targetArgs}`,
|
|
47109
|
+
describe: async () => [
|
|
47110
|
+
`target: ${bindingTarget(binding)}`,
|
|
47111
|
+
`until: ${cfg.until}`,
|
|
47112
|
+
"effect: items modified at or before it are marked landed and never land"
|
|
47113
|
+
],
|
|
47114
|
+
execute: async () => {
|
|
47115
|
+
cfg.deadline?.resume();
|
|
47116
|
+
const p = await explainNotPolled(
|
|
47117
|
+
cfg.filebox,
|
|
47118
|
+
seedPoll(
|
|
47119
|
+
cfg.client,
|
|
47120
|
+
binding.query_id,
|
|
47121
|
+
{ until: cfg.until },
|
|
47122
|
+
{
|
|
47123
|
+
signal: cfg.signal
|
|
47124
|
+
}
|
|
47125
|
+
)
|
|
47126
|
+
);
|
|
47127
|
+
if (cfg.asJson) {
|
|
47128
|
+
writeJSONIndented(cfg.out, p);
|
|
47129
|
+
return;
|
|
47394
47130
|
}
|
|
47395
|
-
|
|
47396
|
-
throw augmentTlsCertError(err);
|
|
47397
|
-
} finally {
|
|
47398
|
-
cpSetup.cleanup();
|
|
47131
|
+
renderPoll(cfg.out, "Seed requested", binding, p);
|
|
47399
47132
|
}
|
|
47400
47133
|
});
|
|
47401
|
-
addServerFlag(cmd);
|
|
47402
|
-
addAccountFlag(cmd);
|
|
47403
|
-
cmd.option(
|
|
47404
|
-
"--connectors-url <url>",
|
|
47405
|
-
"Override the control-plane base URL (e.g. a kubectl port-forward http://localhost:8000)"
|
|
47406
|
-
);
|
|
47407
47134
|
}
|
|
47408
|
-
|
|
47409
|
-
|
|
47410
|
-
|
|
47411
|
-
|
|
47412
|
-
|
|
47413
|
-
|
|
47414
|
-
|
|
47415
|
-
|
|
47416
|
-
|
|
47417
|
-
|
|
47418
|
-
|
|
47419
|
-
|
|
47420
|
-
|
|
47421
|
-
|
|
47422
|
-
|
|
47423
|
-
|
|
47424
|
-
|
|
47425
|
-
|
|
47426
|
-
|
|
47427
|
-
// src/commands/connectorsPoll.ts
|
|
47428
|
-
var DURATION_UNIT_SECONDS2 = { s: 1, m: 60, h: 3600, d: 86400 };
|
|
47429
|
-
var DURATION_BOUNDS = {
|
|
47430
|
-
"--interval": [300, 7 * 86400],
|
|
47431
|
-
"--settle": [0, 86400]
|
|
47432
|
-
};
|
|
47433
|
-
var LOOKBACK_MAX_DAYS = 3650;
|
|
47434
|
-
function parseDurationSeconds(raw2, flag) {
|
|
47435
|
-
const m = /^(\d+)([smhd])?$/.exec(raw2.trim());
|
|
47436
|
-
const seconds = m === null ? NaN : Number(m[1]) * (DURATION_UNIT_SECONDS2[m[2] ?? "s"] ?? 1);
|
|
47437
|
-
const [min, max] = DURATION_BOUNDS[flag] ?? [0, Number.MAX_SAFE_INTEGER];
|
|
47438
|
-
if (m === null) {
|
|
47439
|
-
throw new Error(`invalid ${flag} "${raw2}": expected a duration like 90s, 5m or 1h`);
|
|
47135
|
+
async function runPollItems(cfg) {
|
|
47136
|
+
const binding = await resolvePollBinding(cfg.client, cfg.filebox, cfg.signal);
|
|
47137
|
+
const page = await explainNotPolled(
|
|
47138
|
+
cfg.filebox,
|
|
47139
|
+
listPollItems(
|
|
47140
|
+
cfg.client,
|
|
47141
|
+
binding.query_id,
|
|
47142
|
+
{
|
|
47143
|
+
...cfg.state !== void 0 ? { state: cfg.state } : {},
|
|
47144
|
+
...cfg.limit !== void 0 ? { limit: cfg.limit } : {},
|
|
47145
|
+
...cfg.after !== void 0 ? { after: cfg.after } : {}
|
|
47146
|
+
},
|
|
47147
|
+
{ signal: cfg.signal }
|
|
47148
|
+
)
|
|
47149
|
+
);
|
|
47150
|
+
if (cfg.asJson) {
|
|
47151
|
+
writeJSONIndented(cfg.out, page);
|
|
47152
|
+
return;
|
|
47440
47153
|
}
|
|
47441
|
-
if (
|
|
47442
|
-
|
|
47443
|
-
|
|
47444
|
-
|
|
47154
|
+
if (page.items.length === 0) {
|
|
47155
|
+
cfg.out.write(`No items${cfg.state === void 0 ? "" : ` in state ${cfg.state}`}.
|
|
47156
|
+
`);
|
|
47157
|
+
return;
|
|
47445
47158
|
}
|
|
47446
|
-
|
|
47447
|
-
|
|
47448
|
-
|
|
47449
|
-
|
|
47450
|
-
|
|
47451
|
-
|
|
47452
|
-
|
|
47453
|
-
|
|
47159
|
+
renderItemTable(cfg.out, page.items);
|
|
47160
|
+
if (page.next_after !== null) {
|
|
47161
|
+
const state = cfg.state === void 0 ? "" : ` --state ${cfg.state}`;
|
|
47162
|
+
const limit = cfg.limit === void 0 ? "" : ` --limit ${cfg.limit}`;
|
|
47163
|
+
cfg.out.write(
|
|
47164
|
+
`
|
|
47165
|
+
More: dr connectors poll items ${cfg.filebox}${state}${limit} --after ${page.next_after}
|
|
47166
|
+
`
|
|
47454
47167
|
);
|
|
47455
47168
|
}
|
|
47456
|
-
return days;
|
|
47457
|
-
}
|
|
47458
|
-
var DATE_ONLY_RE = /^(\d{4})-(\d{2})-(\d{2})$/;
|
|
47459
|
-
var DATE_TIME_RE = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2})(?::(\d{2})(?:\.\d+)?)?(?:Z|[+-](\d{2}):(\d{2}))$/;
|
|
47460
|
-
function inRange(value, max, min = 0) {
|
|
47461
|
-
const n = Number(value ?? "0");
|
|
47462
|
-
return n >= min && n <= max;
|
|
47463
|
-
}
|
|
47464
|
-
function isCalendarDay(y, m, d) {
|
|
47465
|
-
if (!inRange(m, 12, 1)) return false;
|
|
47466
|
-
const daysInMonth3 = new Date(Date.UTC(Number(y), Number(m), 0)).getUTCDate();
|
|
47467
|
-
return inRange(d, daysInMonth3, 1);
|
|
47468
|
-
}
|
|
47469
|
-
function parseTimestampFlag(raw2, flag) {
|
|
47470
|
-
const invalid = new Error(
|
|
47471
|
-
`invalid ${flag} "${raw2}": expected YYYY-MM-DD or a timestamp with a zone (e.g. 2026-10-08T17:30:00+03:00)`
|
|
47472
|
-
);
|
|
47473
|
-
const value = raw2.trim();
|
|
47474
|
-
const dateOnly = DATE_ONLY_RE.exec(value);
|
|
47475
|
-
if (dateOnly !== null) {
|
|
47476
|
-
if (!isCalendarDay(dateOnly[1], dateOnly[2], dateOnly[3])) throw invalid;
|
|
47477
|
-
return (/* @__PURE__ */ new Date(`${value}T00:00:00Z`)).toISOString();
|
|
47478
|
-
}
|
|
47479
|
-
const dt = DATE_TIME_RE.exec(value);
|
|
47480
|
-
const valid = dt !== null && isCalendarDay(dt[1], dt[2], dt[3]) && inRange(dt[4], 23) && inRange(dt[5], 59) && inRange(dt[6], 59) && inRange(dt[7], 23) && inRange(dt[8], 59);
|
|
47481
|
-
const ms = valid ? Date.parse(value) : NaN;
|
|
47482
|
-
if (Number.isNaN(ms)) throw invalid;
|
|
47483
|
-
return new Date(ms).toISOString();
|
|
47484
|
-
}
|
|
47485
|
-
function parseSameName(raw2) {
|
|
47486
|
-
if (raw2 === "new-version") return "new_version";
|
|
47487
|
-
if (raw2 === "ignore") return "ignore";
|
|
47488
|
-
throw new Error(`--same-name must be new-version or ignore (got "${raw2}")`);
|
|
47489
|
-
}
|
|
47490
|
-
function parseChoice(raw2, flag, choices) {
|
|
47491
|
-
const match = choices.find((c) => c === raw2);
|
|
47492
|
-
if (match === void 0) {
|
|
47493
|
-
throw new Error(`${flag} must be ${choices.join(" or ")} (got "${raw2}")`);
|
|
47494
|
-
}
|
|
47495
|
-
return match;
|
|
47496
47169
|
}
|
|
47497
|
-
|
|
47498
|
-
|
|
47499
|
-
|
|
47500
|
-
|
|
47501
|
-
if (opts.interval !== void 0) {
|
|
47502
|
-
fields.interval_seconds = parseDurationSeconds(opts.interval, "--interval");
|
|
47503
|
-
}
|
|
47504
|
-
if (opts.since !== void 0) fields.since = parseTimestampFlag(opts.since, "--since");
|
|
47505
|
-
if (opts.lookback !== void 0) fields.lookback_days = parseLookbackDays(opts.lookback);
|
|
47506
|
-
if (opts.settle !== void 0) {
|
|
47507
|
-
fields.settle_seconds = parseDurationSeconds(opts.settle, "--settle");
|
|
47170
|
+
function renderItemResult(cfg, binding, headline, item) {
|
|
47171
|
+
if (cfg.asJson) {
|
|
47172
|
+
writeJSONIndented(cfg.out, item);
|
|
47173
|
+
return;
|
|
47508
47174
|
}
|
|
47509
|
-
|
|
47510
|
-
|
|
47511
|
-
|
|
47512
|
-
function settingsArgs(opts) {
|
|
47513
|
-
const pairs = [
|
|
47514
|
-
["--interval", opts.interval],
|
|
47515
|
-
["--since", opts.since],
|
|
47516
|
-
["--lookback", opts.lookback],
|
|
47517
|
-
["--settle", opts.settle],
|
|
47518
|
-
["--same-name", opts.sameName]
|
|
47519
|
-
];
|
|
47520
|
-
return pairs.filter((p) => p[1] !== void 0).map(([flag, value]) => ` ${flag} ${value.trim()}`).join("");
|
|
47175
|
+
cfg.out.write(`${headline} (${bindingTarget(binding)})
|
|
47176
|
+
`);
|
|
47177
|
+
renderItemTable(cfg.out, [item]);
|
|
47521
47178
|
}
|
|
47522
|
-
async function
|
|
47523
|
-
const
|
|
47524
|
-
const
|
|
47525
|
-
|
|
47526
|
-
|
|
47527
|
-
(q) => byId ? q.filebox_id === Number(wanted) : q.filebox_uuid.toLowerCase() === wanted.toLowerCase()
|
|
47179
|
+
async function runPollRetry(cfg) {
|
|
47180
|
+
const binding = await resolvePollBinding(cfg.client, cfg.filebox, cfg.signal);
|
|
47181
|
+
const item = await explainNotPolled(
|
|
47182
|
+
cfg.filebox,
|
|
47183
|
+
retryPollItem(cfg.client, binding.query_id, cfg.itemId, { signal: cfg.signal })
|
|
47528
47184
|
);
|
|
47529
|
-
|
|
47530
|
-
if (first === void 0) {
|
|
47531
|
-
throw new Error(
|
|
47532
|
-
`no connector binding for filebox ${wanted} in this org (see dr connectors query list)`
|
|
47533
|
-
);
|
|
47534
|
-
}
|
|
47535
|
-
if (matches.length > 1) {
|
|
47536
|
-
const ids = matches.map((q) => `#${q.query_id}`).join(", ");
|
|
47537
|
-
throw new Error(`filebox ${wanted} has several connector bindings (${ids})`);
|
|
47538
|
-
}
|
|
47539
|
-
return first;
|
|
47540
|
-
}
|
|
47541
|
-
function isNotPolled(err) {
|
|
47542
|
-
return err instanceof ControlPlaneError && err.status === 404 && err.body.includes("not polled");
|
|
47543
|
-
}
|
|
47544
|
-
function isPollThrottled(err) {
|
|
47545
|
-
return err instanceof ControlPlaneError && err.status === 409 && (err.body.includes("discover is in flight") || err.body.includes("polled less than"));
|
|
47185
|
+
renderItemResult(cfg, binding, `Item ${cfg.itemId} queued for retry`, item);
|
|
47546
47186
|
}
|
|
47547
|
-
async function
|
|
47548
|
-
|
|
47549
|
-
|
|
47550
|
-
|
|
47551
|
-
|
|
47552
|
-
|
|
47553
|
-
|
|
47554
|
-
|
|
47187
|
+
async function runPollSkip(cfg) {
|
|
47188
|
+
const binding = await resolvePollBinding(cfg.client, cfg.filebox, cfg.signal);
|
|
47189
|
+
cfg.deadline?.pause();
|
|
47190
|
+
await twoStepConfirm({
|
|
47191
|
+
out: cfg.out,
|
|
47192
|
+
action: pollItemConfirmAction(cfg.filebox, cfg.itemId, cfg.scope),
|
|
47193
|
+
token: cfg.token,
|
|
47194
|
+
server: cfg.scope.serverKey,
|
|
47195
|
+
confirmCmd: `dr connectors poll skip ${cfg.filebox} ${cfg.itemId}${cfg.targetArgs}`,
|
|
47196
|
+
describe: async () => [
|
|
47197
|
+
`item: ${cfg.itemId} of ${bindingTarget(binding)}`,
|
|
47198
|
+
"effect: the item is never landed (dr connectors poll retry queues it again)"
|
|
47199
|
+
],
|
|
47200
|
+
execute: async () => {
|
|
47201
|
+
cfg.deadline?.resume();
|
|
47202
|
+
const item = await explainNotPolled(
|
|
47203
|
+
cfg.filebox,
|
|
47204
|
+
skipPollItem(cfg.client, binding.query_id, cfg.itemId, {
|
|
47205
|
+
signal: cfg.signal
|
|
47206
|
+
})
|
|
47555
47207
|
);
|
|
47208
|
+
renderItemResult(cfg, binding, `Item ${cfg.itemId} skipped`, item);
|
|
47556
47209
|
}
|
|
47557
|
-
|
|
47558
|
-
}
|
|
47210
|
+
});
|
|
47559
47211
|
}
|
|
47560
|
-
async function
|
|
47561
|
-
const
|
|
47562
|
-
const
|
|
47563
|
-
|
|
47564
|
-
|
|
47565
|
-
|
|
47566
|
-
|
|
47567
|
-
|
|
47568
|
-
|
|
47569
|
-
);
|
|
47212
|
+
async function withMonolith(opts, action) {
|
|
47213
|
+
const monolithHost = resolve(resolveServer(opts.server)).host;
|
|
47214
|
+
const setup = await setupCall(opts.server, opts.account, {
|
|
47215
|
+
host: monolithHost,
|
|
47216
|
+
authHost: monolithHost
|
|
47217
|
+
});
|
|
47218
|
+
try {
|
|
47219
|
+
await action(setup.client);
|
|
47220
|
+
} finally {
|
|
47221
|
+
setup.cleanup();
|
|
47570
47222
|
}
|
|
47571
|
-
return { id: only.id, name: only.name };
|
|
47572
|
-
}
|
|
47573
|
-
function formatSeconds(sec) {
|
|
47574
|
-
if (sec > 0 && sec % 86400 === 0) return `${sec / 86400}d`;
|
|
47575
|
-
if (sec > 0 && sec % 3600 === 0) return `${sec / 3600}h`;
|
|
47576
|
-
if (sec > 0 && sec % 60 === 0) return `${sec / 60}m`;
|
|
47577
|
-
return `${sec}s`;
|
|
47578
47223
|
}
|
|
47579
|
-
function
|
|
47580
|
-
|
|
47224
|
+
function scopeOf(opts) {
|
|
47225
|
+
const serverKey = resolveServer(opts.server);
|
|
47226
|
+
return {
|
|
47227
|
+
serverKey,
|
|
47228
|
+
account: normaliseAccount(opts.account),
|
|
47229
|
+
controlPlane: resolveConnectorsUrl(serverKey, opts.connectorsUrl)
|
|
47230
|
+
};
|
|
47581
47231
|
}
|
|
47582
|
-
function
|
|
47583
|
-
|
|
47232
|
+
function targetArgsOf(opts) {
|
|
47233
|
+
const url = (opts.connectorsUrl ?? "").trim();
|
|
47234
|
+
return ` --server ${resolveServer(opts.server)}${url === "" ? "" : ` --connectors-url ${url}`}`;
|
|
47584
47235
|
}
|
|
47585
|
-
function
|
|
47586
|
-
|
|
47587
|
-
|
|
47236
|
+
function addCommonFlags(cmd, jsonHelp) {
|
|
47237
|
+
cmd.option("--json", jsonHelp, false);
|
|
47238
|
+
addServerFlag(cmd);
|
|
47239
|
+
addAccountFlag(cmd);
|
|
47240
|
+
addConnectorsUrlFlag(cmd);
|
|
47588
47241
|
}
|
|
47589
|
-
function
|
|
47590
|
-
return
|
|
47242
|
+
function addConfirmTokenFlag(cmd) {
|
|
47243
|
+
return cmd.option("--confirm-token <token>", "Token from a prior plan run \u2014 required to proceed");
|
|
47591
47244
|
}
|
|
47592
|
-
function
|
|
47593
|
-
|
|
47594
|
-
|
|
47595
|
-
|
|
47596
|
-
};
|
|
47597
|
-
const status2 = `${p.enabled ? "enabled" : "disabled"}${p.paused_reason === null ? "" : `, paused: ${oneLine4(p.paused_reason)}`}`;
|
|
47598
|
-
const counts = Object.entries(p.counts).sort(([a], [b]) => a.localeCompare(b)).map(([state, n]) => `${state}=${n}`).join(" ");
|
|
47599
|
-
out.write(`${headline} for ${bindingTarget(binding)}
|
|
47600
|
-
`);
|
|
47601
|
-
row("Status", status2);
|
|
47602
|
-
row("Interval", formatSeconds(p.interval_seconds));
|
|
47603
|
-
row("Date tag", formatDateTag(p.date_tag));
|
|
47604
|
-
row("Since", p.since ?? "-");
|
|
47605
|
-
row("Lookback", `${p.lookback_days}d`);
|
|
47606
|
-
row("Settle", formatSeconds(p.settle_seconds));
|
|
47607
|
-
row("Same name", sameNameLabel(p.change_policy));
|
|
47608
|
-
row("Max attempts", String(p.max_attempts));
|
|
47609
|
-
row("Last poll", `${p.last_poll_at ?? "-"} ${oneLine4(p.last_poll_status ?? "")}`.trimEnd());
|
|
47610
|
-
row("Next poll", p.next_poll_at);
|
|
47611
|
-
row("Failures", String(p.consecutive_failures));
|
|
47612
|
-
row("Discover", p.discover_in_flight ? `in flight since ${p.discover_sent_at ?? "?"}` : "-");
|
|
47613
|
-
row("Last digest", p.last_digest ?? "-");
|
|
47614
|
-
row("Pending seed", p.poll_run_seed_until ?? "-");
|
|
47615
|
-
row(
|
|
47616
|
-
"Pending rekey",
|
|
47617
|
-
p.poll_run_rekey_to === null ? "-" : `to ${sameNameLabel(p.poll_run_rekey_to)}`
|
|
47618
|
-
);
|
|
47619
|
-
row("Items", counts === "" ? "(none)" : counts);
|
|
47620
|
-
}
|
|
47621
|
-
function writeTable26(out, rows) {
|
|
47622
|
-
const widths = [];
|
|
47623
|
-
for (const r of rows) {
|
|
47624
|
-
r.forEach((cell, i) => {
|
|
47625
|
-
widths[i] = Math.max(widths[i] ?? 0, cell.length);
|
|
47626
|
-
});
|
|
47627
|
-
}
|
|
47628
|
-
for (const r of rows) {
|
|
47629
|
-
const cells = r.map(
|
|
47630
|
-
(cell, i) => i === r.length - 1 ? cell : cell.padEnd((widths[i] ?? 0) + 2)
|
|
47631
|
-
);
|
|
47632
|
-
out.write(`${cells.join("").trimEnd()}
|
|
47633
|
-
`);
|
|
47634
|
-
}
|
|
47635
|
-
}
|
|
47636
|
-
function renderItemTable(out, items) {
|
|
47637
|
-
const rows = [["ID", "LABEL", "DATE", "STATE", "ATTEMPTS", "VERSION", "ERROR"]];
|
|
47638
|
-
for (const it of items) {
|
|
47639
|
-
const error = it.last_error ?? (it.skip_reason === null ? null : `skipped: ${it.skip_reason}`);
|
|
47640
|
-
rows.push([
|
|
47641
|
-
String(it.id),
|
|
47642
|
-
oneLine4(it.label),
|
|
47643
|
-
it.item_date ?? "-",
|
|
47644
|
-
it.state,
|
|
47645
|
-
String(it.attempts),
|
|
47646
|
-
it.version_id === null ? "-" : String(it.version_id),
|
|
47647
|
-
error === null ? "-" : truncate3(oneLine4(error), 80)
|
|
47648
|
-
]);
|
|
47649
|
-
}
|
|
47650
|
-
writeTable26(out, rows);
|
|
47651
|
-
}
|
|
47652
|
-
async function runPollEnable(cfg) {
|
|
47653
|
-
const binding = await resolvePollBinding(cfg.client, cfg.filebox, cfg.signal);
|
|
47654
|
-
let existing;
|
|
47655
|
-
try {
|
|
47656
|
-
existing = await getPoll(cfg.client, binding.query_id, { signal: cfg.signal });
|
|
47657
|
-
} catch (err) {
|
|
47658
|
-
if (!isNotPolled(err)) throw err;
|
|
47659
|
-
existing = null;
|
|
47660
|
-
}
|
|
47661
|
-
const s = cfg.settings;
|
|
47662
|
-
if (existing !== null && (s.since !== void 0 || s.lookback_days !== void 0 || s.change_policy !== void 0)) {
|
|
47663
|
-
throw new Error(
|
|
47664
|
-
`filebox ${cfg.filebox} is already polled; change --since, --lookback or --same-name with dr connectors poll set ${cfg.filebox}, which asks what to do with a wider window or a policy switch`
|
|
47665
|
-
);
|
|
47666
|
-
}
|
|
47667
|
-
if (binding.filebox_id <= 0) {
|
|
47668
|
-
throw new Error(
|
|
47669
|
-
`connector-query #${binding.query_id} has no filebox id recorded, so its date tag cannot be read; record it with dr connectors query repoint ${binding.query_id} --filebox-id <id>`
|
|
47670
|
-
);
|
|
47671
|
-
}
|
|
47672
|
-
const dateTag = await readFileboxDateTag(cfg.monolith, binding.filebox_id, cfg.signal);
|
|
47673
|
-
const stored = await putPoll(
|
|
47674
|
-
cfg.client,
|
|
47675
|
-
binding.query_id,
|
|
47676
|
-
{ enabled: true, date_tag: dateTag, ...s },
|
|
47677
|
-
{ signal: cfg.signal }
|
|
47678
|
-
);
|
|
47679
|
-
if (cfg.asJson) {
|
|
47680
|
-
writeJSONIndented(cfg.out, stored);
|
|
47681
|
-
return;
|
|
47682
|
-
}
|
|
47683
|
-
renderPoll(cfg.out, "Polling enabled", binding, stored);
|
|
47684
|
-
}
|
|
47685
|
-
function windowStart(since, lookbackDays, now) {
|
|
47686
|
-
const start = now - lookbackDays * 864e5;
|
|
47687
|
-
return since === null ? start : Math.max(start, Date.parse(since));
|
|
47688
|
-
}
|
|
47689
|
-
function widening(current, changes, now) {
|
|
47690
|
-
if (changes.since === void 0 && changes.lookback_days === void 0) return null;
|
|
47691
|
-
const from = windowStart(current.since, current.lookback_days, now);
|
|
47692
|
-
const to = windowStart(
|
|
47693
|
-
changes.since ?? current.since,
|
|
47694
|
-
changes.lookback_days ?? current.lookback_days,
|
|
47695
|
-
now
|
|
47696
|
-
);
|
|
47697
|
-
if (to >= from) return null;
|
|
47698
|
-
return { from: new Date(from).toISOString(), to: new Date(to).toISOString() };
|
|
47699
|
-
}
|
|
47700
|
-
async function ask(cfg, message, choices, defaultChoice) {
|
|
47701
|
-
const answer = await cfg.prompt(message, choices, defaultChoice);
|
|
47702
|
-
if (answer === null) throw new Error("poll set cancelled; nothing was changed");
|
|
47703
|
-
return parseChoice(answer, "the answer", choices);
|
|
47704
|
-
}
|
|
47705
|
-
var WIDEN_HELP = "seed = mark the newly exposed items as already landed, land = land them all";
|
|
47706
|
-
var POLICY_HELP = "rekey = re-key the ledger from a fresh listing (nothing lands again), seed = mark everything in the window as landed, land = land everything in the window";
|
|
47707
|
-
function pollSetConfirmVerb(widen, onPolicyChange) {
|
|
47708
|
-
return widen === "land" || onPolicyChange === "land" ? "set-land" : "set-seed";
|
|
47709
|
-
}
|
|
47710
|
-
async function runPollSet(cfg) {
|
|
47711
|
-
if (Object.keys(cfg.changes).length === 0) {
|
|
47712
|
-
throw new Error(
|
|
47713
|
-
"poll set: pass at least one of --interval, --since, --lookback, --settle, --same-name"
|
|
47714
|
-
);
|
|
47715
|
-
}
|
|
47716
|
-
const binding = await resolvePollBinding(cfg.client, cfg.filebox, cfg.signal);
|
|
47717
|
-
const current = await explainNotPolled(
|
|
47718
|
-
cfg.filebox,
|
|
47719
|
-
getPoll(cfg.client, binding.query_id, { signal: cfg.signal })
|
|
47720
|
-
);
|
|
47721
|
-
const body = { ...cfg.changes };
|
|
47722
|
-
const detail = [`target: ${bindingTarget(binding)}`];
|
|
47723
|
-
const wider = widening(current, cfg.changes, cfg.now ?? Date.now());
|
|
47724
|
-
if (wider !== null) {
|
|
47725
|
-
const what = `the polling window widens (start ${wider.from} -> ${wider.to})`;
|
|
47726
|
-
if (cfg.widen === void 0 && !cfg.interactive) {
|
|
47727
|
-
throw new Error(`${what}; pass --widen seed or --widen land (${WIDEN_HELP})`);
|
|
47728
|
-
}
|
|
47729
|
-
body.widen = cfg.widen ?? await ask(cfg, `${what}. ${WIDEN_HELP}.`, WIDEN_CHOICES);
|
|
47730
|
-
detail.push(`window: start ${wider.from} -> ${wider.to}, widen ${body.widen}`);
|
|
47731
|
-
} else if (cfg.widen !== void 0) {
|
|
47732
|
-
body.widen = cfg.widen;
|
|
47733
|
-
}
|
|
47734
|
-
const newPolicy = cfg.changes.change_policy;
|
|
47735
|
-
const policySwitch = newPolicy !== void 0 && newPolicy !== current.change_policy;
|
|
47736
|
-
if (policySwitch) {
|
|
47737
|
-
const change = `${sameNameLabel(current.change_policy)} -> ${sameNameLabel(newPolicy)}`;
|
|
47738
|
-
const what = `--same-name changes ${change}`;
|
|
47739
|
-
if (cfg.onPolicyChange === void 0 && !cfg.interactive) {
|
|
47740
|
-
throw new Error(`${what}; pass --on-policy-change rekey, seed or land (${POLICY_HELP})`);
|
|
47741
|
-
}
|
|
47742
|
-
body.on_policy_change = cfg.onPolicyChange ?? await ask(cfg, `${what}. ${POLICY_HELP}.`, POLICY_CHOICES, "rekey");
|
|
47743
|
-
detail.push(`same name: ${change}, on policy change ${body.on_policy_change}`);
|
|
47744
|
-
} else if (cfg.onPolicyChange !== void 0) {
|
|
47745
|
-
body.on_policy_change = cfg.onPolicyChange;
|
|
47746
|
-
}
|
|
47747
|
-
const put = async () => {
|
|
47748
|
-
const stored = await putPoll(cfg.client, binding.query_id, body, {
|
|
47749
|
-
signal: cfg.signal
|
|
47750
|
-
});
|
|
47751
|
-
if (cfg.asJson) {
|
|
47752
|
-
writeJSONIndented(cfg.out, stored);
|
|
47753
|
-
return;
|
|
47754
|
-
}
|
|
47755
|
-
renderPoll(cfg.out, "Polling settings updated", binding, stored);
|
|
47756
|
-
};
|
|
47757
|
-
const landsOrHides = wider !== null && body.widen === "land" || policySwitch && (body.on_policy_change === "seed" || body.on_policy_change === "land");
|
|
47758
|
-
if (!landsOrHides) {
|
|
47759
|
-
await put();
|
|
47760
|
-
return;
|
|
47761
|
-
}
|
|
47762
|
-
const flags = `${body.widen === void 0 ? "" : ` --widen ${body.widen}`}${body.on_policy_change === void 0 ? "" : ` --on-policy-change ${body.on_policy_change}`}`;
|
|
47763
|
-
cfg.deadline?.pause();
|
|
47764
|
-
await twoStepConfirm({
|
|
47765
|
-
out: cfg.out,
|
|
47766
|
-
action: pollConfirmAction(
|
|
47767
|
-
cfg.filebox,
|
|
47768
|
-
pollSetConfirmVerb(body.widen, body.on_policy_change),
|
|
47769
|
-
cfg.scope,
|
|
47770
|
-
{ ...body }
|
|
47771
|
-
),
|
|
47772
|
-
token: cfg.token,
|
|
47773
|
-
server: cfg.scope.serverKey,
|
|
47774
|
-
confirmCmd: `dr connectors poll set ${cfg.filebox}${cfg.settingsArgs}${flags}${cfg.targetArgs}`,
|
|
47775
|
-
describe: async () => detail,
|
|
47776
|
-
execute: async () => {
|
|
47777
|
-
cfg.deadline?.resume();
|
|
47778
|
-
await put();
|
|
47779
|
-
}
|
|
47780
|
-
});
|
|
47781
|
-
}
|
|
47782
|
-
function confirmDigest(scope, payload) {
|
|
47783
|
-
const canonical = JSON.stringify({
|
|
47784
|
-
server: scope.serverKey,
|
|
47785
|
-
account: scope.account ?? "",
|
|
47786
|
-
controlPlane: scope.controlPlane,
|
|
47787
|
-
payload: Object.fromEntries(Object.entries(payload).sort(([a], [b]) => a.localeCompare(b)))
|
|
47788
|
-
});
|
|
47789
|
-
return createHash12("sha256").update(canonical).digest("hex").slice(0, 12);
|
|
47790
|
-
}
|
|
47791
|
-
function pollConfirmAction(filebox, verb, scope, payload = {}) {
|
|
47792
|
-
return {
|
|
47793
|
-
resource: "connector-poll",
|
|
47794
|
-
id: `${filebox.trim()}#${confirmDigest(scope, payload)}`,
|
|
47795
|
-
verb
|
|
47796
|
-
};
|
|
47797
|
-
}
|
|
47798
|
-
function pollItemConfirmAction(filebox, item, scope) {
|
|
47799
|
-
return {
|
|
47800
|
-
resource: "connector-poll-item",
|
|
47801
|
-
id: `${filebox.trim()}/${item}#${confirmDigest(scope, {})}`,
|
|
47802
|
-
verb: "skip"
|
|
47803
|
-
};
|
|
47804
|
-
}
|
|
47805
|
-
async function runPollDisable(cfg) {
|
|
47806
|
-
const binding = await resolvePollBinding(cfg.client, cfg.filebox, cfg.signal);
|
|
47807
|
-
await explainNotPolled(
|
|
47808
|
-
cfg.filebox,
|
|
47809
|
-
getPoll(cfg.client, binding.query_id, { signal: cfg.signal })
|
|
47810
|
-
);
|
|
47811
|
-
const stored = await putPoll(
|
|
47812
|
-
cfg.client,
|
|
47813
|
-
binding.query_id,
|
|
47814
|
-
{ enabled: false },
|
|
47815
|
-
{ signal: cfg.signal }
|
|
47816
|
-
);
|
|
47817
|
-
if (cfg.asJson) {
|
|
47818
|
-
writeJSONIndented(cfg.out, stored);
|
|
47819
|
-
return;
|
|
47820
|
-
}
|
|
47821
|
-
renderPoll(cfg.out, "Polling disabled", binding, stored);
|
|
47822
|
-
}
|
|
47823
|
-
async function runPollGet(cfg) {
|
|
47824
|
-
const binding = await resolvePollBinding(cfg.client, cfg.filebox, cfg.signal);
|
|
47825
|
-
const p = await explainNotPolled(
|
|
47826
|
-
cfg.filebox,
|
|
47827
|
-
getPoll(cfg.client, binding.query_id, { signal: cfg.signal })
|
|
47828
|
-
);
|
|
47829
|
-
if (cfg.asJson) {
|
|
47830
|
-
writeJSONIndented(cfg.out, p);
|
|
47831
|
-
return;
|
|
47832
|
-
}
|
|
47833
|
-
renderPoll(cfg.out, "Polling", binding, p);
|
|
47834
|
-
}
|
|
47835
|
-
async function runPollRun(cfg) {
|
|
47836
|
-
const binding = await resolvePollBinding(cfg.client, cfg.filebox, cfg.signal);
|
|
47837
|
-
let p;
|
|
47838
|
-
try {
|
|
47839
|
-
p = await explainNotPolled(
|
|
47840
|
-
cfg.filebox,
|
|
47841
|
-
runPoll(cfg.client, binding.query_id, { signal: cfg.signal })
|
|
47842
|
-
);
|
|
47843
|
-
} catch (err) {
|
|
47844
|
-
if (isPollThrottled(err)) {
|
|
47845
|
-
throw new Error(
|
|
47846
|
-
`a poll of filebox ${cfg.filebox} is running or ran less than a minute ago; try again shortly`,
|
|
47847
|
-
{ cause: err }
|
|
47848
|
-
);
|
|
47849
|
-
}
|
|
47850
|
-
throw err;
|
|
47851
|
-
}
|
|
47852
|
-
if (cfg.asJson) {
|
|
47853
|
-
writeJSONIndented(cfg.out, p);
|
|
47854
|
-
return;
|
|
47855
|
-
}
|
|
47856
|
-
renderPoll(cfg.out, "Poll requested", binding, p);
|
|
47857
|
-
}
|
|
47858
|
-
async function runPollResume(cfg) {
|
|
47859
|
-
const binding = await resolvePollBinding(cfg.client, cfg.filebox, cfg.signal);
|
|
47860
|
-
const resume = async () => {
|
|
47861
|
-
const p = await explainNotPolled(
|
|
47862
|
-
cfg.filebox,
|
|
47863
|
-
resumePoll(
|
|
47864
|
-
cfg.client,
|
|
47865
|
-
binding.query_id,
|
|
47866
|
-
cfg.source === void 0 ? {} : { source: cfg.source },
|
|
47867
|
-
{ signal: cfg.signal }
|
|
47868
|
-
)
|
|
47869
|
-
);
|
|
47870
|
-
if (cfg.asJson) {
|
|
47871
|
-
writeJSONIndented(cfg.out, p);
|
|
47872
|
-
return;
|
|
47873
|
-
}
|
|
47874
|
-
renderPoll(cfg.out, "Polling resumed", binding, p);
|
|
47875
|
-
};
|
|
47876
|
-
if (cfg.source !== "land") {
|
|
47877
|
-
await resume();
|
|
47878
|
-
return;
|
|
47879
|
-
}
|
|
47880
|
-
cfg.deadline?.pause();
|
|
47881
|
-
await twoStepConfirm({
|
|
47882
|
-
out: cfg.out,
|
|
47883
|
-
action: pollConfirmAction(cfg.filebox, "resume-land", cfg.scope),
|
|
47884
|
-
token: cfg.token,
|
|
47885
|
-
server: cfg.scope.serverKey,
|
|
47886
|
-
confirmCmd: `dr connectors poll resume ${cfg.filebox} --source land${cfg.targetArgs}`,
|
|
47887
|
-
describe: async () => [
|
|
47888
|
-
`target: ${bindingTarget(binding)}`,
|
|
47889
|
-
"source: land every item the changed source holds now"
|
|
47890
|
-
],
|
|
47891
|
-
execute: async () => {
|
|
47892
|
-
cfg.deadline?.resume();
|
|
47893
|
-
await resume();
|
|
47894
|
-
}
|
|
47895
|
-
});
|
|
47896
|
-
}
|
|
47897
|
-
async function countSeedable(cfg, binding) {
|
|
47898
|
-
const untilMs = Date.parse(cfg.until);
|
|
47899
|
-
let count = 0;
|
|
47900
|
-
for (const state of ["seen", "new"]) {
|
|
47901
|
-
let after = null;
|
|
47902
|
-
do {
|
|
47903
|
-
cfg.deadline?.resume();
|
|
47904
|
-
const page = await explainNotPolled(
|
|
47905
|
-
cfg.filebox,
|
|
47906
|
-
listPollItems(
|
|
47907
|
-
cfg.client,
|
|
47908
|
-
binding.query_id,
|
|
47909
|
-
{ state, ...after === null ? {} : { after } },
|
|
47910
|
-
{ signal: cfg.signal }
|
|
47911
|
-
)
|
|
47912
|
-
);
|
|
47913
|
-
count += page.items.filter(
|
|
47914
|
-
(it) => it.source_modified_at !== null && Date.parse(it.source_modified_at) <= untilMs
|
|
47915
|
-
).length;
|
|
47916
|
-
after = page.next_after;
|
|
47917
|
-
} while (after !== null);
|
|
47918
|
-
}
|
|
47919
|
-
return count;
|
|
47920
|
-
}
|
|
47921
|
-
async function runPollSeed(cfg) {
|
|
47922
|
-
const binding = await resolvePollBinding(cfg.client, cfg.filebox, cfg.signal);
|
|
47923
|
-
if (cfg.dryRun) {
|
|
47924
|
-
const wouldSeed = await countSeedable(cfg, binding);
|
|
47925
|
-
if (cfg.asJson) {
|
|
47926
|
-
writeJSONIndented(cfg.out, { dry_run: true, until: cfg.until, would_seed: wouldSeed });
|
|
47927
|
-
return;
|
|
47928
|
-
}
|
|
47929
|
-
cfg.out.write(`Seed dry run for ${bindingTarget(binding)}: nothing was changed.
|
|
47930
|
-
`);
|
|
47931
|
-
cfg.out.write(
|
|
47932
|
-
`${wouldSeed} already-discovered items would be seeded (items not yet discovered are not counted).
|
|
47933
|
-
`
|
|
47934
|
-
);
|
|
47935
|
-
return;
|
|
47936
|
-
}
|
|
47937
|
-
cfg.deadline?.pause();
|
|
47938
|
-
await twoStepConfirm({
|
|
47939
|
-
out: cfg.out,
|
|
47940
|
-
action: pollConfirmAction(cfg.filebox, "seed", cfg.scope, { until: cfg.until }),
|
|
47941
|
-
token: cfg.token,
|
|
47942
|
-
server: cfg.scope.serverKey,
|
|
47943
|
-
confirmCmd: `dr connectors poll seed ${cfg.filebox} --until ${cfg.untilArg}${cfg.targetArgs}`,
|
|
47944
|
-
describe: async () => [
|
|
47945
|
-
`target: ${bindingTarget(binding)}`,
|
|
47946
|
-
`until: ${cfg.until}`,
|
|
47947
|
-
"effect: items modified at or before it are marked landed and never land"
|
|
47948
|
-
],
|
|
47949
|
-
execute: async () => {
|
|
47950
|
-
cfg.deadline?.resume();
|
|
47951
|
-
const p = await explainNotPolled(
|
|
47952
|
-
cfg.filebox,
|
|
47953
|
-
seedPoll(
|
|
47954
|
-
cfg.client,
|
|
47955
|
-
binding.query_id,
|
|
47956
|
-
{ until: cfg.until },
|
|
47957
|
-
{
|
|
47958
|
-
signal: cfg.signal
|
|
47959
|
-
}
|
|
47960
|
-
)
|
|
47961
|
-
);
|
|
47962
|
-
if (cfg.asJson) {
|
|
47963
|
-
writeJSONIndented(cfg.out, p);
|
|
47964
|
-
return;
|
|
47965
|
-
}
|
|
47966
|
-
renderPoll(cfg.out, "Seed requested", binding, p);
|
|
47967
|
-
}
|
|
47968
|
-
});
|
|
47969
|
-
}
|
|
47970
|
-
async function runPollItems(cfg) {
|
|
47971
|
-
const binding = await resolvePollBinding(cfg.client, cfg.filebox, cfg.signal);
|
|
47972
|
-
const page = await explainNotPolled(
|
|
47973
|
-
cfg.filebox,
|
|
47974
|
-
listPollItems(
|
|
47975
|
-
cfg.client,
|
|
47976
|
-
binding.query_id,
|
|
47977
|
-
{
|
|
47978
|
-
...cfg.state !== void 0 ? { state: cfg.state } : {},
|
|
47979
|
-
...cfg.limit !== void 0 ? { limit: cfg.limit } : {},
|
|
47980
|
-
...cfg.after !== void 0 ? { after: cfg.after } : {}
|
|
47981
|
-
},
|
|
47982
|
-
{ signal: cfg.signal }
|
|
47983
|
-
)
|
|
47984
|
-
);
|
|
47985
|
-
if (cfg.asJson) {
|
|
47986
|
-
writeJSONIndented(cfg.out, page);
|
|
47987
|
-
return;
|
|
47988
|
-
}
|
|
47989
|
-
if (page.items.length === 0) {
|
|
47990
|
-
cfg.out.write(`No items${cfg.state === void 0 ? "" : ` in state ${cfg.state}`}.
|
|
47991
|
-
`);
|
|
47992
|
-
return;
|
|
47993
|
-
}
|
|
47994
|
-
renderItemTable(cfg.out, page.items);
|
|
47995
|
-
if (page.next_after !== null) {
|
|
47996
|
-
const state = cfg.state === void 0 ? "" : ` --state ${cfg.state}`;
|
|
47997
|
-
const limit = cfg.limit === void 0 ? "" : ` --limit ${cfg.limit}`;
|
|
47998
|
-
cfg.out.write(
|
|
47999
|
-
`
|
|
48000
|
-
More: dr connectors poll items ${cfg.filebox}${state}${limit} --after ${page.next_after}
|
|
48001
|
-
`
|
|
48002
|
-
);
|
|
48003
|
-
}
|
|
48004
|
-
}
|
|
48005
|
-
function renderItemResult(cfg, binding, headline, item) {
|
|
48006
|
-
if (cfg.asJson) {
|
|
48007
|
-
writeJSONIndented(cfg.out, item);
|
|
48008
|
-
return;
|
|
48009
|
-
}
|
|
48010
|
-
cfg.out.write(`${headline} (${bindingTarget(binding)})
|
|
48011
|
-
`);
|
|
48012
|
-
renderItemTable(cfg.out, [item]);
|
|
48013
|
-
}
|
|
48014
|
-
async function runPollRetry(cfg) {
|
|
48015
|
-
const binding = await resolvePollBinding(cfg.client, cfg.filebox, cfg.signal);
|
|
48016
|
-
const item = await explainNotPolled(
|
|
48017
|
-
cfg.filebox,
|
|
48018
|
-
retryPollItem(cfg.client, binding.query_id, cfg.itemId, { signal: cfg.signal })
|
|
48019
|
-
);
|
|
48020
|
-
renderItemResult(cfg, binding, `Item ${cfg.itemId} queued for retry`, item);
|
|
48021
|
-
}
|
|
48022
|
-
async function runPollSkip(cfg) {
|
|
48023
|
-
const binding = await resolvePollBinding(cfg.client, cfg.filebox, cfg.signal);
|
|
48024
|
-
cfg.deadline?.pause();
|
|
48025
|
-
await twoStepConfirm({
|
|
48026
|
-
out: cfg.out,
|
|
48027
|
-
action: pollItemConfirmAction(cfg.filebox, cfg.itemId, cfg.scope),
|
|
48028
|
-
token: cfg.token,
|
|
48029
|
-
server: cfg.scope.serverKey,
|
|
48030
|
-
confirmCmd: `dr connectors poll skip ${cfg.filebox} ${cfg.itemId}${cfg.targetArgs}`,
|
|
48031
|
-
describe: async () => [
|
|
48032
|
-
`item: ${cfg.itemId} of ${bindingTarget(binding)}`,
|
|
48033
|
-
"effect: the item is never landed (dr connectors poll retry queues it again)"
|
|
48034
|
-
],
|
|
48035
|
-
execute: async () => {
|
|
48036
|
-
cfg.deadline?.resume();
|
|
48037
|
-
const item = await explainNotPolled(
|
|
48038
|
-
cfg.filebox,
|
|
48039
|
-
skipPollItem(cfg.client, binding.query_id, cfg.itemId, {
|
|
48040
|
-
signal: cfg.signal
|
|
48041
|
-
})
|
|
48042
|
-
);
|
|
48043
|
-
renderItemResult(cfg, binding, `Item ${cfg.itemId} skipped`, item);
|
|
48044
|
-
}
|
|
48045
|
-
});
|
|
48046
|
-
}
|
|
48047
|
-
async function withMonolith(opts, action) {
|
|
48048
|
-
const monolithHost = resolve(resolveServer(opts.server)).host;
|
|
48049
|
-
const setup = await setupCall(opts.server, opts.account, {
|
|
48050
|
-
host: monolithHost,
|
|
48051
|
-
authHost: monolithHost
|
|
48052
|
-
});
|
|
48053
|
-
try {
|
|
48054
|
-
await action(setup.client);
|
|
48055
|
-
} finally {
|
|
48056
|
-
setup.cleanup();
|
|
48057
|
-
}
|
|
48058
|
-
}
|
|
48059
|
-
function scopeOf(opts) {
|
|
48060
|
-
const serverKey = resolveServer(opts.server);
|
|
48061
|
-
return {
|
|
48062
|
-
serverKey,
|
|
48063
|
-
account: normaliseAccount(opts.account),
|
|
48064
|
-
controlPlane: resolveConnectorsUrl(serverKey, opts.connectorsUrl)
|
|
48065
|
-
};
|
|
48066
|
-
}
|
|
48067
|
-
function targetArgsOf(opts) {
|
|
48068
|
-
const url = (opts.connectorsUrl ?? "").trim();
|
|
48069
|
-
return ` --server ${resolveServer(opts.server)}${url === "" ? "" : ` --connectors-url ${url}`}`;
|
|
48070
|
-
}
|
|
48071
|
-
function addCommonFlags(cmd, jsonHelp) {
|
|
48072
|
-
cmd.option("--json", jsonHelp, false);
|
|
48073
|
-
addServerFlag(cmd);
|
|
48074
|
-
addAccountFlag(cmd);
|
|
48075
|
-
addConnectorsUrlFlag(cmd);
|
|
48076
|
-
}
|
|
48077
|
-
function addConfirmTokenFlag(cmd) {
|
|
48078
|
-
return cmd.option("--confirm-token <token>", "Token from a prior plan run \u2014 required to proceed");
|
|
48079
|
-
}
|
|
48080
|
-
function addSettingsFlags(cmd) {
|
|
48081
|
-
return cmd.option("--interval <duration>", "how often to poll, 5m to 7d (default: per source)").option("--since <date>", "never pick up items modified before this date or timestamp").option("--lookback <days>", "window every poll lists, 1d to 3650d (default: per source)").option("--settle <duration>", "how long an item must stay unchanged before landing, 0s to 1d").option(
|
|
48082
|
-
"--same-name <policy>",
|
|
48083
|
-
"a changed file under the same name: new-version (lands again) or ignore"
|
|
47245
|
+
function addSettingsFlags(cmd) {
|
|
47246
|
+
return cmd.option("--interval <duration>", "how often to poll, 5m to 7d (default: per source)").option("--since <date>", "never pick up items modified before this date or timestamp").option("--lookback <days>", "window every poll lists, 1d to 3650d (default: per source)").option("--settle <duration>", "how long an item must stay unchanged before landing, 0s to 1d").option(
|
|
47247
|
+
"--same-name <policy>",
|
|
47248
|
+
"a changed file under the same name: new-version (lands again) or ignore"
|
|
48084
47249
|
);
|
|
48085
47250
|
}
|
|
48086
47251
|
function ttyChoicePrompt(message, choices, defaultChoice, io = { input: process.stdin, output: process.stderr }) {
|
|
@@ -48387,6 +47552,996 @@ ${POLL_CONFIRM_HELP}`)
|
|
|
48387
47552
|
addCommonFlags(seedCmd, "Output the binding (or the dry-run count) as JSON");
|
|
48388
47553
|
}
|
|
48389
47554
|
|
|
47555
|
+
// src/commands/connectorsTasks.ts
|
|
47556
|
+
import { readFileSync as readFileSync3 } from "fs";
|
|
47557
|
+
|
|
47558
|
+
// src/api/scheduledTasks.ts
|
|
47559
|
+
var BASE2 = "/scheduled-tasks";
|
|
47560
|
+
async function call2(method, path13, send, what) {
|
|
47561
|
+
let out;
|
|
47562
|
+
try {
|
|
47563
|
+
out = await send();
|
|
47564
|
+
} catch (err) {
|
|
47565
|
+
throw toControlPlaneError(method, path13, err);
|
|
47566
|
+
}
|
|
47567
|
+
if (out === null) throw new Error(`${method} ${path13}: empty response from control-plane ${what}`);
|
|
47568
|
+
return out;
|
|
47569
|
+
}
|
|
47570
|
+
async function createTask(client, fields, reqOptions) {
|
|
47571
|
+
return call2(
|
|
47572
|
+
"POST",
|
|
47573
|
+
BASE2,
|
|
47574
|
+
() => client.postJSON(BASE2, fields, reqOptions),
|
|
47575
|
+
"task create"
|
|
47576
|
+
);
|
|
47577
|
+
}
|
|
47578
|
+
async function listTasks(client, orgId, reqOptions) {
|
|
47579
|
+
const path13 = orgId === void 0 ? BASE2 : `${BASE2}?org_id=${orgId}`;
|
|
47580
|
+
return call2("GET", path13, () => client.getJSON(path13, reqOptions), "task list");
|
|
47581
|
+
}
|
|
47582
|
+
async function getTask(client, taskId, reqOptions) {
|
|
47583
|
+
const path13 = `/scheduled-tasks/${encodeURIComponent(taskId)}`;
|
|
47584
|
+
return call2("GET", path13, () => client.getJSON(path13, reqOptions), "task get");
|
|
47585
|
+
}
|
|
47586
|
+
async function updateTask(client, taskId, fields, reqOptions) {
|
|
47587
|
+
const path13 = `/scheduled-tasks/${encodeURIComponent(taskId)}`;
|
|
47588
|
+
return call2(
|
|
47589
|
+
"PATCH",
|
|
47590
|
+
path13,
|
|
47591
|
+
() => client.patchJSON(path13, fields, reqOptions),
|
|
47592
|
+
"task update"
|
|
47593
|
+
);
|
|
47594
|
+
}
|
|
47595
|
+
async function deleteTask(client, taskId, reqOptions) {
|
|
47596
|
+
const path13 = `/scheduled-tasks/${encodeURIComponent(taskId)}`;
|
|
47597
|
+
try {
|
|
47598
|
+
await client.deleteJSON(path13, reqOptions);
|
|
47599
|
+
} catch (err) {
|
|
47600
|
+
throw toControlPlaneError("DELETE", path13, err);
|
|
47601
|
+
}
|
|
47602
|
+
}
|
|
47603
|
+
async function runTask(client, taskId, reqOptions) {
|
|
47604
|
+
const path13 = `/scheduled-tasks/${encodeURIComponent(taskId)}/run`;
|
|
47605
|
+
return call2(
|
|
47606
|
+
"POST",
|
|
47607
|
+
path13,
|
|
47608
|
+
() => client.postJSON(path13, {}, reqOptions),
|
|
47609
|
+
"task run"
|
|
47610
|
+
);
|
|
47611
|
+
}
|
|
47612
|
+
async function pauseTask(client, taskId, reqOptions) {
|
|
47613
|
+
const path13 = `/scheduled-tasks/${encodeURIComponent(taskId)}/pause`;
|
|
47614
|
+
return call2(
|
|
47615
|
+
"POST",
|
|
47616
|
+
path13,
|
|
47617
|
+
() => client.postJSON(path13, {}, reqOptions),
|
|
47618
|
+
"task pause"
|
|
47619
|
+
);
|
|
47620
|
+
}
|
|
47621
|
+
async function resumeTask(client, taskId, reqOptions) {
|
|
47622
|
+
const path13 = `/scheduled-tasks/${encodeURIComponent(taskId)}/resume`;
|
|
47623
|
+
return call2(
|
|
47624
|
+
"POST",
|
|
47625
|
+
path13,
|
|
47626
|
+
() => client.postJSON(path13, {}, reqOptions),
|
|
47627
|
+
"task resume"
|
|
47628
|
+
);
|
|
47629
|
+
}
|
|
47630
|
+
async function listRuns(client, taskId, limit, reqOptions) {
|
|
47631
|
+
const path13 = `/scheduled-tasks/${encodeURIComponent(taskId)}/runs?limit=${limit}`;
|
|
47632
|
+
return call2(
|
|
47633
|
+
"GET",
|
|
47634
|
+
path13,
|
|
47635
|
+
() => client.getJSON(path13, reqOptions),
|
|
47636
|
+
"task runs"
|
|
47637
|
+
);
|
|
47638
|
+
}
|
|
47639
|
+
|
|
47640
|
+
// src/commands/connectorsTasks.ts
|
|
47641
|
+
var DURATION_UNIT_SECONDS2 = { s: 1, m: 60, h: 3600, d: 86400 };
|
|
47642
|
+
var MIN_EVERY_SECONDS = 60;
|
|
47643
|
+
var MAX_EVERY_SECONDS = 2 ** 31 - 1;
|
|
47644
|
+
function parseEvery(raw2) {
|
|
47645
|
+
const m = /^(\d+)([smhd])?$/.exec(raw2.trim());
|
|
47646
|
+
const seconds = m === null ? NaN : Number(m[1]) * (DURATION_UNIT_SECONDS2[m[2] ?? "s"] ?? 1);
|
|
47647
|
+
if (m === null)
|
|
47648
|
+
throw new Error(`invalid --every "${raw2}": expected a duration like 5m, 1h or 1d`);
|
|
47649
|
+
if (!Number.isSafeInteger(seconds) || seconds < MIN_EVERY_SECONDS || seconds > MAX_EVERY_SECONDS) {
|
|
47650
|
+
throw new Error(`invalid --every "${raw2}": must be at least 1m`);
|
|
47651
|
+
}
|
|
47652
|
+
return seconds;
|
|
47653
|
+
}
|
|
47654
|
+
function parseCron(raw2) {
|
|
47655
|
+
const fields = raw2.trim().split(/\s+/);
|
|
47656
|
+
if (fields.length !== 5 || fields.some((f) => f === "")) {
|
|
47657
|
+
throw new Error(`invalid --cron "${raw2}": expected five fields, e.g. "15 * * * *"`);
|
|
47658
|
+
}
|
|
47659
|
+
return fields.join(" ");
|
|
47660
|
+
}
|
|
47661
|
+
var ROLE_RE = /^[a-z][a-z_]*$/;
|
|
47662
|
+
var DATASOURCE_ID_RE = /^[0-9]{1,18}$/;
|
|
47663
|
+
function parseDatasources(values) {
|
|
47664
|
+
const out = {};
|
|
47665
|
+
for (const raw2 of values) {
|
|
47666
|
+
const eq = raw2.indexOf("=");
|
|
47667
|
+
const role = eq < 0 ? "" : raw2.slice(0, eq).trim();
|
|
47668
|
+
const id = eq < 0 ? "" : raw2.slice(eq + 1).trim();
|
|
47669
|
+
if (!ROLE_RE.test(role) || !DATASOURCE_ID_RE.test(id)) {
|
|
47670
|
+
throw new Error(
|
|
47671
|
+
`invalid --datasource "${raw2}": expected <role>=<datasource id>, e.g. target=4242 (role: lowercase letters and _; id: digits)`
|
|
47672
|
+
);
|
|
47673
|
+
}
|
|
47674
|
+
if (Object.hasOwn(out, role)) throw new Error(`--datasource role "${role}" given twice`);
|
|
47675
|
+
out[role] = id;
|
|
47676
|
+
}
|
|
47677
|
+
const count = Object.keys(out).length;
|
|
47678
|
+
if (count < 1 || count > 4) throw new Error("give between 1 and 4 --datasource roles");
|
|
47679
|
+
return out;
|
|
47680
|
+
}
|
|
47681
|
+
function parseConfig(raw2, readFile10 = readUtf8) {
|
|
47682
|
+
const text2 = raw2.startsWith("@") ? readFile10(raw2.slice(1)) : raw2;
|
|
47683
|
+
let parsed2;
|
|
47684
|
+
try {
|
|
47685
|
+
parsed2 = JSON.parse(text2);
|
|
47686
|
+
} catch (err) {
|
|
47687
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
47688
|
+
throw new Error(`invalid --config JSON: ${msg}`);
|
|
47689
|
+
}
|
|
47690
|
+
if (typeof parsed2 !== "object" || parsed2 === null || Array.isArray(parsed2)) {
|
|
47691
|
+
throw new Error("invalid --config: expected a JSON object");
|
|
47692
|
+
}
|
|
47693
|
+
return parsed2;
|
|
47694
|
+
}
|
|
47695
|
+
function readUtf8(p) {
|
|
47696
|
+
return readFileSync3(p, "utf-8");
|
|
47697
|
+
}
|
|
47698
|
+
function collect(value, previous = []) {
|
|
47699
|
+
return [...previous, value];
|
|
47700
|
+
}
|
|
47701
|
+
function parseScheduleFlags(opts) {
|
|
47702
|
+
if (opts.every !== void 0 && opts.cron !== void 0) {
|
|
47703
|
+
throw new Error("give --every or --cron, not both");
|
|
47704
|
+
}
|
|
47705
|
+
const fields = {};
|
|
47706
|
+
if (opts.artifact !== void 0) fields.artifact_name = opts.artifact;
|
|
47707
|
+
if (opts.flavor !== void 0) fields.runtime_flavor = opts.flavor;
|
|
47708
|
+
if (opts.datasource !== void 0) fields.datasources = parseDatasources(opts.datasource);
|
|
47709
|
+
if (opts.primaryRole !== void 0) fields.primary_role = opts.primaryRole;
|
|
47710
|
+
if (opts.config !== void 0) fields.config = parseConfig(opts.config);
|
|
47711
|
+
if (opts.every !== void 0) fields.every_seconds = parseEvery(opts.every);
|
|
47712
|
+
if (opts.cron !== void 0) fields.cron = parseCron(opts.cron);
|
|
47713
|
+
if (opts.tz !== void 0) fields.timezone = opts.tz;
|
|
47714
|
+
if (fields.primary_role !== void 0 && fields.datasources !== void 0) {
|
|
47715
|
+
if (!Object.hasOwn(fields.datasources, fields.primary_role)) {
|
|
47716
|
+
throw new Error(
|
|
47717
|
+
`--primary-role "${fields.primary_role}" is not one of the --datasource roles`
|
|
47718
|
+
);
|
|
47719
|
+
}
|
|
47720
|
+
}
|
|
47721
|
+
return fields;
|
|
47722
|
+
}
|
|
47723
|
+
function buildCreateFields(name, orgId, opts) {
|
|
47724
|
+
const missing = [
|
|
47725
|
+
["--artifact", opts.artifact],
|
|
47726
|
+
["--flavor", opts.flavor],
|
|
47727
|
+
["--datasource", opts.datasource],
|
|
47728
|
+
["--config", opts.config]
|
|
47729
|
+
].filter(([, v]) => v === void 0).map(([flag]) => flag);
|
|
47730
|
+
if (opts.every === void 0 && opts.cron === void 0) missing.push("--every or --cron");
|
|
47731
|
+
if (missing.length > 0) throw new Error(`missing ${missing.join(", ")}`);
|
|
47732
|
+
const fields = parseScheduleFlags(opts);
|
|
47733
|
+
return {
|
|
47734
|
+
org_id: orgId,
|
|
47735
|
+
name,
|
|
47736
|
+
artifact_name: fields.artifact_name,
|
|
47737
|
+
runtime_flavor: fields.runtime_flavor,
|
|
47738
|
+
datasources: fields.datasources,
|
|
47739
|
+
config: fields.config,
|
|
47740
|
+
...fields.primary_role !== void 0 ? { primary_role: fields.primary_role } : {},
|
|
47741
|
+
...fields.every_seconds !== void 0 ? { every_seconds: fields.every_seconds } : {},
|
|
47742
|
+
...fields.cron !== void 0 ? { cron: fields.cron } : {},
|
|
47743
|
+
...fields.timezone !== void 0 ? { timezone: fields.timezone } : {},
|
|
47744
|
+
...opts.disabled === true ? { enabled: false } : {}
|
|
47745
|
+
};
|
|
47746
|
+
}
|
|
47747
|
+
var UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
47748
|
+
async function resolveTaskId(client, task, signal) {
|
|
47749
|
+
if (UUID_RE.test(task)) return task;
|
|
47750
|
+
const tasks = await listTasks(client, void 0, { signal });
|
|
47751
|
+
const match = tasks.find((t) => t.name === task);
|
|
47752
|
+
if (match === void 0) {
|
|
47753
|
+
throw new Error(
|
|
47754
|
+
`no scheduled task named "${task}" in this org (see: dr connectors tasks list)`
|
|
47755
|
+
);
|
|
47756
|
+
}
|
|
47757
|
+
return match.id;
|
|
47758
|
+
}
|
|
47759
|
+
function explainTaskError(err) {
|
|
47760
|
+
const forbidden = err instanceof ControlPlaneError && err.status === 403 || err instanceof Error && /\bHTTP 403\b/.test(err.message);
|
|
47761
|
+
if (forbidden) {
|
|
47762
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
47763
|
+
const body = err instanceof ControlPlaneError ? err.body : (/HTTP 403[^:]*(?:\([^)]*\))?:\s*([\s\S]*)$/.exec(message)?.[1] ?? "").trim();
|
|
47764
|
+
return withCause(
|
|
47765
|
+
new ControlPlaneError(
|
|
47766
|
+
403,
|
|
47767
|
+
body,
|
|
47768
|
+
`${message}
|
|
47769
|
+
Scheduled tasks need a support user or an org admin: log in as one (--account) \u2014 e.g. dr login-as the customer's support user.`
|
|
47770
|
+
),
|
|
47771
|
+
err
|
|
47772
|
+
);
|
|
47773
|
+
}
|
|
47774
|
+
if (!(err instanceof ControlPlaneError)) return err;
|
|
47775
|
+
const detail = detailOf(err.body);
|
|
47776
|
+
if (err.status === 404) {
|
|
47777
|
+
const message = `scheduled task not found (${detail || err.message})`;
|
|
47778
|
+
return withCause(new ControlPlaneError(404, err.body, message), err);
|
|
47779
|
+
}
|
|
47780
|
+
if (err.status === 409 || err.status === 422) {
|
|
47781
|
+
return withCause(new ControlPlaneError(err.status, err.body, detail || err.message), err);
|
|
47782
|
+
}
|
|
47783
|
+
return err;
|
|
47784
|
+
}
|
|
47785
|
+
function withCause(error, cause) {
|
|
47786
|
+
Object.defineProperty(error, "cause", { value: cause, enumerable: false, configurable: true });
|
|
47787
|
+
return error;
|
|
47788
|
+
}
|
|
47789
|
+
function detailOf(body) {
|
|
47790
|
+
try {
|
|
47791
|
+
const parsed2 = JSON.parse(body);
|
|
47792
|
+
const d = parsed2.detail;
|
|
47793
|
+
if (typeof d === "string") return d;
|
|
47794
|
+
if (Array.isArray(d)) {
|
|
47795
|
+
return d.map(
|
|
47796
|
+
(e) => `${Array.isArray(e.loc) ? e.loc.slice(1).join(".") : ""}: ${e.msg ?? ""}`.trim()
|
|
47797
|
+
).join("; ");
|
|
47798
|
+
}
|
|
47799
|
+
} catch {
|
|
47800
|
+
}
|
|
47801
|
+
return body;
|
|
47802
|
+
}
|
|
47803
|
+
function scheduleText(t) {
|
|
47804
|
+
if (t.schedule_kind === "cron") return `cron "${t.cron ?? ""}" ${t.timezone}`;
|
|
47805
|
+
const s = t.every_seconds ?? 0;
|
|
47806
|
+
const unit = s % 86400 === 0 ? `${s / 86400}d` : s % 3600 === 0 ? `${s / 3600}h` : s % 60 === 0 ? `${s / 60}m` : `${s}s`;
|
|
47807
|
+
return `every ${unit}`;
|
|
47808
|
+
}
|
|
47809
|
+
function stateText(t) {
|
|
47810
|
+
if (!t.enabled) return "disabled";
|
|
47811
|
+
if (t.paused_reason !== null) return `paused (${t.paused_reason})`;
|
|
47812
|
+
return t.running ? "running" : "enabled";
|
|
47813
|
+
}
|
|
47814
|
+
function writeTask(out, t) {
|
|
47815
|
+
const roles = Object.entries(t.datasources).map(([role, id]) => `${role}=${id}${role === t.primary_role ? " (primary)" : ""}`).join(", ");
|
|
47816
|
+
out.write(`Task: ${t.name} (${t.id})
|
|
47817
|
+
`);
|
|
47818
|
+
out.write(` Org: ${t.org_id}
|
|
47819
|
+
`);
|
|
47820
|
+
out.write(` State: ${stateText(t)}
|
|
47821
|
+
`);
|
|
47822
|
+
out.write(` Artifact: ${t.artifact_name} [${t.runtime_flavor}]
|
|
47823
|
+
`);
|
|
47824
|
+
out.write(` Schedule: ${scheduleText(t)}
|
|
47825
|
+
`);
|
|
47826
|
+
out.write(` Next run: ${t.next_run_at}
|
|
47827
|
+
`);
|
|
47828
|
+
out.write(` Last run: ${t.last_run_at ?? "-"} ${t.last_run_status ?? ""}
|
|
47829
|
+
`);
|
|
47830
|
+
if (t.consecutive_failures > 0) out.write(` Failures: ${t.consecutive_failures} in a row
|
|
47831
|
+
`);
|
|
47832
|
+
out.write(` Datasources: ${roles}
|
|
47833
|
+
`);
|
|
47834
|
+
out.write(` Config: ${truncate3(JSON.stringify(t.config), 200)}
|
|
47835
|
+
`);
|
|
47836
|
+
}
|
|
47837
|
+
function writeTaskList(out, tasks) {
|
|
47838
|
+
if (tasks.length === 0) {
|
|
47839
|
+
out.write("No scheduled tasks.\n");
|
|
47840
|
+
return;
|
|
47841
|
+
}
|
|
47842
|
+
for (const t of tasks) {
|
|
47843
|
+
out.write(
|
|
47844
|
+
`${t.name.padEnd(28)} ${stateText(t).padEnd(20)} ${scheduleText(t).padEnd(26)} next ${t.next_run_at} last ${t.last_run_status ?? "-"} ${t.id}
|
|
47845
|
+
`
|
|
47846
|
+
);
|
|
47847
|
+
}
|
|
47848
|
+
}
|
|
47849
|
+
function writeRuns(out, runs) {
|
|
47850
|
+
if (runs.length === 0) {
|
|
47851
|
+
out.write("No runs yet.\n");
|
|
47852
|
+
return;
|
|
47853
|
+
}
|
|
47854
|
+
for (const r of runs) {
|
|
47855
|
+
const tail = r.error !== null ? ` error: ${truncate3(r.error, 160)}` : r.summary !== null ? ` ${truncate3(JSON.stringify(r.summary), 160)}` : "";
|
|
47856
|
+
out.write(
|
|
47857
|
+
`${r.started_at} ${r.status.padEnd(15)} ${r.trigger.padEnd(8)} attempt ${r.attempt} ${r.artifact_version ?? "-"}${tail}
|
|
47858
|
+
`
|
|
47859
|
+
);
|
|
47860
|
+
}
|
|
47861
|
+
}
|
|
47862
|
+
function withFlags(cmd) {
|
|
47863
|
+
cmd.option("--json", "Output raw JSON instead of a summary", false);
|
|
47864
|
+
addServerFlag(cmd);
|
|
47865
|
+
addAccountFlag(cmd);
|
|
47866
|
+
addConnectorsUrlFlag(cmd);
|
|
47867
|
+
return cmd;
|
|
47868
|
+
}
|
|
47869
|
+
function addScheduleFlags(cmd) {
|
|
47870
|
+
return cmd.option("--artifact <name>", "published artifact name, e.g. sftp_export__daily_sales").option("--flavor <flavor>", "runner flavor the artifact was published for, e.g. sftp").option("--datasource <role=id>", "monolith datasource for a role (repeat per role)", collect).option(
|
|
47871
|
+
"--primary-role <role>",
|
|
47872
|
+
"the role passed as the primary datasource (default: the first)"
|
|
47873
|
+
).option("--config <json|@file>", "task config: a JSON object, or @path to a JSON file").option("--every <duration>", "run on an interval, e.g. 5m, 1h (at least 1m)").option("--cron <expr>", 'run on a five-field cron line, e.g. "15 * * * *"').option("--tz <iana>", "timezone the cron line is read in (default UTC)");
|
|
47874
|
+
}
|
|
47875
|
+
async function onTask(task, opts, act, print) {
|
|
47876
|
+
await withConnectorsSetup(opts, async ({ client, signal }) => {
|
|
47877
|
+
try {
|
|
47878
|
+
const taskId = await resolveTaskId(client, task, signal);
|
|
47879
|
+
const result = await act(client, taskId, signal);
|
|
47880
|
+
if (opts.json === true) writeJSONIndented(process.stdout, result);
|
|
47881
|
+
else print(process.stdout, result);
|
|
47882
|
+
} catch (err) {
|
|
47883
|
+
throw explainTaskError(err);
|
|
47884
|
+
}
|
|
47885
|
+
});
|
|
47886
|
+
}
|
|
47887
|
+
function registerConnectorsTasks(connectors) {
|
|
47888
|
+
const tasks = connectors.command("tasks").description("DRIP scheduled tasks: run an artifact on a schedule (support user or org admin)");
|
|
47889
|
+
withFlags(
|
|
47890
|
+
addScheduleFlags(
|
|
47891
|
+
tasks.command("create").description("Create a scheduled task").argument("<name>", "task name, unique in the org (a-z, 0-9, _ and -)").option("--org-id <id>", "org id (defaults to the JWT org)").option("--disabled", "create it disabled", false)
|
|
47892
|
+
)
|
|
47893
|
+
).action(
|
|
47894
|
+
async (name, opts) => {
|
|
47895
|
+
const explicitOrg = opts.orgId !== void 0 ? parsePositiveIntArg2(opts.orgId, "--org-id") : void 0;
|
|
47896
|
+
const parsed2 = buildCreateFields(name, explicitOrg ?? 0, opts);
|
|
47897
|
+
await withConnectorsSetup(opts, async ({ client, signal }) => {
|
|
47898
|
+
const fields = { ...parsed2, org_id: explicitOrg ?? client.getCurrentOrgId() };
|
|
47899
|
+
try {
|
|
47900
|
+
const t = await createTask(client, fields, { signal });
|
|
47901
|
+
if (opts.json === true) writeJSONIndented(process.stdout, t);
|
|
47902
|
+
else writeTask(process.stdout, t);
|
|
47903
|
+
} catch (err) {
|
|
47904
|
+
throw explainTaskError(err);
|
|
47905
|
+
}
|
|
47906
|
+
});
|
|
47907
|
+
}
|
|
47908
|
+
);
|
|
47909
|
+
withFlags(
|
|
47910
|
+
tasks.command("list").description("List the org's scheduled tasks").option("--org-id <id>", "org id (defaults to the JWT org)")
|
|
47911
|
+
).action(async (opts) => {
|
|
47912
|
+
await withConnectorsSetup(opts, async ({ client, signal }) => {
|
|
47913
|
+
const orgId = opts.orgId !== void 0 ? parsePositiveIntArg2(opts.orgId, "--org-id") : void 0;
|
|
47914
|
+
try {
|
|
47915
|
+
const list28 = await listTasks(client, orgId, { signal });
|
|
47916
|
+
if (opts.json === true) writeJSONIndented(process.stdout, list28);
|
|
47917
|
+
else writeTaskList(process.stdout, list28);
|
|
47918
|
+
} catch (err) {
|
|
47919
|
+
throw explainTaskError(err);
|
|
47920
|
+
}
|
|
47921
|
+
});
|
|
47922
|
+
});
|
|
47923
|
+
withFlags(
|
|
47924
|
+
tasks.command("get").description("Show a scheduled task").argument("<task>", "task id or name")
|
|
47925
|
+
).action(
|
|
47926
|
+
async (task, opts) => onTask(
|
|
47927
|
+
task,
|
|
47928
|
+
opts,
|
|
47929
|
+
(c, id, signal) => getTask(c, id, { signal }),
|
|
47930
|
+
(o, r) => writeTask(o, r)
|
|
47931
|
+
)
|
|
47932
|
+
);
|
|
47933
|
+
withFlags(
|
|
47934
|
+
addScheduleFlags(
|
|
47935
|
+
tasks.command("update").description("Change a scheduled task (only the flags given change)").argument("<task>", "task id or name").option("--enable", "enable the task").option("--disable", "disable the task")
|
|
47936
|
+
)
|
|
47937
|
+
).action(
|
|
47938
|
+
async (task, opts) => {
|
|
47939
|
+
if (opts.enable === true && opts.disable === true)
|
|
47940
|
+
throw new Error("give --enable or --disable, not both");
|
|
47941
|
+
const fields = parseScheduleFlags(opts);
|
|
47942
|
+
if (opts.enable === true) fields.enabled = true;
|
|
47943
|
+
if (opts.disable === true) fields.enabled = false;
|
|
47944
|
+
if (Object.keys(fields).length === 0)
|
|
47945
|
+
throw new Error("nothing to change: give at least one flag");
|
|
47946
|
+
await onTask(
|
|
47947
|
+
task,
|
|
47948
|
+
opts,
|
|
47949
|
+
(c, id, signal) => updateTask(c, id, fields, { signal }),
|
|
47950
|
+
(o, r) => writeTask(o, r)
|
|
47951
|
+
);
|
|
47952
|
+
}
|
|
47953
|
+
);
|
|
47954
|
+
withFlags(
|
|
47955
|
+
tasks.command("delete").description("Delete a scheduled task and its run history").argument("<task>", "task id or name")
|
|
47956
|
+
).action(
|
|
47957
|
+
async (task, opts) => onTask(
|
|
47958
|
+
task,
|
|
47959
|
+
opts,
|
|
47960
|
+
async (c, id, signal) => {
|
|
47961
|
+
await deleteTask(c, id, { signal });
|
|
47962
|
+
return { deleted: id };
|
|
47963
|
+
},
|
|
47964
|
+
(o, r) => o.write(`Deleted scheduled task ${r.deleted}
|
|
47965
|
+
`)
|
|
47966
|
+
)
|
|
47967
|
+
);
|
|
47968
|
+
withFlags(
|
|
47969
|
+
tasks.command("run").description("Run a task now (does not move its schedule)").argument("<task>", "task id or name")
|
|
47970
|
+
).action(
|
|
47971
|
+
async (task, opts) => onTask(
|
|
47972
|
+
task,
|
|
47973
|
+
opts,
|
|
47974
|
+
(c, id, signal) => runTask(c, id, { signal }),
|
|
47975
|
+
(o, r) => {
|
|
47976
|
+
const run2 = r;
|
|
47977
|
+
o.write(`Queued run ${run2.id} (${run2.status}); see: dr connectors tasks runs ${task}
|
|
47978
|
+
`);
|
|
47979
|
+
}
|
|
47980
|
+
)
|
|
47981
|
+
);
|
|
47982
|
+
withFlags(
|
|
47983
|
+
tasks.command("pause").description("Pause a scheduled task").argument("<task>", "task id or name")
|
|
47984
|
+
).action(
|
|
47985
|
+
async (task, opts) => onTask(
|
|
47986
|
+
task,
|
|
47987
|
+
opts,
|
|
47988
|
+
(c, id, signal) => pauseTask(c, id, { signal }),
|
|
47989
|
+
(o, r) => writeTask(o, r)
|
|
47990
|
+
)
|
|
47991
|
+
);
|
|
47992
|
+
withFlags(
|
|
47993
|
+
tasks.command("resume").description("Resume a paused task (clears its failure count, schedules the next run)").argument("<task>", "task id or name")
|
|
47994
|
+
).action(
|
|
47995
|
+
async (task, opts) => onTask(
|
|
47996
|
+
task,
|
|
47997
|
+
opts,
|
|
47998
|
+
(c, id, signal) => resumeTask(c, id, { signal }),
|
|
47999
|
+
(o, r) => writeTask(o, r)
|
|
48000
|
+
)
|
|
48001
|
+
);
|
|
48002
|
+
withFlags(
|
|
48003
|
+
tasks.command("runs").description("A task's run history, newest first").argument("<task>", "task id or name").option("--limit <n>", "how many runs (1-200)", "20")
|
|
48004
|
+
).action(async (task, opts) => {
|
|
48005
|
+
const limit = parsePositiveIntArg2(opts.limit, "--limit");
|
|
48006
|
+
if (limit > 200) throw new Error("invalid --limit: at most 200");
|
|
48007
|
+
await onTask(
|
|
48008
|
+
task,
|
|
48009
|
+
opts,
|
|
48010
|
+
(c, id, signal) => listRuns(c, id, limit, { signal }),
|
|
48011
|
+
(o, r) => writeRuns(o, r)
|
|
48012
|
+
);
|
|
48013
|
+
});
|
|
48014
|
+
}
|
|
48015
|
+
|
|
48016
|
+
// src/commands/connectorsAws.ts
|
|
48017
|
+
var ROLE_ARN_RE = /^arn:aws[a-z-]*:iam::\d{12}:role\/\S+$/;
|
|
48018
|
+
var OTHER_ORG_HINT = "if your session is valid, the connection or --org-id may belong to another org than the --account you are logged in as";
|
|
48019
|
+
function explainAwsError(err, id) {
|
|
48020
|
+
if (!(err instanceof ControlPlaneError)) {
|
|
48021
|
+
if (err instanceof Error && /\bHTTP 403\b/.test(err.message)) {
|
|
48022
|
+
return withCause(
|
|
48023
|
+
new ControlPlaneError(
|
|
48024
|
+
403,
|
|
48025
|
+
"",
|
|
48026
|
+
`${err.message}
|
|
48027
|
+
${OTHER_ORG_HINT}`
|
|
48028
|
+
),
|
|
48029
|
+
err
|
|
48030
|
+
);
|
|
48031
|
+
}
|
|
48032
|
+
return err;
|
|
48033
|
+
}
|
|
48034
|
+
if (!hasDetail(err.body)) return err;
|
|
48035
|
+
const detail = detailOf(err.body) || err.message;
|
|
48036
|
+
const messages = {
|
|
48037
|
+
403: `forbidden: ${detail} (${OTHER_ORG_HINT})`,
|
|
48038
|
+
404: `AWS connection${id === void 0 ? "" : ` ${id}`} not found: ${detail}`,
|
|
48039
|
+
409: `conflict: ${detail}`,
|
|
48040
|
+
422: `rejected: ${detail}`,
|
|
48041
|
+
503: `the control plane cannot broker AWS roles on this site: ${detail}`
|
|
48042
|
+
};
|
|
48043
|
+
const message = messages[err.status];
|
|
48044
|
+
if (message === void 0) return err;
|
|
48045
|
+
return withCause(new ControlPlaneError(err.status, err.body, message), err);
|
|
48046
|
+
}
|
|
48047
|
+
function hasDetail(body) {
|
|
48048
|
+
try {
|
|
48049
|
+
const parsed2 = JSON.parse(body);
|
|
48050
|
+
return typeof parsed2 === "object" && parsed2 !== null && "detail" in parsed2;
|
|
48051
|
+
} catch {
|
|
48052
|
+
return false;
|
|
48053
|
+
}
|
|
48054
|
+
}
|
|
48055
|
+
function oneLine4(s) {
|
|
48056
|
+
return s.replace(/\s+/g, " ").trim();
|
|
48057
|
+
}
|
|
48058
|
+
function writeConnection(out, c) {
|
|
48059
|
+
const row = (label, value) => {
|
|
48060
|
+
out.write(` ${`${label}:`.padEnd(14)}${value}
|
|
48061
|
+
`);
|
|
48062
|
+
};
|
|
48063
|
+
out.write(`AWS connection #${c.id} (org ${c.org_id})
|
|
48064
|
+
`);
|
|
48065
|
+
row("Datasource", c.secret_ref);
|
|
48066
|
+
row("Role ARN", c.role_arn);
|
|
48067
|
+
row("Status", `${c.status}${c.enabled ? "" : " (disabled)"}`);
|
|
48068
|
+
row("Broker ARN", c.broker_arn ?? "-");
|
|
48069
|
+
row("External ID", c.external_id);
|
|
48070
|
+
row("Expires", c.expires_at ?? "-");
|
|
48071
|
+
row("Last assumed", c.last_assumed_at ?? "-");
|
|
48072
|
+
row("Last error", c.last_error === null ? "-" : oneLine4(c.last_error));
|
|
48073
|
+
row("Failures", String(c.consecutive_failures));
|
|
48074
|
+
if (c.trust_statement === null) {
|
|
48075
|
+
out.write("\nTrust statement: none returned by the control plane.\n");
|
|
48076
|
+
} else {
|
|
48077
|
+
out.write("\nTrust statement (add it to the customer role's trust policy):\n");
|
|
48078
|
+
out.write(`${JSON.stringify(c.trust_statement, null, 2)}
|
|
48079
|
+
`);
|
|
48080
|
+
}
|
|
48081
|
+
out.write(
|
|
48082
|
+
"\nThe customer role's permissions should be read-only on their bucket/prefix (plus kms:Decrypt for a customer-managed KMS key).\n"
|
|
48083
|
+
);
|
|
48084
|
+
}
|
|
48085
|
+
function writeConnectionTable(out, list28) {
|
|
48086
|
+
if (list28.length === 0) {
|
|
48087
|
+
out.write("No AWS connections for this org.\n");
|
|
48088
|
+
return;
|
|
48089
|
+
}
|
|
48090
|
+
const rows = [["ID", "SECRET_REF", "ROLE_ARN", "STATUS", "EXPIRES_AT", "LAST_ERROR"]];
|
|
48091
|
+
for (const c of list28) {
|
|
48092
|
+
rows.push([
|
|
48093
|
+
String(c.id),
|
|
48094
|
+
oneLine4(c.secret_ref),
|
|
48095
|
+
oneLine4(c.role_arn),
|
|
48096
|
+
c.status,
|
|
48097
|
+
c.expires_at ?? "-",
|
|
48098
|
+
c.last_error === null ? "-" : truncate3(oneLine4(c.last_error), 60)
|
|
48099
|
+
]);
|
|
48100
|
+
}
|
|
48101
|
+
const widths = rows[0].map((_, i) => Math.max(...rows.map((r) => r[i].length)));
|
|
48102
|
+
for (const r of rows) {
|
|
48103
|
+
out.write(
|
|
48104
|
+
`${r.map((cell, i) => cell.padEnd(widths[i] + 2)).join("").trimEnd()}
|
|
48105
|
+
`
|
|
48106
|
+
);
|
|
48107
|
+
}
|
|
48108
|
+
}
|
|
48109
|
+
function writeRotated(out, c) {
|
|
48110
|
+
out.write(`External ID rotated for AWS connection #${c.id} (org ${c.org_id}).
|
|
48111
|
+
`);
|
|
48112
|
+
out.write(`New external ID: ${c.external_id}
|
|
48113
|
+
`);
|
|
48114
|
+
out.write(
|
|
48115
|
+
`AWS refuses this connection until the customer replaces the trust statement in the trust policy of ${oneLine4(c.role_arn)} with the one below; the old external ID no longer works.
|
|
48116
|
+
|
|
48117
|
+
`
|
|
48118
|
+
);
|
|
48119
|
+
if (c.trust_statement === null) {
|
|
48120
|
+
out.write("New trust statement: none returned by the control plane.\n");
|
|
48121
|
+
} else {
|
|
48122
|
+
out.write(`New trust statement:
|
|
48123
|
+
${JSON.stringify(c.trust_statement, null, 2)}
|
|
48124
|
+
`);
|
|
48125
|
+
}
|
|
48126
|
+
}
|
|
48127
|
+
function withFlags2(cmd) {
|
|
48128
|
+
cmd.option("--json", "Output raw JSON instead of a summary", false);
|
|
48129
|
+
addServerFlag(cmd);
|
|
48130
|
+
addAccountFlag(cmd);
|
|
48131
|
+
addConnectorsUrlFlag(cmd);
|
|
48132
|
+
return cmd;
|
|
48133
|
+
}
|
|
48134
|
+
function parseRoleArn(arn) {
|
|
48135
|
+
if (!ROLE_ARN_RE.test(arn)) {
|
|
48136
|
+
throw new Error(
|
|
48137
|
+
`invalid --role-arn "${arn}": expected arn:aws:iam::<12-digit account>:role/<name>`
|
|
48138
|
+
);
|
|
48139
|
+
}
|
|
48140
|
+
return arn;
|
|
48141
|
+
}
|
|
48142
|
+
function awsConfirmAction(verb, id, opts, c) {
|
|
48143
|
+
const state = { role_arn: c.role_arn, external_id: c.external_id };
|
|
48144
|
+
return {
|
|
48145
|
+
resource: "aws-connection",
|
|
48146
|
+
id: `${id}#${confirmDigest(scopeOf(opts), state)}`,
|
|
48147
|
+
verb
|
|
48148
|
+
};
|
|
48149
|
+
}
|
|
48150
|
+
async function verifyAgainstCurrent(action, token, verb, id) {
|
|
48151
|
+
try {
|
|
48152
|
+
await preflightConfirmToken(action, token);
|
|
48153
|
+
} catch (err) {
|
|
48154
|
+
if (!(err instanceof Error) || err.message !== ERR_TOKEN_MISMATCH) throw err;
|
|
48155
|
+
throw new Error(
|
|
48156
|
+
`confirm token does not match ${verb} of AWS connection ${id} as it is now: the connection changed since the plan (role or external ID); plan again. (A token planned for another verb, connection, server, account or --connectors-url is refused the same way.)`,
|
|
48157
|
+
{ cause: err }
|
|
48158
|
+
);
|
|
48159
|
+
}
|
|
48160
|
+
}
|
|
48161
|
+
async function runGated(verb, id, opts, effect, act, print) {
|
|
48162
|
+
await withConnectorsSetup(opts, async (setup) => {
|
|
48163
|
+
try {
|
|
48164
|
+
const c = await getAwsConnection(setup.client, id, { signal: setup.signal });
|
|
48165
|
+
const action = awsConfirmAction(verb, id, opts, c);
|
|
48166
|
+
await verifyAgainstCurrent(action, opts.confirmToken, verb, id);
|
|
48167
|
+
const detail = [
|
|
48168
|
+
`connection: #${c.id} (org ${c.org_id})`,
|
|
48169
|
+
`datasource: ${oneLine4(c.secret_ref)}`,
|
|
48170
|
+
`role: ${oneLine4(c.role_arn)}`,
|
|
48171
|
+
`status: ${c.status}${c.enabled ? "" : " (disabled)"}`,
|
|
48172
|
+
`external ID: ${c.external_id}`,
|
|
48173
|
+
`effect: ${effect}`
|
|
48174
|
+
];
|
|
48175
|
+
setup.pauseTimeout();
|
|
48176
|
+
await twoStepConfirm({
|
|
48177
|
+
out: process.stdout,
|
|
48178
|
+
action,
|
|
48179
|
+
token: opts.confirmToken,
|
|
48180
|
+
server: scopeOf(opts).serverKey,
|
|
48181
|
+
confirmCmd: `dr connectors aws ${verb} ${id}${targetArgsOf(opts)}`,
|
|
48182
|
+
describe: async () => detail,
|
|
48183
|
+
execute: async () => {
|
|
48184
|
+
setup.extendTimeout(DEFAULT_CALL_TIMEOUT_MS);
|
|
48185
|
+
const now = await getAwsConnection(setup.client, id, { signal: setup.signal });
|
|
48186
|
+
if (now.role_arn !== c.role_arn || now.external_id !== c.external_id) {
|
|
48187
|
+
throw new Error(
|
|
48188
|
+
`AWS connection ${id} changed while waiting for confirmation (role or external ID); nothing was sent. Plan again.`
|
|
48189
|
+
);
|
|
48190
|
+
}
|
|
48191
|
+
const result = await act(setup);
|
|
48192
|
+
if (opts.json === true) writeJSONIndented(process.stdout, result);
|
|
48193
|
+
else print(process.stdout, result);
|
|
48194
|
+
}
|
|
48195
|
+
});
|
|
48196
|
+
} catch (err) {
|
|
48197
|
+
throw explainAwsError(err, id);
|
|
48198
|
+
}
|
|
48199
|
+
});
|
|
48200
|
+
}
|
|
48201
|
+
async function run(opts, act, print, id) {
|
|
48202
|
+
await withConnectorsSetup(opts, async (setup) => {
|
|
48203
|
+
let result;
|
|
48204
|
+
try {
|
|
48205
|
+
result = await act(setup);
|
|
48206
|
+
} catch (err) {
|
|
48207
|
+
throw explainAwsError(err, id);
|
|
48208
|
+
}
|
|
48209
|
+
if (opts.json === true) writeJSONIndented(process.stdout, result);
|
|
48210
|
+
else print(process.stdout, result);
|
|
48211
|
+
});
|
|
48212
|
+
}
|
|
48213
|
+
function registerConnectorsAws(connectors) {
|
|
48214
|
+
const aws = connectors.command("aws").description("Customer IAM roles DRIP assumes to read the customer's S3 (external ID, trust)");
|
|
48215
|
+
withFlags2(
|
|
48216
|
+
aws.command("create").description("Register a customer IAM role for a datasource; prints the trust statement").requiredOption("--secret-ref <datasourceId>", "monolith datasource id the role serves").requiredOption("--role-arn <arn>", "customer role, e.g. arn:aws:iam::123456789012:role/x").option("--org-id <id>", "org id (defaults to the JWT org)")
|
|
48217
|
+
).action(async (opts) => {
|
|
48218
|
+
const secretRef = opts.secretRef.trim();
|
|
48219
|
+
if (secretRef === "" || secretRef.length > 200) {
|
|
48220
|
+
throw new Error(`invalid --secret-ref "${opts.secretRef}": expected 1-200 characters`);
|
|
48221
|
+
}
|
|
48222
|
+
parseRoleArn(opts.roleArn);
|
|
48223
|
+
const orgId = opts.orgId === void 0 ? void 0 : parsePositiveIntArg2(opts.orgId, "--org-id");
|
|
48224
|
+
const fields = {
|
|
48225
|
+
secret_ref: secretRef,
|
|
48226
|
+
role_arn: opts.roleArn,
|
|
48227
|
+
...orgId !== void 0 ? { org_id: orgId } : {}
|
|
48228
|
+
};
|
|
48229
|
+
await run(
|
|
48230
|
+
opts,
|
|
48231
|
+
({ client, signal }) => createAwsConnection(client, fields, { signal }),
|
|
48232
|
+
writeConnection
|
|
48233
|
+
);
|
|
48234
|
+
});
|
|
48235
|
+
withFlags2(
|
|
48236
|
+
aws.command("list").description("List the org's AWS connections").option("--secret-ref <datasourceId>", "only the connection of this datasource")
|
|
48237
|
+
).action(async (opts) => {
|
|
48238
|
+
await run(
|
|
48239
|
+
opts,
|
|
48240
|
+
({ client, signal }) => listAwsConnections(client, opts.secretRef, { signal }),
|
|
48241
|
+
writeConnectionTable
|
|
48242
|
+
);
|
|
48243
|
+
});
|
|
48244
|
+
withFlags2(
|
|
48245
|
+
aws.command("get").description("Show an AWS connection and its trust statement").argument("<id>", "connection id")
|
|
48246
|
+
).action(async (idArg, opts) => {
|
|
48247
|
+
const id = parsePositiveIntArg2(idArg, "connection id");
|
|
48248
|
+
await run(
|
|
48249
|
+
opts,
|
|
48250
|
+
({ client, signal }) => getAwsConnection(client, id, { signal }),
|
|
48251
|
+
writeConnection,
|
|
48252
|
+
id
|
|
48253
|
+
);
|
|
48254
|
+
});
|
|
48255
|
+
for (const [verb, enabled, help] of [
|
|
48256
|
+
["enable", true, "Enable an AWS connection; also retries now (stored credentials are dropped)"],
|
|
48257
|
+
["disable", false, "Disable an AWS connection (stored credentials are dropped)"]
|
|
48258
|
+
]) {
|
|
48259
|
+
withFlags2(aws.command(verb).description(help).argument("<id>", "connection id")).action(
|
|
48260
|
+
async (idArg, opts) => {
|
|
48261
|
+
const id = parsePositiveIntArg2(idArg, "connection id");
|
|
48262
|
+
await run(
|
|
48263
|
+
opts,
|
|
48264
|
+
({ client, signal }) => updateAwsConnection(client, id, { enabled }, { signal }),
|
|
48265
|
+
(out, c) => {
|
|
48266
|
+
writeConnection(out, c);
|
|
48267
|
+
if (enabled)
|
|
48268
|
+
out.write(
|
|
48269
|
+
"\nStored credentials were dropped; the poller assumes the role again now.\n"
|
|
48270
|
+
);
|
|
48271
|
+
},
|
|
48272
|
+
id
|
|
48273
|
+
);
|
|
48274
|
+
}
|
|
48275
|
+
);
|
|
48276
|
+
}
|
|
48277
|
+
withFlags2(
|
|
48278
|
+
aws.command("set-role").description("Point an AWS connection at another customer role (stored credentials dropped)").argument("<id>", "connection id").requiredOption("--role-arn <arn>", "customer role, e.g. arn:aws:iam::123456789012:role/x")
|
|
48279
|
+
).action(async (idArg, opts) => {
|
|
48280
|
+
const id = parsePositiveIntArg2(idArg, "connection id");
|
|
48281
|
+
const roleArn = parseRoleArn(opts.roleArn);
|
|
48282
|
+
await run(
|
|
48283
|
+
opts,
|
|
48284
|
+
({ client, signal }) => updateAwsConnection(client, id, { role_arn: roleArn }, { signal }),
|
|
48285
|
+
(out, c) => {
|
|
48286
|
+
writeConnection(out, c);
|
|
48287
|
+
out.write(
|
|
48288
|
+
`
|
|
48289
|
+
The new role must trust the broker with the same external ID (${c.external_id}): add the trust statement above to its trust policy. Stored credentials were dropped; the poller assumes the new role next.
|
|
48290
|
+
`
|
|
48291
|
+
);
|
|
48292
|
+
},
|
|
48293
|
+
id
|
|
48294
|
+
);
|
|
48295
|
+
});
|
|
48296
|
+
const confirmHelp = "\nPrints a plan and a --confirm-token; re-run the printed command, token included, to apply it.\nOn production a person must also type a word in a dialog.";
|
|
48297
|
+
withFlags2(
|
|
48298
|
+
aws.command("rotate-external-id").description("Issue a new external ID; AWS refuses the role until the customer updates it").argument("<id>", "connection id").option("--confirm-token <token>", "Token from a prior plan run \u2014 required to proceed").addHelpText("after", confirmHelp)
|
|
48299
|
+
).action(async (idArg, opts) => {
|
|
48300
|
+
const id = parsePositiveIntArg2(idArg, "connection id");
|
|
48301
|
+
await runGated(
|
|
48302
|
+
"rotate-external-id",
|
|
48303
|
+
id,
|
|
48304
|
+
opts,
|
|
48305
|
+
"a new external ID; AWS refuses the role until the customer updates its trust policy",
|
|
48306
|
+
({ client, signal }) => rotateAwsExternalId(client, id, { signal }),
|
|
48307
|
+
writeRotated
|
|
48308
|
+
);
|
|
48309
|
+
});
|
|
48310
|
+
withFlags2(
|
|
48311
|
+
aws.command("delete").description("Delete an AWS connection: the role and its stored credentials").argument("<id>", "connection id").option("--confirm-token <token>", "Token from a prior plan run \u2014 required to proceed").addHelpText("after", confirmHelp)
|
|
48312
|
+
).action(async (idArg, opts) => {
|
|
48313
|
+
const id = parsePositiveIntArg2(idArg, "connection id");
|
|
48314
|
+
await runGated(
|
|
48315
|
+
"delete",
|
|
48316
|
+
id,
|
|
48317
|
+
opts,
|
|
48318
|
+
"removes the role and its stored credentials; the connector can no longer read S3",
|
|
48319
|
+
async ({ client, signal }) => {
|
|
48320
|
+
await deleteAwsConnection(client, id, { signal });
|
|
48321
|
+
return { deleted: id };
|
|
48322
|
+
},
|
|
48323
|
+
(out, r) => out.write(`Deleted AWS connection ${r.deleted}
|
|
48324
|
+
`)
|
|
48325
|
+
);
|
|
48326
|
+
});
|
|
48327
|
+
}
|
|
48328
|
+
|
|
48329
|
+
// src/commands/connectorsDevFetch.ts
|
|
48330
|
+
import { promises as fs19 } from "fs";
|
|
48331
|
+
import * as path9 from "path";
|
|
48332
|
+
import { randomUUID as randomUUID2 } from "crypto";
|
|
48333
|
+
function buildDevEnvelope(i) {
|
|
48334
|
+
const config = {
|
|
48335
|
+
...i.publicConfig,
|
|
48336
|
+
...i.queryConfig,
|
|
48337
|
+
// Coerce absent values to null (not undefined) so JSON.stringify keeps the
|
|
48338
|
+
// key — matches build_hook_job's dq.get(...) -> None; a dropped objectType
|
|
48339
|
+
// becomes a cryptic KeyError in the runner's dispatch.
|
|
48340
|
+
objectType: i.dataQuery.objectType ?? null,
|
|
48341
|
+
type: i.dataQuery.type ?? null,
|
|
48342
|
+
query: i.dataQuery.query
|
|
48343
|
+
};
|
|
48344
|
+
if (i.period) config.period = i.period;
|
|
48345
|
+
return {
|
|
48346
|
+
correlation_id: i.correlationId,
|
|
48347
|
+
config,
|
|
48348
|
+
secret_ref: i.secretRef,
|
|
48349
|
+
cred_token: "dev-local",
|
|
48350
|
+
output_sas: "file:///out/output.csv",
|
|
48351
|
+
input_sas: null,
|
|
48352
|
+
cursor: null,
|
|
48353
|
+
contract_version: i.contractVersion,
|
|
48354
|
+
metadata: {}
|
|
48355
|
+
};
|
|
48356
|
+
}
|
|
48357
|
+
function asObject(v) {
|
|
48358
|
+
if (typeof v === "object" && v !== null && !Array.isArray(v)) {
|
|
48359
|
+
return v;
|
|
48360
|
+
}
|
|
48361
|
+
return null;
|
|
48362
|
+
}
|
|
48363
|
+
function extractDataQueryFields(raw2) {
|
|
48364
|
+
const root = asObject(decodeShape(raw2));
|
|
48365
|
+
if (root === null || Object.keys(root).length === 0) {
|
|
48366
|
+
throw new Error(
|
|
48367
|
+
"no data-query attached to this filebox (empty data_query response) \u2014 attach one in the filebox UI first"
|
|
48368
|
+
);
|
|
48369
|
+
}
|
|
48370
|
+
const cfg = asObject(root.config);
|
|
48371
|
+
const src = cfg ?? root;
|
|
48372
|
+
const query = src.query;
|
|
48373
|
+
let queryObj;
|
|
48374
|
+
if (query === void 0 || query === null) {
|
|
48375
|
+
queryObj = {};
|
|
48376
|
+
} else {
|
|
48377
|
+
const asObj = asObject(query);
|
|
48378
|
+
if (asObj === null) {
|
|
48379
|
+
throw new Error(
|
|
48380
|
+
`data-query \`query\` is not an object (got ${typeof query}) \u2014 expected the Sync-query field`
|
|
48381
|
+
);
|
|
48382
|
+
}
|
|
48383
|
+
queryObj = asObj;
|
|
48384
|
+
}
|
|
48385
|
+
return { objectType: queryObj.objectType, type: src.type, query: queryObj };
|
|
48386
|
+
}
|
|
48387
|
+
function extractCredentials(integration) {
|
|
48388
|
+
const cfg = asObject(integration.config);
|
|
48389
|
+
if (cfg === null || Object.keys(cfg).length === 0) {
|
|
48390
|
+
throw new Error(
|
|
48391
|
+
`datasource ${integration.id} has an empty/missing \`config\` \u2014 nothing to use as connector credentials`
|
|
48392
|
+
);
|
|
48393
|
+
}
|
|
48394
|
+
return { ...cfg };
|
|
48395
|
+
}
|
|
48396
|
+
var ENV_FILE = "env.json";
|
|
48397
|
+
var SECRETS_FILE = "secrets.json";
|
|
48398
|
+
var CONNECTION_FILE = "connection.py";
|
|
48399
|
+
var QUERY_FILE = "query.py";
|
|
48400
|
+
async function writeRunDir(input) {
|
|
48401
|
+
let connectionSource;
|
|
48402
|
+
try {
|
|
48403
|
+
connectionSource = await fs19.readFile(input.connectionPath, "utf-8");
|
|
48404
|
+
} catch {
|
|
48405
|
+
throw new Error(`${input.connectionPath}: connection source not found or unreadable`);
|
|
48406
|
+
}
|
|
48407
|
+
let querySource;
|
|
48408
|
+
try {
|
|
48409
|
+
querySource = await fs19.readFile(input.queryPath, "utf-8");
|
|
48410
|
+
} catch {
|
|
48411
|
+
throw new Error(`${input.queryPath}: dataquery source not found or unreadable`);
|
|
48412
|
+
}
|
|
48413
|
+
await fs19.mkdir(input.outDir, { recursive: true });
|
|
48414
|
+
await fs19.writeFile(
|
|
48415
|
+
path9.join(input.outDir, ENV_FILE),
|
|
48416
|
+
JSON.stringify(input.env, null, 2) + "\n",
|
|
48417
|
+
"utf-8"
|
|
48418
|
+
);
|
|
48419
|
+
const secretsPath = path9.join(input.outDir, SECRETS_FILE);
|
|
48420
|
+
await fs19.writeFile(
|
|
48421
|
+
secretsPath,
|
|
48422
|
+
JSON.stringify({ [input.secretRef]: input.credentials }, null, 2) + "\n",
|
|
48423
|
+
"utf-8"
|
|
48424
|
+
);
|
|
48425
|
+
if (process.platform !== "win32") {
|
|
48426
|
+
await fs19.chmod(secretsPath, 384);
|
|
48427
|
+
}
|
|
48428
|
+
await fs19.writeFile(path9.join(input.outDir, CONNECTION_FILE), connectionSource, "utf-8");
|
|
48429
|
+
await fs19.writeFile(path9.join(input.outDir, QUERY_FILE), querySource, "utf-8");
|
|
48430
|
+
}
|
|
48431
|
+
async function ensureGitignored(entry) {
|
|
48432
|
+
const gitignorePath = path9.join(process.cwd(), ".gitignore");
|
|
48433
|
+
let current = "";
|
|
48434
|
+
try {
|
|
48435
|
+
current = await fs19.readFile(gitignorePath, "utf-8");
|
|
48436
|
+
} catch {
|
|
48437
|
+
}
|
|
48438
|
+
const normalized = entry.split("\\").join("/").replace(/\/+$/, "") + "/";
|
|
48439
|
+
const lines = current.split(/\r?\n/).map((l) => l.trim());
|
|
48440
|
+
if (lines.includes(normalized) || lines.includes(entry)) return;
|
|
48441
|
+
const sep = current === "" || current.endsWith("\n") ? "" : "\n";
|
|
48442
|
+
await fs19.writeFile(gitignorePath, `${current}${sep}${normalized}
|
|
48443
|
+
`, "utf-8");
|
|
48444
|
+
}
|
|
48445
|
+
function parsePositiveInt3(raw2, label) {
|
|
48446
|
+
const n = Number(raw2);
|
|
48447
|
+
if (!Number.isInteger(n) || String(n) !== raw2.trim() || n <= 0) {
|
|
48448
|
+
throw new Error(`invalid ${label}: "${raw2}"`);
|
|
48449
|
+
}
|
|
48450
|
+
return n;
|
|
48451
|
+
}
|
|
48452
|
+
function registerDevFetch(connectors) {
|
|
48453
|
+
const cmd = connectors.command("dev-fetch").description(
|
|
48454
|
+
"Assemble a local run dir (env.json + secrets.json + connection.py + query.py) for dev-run.sh"
|
|
48455
|
+
).argument("<fileboxUuid>", "filebox UUID of the connector binding").requiredOption(
|
|
48456
|
+
"--datasource <id>",
|
|
48457
|
+
"datasource (data-integration) id whose credentials to fetch"
|
|
48458
|
+
).requiredOption(
|
|
48459
|
+
"--document-id <n>",
|
|
48460
|
+
"numeric filebox document id (no uuid->id lookup exists; required)"
|
|
48461
|
+
).requiredOption("--connection <path>", "path to the connector's connection .py source").requiredOption("--query <path>", "path to the connector's dataquery .py source").option("--from <date>", "period start (paired with --to)").option("--to <date>", "period end (paired with --from)").option("-o, --output <dir>", "output run dir (default ./dev-run/<fileboxUuid>/)").action(async (fileboxUuid, opts) => {
|
|
48462
|
+
const datasourceId = parsePositiveInt3(opts.datasource, "datasource id");
|
|
48463
|
+
const documentId = parsePositiveInt3(opts.documentId, "document-id");
|
|
48464
|
+
if (opts.from === void 0 !== (opts.to === void 0)) {
|
|
48465
|
+
throw new Error("--from and --to must be provided together");
|
|
48466
|
+
}
|
|
48467
|
+
const defaultRoot = path9.join("dev-run", fileboxUuid);
|
|
48468
|
+
const outDir = opts.output ?? defaultRoot;
|
|
48469
|
+
const gitignoreEntry = opts.output ?? "dev-run";
|
|
48470
|
+
const serverKey = resolveServer(opts.server);
|
|
48471
|
+
const host = resolveConnectorsUrl(serverKey, opts.connectorsUrl);
|
|
48472
|
+
const monolithHost = resolve(serverKey).host;
|
|
48473
|
+
const cpSetup = await setupCall(opts.server, opts.account, { host, authHost: monolithHost });
|
|
48474
|
+
try {
|
|
48475
|
+
const binding = await getBinding(cpSetup.client, fileboxUuid, {
|
|
48476
|
+
signal: cpSetup.signal
|
|
48477
|
+
});
|
|
48478
|
+
const monoSetup = await setupCall(opts.server, opts.account, {
|
|
48479
|
+
host: monolithHost,
|
|
48480
|
+
authHost: monolithHost
|
|
48481
|
+
});
|
|
48482
|
+
try {
|
|
48483
|
+
const rawDataQuery = await getDataQueryRaw(monoSetup.client, documentId, {
|
|
48484
|
+
signal: monoSetup.signal
|
|
48485
|
+
});
|
|
48486
|
+
const dataQuery = extractDataQueryFields(rawDataQuery);
|
|
48487
|
+
if (dataQuery.objectType === void 0 || dataQuery.objectType === null || dataQuery.objectType === "") {
|
|
48488
|
+
process.stderr.write(
|
|
48489
|
+
'WARNING: objectType is not set inside the Sync-query (config.query.objectType). get_query_class will receive None \u2014 fine only if the connector has a single default dataset. Add it to the data-query, e.g. config.query.objectType = "<dataset>".\n'
|
|
48490
|
+
);
|
|
48491
|
+
}
|
|
48492
|
+
const integration = await itGet(monoSetup.client, datasourceId, {
|
|
48493
|
+
signal: monoSetup.signal
|
|
48494
|
+
});
|
|
48495
|
+
const credentials = extractCredentials(integration);
|
|
48496
|
+
let period;
|
|
48497
|
+
if (opts.from !== void 0 && opts.to !== void 0) {
|
|
48498
|
+
period = { from: opts.from, to: opts.to };
|
|
48499
|
+
}
|
|
48500
|
+
const env = buildDevEnvelope({
|
|
48501
|
+
correlationId: randomUUID2(),
|
|
48502
|
+
secretRef: binding.secret_ref,
|
|
48503
|
+
publicConfig: binding.public_config,
|
|
48504
|
+
queryConfig: binding.query_config,
|
|
48505
|
+
dataQuery,
|
|
48506
|
+
...period !== void 0 ? { period } : {},
|
|
48507
|
+
// Deliberate sandbox stub: the fetch/dispatch path doesn't branch on
|
|
48508
|
+
// contract_version, so a fixed "1" is sufficient for a local run.
|
|
48509
|
+
contractVersion: "1"
|
|
48510
|
+
});
|
|
48511
|
+
await writeRunDir({
|
|
48512
|
+
outDir,
|
|
48513
|
+
env,
|
|
48514
|
+
secretRef: binding.secret_ref,
|
|
48515
|
+
credentials,
|
|
48516
|
+
connectionPath: opts.connection,
|
|
48517
|
+
queryPath: opts.query
|
|
48518
|
+
});
|
|
48519
|
+
await ensureGitignored(gitignoreEntry);
|
|
48520
|
+
process.stdout.write(`Wrote run dir: ${outDir}
|
|
48521
|
+
`);
|
|
48522
|
+
process.stdout.write(
|
|
48523
|
+
` env.json (secret_ref=${binding.secret_ref}, flavor=${binding.runtime_flavor})
|
|
48524
|
+
`
|
|
48525
|
+
);
|
|
48526
|
+
process.stdout.write(" secrets.json (credentials NOT shown)\n");
|
|
48527
|
+
process.stdout.write(" connection.py, query.py\n");
|
|
48528
|
+
} finally {
|
|
48529
|
+
monoSetup.cleanup();
|
|
48530
|
+
}
|
|
48531
|
+
} catch (err) {
|
|
48532
|
+
throw augmentTlsCertError(err);
|
|
48533
|
+
} finally {
|
|
48534
|
+
cpSetup.cleanup();
|
|
48535
|
+
}
|
|
48536
|
+
});
|
|
48537
|
+
addServerFlag(cmd);
|
|
48538
|
+
addAccountFlag(cmd);
|
|
48539
|
+
cmd.option(
|
|
48540
|
+
"--connectors-url <url>",
|
|
48541
|
+
"Override the control-plane base URL (e.g. a kubectl port-forward http://localhost:8000)"
|
|
48542
|
+
);
|
|
48543
|
+
}
|
|
48544
|
+
|
|
48390
48545
|
// src/commands/connectors.ts
|
|
48391
48546
|
var CONTROL_PLANE_HOST = (subDomain) => `https://${subDomain}-connectors-control-plane.datarails.io`;
|
|
48392
48547
|
function resolveConnectorsUrl(serverKey, connectorsUrlFlag) {
|