@erdoai/cli 0.202.0 → 0.204.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +157 -10
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -79,6 +79,50 @@ function formatRefreshJobRow(r) {
|
|
|
79
79
|
if (r.keep_running) notes.push("keep-running");
|
|
80
80
|
return `${r.id} ${r.state} ${r.name} refresh${notes.length ? ` ${notes.join("; ")}` : ""}`;
|
|
81
81
|
}
|
|
82
|
+
var FRESHNESS_TARGET_MIN_SECONDS = 5 * 60;
|
|
83
|
+
var FRESHNESS_TARGET_MAX_SECONDS = 366 * 24 * 60 * 60;
|
|
84
|
+
var FRESHNESS_UNITS = { s: 1, m: 60, h: 60 * 60, d: 24 * 60 * 60, w: 7 * 24 * 60 * 60 };
|
|
85
|
+
function parseFreshnessTarget(value) {
|
|
86
|
+
const text = value.trim().toLowerCase();
|
|
87
|
+
if (text === "off" || text === "none" || text === "0") return 0;
|
|
88
|
+
let seconds;
|
|
89
|
+
if (/^\d+$/.test(text)) {
|
|
90
|
+
seconds = Number(text);
|
|
91
|
+
} else {
|
|
92
|
+
const parts = text.match(/^(?:\d+[smhdw])+$/) ? text.match(/\d+[smhdw]/g) : null;
|
|
93
|
+
if (!parts) {
|
|
94
|
+
throw new Error(`--freshness-target must be a duration like 30m, 2h, 1d or 1h30m, a number of seconds, or off (got ${JSON.stringify(value)})`);
|
|
95
|
+
}
|
|
96
|
+
seconds = parts.reduce((total, part) => total + Number(part.slice(0, -1)) * FRESHNESS_UNITS[part.slice(-1)], 0);
|
|
97
|
+
}
|
|
98
|
+
if (seconds === 0) return 0;
|
|
99
|
+
if (seconds < FRESHNESS_TARGET_MIN_SECONDS || seconds > FRESHNESS_TARGET_MAX_SECONDS) {
|
|
100
|
+
throw new Error("--freshness-target must be between 5m and 366d, or off to clear it");
|
|
101
|
+
}
|
|
102
|
+
return seconds;
|
|
103
|
+
}
|
|
104
|
+
function formatFreshnessTarget(seconds) {
|
|
105
|
+
const units = [[24 * 60 * 60, "day"], [60 * 60, "hour"], [60, "minute"]];
|
|
106
|
+
const plural = (n, unit) => `${n} ${unit}${n === 1 ? "" : "s"}`;
|
|
107
|
+
for (let i = 0; i < units.length; i++) {
|
|
108
|
+
const [size, name] = units[i];
|
|
109
|
+
if (seconds < size) continue;
|
|
110
|
+
let text = plural(Math.floor(seconds / size), name);
|
|
111
|
+
if (i + 1 < units.length) {
|
|
112
|
+
const minor = Math.floor(seconds % size / units[i + 1][0]);
|
|
113
|
+
if (minor > 0) text += ` ${plural(minor, units[i + 1][1])}`;
|
|
114
|
+
}
|
|
115
|
+
return text;
|
|
116
|
+
}
|
|
117
|
+
return plural(seconds, "second");
|
|
118
|
+
}
|
|
119
|
+
function describeFreshnessTarget(state) {
|
|
120
|
+
if (!state.freshness_target_seconds) return void 0;
|
|
121
|
+
const target = `data no older than ${formatFreshnessTarget(state.freshness_target_seconds)} \u2014 its owner is alerted when it falls behind, and it is never paused for being unread`;
|
|
122
|
+
if (!state.freshness_alert) return target;
|
|
123
|
+
const since = state.freshness_alert_raised_at ? ` since ${state.freshness_alert_raised_at}` : "";
|
|
124
|
+
return `${target}. ALERTING${since}: ${state.freshness_alert}`;
|
|
125
|
+
}
|
|
82
126
|
|
|
83
127
|
// src/index.ts
|
|
84
128
|
import { readFileSync as readFileSync5, writeFileSync as writeFileSync3 } from "fs";
|
|
@@ -1715,6 +1759,12 @@ var ErdoClient = class {
|
|
|
1715
1759
|
updateDatasetSchema(slug, input) {
|
|
1716
1760
|
return this.request("POST", `/v1/datasets/${encodeURIComponent(slug)}/schema`, input);
|
|
1717
1761
|
}
|
|
1762
|
+
// Owner, record role and row provenance are declarations on a durable
|
|
1763
|
+
// dataset, set through the same schema route as class and purpose so one
|
|
1764
|
+
// owner (the dataset service) validates and persists them.
|
|
1765
|
+
setDatasetStewardship(slug, input) {
|
|
1766
|
+
return this.updateDatasetSchema(slug, input);
|
|
1767
|
+
}
|
|
1718
1768
|
// Dataset row writes are deliberately never replayed after a network
|
|
1719
1769
|
// failure: the server may have committed the rows before the response was
|
|
1720
1770
|
// lost, and repeating an append or upsert can change the result.
|
|
@@ -8766,7 +8816,7 @@ datasetsCmd.command("ensure <purpose>").description(
|
|
|
8766
8816
|
fail(e);
|
|
8767
8817
|
}
|
|
8768
8818
|
});
|
|
8769
|
-
datasetsCmd.command("update-schema <slug>").description("Evolve an existing dataset's column contract. For populated lead-identity datasets, use add_column operations only; existing rows and lead ids are preserved.").requiredOption("--input <file>", "JSON request containing operations and optional class/purpose changes").action(async (slug, opts) => {
|
|
8819
|
+
datasetsCmd.command("update-schema <slug>").description("Evolve an existing dataset's column contract. For populated lead-identity datasets, use add_column operations only; existing rows and lead ids are preserved.").requiredOption("--input <file>", "JSON request containing operations and optional class/purpose/owner/record/provenance changes").action(async (slug, opts) => {
|
|
8770
8820
|
try {
|
|
8771
8821
|
const input = JSON.parse(readFileSync5(opts.input, "utf8"));
|
|
8772
8822
|
if (!input || typeof input !== "object" || Array.isArray(input)) {
|
|
@@ -8790,6 +8840,91 @@ datasetsCmd.command("delete <slug>").description("Permanently delete a dataset a
|
|
|
8790
8840
|
fail(e);
|
|
8791
8841
|
}
|
|
8792
8842
|
});
|
|
8843
|
+
function printDatasetStewardship(res) {
|
|
8844
|
+
if (!res) return;
|
|
8845
|
+
const lines = [];
|
|
8846
|
+
if (res.owner) {
|
|
8847
|
+
const who = res.owner.email || res.owner.user_id;
|
|
8848
|
+
lines.push(["owner", res.owner.name ? `${res.owner.name} <${who}>` : who]);
|
|
8849
|
+
}
|
|
8850
|
+
if (res.record_role === "mirror") lines.push(["record", `mirror of ${res.record_of || "(unnamed system)"}`]);
|
|
8851
|
+
else if (res.record_role) lines.push(["record", "this dataset is the record"]);
|
|
8852
|
+
if (res.row_provenance) {
|
|
8853
|
+
lines.push(["provenance", `required: every row needs ${res.row_provenance.source_column} and ${res.row_provenance.checked_at_column}`]);
|
|
8854
|
+
}
|
|
8855
|
+
if (!lines.length) return;
|
|
8856
|
+
console.log("");
|
|
8857
|
+
printAlignedTable(["declares", "value"], lines);
|
|
8858
|
+
}
|
|
8859
|
+
function printStewardshipResult(slug, res) {
|
|
8860
|
+
console.log(slug);
|
|
8861
|
+
if (res.stewardship) printDatasetStewardship(res.stewardship);
|
|
8862
|
+
else console.log("(no owner, record role or provenance requirement declared)");
|
|
8863
|
+
}
|
|
8864
|
+
datasetsCmd.command("set-owner <slug> [owner]").description(
|
|
8865
|
+
"Name the person who answers for a durable dataset: their email, user id, or `me`. The owner must be a member of the dataset's organization. Ownership is accountability, not access; sharing still decides who can read or edit."
|
|
8866
|
+
).option("--clear", "remove the dataset's owner").action(async (slug, owner, opts) => {
|
|
8867
|
+
try {
|
|
8868
|
+
if (opts.clear === Boolean(owner)) {
|
|
8869
|
+
fail(new Error("pass an owner to set, or --clear to remove one"));
|
|
8870
|
+
return;
|
|
8871
|
+
}
|
|
8872
|
+
const res = await new ErdoClient().setDatasetStewardship(slug, opts.clear ? { clear_owner: true } : { owner });
|
|
8873
|
+
printStewardshipResult(slug, res);
|
|
8874
|
+
} catch (e) {
|
|
8875
|
+
fail(e);
|
|
8876
|
+
}
|
|
8877
|
+
});
|
|
8878
|
+
datasetsCmd.command("set-record <slug> [role]").description(
|
|
8879
|
+
'Say whether a durable dataset is the record for its rows (`record`) or a copy of a record another system holds (`mirror`, with --of naming it, e.g. "Attio people"), so people fix a fact once, in the right place.'
|
|
8880
|
+
).option("--of <system>", "with mirror: where the record lives").option("--clear", "remove the dataset's record role").action(async (slug, role, opts) => {
|
|
8881
|
+
try {
|
|
8882
|
+
if (opts.clear) {
|
|
8883
|
+
if (role || opts.of) {
|
|
8884
|
+
fail(new Error("pass a role or --clear, not both"));
|
|
8885
|
+
return;
|
|
8886
|
+
}
|
|
8887
|
+
printStewardshipResult(slug, await new ErdoClient().setDatasetStewardship(slug, { clear_record_role: true }));
|
|
8888
|
+
return;
|
|
8889
|
+
}
|
|
8890
|
+
if (role !== "record" && role !== "mirror") {
|
|
8891
|
+
fail(new Error("pass `record` or `mirror` (with --of <system>), or --clear"));
|
|
8892
|
+
return;
|
|
8893
|
+
}
|
|
8894
|
+
if (role === "mirror" && !opts.of?.trim()) {
|
|
8895
|
+
fail(new Error('a mirror needs --of naming where the record lives, e.g. --of "Attio people"'));
|
|
8896
|
+
return;
|
|
8897
|
+
}
|
|
8898
|
+
if (role === "record" && opts.of) {
|
|
8899
|
+
fail(new Error("--of applies only to mirror; a record dataset is itself the record"));
|
|
8900
|
+
return;
|
|
8901
|
+
}
|
|
8902
|
+
printStewardshipResult(slug, await new ErdoClient().setDatasetStewardship(slug, {
|
|
8903
|
+
record_role: role,
|
|
8904
|
+
...opts.of ? { record_of: opts.of } : {}
|
|
8905
|
+
}));
|
|
8906
|
+
} catch (e) {
|
|
8907
|
+
fail(e);
|
|
8908
|
+
}
|
|
8909
|
+
});
|
|
8910
|
+
datasetsCmd.command("require-provenance <slug>").description(
|
|
8911
|
+
"Require every row of a durable dataset to say where its facts came from and when they were last checked. Later writes whose rows leave either column blank are refused, naming the rows. Both columns must be in the declared schema when the dataset has one. Use --off to remove the requirement."
|
|
8912
|
+
).option("--source-column <name>", "column holding each row's source (default source_url)").option("--checked-at-column <name>", "column holding when each row was last checked (default checked_at)").option("--off", "remove the requirement").action(async (slug, opts) => {
|
|
8913
|
+
try {
|
|
8914
|
+
if (opts.off && (opts.sourceColumn || opts.checkedAtColumn)) {
|
|
8915
|
+
fail(new Error("--off removes the requirement; it takes no column names"));
|
|
8916
|
+
return;
|
|
8917
|
+
}
|
|
8918
|
+
const res = await new ErdoClient().setDatasetStewardship(slug, opts.off ? { provenance_required: false } : {
|
|
8919
|
+
provenance_required: true,
|
|
8920
|
+
...opts.sourceColumn ? { provenance_source_column: opts.sourceColumn } : {},
|
|
8921
|
+
...opts.checkedAtColumn ? { provenance_checked_at_column: opts.checkedAtColumn } : {}
|
|
8922
|
+
});
|
|
8923
|
+
printStewardshipResult(slug, res);
|
|
8924
|
+
} catch (e) {
|
|
8925
|
+
fail(e);
|
|
8926
|
+
}
|
|
8927
|
+
});
|
|
8793
8928
|
datasetsCmd.command("set-purpose <slug> [purpose]").description(
|
|
8794
8929
|
"Set (or with --clear, remove) the dataset's purpose \u2014 the org-vocabulary role `datasets purposes` reports. Use it to CORRECT a purpose that names the wrong dataset: one purpose maps to one dataset per org, so a purpose left on the wrong one points everything that resolves through the vocabulary at the wrong rows. Setting a purpose another dataset carries is an atomic MOVE: the old carrier is cleared in the same transaction, so one call on the right dataset lands it (the response says which dataset it moved from). Use --sibling only when deliberately running a parallel table for the same purpose."
|
|
8795
8930
|
).option("--clear", "remove the dataset's purpose instead of setting one").option(
|
|
@@ -8940,7 +9075,7 @@ Read one with: erdo datasets fetch ${res.dataset_slug} --revision <id> --sql "SE
|
|
|
8940
9075
|
});
|
|
8941
9076
|
datasetsCmd.command("schema <slug>").alias("describe").description(
|
|
8942
9077
|
"Show physical columns by default, or the declared write contract with --declared. File datasets read the stored `data` table; integration datasets show their discovered resource tables and columns. For an integration dataset, choose a resource/table explicitly when fetching: `datasets fetch <slug> --sql 'SELECT * FROM <table>'`."
|
|
8943
|
-
).option("--json", "print the raw JSON result instead of a table").option("--declared", "read the declared write contract (column names, type hints, required flags, semantic roles, currency) instead of the physical columns").action(async (slug, opts) => {
|
|
9078
|
+
).option("--json", "print the raw JSON result instead of a table").option("--declared", "read the declared write contract (column names, type hints, required flags, semantic roles, currency, plus the dataset's owner, record role and provenance requirement) instead of the physical columns").action(async (slug, opts) => {
|
|
8944
9079
|
try {
|
|
8945
9080
|
const client = new ErdoClient();
|
|
8946
9081
|
if (opts.declared) {
|
|
@@ -8956,16 +9091,15 @@ datasetsCmd.command("schema <slug>").alias("describe").description(
|
|
|
8956
9091
|
}
|
|
8957
9092
|
if (schema == null) {
|
|
8958
9093
|
console.log(`${slug} has no declared column contract. Use datasets schema ${slug} to inspect physical columns.`);
|
|
8959
|
-
|
|
8960
|
-
}
|
|
8961
|
-
if (!schema.columns.length) {
|
|
9094
|
+
} else if (!schema.columns.length) {
|
|
8962
9095
|
console.log(`${slug}'s declared column contract contains no columns.`);
|
|
8963
|
-
|
|
9096
|
+
} else {
|
|
9097
|
+
printAlignedTable(
|
|
9098
|
+
["column", "type", "required", "semantic role", "currency", "description"],
|
|
9099
|
+
schema.columns.map((column) => [column.name, column.type || "", column.required ? "yes" : "no", column.semantic_role || "", column.currency || "", column.description || ""])
|
|
9100
|
+
);
|
|
8964
9101
|
}
|
|
8965
|
-
|
|
8966
|
-
["column", "type", "required", "semantic role", "currency", "description"],
|
|
8967
|
-
schema.columns.map((column) => [column.name, column.type || "", column.required ? "yes" : "no", column.semantic_role || "", column.currency || "", column.description || ""])
|
|
8968
|
-
);
|
|
9102
|
+
printDatasetStewardship(res2.stewardship);
|
|
8969
9103
|
return;
|
|
8970
9104
|
}
|
|
8971
9105
|
const res = await client.getDatasetResourceSchema(slug);
|
|
@@ -9128,6 +9262,8 @@ refreshCmd.command("show <slug>").description(
|
|
|
9128
9262
|
const pendingPause = describePendingPause(res);
|
|
9129
9263
|
if (pendingPause) rows.push(["pause warning", pendingPause]);
|
|
9130
9264
|
if (res.keep_running) rows.push(["keep running", "yes \u2014 never paused for being unread"]);
|
|
9265
|
+
const freshness = describeFreshnessTarget(res);
|
|
9266
|
+
if (freshness) rows.push(["freshness target", freshness]);
|
|
9131
9267
|
const eventTriggers = describeEventTriggers(res);
|
|
9132
9268
|
if (eventTriggers) rows.push(["event triggers", eventTriggers]);
|
|
9133
9269
|
if (res.schedule) {
|
|
@@ -9231,6 +9367,9 @@ refreshCmd.command("set <slug>").description(
|
|
|
9231
9367
|
"--debounce <seconds>",
|
|
9232
9368
|
"minimum gap between two view-triggered refreshes",
|
|
9233
9369
|
(v) => parseInt(v, 10)
|
|
9370
|
+
).option(
|
|
9371
|
+
"--freshness-target <duration>",
|
|
9372
|
+
"for a durable dataset people or systems rely on: the longest its data may go without a successful refresh or a write, e.g. 2h, 1d or 1h30m (5m to 366d). If it goes stale, or its refresh is paused or failing, the refresh's owner gets one attention item that resolves itself once the data is fresh again, and the refresh is never paused for being unread. Omit to keep the saved target; pass off to clear it"
|
|
9234
9373
|
).option(
|
|
9235
9374
|
"--live-write-mode <append|replace>",
|
|
9236
9375
|
"for the live strategies: append for a time-series that accumulates, replace for current state that overwrites on each event"
|
|
@@ -9340,6 +9479,9 @@ refreshCmd.command("set <slug>").description(
|
|
|
9340
9479
|
body.refresh_on_view_stale_after_seconds = opts.staleAfter;
|
|
9341
9480
|
}
|
|
9342
9481
|
if (opts.debounce !== void 0) body.refresh_on_view_debounce_seconds = opts.debounce;
|
|
9482
|
+
if (opts.freshnessTarget !== void 0) {
|
|
9483
|
+
body.freshness_target_seconds = parseFreshnessTarget(opts.freshnessTarget);
|
|
9484
|
+
}
|
|
9343
9485
|
if (opts.liveWriteMode) body.live_write_mode = opts.liveWriteMode;
|
|
9344
9486
|
if (opts.payloadSchema) {
|
|
9345
9487
|
body.payload_schema = readRefreshJSONFile(opts.payloadSchema, "--payload-schema");
|
|
@@ -9365,6 +9507,11 @@ refreshCmd.command("set <slug>").description(
|
|
|
9365
9507
|
job_enabled: body.job_enabled,
|
|
9366
9508
|
schedule_enabled: body.schedule_enabled
|
|
9367
9509
|
}));
|
|
9510
|
+
if (res.freshness_target_seconds) {
|
|
9511
|
+
console.log(`freshness target: ${formatFreshnessTarget(res.freshness_target_seconds)} \u2014 its owner is alerted when the data falls behind`);
|
|
9512
|
+
} else if (body.freshness_target_seconds === 0) {
|
|
9513
|
+
console.log("freshness target: cleared");
|
|
9514
|
+
}
|
|
9368
9515
|
if (res.job_id) console.log(`job: ${res.job_id}`);
|
|
9369
9516
|
if (res.webhook_url) {
|
|
9370
9517
|
console.log(`webhook url: ${res.webhook_url}`);
|