@erdoai/cli 0.240.0 → 0.241.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +133 -0
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1663,6 +1663,33 @@ var ErdoClient = class {
|
|
|
1663
1663
|
const qs = q.toString();
|
|
1664
1664
|
return this.request("GET", `/v1/datasets${qs ? `?${qs}` : ""}`);
|
|
1665
1665
|
}
|
|
1666
|
+
listDatasetAssertions(slug) {
|
|
1667
|
+
return this.request("GET", `/v1/datasets/${encodeURIComponent(slug)}/assertions`);
|
|
1668
|
+
}
|
|
1669
|
+
setDatasetAssertion(datasetSlug, assertionSlug, input) {
|
|
1670
|
+
return this.request(
|
|
1671
|
+
"PUT",
|
|
1672
|
+
`/v1/datasets/${encodeURIComponent(datasetSlug)}/assertions/${encodeURIComponent(assertionSlug)}`,
|
|
1673
|
+
input,
|
|
1674
|
+
false
|
|
1675
|
+
);
|
|
1676
|
+
}
|
|
1677
|
+
runDatasetAssertions(datasetSlug, slugs) {
|
|
1678
|
+
return this.request(
|
|
1679
|
+
"POST",
|
|
1680
|
+
`/v1/datasets/${encodeURIComponent(datasetSlug)}/assertions/run`,
|
|
1681
|
+
slugs?.length ? { slugs } : {},
|
|
1682
|
+
false
|
|
1683
|
+
);
|
|
1684
|
+
}
|
|
1685
|
+
retireDatasetAssertion(datasetSlug, assertionSlug) {
|
|
1686
|
+
return this.request(
|
|
1687
|
+
"DELETE",
|
|
1688
|
+
`/v1/datasets/${encodeURIComponent(datasetSlug)}/assertions/${encodeURIComponent(assertionSlug)}`,
|
|
1689
|
+
void 0,
|
|
1690
|
+
false
|
|
1691
|
+
);
|
|
1692
|
+
}
|
|
1666
1693
|
getDatasetRowActions(slug, opts) {
|
|
1667
1694
|
const q = new URLSearchParams({ dataset: slug });
|
|
1668
1695
|
if (opts?.include_receipts) q.set("include_receipts", "true");
|
|
@@ -8726,6 +8753,108 @@ datasetPermissionsCmd.command("remove-user <slug> <user-id>").option("--json", "
|
|
|
8726
8753
|
fail(e);
|
|
8727
8754
|
}
|
|
8728
8755
|
});
|
|
8756
|
+
function printDatasetAssertionsSummary(summary, dataset) {
|
|
8757
|
+
const counts = [
|
|
8758
|
+
`passing ${summary.passing}`,
|
|
8759
|
+
`failing ${summary.failing}`,
|
|
8760
|
+
`stale ${summary.stale}`,
|
|
8761
|
+
`could_not_run ${summary.could_not_run}`,
|
|
8762
|
+
`unchecked ${summary.unchecked}`
|
|
8763
|
+
];
|
|
8764
|
+
const slugs = summary.failing_slugs?.length ? `; failing: ${summary.failing_slugs.join(", ")}` : "";
|
|
8765
|
+
console.log(`${dataset ? `${dataset}: ` : ""}assertions \u2014 ${counts.join(", ")}${slugs}`);
|
|
8766
|
+
}
|
|
8767
|
+
function assertionRows(assertions) {
|
|
8768
|
+
const rank = { failing: 0, could_not_run: 1, stale: 2, unchecked: 3, passing: 4 };
|
|
8769
|
+
return [...assertions].sort((a, b) => (rank[a.reader_state] ?? 99) - (rank[b.reader_state] ?? 99) || a.slug.localeCompare(b.slug)).map((a) => [a.slug, a.reader_state, a.result?.failing_count ?? "", a.description]);
|
|
8770
|
+
}
|
|
8771
|
+
function printDatasetAssertions(res, json) {
|
|
8772
|
+
if (json) {
|
|
8773
|
+
print(res);
|
|
8774
|
+
return;
|
|
8775
|
+
}
|
|
8776
|
+
printAlignedTable(["slug", "state", "failing", "description"], assertionRows(res.assertions ?? []));
|
|
8777
|
+
printDatasetAssertionsSummary(res.summary, res.dataset_slug);
|
|
8778
|
+
}
|
|
8779
|
+
function assertionInputFromOptions(opts) {
|
|
8780
|
+
const templateFlags = [opts.notNull, opts.unique, opts.acceptedValues, opts.range, opts.expression].filter((v) => v !== void 0);
|
|
8781
|
+
if (opts.sql !== void 0 && templateFlags.length) throw new Error("--sql cannot be combined with a template rule");
|
|
8782
|
+
if (opts.sql === void 0 && templateFlags.length !== 1) throw new Error("choose exactly one of --sql, --not-null, --unique, --accepted-values, --range, or --expression");
|
|
8783
|
+
if (opts.sql !== void 0 && opts.where !== void 0) throw new Error("--where applies only to template rules");
|
|
8784
|
+
const input = { description: opts.description };
|
|
8785
|
+
if (opts.keyColumn) input.key_column = opts.keyColumn;
|
|
8786
|
+
if (opts.sql !== void 0) {
|
|
8787
|
+
input.sql = readMaybeFile(opts.sql) ?? "";
|
|
8788
|
+
return input;
|
|
8789
|
+
}
|
|
8790
|
+
if (opts.notNull !== void 0) input.template = { kind: "not_null", column: opts.notNull, where: opts.where };
|
|
8791
|
+
else if (opts.unique !== void 0) input.template = { kind: "unique", columns: opts.unique.split(",").map((v) => v.trim()).filter(Boolean), where: opts.where };
|
|
8792
|
+
else if (opts.acceptedValues !== void 0) {
|
|
8793
|
+
const equals = opts.acceptedValues.indexOf("=");
|
|
8794
|
+
if (equals < 1) throw new Error("--accepted-values must be column=value1,value2");
|
|
8795
|
+
input.template = { kind: "accepted_values", column: opts.acceptedValues.slice(0, equals).trim(), values: opts.acceptedValues.slice(equals + 1).split(",").map((v) => v.trim()), where: opts.where };
|
|
8796
|
+
} else if (opts.range !== void 0) {
|
|
8797
|
+
if (opts.min === void 0 && opts.max === void 0) throw new Error("--range needs --min, --max, or both");
|
|
8798
|
+
const min = opts.min === void 0 ? void 0 : Number(opts.min);
|
|
8799
|
+
const max = opts.max === void 0 ? void 0 : Number(opts.max);
|
|
8800
|
+
if (min !== void 0 && !Number.isFinite(min) || max !== void 0 && !Number.isFinite(max)) throw new Error("--min and --max must be finite numbers");
|
|
8801
|
+
input.template = { kind: "range", column: opts.range, ...min === void 0 ? {} : { min }, ...max === void 0 ? {} : { max }, where: opts.where };
|
|
8802
|
+
} else input.template = { kind: "expression", predicate: opts.expression, where: opts.where };
|
|
8803
|
+
return input;
|
|
8804
|
+
}
|
|
8805
|
+
var datasetAssertionsCmd = datasetsCmd.command("assertions").description("Define, run, and inspect read-only data assertions");
|
|
8806
|
+
datasetAssertionsCmd.command("list <dataset>").description("List assertions and their latest reader state").option("--json", "print the raw JSON result").action(async (dataset, opts) => {
|
|
8807
|
+
try {
|
|
8808
|
+
printDatasetAssertions(await new ErdoClient().listDatasetAssertions(dataset), Boolean(opts.json));
|
|
8809
|
+
} catch (e) {
|
|
8810
|
+
fail(e);
|
|
8811
|
+
}
|
|
8812
|
+
});
|
|
8813
|
+
datasetAssertionsCmd.command("set <dataset> <slug>").description("Create or replace an assertion rule").requiredOption("--description <text>", "what must hold and why").option("--sql <sql>", "a read-only SELECT/WITH query; prefix with @ to read a file").option("--not-null <column>", "require a column to be non-null").option("--unique <columns>", "require one or more comma-separated columns to be unique").option("--accepted-values <column=values>", "allow only comma-separated values").option("--range <column>", "require a numeric value within --min/--max").option("--min <n>", "inclusive range minimum").option("--max <n>", "inclusive range maximum").option("--expression <predicate>", "require a SQL predicate for every row").option("--where <predicate>", "scope a template rule to matching rows").option("--key-column <column>", "put this column first in failure samples").action(async (dataset, slug, opts) => {
|
|
8814
|
+
try {
|
|
8815
|
+
const result = await new ErdoClient().setDatasetAssertion(dataset, slug, assertionInputFromOptions(opts));
|
|
8816
|
+
console.log(`${result.assertion.slug} set v${result.assertion.version}`);
|
|
8817
|
+
} catch (e) {
|
|
8818
|
+
fail(e);
|
|
8819
|
+
}
|
|
8820
|
+
});
|
|
8821
|
+
datasetAssertionsCmd.command("run <dataset>").description("Run assertions synchronously; exits 1 when any is failing or could_not_run").option("--slug <slug>", "run only this assertion (repeatable)", collect, []).option("--json", "print the raw JSON result").action(async (dataset, opts) => {
|
|
8822
|
+
try {
|
|
8823
|
+
const res = await new ErdoClient().runDatasetAssertions(dataset, opts.slug.length ? opts.slug : void 0);
|
|
8824
|
+
printDatasetAssertions(res, Boolean(opts.json));
|
|
8825
|
+
if (res.assertions.some((a) => a.reader_state === "failing" || a.reader_state === "could_not_run")) process.exitCode = 1;
|
|
8826
|
+
} catch (e) {
|
|
8827
|
+
fail(e);
|
|
8828
|
+
}
|
|
8829
|
+
});
|
|
8830
|
+
datasetAssertionsCmd.command("retire <dataset> <slug>").description("Retire an assertion without deleting its history").action(async (dataset, slug) => {
|
|
8831
|
+
try {
|
|
8832
|
+
const res = await new ErdoClient().retireDatasetAssertion(dataset, slug);
|
|
8833
|
+
console.log(res.retired ? `retired ${res.slug}` : `not retired ${res.slug}`);
|
|
8834
|
+
} catch (e) {
|
|
8835
|
+
fail(e);
|
|
8836
|
+
}
|
|
8837
|
+
});
|
|
8838
|
+
datasetAssertionsCmd.command("results <dataset> <slug>").description("Show one assertion and its latest failure samples").option("--json", "print the raw JSON result").action(async (dataset, slug, opts) => {
|
|
8839
|
+
try {
|
|
8840
|
+
const res = await new ErdoClient().listDatasetAssertions(dataset);
|
|
8841
|
+
const assertion = (res.assertions ?? []).find((a) => a.slug === slug);
|
|
8842
|
+
if (!assertion) throw new Error(`assertion ${slug} was not found in dataset ${dataset}`);
|
|
8843
|
+
if (opts.json) {
|
|
8844
|
+
print(assertion);
|
|
8845
|
+
return;
|
|
8846
|
+
}
|
|
8847
|
+
console.log(`${assertion.slug} ${assertion.reader_state} ${assertion.description}`);
|
|
8848
|
+
if (assertion.result) {
|
|
8849
|
+
console.log(`failing rows: ${assertion.result.failing_count}`);
|
|
8850
|
+
if (assertion.result.reason) console.log(`reason: ${assertion.result.reason}`);
|
|
8851
|
+
const samples = assertion.result.samples;
|
|
8852
|
+
if (samples?.rows?.length) printAlignedTable(samples.columns, samples.rows);
|
|
8853
|
+
} else console.log("no result yet");
|
|
8854
|
+
} catch (e) {
|
|
8855
|
+
fail(e);
|
|
8856
|
+
}
|
|
8857
|
+
});
|
|
8729
8858
|
datasetsCmd.command("list").description("List datasets").option(
|
|
8730
8859
|
"--class <class>",
|
|
8731
8860
|
"filter by structural class: 'table' or 'scratch', or 'all' to include scratch analysis by-products (default hides scratch)"
|
|
@@ -9459,6 +9588,7 @@ datasetsCmd.command("schema <slug>").alias("describe").description(
|
|
|
9459
9588
|
);
|
|
9460
9589
|
}
|
|
9461
9590
|
printDatasetStewardship(res2.stewardship);
|
|
9591
|
+
if (res2.assertions) printDatasetAssertionsSummary(res2.assertions.summary, slug);
|
|
9462
9592
|
return;
|
|
9463
9593
|
}
|
|
9464
9594
|
const res = await client.getDatasetResourceSchema(slug);
|
|
@@ -9474,6 +9604,7 @@ datasetsCmd.command("schema <slug>").alias("describe").description(
|
|
|
9474
9604
|
}
|
|
9475
9605
|
if (!resources.length) {
|
|
9476
9606
|
console.log(`${slug} has no discovered integration resources yet.`);
|
|
9607
|
+
if (res.assertions) printDatasetAssertionsSummary(res.assertions.summary, slug);
|
|
9477
9608
|
return;
|
|
9478
9609
|
}
|
|
9479
9610
|
const rows = resources.flatMap(
|
|
@@ -9486,6 +9617,7 @@ datasetsCmd.command("schema <slug>").alias("describe").description(
|
|
|
9486
9617
|
printAlignedTable(["table", "column", "type"], rows);
|
|
9487
9618
|
console.log(`
|
|
9488
9619
|
Fetch a selected table with: erdo datasets fetch ${slug} --sql "SELECT * FROM <table>"`);
|
|
9620
|
+
if (res.assertions) printDatasetAssertionsSummary(res.assertions.summary, slug);
|
|
9489
9621
|
return;
|
|
9490
9622
|
}
|
|
9491
9623
|
if (res.schema_provenance !== "dataset_storage" || !Array.isArray(res.columns) || !Array.isArray(res.rows))
|
|
@@ -9504,6 +9636,7 @@ Fetch a selected table with: erdo datasets fetch ${slug} --sql "SELECT * FROM <t
|
|
|
9504
9636
|
["column", "type"],
|
|
9505
9637
|
res.rows.map((r) => [r[nameIdx], r[typeIdx]])
|
|
9506
9638
|
);
|
|
9639
|
+
if (res.assertions) printDatasetAssertionsSummary(res.assertions.summary, slug);
|
|
9507
9640
|
} catch (e) {
|
|
9508
9641
|
fail(e);
|
|
9509
9642
|
}
|