keepset 0.4.0 → 0.5.0

package/dist/{bundled-DN3sSHfu.js → bundled-P_132GDn.js}

@@ -46189,4 +46189,4 @@ function getManifest() {
 	return catalogue().manifest;
 }
 //#endregion
-export { getCategory as A, DatasetNotFoundError as B, SnapshotSchema as C, waybackUrl as D, parseEntry as E, GraphNodeSchema as F, NativeParseError as H, GraphV1Schema as I, JsonScalarSchema as L, kindOf as M, listCategories as N, CategoryIdSchema as O, GraphEdgeSchema as P, ArchiveError as R, RelativePathSchema as S, WaybackTimestampSchema as T, RecipeError as U, KeepsetError as V, UnknownCategoryError as W, IdSchema as _, createCatalogue as a, RESERVED_META_KEYS as b, ManifestSchema as c, ArchiveMemberSchema as d, ArchiveSchema as f, HttpUrlSchema as g, EntrySchema as h, CatalogueIndexSchema as i, getTarget as j, countTarget as k, SemVerSchema as l, CrosswalkReferenceSchema as m, getManifest as n, getOwn as o, CitationSchema as p, listDatasets as r, LineageSchema as s, getDataset as t, parseManifest as u, LicenceSchema as v, SourceSchema as w, RecipeSchema as x, ParserSchema as y, CatalogueSyntaxError as z };
+export { countTarget as A, CatalogueSyntaxError as B, SnapshotSchema as C, toIssues as D, parseEntry as E, GraphEdgeSchema as F, UnknownCategoryError as G, KeepsetError as H, GraphNodeSchema as I, GraphV1Schema as L, getTarget as M, kindOf as N, waybackUrl as O, listCategories as P, JsonScalarSchema as R, RelativePathSchema as S, WaybackTimestampSchema as T, NativeParseError as U, DatasetNotFoundError as V, RecipeError as W, IdSchema as _, createCatalogue as a, RESERVED_META_KEYS as b, ManifestSchema as c, ArchiveMemberSchema as d, ArchiveSchema as f, HttpUrlSchema as g, EntrySchema as h, CatalogueIndexSchema as i, getCategory as j, CategoryIdSchema as k, SemVerSchema as l, CrosswalkReferenceSchema as m, getManifest as n, getOwn as o, CitationSchema as p, listDatasets as r, LineageSchema as s, getDataset as t, parseManifest as u, LicenceSchema as v, SourceSchema as w, RecipeSchema as x, ParserSchema as y, ArchiveError as z };

package/dist/index.d.ts

@@ -69,4 +69,12 @@ export declare function getManifest(): Manifest;
  */
 export declare function serialise(doc: Entry | Manifest | Crosswalk): string;
 //#endregion
+//#region src/validate.d.ts
+/**
+ * Checks a dataset as the catalogue holds it: the entry against the entry schema, and its crosswalk against twindex's structural validation and the entry's category. A catalogue index is checked the same way when it is built, so a bundled dataset is valid unless the package itself is broken; the check is what a contributor runs on an entry they are writing.
+ * @param dataset - The dataset.
+ * @returns Every issue found, with paths from the dataset's root (`entry.` or `crosswalk.`); empty when the dataset is valid.
+ */
+export declare function validateDataset(dataset: Dataset): SyntaxIssue[];
+//#endregion
 export { type Archive, type ArchiveMember, ArchiveMemberSchema, ArchiveSchema, type Catalogue, type CatalogueIndex, CatalogueIndexSchema, type Category, type CategoryId, CategoryIdSchema, type CategoryTarget, type CategoryTargets, type Citation, CitationSchema, type CountedKey, type CrosswalkReference, CrosswalkReferenceSchema, type Dataset, type DatasetQuery, type Entry, EntrySchema, type GraphEdge, GraphEdgeSchema, type GraphNode, GraphNodeSchema, type GraphV1, GraphV1Schema, HttpUrlSchema, IdSchema, type JsonScalar, JsonScalarSchema, type Licence, LicenceSchema, type Lineage, LineageSchema, type Manifest, ManifestSchema, type Parser, ParserSchema, RESERVED_META_KEYS, type Recipe, RecipeSchema, type RecipeStep, RelativePathSchema, SemVerSchema, type Snapshot, SnapshotSchema, type Source, SourceSchema, type SyntaxIssue, WaybackTimestampSchema, countTarget, createCatalogue, getCategory, getTarget, kindOf, listCategories, parseEntry, parseManifest, waybackUrl };

package/dist/index.js

@@ -1,4 +1,5 @@
-import { A as getCategory, B as DatasetNotFoundError, C as SnapshotSchema, D as waybackUrl, E as parseEntry, F as GraphNodeSchema, H as NativeParseError, I as GraphV1Schema, L as JsonScalarSchema, M as kindOf, N as listCategories, O as CategoryIdSchema, P as GraphEdgeSchema, R as ArchiveError, S as RelativePathSchema, T as WaybackTimestampSchema, U as RecipeError, V as KeepsetError, W as UnknownCategoryError, _ as IdSchema, a as createCatalogue, b as RESERVED_META_KEYS, c as ManifestSchema, d as ArchiveMemberSchema, f as ArchiveSchema, g as HttpUrlSchema, h as EntrySchema, i as CatalogueIndexSchema, j as getTarget, k as countTarget, l as SemVerSchema, m as CrosswalkReferenceSchema, n as getManifest, p as CitationSchema, r as listDatasets, s as LineageSchema, t as getDataset, u as parseManifest, v as LicenceSchema, w as SourceSchema, x as RecipeSchema, y as ParserSchema, z as CatalogueSyntaxError } from "./bundled-DN3sSHfu.js";
+import { A as countTarget, B as CatalogueSyntaxError, C as SnapshotSchema, D as toIssues, E as parseEntry, F as GraphEdgeSchema, G as UnknownCategoryError, H as KeepsetError, I as GraphNodeSchema, L as GraphV1Schema, M as getTarget, N as kindOf, O as waybackUrl, P as listCategories, R as JsonScalarSchema, S as RelativePathSchema, T as WaybackTimestampSchema, U as NativeParseError, V as DatasetNotFoundError, W as RecipeError, _ as IdSchema, a as createCatalogue, b as RESERVED_META_KEYS, c as ManifestSchema, d as ArchiveMemberSchema, f as ArchiveSchema, g as HttpUrlSchema, h as EntrySchema, i as CatalogueIndexSchema, j as getCategory, k as CategoryIdSchema, l as SemVerSchema, m as CrosswalkReferenceSchema, n as getManifest, p as CitationSchema, r as listDatasets, s as LineageSchema, t as getDataset, u as parseManifest, v as LicenceSchema, w as SourceSchema, x as RecipeSchema, y as ParserSchema, z as ArchiveError } from "./bundled-P_132GDn.js";
+import { validateCrosswalk } from "twindex";
 //#region src/serialise.ts
 /**
 * Copies a JSON value with every object's keys in ascending code-unit order, the order the house JSON lint (`json/sort-keys`) requires of committed JSON. `Object.fromEntries` defines each key as an own data property, so a key such as `__proto__` stays a key.
@@ -27,4 +28,33 @@ function serialise(doc) {
 	return canonicalJson(doc);
 }
 //#endregion
-export { ArchiveError, ArchiveMemberSchema, ArchiveSchema, CatalogueIndexSchema, CatalogueSyntaxError, CategoryIdSchema, CitationSchema, CrosswalkReferenceSchema, DatasetNotFoundError, EntrySchema, GraphEdgeSchema, GraphNodeSchema, GraphV1Schema, HttpUrlSchema, IdSchema, JsonScalarSchema, KeepsetError, LicenceSchema, LineageSchema, ManifestSchema, NativeParseError, ParserSchema, RESERVED_META_KEYS, RecipeError, RecipeSchema, RelativePathSchema, SemVerSchema, SnapshotSchema, SourceSchema, UnknownCategoryError, WaybackTimestampSchema, countTarget, createCatalogue, getCategory, getDataset, getManifest, getTarget, kindOf, listCategories, listDatasets, parseEntry, parseManifest, serialise, waybackUrl };
+//#region src/validate.ts
+/**
+* Checks a dataset as the catalogue holds it: the entry against the entry schema, and its crosswalk against twindex's structural validation and the entry's category. A catalogue index is checked the same way when it is built, so a bundled dataset is valid unless the package itself is broken; the check is what a contributor runs on an entry they are writing.
+* @param dataset - The dataset.
+* @returns Every issue found, with paths from the dataset's root (`entry.` or `crosswalk.`); empty when the dataset is valid.
+*/
+function validateDataset(dataset) {
+	const issues = [];
+	const entry = EntrySchema.safeParse(dataset.entry);
+	if (!entry.success) issues.push(...toIssues(entry.error).map((issue) => ({
+		path: `entry.${issue.path}`,
+		message: issue.message
+	})));
+	const { rules, oneWay } = dataset.crosswalk;
+	for (const issue of validateCrosswalk(dataset.crosswalk)) {
+		const { ruleIndex } = issue;
+		const path = ruleIndex >= 0 && ruleIndex < rules.length ? `crosswalk.rules.${String(ruleIndex)}` : ruleIndex >= rules.length && ruleIndex < rules.length + oneWay.length ? `crosswalk.oneWay.${String(ruleIndex - rules.length)}` : "crosswalk";
+		issues.push({
+			path,
+			message: issue.message
+		});
+	}
+	if (dataset.crosswalk.target !== dataset.entry.category) issues.push({
+		path: "crosswalk.target",
+		message: `must be the entry's category, ${dataset.entry.category}`
+	});
+	return issues;
+}
+//#endregion
+export { ArchiveError, ArchiveMemberSchema, ArchiveSchema, CatalogueIndexSchema, CatalogueSyntaxError, CategoryIdSchema, CitationSchema, CrosswalkReferenceSchema, DatasetNotFoundError, EntrySchema, GraphEdgeSchema, GraphNodeSchema, GraphV1Schema, HttpUrlSchema, IdSchema, JsonScalarSchema, KeepsetError, LicenceSchema, LineageSchema, ManifestSchema, NativeParseError, ParserSchema, RESERVED_META_KEYS, RecipeError, RecipeSchema, RelativePathSchema, SemVerSchema, SnapshotSchema, SourceSchema, UnknownCategoryError, WaybackTimestampSchema, countTarget, createCatalogue, getCategory, getDataset, getManifest, getTarget, kindOf, listCategories, listDatasets, parseEntry, parseManifest, serialise, validateDataset, waybackUrl };

package/dist/node/fetch-dataset.d.ts

@@ -55,4 +55,30 @@ export declare function fetchEntry(dataset: Dataset, options?: FetchOptions): Pr
  * @throws DatasetNotFoundError when no entry has that id, and RecipeError naming the entry and the failing step when the recipe fails.
  */
 export declare function fetchDataset(id: string, options?: FetchOptions): Promise<FetchedDataset>;
+/** Whether decoding a dataset's encoded native document gives the native document back, with the provenance of the bytes it was checked on. */
+export type RoundTripReport = {
+  readonly holds: true;
+  readonly provenance: Provenance;
+} | {
+  readonly holds: false;
+  readonly path: string;
+  readonly message: string;
+  readonly provenance: Provenance;
+};
+/**
+ * Checks a dataset's crosswalk on its real data: runs the recipe to the native document, then twindex's `assertRoundTrip`, which encodes it, decodes the result and compares the two over every path the crosswalk promises to preserve.
+ * @param dataset - A dataset from any catalogue.
+ * @param options - The network port, abort signal and clock.
+ * @returns Whether the round trip holds, and where it first differs when it does not.
+ * @throws RecipeError naming the entry and the failing step when the recipe fails before the comparison, or when the crosswalk cannot run.
+ */
+export declare function roundTripEntry(dataset: Dataset, options?: FetchOptions): Promise<RoundTripReport>;
+/**
+ * Runs `roundTripEntry` on a dataset of the bundled catalogue by id.
+ * @param id - The entry id.
+ * @param options - The network port, abort signal and clock.
+ * @returns Whether the round trip holds.
+ * @throws DatasetNotFoundError when no entry has that id, and RecipeError as `roundTripEntry` does.
+ */
+export declare function roundTripDataset(id: string, options?: FetchOptions): Promise<RoundTripReport>;
 //#endregion

package/dist/node/fetch-dataset.js

@@ -1,5 +1,5 @@
-import { H as NativeParseError, R as ArchiveError, U as RecipeError, V as KeepsetError, j as getTarget, k as countTarget, o as getOwn, t as getDataset } from "../bundled-DN3sSHfu.js";
-import { encode } from "twindex";
+import { A as countTarget, H as KeepsetError, M as getTarget, U as NativeParseError, W as RecipeError, o as getOwn, t as getDataset, z as ArchiveError } from "../bundled-P_132GDn.js";
+import { RoundTripMismatchError, assertRoundTrip, encode } from "twindex";
 import { createHash } from "node:crypto";
 import { gunzipSync } from "node:zlib";
 import { unzipSync } from "fflate";
@@ -832,14 +832,13 @@ function metaOf(entry) {
 	};
 }
 /**
-* Runs a dataset's recipe: downloads the source (from its pinned snapshot when the source is dead), unpacks the archive, parses the native shape, applies the twindex crosswalk and validates the result against the category target. The native document the crosswalk receives is the parser's output with `meta` built from the entry.
-* @param dataset - A dataset from any catalogue.
+* Runs a recipe's download, extract and parse steps, and builds the native document the crosswalk receives: the parser's output with `meta` built from the entry.
+* @param dataset - The dataset.
 * @param options - The network port, abort signal and clock.
-* @returns The resolved target with its provenance and any count warnings.
-* @throws RecipeError naming the entry and the failing step; nothing partial or default-shaped is ever returned.
+* @returns The native document and the provenance of its bytes.
 */
-async function fetchEntry(dataset, options = {}) {
-	const { entry, crosswalk } = dataset;
+async function runToNative(dataset, options) {
+	const { entry } = dataset;
 	const { fetch: fetchPort = downloadRaw, signal, now = () => /* @__PURE__ */ new Date() } = options;
 	const { recipe } = entry;
 	if (recipe.status === "not-fetchable") throw new RecipeError(entry.id, "recipe", `the entry has no fetchable recipe: ${recipe.reason}`);
@@ -853,22 +852,13 @@ async function fetchEntry(dataset, options = {}) {
 		return new Uint8Array(await response.arrayBuffer());
 	});
 	const unpacked = await step(entry.id, "extract", () => unpack(bytes, recipe.archive));
-	const native = await step(entry.id, "parse", () => parseNative(unpacked.bytes, recipe.parser));
-	const encoded = await step(entry.id, "crosswalk", () => encode(crosswalk, {
-		meta: metaOf(entry),
-		nodes: native.nodes,
-		edges: native.edges
-	}));
-	const data = await step(entry.id, "validate", () => {
-		const result = getTarget(entry.category).safeParse(encoded);
-		if (!result.success) throw new KeepsetError(`the result is not ${entry.category}: ${result.error.issues.map((issue) => `${issue.path.map(String).join(".") || "(root)"}: ${issue.message}`).join("; ")}`);
-		return result.data;
-	});
-	const counts = countTarget(entry.category, data);
-	const warnings = Object.entries(entry.expectedCounts).filter(([key, expected]) => counts.get(key) !== expected).map(([key, expected]) => `${key}: the entry records ${String(expected)}, the fetched result has ${String(counts.get(key))}`);
+	const parsed = await step(entry.id, "parse", () => parseNative(unpacked.bytes, recipe.parser));
 	return {
-		dataset,
-		data,
+		native: {
+			meta: metaOf(entry),
+			nodes: parsed.nodes,
+			edges: parsed.edges
+		},
 		provenance: {
 			entryId: entry.id,
 			catalogue: dataset.catalogue,
@@ -883,8 +873,31 @@ async function fetchEntry(dataset, options = {}) {
 			sha256: createHash("sha256").update(bytes).digest("hex"),
 			byteLength: bytes.byteLength,
 			...unpacked.member === void 0 ? {} : { member: unpacked.member }
-		},
-		warnings
+		}
+	};
+}
+/**
+* Runs a dataset's recipe: downloads the source (from its pinned snapshot when the source is dead), unpacks the archive, parses the native shape, applies the twindex crosswalk and validates the result against the category target. The native document the crosswalk receives is the parser's output with `meta` built from the entry.
+* @param dataset - A dataset from any catalogue.
+* @param options - The network port, abort signal and clock.
+* @returns The resolved target with its provenance and any count warnings.
+* @throws RecipeError naming the entry and the failing step; nothing partial or default-shaped is ever returned.
+*/
+async function fetchEntry(dataset, options = {}) {
+	const { entry, crosswalk } = dataset;
+	const { native, provenance } = await runToNative(dataset, options);
+	const encoded = await step(entry.id, "crosswalk", () => encode(crosswalk, native));
+	const data = await step(entry.id, "validate", () => {
+		const result = getTarget(entry.category).safeParse(encoded);
+		if (!result.success) throw new KeepsetError(`the result is not ${entry.category}: ${result.error.issues.map((issue) => `${issue.path.map(String).join(".") || "(root)"}: ${issue.message}`).join("; ")}`);
+		return result.data;
+	});
+	const counts = countTarget(entry.category, data);
+	return {
+		dataset,
+		data,
+		provenance,
+		warnings: Object.entries(entry.expectedCounts).filter(([key, expected]) => counts.get(key) !== expected).map(([key, expected]) => `${key}: the entry records ${String(expected)}, the fetched result has ${String(counts.get(key))}`)
 	};
 }
 /**
@@ -897,5 +910,40 @@ async function fetchEntry(dataset, options = {}) {
 async function fetchDataset(id, options = {}) {
 	return fetchEntry(getDataset(id), options);
 }
+/**
+* Checks a dataset's crosswalk on its real data: runs the recipe to the native document, then twindex's `assertRoundTrip`, which encodes it, decodes the result and compares the two over every path the crosswalk promises to preserve.
+* @param dataset - A dataset from any catalogue.
+* @param options - The network port, abort signal and clock.
+* @returns Whether the round trip holds, and where it first differs when it does not.
+* @throws RecipeError naming the entry and the failing step when the recipe fails before the comparison, or when the crosswalk cannot run.
+*/
+async function roundTripEntry(dataset, options = {}) {
+	const { native, provenance } = await runToNative(dataset, options);
+	try {
+		assertRoundTrip(dataset.crosswalk, [native]);
+	} catch (error) {
+		if (error instanceof RoundTripMismatchError) return {
+			holds: false,
+			path: error.path,
+			message: error.message,
+			provenance
+		};
+		throw new RecipeError(dataset.entry.id, "crosswalk", error instanceof Error ? error.message : String(error), { cause: error });
+	}
+	return {
+		holds: true,
+		provenance
+	};
+}
+/**
+* Runs `roundTripEntry` on a dataset of the bundled catalogue by id.
+* @param id - The entry id.
+* @param options - The network port, abort signal and clock.
+* @returns Whether the round trip holds.
+* @throws DatasetNotFoundError when no entry has that id, and RecipeError as `roundTripEntry` does.
+*/
+async function roundTripDataset(id, options = {}) {
+	return roundTripEntry(getDataset(id), options);
+}
 //#endregion
-export { downloadUrl, fetchDataset, fetchEntry };
+export { downloadUrl, fetchDataset, fetchEntry, roundTripDataset, roundTripEntry };

package/dist-cli/keepset.mjs

@@ -0,0 +1,246 @@
+#!/usr/bin/env node
+import { writeFileSync } from "node:fs";
+import { KeepsetError, countTarget, getDataset, listDatasets, validateDataset } from "keepset";
+import { fetchDataset, roundTripDataset } from "keepset/node";
+import { parseArgs } from "node:util";
+//#region package.json
+var version = "0.5.0";
+const usage = `Usage: keepset <command> [options]
+
+Commands:
+  search [text] [--kind <kind>] [--domain <domain>] [--category <id>] [--json]
+                      list the datasets whose id, name, description, domain or
+                      tags contain the text, filtered by kind, domain or category
+  show <id> [--json]  print a dataset's metadata, recipe and crosswalk
+  fetch <id> [--out <file>]
+                      run the dataset's recipe and write the normalised result
+                      to <file> (default <id>.json), with its provenance beside
+                      it in <file without .json>.provenance.json
+  validate <id>       check the entry against the entry schema and its
+                      crosswalk against twindex's validation
+  roundtrip <id>      fetch the dataset and check that decoding the encoded
+                      native document gives it back
+
+Options:
+  --help, -h          print this help
+  --version, -v       print the version
+
+Exit status: 0 for a result, a valid entry or a round trip that holds; 1 for an
+invalid entry or a round trip that does not hold; 2 when there is no answer:
+bad usage, an unknown dataset or a recipe that failed.
+`;
+function parseCommandLine(args) {
+	return parseArgs({
+		args: [...args],
+		allowPositionals: true,
+		strict: true,
+		options: {
+			kind: { type: "string" },
+			domain: { type: "string" },
+			category: { type: "string" },
+			json: { type: "boolean" },
+			out: { type: "string" },
+			help: {
+				type: "boolean",
+				short: "h"
+			},
+			version: {
+				type: "boolean",
+				short: "v"
+			}
+		}
+	});
+}
+/**
+* JSON as the command prints it: two-space indented, with a trailing newline.
+* @param value - The value.
+* @returns The text.
+*/
+function json(value) {
+	return `${JSON.stringify(value, void 0, 2)}\n`;
+}
+/**
+* One line per dataset for `search`: id, name, category and whether its recipe can run, tab-separated so the output pipes into other tools.
+* @param datasets - The datasets.
+* @returns The text.
+*/
+function formatSearch(datasets) {
+	return datasets.map(({ entry }) => `${entry.id}\t${entry.name}\t${entry.category}\t${entry.recipe.status}\n`).join("");
+}
+/**
+* The human-readable form of `show`: the descriptor, sources, licence, citations, recipe and crosswalk rules, one labelled section after another.
+* @param dataset - The dataset.
+* @returns The text.
+*/
+function formatShow(dataset) {
+	const { entry, crosswalk } = dataset;
+	const lines = [`${entry.name} (${entry.id})`];
+	if (entry.description !== void 0) lines.push("", entry.description);
+	lines.push("", `category: ${entry.category} (kind ${dataset.kind})`, `catalogue: ${dataset.catalogue.id} ${dataset.catalogue.version}`);
+	if (entry.domain !== void 0) lines.push(`domain: ${entry.domain}`);
+	if (entry.tags.length > 0) lines.push(`tags: ${entry.tags.join(", ")}`);
+	const { licence } = entry;
+	lines.push(`licence: ${licence.kind === "spdx" ? licence.id : licence.kind === "custom" ? `${licence.name} <${licence.url}>` : "not recorded by the source"}`);
+	lines.push("", "sources:");
+	for (const [index, source] of entry.sources.entries()) {
+		const snapshot = source.snapshot === void 0 ? "" : `, snapshot ${source.snapshot.timestamp}`;
+		lines.push(`  [${String(index)}] ${source.url} (${source.status}, retrieved ${source.retrieved}${snapshot})`);
+	}
+	if (entry.citations.length > 0) {
+		lines.push("", "citations:");
+		for (const citation of entry.citations) {
+			const link = citation.doi === void 0 ? citation.url === void 0 ? "" : ` <${citation.url}>` : ` doi:${citation.doi}`;
+			lines.push(`  ${citation.text}${link}`);
+		}
+	}
+	const { recipe } = entry;
+	lines.push("", "recipe:");
+	if (recipe.status === "not-fetchable") lines.push(`  not fetchable: ${recipe.reason}`);
+	else {
+		const { member } = "member" in recipe.archive ? recipe.archive : { member: void 0 };
+		const memberText = member === void 0 ? "" : "path" in member ? `, member ${member.path}` : `, member ending ${member.extensions.join(" or ")}`;
+		const { format, ...options } = recipe.parser;
+		lines.push(`  source: [${String(recipe.source)}]`, `  archive: ${recipe.archive.kind}${memberText}`, `  parser: ${format} ${JSON.stringify(options)}`);
+	}
+	lines.push("", `crosswalk: ${crosswalk.source} -> ${crosswalk.target}`);
+	for (const rule of crosswalk.rules) lines.push(`  ${rule.op} ${rule.from} -> ${rule.to}`);
+	for (const rule of crosswalk.oneWay) lines.push(rule.op === "drop" ? `  drop ${rule.from} (encode)` : `  default ${rule.to} = ${JSON.stringify(rule.value)} (${rule.direction})`);
+	const counts = Object.entries(entry.expectedCounts);
+	if (counts.length > 0) lines.push("", `expected counts: ${counts.map(([key, count]) => `${key} ${String(count)}`).join(", ")}`);
+	const properties = Object.entries(entry.properties);
+	if (properties.length > 0) lines.push("", "properties:", ...properties.map(([key, value]) => `  ${key}: ${JSON.stringify(value)}`));
+	return `${lines.join("\n")}\n`;
+}
+/**
+* Where `fetch` writes provenance: beside the result, `.json` replaced by `.provenance.json` (or the suffix appended to a path without one).
+* @param out - The result path.
+* @returns The provenance path.
+*/
+function provenancePath(out) {
+	return out.endsWith(".json") ? `${out.slice(0, -5)}.provenance.json` : `${out}.provenance.json`;
+}
+/**
+* Formats a list of validation issues, one per line.
+* @param issues - The issues.
+* @returns The text.
+*/
+function formatIssues(issues) {
+	return issues.map((issue) => `${issue.path}: ${issue.message}\n`).join("");
+}
+/**
+* Runs the command. A KeepsetError from the library (an unknown dataset, a failing recipe) is the command's failure to answer and becomes exit status 2 with its message; anything else is a bug and propagates.
+* @param args - The arguments after the command name.
+* @param library - The library calls.
+* @param io - Output.
+* @returns The exit status.
+*/
+async function run(args, library, io) {
+	let parsed;
+	try {
+		parsed = parseCommandLine(args);
+	} catch (error) {
+		io.writeStderr(`${error instanceof Error ? error.message : String(error)}\n\n${usage}`);
+		return 2;
+	}
+	const { values, positionals } = parsed;
+	if (values.version === true) {
+		io.writeStdout(`${io.version}\n`);
+		return 0;
+	}
+	if (values.help === true) {
+		io.writeStdout(usage);
+		return 0;
+	}
+	const [command, ...rest] = positionals;
+	const reject = (message) => {
+		io.writeStderr(`${message}\n\n${usage}`);
+		return 2;
+	};
+	try {
+		if (command === "search") {
+			if (rest.length > 1) return reject("search takes at most one text argument; quote a phrase");
+			const [text] = rest;
+			const query = {
+				...text === void 0 ? {} : { text },
+				...values.kind === void 0 ? {} : { kind: values.kind },
+				...values.domain === void 0 ? {} : { domain: values.domain },
+				...values.category === void 0 ? {} : { category: values.category }
+			};
+			const datasets = library.listDatasets(query);
+			io.writeStdout(values.json === true ? json(datasets.map((dataset) => dataset.entry)) : formatSearch(datasets));
+			return 0;
+		}
+		const [id, ...extra] = rest;
+		if (command === void 0) return reject("expected a command");
+		if (![
+			"show",
+			"fetch",
+			"validate",
+			"roundtrip"
+		].includes(command)) return reject(`unknown command ${command}`);
+		if (id === void 0 || extra.length > 0) return reject(`${command} takes exactly one dataset id`);
+		if (command === "show") {
+			const dataset = library.getDataset(id);
+			io.writeStdout(values.json === true ? json(dataset) : formatShow(dataset));
+			return 0;
+		}
+		if (command === "validate") {
+			const issues = library.validateDataset(library.getDataset(id));
+			if (issues.length > 0) {
+				io.writeStdout(formatIssues(issues));
+				return 1;
+			}
+			io.writeStdout(`${id} is valid\n`);
+			return 0;
+		}
+		if (command === "fetch") {
+			const fetched = await library.fetchDataset(id);
+			const out = values.out ?? `${id}.json`;
+			io.writeFile(out, json(fetched.data));
+			io.writeFile(provenancePath(out), json(fetched.provenance));
+			for (const warning of fetched.warnings) io.writeStderr(`warning: ${warning}\n`);
+			const counts = [...library.countTarget(fetched)].map(([key, count]) => `${key} ${String(count)}`).join(", ");
+			io.writeStdout(`wrote ${out} (${counts}) and ${provenancePath(out)}\n`);
+			return 0;
+		}
+		const report = await library.roundTripDataset(id);
+		if (report.holds) {
+			io.writeStdout(`${id}: the round trip holds\n`);
+			return 0;
+		}
+		io.writeStdout(`${id}: the round trip does not hold at ${report.path === "" ? "(root)" : report.path}: ${report.message}\n`);
+		return 1;
+	} catch (error) {
+		if (error instanceof KeepsetError) {
+			io.writeStderr(`${error.message}\n`);
+			return 2;
+		}
+		throw error;
+	}
+}
+//#endregion
+//#region cli/keepset.ts
+process.stdout.on("error", (error) => {
+	if (error.code !== "EPIPE") throw error;
+});
+process.exitCode = await run(process.argv.slice(2), {
+	listDatasets,
+	getDataset,
+	validateDataset,
+	fetchDataset: async (id) => fetchDataset(id),
+	roundTripDataset: async (id) => roundTripDataset(id),
+	countTarget: (fetched) => countTarget(fetched.dataset.entry.category, fetched.data)
+}, {
+	writeStdout: (text) => {
+		process.stdout.write(text);
+	},
+	writeStderr: (text) => {
+		process.stderr.write(text);
+	},
+	writeFile: (path, text) => {
+		writeFileSync(path, text, "utf8");
+	},
+	version
+});
+//#endregion
+export {};

package/package.json

@@ -1,7 +1,10 @@
 {
   "name": "keepset",
   "description": "Library, CLI and statically generated catalogue for dataset fetch-and-normalise recipes",
-  "version": "0.4.0",
+  "version": "0.5.0",
+  "bin": {
+    "keepset": "./dist-cli/keepset.mjs"
+  },
   "dependencies": {
     "fflate": "0.8.3",
     "twindex": "0.1.1",
@@ -49,6 +52,7 @@
   },
   "files": [
     "dist",
+    "dist-cli",
     "schemas"
   ],
   "keywords": [

package/schemas/entry.schema.json

@@ -1 +1 @@
-{"$defs":{"Crosswalk":{"$ref":"https://cdn.jsdelivr.net/npm/twindex@0.1.1/schemas/twindex.schema.json"},"Entry":{"additionalProperties":false,"properties":{"category":{"enum":["graph/v1"],"type":"string"},"citations":{"items":{"additionalProperties":false,"properties":{"doi":{"pattern":"^10\\.\\d{4,9}\\/\\S+$","type":"string"},"text":{"minLength":1,"type":"string"},"url":{"format":"uri","type":"string"}},"required":["text"],"type":"object"},"type":"array"},"crosswalk":{"anyOf":[{"$ref":"#/$defs/Crosswalk"},{"additionalProperties":false,"properties":{"file":{"type":"string"}},"required":["file"],"type":"object"}]},"description":{"minLength":1,"type":"string"},"domain":{"minLength":1,"type":"string"},"expectedCounts":{"additionalProperties":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"propertyNames":{"pattern":"^[A-Za-z][A-Za-z0-9]*$","type":"string"},"type":"object"},"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"licence":{"oneOf":[{"additionalProperties":false,"properties":{"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9.+-]*$","type":"string"},"kind":{"const":"spdx","type":"string"}},"required":["kind","id"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"custom","type":"string"},"name":{"minLength":1,"type":"string"},"url":{"format":"uri","type":"string"}},"required":["kind","name","url"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"unrecorded","type":"string"}},"required":["kind"],"type":"object"}]},"name":{"minLength":1,"type":"string"},"properties":{"additionalProperties":{"type":["string","number","boolean","null"]},"propertyNames":{"pattern":"^[A-Za-z][A-Za-z0-9]*$","type":"string"},"type":"object"},"recipe":{"oneOf":[{"additionalProperties":false,"properties":{"archive":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"none","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"gzip","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"zip","type":"string"},"member":{"anyOf":[{"additionalProperties":false,"properties":{"path":{"type":"string"}},"required":["path"],"type":"object"},{"additionalProperties":false,"properties":{"extensions":{"items":{"pattern":"^\\.[A-Za-z0-9][A-Za-z0-9.]*$","type":"string"},"minItems":1,"type":"array"}},"required":["extensions"],"type":"object"}]}},"required":["kind","member"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"tar","type":"string"},"member":{"anyOf":[{"additionalProperties":false,"properties":{"path":{"type":"string"}},"required":["path"],"type":"object"},{"additionalProperties":false,"properties":{"extensions":{"items":{"pattern":"^\\.[A-Za-z0-9][A-Za-z0-9.]*$","type":"string"},"minItems":1,"type":"array"}},"required":["extensions"],"type":"object"}]}},"required":["kind","member"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"tar.gz","type":"string"},"member":{"anyOf":[{"additionalProperties":false,"properties":{"path":{"type":"string"}},"required":["path"],"type":"object"},{"additionalProperties":false,"properties":{"extensions":{"items":{"pattern":"^\\.[A-Za-z0-9][A-Za-z0-9.]*$","type":"string"},"minItems":1,"type":"array"}},"required":["extensions"],"type":"object"}]}},"required":["kind","member"],"type":"object"}]},"parser":{"oneOf":[{"additionalProperties":false,"properties":{"commentPrefixes":{"items":{"minLength":1,"type":"string"},"type":"array"},"delimiter":{"enum":["detect","whitespace","tab","comma","space"],"type":"string"},"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"edge-list","type":"string"},"skipLines":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"sourceColumn":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"targetColumn":{"maximum":9007199254740991,"minimum":0,"type":"integer"}},"required":["format","encoding","delimiter","skipLines","commentPrefixes","sourceColumn","targetColumn"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"matrix-market","type":"string"}},"required":["format","encoding"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"gml","type":"string"}},"required":["format","encoding"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"pajek","type":"string"}},"required":["format","encoding"],"type":"object"}]},"source":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"status":{"const":"fetchable","type":"string"}},"required":["status","source","archive","parser"],"type":"object"},{"additionalProperties":false,"properties":{"reason":{"minLength":1,"type":"string"},"status":{"const":"not-fetchable","type":"string"}},"required":["status","reason"],"type":"object"}]},"sources":{"items":{"additionalProperties":false,"properties":{"retrieved":{"format":"date","pattern":"^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))$","type":"string"},"snapshot":{"additionalProperties":false,"properties":{"timestamp":{"pattern":"^\\d{8}(?:\\d{6})?$","type":"string"},"url":{"format":"uri","type":"string"}},"required":["url","timestamp"],"type":"object"},"status":{"enum":["live","dead"],"type":"string"},"url":{"format":"uri","type":"string"}},"required":["url","retrieved","status"],"type":"object"},"minItems":1,"type":"array"},"tags":{"items":{"minLength":1,"type":"string"},"type":"array"}},"required":["id","name","tags","category","sources","licence","citations","recipe","crosswalk","expectedCounts","properties"],"type":"object"}},"$id":"https://cdn.jsdelivr.net/npm/keepset@0.4.0/schemas/entry.schema.json","$ref":"#/$defs/Entry","$schema":"https://json-schema.org/draft/2020-12/schema"}
+{"$defs":{"Crosswalk":{"$ref":"https://cdn.jsdelivr.net/npm/twindex@0.1.1/schemas/twindex.schema.json"},"Entry":{"additionalProperties":false,"properties":{"category":{"enum":["graph/v1"],"type":"string"},"citations":{"items":{"additionalProperties":false,"properties":{"doi":{"pattern":"^10\\.\\d{4,9}\\/\\S+$","type":"string"},"text":{"minLength":1,"type":"string"},"url":{"format":"uri","type":"string"}},"required":["text"],"type":"object"},"type":"array"},"crosswalk":{"anyOf":[{"$ref":"#/$defs/Crosswalk"},{"additionalProperties":false,"properties":{"file":{"type":"string"}},"required":["file"],"type":"object"}]},"description":{"minLength":1,"type":"string"},"domain":{"minLength":1,"type":"string"},"expectedCounts":{"additionalProperties":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"propertyNames":{"pattern":"^[A-Za-z][A-Za-z0-9]*$","type":"string"},"type":"object"},"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"licence":{"oneOf":[{"additionalProperties":false,"properties":{"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9.+-]*$","type":"string"},"kind":{"const":"spdx","type":"string"}},"required":["kind","id"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"custom","type":"string"},"name":{"minLength":1,"type":"string"},"url":{"format":"uri","type":"string"}},"required":["kind","name","url"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"unrecorded","type":"string"}},"required":["kind"],"type":"object"}]},"name":{"minLength":1,"type":"string"},"properties":{"additionalProperties":{"type":["string","number","boolean","null"]},"propertyNames":{"pattern":"^[A-Za-z][A-Za-z0-9]*$","type":"string"},"type":"object"},"recipe":{"oneOf":[{"additionalProperties":false,"properties":{"archive":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"none","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"gzip","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"zip","type":"string"},"member":{"anyOf":[{"additionalProperties":false,"properties":{"path":{"type":"string"}},"required":["path"],"type":"object"},{"additionalProperties":false,"properties":{"extensions":{"items":{"pattern":"^\\.[A-Za-z0-9][A-Za-z0-9.]*$","type":"string"},"minItems":1,"type":"array"}},"required":["extensions"],"type":"object"}]}},"required":["kind","member"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"tar","type":"string"},"member":{"anyOf":[{"additionalProperties":false,"properties":{"path":{"type":"string"}},"required":["path"],"type":"object"},{"additionalProperties":false,"properties":{"extensions":{"items":{"pattern":"^\\.[A-Za-z0-9][A-Za-z0-9.]*$","type":"string"},"minItems":1,"type":"array"}},"required":["extensions"],"type":"object"}]}},"required":["kind","member"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"tar.gz","type":"string"},"member":{"anyOf":[{"additionalProperties":false,"properties":{"path":{"type":"string"}},"required":["path"],"type":"object"},{"additionalProperties":false,"properties":{"extensions":{"items":{"pattern":"^\\.[A-Za-z0-9][A-Za-z0-9.]*$","type":"string"},"minItems":1,"type":"array"}},"required":["extensions"],"type":"object"}]}},"required":["kind","member"],"type":"object"}]},"parser":{"oneOf":[{"additionalProperties":false,"properties":{"commentPrefixes":{"items":{"minLength":1,"type":"string"},"type":"array"},"delimiter":{"enum":["detect","whitespace","tab","comma","space"],"type":"string"},"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"edge-list","type":"string"},"skipLines":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"sourceColumn":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"targetColumn":{"maximum":9007199254740991,"minimum":0,"type":"integer"}},"required":["format","encoding","delimiter","skipLines","commentPrefixes","sourceColumn","targetColumn"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"matrix-market","type":"string"}},"required":["format","encoding"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"gml","type":"string"}},"required":["format","encoding"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"pajek","type":"string"}},"required":["format","encoding"],"type":"object"}]},"source":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"status":{"const":"fetchable","type":"string"}},"required":["status","source","archive","parser"],"type":"object"},{"additionalProperties":false,"properties":{"reason":{"minLength":1,"type":"string"},"status":{"const":"not-fetchable","type":"string"}},"required":["status","reason"],"type":"object"}]},"sources":{"items":{"additionalProperties":false,"properties":{"retrieved":{"format":"date","pattern":"^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))$","type":"string"},"snapshot":{"additionalProperties":false,"properties":{"timestamp":{"pattern":"^\\d{8}(?:\\d{6})?$","type":"string"},"url":{"format":"uri","type":"string"}},"required":["url","timestamp"],"type":"object"},"status":{"enum":["live","dead"],"type":"string"},"url":{"format":"uri","type":"string"}},"required":["url","retrieved","status"],"type":"object"},"minItems":1,"type":"array"},"tags":{"items":{"minLength":1,"type":"string"},"type":"array"}},"required":["id","name","tags","category","sources","licence","citations","recipe","crosswalk","expectedCounts","properties"],"type":"object"}},"$id":"https://cdn.jsdelivr.net/npm/keepset@0.5.0/schemas/entry.schema.json","$ref":"#/$defs/Entry","$schema":"https://json-schema.org/draft/2020-12/schema"}

package/schemas/manifest.schema.json

@@ -1 +1 @@
-{"$defs":{"Manifest":{"additionalProperties":false,"properties":{"description":{"minLength":1,"type":"string"},"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"lineage":{"additionalProperties":false,"properties":{"catalogue":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"repository":{"format":"uri","type":"string"},"version":{"pattern":"^(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)(?:-[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?(?:\\+[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?$","type":"string"}},"required":["catalogue","version","repository"],"type":"object"},"name":{"minLength":1,"type":"string"},"repository":{"format":"uri","type":"string"},"version":{"pattern":"^(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)(?:-[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?(?:\\+[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?$","type":"string"}},"required":["id","name","description","version","repository"],"type":"object"}},"$id":"https://cdn.jsdelivr.net/npm/keepset@0.4.0/schemas/manifest.schema.json","$ref":"#/$defs/Manifest","$schema":"https://json-schema.org/draft/2020-12/schema"}
+{"$defs":{"Manifest":{"additionalProperties":false,"properties":{"description":{"minLength":1,"type":"string"},"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"lineage":{"additionalProperties":false,"properties":{"catalogue":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"repository":{"format":"uri","type":"string"},"version":{"pattern":"^(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)(?:-[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?(?:\\+[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?$","type":"string"}},"required":["catalogue","version","repository"],"type":"object"},"name":{"minLength":1,"type":"string"},"repository":{"format":"uri","type":"string"},"version":{"pattern":"^(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)(?:-[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?(?:\\+[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?$","type":"string"}},"required":["id","name","description","version","repository"],"type":"object"}},"$id":"https://cdn.jsdelivr.net/npm/keepset@0.5.0/schemas/manifest.schema.json","$ref":"#/$defs/Manifest","$schema":"https://json-schema.org/draft/2020-12/schema"}