keepset 0.22.1 → 0.24.0

package/dist/node/fetch-dataset.js

@@ -1,4 +1,4 @@
-import { $ as NativeParseError, F as getOwn, J as ArchiveError, L as countTarget, Q as KeepsetError, et as RecipeError, t as getDataset, z as getTarget } from "../bundled-CAsUNCDb.js";
+import { $ as NativeParseError, F as getOwn, J as ArchiveError, L as countTarget, Q as KeepsetError, et as RecipeError, t as getDataset, z as getTarget } from "../bundled-BWINQKLQ.js";
 import { RoundTripMismatchError, assertRoundTrip, encode } from "twindex";
 import { z } from "zod";
 import { createHash } from "node:crypto";
@@ -104,7 +104,7 @@ function parseEdgeList(text, options) {
 //#endregion
 //#region src/parsers/gml.ts
 const KEY = /[A-Za-z_][A-Za-z0-9_]*/y;
-const NUMBER = /[+-]?(?:\d+\.?\d*|\.\d+)(?:[Ee][+-]?\d+)?/y;
+const NUMBER$1 = /[+-]?(?:\d+\.?\d*|\.\d+)(?:[Ee][+-]?\d+)?/y;
 /**
 * Matches a sticky pattern at a position.
 * @param pattern - A sticky regular expression.
@@ -155,7 +155,7 @@ function tokenise(text) {
 			position = end + 1;
 		} else {
 			const key = matchAt(KEY, text, position);
-			const number = key === void 0 ? matchAt(NUMBER, text, position) : void 0;
+			const number = key === void 0 ? matchAt(NUMBER$1, text, position) : void 0;
 			const match = key ?? number;
 			if (match === void 0) throw new NativeParseError("gml", line, `unexpected character ${JSON.stringify(char)}`);
 			tokens.push({
@@ -832,6 +832,89 @@ function parsePajek(text) {
 	};
 }
 //#endregion
+//#region src/parsers/solomon.ts
+const FORMAT = "solomon";
+/** A line of integers or decimals separated by any run of spaces and tabs, as the files write rows (Solomon's with padded spaces, the Tenerife set with tabs). */
+const NUMBER = /^-?\d+(?:\.\d+)?$/;
+/**
+* Reads a token as a finite number.
+* @param token - The token.
+* @param line - The 1-based line, for the error.
+* @param what - What the token should be, for the error.
+* @returns The number.
+* @throws NativeParseError when the token is not a decimal number.
+*/
+function numberOf(token, line, what) {
+	if (!NUMBER.test(token)) throw new NativeParseError(FORMAT, line, `expected ${what}, found ${JSON.stringify(token)}`);
+	return Number(token);
+}
+/**
+* Parses a Solomon style vehicle routing instance (R. M. Solomon, 1987): a name on the first line, a `VEHICLE` section holding the number of vehicles and their capacity, and a `CUSTOMER` section of one row per node, the depot first, with its id, position, demand, ready time, due date and service time. Columns are separated by any run of spaces or tabs, so the original files and the tab separated Tenerife set read alike, and blank or whitespace only lines are skipped. The header lines (`NUMBER CAPACITY` and the column titles) are not read: a row is a line of exactly seven numbers after the `CUSTOMER` keyword, and anything else after it is an error. Node ids are kept as the file writes them, so a file whose ids are not consecutive keeps its own. The time quantities are in the unit the source uses, which the format does not record.
+* @param text - The file text.
+* @returns The instance: its name, vehicle count and capacity as `meta`, and its nodes in file order.
+* @throws NativeParseError naming the line for a missing name, `VEHICLE` or `CUSTOMER` section, a vehicle line that is not two numbers, a customer row that is not seven numbers, a repeated node id, or a file with no customer.
+*/
+function parseSolomon(text) {
+	const all = lines(text);
+	const present = (index) => (all[index] ?? "").trim();
+	const name = present(0);
+	if (name === "") throw new NativeParseError(FORMAT, 1, "expected the instance name on the first line");
+	const meta = { name };
+	const nodes = [];
+	const seen = /* @__PURE__ */ new Set();
+	let section = "preamble";
+	let awaitingHeader = false;
+	for (const [index, raw] of all.entries()) {
+		const line = index + 1;
+		const trimmed = raw.trim();
+		if (index === 0 || trimmed === "") continue;
+		const keyword = trimmed.toUpperCase();
+		if (keyword === "VEHICLE" || keyword === "CUSTOMER") {
+			section = keyword === "VEHICLE" ? "vehicle" : "customer";
+			awaitingHeader = true;
+			continue;
+		}
+		if (section === "preamble") throw new NativeParseError(FORMAT, line, `expected VEHICLE or CUSTOMER, found ${JSON.stringify(trimmed)}`);
+		if (awaitingHeader) {
+			awaitingHeader = false;
+			continue;
+		}
+		const tokens = trimmed.split(/\s+/);
+		if (section === "vehicle") {
+			const [vehicles, capacity, ...extra] = tokens;
+			if (vehicles === void 0 || capacity === void 0 || extra.length > 0) throw new NativeParseError(FORMAT, line, `expected the vehicle count and capacity, found ${JSON.stringify(trimmed)}`);
+			meta.vehicles = numberOf(vehicles, line, "the vehicle count");
+			meta.capacity = numberOf(capacity, line, "the vehicle capacity");
+			continue;
+		}
+		const [id, x, y, demand, ready, due, service, ...extra] = tokens;
+		if (id === void 0 || x === void 0 || y === void 0 || demand === void 0 || ready === void 0 || due === void 0 || service === void 0 || extra.length > 0) throw new NativeParseError(FORMAT, line, `expected seven numbers (id, x, y, demand, ready, due, service), found ${JSON.stringify(trimmed)}`);
+		if (!/^\d+$/.test(id)) throw new NativeParseError(FORMAT, line, `expected a whole number node id, found ${JSON.stringify(id)}`);
+		if (seen.has(id)) throw new NativeParseError(FORMAT, line, `node ${id} appears twice`);
+		seen.add(id);
+		nodes.push({
+			id,
+			x: numberOf(x, line, "x"),
+			y: numberOf(y, line, "y"),
+			demand: numberOf(demand, line, "the demand"),
+			ready: numberOf(ready, line, "the ready time"),
+			due: numberOf(due, line, "the due date"),
+			service: numberOf(service, line, "the service time")
+		});
+	}
+	if (section !== "customer") throw new NativeParseError(FORMAT, all.length, "expected a CUSTOMER section");
+	if (nodes.length === 0) throw new NativeParseError(FORMAT, all.length, "the CUSTOMER section has no rows");
+	if (meta.vehicles === void 0 || meta.capacity === void 0) throw new NativeParseError(FORMAT, all.length, "expected a VEHICLE section with the vehicle count and capacity");
+	return {
+		meta: {
+			name: meta.name,
+			vehicles: meta.vehicles,
+			capacity: meta.capacity
+		},
+		nodes
+	};
+}
+//#endregion
 //#region src/parsers/parse-native.ts
 /** Bytes converted per String.fromCharCode call: spreading a whole file would exceed the engine's argument-count limit, and this many stays well inside every engine's. */
 const LATIN1_CHUNK = 32768;
@@ -887,6 +970,7 @@ function parseNative(bytes, parser) {
 	if (parser.format === "edge-list") return parseEdgeList(text, parser);
 	if (parser.format === "matrix-market") return parseMatrixMarket(text);
 	if (parser.format === "gml") return parseGml(text);
+	if (parser.format === "solomon") return parseSolomon(text);
 	return parsePajek(text);
 }
 /** The ustar header layout (POSIX.1-1988, pax(1) "ustar Interchange Format"): each field's byte offset and length. */
@@ -1285,6 +1369,18 @@ function selects(path, member) {
 	return member.extensions.some((extension) => path.toLowerCase().endsWith(extension.toLowerCase()));
 }
 /**
+* Picks every member a selector matches, in path order, so a collection reads in the same order wherever it is unpacked.
+* @param paths - Every member path.
+* @param member - The recipe's selector.
+* @returns The matching paths, sorted, at least one.
+* @throws ArchiveError when no member matches, listing the members.
+*/
+function pickAll(paths, member) {
+	const matched = paths.filter((path) => selects(path, member)).sort((left, right) => left < right ? -1 : left > right ? 1 : 0);
+	if (matched.length === 0) throw new ArchiveError(`expected at least one ${"path" in member ? `member ${member.path}` : `member ending ${member.extensions.join(" or ")}`}, found none among: ${paths.join(", ")}`);
+	return matched;
+}
+/**
 * Picks the one member a selector matches.
 * @param paths - Every member path.
 * @param member - The recipe's selector.
@@ -1338,6 +1434,42 @@ function unpack(bytes, archive) {
 	};
 }
 /**
+* Unpacks downloaded bytes into every file a collection's selector matches.
+* @param bytes - The downloaded bytes.
+* @param archive - The recipe's archive layout, which selects members with `members`.
+* @returns Each matching file, in path order.
+* @throws ArchiveError when the bytes are not a well-formed archive of the stated kind or the selector matches no member.
+*/
+function unpackMembers(bytes, archive) {
+	if (archive.kind === "zip") {
+		const names = [];
+		unzip(bytes, (file) => {
+			names.push(file.name);
+			return false;
+		});
+		const paths = pickAll(names, archive.members);
+		const chosen = new Set(paths);
+		const files = unzip(bytes, (candidate) => chosen.has(candidate.name));
+		return paths.map((path) => {
+			const file = getOwn(files, path);
+			if (file === void 0) throw new ArchiveError(`zip member ${path} could not be read`);
+			return {
+				path,
+				bytes: file
+			};
+		});
+	}
+	const members = readTar(archive.kind === "tar" ? bytes : gunzip(bytes));
+	return pickAll(members.map((member) => member.path), archive.members).map((path) => {
+		const found = members.find((member) => member.path === path);
+		if (found === void 0) throw new ArchiveError(`tar member ${path} could not be read`);
+		return {
+			path,
+			bytes: found.data
+		};
+	});
+}
+/**
 * Gunzips bytes, rethrowing zlib's failure in the package's taxonomy.
 * @param bytes - Gzip data.
 * @returns The decompressed bytes.
@@ -1682,6 +1814,56 @@ async function downloadCkanDatastore(dump, pageSize, port, signal) {
 		signal
 	});
 }
+const OpenFemaCountSchema = z.object({ metadata: z.object({ count: z.int().min(0) }) });
+/**
+* Downloads a whole OpenFEMA entity, or the rows of it the source URL selects, as one CSV: the row count is read from the API's own metadata, then pages of `pageSize` rows in the order of the entity's id, each checked to hold the rows it should, and joined under the first page's header.
+* @param entity - The entity's URL with any `$select` or `$filter` but no `$top`, `$skip`, `$orderby` or `$format`, such as `https://www.fema.gov/api/open/v3/NfipClaims?$select=id,state`.
+* @param pageSize - How many rows a page holds.
+* @param port - The network port.
+* @param signal - Aborts the download.
+* @returns The bytes of one CSV, which is what the parser reads and what the provenance hashes.
+* @throws KeepsetError when the source URL sets a paging parameter, the entity does not report a count, or a page is not what it should be.
+*/
+async function downloadOpenFema(entity, pageSize, port, signal) {
+	const base = new URL(entity);
+	for (const name of [
+		"$top",
+		"$skip",
+		"$orderby",
+		"$format",
+		"$inlinecount"
+	]) if (base.searchParams.has(name)) throw new KeepsetError(`${entity} sets ${name}, which the download sets itself`);
+	const withParameters = (parameters) => {
+		const url = new URL(entity);
+		for (const [name, value] of Object.entries(parameters)) url.searchParams.set(name, value);
+		return url.href;
+	};
+	const countUrl = withParameters({
+		$inlinecount: "allpages",
+		$top: "1"
+	});
+	let body;
+	try {
+		body = JSON.parse(new TextDecoder("utf-8").decode(await withAttempts(async () => getBytes(port, countUrl, signal), signal)));
+	} catch (error) {
+		throw new KeepsetError(`${countUrl} did not answer JSON (${error instanceof Error ? error.message : String(error)})`);
+	}
+	const count = OpenFemaCountSchema.safeParse(body);
+	if (!count.success) throw new KeepsetError(`${entity} did not report a row count`);
+	return downloadPages({
+		resource: entity,
+		pageUrl: (offset) => withParameters({
+			$format: "csv",
+			$orderby: "id",
+			$top: String(pageSize),
+			$skip: String(offset)
+		}),
+		pageSize,
+		count: count.data.metadata.count,
+		port,
+		signal
+	});
+}
 /**
 * Downloads a paged source as the one file its pages make, in the way its recipe's paging says.
 * @param url - The source's URL.
@@ -1693,7 +1875,8 @@ async function downloadCkanDatastore(dump, pageSize, port, signal) {
 */
 async function downloadPaged(url, paging, port, signal) {
 	if (paging.kind === "arcgis-query") return downloadArcgisLayer(url, paging.orderBy, port, signal);
-	return paging.kind === "socrata-resource" ? downloadSocrataResource(url, paging.pageSize, port, signal) : downloadCkanDatastore(url, paging.pageSize, port, signal);
+	if (paging.kind === "socrata-resource") return downloadSocrataResource(url, paging.pageSize, port, signal);
+	return paging.kind === "ckan-datastore" ? downloadCkanDatastore(url, paging.pageSize, port, signal) : downloadOpenFema(url, paging.pageSize, port, signal);
 }
 //#endregion
 //#region src/node/fetch-dataset.ts
@@ -1757,28 +1940,54 @@ async function runToNative(dataset, options) {
 		if (!response.ok) throw new KeepsetError(`${fetchedUrl} answered ${String(response.status)} ${response.statusText}`);
 		return new Uint8Array(await response.arrayBuffer());
 	});
-	const unpacked = await step(entry.id, "extract", () => unpack(bytes, recipe.archive));
+	const provenanceOf = (extra) => ({
+		entryId: entry.id,
+		catalogue: dataset.catalogue,
+		category: entry.category,
+		source: {
+			url: source.url,
+			status: source.status,
+			...source.snapshot === void 0 ? {} : { snapshot: source.snapshot }
+		},
+		fetchedUrl,
+		retrievedAt,
+		sha256: createHash("sha256").update(bytes).digest("hex"),
+		byteLength: bytes.byteLength,
+		...extra
+	});
+	const { archive } = recipe;
+	if ("members" in archive) {
+		const files = await step(entry.id, "extract", () => unpackMembers(bytes, archive));
+		const members = await step(entry.id, "parse", () => files.map((file) => {
+			try {
+				return {
+					id: file.path,
+					meta: {},
+					...parseNative(file.bytes, recipe.parser)
+				};
+			} catch (error) {
+				throw new KeepsetError(`member ${file.path}: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
+			}
+		}));
+		return {
+			native: {
+				meta: metaOf(entry),
+				members
+			},
+			provenance: provenanceOf({ members: files.map((file) => file.path) })
+		};
+	}
+	const unpacked = await step(entry.id, "extract", () => unpack(bytes, archive));
 	const parsed = await step(entry.id, "parse", () => parseNative(unpacked.bytes, recipe.parser));
 	return {
 		native: {
-			meta: metaOf(entry),
-			...parsed
+			...parsed,
+			meta: {
+				..."meta" in parsed ? parsed.meta : {},
+				...metaOf(entry)
+			}
 		},
-		provenance: {
-			entryId: entry.id,
-			catalogue: dataset.catalogue,
-			category: entry.category,
-			source: {
-				url: source.url,
-				status: source.status,
-				...source.snapshot === void 0 ? {} : { snapshot: source.snapshot }
-			},
-			fetchedUrl,
-			retrievedAt,
-			sha256: createHash("sha256").update(bytes).digest("hex"),
-			byteLength: bytes.byteLength,
-			...unpacked.member === void 0 ? {} : { member: unpacked.member }
-		}
+		provenance: provenanceOf(unpacked.member === void 0 ? {} : { member: unpacked.member })
 	};
 }
 /**

package/dist-cli/keepset.mjs

@@ -4,7 +4,7 @@ import { KeepsetError, NAME_SEPARATOR, countTarget, getDataset, listDatasets, li
 import { fetchDataset, roundTripDataset } from "keepset/node";
 import { parseArgs } from "node:util";
 //#region package.json
-var version = "0.22.1";
+var version = "0.24.0";
 const usage = `Usage: keepset <command> [options]
 
 Commands:

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "keepset",
   "description": "Library, CLI and statically generated catalogue for dataset fetch-and-normalise recipes",
-  "version": "0.22.1",
+  "version": "0.24.0",
   "bin": {
     "keepset": "./dist-cli/keepset.mjs"
   },

package/schemas/entry.schema.json

@@ -1 +1 @@
-{"$defs":{"Crosswalk":{"$ref":"https://cdn.jsdelivr.net/npm/twindex@0.1.4/schemas/twindex.schema.json"},"Entry":{"additionalProperties":false,"properties":{"aliases":{"items":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*(?:\\/[A-Za-z0-9][A-Za-z0-9._-]*)*$","type":"string"},"minItems":1,"type":"array"},"category":{"enum":["graph/v1","or/v1"],"type":"string"},"citations":{"items":{"additionalProperties":false,"properties":{"doi":{"pattern":"^10\\.\\d{4,9}\\/\\S+$","type":"string"},"text":{"minLength":1,"type":"string"},"url":{"format":"uri","type":"string"}},"required":["text"],"type":"object"},"type":"array"},"crosswalk":{"anyOf":[{"$ref":"#/$defs/Crosswalk"},{"additionalProperties":false,"properties":{"file":{"type":"string"}},"required":["file"],"type":"object"}]},"description":{"minLength":1,"type":"string"},"domain":{"minLength":1,"type":"string"},"expectedCounts":{"additionalProperties":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"propertyNames":{"pattern":"^[A-Za-z][A-Za-z0-9]*$","type":"string"},"type":"object"},"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"licence":{"oneOf":[{"additionalProperties":false,"properties":{"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9.+-]*$","type":"string"},"kind":{"const":"spdx","type":"string"}},"required":["kind","id"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"custom","type":"string"},"name":{"minLength":1,"type":"string"},"url":{"format":"uri","type":"string"}},"required":["kind","name","url"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"unrecorded","type":"string"}},"required":["kind"],"type":"object"}]},"name":{"minLength":1,"type":"string"},"namespace":{"items":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"minItems":1,"type":"array"},"properties":{"additionalProperties":{"type":["string","number","boolean","null"]},"propertyNames":{"pattern":"^[A-Za-z][A-Za-z0-9]*$","type":"string"},"type":"object"},"recipe":{"oneOf":[{"additionalProperties":false,"properties":{"archive":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"none","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"gzip","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"zip","type":"string"},"member":{"anyOf":[{"additionalProperties":false,"properties":{"path":{"type":"string"}},"required":["path"],"type":"object"},{"additionalProperties":false,"properties":{"extensions":{"items":{"pattern":"^\\.[A-Za-z0-9][A-Za-z0-9.]*$","type":"string"},"minItems":1,"type":"array"}},"required":["extensions"],"type":"object"}]}},"required":["kind","member"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"tar","type":"string"},"member":{"anyOf":[{"additionalProperties":false,"properties":{"path":{"type":"string"}},"required":["path"],"type":"object"},{"additionalProperties":false,"properties":{"extensions":{"items":{"pattern":"^\\.[A-Za-z0-9][A-Za-z0-9.]*$","type":"string"},"minItems":1,"type":"array"}},"required":["extensions"],"type":"object"}]}},"required":["kind","member"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"tar.gz","type":"string"},"member":{"anyOf":[{"additionalProperties":false,"properties":{"path":{"type":"string"}},"required":["path"],"type":"object"},{"additionalProperties":false,"properties":{"extensions":{"items":{"pattern":"^\\.[A-Za-z0-9][A-Za-z0-9.]*$","type":"string"},"minItems":1,"type":"array"}},"required":["extensions"],"type":"object"}]}},"required":["kind","member"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"xlsx","type":"string"},"sheet":{"minLength":1,"type":"string"}},"required":["kind","sheet"],"type":"object"}]},"paging":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"arcgis-query","type":"string"},"orderBy":{"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"}},"required":["kind","orderBy"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"socrata-resource","type":"string"},"pageSize":{"maximum":9007199254740991,"minimum":1,"type":"integer"}},"required":["kind","pageSize"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"ckan-datastore","type":"string"},"pageSize":{"maximum":9007199254740991,"minimum":1,"type":"integer"}},"required":["kind","pageSize"],"type":"object"}]},"parser":{"oneOf":[{"additionalProperties":false,"properties":{"commentPrefixes":{"items":{"minLength":1,"type":"string"},"type":"array"},"delimiter":{"enum":["detect","whitespace","tab","comma","space"],"type":"string"},"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"edge-list","type":"string"},"skipLines":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"sourceColumn":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"targetColumn":{"maximum":9007199254740991,"minimum":0,"type":"integer"}},"required":["format","encoding","delimiter","skipLines","commentPrefixes","sourceColumn","targetColumn"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"matrix-market","type":"string"}},"required":["format","encoding"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"gml","type":"string"}},"required":["format","encoding"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"pajek","type":"string"}},"required":["format","encoding"],"type":"object"},{"additionalProperties":false,"properties":{"columns":{"items":{"pattern":"^[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?$","type":"string"},"type":"array"},"delimiter":{"enum":["comma","tab","semicolon"],"type":"string"},"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"csv-table","type":"string"},"idColumn":{"anyOf":[{"pattern":"^[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?$","type":"string"},{"type":"null"}]},"nullValues":{"items":{"minLength":1,"type":"string"},"type":"array"},"numericColumns":{"items":{"additionalProperties":false,"properties":{"column":{"pattern":"^[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?$","type":"string"},"factor":{"exclusiveMinimum":0,"type":"number"}},"required":["column","factor"],"type":"object"},"type":"array"},"skipLines":{"maximum":9007199254740991,"minimum":0,"type":"integer"}},"required":["format","encoding","delimiter","skipLines","idColumn","numericColumns"],"type":"object"},{"additionalProperties":false,"properties":{"columns":{"items":{"pattern":"^[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?$","type":"string"},"minItems":1,"type":"array"},"dateColumns":{"items":{"pattern":"^[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?$","type":"string"},"type":"array"},"encoding":{"const":"utf-8","type":"string"},"format":{"const":"geojson","type":"string"},"idColumn":{"anyOf":[{"pattern":"^[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?$","type":"string"},{"type":"null"}]},"numericColumns":{"items":{"additionalProperties":false,"properties":{"column":{"pattern":"^[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?$","type":"string"},"factor":{"exclusiveMinimum":0,"type":"number"}},"required":["column","factor"],"type":"object"},"type":"array"}},"required":["format","encoding","idColumn","numericColumns","dateColumns","columns"],"type":"object"}]},"source":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"status":{"const":"fetchable","type":"string"}},"required":["status","source","archive","parser"],"type":"object"},{"additionalProperties":false,"properties":{"reason":{"minLength":1,"type":"string"},"status":{"const":"not-fetchable","type":"string"}},"required":["status","reason"],"type":"object"}]},"sources":{"items":{"additionalProperties":false,"properties":{"retrieved":{"format":"date","pattern":"^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))$","type":"string"},"snapshot":{"additionalProperties":false,"properties":{"timestamp":{"pattern":"^\\d{8}(?:\\d{6})?$","type":"string"},"url":{"format":"uri","type":"string"}},"required":["url","timestamp"],"type":"object"},"status":{"enum":["live","dead"],"type":"string"},"url":{"format":"uri","type":"string"}},"required":["url","retrieved","status"],"type":"object"},"minItems":1,"type":"array"},"tags":{"items":{"minLength":1,"type":"string"},"type":"array"}},"required":["id","name","tags","category","sources","licence","citations","recipe","crosswalk","expectedCounts","properties"],"type":"object"}},"$id":"https://cdn.jsdelivr.net/npm/keepset@0.22.1/schemas/entry.schema.json","$ref":"#/$defs/Entry","$schema":"https://json-schema.org/draft/2020-12/schema"}
+{"$defs":{"Crosswalk":{"$ref":"https://cdn.jsdelivr.net/npm/twindex@0.1.4/schemas/twindex.schema.json"},"Entry":{"additionalProperties":false,"properties":{"aliases":{"items":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*(?:\\/[A-Za-z0-9][A-Za-z0-9._-]*)*$","type":"string"},"minItems":1,"type":"array"},"category":{"enum":["graph/v1","or/v1","or-set/v1"],"type":"string"},"citations":{"items":{"additionalProperties":false,"properties":{"doi":{"pattern":"^10\\.\\d{4,9}\\/\\S+$","type":"string"},"text":{"minLength":1,"type":"string"},"url":{"format":"uri","type":"string"}},"required":["text"],"type":"object"},"type":"array"},"crosswalk":{"anyOf":[{"$ref":"#/$defs/Crosswalk"},{"additionalProperties":false,"properties":{"file":{"type":"string"}},"required":["file"],"type":"object"}]},"description":{"minLength":1,"type":"string"},"domain":{"minLength":1,"type":"string"},"expectedCounts":{"additionalProperties":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"propertyNames":{"pattern":"^[A-Za-z][A-Za-z0-9]*$","type":"string"},"type":"object"},"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"licence":{"oneOf":[{"additionalProperties":false,"properties":{"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9.+-]*$","type":"string"},"kind":{"const":"spdx","type":"string"}},"required":["kind","id"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"custom","type":"string"},"name":{"minLength":1,"type":"string"},"url":{"format":"uri","type":"string"}},"required":["kind","name","url"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"unrecorded","type":"string"}},"required":["kind"],"type":"object"}]},"name":{"minLength":1,"type":"string"},"namespace":{"items":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"minItems":1,"type":"array"},"properties":{"additionalProperties":{"type":["string","number","boolean","null"]},"propertyNames":{"pattern":"^[A-Za-z][A-Za-z0-9]*$","type":"string"},"type":"object"},"recipe":{"oneOf":[{"additionalProperties":false,"properties":{"archive":{"anyOf":[{"additionalProperties":false,"properties":{"kind":{"const":"none","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"gzip","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"enum":["zip","tar","tar.gz"],"type":"string"},"member":{"anyOf":[{"additionalProperties":false,"properties":{"path":{"type":"string"}},"required":["path"],"type":"object"},{"additionalProperties":false,"properties":{"extensions":{"items":{"pattern":"^\\.[A-Za-z0-9][A-Za-z0-9.]*$","type":"string"},"minItems":1,"type":"array"}},"required":["extensions"],"type":"object"}]}},"required":["kind","member"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"enum":["zip","tar","tar.gz"],"type":"string"},"members":{"anyOf":[{"additionalProperties":false,"properties":{"path":{"type":"string"}},"required":["path"],"type":"object"},{"additionalProperties":false,"properties":{"extensions":{"items":{"pattern":"^\\.[A-Za-z0-9][A-Za-z0-9.]*$","type":"string"},"minItems":1,"type":"array"}},"required":["extensions"],"type":"object"}]}},"required":["kind","members"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"xlsx","type":"string"},"sheet":{"minLength":1,"type":"string"}},"required":["kind","sheet"],"type":"object"}]},"paging":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"arcgis-query","type":"string"},"orderBy":{"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"}},"required":["kind","orderBy"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"socrata-resource","type":"string"},"pageSize":{"maximum":9007199254740991,"minimum":1,"type":"integer"}},"required":["kind","pageSize"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"ckan-datastore","type":"string"},"pageSize":{"maximum":9007199254740991,"minimum":1,"type":"integer"}},"required":["kind","pageSize"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"openfema","type":"string"},"pageSize":{"maximum":9007199254740991,"minimum":1,"type":"integer"}},"required":["kind","pageSize"],"type":"object"}]},"parser":{"oneOf":[{"additionalProperties":false,"properties":{"commentPrefixes":{"items":{"minLength":1,"type":"string"},"type":"array"},"delimiter":{"enum":["detect","whitespace","tab","comma","space"],"type":"string"},"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"edge-list","type":"string"},"skipLines":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"sourceColumn":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"targetColumn":{"maximum":9007199254740991,"minimum":0,"type":"integer"}},"required":["format","encoding","delimiter","skipLines","commentPrefixes","sourceColumn","targetColumn"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"matrix-market","type":"string"}},"required":["format","encoding"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"gml","type":"string"}},"required":["format","encoding"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"pajek","type":"string"}},"required":["format","encoding"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"solomon","type":"string"}},"required":["format","encoding"],"type":"object"},{"additionalProperties":false,"properties":{"columns":{"items":{"pattern":"^[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?$","type":"string"},"type":"array"},"delimiter":{"enum":["comma","tab","semicolon"],"type":"string"},"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"csv-table","type":"string"},"idColumn":{"anyOf":[{"pattern":"^[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?$","type":"string"},{"type":"null"}]},"nullValues":{"items":{"minLength":1,"type":"string"},"type":"array"},"numericColumns":{"items":{"additionalProperties":false,"properties":{"column":{"pattern":"^[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?$","type":"string"},"factor":{"exclusiveMinimum":0,"type":"number"}},"required":["column","factor"],"type":"object"},"type":"array"},"skipLines":{"maximum":9007199254740991,"minimum":0,"type":"integer"}},"required":["format","encoding","delimiter","skipLines","idColumn","numericColumns"],"type":"object"},{"additionalProperties":false,"properties":{"columns":{"items":{"pattern":"^[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?$","type":"string"},"minItems":1,"type":"array"},"dateColumns":{"items":{"pattern":"^[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?$","type":"string"},"type":"array"},"encoding":{"const":"utf-8","type":"string"},"format":{"const":"geojson","type":"string"},"idColumn":{"anyOf":[{"pattern":"^[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?$","type":"string"},{"type":"null"}]},"numericColumns":{"items":{"additionalProperties":false,"properties":{"column":{"pattern":"^[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?$","type":"string"},"factor":{"exclusiveMinimum":0,"type":"number"}},"required":["column","factor"],"type":"object"},"type":"array"}},"required":["format","encoding","idColumn","numericColumns","dateColumns","columns"],"type":"object"}]},"source":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"status":{"const":"fetchable","type":"string"}},"required":["status","source","archive","parser"],"type":"object"},{"additionalProperties":false,"properties":{"reason":{"minLength":1,"type":"string"},"status":{"const":"not-fetchable","type":"string"}},"required":["status","reason"],"type":"object"}]},"sources":{"items":{"additionalProperties":false,"properties":{"retrieved":{"format":"date","pattern":"^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))$","type":"string"},"snapshot":{"additionalProperties":false,"properties":{"timestamp":{"pattern":"^\\d{8}(?:\\d{6})?$","type":"string"},"url":{"format":"uri","type":"string"}},"required":["url","timestamp"],"type":"object"},"status":{"enum":["live","dead"],"type":"string"},"url":{"format":"uri","type":"string"}},"required":["url","retrieved","status"],"type":"object"},"minItems":1,"type":"array"},"tags":{"items":{"minLength":1,"type":"string"},"type":"array"}},"required":["id","name","tags","category","sources","licence","citations","recipe","crosswalk","expectedCounts","properties"],"type":"object"}},"$id":"https://cdn.jsdelivr.net/npm/keepset@0.24.0/schemas/entry.schema.json","$ref":"#/$defs/Entry","$schema":"https://json-schema.org/draft/2020-12/schema"}

package/schemas/manifest.schema.json

@@ -1 +1 @@
-{"$defs":{"Manifest":{"additionalProperties":false,"properties":{"description":{"minLength":1,"type":"string"},"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"lineage":{"additionalProperties":false,"properties":{"catalogue":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"repository":{"format":"uri","type":"string"},"version":{"pattern":"^(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)(?:-[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?(?:\\+[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?$","type":"string"}},"required":["catalogue","version","repository"],"type":"object"},"name":{"minLength":1,"type":"string"},"namespaceAliases":{"additionalProperties":{"items":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"minItems":1,"type":"array"},"propertyNames":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"type":"object"},"repository":{"format":"uri","type":"string"},"version":{"pattern":"^(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)(?:-[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?(?:\\+[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?$","type":"string"}},"required":["id","name","description","version","repository"],"type":"object"}},"$id":"https://cdn.jsdelivr.net/npm/keepset@0.22.1/schemas/manifest.schema.json","$ref":"#/$defs/Manifest","$schema":"https://json-schema.org/draft/2020-12/schema"}
+{"$defs":{"Manifest":{"additionalProperties":false,"properties":{"description":{"minLength":1,"type":"string"},"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"lineage":{"additionalProperties":false,"properties":{"catalogue":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"repository":{"format":"uri","type":"string"},"version":{"pattern":"^(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)(?:-[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?(?:\\+[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?$","type":"string"}},"required":["catalogue","version","repository"],"type":"object"},"name":{"minLength":1,"type":"string"},"namespaceAliases":{"additionalProperties":{"items":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"minItems":1,"type":"array"},"propertyNames":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"type":"object"},"repository":{"format":"uri","type":"string"},"version":{"pattern":"^(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)(?:-[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?(?:\\+[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?$","type":"string"}},"required":["id","name","description","version","repository"],"type":"object"}},"$id":"https://cdn.jsdelivr.net/npm/keepset@0.24.0/schemas/manifest.schema.json","$ref":"#/$defs/Manifest","$schema":"https://json-schema.org/draft/2020-12/schema"}