keepset 0.20.0 → 0.21.0

package/dist/{categories-BVROO4tM.d.ts → categories-frr8n_SI.d.ts}

@@ -176,6 +176,9 @@ declare const RecipeSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
   paging: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
     kind: z.ZodLiteral<"arcgis-query">;
     orderBy: z.ZodString;
+  }, z.core.$strict>, z.ZodObject<{
+    kind: z.ZodLiteral<"socrata-resource">;
+    pageSize: z.ZodInt;
   }, z.core.$strict>], "kind">>;
   archive: z.ZodDiscriminatedUnion<[z.ZodObject<{
     kind: z.ZodLiteral<"none">;
@@ -332,6 +335,9 @@ declare const EntrySchema: z.ZodObject<{
     paging: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
       kind: z.ZodLiteral<"arcgis-query">;
       orderBy: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
+      kind: z.ZodLiteral<"socrata-resource">;
+      pageSize: z.ZodInt;
     }, z.core.$strict>], "kind">>;
     archive: z.ZodDiscriminatedUnion<[z.ZodObject<{
       kind: z.ZodLiteral<"none">;
@@ -541,6 +547,9 @@ declare const EntrySchema: z.ZodObject<{
       paging?: {
         kind: "arcgis-query";
         orderBy: string;
+      } | {
+        kind: "socrata-resource";
+        pageSize: number;
       } | undefined;
     } | {
       status: "not-fetchable";
@@ -695,6 +704,9 @@ declare const CatalogueIndexSchema: z.ZodObject<{
       paging: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
         kind: z.ZodLiteral<"arcgis-query">;
         orderBy: z.ZodString;
+      }, z.core.$strict>, z.ZodObject<{
+        kind: z.ZodLiteral<"socrata-resource">;
+        pageSize: z.ZodInt;
       }, z.core.$strict>], "kind">>;
       archive: z.ZodDiscriminatedUnion<[z.ZodObject<{
         kind: z.ZodLiteral<"none">;
@@ -904,6 +916,9 @@ declare const CatalogueIndexSchema: z.ZodObject<{
         paging?: {
           kind: "arcgis-query";
           orderBy: string;
+        } | {
+          kind: "socrata-resource";
+          pageSize: number;
         } | undefined;
       } | {
         status: "not-fetchable";
@@ -1039,6 +1054,9 @@ declare const CatalogueIndexSchema: z.ZodObject<{
         paging?: {
           kind: "arcgis-query";
           orderBy: string;
+        } | {
+          kind: "socrata-resource";
+          pageSize: number;
         } | undefined;
       } | {
         status: "not-fetchable";

package/dist/index.d.ts

@@ -1,4 +1,4 @@
-import { $ as Parser, A as LineageSchema, B as Citation, C as CatalogueIndex, D as NamespaceNode, E as DatasetQuery, F as AliasSchema, G as Entry, H as ColumnNameSchema, I as Archive, J as IdSchema, K as EntrySchema, L as ArchiveMember, M as ManifestSchema, N as SemVerSchema, O as createCatalogue, P as parseManifest, Q as NamespaceSchema, R as ArchiveMemberSchema, S as Catalogue, T as Dataset, U as CrosswalkReference, V as CitationSchema, W as CrosswalkReferenceSchema, X as LicenceSchema, Y as Licence, Z as Namespace, _ as GraphEdgeSchema, a as CategoryTargets, at as Snapshot, b as GraphV1, c as getCategory, ct as SourceSchema, d as listCategories, dt as waybackUrl, et as ParserSchema, f as OrNode, g as GraphEdge, h as OrV1Schema, i as CategoryTarget, it as RelativePathSchema, j as Manifest, k as Lineage, l as getTarget, lt as WaybackTimestampSchema, m as OrV1, n as CategoryId, nt as Recipe, o as CountedKey, ot as SnapshotSchema, p as OrNodeSchema, q as HttpUrlSchema, r as CategoryIdSchema, rt as RecipeSchema, s as countTarget, st as Source, t as Category, tt as RESERVED_META_KEYS, u as kindOf, ut as parseEntry, v as GraphNode, w as CatalogueIndexSchema, x as GraphV1Schema, y as GraphNodeSchema, z as ArchiveSchema } from "./categories-BVROO4tM.js";
+import { $ as Parser, A as LineageSchema, B as Citation, C as CatalogueIndex, D as NamespaceNode, E as DatasetQuery, F as AliasSchema, G as Entry, H as ColumnNameSchema, I as Archive, J as IdSchema, K as EntrySchema, L as ArchiveMember, M as ManifestSchema, N as SemVerSchema, O as createCatalogue, P as parseManifest, Q as NamespaceSchema, R as ArchiveMemberSchema, S as Catalogue, T as Dataset, U as CrosswalkReference, V as CitationSchema, W as CrosswalkReferenceSchema, X as LicenceSchema, Y as Licence, Z as Namespace, _ as GraphEdgeSchema, a as CategoryTargets, at as Snapshot, b as GraphV1, c as getCategory, ct as SourceSchema, d as listCategories, dt as waybackUrl, et as ParserSchema, f as OrNode, g as GraphEdge, h as OrV1Schema, i as CategoryTarget, it as RelativePathSchema, j as Manifest, k as Lineage, l as getTarget, lt as WaybackTimestampSchema, m as OrV1, n as CategoryId, nt as Recipe, o as CountedKey, ot as SnapshotSchema, p as OrNodeSchema, q as HttpUrlSchema, r as CategoryIdSchema, rt as RecipeSchema, s as countTarget, st as Source, t as Category, tt as RESERVED_META_KEYS, u as kindOf, ut as parseEntry, v as GraphNode, w as CatalogueIndexSchema, x as GraphV1Schema, y as GraphNodeSchema, z as ArchiveSchema } from "./categories-frr8n_SI.js";
 import { Crosswalk } from "twindex";
 import { z } from "zod";
 //#region src/errors.d.ts

package/dist/index.js

@@ -1,4 +1,4 @@
-import { $ as NativeParseError, A as toIssues, B as kindOf, C as RESERVED_META_KEYS, D as SourceSchema, E as SnapshotSchema, G as GraphNodeSchema, H as OrNodeSchema, I as CategoryIdSchema, J as ArchiveError, K as GraphV1Schema, L as countTarget, M as NAME_SEPARATOR, N as groupsOf, O as WaybackTimestampSchema, P as namesOf, Q as KeepsetError, R as getCategory, S as ParserSchema, T as RelativePathSchema, U as OrV1Schema, V as listCategories, W as GraphEdgeSchema, X as DatasetAmbiguousError, Y as CatalogueSyntaxError, Z as DatasetNotFoundError, _ as EntrySchema, a as CatalogueIndexSchema, b as LicenceSchema, c as ManifestSchema, d as AliasSchema, et as RecipeError, f as ArchiveMemberSchema, g as CrosswalkReferenceSchema, h as ColumnNameSchema, i as listNamespaces, j as waybackUrl, k as parseEntry, l as SemVerSchema, m as CitationSchema, n as getManifest, o as createCatalogue, p as ArchiveSchema, q as JsonScalarSchema, r as listDatasets, s as LineageSchema, t as getDataset, tt as UnknownCategoryError, u as parseManifest, v as HttpUrlSchema, w as RecipeSchema, x as NamespaceSchema, y as IdSchema, z as getTarget } from "./bundled-BmqpHjOH.js";
+import { $ as NativeParseError, A as toIssues, B as kindOf, C as RESERVED_META_KEYS, D as SourceSchema, E as SnapshotSchema, G as GraphNodeSchema, H as OrNodeSchema, I as CategoryIdSchema, J as ArchiveError, K as GraphV1Schema, L as countTarget, M as NAME_SEPARATOR, N as groupsOf, O as WaybackTimestampSchema, P as namesOf, Q as KeepsetError, R as getCategory, S as ParserSchema, T as RelativePathSchema, U as OrV1Schema, V as listCategories, W as GraphEdgeSchema, X as DatasetAmbiguousError, Y as CatalogueSyntaxError, Z as DatasetNotFoundError, _ as EntrySchema, a as CatalogueIndexSchema, b as LicenceSchema, c as ManifestSchema, d as AliasSchema, et as RecipeError, f as ArchiveMemberSchema, g as CrosswalkReferenceSchema, h as ColumnNameSchema, i as listNamespaces, j as waybackUrl, k as parseEntry, l as SemVerSchema, m as CitationSchema, n as getManifest, o as createCatalogue, p as ArchiveSchema, q as JsonScalarSchema, r as listDatasets, s as LineageSchema, t as getDataset, tt as UnknownCategoryError, u as parseManifest, v as HttpUrlSchema, w as RecipeSchema, x as NamespaceSchema, y as IdSchema, z as getTarget } from "./bundled-NjZZ73vT.js";
 import { validateCrosswalk } from "twindex";
 //#region src/serialise.ts
 /**

package/dist/node/fetch-dataset.d.ts

@@ -1,4 +1,4 @@
-import { G as Entry, T as Dataset, at as Snapshot, i as CategoryTarget, st as Source } from "../categories-BVROO4tM.js";
+import { G as Entry, T as Dataset, at as Snapshot, i as CategoryTarget, st as Source } from "../categories-frr8n_SI.js";
 //#region src/node/fetch-dataset.d.ts
 /** The network port a recipe downloads through: `downloadRaw` by default, which returns the bytes exactly as served, or a stand-in under test. */
 export type FetchPort = (url: string, init: {

package/dist/node/fetch-dataset.js

@@ -1,4 +1,4 @@
-import { $ as NativeParseError, F as getOwn, J as ArchiveError, L as countTarget, Q as KeepsetError, et as RecipeError, t as getDataset, z as getTarget } from "../bundled-BmqpHjOH.js";
+import { $ as NativeParseError, F as getOwn, J as ArchiveError, L as countTarget, Q as KeepsetError, et as RecipeError, t as getDataset, z as getTarget } from "../bundled-NjZZ73vT.js";
 import { RoundTripMismatchError, assertRoundTrip, encode } from "twindex";
 import { z } from "zod";
 import { createHash } from "node:crypto";
@@ -390,8 +390,8 @@ function columnNames(cells, idColumn, line) {
 	return names;
 }
 /** The code unit of each character the scanner stops at. */
-const QUOTE = "\"".charCodeAt(0);
-const LINE_FEED = "\n".charCodeAt(0);
+const QUOTE$1 = "\"".charCodeAt(0);
+const LINE_FEED$1 = "\n".charCodeAt(0);
 const CARRIAGE_RETURN = "\r".charCodeAt(0);
 /** Scans a table file's text as it arrives in pieces, holding what a record in progress needs between them: the fields and field so far, whether a quoted field is open, and the line the record began on and the line the scan has reached. */
 var RecordScanner = class {
@@ -429,7 +429,7 @@ var RecordScanner = class {
 					completed,
 					carry: text.slice(index)
 				};
-				if (text.charCodeAt(index + 1) === QUOTE) {
+				if (text.charCodeAt(index + 1) === QUOTE$1) {
 					this.field += "\"";
 					index += 2;
 				} else {
@@ -439,7 +439,7 @@ var RecordScanner = class {
 				continue;
 			}
 			const code = text.charCodeAt(index);
-			if (code === QUOTE) {
+			if (code === QUOTE$1) {
 				if (this.field !== "") throw new NativeParseError("csv-table", this.line, "a quote opens inside an unquoted field");
 				this.quoted = true;
 				index += 1;
@@ -447,19 +447,19 @@ var RecordScanner = class {
 				this.fields.push(this.field);
 				this.field = "";
 				index += 1;
-			} else if (code === LINE_FEED || code === CARRIAGE_RETURN) {
+			} else if (code === LINE_FEED$1 || code === CARRIAGE_RETURN) {
 				if (code === CARRIAGE_RETURN && index + 1 >= text.length && !final) return {
 					completed,
 					carry: text.slice(index)
 				};
-				index += code === CARRIAGE_RETURN && text.charCodeAt(index + 1) === LINE_FEED ? 2 : 1;
+				index += code === CARRIAGE_RETURN && text.charCodeAt(index + 1) === LINE_FEED$1 ? 2 : 1;
 				this.line += 1;
 				completed.push(this.endRecord());
 			} else {
 				let next = index + 1;
 				while (next < text.length) {
 					const following = text.charCodeAt(next);
-					if (following === QUOTE || following === this.delimiterCode || following === LINE_FEED || following === CARRIAGE_RETURN) break;
+					if (following === QUOTE$1 || following === this.delimiterCode || following === LINE_FEED$1 || following === CARRIAGE_RETURN) break;
 					next += 1;
 				}
 				this.field += text.slice(index, next);
@@ -1373,20 +1373,40 @@ const REDIRECTS = new Set(Object.values({
 }));
 /** What the downloader says it is, so a server operator can tell where the traffic comes from. */
 const USER_AGENT = "keepset (+https://github.com/Mearman/keepset)";
+/** The failure of a connection that sent nothing for the stall time. */
+var StalledError = class extends KeepsetError {};
+/**
+* Whether a failure is the connection's and may pass: a stall, or a socket error such as a reset, a broken pipe or a timeout, which carries an `E` code. A bad URL, a redirect that leads nowhere and an abort are not.
+* @param error - The failure.
+* @returns True when asking again may succeed.
+*/
+function isConnectionFailure(error) {
+	return error instanceof StalledError || typeof error === "object" && error !== null && "code" in error && typeof error.code === "string" && /^E[A-Z]+$/.test(error.code) && error.code !== "ERR_INVALID_URL";
+}
 /**
 * Sends one GET and resolves with the response head, its body still unread.
 * @param url - The URL.
 * @param signal - Aborts the request.
+* @param stall - How many milliseconds the connection may send nothing, in the head or the body that follows.
 * @returns The response.
+* @throws KeepsetError when the connection stalls.
 */
-async function get(url, signal) {
+async function get(url, signal, stall) {
 	const send = url.protocol === "https:" ? request$1 : request;
 	return new Promise((resolve, reject) => {
+		let answered;
 		const request = send(url, {
 			method: "GET",
 			headers: { "user-agent": USER_AGENT },
 			...signal === void 0 ? {} : { signal }
-		}, resolve);
+		}, (response) => {
+			answered = response;
+			resolve(response);
+		});
+		request.setTimeout(stall, () => {
+			const dead = new StalledError(`${url.href} sent nothing for ${String(stall)} milliseconds, so the connection was given up as dead`);
+			(answered ?? request).destroy(dead);
+		});
 		request.on("error", reject);
 		request.end();
 	});
@@ -1407,13 +1427,13 @@ async function readBody(response) {
 /**
 * Requests one URL, following its redirects by recursion.
 * @param url - The URL to request.
-* @param signal - Aborts the request.
+* @param init - The request options.
 * @param redirects - How many redirects led here.
 * @returns The final response.
 */
-async function follow(url, signal, redirects) {
+async function follow(url, init, redirects) {
 	if (url.protocol !== "http:" && url.protocol !== "https:") throw new KeepsetError(`${url.href} is not an http or https URL`);
-	const response = await get(url, signal);
+	const response = await get(url, init.signal, init.stallMilliseconds ?? 12e4);
 	const { statusCode: status, statusMessage } = response;
 	if (status === void 0 || statusMessage === void 0) throw new KeepsetError(`${url.href} answered without a status line`);
 	if (!REDIRECTS.has(status)) return new Response(await readBody(response), {
@@ -1424,17 +1444,36 @@ async function follow(url, signal, redirects) {
 	const { location } = response.headers;
 	if (location === void 0) throw new KeepsetError(`${url.href} answered ${String(status)} without a location`);
 	if (redirects === MAX_REDIRECTS) throw new KeepsetError(`${url.href} redirected more than ${String(MAX_REDIRECTS)} times`);
-	return follow(new URL(location, url), signal, redirects + 1);
+	return follow(new URL(location, url), init, redirects + 1);
 }
 /**
 * Downloads a URL's bytes exactly as the server sends them, following redirects. Unlike the platform `fetch`, it never undoes a `Content-Encoding`: servers commonly label a `.gz` file `Content-Encoding: gzip`, and a recipe that unpacks gzip needs the file as published, which is also what its SHA-256 is recorded over. No `Accept-Encoding` is sent, so a server that compresses on the fly has no reason to.
 * @param url - An http or https URL.
 * @param init - The request options.
 * @returns A Response holding the raw bytes, with the final status.
-* @throws KeepsetError when a redirect has no location, leaves http and https, or passes the redirect limit.
+* @throws KeepsetError when a redirect has no location, leaves http and https, or passes the redirect limit; the connection's failure, after the tries, when it keeps failing or sends nothing for longer than the stall time.
 */
 async function downloadRaw(url, init) {
-	return follow(new URL(url), init.signal, 0);
+	return tryDownload(url, init, 1);
+}
+/**
+* One try of a download, and the tries after it while the connection fails.
+* @param url - The URL.
+* @param init - The request options.
+* @param attempt - Which try this is, from 1.
+* @returns The response.
+* @throws The last failure when the tries are used up, or at once for a failure that is not the connection's.
+*/
+async function tryDownload(url, init, attempt) {
+	try {
+		return await follow(new URL(url), init, 0);
+	} catch (error) {
+		if (attempt >= 3 || init.signal?.aborted === true || !isConnectionFailure(error)) throw error;
+		await new Promise((resolve) => {
+			setTimeout(resolve, (init.retryDelayMilliseconds ?? 3e3) * attempt);
+		});
+		return tryDownload(url, init, attempt + 1);
+	}
 }
 //#endregion
 //#region src/node/paged.ts
@@ -1481,7 +1520,7 @@ const FeatureListSchema = z.array(z.unknown());
 async function readPages(layer, held) {
 	if (held.length >= layer.count) return held;
 	const url = `${layer.query}?where=1%3D1&outFields=*&orderByFields=${encodeURIComponent(layer.orderBy)}&resultOffset=${String(held.length)}&f=geojson`;
-	const page = await getJson(layer.port, url, layer.signal);
+	const page = await withAttempts(async () => getJson(layer.port, url, layer.signal), layer.signal);
 	const features = FeatureListSchema.safeParse(typeof page === "object" && page !== null && "features" in page ? page.features : void 0);
 	if (!features.success || features.data.length === 0) throw new KeepsetError(`${layer.query} gave no features at offset ${String(held.length)}, before the ${String(layer.count)} it reports, so it does not page by offset`);
 	return readPages(layer, [...held, ...features.data]);
@@ -1511,6 +1550,95 @@ async function downloadArcgisLayer(layerUrl, orderBy, port, signal) {
 		features: pages
 	}));
 }
+/** How many times a page is requested before the download fails: a connection can drop or go silent in one request of dozens and succeed in the next, and the pages are read by offset so asking again is exact. */
+const PAGE_ATTEMPTS = 3;
+/**
+* Runs one page read, asking again when it fails, so one bad response of a long download does not lose the rest.
+* @param read - The page read.
+* @param signal - Aborts the download, which is not asked again.
+* @param attempt - Which attempt this is, from 1.
+* @returns What the read returns.
+* @throws The last failure, when the attempts are used up or the download was aborted.
+*/
+async function withAttempts(read, signal, attempt = 1) {
+	try {
+		return await read();
+	} catch (error) {
+		if (attempt >= PAGE_ATTEMPTS || signal?.aborted === true) throw error;
+		return withAttempts(read, signal, attempt + 1);
+	}
+}
+const LINE_FEED = 10;
+const QUOTE = 34;
+/**
+* How many CSV records a response holds: the line feeds outside a quoted field, and one more when the last record has no line feed.
+* @param bytes - A CSV response.
+* @returns The number of records, the header included.
+*/
+function recordsIn(bytes) {
+	let records = 0;
+	let quoted = false;
+	for (const byte of bytes) if (byte === QUOTE) quoted = !quoted;
+	else if (byte === LINE_FEED && !quoted) records += 1;
+	return bytes.length > 0 && bytes[bytes.length - 1] !== LINE_FEED ? records + 1 : records;
+}
+/**
+* Reads one response of a Socrata resource endpoint as bytes.
+* @param port - The network port.
+* @param url - The request URL.
+* @param signal - Aborts the request.
+* @returns The response body.
+* @throws KeepsetError when the server does not answer successfully.
+*/
+async function getBytes(port, url, signal) {
+	const response = await port(url, signal === void 0 ? {} : { signal });
+	if (!response.ok) throw new KeepsetError(`${url} answered ${String(response.status)} ${response.statusText}`);
+	return new Uint8Array(await response.arrayBuffer());
+}
+/**
+* Reads a resource's pages, each starting where the rows held so far end, and checks each holds exactly the rows it should, so a response cut short is an error and not a table with rows missing.
+* @param read - The resource read.
+* @param held - The pages read so far, each after the first without its header record.
+* @param offset - How many rows the pages hold.
+* @returns Every page.
+* @throws KeepsetError when a page holds a different number of rows than the count says it should, or its header differs from the first page's.
+*/
+async function readResourcePages(read, held, offset) {
+	if (offset >= read.count) return held;
+	const expected = Math.min(read.pageSize, read.count - offset);
+	const page = await withAttempts(async () => {
+		const bytes = await getBytes(read.port, `${read.resource}?%24order=%3Aid&%24limit=${String(read.pageSize)}&%24offset=${String(offset)}`, read.signal);
+		const rows = recordsIn(bytes) - 1;
+		if (rows !== expected) throw new KeepsetError(`${read.resource} gave ${String(rows)} rows at offset ${String(offset)} where ${String(expected)} of its ${String(read.count)} were due, so a response was cut short or the table changed during the download`);
+		return bytes;
+	}, read.signal);
+	const first = held[0];
+	const headerEnd = page.indexOf(LINE_FEED) + 1;
+	if (first !== void 0 && !page.subarray(0, headerEnd).every((byte, index) => byte === first[index])) throw new KeepsetError(`${read.resource} gave a different header at offset ${String(offset)} than at the start`);
+	return readResourcePages(read, [...held, first === void 0 ? page : page.subarray(headerEnd)], offset + expected);
+}
+/**
+* Downloads a whole Socrata resource as one CSV: the row count is read first, then pages of `pageSize` rows in the order of the rows' ids, each checked to hold the rows it should, and joined under the first page's header.
+* @param resource - The resource's URL, such as `https://data.example.org/resource/abcd-1234.csv`, without a query.
+* @param pageSize - How many rows a page holds.
+* @param port - The network port.
+* @param signal - Aborts the download.
+* @returns The bytes of one CSV, which is what the parser reads and what the provenance hashes.
+* @throws KeepsetError when the resource does not report a count, or a page is not what it should be.
+*/
+async function downloadSocrataResource(resource, pageSize, port, signal) {
+	const reported = /^"count"\r?\n"(\d+)"/.exec(new TextDecoder("utf-8").decode(await getBytes(port, `${resource}?%24select=count(*)`, signal)));
+	if (reported?.[1] === void 0) throw new KeepsetError(`${resource} did not report a row count`);
+	const pages = await readResourcePages({
+		resource,
+		pageSize,
+		count: Number(reported[1]),
+		port,
+		signal
+	}, [], 0);
+	if (pages.length === 0) throw new KeepsetError(`${resource} has no rows`);
+	return Buffer.concat(pages);
+}
 /**
 * Downloads a paged source as the one file its pages make, in the way its recipe's paging says.
 * @param url - The source's URL.
@@ -1521,7 +1649,7 @@ async function downloadArcgisLayer(layerUrl, orderBy, port, signal) {
 * @throws KeepsetError when a page cannot be read or the pages do not add up to what the source reports.
 */
 async function downloadPaged(url, paging, port, signal) {
-	return downloadArcgisLayer(url, paging.orderBy, port, signal);
+	return paging.kind === "arcgis-query" ? downloadArcgisLayer(url, paging.orderBy, port, signal) : downloadSocrataResource(url, paging.pageSize, port, signal);
 }
 //#endregion
 //#region src/node/fetch-dataset.ts

package/dist-cli/keepset.mjs

@@ -4,7 +4,7 @@ import { KeepsetError, NAME_SEPARATOR, countTarget, getDataset, listDatasets, li
 import { fetchDataset, roundTripDataset } from "keepset/node";
 import { parseArgs } from "node:util";
 //#region package.json
-var version = "0.20.0";
+var version = "0.21.0";
 const usage = `Usage: keepset <command> [options]
 
 Commands:

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "keepset",
   "description": "Library, CLI and statically generated catalogue for dataset fetch-and-normalise recipes",
-  "version": "0.20.0",
+  "version": "0.21.0",
   "bin": {
     "keepset": "./dist-cli/keepset.mjs"
   },

package/schemas/entry.schema.json

@@ -1 +1 @@
-{"$defs":{"Crosswalk":{"$ref":"https://cdn.jsdelivr.net/npm/twindex@0.1.4/schemas/twindex.schema.json"},"Entry":{"additionalProperties":false,"properties":{"aliases":{"items":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*(?:\\/[A-Za-z0-9][A-Za-z0-9._-]*)*$","type":"string"},"minItems":1,"type":"array"},"category":{"enum":["graph/v1","or/v1"],"type":"string"},"citations":{"items":{"additionalProperties":false,"properties":{"doi":{"pattern":"^10\\.\\d{4,9}\\/\\S+$","type":"string"},"text":{"minLength":1,"type":"string"},"url":{"format":"uri","type":"string"}},"required":["text"],"type":"object"},"type":"array"},"crosswalk":{"anyOf":[{"$ref":"#/$defs/Crosswalk"},{"additionalProperties":false,"properties":{"file":{"type":"string"}},"required":["file"],"type":"object"}]},"description":{"minLength":1,"type":"string"},"domain":{"minLength":1,"type":"string"},"expectedCounts":{"additionalProperties":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"propertyNames":{"pattern":"^[A-Za-z][A-Za-z0-9]*$","type":"string"},"type":"object"},"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"licence":{"oneOf":[{"additionalProperties":false,"properties":{"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9.+-]*$","type":"string"},"kind":{"const":"spdx","type":"string"}},"required":["kind","id"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"custom","type":"string"},"name":{"minLength":1,"type":"string"},"url":{"format":"uri","type":"string"}},"required":["kind","name","url"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"unrecorded","type":"string"}},"required":["kind"],"type":"object"}]},"name":{"minLength":1,"type":"string"},"namespace":{"items":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"minItems":1,"type":"array"},"properties":{"additionalProperties":{"type":["string","number","boolean","null"]},"propertyNames":{"pattern":"^[A-Za-z][A-Za-z0-9]*$","type":"string"},"type":"object"},"recipe":{"oneOf":[{"additionalProperties":false,"properties":{"archive":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"none","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"gzip","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"zip","type":"string"},"member":{"anyOf":[{"additionalProperties":false,"properties":{"path":{"type":"string"}},"required":["path"],"type":"object"},{"additionalProperties":false,"properties":{"extensions":{"items":{"pattern":"^\\.[A-Za-z0-9][A-Za-z0-9.]*$","type":"string"},"minItems":1,"type":"array"}},"required":["extensions"],"type":"object"}]}},"required":["kind","member"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"tar","type":"string"},"member":{"anyOf":[{"additionalProperties":false,"properties":{"path":{"type":"string"}},"required":["path"],"type":"object"},{"additionalProperties":false,"properties":{"extensions":{"items":{"pattern":"^\\.[A-Za-z0-9][A-Za-z0-9.]*$","type":"string"},"minItems":1,"type":"array"}},"required":["extensions"],"type":"object"}]}},"required":["kind","member"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"tar.gz","type":"string"},"member":{"anyOf":[{"additionalProperties":false,"properties":{"path":{"type":"string"}},"required":["path"],"type":"object"},{"additionalProperties":false,"properties":{"extensions":{"items":{"pattern":"^\\.[A-Za-z0-9][A-Za-z0-9.]*$","type":"string"},"minItems":1,"type":"array"}},"required":["extensions"],"type":"object"}]}},"required":["kind","member"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"xlsx","type":"string"},"sheet":{"minLength":1,"type":"string"}},"required":["kind","sheet"],"type":"object"}]},"paging":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"arcgis-query","type":"string"},"orderBy":{"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"}},"required":["kind","orderBy"],"type":"object"}]},"parser":{"oneOf":[{"additionalProperties":false,"properties":{"commentPrefixes":{"items":{"minLength":1,"type":"string"},"type":"array"},"delimiter":{"enum":["detect","whitespace","tab","comma","space"],"type":"string"},"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"edge-list","type":"string"},"skipLines":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"sourceColumn":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"targetColumn":{"maximum":9007199254740991,"minimum":0,"type":"integer"}},"required":["format","encoding","delimiter","skipLines","commentPrefixes","sourceColumn","targetColumn"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"matrix-market","type":"string"}},"required":["format","encoding"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"gml","type":"string"}},"required":["format","encoding"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"pajek","type":"string"}},"required":["format","encoding"],"type":"object"},{"additionalProperties":false,"properties":{"columns":{"items":{"pattern":"^[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?$","type":"string"},"type":"array"},"delimiter":{"enum":["comma","tab","semicolon"],"type":"string"},"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"csv-table","type":"string"},"idColumn":{"anyOf":[{"pattern":"^[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?$","type":"string"},{"type":"null"}]},"numericColumns":{"items":{"additionalProperties":false,"properties":{"column":{"pattern":"^[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?$","type":"string"},"factor":{"exclusiveMinimum":0,"type":"number"}},"required":["column","factor"],"type":"object"},"type":"array"},"skipLines":{"maximum":9007199254740991,"minimum":0,"type":"integer"}},"required":["format","encoding","delimiter","skipLines","idColumn","numericColumns"],"type":"object"},{"additionalProperties":false,"properties":{"columns":{"items":{"pattern":"^[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?$","type":"string"},"minItems":1,"type":"array"},"dateColumns":{"items":{"pattern":"^[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?$","type":"string"},"type":"array"},"encoding":{"const":"utf-8","type":"string"},"format":{"const":"geojson","type":"string"},"idColumn":{"anyOf":[{"pattern":"^[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?$","type":"string"},{"type":"null"}]},"numericColumns":{"items":{"additionalProperties":false,"properties":{"column":{"pattern":"^[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?$","type":"string"},"factor":{"exclusiveMinimum":0,"type":"number"}},"required":["column","factor"],"type":"object"},"type":"array"}},"required":["format","encoding","idColumn","numericColumns","dateColumns","columns"],"type":"object"}]},"source":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"status":{"const":"fetchable","type":"string"}},"required":["status","source","archive","parser"],"type":"object"},{"additionalProperties":false,"properties":{"reason":{"minLength":1,"type":"string"},"status":{"const":"not-fetchable","type":"string"}},"required":["status","reason"],"type":"object"}]},"sources":{"items":{"additionalProperties":false,"properties":{"retrieved":{"format":"date","pattern":"^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))$","type":"string"},"snapshot":{"additionalProperties":false,"properties":{"timestamp":{"pattern":"^\\d{8}(?:\\d{6})?$","type":"string"},"url":{"format":"uri","type":"string"}},"required":["url","timestamp"],"type":"object"},"status":{"enum":["live","dead"],"type":"string"},"url":{"format":"uri","type":"string"}},"required":["url","retrieved","status"],"type":"object"},"minItems":1,"type":"array"},"tags":{"items":{"minLength":1,"type":"string"},"type":"array"}},"required":["id","name","tags","category","sources","licence","citations","recipe","crosswalk","expectedCounts","properties"],"type":"object"}},"$id":"https://cdn.jsdelivr.net/npm/keepset@0.20.0/schemas/entry.schema.json","$ref":"#/$defs/Entry","$schema":"https://json-schema.org/draft/2020-12/schema"}
+{"$defs":{"Crosswalk":{"$ref":"https://cdn.jsdelivr.net/npm/twindex@0.1.4/schemas/twindex.schema.json"},"Entry":{"additionalProperties":false,"properties":{"aliases":{"items":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*(?:\\/[A-Za-z0-9][A-Za-z0-9._-]*)*$","type":"string"},"minItems":1,"type":"array"},"category":{"enum":["graph/v1","or/v1"],"type":"string"},"citations":{"items":{"additionalProperties":false,"properties":{"doi":{"pattern":"^10\\.\\d{4,9}\\/\\S+$","type":"string"},"text":{"minLength":1,"type":"string"},"url":{"format":"uri","type":"string"}},"required":["text"],"type":"object"},"type":"array"},"crosswalk":{"anyOf":[{"$ref":"#/$defs/Crosswalk"},{"additionalProperties":false,"properties":{"file":{"type":"string"}},"required":["file"],"type":"object"}]},"description":{"minLength":1,"type":"string"},"domain":{"minLength":1,"type":"string"},"expectedCounts":{"additionalProperties":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"propertyNames":{"pattern":"^[A-Za-z][A-Za-z0-9]*$","type":"string"},"type":"object"},"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"licence":{"oneOf":[{"additionalProperties":false,"properties":{"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9.+-]*$","type":"string"},"kind":{"const":"spdx","type":"string"}},"required":["kind","id"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"custom","type":"string"},"name":{"minLength":1,"type":"string"},"url":{"format":"uri","type":"string"}},"required":["kind","name","url"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"unrecorded","type":"string"}},"required":["kind"],"type":"object"}]},"name":{"minLength":1,"type":"string"},"namespace":{"items":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"minItems":1,"type":"array"},"properties":{"additionalProperties":{"type":["string","number","boolean","null"]},"propertyNames":{"pattern":"^[A-Za-z][A-Za-z0-9]*$","type":"string"},"type":"object"},"recipe":{"oneOf":[{"additionalProperties":false,"properties":{"archive":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"none","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"gzip","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"zip","type":"string"},"member":{"anyOf":[{"additionalProperties":false,"properties":{"path":{"type":"string"}},"required":["path"],"type":"object"},{"additionalProperties":false,"properties":{"extensions":{"items":{"pattern":"^\\.[A-Za-z0-9][A-Za-z0-9.]*$","type":"string"},"minItems":1,"type":"array"}},"required":["extensions"],"type":"object"}]}},"required":["kind","member"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"tar","type":"string"},"member":{"anyOf":[{"additionalProperties":false,"properties":{"path":{"type":"string"}},"required":["path"],"type":"object"},{"additionalProperties":false,"properties":{"extensions":{"items":{"pattern":"^\\.[A-Za-z0-9][A-Za-z0-9.]*$","type":"string"},"minItems":1,"type":"array"}},"required":["extensions"],"type":"object"}]}},"required":["kind","member"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"tar.gz","type":"string"},"member":{"anyOf":[{"additionalProperties":false,"properties":{"path":{"type":"string"}},"required":["path"],"type":"object"},{"additionalProperties":false,"properties":{"extensions":{"items":{"pattern":"^\\.[A-Za-z0-9][A-Za-z0-9.]*$","type":"string"},"minItems":1,"type":"array"}},"required":["extensions"],"type":"object"}]}},"required":["kind","member"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"xlsx","type":"string"},"sheet":{"minLength":1,"type":"string"}},"required":["kind","sheet"],"type":"object"}]},"paging":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"arcgis-query","type":"string"},"orderBy":{"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"}},"required":["kind","orderBy"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"socrata-resource","type":"string"},"pageSize":{"maximum":9007199254740991,"minimum":1,"type":"integer"}},"required":["kind","pageSize"],"type":"object"}]},"parser":{"oneOf":[{"additionalProperties":false,"properties":{"commentPrefixes":{"items":{"minLength":1,"type":"string"},"type":"array"},"delimiter":{"enum":["detect","whitespace","tab","comma","space"],"type":"string"},"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"edge-list","type":"string"},"skipLines":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"sourceColumn":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"targetColumn":{"maximum":9007199254740991,"minimum":0,"type":"integer"}},"required":["format","encoding","delimiter","skipLines","commentPrefixes","sourceColumn","targetColumn"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"matrix-market","type":"string"}},"required":["format","encoding"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"gml","type":"string"}},"required":["format","encoding"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"pajek","type":"string"}},"required":["format","encoding"],"type":"object"},{"additionalProperties":false,"properties":{"columns":{"items":{"pattern":"^[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?$","type":"string"},"type":"array"},"delimiter":{"enum":["comma","tab","semicolon"],"type":"string"},"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"csv-table","type":"string"},"idColumn":{"anyOf":[{"pattern":"^[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?$","type":"string"},{"type":"null"}]},"numericColumns":{"items":{"additionalProperties":false,"properties":{"column":{"pattern":"^[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?$","type":"string"},"factor":{"exclusiveMinimum":0,"type":"number"}},"required":["column","factor"],"type":"object"},"type":"array"},"skipLines":{"maximum":9007199254740991,"minimum":0,"type":"integer"}},"required":["format","encoding","delimiter","skipLines","idColumn","numericColumns"],"type":"object"},{"additionalProperties":false,"properties":{"columns":{"items":{"pattern":"^[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?$","type":"string"},"minItems":1,"type":"array"},"dateColumns":{"items":{"pattern":"^[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?$","type":"string"},"type":"array"},"encoding":{"const":"utf-8","type":"string"},"format":{"const":"geojson","type":"string"},"idColumn":{"anyOf":[{"pattern":"^[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?$","type":"string"},{"type":"null"}]},"numericColumns":{"items":{"additionalProperties":false,"properties":{"column":{"pattern":"^[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?$","type":"string"},"factor":{"exclusiveMinimum":0,"type":"number"}},"required":["column","factor"],"type":"object"},"type":"array"}},"required":["format","encoding","idColumn","numericColumns","dateColumns","columns"],"type":"object"}]},"source":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"status":{"const":"fetchable","type":"string"}},"required":["status","source","archive","parser"],"type":"object"},{"additionalProperties":false,"properties":{"reason":{"minLength":1,"type":"string"},"status":{"const":"not-fetchable","type":"string"}},"required":["status","reason"],"type":"object"}]},"sources":{"items":{"additionalProperties":false,"properties":{"retrieved":{"format":"date","pattern":"^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))$","type":"string"},"snapshot":{"additionalProperties":false,"properties":{"timestamp":{"pattern":"^\\d{8}(?:\\d{6})?$","type":"string"},"url":{"format":"uri","type":"string"}},"required":["url","timestamp"],"type":"object"},"status":{"enum":["live","dead"],"type":"string"},"url":{"format":"uri","type":"string"}},"required":["url","retrieved","status"],"type":"object"},"minItems":1,"type":"array"},"tags":{"items":{"minLength":1,"type":"string"},"type":"array"}},"required":["id","name","tags","category","sources","licence","citations","recipe","crosswalk","expectedCounts","properties"],"type":"object"}},"$id":"https://cdn.jsdelivr.net/npm/keepset@0.21.0/schemas/entry.schema.json","$ref":"#/$defs/Entry","$schema":"https://json-schema.org/draft/2020-12/schema"}

package/schemas/manifest.schema.json

@@ -1 +1 @@
-{"$defs":{"Manifest":{"additionalProperties":false,"properties":{"description":{"minLength":1,"type":"string"},"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"lineage":{"additionalProperties":false,"properties":{"catalogue":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"repository":{"format":"uri","type":"string"},"version":{"pattern":"^(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)(?:-[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?(?:\\+[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?$","type":"string"}},"required":["catalogue","version","repository"],"type":"object"},"name":{"minLength":1,"type":"string"},"namespaceAliases":{"additionalProperties":{"items":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"minItems":1,"type":"array"},"propertyNames":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"type":"object"},"repository":{"format":"uri","type":"string"},"version":{"pattern":"^(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)(?:-[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?(?:\\+[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?$","type":"string"}},"required":["id","name","description","version","repository"],"type":"object"}},"$id":"https://cdn.jsdelivr.net/npm/keepset@0.20.0/schemas/manifest.schema.json","$ref":"#/$defs/Manifest","$schema":"https://json-schema.org/draft/2020-12/schema"}
+{"$defs":{"Manifest":{"additionalProperties":false,"properties":{"description":{"minLength":1,"type":"string"},"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"lineage":{"additionalProperties":false,"properties":{"catalogue":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"repository":{"format":"uri","type":"string"},"version":{"pattern":"^(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)(?:-[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?(?:\\+[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?$","type":"string"}},"required":["catalogue","version","repository"],"type":"object"},"name":{"minLength":1,"type":"string"},"namespaceAliases":{"additionalProperties":{"items":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"minItems":1,"type":"array"},"propertyNames":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"type":"object"},"repository":{"format":"uri","type":"string"},"version":{"pattern":"^(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)(?:-[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?(?:\\+[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?$","type":"string"}},"required":["id","name","description","version","repository"],"type":"object"}},"$id":"https://cdn.jsdelivr.net/npm/keepset@0.21.0/schemas/manifest.schema.json","$ref":"#/$defs/Manifest","$schema":"https://json-schema.org/draft/2020-12/schema"}