keepset 0.19.0 → 0.21.0

package/dist/node/fetch-dataset.js

@@ -1,4 +1,4 @@
-import { $ as NativeParseError, F as getOwn, J as ArchiveError, L as countTarget, Q as KeepsetError, et as RecipeError, t as getDataset, z as getTarget } from "../bundled-8oHQPxcd.js";
+import { $ as NativeParseError, F as getOwn, J as ArchiveError, L as countTarget, Q as KeepsetError, et as RecipeError, t as getDataset, z as getTarget } from "../bundled-NjZZ73vT.js";
 import { RoundTripMismatchError, assertRoundTrip, encode } from "twindex";
 import { z } from "zod";
 import { createHash } from "node:crypto";
@@ -356,7 +356,7 @@ const DELIMITERS = {
 	semicolon: ";"
 };
 /** A number as a decimal literal, with an optional sign and exponent. `Number` alone would also accept hex, binary, `Infinity` and the empty string. */
-const DECIMAL = /^[+-]?(?:\d+\.?\d*|\.\d+)(?:[eE][+-]?\d+)?$/;
+const DECIMAL$1 = /^[+-]?(?:\d+\.?\d*|\.\d+)(?:[eE][+-]?\d+)?$/;
 /** The byte order mark some exporters put before the first header. */
 const BYTE_ORDER_MARK = "";
 /**
@@ -390,8 +390,8 @@ function columnNames(cells, idColumn, line) {
 	return names;
 }
 /** The code unit of each character the scanner stops at. */
-const QUOTE = "\"".charCodeAt(0);
-const LINE_FEED = "\n".charCodeAt(0);
+const QUOTE$1 = "\"".charCodeAt(0);
+const LINE_FEED$1 = "\n".charCodeAt(0);
 const CARRIAGE_RETURN = "\r".charCodeAt(0);
 /** Scans a table file's text as it arrives in pieces, holding what a record in progress needs between them: the fields and field so far, whether a quoted field is open, and the line the record began on and the line the scan has reached. */
 var RecordScanner = class {
@@ -429,7 +429,7 @@ var RecordScanner = class {
 					completed,
 					carry: text.slice(index)
 				};
-				if (text.charCodeAt(index + 1) === QUOTE) {
+				if (text.charCodeAt(index + 1) === QUOTE$1) {
 					this.field += "\"";
 					index += 2;
 				} else {
@@ -439,7 +439,7 @@ var RecordScanner = class {
 				continue;
 			}
 			const code = text.charCodeAt(index);
-			if (code === QUOTE) {
+			if (code === QUOTE$1) {
 				if (this.field !== "") throw new NativeParseError("csv-table", this.line, "a quote opens inside an unquoted field");
 				this.quoted = true;
 				index += 1;
@@ -447,19 +447,19 @@ var RecordScanner = class {
 				this.fields.push(this.field);
 				this.field = "";
 				index += 1;
-			} else if (code === LINE_FEED || code === CARRIAGE_RETURN) {
+			} else if (code === LINE_FEED$1 || code === CARRIAGE_RETURN) {
 				if (code === CARRIAGE_RETURN && index + 1 >= text.length && !final) return {
 					completed,
 					carry: text.slice(index)
 				};
-				index += code === CARRIAGE_RETURN && text.charCodeAt(index + 1) === LINE_FEED ? 2 : 1;
+				index += code === CARRIAGE_RETURN && text.charCodeAt(index + 1) === LINE_FEED$1 ? 2 : 1;
 				this.line += 1;
 				completed.push(this.endRecord());
 			} else {
 				let next = index + 1;
 				while (next < text.length) {
 					const following = text.charCodeAt(next);
-					if (following === QUOTE || following === this.delimiterCode || following === LINE_FEED || following === CARRIAGE_RETURN) break;
+					if (following === QUOTE$1 || following === this.delimiterCode || following === LINE_FEED$1 || following === CARRIAGE_RETURN) break;
 					next += 1;
 				}
 				this.field += text.slice(index, next);
@@ -559,7 +559,7 @@ function* withoutByteOrderMark(text) {
 */
 function numeric$1(cell, factor, column, line) {
 	if (cell === "") return null;
-	if (!DECIMAL.test(cell)) throw new NativeParseError("csv-table", line, `column ${column} is declared numeric but holds ${JSON.stringify(cell)}`);
+	if (!DECIMAL$1.test(cell)) throw new NativeParseError("csv-table", line, `column ${column} is declared numeric but holds ${JSON.stringify(cell)}`);
 	return Number(cell) * factor;
 }
 /**
@@ -933,7 +933,7 @@ const BYTE_RADIX = 256;
 * @param length - Field length.
 * @returns The field text.
 */
-function field(bytes, offset, length) {
+function field$1(bytes, offset, length) {
 	const slice = bytes.subarray(offset, offset + length);
 	const end = slice.indexOf(0);
 	return new TextDecoder().decode(end === -1 ? slice : slice.subarray(0, end));
@@ -952,7 +952,7 @@ function numeric(bytes, offset, length) {
 		for (const byte of bytes.subarray(offset + 1, offset + length)) value = value * BYTE_RADIX + byte;
 		return value;
 	}
-	const text = field(bytes, offset, length).trim();
+	const text = field$1(bytes, offset, length).trim();
 	if (!/^[0-7]*$/.test(text)) throw new ArchiveError(`tar header field is not octal: ${JSON.stringify(text)}`);
 	return text === "" ? 0 : Number.parseInt(text, 8);
 }
@@ -988,21 +988,21 @@ function readTar(bytes) {
 		for (const [index, byte] of header.entries()) sum += index >= checksum.offset && index < checksum.offset + checksum.length ? 32 : byte;
 		if (sum !== numeric(header, checksum.offset, checksum.length)) throw new ArchiveError(`tar header at byte ${String(offset)} fails its checksum`);
 		const size = numeric(header, HEADER.size.offset, HEADER.size.length);
-		const type = field(header, HEADER.type.offset, HEADER.type.length);
+		const type = field$1(header, HEADER.type.offset, HEADER.type.length);
 		const dataStart = offset + 512;
 		if (dataStart + size > bytes.length) throw new ArchiveError(`tar member at byte ${String(offset)} runs past the end of the archive`);
 		const data = bytes.subarray(dataStart, dataStart + size);
 		offset = dataStart + Math.ceil(size / 512) * 512;
 		if (type === "L") {
-			longName = field(data, 0, data.length);
+			longName = field$1(data, 0, data.length);
 			continue;
 		}
 		if (type === "x") {
 			longName = paxPath(data);
 			continue;
 		}
-		const name = field(header, HEADER.name.offset, HEADER.name.length);
-		const prefix = field(header, HEADER.magic.offset, HEADER.magic.length).startsWith("ustar") ? field(header, HEADER.prefix.offset, HEADER.prefix.length) : "";
+		const name = field$1(header, HEADER.name.offset, HEADER.name.length);
+		const prefix = field$1(header, HEADER.magic.offset, HEADER.magic.length).startsWith("ustar") ? field$1(header, HEADER.prefix.offset, HEADER.prefix.length) : "";
 		const path = longName ?? (prefix === "" ? name : `${prefix}/${name}`);
 		longName = void 0;
 		if (type === "0" || type === "" || type === "7") members.push({
@@ -1013,6 +1013,255 @@ function readTar(bytes) {
 	return members;
 }
 //#endregion
+//#region src/node/xlsx.ts
+const HEXADECIMAL = 16;
+const DECIMAL = 10;
+const LETTERS = 26;
+const FIRST_LETTER = "A".charCodeAt(0);
+const SECONDS_PER_DAY = 86400;
+const MILLISECONDS = 1e3;
+/** The serial number Excel gives its 29th of February 1900, a day that never existed: serials above it count from 30 December 1899 and serials below it from 31 December 1899. */
+const LEAP_BUG_SERIAL = 60;
+/** The day a serial number counts from: 1 January 1904 in a workbook that says so, otherwise 31 December 1899 for the serials before Excel's false 29th of February 1900 and 30 December 1899 for those after it. */
+const EPOCH_1904 = "1904-01-01T00:00:00Z";
+const EPOCH_BEFORE_LEAP_BUG = "1899-12-31T00:00:00Z";
+const EPOCH_AFTER_LEAP_BUG = "1899-12-30T00:00:00Z";
+const DATE_LENGTH = 10;
+const DATE_TIME_LENGTH = 19;
+const SECONDS_PER_MINUTE = 60;
+const MINUTES_PER_HOUR = 60;
+/**
+* Replaces the XML entities and character references of a text.
+* @param text - Escaped text.
+* @returns The text.
+*/
+function unescape(text) {
+	return text.replaceAll(/&(#x[0-9A-Fa-f]+|#[0-9]+|amp|lt|gt|quot|apos);/g, (_whole, entity) => {
+		if (entity.startsWith("#x")) return String.fromCodePoint(Number.parseInt(entity.slice(2), HEXADECIMAL));
+		if (entity.startsWith("#")) return String.fromCodePoint(Number.parseInt(entity.slice(1), DECIMAL));
+		return {
+			amp: "&",
+			lt: "<",
+			gt: ">",
+			quot: "\"",
+			apos: "'"
+		}[entity] ?? "";
+	});
+}
+/**
+* The attributes of a start tag's text.
+* @param tag - The text between `<` and `>`, name included.
+* @returns The attributes, unescaped.
+*/
+function attributesOf(tag) {
+	const attributes = {};
+	for (const [, name, value] of tag.matchAll(/([\w:.-]+)\s*=\s*"([^"]*)"/g)) if (name !== void 0 && value !== void 0) attributes[name] = unescape(value);
+	return attributes;
+}
+/**
+* The start tags of one element name in an XML part, in order.
+* @param xml - The part's text.
+* @param name - The element name.
+* @returns The attributes of each start tag.
+*/
+function elements(xml, name) {
+	return [...xml.matchAll(new RegExp(`<${name}(?=[\\s/>])([^>]*)>`, "g"))].map((match) => attributesOf(match[1] ?? ""));
+}
+/**
+* The strings of a workbook's shared string table, in index order: each item's text runs joined, the phonetic runs of East Asian text left out.
+* @param xml - The `sharedStrings.xml` text.
+* @returns The strings.
+*/
+function sharedStrings(xml) {
+	return [...xml.matchAll(/<si(?:\s[^>]*)?>([\s\S]*?)<\/si>/g)].map((item) => {
+		return [...(item[1] ?? "").replaceAll(/<rPh[\s\S]*?<\/rPh>/g, "").matchAll(/<t(?:\s[^>]*)?>([\s\S]*?)<\/t>/g)].map((text) => unescape(text[1] ?? "")).join("");
+	});
+}
+/** The built-in number format ids that show a date or a time, as inclusive ranges (ECMA-376, 18.8.30): the short and long dates and times, the locale dates, and the elapsed and minute times. */
+const BUILT_IN_DATE_FORMATS = [
+	{
+		first: 14,
+		last: 22
+	},
+	{
+		first: 27,
+		last: 36
+	},
+	{
+		first: 45,
+		last: 47
+	},
+	{
+		first: 50,
+		last: 58
+	}
+];
+/**
+* Whether a custom number format shows a date or a time: it names a year, month, day, hour or second once its quoted text, escaped characters and bracketed conditions and colours are removed.
+* @param code - The format code.
+* @returns True for a date or time format.
+*/
+function isDateFormat(code) {
+	return /[ymdhs]/i.test(code.replaceAll(/"[^"]*"|\\.|\[[^\]]*\]/g, ""));
+}
+/**
+* Which cell styles show dates, by the style's index.
+* @param xml - The `styles.xml` text, or undefined when the workbook has none.
+* @returns The style indexes whose number format is a date or a time.
+*/
+function dateStyles(xml) {
+	if (xml === void 0) return /* @__PURE__ */ new Set();
+	const custom = /* @__PURE__ */ new Map();
+	for (const format of elements(xml, "numFmt")) if (format["numFmtId"] !== void 0 && format["formatCode"] !== void 0) custom.set(Number(format["numFmtId"]), format["formatCode"]);
+	const cellFormats = /<cellXfs(?:\s[^>]*)?>([\s\S]*?)<\/cellXfs>/.exec(xml)?.[1] ?? "";
+	const styles = /* @__PURE__ */ new Set();
+	for (const [index, format] of elements(cellFormats, "xf").entries()) {
+		const id = Number(format["numFmtId"] ?? "0");
+		const code = custom.get(id);
+		if (code === void 0 ? BUILT_IN_DATE_FORMATS.some((range) => id >= range.first && id <= range.last) : isDateFormat(code)) styles.add(index);
+	}
+	return styles;
+}
+/**
+* A date serial number as ISO 8601 text: a date when it has no time of day, a date and time when it has, and a time alone for a serial below one day in a 1900 workbook.
+* @param serial - The serial number, with the time of day as its fraction.
+* @param date1904 - Whether the workbook counts days from 1 January 1904.
+* @returns The text.
+* @throws ArchiveError for the serial of the 29th of February 1900, which is not a date.
+*/
+function isoOf(serial, date1904) {
+	const whole = Math.floor(serial);
+	const seconds = Math.round((serial - whole) * SECONDS_PER_DAY);
+	if (!date1904 && whole === 0) return [
+		Math.floor(seconds / 3600),
+		Math.floor(seconds / SECONDS_PER_MINUTE) % MINUTES_PER_HOUR,
+		seconds % SECONDS_PER_MINUTE
+	].map((part) => String(part).padStart(2, "0")).join(":");
+	if (!date1904 && whole === LEAP_BUG_SERIAL) throw new ArchiveError("a cell holds the serial of 29 February 1900, which is not a date");
+	return new Date(Date.parse(date1904 ? EPOCH_1904 : whole < LEAP_BUG_SERIAL ? EPOCH_BEFORE_LEAP_BUG : EPOCH_AFTER_LEAP_BUG) + (whole * SECONDS_PER_DAY + seconds) * MILLISECONDS).toISOString().slice(0, seconds === 0 ? DATE_LENGTH : DATE_TIME_LENGTH);
+}
+/**
+* The 0-based column of a cell reference such as `AB12`.
+* @param reference - The reference.
+* @returns The column index.
+*/
+function columnOf(reference) {
+	let index = 0;
+	for (const letter of reference.replaceAll(/[^A-Za-z]/g, "")) index = index * LETTERS + (letter.toUpperCase().charCodeAt(0) - FIRST_LETTER + 1);
+	return index - 1;
+}
+/**
+* One CSV field: quoted when it holds a comma, a quote or a line break.
+* @param text - The cell text.
+* @returns The field.
+*/
+function field(text) {
+	return /[",\r\n]/.test(text) ? `"${text.replaceAll("\"", "\"\"")}"` : text;
+}
+/**
+* Finds the part that holds a named sheet.
+* @param workbook - The `workbook.xml` text.
+* @param rels - The workbook's relationships text.
+* @param name - The sheet's name.
+* @returns The part's path inside the archive.
+* @throws ArchiveError when the workbook has no such sheet or its relationship is missing.
+*/
+function sheetPath(workbook, rels, name) {
+	const sheets = elements(workbook, "sheet");
+	const wanted = sheets.find((sheet) => sheet["name"] === name);
+	if (wanted === void 0) throw new ArchiveError(`the workbook has no sheet ${name}; it has ${sheets.map((sheet) => sheet["name"] ?? "").join(", ")}`);
+	const target = elements(rels, "Relationship").find((relation) => relation["Id"] === wanted["r:id"])?.["Target"];
+	if (target === void 0) throw new ArchiveError(`the workbook has no relationship for the sheet ${name}`);
+	return target.startsWith("/") ? target.slice(1) : `xl/${target}`;
+}
+/**
+* Reads a worksheet's cells into text by row and column.
+* @param xml - The worksheet's text.
+* @param strings - The shared strings.
+* @param dates - The date styles.
+* @param date1904 - Whether the workbook counts days from 1 January 1904.
+* @returns The rows by 1-based row number, each a sparse list of the cells' text, and the widest row's column count.
+* @throws ArchiveError when a cell names a shared string the workbook lacks or a date cell holds no number.
+*/
+function readRows(xml, strings, dates, date1904) {
+	const rows = /* @__PURE__ */ new Map();
+	let width = 0;
+	let next = 1;
+	for (const row of xml.matchAll(/<row(?=[\s>])([^>]*?)(?:\/>|>([\s\S]*?)<\/row>)/g)) {
+		const number = Number(attributesOf(row[1] ?? "")["r"] ?? next);
+		next = number + 1;
+		const cells = [];
+		for (const cell of (row[2] ?? "").matchAll(/<c(?=[\s>])([^>]*?)(?:\/>|>([\s\S]*?)<\/c>)/g)) {
+			const attributes = attributesOf(cell[1] ?? "");
+			const column = attributes["r"] === void 0 ? cells.length : columnOf(attributes["r"]);
+			const body = cell[2] ?? "";
+			const value = /<v(?:\s[^>]*)?>([\s\S]*?)<\/v>/.exec(body)?.[1];
+			let text;
+			if (attributes["t"] === "inlineStr") text = [...body.matchAll(/<t(?:\s[^>]*)?>([\s\S]*?)<\/t>/g)].map((run) => unescape(run[1] ?? "")).join("");
+			else if (value === void 0) text = void 0;
+			else if (attributes["t"] === "s") {
+				text = strings[Number(value)];
+				if (text === void 0) throw new ArchiveError(`row ${String(number)} names the shared string ${value}, which the workbook does not have`);
+			} else if (attributes["t"] === "b") text = value === "0" ? "FALSE" : "TRUE";
+			else if (attributes["t"] === "str" || attributes["t"] === "e") text = unescape(value);
+			else if (dates.has(Number(attributes["s"] ?? "0"))) {
+				const serial = Number(value);
+				if (!Number.isFinite(serial) || serial < 0) throw new ArchiveError(`row ${String(number)} has a date cell holding ${value}`);
+				text = isoOf(serial, date1904);
+			} else text = unescape(value);
+			if (text !== void 0 && text !== "") {
+				cells[column] = text;
+				width = Math.max(width, column + 1);
+			}
+		}
+		rows.set(number, cells);
+	}
+	return {
+		rows,
+		width
+	};
+}
+/**
+* Writes one sheet of an xlsx workbook as CSV text, so a recipe reads it with the table parser: one line for every row number from the first to the last (a row the sheet leaves out is an empty line, so the lines above the header are the rows an editor shows), every line the width of the sheet's widest row, a date cell as ISO 8601 text, a boolean as TRUE or FALSE, a number as the text the workbook stores, and a formula as its stored result.
+* @param bytes - The workbook.
+* @param sheet - The sheet's name.
+* @returns The CSV, in UTF-8.
+* @throws ArchiveError when the bytes are not a workbook, the sheet is missing, or a cell cannot be read.
+*/
+function sheetToCsv(bytes, sheet) {
+	let parts;
+	try {
+		const wanted = /* @__PURE__ */ new Set([
+			"xl/workbook.xml",
+			"xl/_rels/workbook.xml.rels",
+			"xl/sharedStrings.xml",
+			"xl/styles.xml"
+		]);
+		parts = unzipSync(bytes, { filter: (file) => wanted.has(file.name) });
+	} catch (error) {
+		throw new ArchiveError(`not a valid workbook: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
+	}
+	const text = (path) => {
+		const part = getOwn(parts, path);
+		return part === void 0 ? void 0 : new TextDecoder("utf-8").decode(part);
+	};
+	const workbook = text("xl/workbook.xml");
+	const rels = text("xl/_rels/workbook.xml.rels");
+	if (workbook === void 0 || rels === void 0) throw new ArchiveError("the file is not an xlsx workbook: it has no xl/workbook.xml with its relationships");
+	const path = sheetPath(workbook, rels, sheet);
+	const sheetBytes = getOwn(unzipSync(bytes, { filter: (file) => file.name === path }), path);
+	if (sheetBytes === void 0) throw new ArchiveError(`the workbook names the sheet ${sheet} in ${path}, which the file does not have`);
+	const strings = text("xl/sharedStrings.xml");
+	const { rows, width } = readRows(new TextDecoder("utf-8").decode(sheetBytes), strings === void 0 ? [] : sharedStrings(strings), dateStyles(text("xl/styles.xml")), /<workbookPr(?=[\s/>])[^>]*date1904="(?:1|true)"/.test(workbook));
+	const last = Math.max(0, ...rows.keys());
+	const lines = [];
+	for (let number = 1; number <= last; number += 1) {
+		const cells = rows.get(number) ?? [];
+		lines.push(Array.from({ length: width }, (_empty, column) => field(cells[column] ?? "")).join(","));
+	}
+	return new TextEncoder().encode(lines.length === 0 ? "" : `${lines.join("\n")}\n`);
+}
+//#endregion
 //#region src/node/archive.ts
 /**
 * Whether an archive member is a macOS resource fork (`__MACOSX/` or an AppleDouble `._` file), which carries Finder metadata rather than data and never matches a selector.
@@ -1059,6 +1308,10 @@ function pick(paths, member) {
 function unpack(bytes, archive) {
 	if (archive.kind === "none") return { bytes };
 	if (archive.kind === "gzip") return { bytes: gunzip(bytes) };
+	if (archive.kind === "xlsx") return {
+		bytes: sheetToCsv(bytes, archive.sheet),
+		member: archive.sheet
+	};
 	if (archive.kind === "zip") {
 		const names = [];
 		unzip(bytes, (file) => {
@@ -1120,20 +1373,40 @@ const REDIRECTS = new Set(Object.values({
 }));
 /** What the downloader says it is, so a server operator can tell where the traffic comes from. */
 const USER_AGENT = "keepset (+https://github.com/Mearman/keepset)";
+/** The failure of a connection that sent nothing for the stall time. */
+var StalledError = class extends KeepsetError {};
+/**
+* Whether a failure is the connection's and may pass: a stall, or a socket error such as a reset, a broken pipe or a timeout, which carries an `E` code. A bad URL, a redirect that leads nowhere and an abort are not.
+* @param error - The failure.
+* @returns True when asking again may succeed.
+*/
+function isConnectionFailure(error) {
+	return error instanceof StalledError || typeof error === "object" && error !== null && "code" in error && typeof error.code === "string" && /^E[A-Z]+$/.test(error.code) && error.code !== "ERR_INVALID_URL";
+}
 /**
 * Sends one GET and resolves with the response head, its body still unread.
 * @param url - The URL.
 * @param signal - Aborts the request.
+* @param stall - How many milliseconds the connection may send nothing, in the head or the body that follows.
 * @returns The response.
+* @throws KeepsetError when the connection stalls.
 */
-async function get(url, signal) {
+async function get(url, signal, stall) {
 	const send = url.protocol === "https:" ? request$1 : request;
 	return new Promise((resolve, reject) => {
+		let answered;
 		const request = send(url, {
 			method: "GET",
 			headers: { "user-agent": USER_AGENT },
 			...signal === void 0 ? {} : { signal }
-		}, resolve);
+		}, (response) => {
+			answered = response;
+			resolve(response);
+		});
+		request.setTimeout(stall, () => {
+			const dead = new StalledError(`${url.href} sent nothing for ${String(stall)} milliseconds, so the connection was given up as dead`);
+			(answered ?? request).destroy(dead);
+		});
 		request.on("error", reject);
 		request.end();
 	});
@@ -1154,13 +1427,13 @@ async function readBody(response) {
 /**
 * Requests one URL, following its redirects by recursion.
 * @param url - The URL to request.
-* @param signal - Aborts the request.
+* @param init - The request options.
 * @param redirects - How many redirects led here.
 * @returns The final response.
 */
-async function follow(url, signal, redirects) {
+async function follow(url, init, redirects) {
 	if (url.protocol !== "http:" && url.protocol !== "https:") throw new KeepsetError(`${url.href} is not an http or https URL`);
-	const response = await get(url, signal);
+	const response = await get(url, init.signal, init.stallMilliseconds ?? 12e4);
 	const { statusCode: status, statusMessage } = response;
 	if (status === void 0 || statusMessage === void 0) throw new KeepsetError(`${url.href} answered without a status line`);
 	if (!REDIRECTS.has(status)) return new Response(await readBody(response), {
@@ -1171,17 +1444,36 @@ async function follow(url, signal, redirects) {
 	const { location } = response.headers;
 	if (location === void 0) throw new KeepsetError(`${url.href} answered ${String(status)} without a location`);
 	if (redirects === MAX_REDIRECTS) throw new KeepsetError(`${url.href} redirected more than ${String(MAX_REDIRECTS)} times`);
-	return follow(new URL(location, url), signal, redirects + 1);
+	return follow(new URL(location, url), init, redirects + 1);
 }
 /**
 * Downloads a URL's bytes exactly as the server sends them, following redirects. Unlike the platform `fetch`, it never undoes a `Content-Encoding`: servers commonly label a `.gz` file `Content-Encoding: gzip`, and a recipe that unpacks gzip needs the file as published, which is also what its SHA-256 is recorded over. No `Accept-Encoding` is sent, so a server that compresses on the fly has no reason to.
 * @param url - An http or https URL.
 * @param init - The request options.
 * @returns A Response holding the raw bytes, with the final status.
-* @throws KeepsetError when a redirect has no location, leaves http and https, or passes the redirect limit.
+* @throws KeepsetError when a redirect has no location, leaves http and https, or passes the redirect limit; the connection's failure, after the tries, when it keeps failing or sends nothing for longer than the stall time.
 */
 async function downloadRaw(url, init) {
-	return follow(new URL(url), init.signal, 0);
+	return tryDownload(url, init, 1);
+}
+/**
+* One try of a download, and the tries after it while the connection fails.
+* @param url - The URL.
+* @param init - The request options.
+* @param attempt - Which try this is, from 1.
+* @returns The response.
+* @throws The last failure when the tries are used up, or at once for a failure that is not the connection's.
+*/
+async function tryDownload(url, init, attempt) {
+	try {
+		return await follow(new URL(url), init, 0);
+	} catch (error) {
+		if (attempt >= 3 || init.signal?.aborted === true || !isConnectionFailure(error)) throw error;
+		await new Promise((resolve) => {
+			setTimeout(resolve, (init.retryDelayMilliseconds ?? 3e3) * attempt);
+		});
+		return tryDownload(url, init, attempt + 1);
+	}
 }
 //#endregion
 //#region src/node/paged.ts
@@ -1228,7 +1520,7 @@ const FeatureListSchema = z.array(z.unknown());
 async function readPages(layer, held) {
 	if (held.length >= layer.count) return held;
 	const url = `${layer.query}?where=1%3D1&outFields=*&orderByFields=${encodeURIComponent(layer.orderBy)}&resultOffset=${String(held.length)}&f=geojson`;
-	const page = await getJson(layer.port, url, layer.signal);
+	const page = await withAttempts(async () => getJson(layer.port, url, layer.signal), layer.signal);
 	const features = FeatureListSchema.safeParse(typeof page === "object" && page !== null && "features" in page ? page.features : void 0);
 	if (!features.success || features.data.length === 0) throw new KeepsetError(`${layer.query} gave no features at offset ${String(held.length)}, before the ${String(layer.count)} it reports, so it does not page by offset`);
 	return readPages(layer, [...held, ...features.data]);
@@ -1258,6 +1550,95 @@ async function downloadArcgisLayer(layerUrl, orderBy, port, signal) {
 		features: pages
 	}));
 }
+/** How many times a page is requested before the download fails: a connection can drop or go silent in one request of dozens and succeed in the next, and the pages are read by offset so asking again is exact. */
+const PAGE_ATTEMPTS = 3;
+/**
+* Runs one page read, asking again when it fails, so one bad response of a long download does not lose the rest.
+* @param read - The page read.
+* @param signal - Aborts the download, which is not asked again.
+* @param attempt - Which attempt this is, from 1.
+* @returns What the read returns.
+* @throws The last failure, when the attempts are used up or the download was aborted.
+*/
+async function withAttempts(read, signal, attempt = 1) {
+	try {
+		return await read();
+	} catch (error) {
+		if (attempt >= PAGE_ATTEMPTS || signal?.aborted === true) throw error;
+		return withAttempts(read, signal, attempt + 1);
+	}
+}
+const LINE_FEED = 10;
+const QUOTE = 34;
+/**
+* How many CSV records a response holds: the line feeds outside a quoted field, and one more when the last record has no line feed.
+* @param bytes - A CSV response.
+* @returns The number of records, the header included.
+*/
+function recordsIn(bytes) {
+	let records = 0;
+	let quoted = false;
+	for (const byte of bytes) if (byte === QUOTE) quoted = !quoted;
+	else if (byte === LINE_FEED && !quoted) records += 1;
+	return bytes.length > 0 && bytes[bytes.length - 1] !== LINE_FEED ? records + 1 : records;
+}
+/**
+* Reads one response of a Socrata resource endpoint as bytes.
+* @param port - The network port.
+* @param url - The request URL.
+* @param signal - Aborts the request.
+* @returns The response body.
+* @throws KeepsetError when the server does not answer successfully.
+*/
+async function getBytes(port, url, signal) {
+	const response = await port(url, signal === void 0 ? {} : { signal });
+	if (!response.ok) throw new KeepsetError(`${url} answered ${String(response.status)} ${response.statusText}`);
+	return new Uint8Array(await response.arrayBuffer());
+}
+/**
+* Reads a resource's pages, each starting where the rows held so far end, and checks each holds exactly the rows it should, so a response cut short is an error and not a table with rows missing.
+* @param read - The resource read.
+* @param held - The pages read so far, each after the first without its header record.
+* @param offset - How many rows the pages hold.
+* @returns Every page.
+* @throws KeepsetError when a page holds a different number of rows than the count says it should, or its header differs from the first page's.
+*/
+async function readResourcePages(read, held, offset) {
+	if (offset >= read.count) return held;
+	const expected = Math.min(read.pageSize, read.count - offset);
+	const page = await withAttempts(async () => {
+		const bytes = await getBytes(read.port, `${read.resource}?%24order=%3Aid&%24limit=${String(read.pageSize)}&%24offset=${String(offset)}`, read.signal);
+		const rows = recordsIn(bytes) - 1;
+		if (rows !== expected) throw new KeepsetError(`${read.resource} gave ${String(rows)} rows at offset ${String(offset)} where ${String(expected)} of its ${String(read.count)} were due, so a response was cut short or the table changed during the download`);
+		return bytes;
+	}, read.signal);
+	const first = held[0];
+	const headerEnd = page.indexOf(LINE_FEED) + 1;
+	if (first !== void 0 && !page.subarray(0, headerEnd).every((byte, index) => byte === first[index])) throw new KeepsetError(`${read.resource} gave a different header at offset ${String(offset)} than at the start`);
+	return readResourcePages(read, [...held, first === void 0 ? page : page.subarray(headerEnd)], offset + expected);
+}
+/**
+* Downloads a whole Socrata resource as one CSV: the row count is read first, then pages of `pageSize` rows in the order of the rows' ids, each checked to hold the rows it should, and joined under the first page's header.
+* @param resource - The resource's URL, such as `https://data.example.org/resource/abcd-1234.csv`, without a query.
+* @param pageSize - How many rows a page holds.
+* @param port - The network port.
+* @param signal - Aborts the download.
+* @returns The bytes of one CSV, which is what the parser reads and what the provenance hashes.
+* @throws KeepsetError when the resource does not report a count, or a page is not what it should be.
+*/
+async function downloadSocrataResource(resource, pageSize, port, signal) {
+	const reported = /^"count"\r?\n"(\d+)"/.exec(new TextDecoder("utf-8").decode(await getBytes(port, `${resource}?%24select=count(*)`, signal)));
+	if (reported?.[1] === void 0) throw new KeepsetError(`${resource} did not report a row count`);
+	const pages = await readResourcePages({
+		resource,
+		pageSize,
+		count: Number(reported[1]),
+		port,
+		signal
+	}, [], 0);
+	if (pages.length === 0) throw new KeepsetError(`${resource} has no rows`);
+	return Buffer.concat(pages);
+}
 /**
 * Downloads a paged source as the one file its pages make, in the way its recipe's paging says.
 * @param url - The source's URL.
@@ -1268,7 +1649,7 @@ async function downloadArcgisLayer(layerUrl, orderBy, port, signal) {
 * @throws KeepsetError when a page cannot be read or the pages do not add up to what the source reports.
 */
 async function downloadPaged(url, paging, port, signal) {
-	return downloadArcgisLayer(url, paging.orderBy, port, signal);
+	return paging.kind === "arcgis-query" ? downloadArcgisLayer(url, paging.orderBy, port, signal) : downloadSocrataResource(url, paging.pageSize, port, signal);
 }
 //#endregion
 //#region src/node/fetch-dataset.ts

package/dist-cli/keepset.mjs

@@ -4,7 +4,7 @@ import { KeepsetError, NAME_SEPARATOR, countTarget, getDataset, listDatasets, li
 import { fetchDataset, roundTripDataset } from "keepset/node";
 import { parseArgs } from "node:util";
 //#region package.json
-var version = "0.19.0";
+var version = "0.21.0";
 const usage = `Usage: keepset <command> [options]
 
 Commands:

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "keepset",
   "description": "Library, CLI and statically generated catalogue for dataset fetch-and-normalise recipes",
-  "version": "0.19.0",
+  "version": "0.21.0",
   "bin": {
     "keepset": "./dist-cli/keepset.mjs"
   },