keepset 0.3.0 → 0.4.0

package/dist/{categories-BIZJUKUw.d.ts → categories-CUf1VecC.d.ts}

@@ -57,11 +57,11 @@ declare const CitationSchema: z.ZodObject<{
 type Citation = z.infer<typeof CitationSchema>;
 /** A relative path inside an archive or a catalogue: `/`-separated segments, none empty, `.` or `..`, so it can never climb out of the directory it is resolved against. */
 declare const RelativePathSchema: z.ZodString;
-/** Which file of an archive to read: a member path, matched against a member's whole path or its trailing segments (so `net.gml` finds `net/net.gml`), or the member whose name ends with an extension. Either must match exactly one member when the recipe runs; macOS resource-fork members (`__MACOSX/` and `._` files) never match. */
+/** Which file of an archive to read: a member path, matched against a member's whole path or its trailing segments (so `net.gml` finds `net/net.gml`), or the member whose name ends with one of a list of extensions (a format can have several, such as Pajek's `.net` and `.paj`). Either must match exactly one member when the recipe runs; macOS resource-fork members (`__MACOSX/` and `._` files) never match. */
 declare const ArchiveMemberSchema: z.ZodUnion<readonly [z.ZodObject<{
   path: z.ZodString;
 }, z.core.$strict>, z.ZodObject<{
-  extension: z.ZodString;
+  extensions: z.ZodArray<z.ZodString>;
 }, z.core.$strict>]>;
 type ArchiveMember = z.infer<typeof ArchiveMemberSchema>;
 /** How the downloaded bytes unpack into the one file the parser reads. */
@@ -74,21 +74,21 @@ declare const ArchiveSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
   member: z.ZodUnion<readonly [z.ZodObject<{
     path: z.ZodString;
   }, z.core.$strict>, z.ZodObject<{
-    extension: z.ZodString;
+    extensions: z.ZodArray<z.ZodString>;
   }, z.core.$strict>]>;
 }, z.core.$strict>, z.ZodObject<{
   kind: z.ZodLiteral<"tar">;
   member: z.ZodUnion<readonly [z.ZodObject<{
     path: z.ZodString;
   }, z.core.$strict>, z.ZodObject<{
-    extension: z.ZodString;
+    extensions: z.ZodArray<z.ZodString>;
   }, z.core.$strict>]>;
 }, z.core.$strict>, z.ZodObject<{
   kind: z.ZodLiteral<"tar.gz">;
   member: z.ZodUnion<readonly [z.ZodObject<{
     path: z.ZodString;
   }, z.core.$strict>, z.ZodObject<{
-    extension: z.ZodString;
+    extensions: z.ZodArray<z.ZodString>;
   }, z.core.$strict>]>;
 }, z.core.$strict>], "kind">;
 type Archive = z.infer<typeof ArchiveSchema>;
@@ -143,21 +143,21 @@ declare const RecipeSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
     member: z.ZodUnion<readonly [z.ZodObject<{
       path: z.ZodString;
     }, z.core.$strict>, z.ZodObject<{
-      extension: z.ZodString;
+      extensions: z.ZodArray<z.ZodString>;
     }, z.core.$strict>]>;
   }, z.core.$strict>, z.ZodObject<{
     kind: z.ZodLiteral<"tar">;
     member: z.ZodUnion<readonly [z.ZodObject<{
       path: z.ZodString;
     }, z.core.$strict>, z.ZodObject<{
-      extension: z.ZodString;
+      extensions: z.ZodArray<z.ZodString>;
     }, z.core.$strict>]>;
   }, z.core.$strict>, z.ZodObject<{
     kind: z.ZodLiteral<"tar.gz">;
     member: z.ZodUnion<readonly [z.ZodObject<{
       path: z.ZodString;
     }, z.core.$strict>, z.ZodObject<{
-      extension: z.ZodString;
+      extensions: z.ZodArray<z.ZodString>;
     }, z.core.$strict>]>;
   }, z.core.$strict>], "kind">;
   parser: z.ZodDiscriminatedUnion<[z.ZodObject<{
@@ -261,21 +261,21 @@ declare const EntrySchema: z.ZodObject<{
       member: z.ZodUnion<readonly [z.ZodObject<{
         path: z.ZodString;
       }, z.core.$strict>, z.ZodObject<{
-        extension: z.ZodString;
+        extensions: z.ZodArray<z.ZodString>;
       }, z.core.$strict>]>;
     }, z.core.$strict>, z.ZodObject<{
       kind: z.ZodLiteral<"tar">;
       member: z.ZodUnion<readonly [z.ZodObject<{
         path: z.ZodString;
       }, z.core.$strict>, z.ZodObject<{
-        extension: z.ZodString;
+        extensions: z.ZodArray<z.ZodString>;
       }, z.core.$strict>]>;
     }, z.core.$strict>, z.ZodObject<{
       kind: z.ZodLiteral<"tar.gz">;
       member: z.ZodUnion<readonly [z.ZodObject<{
         path: z.ZodString;
       }, z.core.$strict>, z.ZodObject<{
-        extension: z.ZodString;
+        extensions: z.ZodArray<z.ZodString>;
       }, z.core.$strict>]>;
     }, z.core.$strict>], "kind">;
     parser: z.ZodDiscriminatedUnion<[z.ZodObject<{
@@ -367,21 +367,21 @@ declare const EntrySchema: z.ZodObject<{
         member: {
           path: string;
         } | {
-          extension: string;
+          extensions: string[];
         };
       } | {
         kind: "tar";
         member: {
           path: string;
         } | {
-          extension: string;
+          extensions: string[];
         };
       } | {
         kind: "tar.gz";
         member: {
           path: string;
         } | {
-          extension: string;
+          extensions: string[];
         };
       };
       parser: {
@@ -552,21 +552,21 @@ declare const CatalogueIndexSchema: z.ZodObject<{
         member: z.ZodUnion<readonly [z.ZodObject<{
           path: z.ZodString;
         }, z.core.$strict>, z.ZodObject<{
-          extension: z.ZodString;
+          extensions: z.ZodArray<z.ZodString>;
         }, z.core.$strict>]>;
       }, z.core.$strict>, z.ZodObject<{
         kind: z.ZodLiteral<"tar">;
         member: z.ZodUnion<readonly [z.ZodObject<{
           path: z.ZodString;
         }, z.core.$strict>, z.ZodObject<{
-          extension: z.ZodString;
+          extensions: z.ZodArray<z.ZodString>;
         }, z.core.$strict>]>;
       }, z.core.$strict>, z.ZodObject<{
         kind: z.ZodLiteral<"tar.gz">;
         member: z.ZodUnion<readonly [z.ZodObject<{
           path: z.ZodString;
         }, z.core.$strict>, z.ZodObject<{
-          extension: z.ZodString;
+          extensions: z.ZodArray<z.ZodString>;
         }, z.core.$strict>]>;
       }, z.core.$strict>], "kind">;
       parser: z.ZodDiscriminatedUnion<[z.ZodObject<{
@@ -658,21 +658,21 @@ declare const CatalogueIndexSchema: z.ZodObject<{
           member: {
             path: string;
           } | {
-            extension: string;
+            extensions: string[];
           };
         } | {
           kind: "tar";
           member: {
             path: string;
           } | {
-            extension: string;
+            extensions: string[];
           };
         } | {
           kind: "tar.gz";
           member: {
             path: string;
           } | {
-            extension: string;
+            extensions: string[];
           };
         };
         parser: {
@@ -762,21 +762,21 @@ declare const CatalogueIndexSchema: z.ZodObject<{
           member: {
             path: string;
           } | {
-            extension: string;
+            extensions: string[];
           };
         } | {
           kind: "tar";
           member: {
             path: string;
           } | {
-            extension: string;
+            extensions: string[];
           };
         } | {
           kind: "tar.gz";
           member: {
             path: string;
           } | {
-            extension: string;
+            extensions: string[];
           };
         };
         parser: {

package/dist/index.d.ts

@@ -1,4 +1,4 @@
-import { $ as SnapshotSchema, A as SemVerSchema, B as Entry, C as Dataset, D as LineageSchema, E as Lineage, F as ArchiveSchema, G as LicenceSchema, H as HttpUrlSchema, I as Citation, J as RESERVED_META_KEYS, K as Parser, L as CitationSchema, M as Archive, N as ArchiveMember, O as Manifest, P as ArchiveMemberSchema, Q as Snapshot, R as CrosswalkReference, S as CatalogueIndexSchema, T as createCatalogue, U as IdSchema, V as EntrySchema, W as Licence, X as RecipeSchema, Y as Recipe, Z as RelativePathSchema, _ as GraphV1Schema, a as CategoryTargets, b as Catalogue, c as getCategory, d as listCategories, et as Source, f as GraphEdge, g as GraphV1, h as GraphNodeSchema, i as CategoryTarget, it as waybackUrl, j as parseManifest, k as ManifestSchema, l as getTarget, m as GraphNode, n as CategoryId, nt as WaybackTimestampSchema, o as CountedKey, p as GraphEdgeSchema, q as ParserSchema, r as CategoryIdSchema, rt as parseEntry, s as countTarget, t as Category, tt as SourceSchema, u as kindOf, v as JsonScalar, w as DatasetQuery, x as CatalogueIndex, y as JsonScalarSchema, z as CrosswalkReferenceSchema } from "./categories-BIZJUKUw.js";
+import { $ as SnapshotSchema, A as SemVerSchema, B as Entry, C as Dataset, D as LineageSchema, E as Lineage, F as ArchiveSchema, G as LicenceSchema, H as HttpUrlSchema, I as Citation, J as RESERVED_META_KEYS, K as Parser, L as CitationSchema, M as Archive, N as ArchiveMember, O as Manifest, P as ArchiveMemberSchema, Q as Snapshot, R as CrosswalkReference, S as CatalogueIndexSchema, T as createCatalogue, U as IdSchema, V as EntrySchema, W as Licence, X as RecipeSchema, Y as Recipe, Z as RelativePathSchema, _ as GraphV1Schema, a as CategoryTargets, b as Catalogue, c as getCategory, d as listCategories, et as Source, f as GraphEdge, g as GraphV1, h as GraphNodeSchema, i as CategoryTarget, it as waybackUrl, j as parseManifest, k as ManifestSchema, l as getTarget, m as GraphNode, n as CategoryId, nt as WaybackTimestampSchema, o as CountedKey, p as GraphEdgeSchema, q as ParserSchema, r as CategoryIdSchema, rt as parseEntry, s as countTarget, t as Category, tt as SourceSchema, u as kindOf, v as JsonScalar, w as DatasetQuery, x as CatalogueIndex, y as JsonScalarSchema, z as CrosswalkReferenceSchema } from "./categories-CUf1VecC.js";
 import { Crosswalk } from "twindex";
 //#region src/errors.d.ts
 /** The root of this package's error taxonomy: every error keepset throws is a `KeepsetError`, so callers can catch the family with one check. */

package/dist/index.js

@@ -1,4 +1,4 @@
-import { A as getCategory, B as DatasetNotFoundError, C as SnapshotSchema, D as waybackUrl, E as parseEntry, F as GraphNodeSchema, H as NativeParseError, I as GraphV1Schema, L as JsonScalarSchema, M as kindOf, N as listCategories, O as CategoryIdSchema, P as GraphEdgeSchema, R as ArchiveError, S as RelativePathSchema, T as WaybackTimestampSchema, U as RecipeError, V as KeepsetError, W as UnknownCategoryError, _ as IdSchema, a as createCatalogue, b as RESERVED_META_KEYS, c as ManifestSchema, d as ArchiveMemberSchema, f as ArchiveSchema, g as HttpUrlSchema, h as EntrySchema, i as CatalogueIndexSchema, j as getTarget, k as countTarget, l as SemVerSchema, m as CrosswalkReferenceSchema, n as getManifest, p as CitationSchema, r as listDatasets, s as LineageSchema, t as getDataset, u as parseManifest, v as LicenceSchema, w as SourceSchema, x as RecipeSchema, y as ParserSchema, z as CatalogueSyntaxError } from "./bundled--fOitQlP.js";
+import { A as getCategory, B as DatasetNotFoundError, C as SnapshotSchema, D as waybackUrl, E as parseEntry, F as GraphNodeSchema, H as NativeParseError, I as GraphV1Schema, L as JsonScalarSchema, M as kindOf, N as listCategories, O as CategoryIdSchema, P as GraphEdgeSchema, R as ArchiveError, S as RelativePathSchema, T as WaybackTimestampSchema, U as RecipeError, V as KeepsetError, W as UnknownCategoryError, _ as IdSchema, a as createCatalogue, b as RESERVED_META_KEYS, c as ManifestSchema, d as ArchiveMemberSchema, f as ArchiveSchema, g as HttpUrlSchema, h as EntrySchema, i as CatalogueIndexSchema, j as getTarget, k as countTarget, l as SemVerSchema, m as CrosswalkReferenceSchema, n as getManifest, p as CitationSchema, r as listDatasets, s as LineageSchema, t as getDataset, u as parseManifest, v as LicenceSchema, w as SourceSchema, x as RecipeSchema, y as ParserSchema, z as CatalogueSyntaxError } from "./bundled-DN3sSHfu.js";
 //#region src/serialise.ts
 /**
 * Copies a JSON value with every object's keys in ascending code-unit order, the order the house JSON lint (`json/sort-keys`) requires of committed JSON. `Object.fromEntries` defines each key as an own data property, so a key such as `__proto__` stays a key.
@@ -11,12 +11,20 @@ function sortKeys(value) {
 	return value;
 }
 /**
+* Serialises any JSON value in the committed form: two-space indented, keys in ascending order, with a trailing newline.
+* @param value - A JSON value.
+* @returns The file text.
+*/
+function canonicalJson(value) {
+	return `${JSON.stringify(sortKeys(value), void 0, 2)}\n`;
+}
+/**
 * Serialises a catalogue document (an entry, a manifest or a crosswalk file) as its file holds it: two-space indented JSON with keys in ascending order and a trailing newline. The form depends only on the document's content, so parsing a file written this way and serialising the result reproduces the file byte for byte.
 * @param doc - A parsed entry, manifest or crosswalk.
 * @returns The file text.
 */
 function serialise(doc) {
-	return `${JSON.stringify(sortKeys(doc), void 0, 2)}\n`;
+	return canonicalJson(doc);
 }
 //#endregion
 export { ArchiveError, ArchiveMemberSchema, ArchiveSchema, CatalogueIndexSchema, CatalogueSyntaxError, CategoryIdSchema, CitationSchema, CrosswalkReferenceSchema, DatasetNotFoundError, EntrySchema, GraphEdgeSchema, GraphNodeSchema, GraphV1Schema, HttpUrlSchema, IdSchema, JsonScalarSchema, KeepsetError, LicenceSchema, LineageSchema, ManifestSchema, NativeParseError, ParserSchema, RESERVED_META_KEYS, RecipeError, RecipeSchema, RelativePathSchema, SemVerSchema, SnapshotSchema, SourceSchema, UnknownCategoryError, WaybackTimestampSchema, countTarget, createCatalogue, getCategory, getDataset, getManifest, getTarget, kindOf, listCategories, listDatasets, parseEntry, parseManifest, serialise, waybackUrl };

package/dist/node/fetch-dataset.d.ts

@@ -1,6 +1,6 @@
-import { B as Entry, C as Dataset, Q as Snapshot, et as Source, i as CategoryTarget } from "../categories-BIZJUKUw.js";
+import { B as Entry, C as Dataset, Q as Snapshot, et as Source, i as CategoryTarget } from "../categories-CUf1VecC.js";
 //#region src/node/fetch-dataset.d.ts
-/** The network port a recipe downloads through: the platform `fetch`, or a stand-in under test. */
+/** The network port a recipe downloads through: `downloadRaw` by default, which returns the bytes exactly as served, or a stand-in under test. */
 export type FetchPort = (url: string, init: {
   readonly signal?: AbortSignal;
 }) => Promise<Response>;
@@ -48,7 +48,7 @@ export declare function downloadUrl(source: Source): string;
  */
 export declare function fetchEntry(dataset: Dataset, options?: FetchOptions): Promise<FetchedDataset>;
 /**
- * Fetches a dataset of the bundled catalogue by id and resolves it to its category's target shape, so `fetchDataset("cora")` yields graph/v1 `{ meta, nodes, edges }` directly. Node-only: it downloads with the platform `fetch` and unpacks with Node's zlib.
+ * Fetches a dataset of the bundled catalogue by id and resolves it to its category's target shape, so `fetchDataset("cora")` yields graph/v1 `{ meta, nodes, edges }` directly. Node-only: it downloads with Node's http and https clients and unpacks with Node's zlib.
  * @param id - The entry id.
  * @param options - The network port, abort signal and clock.
  * @returns The resolved target with its provenance and any count warnings.

package/dist/node/fetch-dataset.js

@@ -1,8 +1,10 @@
-import { H as NativeParseError, R as ArchiveError, U as RecipeError, V as KeepsetError, j as getTarget, k as countTarget, o as getOwn, t as getDataset } from "../bundled--fOitQlP.js";
+import { H as NativeParseError, R as ArchiveError, U as RecipeError, V as KeepsetError, j as getTarget, k as countTarget, o as getOwn, t as getDataset } from "../bundled-DN3sSHfu.js";
 import { encode } from "twindex";
 import { createHash } from "node:crypto";
 import { gunzipSync } from "node:zlib";
 import { unzipSync } from "fflate";
+import { request } from "node:http";
+import { request as request$1 } from "node:https";
 //#region src/parsers/native.ts
 /**
 * Collects node ids in order of first appearance, so a parser that learns its nodes from its edges emits them deterministically.
@@ -346,8 +348,9 @@ function parseMatrixMarket(text) {
 }
 //#endregion
 //#region src/parsers/pajek.ts
-/** Section keywords, lower-cased, mapped to how their lines are read. `*Partition` and `*Vector` hold node attributes this version does not carry, so their lines are skipped, as are the `*Vertices` lines inside them. */
+/** Section keywords, lower-cased, mapped to how their lines are read. `*Network` names the network and starts no section. `*Partition` and `*Vector` hold node attributes this version does not carry, so their lines are skipped, as are the `*Vertices` lines inside them. */
 const sections = /* @__PURE__ */ new Map([
+	["*network", "network"],
 	["*vertices", "vertices"],
 	["*arcs", "pairs"],
 	["*edges", "pairs"],
@@ -358,6 +361,18 @@ const sections = /* @__PURE__ */ new Map([
 	["*vector", "attributes"]
 ]);
 /**
+* Resolves a section keyword, which Pajek reads case-insensitively and accepts abbreviated (`*net` for `*Network`, `*vert` for `*Vertices`): an exact keyword first, otherwise the shortest keyword it abbreviates, so `*edge` is `*Edges` rather than `*Edgeslist`.
+* @param keyword - The keyword as written.
+* @returns How its section is read, or undefined for a keyword that abbreviates none.
+*/
+function sectionOf(keyword) {
+	const name = keyword.toLowerCase();
+	const exact = sections.get(name);
+	if (exact !== void 0) return exact;
+	const [shortest] = [...sections.keys()].filter((candidate) => candidate.startsWith(name)).sort((left, right) => left.length - right.length);
+	return shortest === void 0 ? void 0 : sections.get(shortest);
+}
+/**
 * Reads a vertex number, which Pajek numbers from 1 to the declared vertex count.
 * @param token - The token.
 * @param count - The declared vertex count.
@@ -388,14 +403,13 @@ function parsePajek(text) {
 		const tokens = content.split(/\s+/);
 		const [keyword] = tokens;
 		if (keyword?.startsWith("*") === true) {
-			const name = keyword.toLowerCase();
-			if (name === "*network") {
+			const next = sectionOf(keyword);
+			if (next === void 0) throw new NativeParseError("pajek", lineNumber, `unknown section ${keyword}`);
+			if (next === "network") {
 				networks++;
 				if (networks > 1) throw new NativeParseError("pajek", lineNumber, "a project file holding more than one network is not supported");
 				continue;
 			}
-			const next = sections.get(name);
-			if (next === void 0) throw new NativeParseError("pajek", lineNumber, `unknown section ${keyword}`);
 			if (next === "vertices" && section !== "attributes") {
 				if (count !== void 0) throw new NativeParseError("pajek", lineNumber, "a network declares *Vertices twice");
 				const declared = tokens[1];
@@ -616,7 +630,7 @@ function isResourceFork(path) {
 	return path.startsWith("__MACOSX/") || path.split("/").some((segment) => segment.startsWith("._"));
 }
 /**
-* Whether a member path matches a selector: a path selector matches the whole member path or its trailing segments, an extension selector the end of the member's name.
+* Whether a member path matches a selector: a path selector matches the whole member path or its trailing segments, an extensions selector the end of the member's name, ignoring case.
 * @param path - The member path.
 * @param member - The recipe's selector.
 * @returns True on a match.
@@ -624,7 +638,7 @@ function isResourceFork(path) {
 function selects(path, member) {
 	if (isResourceFork(path) || path.endsWith("/")) return false;
 	if ("path" in member) return path === member.path || path.endsWith(`/${member.path}`);
-	return path.toLowerCase().endsWith(member.extension.toLowerCase());
+	return member.extensions.some((extension) => path.toLowerCase().endsWith(extension.toLowerCase()));
 }
 /**
 * Picks the one member a selector matches.
@@ -637,7 +651,7 @@ function pick(paths, member) {
 	const matched = paths.filter((path) => selects(path, member));
 	const [only] = matched;
 	if (matched.length !== 1 || only === void 0) {
-		const wanted = "path" in member ? `member ${member.path}` : `one member ending ${member.extension}`;
+		const wanted = "path" in member ? `member ${member.path}` : `one member ending ${member.extensions.join(" or ")}`;
 		throw new ArchiveError(`expected ${wanted}, found ${String(matched.length)} among: ${paths.join(", ")}`);
 	}
 	return only;
@@ -701,6 +715,82 @@ function unzip(bytes, filter) {
 	}
 }
 //#endregion
+//#region src/node/download.ts
+/** The most redirects one download follows: the Fetch Standard's own limit (https://fetch.spec.whatwg.org/#http-redirect-fetch, step 5). */
+const MAX_REDIRECTS = 20;
+const REDIRECTS = new Set(Object.values({
+	movedPermanently: 301,
+	found: 302,
+	seeOther: 303,
+	temporaryRedirect: 307,
+	permanentRedirect: 308
+}));
+/** What the downloader says it is, so a server operator can tell where the traffic comes from. */
+const USER_AGENT = "keepset (+https://github.com/ExaDev/keepset)";
+/**
+* Sends one GET and resolves with the response head, its body still unread.
+* @param url - The URL.
+* @param signal - Aborts the request.
+* @returns The response.
+*/
+async function get(url, signal) {
+	const send = url.protocol === "https:" ? request$1 : request;
+	return new Promise((resolve, reject) => {
+		const request = send(url, {
+			method: "GET",
+			headers: { "user-agent": USER_AGENT },
+			...signal === void 0 ? {} : { signal }
+		}, resolve);
+		request.on("error", reject);
+		request.end();
+	});
+}
+/**
+* Reads a response body to the end.
+* @param response - The response.
+* @returns The body's bytes.
+*/
+async function readBody(response) {
+	const chunks = [];
+	for await (const chunk of response) {
+		if (!(chunk instanceof Uint8Array)) throw new KeepsetError("the response body yielded a chunk that is not bytes");
+		chunks.push(chunk);
+	}
+	return Buffer.concat(chunks);
+}
+/**
+* Requests one URL, following its redirects by recursion.
+* @param url - The URL to request.
+* @param signal - Aborts the request.
+* @param redirects - How many redirects led here.
+* @returns The final response.
+*/
+async function follow(url, signal, redirects) {
+	if (url.protocol !== "http:" && url.protocol !== "https:") throw new KeepsetError(`${url.href} is not an http or https URL`);
+	const response = await get(url, signal);
+	const { statusCode: status, statusMessage } = response;
+	if (status === void 0 || statusMessage === void 0) throw new KeepsetError(`${url.href} answered without a status line`);
+	if (!REDIRECTS.has(status)) return new Response(await readBody(response), {
+		status,
+		statusText: statusMessage
+	});
+	response.resume();
+	const { location } = response.headers;
+	if (location === void 0) throw new KeepsetError(`${url.href} answered ${String(status)} without a location`);
+	if (redirects === MAX_REDIRECTS) throw new KeepsetError(`${url.href} redirected more than ${String(MAX_REDIRECTS)} times`);
+	return follow(new URL(location, url), signal, redirects + 1);
+}
+/**
+* Downloads a URL's bytes exactly as the server sends them, following redirects. Unlike the platform `fetch`, it never undoes a `Content-Encoding`: servers commonly label a `.gz` file `Content-Encoding: gzip`, and a recipe that unpacks gzip needs the file as published, which is also what its SHA-256 is recorded over. No `Accept-Encoding` is sent, so a server that compresses on the fly has no reason to.
+* @param url - An http or https URL.
+* @param init - The request options.
+* @returns A Response holding the raw bytes, with the final status.
+* @throws KeepsetError when a redirect has no location, leaves http and https, or passes the redirect limit.
+*/
+async function downloadRaw(url, init) {
+	return follow(new URL(url), init.signal, 0);
+}
+//#endregion
 //#region src/node/fetch-dataset.ts
 /**
 * The URL a recipe downloads a source from. A dead source is fetched from its pinned snapshot in the Wayback Machine's `id_` form, which serves the archived bytes as they were captured rather than wrapped in the archive's page chrome.
@@ -750,7 +840,7 @@ function metaOf(entry) {
 */
 async function fetchEntry(dataset, options = {}) {
 	const { entry, crosswalk } = dataset;
-	const { fetch: fetchPort = globalThis.fetch, signal, now = () => /* @__PURE__ */ new Date() } = options;
+	const { fetch: fetchPort = downloadRaw, signal, now = () => /* @__PURE__ */ new Date() } = options;
 	const { recipe } = entry;
 	if (recipe.status === "not-fetchable") throw new RecipeError(entry.id, "recipe", `the entry has no fetchable recipe: ${recipe.reason}`);
 	const source = entry.sources[recipe.source];
@@ -798,7 +888,7 @@ async function fetchEntry(dataset, options = {}) {
 	};
 }
 /**
-* Fetches a dataset of the bundled catalogue by id and resolves it to its category's target shape, so `fetchDataset("cora")` yields graph/v1 `{ meta, nodes, edges }` directly. Node-only: it downloads with the platform `fetch` and unpacks with Node's zlib.
+* Fetches a dataset of the bundled catalogue by id and resolves it to its category's target shape, so `fetchDataset("cora")` yields graph/v1 `{ meta, nodes, edges }` directly. Node-only: it downloads with Node's http and https clients and unpacks with Node's zlib.
 * @param id - The entry id.
 * @param options - The network port, abort signal and clock.
 * @returns The resolved target with its provenance and any count warnings.

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "keepset",
   "description": "Library, CLI and statically generated catalogue for dataset fetch-and-normalise recipes",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "dependencies": {
     "fflate": "0.8.3",
     "twindex": "0.1.1",
@@ -79,6 +79,8 @@
     "lint:check": "eslint . --cache --max-warnings 0",
     "prepare": "husky",
     "prepublishOnly": "pnpm run lint:check && pnpm run build && pnpm run typecheck && pnpm run test && publint && attw --pack --profile esm-only",
+    "seed:check": "tsx scripts/seed/import-phd-catalogue.ts --pipeline scripts/seed/sources/pipeline-catalog.json --graphbox scripts/seed/sources/graphbox-catalog.json --check",
+    "seed:import": "tsx scripts/seed/import-phd-catalogue.ts",
     "test": "vitest run",
     "typecheck": "tsc -p tsconfig.json && tsc -p tsconfig.node.json"
   },

package/schemas/entry.schema.json

@@ -1 +1 @@
-{"$defs":{"Crosswalk":{"$ref":"https://cdn.jsdelivr.net/npm/twindex@0.1.1/schemas/twindex.schema.json"},"Entry":{"additionalProperties":false,"properties":{"category":{"enum":["graph/v1"],"type":"string"},"citations":{"items":{"additionalProperties":false,"properties":{"doi":{"pattern":"^10\\.\\d{4,9}\\/\\S+$","type":"string"},"text":{"minLength":1,"type":"string"},"url":{"format":"uri","type":"string"}},"required":["text"],"type":"object"},"type":"array"},"crosswalk":{"anyOf":[{"$ref":"#/$defs/Crosswalk"},{"additionalProperties":false,"properties":{"file":{"type":"string"}},"required":["file"],"type":"object"}]},"description":{"minLength":1,"type":"string"},"domain":{"minLength":1,"type":"string"},"expectedCounts":{"additionalProperties":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"propertyNames":{"pattern":"^[A-Za-z][A-Za-z0-9]*$","type":"string"},"type":"object"},"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"licence":{"oneOf":[{"additionalProperties":false,"properties":{"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9.+-]*$","type":"string"},"kind":{"const":"spdx","type":"string"}},"required":["kind","id"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"custom","type":"string"},"name":{"minLength":1,"type":"string"},"url":{"format":"uri","type":"string"}},"required":["kind","name","url"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"unrecorded","type":"string"}},"required":["kind"],"type":"object"}]},"name":{"minLength":1,"type":"string"},"properties":{"additionalProperties":{"type":["string","number","boolean","null"]},"propertyNames":{"pattern":"^[A-Za-z][A-Za-z0-9]*$","type":"string"},"type":"object"},"recipe":{"oneOf":[{"additionalProperties":false,"properties":{"archive":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"none","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"gzip","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"zip","type":"string"},"member":{"anyOf":[{"additionalProperties":false,"properties":{"path":{"type":"string"}},"required":["path"],"type":"object"},{"additionalProperties":false,"properties":{"extension":{"pattern":"^\\.[A-Za-z0-9][A-Za-z0-9.]*$","type":"string"}},"required":["extension"],"type":"object"}]}},"required":["kind","member"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"tar","type":"string"},"member":{"anyOf":[{"additionalProperties":false,"properties":{"path":{"type":"string"}},"required":["path"],"type":"object"},{"additionalProperties":false,"properties":{"extension":{"pattern":"^\\.[A-Za-z0-9][A-Za-z0-9.]*$","type":"string"}},"required":["extension"],"type":"object"}]}},"required":["kind","member"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"tar.gz","type":"string"},"member":{"anyOf":[{"additionalProperties":false,"properties":{"path":{"type":"string"}},"required":["path"],"type":"object"},{"additionalProperties":false,"properties":{"extension":{"pattern":"^\\.[A-Za-z0-9][A-Za-z0-9.]*$","type":"string"}},"required":["extension"],"type":"object"}]}},"required":["kind","member"],"type":"object"}]},"parser":{"oneOf":[{"additionalProperties":false,"properties":{"commentPrefixes":{"items":{"minLength":1,"type":"string"},"type":"array"},"delimiter":{"enum":["detect","whitespace","tab","comma","space"],"type":"string"},"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"edge-list","type":"string"},"skipLines":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"sourceColumn":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"targetColumn":{"maximum":9007199254740991,"minimum":0,"type":"integer"}},"required":["format","encoding","delimiter","skipLines","commentPrefixes","sourceColumn","targetColumn"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"matrix-market","type":"string"}},"required":["format","encoding"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"gml","type":"string"}},"required":["format","encoding"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"pajek","type":"string"}},"required":["format","encoding"],"type":"object"}]},"source":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"status":{"const":"fetchable","type":"string"}},"required":["status","source","archive","parser"],"type":"object"},{"additionalProperties":false,"properties":{"reason":{"minLength":1,"type":"string"},"status":{"const":"not-fetchable","type":"string"}},"required":["status","reason"],"type":"object"}]},"sources":{"items":{"additionalProperties":false,"properties":{"retrieved":{"format":"date","pattern":"^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))$","type":"string"},"snapshot":{"additionalProperties":false,"properties":{"timestamp":{"pattern":"^\\d{8}(?:\\d{6})?$","type":"string"},"url":{"format":"uri","type":"string"}},"required":["url","timestamp"],"type":"object"},"status":{"enum":["live","dead"],"type":"string"},"url":{"format":"uri","type":"string"}},"required":["url","retrieved","status"],"type":"object"},"minItems":1,"type":"array"},"tags":{"items":{"minLength":1,"type":"string"},"type":"array"}},"required":["id","name","tags","category","sources","licence","citations","recipe","crosswalk","expectedCounts","properties"],"type":"object"}},"$id":"https://cdn.jsdelivr.net/npm/keepset@0.3.0/schemas/entry.schema.json","$ref":"#/$defs/Entry","$schema":"https://json-schema.org/draft/2020-12/schema"}
+{"$defs":{"Crosswalk":{"$ref":"https://cdn.jsdelivr.net/npm/twindex@0.1.1/schemas/twindex.schema.json"},"Entry":{"additionalProperties":false,"properties":{"category":{"enum":["graph/v1"],"type":"string"},"citations":{"items":{"additionalProperties":false,"properties":{"doi":{"pattern":"^10\\.\\d{4,9}\\/\\S+$","type":"string"},"text":{"minLength":1,"type":"string"},"url":{"format":"uri","type":"string"}},"required":["text"],"type":"object"},"type":"array"},"crosswalk":{"anyOf":[{"$ref":"#/$defs/Crosswalk"},{"additionalProperties":false,"properties":{"file":{"type":"string"}},"required":["file"],"type":"object"}]},"description":{"minLength":1,"type":"string"},"domain":{"minLength":1,"type":"string"},"expectedCounts":{"additionalProperties":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"propertyNames":{"pattern":"^[A-Za-z][A-Za-z0-9]*$","type":"string"},"type":"object"},"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"licence":{"oneOf":[{"additionalProperties":false,"properties":{"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9.+-]*$","type":"string"},"kind":{"const":"spdx","type":"string"}},"required":["kind","id"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"custom","type":"string"},"name":{"minLength":1,"type":"string"},"url":{"format":"uri","type":"string"}},"required":["kind","name","url"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"unrecorded","type":"string"}},"required":["kind"],"type":"object"}]},"name":{"minLength":1,"type":"string"},"properties":{"additionalProperties":{"type":["string","number","boolean","null"]},"propertyNames":{"pattern":"^[A-Za-z][A-Za-z0-9]*$","type":"string"},"type":"object"},"recipe":{"oneOf":[{"additionalProperties":false,"properties":{"archive":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"none","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"gzip","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"zip","type":"string"},"member":{"anyOf":[{"additionalProperties":false,"properties":{"path":{"type":"string"}},"required":["path"],"type":"object"},{"additionalProperties":false,"properties":{"extensions":{"items":{"pattern":"^\\.[A-Za-z0-9][A-Za-z0-9.]*$","type":"string"},"minItems":1,"type":"array"}},"required":["extensions"],"type":"object"}]}},"required":["kind","member"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"tar","type":"string"},"member":{"anyOf":[{"additionalProperties":false,"properties":{"path":{"type":"string"}},"required":["path"],"type":"object"},{"additionalProperties":false,"properties":{"extensions":{"items":{"pattern":"^\\.[A-Za-z0-9][A-Za-z0-9.]*$","type":"string"},"minItems":1,"type":"array"}},"required":["extensions"],"type":"object"}]}},"required":["kind","member"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"tar.gz","type":"string"},"member":{"anyOf":[{"additionalProperties":false,"properties":{"path":{"type":"string"}},"required":["path"],"type":"object"},{"additionalProperties":false,"properties":{"extensions":{"items":{"pattern":"^\\.[A-Za-z0-9][A-Za-z0-9.]*$","type":"string"},"minItems":1,"type":"array"}},"required":["extensions"],"type":"object"}]}},"required":["kind","member"],"type":"object"}]},"parser":{"oneOf":[{"additionalProperties":false,"properties":{"commentPrefixes":{"items":{"minLength":1,"type":"string"},"type":"array"},"delimiter":{"enum":["detect","whitespace","tab","comma","space"],"type":"string"},"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"edge-list","type":"string"},"skipLines":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"sourceColumn":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"targetColumn":{"maximum":9007199254740991,"minimum":0,"type":"integer"}},"required":["format","encoding","delimiter","skipLines","commentPrefixes","sourceColumn","targetColumn"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"matrix-market","type":"string"}},"required":["format","encoding"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"gml","type":"string"}},"required":["format","encoding"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"pajek","type":"string"}},"required":["format","encoding"],"type":"object"}]},"source":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"status":{"const":"fetchable","type":"string"}},"required":["status","source","archive","parser"],"type":"object"},{"additionalProperties":false,"properties":{"reason":{"minLength":1,"type":"string"},"status":{"const":"not-fetchable","type":"string"}},"required":["status","reason"],"type":"object"}]},"sources":{"items":{"additionalProperties":false,"properties":{"retrieved":{"format":"date","pattern":"^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))$","type":"string"},"snapshot":{"additionalProperties":false,"properties":{"timestamp":{"pattern":"^\\d{8}(?:\\d{6})?$","type":"string"},"url":{"format":"uri","type":"string"}},"required":["url","timestamp"],"type":"object"},"status":{"enum":["live","dead"],"type":"string"},"url":{"format":"uri","type":"string"}},"required":["url","retrieved","status"],"type":"object"},"minItems":1,"type":"array"},"tags":{"items":{"minLength":1,"type":"string"},"type":"array"}},"required":["id","name","tags","category","sources","licence","citations","recipe","crosswalk","expectedCounts","properties"],"type":"object"}},"$id":"https://cdn.jsdelivr.net/npm/keepset@0.4.0/schemas/entry.schema.json","$ref":"#/$defs/Entry","$schema":"https://json-schema.org/draft/2020-12/schema"}

package/schemas/manifest.schema.json

@@ -1 +1 @@
-{"$defs":{"Manifest":{"additionalProperties":false,"properties":{"description":{"minLength":1,"type":"string"},"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"lineage":{"additionalProperties":false,"properties":{"catalogue":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"repository":{"format":"uri","type":"string"},"version":{"pattern":"^(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)(?:-[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?(?:\\+[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?$","type":"string"}},"required":["catalogue","version","repository"],"type":"object"},"name":{"minLength":1,"type":"string"},"repository":{"format":"uri","type":"string"},"version":{"pattern":"^(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)(?:-[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?(?:\\+[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?$","type":"string"}},"required":["id","name","description","version","repository"],"type":"object"}},"$id":"https://cdn.jsdelivr.net/npm/keepset@0.3.0/schemas/manifest.schema.json","$ref":"#/$defs/Manifest","$schema":"https://json-schema.org/draft/2020-12/schema"}
+{"$defs":{"Manifest":{"additionalProperties":false,"properties":{"description":{"minLength":1,"type":"string"},"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"lineage":{"additionalProperties":false,"properties":{"catalogue":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"repository":{"format":"uri","type":"string"},"version":{"pattern":"^(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)(?:-[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?(?:\\+[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?$","type":"string"}},"required":["catalogue","version","repository"],"type":"object"},"name":{"minLength":1,"type":"string"},"repository":{"format":"uri","type":"string"},"version":{"pattern":"^(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)(?:-[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?(?:\\+[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?$","type":"string"}},"required":["id","name","description","version","repository"],"type":"object"}},"$id":"https://cdn.jsdelivr.net/npm/keepset@0.4.0/schemas/manifest.schema.json","$ref":"#/$defs/Manifest","$schema":"https://json-schema.org/draft/2020-12/schema"}