keepset 0.22.1 → 0.24.0

package/README.md

@@ -63,7 +63,7 @@ The catalogue itself is a manifest (`catalogue/manifest.json`: id, version, repo
 
 ## Categories and normalisation targets
 
-A category names a normalisation target: the shape every entry in that category resolves to, whatever its native shape. `graph/v1` is `{ meta, nodes, edges }` with unique node ids and edges whose endpoints name nodes; its entries are graph datasets taken from upstream collections such as SuiteSparse, Netzschleuder, SNAP, KONECT and OGB. `or/v1` is `{ meta, nodes }`, an operations-research instance as the nodes it offers to visit, each flat with optional coordinates, prize, prize probability, duration, travel time, skill, territory, lifecycle status and times, and deadline, read from tabular sources by the `csv-table` parser (a delimited file, a member of an archive, a sheet of an Excel workbook, or an open data portal read a page at a time) and from ArcGIS feature layers by the `geojson` parser. Its entries come from a survey of open operations-research datasets, imported with a curation that records, for each source, what its download gives and how its columns map onto the node; a dataset whose source has no usable table is imported without a recipe and says why. A new category starts by defining its target schema in `src/targets/` and registering it in the category registry; an entry cannot name a category that does not exist. The statically generated site renders every entry with schema.org Dataset JSON-LD, so the catalogue surfaces in the dataset discovery layers.
+A category names a normalisation target: the shape every entry in that category resolves to, whatever its native shape. `graph/v1` is `{ meta, nodes, edges }` with unique node ids and edges whose endpoints name nodes; its entries are graph datasets taken from upstream collections such as SuiteSparse, Netzschleuder, SNAP, KONECT and OGB. `or/v1` is `{ meta, nodes }`, an operations-research instance as the nodes it offers to visit, each flat with optional coordinates, prize, prize probability, duration, travel time, skill, territory, lifecycle status and times, and deadline, read from tabular sources by the `csv-table` parser (a delimited file, a member of an archive, a sheet of an Excel workbook, or an open data portal read a page at a time) and from ArcGIS feature layers by the `geojson` parser. Its entries come from a survey of open operations-research datasets, imported with a curation that records, for each source, what its download gives and how its columns map onto the node; a dataset whose source has no usable table is imported without a recipe and says why. `or-set/v1` is `{ meta, instances }`, a collection of operations-research instances such as a benchmark suite, each instance an `or/v1` document under an id of its own (for a collection read from an archive, the path of the member it came from): a recipe selects every matching archive member with `members`, the parser reads each one, and a member that does not parse fails the fetch and is named. A new category starts by defining its target schema in `src/targets/` and registering it in the category registry; an entry cannot name a category that does not exist. The statically generated site renders every entry with schema.org Dataset JSON-LD, so the catalogue surfaces in the dataset discovery layers.
 
 ## Licensing
 

package/dist/{bundled-CAsUNCDb.js → bundled-BWINQKLQ.js}

@@ -113,14 +113,16 @@ const JsonScalarSchema = z.union([
 * Reports each node whose id an earlier node already has, as an issue on that node's id.
 * @param nodes - The nodes of a target document, in order.
 * @param ctx - The refinement context to report to.
+* @param prefix - The path of the object holding the `nodes` array, for a document that nests nodes inside another.
 * @returns The set of ids the nodes carry.
 */
-function addDuplicateIdIssues(nodes, ctx) {
+function addDuplicateIdIssues(nodes, ctx, prefix = []) {
 	const ids = /* @__PURE__ */ new Set();
 	for (const [index, node] of nodes.entries()) {
 		if (ids.has(node.id)) ctx.addIssue({
 			code: "custom",
 			path: [
+				...prefix,
 				"nodes",
 				index,
 				"id"
@@ -202,22 +204,65 @@ const OrV1Schema = define(z.object({
 	addDuplicateIdIssues(instance.nodes, ctx);
 }));
 //#endregion
+//#region src/targets/or-set-v1.ts
+/** One instance of a set: an or/v1 instance, `{ meta, nodes }`, with the id that names it within the set (for a collection read from an archive, the member's path). */
+const OrInstanceSchema = z.object({
+	id: z.string().min(1),
+	meta: recordOf(z.string().min(1), JsonScalarSchema),
+	nodes: z.array(OrNodeSchema)
+}).strict();
+/**
+* The or-set/v1 normalisation target: `{ meta, instances }`, a collection of operations-research instances such as a benchmark suite, each instance an or/v1 document (`{ meta, nodes }`) under an id of its own. Beyond the shapes it checks that instance ids are unique in the set and node ids unique within each instance. A node id need not be unique across instances, because every instance numbers its own nodes.
+*/
+const OrSetV1Schema = define(z.object({
+	meta: recordOf(z.string().min(1), JsonScalarSchema),
+	instances: z.array(OrInstanceSchema)
+}).strict().superRefine((set, ctx) => {
+	const ids = /* @__PURE__ */ new Set();
+	for (const [index, instance] of set.instances.entries()) {
+		if (ids.has(instance.id)) ctx.addIssue({
+			code: "custom",
+			path: [
+				"instances",
+				index,
+				"id"
+			],
+			message: `duplicate instance id ${JSON.stringify(instance.id)}`
+		});
+		ids.add(instance.id);
+		addDuplicateIdIssues(instance.nodes, ctx, ["instances", index]);
+	}
+}));
+//#endregion
 //#region src/categories.ts
 /** Every category id the registry knows. A category id is `<kind>/v<version>`: the kind of data, and the version of its normalisation target, so a breaking change to a target is a new category rather than an edit to an old one. */
-const CategoryIdSchema = z.enum(["graph/v1", "or/v1"]);
+const CategoryIdSchema = z.enum([
+	"graph/v1",
+	"or/v1",
+	"or-set/v1"
+]);
 /** The registry. Adding a category is adding its id to `CategoryIdSchema` and its entry here, with its target schema beside graph/v1's in src/targets/: the mapped type makes the compiler reject a registry that misses an id or has one too many. */
 const registry = {
 	"graph/v1": {
 		id: "graph/v1",
 		description: "A graph as { meta, nodes, edges }: a flat record of scalar metadata, nodes carrying at least an id, and edges carrying at least source and target node ids, with optional node and edge types.",
+		shape: "document",
 		target: GraphV1Schema,
 		counted: ["nodes", "edges"]
 	},
 	"or/v1": {
 		id: "or/v1",
 		description: "An operations-research instance as { meta, nodes }: a flat record of scalar metadata and nodes carrying at least an id, with optional coordinates, prize, prize probability, duration, travel time, skill, territory, lifecycle status and times, and deadline.",
+		shape: "document",
 		target: OrV1Schema,
 		counted: ["nodes"]
+	},
+	"or-set/v1": {
+		id: "or-set/v1",
+		description: "A collection of operations-research instances as { meta, instances }: a flat record of scalar metadata and instances, each an or/v1 document ({ meta, nodes }) under an id of its own, read from every matching member of an archive.",
+		shape: "collection",
+		target: OrSetV1Schema,
+		counted: ["instances"]
 	}
 };
 function getCategory(id) {
@@ -410,22 +455,26 @@ const CitationSchema = z.object({
 const RelativePathSchema = z.string().refine((path) => path.split("/").every((segment) => segment !== "" && segment !== "." && segment !== ".."), { message: "must be a relative path of non-empty segments with no '.' or '..'" });
 /** Which file of an archive to read: a member path, matched against a member's whole path or its trailing segments (so `net.gml` finds `net/net.gml`), or the member whose name ends with one of a list of extensions (a format can have several, such as Pajek's `.net` and `.paj`). Either must match exactly one member when the recipe runs; macOS resource-fork members (`__MACOSX/` and `._` files) never match. */
 const ArchiveMemberSchema = z.union([z.object({ path: RelativePathSchema }).strict(), z.object({ extensions: z.array(z.string().regex(/^\.[A-Za-z0-9][A-Za-z0-9.]*$/, { message: "must be a file extension such as .mtx" })).min(1) }).strict()]);
-/** How the downloaded bytes unpack into the one file the parser reads. A workbook (`xlsx`) is read as the CSV of its named sheet, with dates as ISO 8601 text, for the `csv-table` parser. */
-const ArchiveSchema = z.discriminatedUnion("kind", [
+/** The archive kinds that hold several files, from which a recipe selects one (`member`) or every one a selector matches (`members`). */
+const FileArchiveKindSchema = z.enum([
+	"zip",
+	"tar",
+	"tar.gz"
+]);
+/** An archive a collection is read from: every file a selector matches, at least one. */
+const CollectionArchiveSchema = z.object({
+	kind: FileArchiveKindSchema,
+	members: ArchiveMemberSchema
+}).strict();
+/** How the downloaded bytes unpack into the file the parser reads, or for a collection the files it reads one by one. A `members` selector gives every member it matches, at least one, in path order. A workbook (`xlsx`) is read as the CSV of its named sheet, with dates as ISO 8601 text, for the `csv-table` parser. */
+const ArchiveSchema = z.union([
 	z.object({ kind: z.literal("none") }).strict(),
 	z.object({ kind: z.literal("gzip") }).strict(),
 	z.object({
-		kind: z.literal("zip"),
-		member: ArchiveMemberSchema
-	}).strict(),
-	z.object({
-		kind: z.literal("tar"),
-		member: ArchiveMemberSchema
-	}).strict(),
-	z.object({
-		kind: z.literal("tar.gz"),
+		kind: FileArchiveKindSchema,
 		member: ArchiveMemberSchema
 	}).strict(),
+	CollectionArchiveSchema,
 	z.object({
 		kind: z.literal("xlsx"),
 		sheet: z.string().min(1)
@@ -468,6 +517,10 @@ const ParserSchema = z.discriminatedUnion("format", [
 		format: z.literal("pajek"),
 		encoding: EncodingSchema
 	}).strict(),
+	z.object({
+		format: z.literal("solomon"),
+		encoding: EncodingSchema
+	}).strict(),
 	z.object({
 		format: z.literal("csv-table"),
 		encoding: EncodingSchema,
@@ -516,7 +569,7 @@ const ParserSchema = z.discriminatedUnion("format", [
 		path: ["dateColumns"]
 	})
 ]);
-/** Where a source is a paged query and not one file. An ArcGIS feature layer returns a limited number of features per request: the source URL is the layer's, the download reads every page in `orderBy` order and the parser reads the one GeoJSON collection they make. A Socrata resource endpoint (`/resource/<id>.csv`) is read `pageSize` rows at a time in the order of its row ids, and a CKAN datastore dump (`/datastore/dump/<id>`) `pageSize` rows at a time, each page a CSV with its own header; the download joins them into the one CSV the parser reads, so no single response has to survive a long transfer. */
+/** Where a source is a paged query and not one file. An ArcGIS feature layer returns a limited number of features per request: the source URL is the layer's, the download reads every page in `orderBy` order and the parser reads the one GeoJSON collection they make. A Socrata resource endpoint (`/resource/<id>.csv`) is read `pageSize` rows at a time in the order of its row ids, a CKAN datastore dump (`/datastore/dump/<id>`) and an OpenFEMA entity (`/api/open/v<n>/<Entity>`, with any `$select` or `$filter` in the source URL) `pageSize` rows at a time, each page a CSV with its own header; the download joins them into the one CSV the parser reads, so no single response has to survive a long transfer. */
 const PagingSchema = z.discriminatedUnion("kind", [
 	z.object({
 		kind: z.literal("arcgis-query"),
@@ -529,6 +582,10 @@ const PagingSchema = z.discriminatedUnion("kind", [
 	z.object({
 		kind: z.literal("ckan-datastore"),
 		pageSize: z.int().min(1)
+	}).strict(),
+	z.object({
+		kind: z.literal("openfema"),
+		pageSize: z.int().min(1)
 	}).strict()
 ]);
 /** How to obtain the dataset: a fetchable recipe naming the source (by index into `sources`), the archive layout and the parser, or a statement that no declarative recipe exists yet and why. */
@@ -583,6 +640,24 @@ const EntrySchema = define(z.object({
 		path: ["recipe", "source"],
 		message: `names source ${String(entry.recipe.source)} but the entry has ${String(entry.sources.length)}`
 	});
+	if (entry.recipe.status === "fetchable") {
+		const selectsMembers = "members" in entry.recipe.archive;
+		const { shape } = getCategory(entry.category);
+		if (shape === "collection" && !selectsMembers) ctx.addIssue({
+			code: "custom",
+			path: ["recipe", "archive"],
+			message: `${entry.category} is a collection, so the archive must select its members with \`members\``
+		});
+		if (shape === "document" && selectsMembers) ctx.addIssue({
+			code: "custom",
+			path: [
+				"recipe",
+				"archive",
+				"members"
+			],
+			message: `${entry.category} is one document, so the archive selects one file with \`member\``
+		});
+	}
 	if ("target" in entry.crosswalk && entry.crosswalk.target !== entry.category) ctx.addIssue({
 		code: "custom",
 		path: ["crosswalk", "target"],
@@ -905,7 +980,7 @@ var catalogue_default = {
 		"id": "keepset",
 		"name": "keepset",
 		"description": "The keepset catalogue: datasets with machine-readable recipes that fetch each one and normalise it into its category's target shape.",
-		"version": "1.10.0",
+		"version": "1.12.0",
 		"repository": "https://github.com/Mearman/keepset",
 		"namespaceAliases": {
 			"networkrepository": ["nr"],
@@ -6414,14 +6489,90 @@ var catalogue_default = {
 				"url": "https://www.fema.gov/openfema-data-page/fima-nfip-redacted-claims-v2",
 				"retrieved": "2026-10-19",
 				"status": "live"
+			}, {
+				"url": "https://www.fema.gov/api/open/v3/NfipClaims?$select=id,openDate,latitude,longitude,amountPaidOnBuildingClaim",
+				"retrieved": "2026-10-19",
+				"status": "live"
 			}],
-			"licence": { "kind": "unrecorded" },
+			"licence": {
+				"kind": "custom",
+				"name": "OpenFEMA Terms and Conditions",
+				"url": "https://www.fema.gov/about/openfema/terms-conditions"
+			},
 			"citations": [],
 			"recipe": {
-				"status": "not-fetchable",
-				"reason": "The OpenFEMA v3 API pages claims anonymously as CSV, but the table has millions of rows whose locations are rounded to a tenth of a degree, the terms of use limit it to statistical use without reidentification, and keepset has no recipe paging for this API."
+				"status": "fetchable",
+				"source": 1,
+				"paging": {
+					"kind": "openfema",
+					"pageSize": 5e4
+				},
+				"archive": { "kind": "none" },
+				"parser": {
+					"format": "csv-table",
+					"encoding": "utf-8",
+					"delimiter": "comma",
+					"skipLines": 0,
+					"idColumn": "id",
+					"numericColumns": [
+						{
+							"column": "latitude",
+							"factor": 1
+						},
+						{
+							"column": "longitude",
+							"factor": 1
+						},
+						{
+							"column": "amountPaidOnBuildingClaim",
+							"factor": 1
+						}
+					],
+					"columns": [
+						"latitude",
+						"longitude",
+						"openDate",
+						"amountPaidOnBuildingClaim"
+					]
+				}
+			},
+			"crosswalk": {
+				"source": "csv-table",
+				"target": "or/v1",
+				"rules": [
+					{
+						"op": "move",
+						"from": "meta",
+						"to": "meta"
+					},
+					{
+						"op": "move",
+						"from": "nodes[].id",
+						"to": "nodes[].id"
+					},
+					{
+						"op": "move",
+						"from": "nodes[].latitude",
+						"to": "nodes[].lat"
+					},
+					{
+						"op": "move",
+						"from": "nodes[].longitude",
+						"to": "nodes[].lon"
+					},
+					{
+						"op": "move",
+						"from": "nodes[].openDate",
+						"to": "nodes[].opened"
+					},
+					{
+						"op": "move",
+						"from": "nodes[].amountPaidOnBuildingClaim",
+						"to": "nodes[].prize"
+					}
+				],
+				"oneWay": []
 			},
-			"crosswalk": { "file": "crosswalks/or-v1/nodes.json" },
 			"expectedCounts": {},
 			"properties": { "surveyId": "or-122" }
 		},
@@ -7025,7 +7176,7 @@ var catalogue_default = {
 				"duration",
 				"travelTime"
 			],
-			"category": "or/v1",
+			"category": "or-set/v1",
 			"sources": [{
 				"url": "https://github.com/psxjpc/MOVRPTW-Generator",
 				"retrieved": "2026-10-19",
@@ -7041,12 +7192,95 @@ var catalogue_default = {
 			},
 			"citations": [],
 			"recipe": {
-				"status": "not-fetchable",
-				"reason": "The download is a collection of Solomon style instance files and distance matrices, and a collection is more than one or/v1 document, so no recipe reads it."
+				"status": "fetchable",
+				"source": 1,
+				"archive": {
+					"kind": "zip",
+					"members": { "extensions": [
+						".tw0Specs.dat",
+						".tw1Specs.dat",
+						".tw2Specs.dat",
+						".tw3Specs.dat",
+						".tw4Specs.dat"
+					] }
+				},
+				"parser": {
+					"format": "solomon",
+					"encoding": "utf-8"
+				}
 			},
-			"crosswalk": { "file": "crosswalks/or-v1/nodes.json" },
-			"expectedCounts": {},
-			"properties": { "surveyId": "or-9" }
+			"crosswalk": {
+				"source": "solomon",
+				"target": "or-set/v1",
+				"rules": [
+					{
+						"op": "move",
+						"from": "meta",
+						"to": "meta"
+					},
+					{
+						"op": "move",
+						"from": "members[].id",
+						"to": "instances[].id"
+					},
+					{
+						"op": "move",
+						"from": "members[].meta.name",
+						"to": "instances[].meta.name"
+					},
+					{
+						"op": "move",
+						"from": "members[].meta.vehicles",
+						"to": "instances[].meta.vehicles"
+					},
+					{
+						"op": "move",
+						"from": "members[].meta.capacity",
+						"to": "instances[].meta.capacity"
+					},
+					{
+						"op": "move",
+						"from": "members[].nodes[].id",
+						"to": "instances[].nodes[].id"
+					},
+					{
+						"op": "move",
+						"from": "members[].nodes[].due",
+						"to": "instances[].nodes[].deadline"
+					},
+					{
+						"op": "move",
+						"from": "members[].nodes[].service",
+						"to": "instances[].nodes[].duration"
+					},
+					{
+						"op": "move",
+						"from": "members[].nodes[].x",
+						"to": "instances[].nodes[].lat"
+					},
+					{
+						"op": "move",
+						"from": "members[].nodes[].y",
+						"to": "instances[].nodes[].lon"
+					},
+					{
+						"op": "move",
+						"from": "members[].nodes[].demand",
+						"to": "instances[].nodes[].demand"
+					},
+					{
+						"op": "move",
+						"from": "members[].nodes[].ready",
+						"to": "instances[].nodes[].ready"
+					}
+				],
+				"oneWay": []
+			},
+			"expectedCounts": { "instances": 45 },
+			"properties": {
+				"surveyId": "or-9",
+				"timeUnit": "seconds"
+			}
 		},
 		{
 			"id": "github-zhangchenhaoseu-jd-cold-chain-nodes",
@@ -8506,21 +8740,102 @@ var catalogue_default = {
 				"duration",
 				"travelTime"
 			],
-			"category": "or/v1",
+			"category": "or-set/v1",
 			"sources": [{
 				"url": "https://neo.lcc.uma.es/vrp/vrp-instances/capacitated-vrp-with-time-windows-instances/",
 				"retrieved": "2026-10-19",
 				"status": "live"
+			}, {
+				"url": "https://neo.lcc.uma.es/vrp/wp-content/data/instances/solomon/solomon_100.zip",
+				"retrieved": "2026-10-19",
+				"status": "live"
 			}],
 			"licence": { "kind": "unrecorded" },
 			"citations": [],
 			"recipe": {
-				"status": "not-fetchable",
-				"reason": "The download is a collection of Solomon instance files (planar coordinates, demands, service times and windows in abstract units, not seconds), and a collection is more than one or/v1 document, so no recipe reads it."
+				"status": "fetchable",
+				"source": 1,
+				"archive": {
+					"kind": "zip",
+					"members": { "extensions": [".txt"] }
+				},
+				"parser": {
+					"format": "solomon",
+					"encoding": "utf-8"
+				}
 			},
-			"crosswalk": { "file": "crosswalks/or-v1/nodes.json" },
-			"expectedCounts": {},
-			"properties": { "surveyId": "or-5" }
+			"crosswalk": {
+				"source": "solomon",
+				"target": "or-set/v1",
+				"rules": [
+					{
+						"op": "move",
+						"from": "meta",
+						"to": "meta"
+					},
+					{
+						"op": "move",
+						"from": "members[].id",
+						"to": "instances[].id"
+					},
+					{
+						"op": "move",
+						"from": "members[].meta.name",
+						"to": "instances[].meta.name"
+					},
+					{
+						"op": "move",
+						"from": "members[].meta.vehicles",
+						"to": "instances[].meta.vehicles"
+					},
+					{
+						"op": "move",
+						"from": "members[].meta.capacity",
+						"to": "instances[].meta.capacity"
+					},
+					{
+						"op": "move",
+						"from": "members[].nodes[].id",
+						"to": "instances[].nodes[].id"
+					},
+					{
+						"op": "move",
+						"from": "members[].nodes[].due",
+						"to": "instances[].nodes[].deadline"
+					},
+					{
+						"op": "move",
+						"from": "members[].nodes[].x",
+						"to": "instances[].nodes[].x"
+					},
+					{
+						"op": "move",
+						"from": "members[].nodes[].y",
+						"to": "instances[].nodes[].y"
+					},
+					{
+						"op": "move",
+						"from": "members[].nodes[].demand",
+						"to": "instances[].nodes[].demand"
+					},
+					{
+						"op": "move",
+						"from": "members[].nodes[].ready",
+						"to": "instances[].nodes[].ready"
+					},
+					{
+						"op": "move",
+						"from": "members[].nodes[].service",
+						"to": "instances[].nodes[].service"
+					}
+				],
+				"oneWay": []
+			},
+			"expectedCounts": { "instances": 56 },
+			"properties": {
+				"surveyId": "or-5",
+				"timeUnit": "abstract units"
+			}
 		},
 		{
 			"id": "leeds-fixed-penalty-notices",