keepset 0.1.0 → 0.3.0

package/dist/index.d.ts

@@ -1,100 +1,9 @@
-import { z } from "zod";
-//#region src/targets/graph-v1.d.ts
-/** A JSON scalar: the only values graph/v1 metadata may hold, so `meta` stays a flat record any consumer can tabulate. `z.number()` already rejects NaN and the infinities, which JSON cannot carry. */
-export declare const JsonScalarSchema: z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean, z.ZodNull]>;
-type JsonScalar = z.infer<typeof JsonScalarSchema>;
-/** A node: an id, an optional type, and whatever else the dataset carries. The target fixes only what every graph consumer needs, so extra fields pass through untouched. */
-export declare const GraphNodeSchema: z.ZodObject<{
-  id: z.ZodString;
-  type: z.ZodOptional<z.ZodString>;
-}, z.core.$loose>;
-type GraphNode = z.infer<typeof GraphNodeSchema>;
-/** An edge: source and target node ids, an optional type, and whatever else the dataset carries. Direction is the dataset's own: graph/v1 records it in `meta`, never by duplicating edges. */
-export declare const GraphEdgeSchema: z.ZodObject<{
-  source: z.ZodString;
-  target: z.ZodString;
-  type: z.ZodOptional<z.ZodString>;
-}, z.core.$loose>;
-type GraphEdge = z.infer<typeof GraphEdgeSchema>;
-/**
- * The graph/v1 normalisation target: `{ meta, nodes, edges }`, the shape the thesis pipeline's benchmark cache already uses. It describes the result `fetchDataset` resolves a graph entry to, not what a source file may contain. Beyond the field shapes it checks the two properties every consumer relies on: node ids are unique, and every edge endpoint names a node.
- */
-export declare const GraphV1Schema: z.ZodObject<{
-  meta: z.ZodRecord<z.ZodString, z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean, z.ZodNull]>>;
-  nodes: z.ZodArray<z.ZodObject<{
-    id: z.ZodString;
-    type: z.ZodOptional<z.ZodString>;
-  }, z.core.$loose>>;
-  edges: z.ZodArray<z.ZodObject<{
-    source: z.ZodString;
-    target: z.ZodString;
-    type: z.ZodOptional<z.ZodString>;
-  }, z.core.$loose>>;
-}, z.core.$strict> & {
-  is: (value: unknown) => value is {
-    meta: Record<string, string | number | boolean | null>;
-    nodes: {
-      [x: string]: unknown;
-      id: string;
-      type?: string | undefined;
-    }[];
-    edges: {
-      [x: string]: unknown;
-      source: string;
-      target: string;
-      type?: string | undefined;
-    }[];
-  };
-};
-type GraphV1 = z.infer<typeof GraphV1Schema>;
-//#endregion
-//#region src/categories.d.ts
-/** Every category id the registry knows. A category id is `<kind>/v<version>`: the kind of data, and the version of its normalisation target, so a breaking change to a target is a new category rather than an edit to an old one. */
-export declare const CategoryIdSchema: z.ZodEnum<{
-  "graph/v1": "graph/v1";
-}>;
-type CategoryId = z.infer<typeof CategoryIdSchema>;
-/** The normalisation target each category resolves to, keyed by category id. */
-interface CategoryTargets {
-  readonly "graph/v1": GraphV1;
-}
-/** The normalised shape of any category: what `fetchDataset` can resolve to. */
-type CategoryTarget = CategoryTargets[CategoryId];
-/** One category: its id, what it holds, and the schema of its normalisation target. */
-interface Category<Id extends CategoryId = CategoryId> {
-  readonly id: Id;
-  readonly description: string;
-  readonly target: z.ZodType<CategoryTargets[Id]>;
-}
-/**
- * Looks up a category by id.
- * @param id - Any string; category ids usually come from entry documents, so the argument is untrusted.
- * @returns The category.
- * @throws UnknownCategoryError when no category has that id.
- */
-export declare function getCategory<Id extends CategoryId>(id: Id): Category<Id>;
-export declare function getCategory(id: string): Category;
-/**
- * Returns the schema of a category's normalisation target.
- * @param id - Any string, as for `getCategory`.
- * @returns The target schema.
- * @throws UnknownCategoryError when no category has that id.
- */
-export declare function getTarget<Id extends CategoryId>(id: Id): z.ZodType<CategoryTargets[Id]>;
-export declare function getTarget(id: string): z.ZodType<CategoryTarget>;
-/**
- * The kind of data a category holds: the part of its id before the version.
- * @param id - A category id.
- * @returns The kind, such as `graph`.
- */
-export declare function kindOf(id: CategoryId): string;
-/** @returns Every registered category, in id order. */
-export declare function listCategories(): readonly Category[];
-//#endregion
+import { $ as SnapshotSchema, A as SemVerSchema, B as Entry, C as Dataset, D as LineageSchema, E as Lineage, F as ArchiveSchema, G as LicenceSchema, H as HttpUrlSchema, I as Citation, J as RESERVED_META_KEYS, K as Parser, L as CitationSchema, M as Archive, N as ArchiveMember, O as Manifest, P as ArchiveMemberSchema, Q as Snapshot, R as CrosswalkReference, S as CatalogueIndexSchema, T as createCatalogue, U as IdSchema, V as EntrySchema, W as Licence, X as RecipeSchema, Y as Recipe, Z as RelativePathSchema, _ as GraphV1Schema, a as CategoryTargets, b as Catalogue, c as getCategory, d as listCategories, et as Source, f as GraphEdge, g as GraphV1, h as GraphNodeSchema, i as CategoryTarget, it as waybackUrl, j as parseManifest, k as ManifestSchema, l as getTarget, m as GraphNode, n as CategoryId, nt as WaybackTimestampSchema, o as CountedKey, p as GraphEdgeSchema, q as ParserSchema, r as CategoryIdSchema, rt as parseEntry, s as countTarget, t as Category, tt as SourceSchema, u as kindOf, v as JsonScalar, w as DatasetQuery, x as CatalogueIndex, y as JsonScalarSchema, z as CrosswalkReferenceSchema } from "./categories-BIZJUKUw.js";
+import { Crosswalk } from "twindex";
 //#region src/errors.d.ts
 /** The root of this package's error taxonomy: every error keepset throws is a `KeepsetError`, so callers can catch the family with one check. */
 export declare class KeepsetError extends Error {
-  constructor(message: string);
+  constructor(message: string, options?: ErrorOptions);
 }
 /** Thrown when a category id names no category in the registry. Carries the ids that do exist, so the message says what would have been accepted. */
 export declare class UnknownCategoryError extends KeepsetError {
@@ -102,5 +11,62 @@ export declare class UnknownCategoryError extends KeepsetError {
   readonly knownIds: readonly string[];
   constructor(categoryId: string, knownIds: readonly string[]);
 }
+/** One reason a catalogue document failed its schema: the dotted path of the offending value and what is wrong with it. Zod's own issue objects are collapsed into these so the public error surface stays stable across Zod versions. */
+interface SyntaxIssue {
+  readonly path: string;
+  readonly message: string;
+}
+/** Thrown when an entry, a catalogue manifest or a catalogue index fails its schema. Carries every issue found, not just the first, so one failed parse reports the whole document. */
+export declare class CatalogueSyntaxError extends KeepsetError {
+  readonly document: "entry" | "manifest" | "catalogue";
+  readonly issues: readonly SyntaxIssue[];
+  constructor(document: "entry" | "manifest" | "catalogue", issues: readonly SyntaxIssue[]);
+}
+/** Thrown by a parser when a file does not have the format its recipe names. `line` is 1-based, counting every line of the file, so it points at the offending line in an editor. */
+export declare class NativeParseError extends KeepsetError {
+  readonly format: string;
+  readonly line: number;
+  constructor(format: string, line: number, message: string);
+}
+/** Thrown when an archive does not hold exactly one member the recipe's member selector matches, or is not a well-formed archive of its stated kind. */
+export declare class ArchiveError extends KeepsetError {}
+/** Thrown when no entry in the catalogue has the requested id. */
+export declare class DatasetNotFoundError extends KeepsetError {
+  readonly datasetId: string;
+  constructor(datasetId: string);
+}
+/** The step of a recipe that failed: the recipe itself (not fetchable), downloading the source, unpacking the archive, parsing the native shape, applying the crosswalk, or validating the result against the category target. */
+type RecipeStep = "recipe" | "download" | "extract" | "parse" | "crosswalk" | "validate";
+/** Thrown when running an entry's recipe fails. Names the entry and the failing step, and carries the underlying error as `cause`; a recipe never resolves to a partial or default-shaped result. */
+export declare class RecipeError extends KeepsetError {
+  readonly entryId: string;
+  readonly step: RecipeStep;
+  constructor(entryId: string, step: RecipeStep, message: string, options?: ErrorOptions);
+}
+//#endregion
+//#region src/bundled.d.ts
+/**
+ * Lists the datasets in the bundled catalogue that match a query. Browser-safe: a pure read over the index bundled with the package.
+ * @param query - The filters; all must match, and none given lists everything.
+ * @returns The matching datasets, in entry id order.
+ */
+export declare function listDatasets(query?: DatasetQuery): Dataset[];
+/**
+ * Returns one dataset of the bundled catalogue: its entry (metadata and recipe) and its resolved crosswalk. Browser-safe.
+ * @param id - The entry id.
+ * @returns The dataset.
+ * @throws DatasetNotFoundError when no entry has that id.
+ */
+export declare function getDataset(id: string): Dataset;
+/** @returns The bundled catalogue's manifest: its id, version and repository. */
+export declare function getManifest(): Manifest;
+//#endregion
+//#region src/serialise.d.ts
+/**
+ * Serialises a catalogue document (an entry, a manifest or a crosswalk file) as its file holds it: two-space indented JSON with keys in ascending order and a trailing newline. The form depends only on the document's content, so parsing a file written this way and serialising the result reproduces the file byte for byte.
+ * @param doc - A parsed entry, manifest or crosswalk.
+ * @returns The file text.
+ */
+export declare function serialise(doc: Entry | Manifest | Crosswalk): string;
 //#endregion
-export type { Category, CategoryId, CategoryTarget, CategoryTargets, GraphEdge, GraphNode, GraphV1, JsonScalar };
+export { type Archive, type ArchiveMember, ArchiveMemberSchema, ArchiveSchema, type Catalogue, type CatalogueIndex, CatalogueIndexSchema, type Category, type CategoryId, CategoryIdSchema, type CategoryTarget, type CategoryTargets, type Citation, CitationSchema, type CountedKey, type CrosswalkReference, CrosswalkReferenceSchema, type Dataset, type DatasetQuery, type Entry, EntrySchema, type GraphEdge, GraphEdgeSchema, type GraphNode, GraphNodeSchema, type GraphV1, GraphV1Schema, HttpUrlSchema, IdSchema, type JsonScalar, JsonScalarSchema, type Licence, LicenceSchema, type Lineage, LineageSchema, type Manifest, ManifestSchema, type Parser, ParserSchema, RESERVED_META_KEYS, type Recipe, RecipeSchema, type RecipeStep, RelativePathSchema, SemVerSchema, type Snapshot, SnapshotSchema, type Source, SourceSchema, type SyntaxIssue, WaybackTimestampSchema, countTarget, createCatalogue, getCategory, getTarget, kindOf, listCategories, parseEntry, parseManifest, waybackUrl };

package/dist/index.js

@@ -1,114 +1,22 @@
-import { z } from "zod";
-//#region src/errors.ts
-/** The root of this package's error taxonomy: every error keepset throws is a `KeepsetError`, so callers can catch the family with one check. */
-var KeepsetError = class extends Error {
-	constructor(message) {
-		super(message);
-		this.name = new.target.name;
-	}
-};
-/** Thrown when a category id names no category in the registry. Carries the ids that do exist, so the message says what would have been accepted. */
-var UnknownCategoryError = class extends KeepsetError {
-	categoryId;
-	knownIds;
-	constructor(categoryId, knownIds) {
-		super(`unknown category ${JSON.stringify(categoryId)}: known categories are ${knownIds.join(", ")}`);
-		this.categoryId = categoryId;
-		this.knownIds = knownIds;
-	}
-};
-//#endregion
-//#region src/define.ts
-/**
-* Attaches a type guard to a Zod schema (the schema-attached guard pattern: validation, inference and narrowing share one definition, so the three cannot drift). This module is isolated because it is the one place in the package that augments an object in place with Object.assign, which the house lint rule that eslint.config.ts turns off for this file alone would otherwise reject: object spread is no alternative, as it copies own properties only and drops the Zod prototype's parse/refine methods.
-* @param schema - The schema to augment.
-* @returns The same schema object, now also carrying `is`.
-*/
-function define(schema) {
-	return Object.assign(schema, { is(value) {
-		return schema.safeParse(value).success;
-	} });
-}
-//#endregion
-//#region src/targets/graph-v1.ts
-/** A JSON scalar: the only values graph/v1 metadata may hold, so `meta` stays a flat record any consumer can tabulate. `z.number()` already rejects NaN and the infinities, which JSON cannot carry. */
-const JsonScalarSchema = z.union([
-	z.string(),
-	z.number(),
-	z.boolean(),
-	z.null()
-]);
-/** A node: an id, an optional type, and whatever else the dataset carries. The target fixes only what every graph consumer needs, so extra fields pass through untouched. */
-const GraphNodeSchema = z.looseObject({
-	id: z.string().min(1),
-	type: z.string().min(1).optional()
-});
-/** An edge: source and target node ids, an optional type, and whatever else the dataset carries. Direction is the dataset's own: graph/v1 records it in `meta`, never by duplicating edges. */
-const GraphEdgeSchema = z.looseObject({
-	source: z.string().min(1),
-	target: z.string().min(1),
-	type: z.string().min(1).optional()
-});
+import { A as getCategory, B as DatasetNotFoundError, C as SnapshotSchema, D as waybackUrl, E as parseEntry, F as GraphNodeSchema, H as NativeParseError, I as GraphV1Schema, L as JsonScalarSchema, M as kindOf, N as listCategories, O as CategoryIdSchema, P as GraphEdgeSchema, R as ArchiveError, S as RelativePathSchema, T as WaybackTimestampSchema, U as RecipeError, V as KeepsetError, W as UnknownCategoryError, _ as IdSchema, a as createCatalogue, b as RESERVED_META_KEYS, c as ManifestSchema, d as ArchiveMemberSchema, f as ArchiveSchema, g as HttpUrlSchema, h as EntrySchema, i as CatalogueIndexSchema, j as getTarget, k as countTarget, l as SemVerSchema, m as CrosswalkReferenceSchema, n as getManifest, p as CitationSchema, r as listDatasets, s as LineageSchema, t as getDataset, u as parseManifest, v as LicenceSchema, w as SourceSchema, x as RecipeSchema, y as ParserSchema, z as CatalogueSyntaxError } from "./bundled--fOitQlP.js";
+//#region src/serialise.ts
 /**
-* The graph/v1 normalisation target: `{ meta, nodes, edges }`, the shape the thesis pipeline's benchmark cache already uses. It describes the result `fetchDataset` resolves a graph entry to, not what a source file may contain. Beyond the field shapes it checks the two properties every consumer relies on: node ids are unique, and every edge endpoint names a node.
+* Copies a JSON value with every object's keys in ascending code-unit order, the order the house JSON lint (`json/sort-keys`) requires of committed JSON. `Object.fromEntries` defines each key as an own data property, so a key such as `__proto__` stays a key.
+* @param value - A JSON value.
+* @returns The same value with sorted keys.
 */
-const GraphV1Schema = define(z.object({
-	meta: z.record(z.string(), JsonScalarSchema),
-	nodes: z.array(GraphNodeSchema),
-	edges: z.array(GraphEdgeSchema)
-}).strict().superRefine((graph, ctx) => {
-	const ids = /* @__PURE__ */ new Set();
-	for (const [index, node] of graph.nodes.entries()) {
-		if (ids.has(node.id)) ctx.addIssue({
-			code: "custom",
-			path: [
-				"nodes",
-				index,
-				"id"
-			],
-			message: `duplicate node id ${JSON.stringify(node.id)}`
-		});
-		ids.add(node.id);
-	}
-	for (const [index, edge] of graph.edges.entries()) for (const end of ["source", "target"]) if (!ids.has(edge[end])) ctx.addIssue({
-		code: "custom",
-		path: [
-			"edges",
-			index,
-			end
-		],
-		message: `edge ${end} ${JSON.stringify(edge[end])} names no node`
-	});
-}));
-//#endregion
-//#region src/categories.ts
-/** Every category id the registry knows. A category id is `<kind>/v<version>`: the kind of data, and the version of its normalisation target, so a breaking change to a target is a new category rather than an edit to an old one. */
-const CategoryIdSchema = z.enum(["graph/v1"]);
-/** The registry. Adding a category is adding its id to `CategoryIdSchema` and its entry here, with its target schema beside graph/v1's in src/targets/: the mapped type makes the compiler reject a registry that misses an id or has one too many. */
-const registry = { "graph/v1": {
-	id: "graph/v1",
-	description: "A graph as { meta, nodes, edges }: a flat record of scalar metadata, nodes carrying at least an id, and edges carrying at least source and target node ids, with optional node and edge types.",
-	target: GraphV1Schema
-} };
-function getCategory(id) {
-	const parsed = CategoryIdSchema.safeParse(id);
-	if (!parsed.success) throw new UnknownCategoryError(id, CategoryIdSchema.options);
-	return registry[parsed.data];
-}
-function getTarget(id) {
-	return getCategory(id).target;
+function sortKeys(value) {
+	if (Array.isArray(value)) return value.map(sortKeys);
+	if (typeof value === "object" && value !== null) return Object.fromEntries(Object.entries(value).sort(([left], [right]) => left < right ? -1 : 1).map(([key, item]) => [key, sortKeys(item)]));
+	return value;
 }
 /**
-* The kind of data a category holds: the part of its id before the version.
-* @param id - A category id.
-* @returns The kind, such as `graph`.
+* Serialises a catalogue document (an entry, a manifest or a crosswalk file) as its file holds it: two-space indented JSON with keys in ascending order and a trailing newline. The form depends only on the document's content, so parsing a file written this way and serialising the result reproduces the file byte for byte.
+* @param doc - A parsed entry, manifest or crosswalk.
+* @returns The file text.
 */
-function kindOf(id) {
-	return id.slice(0, id.indexOf("/"));
-}
-/** @returns Every registered category, in id order. */
-function listCategories() {
-	return CategoryIdSchema.options.map((id) => getCategory(id));
+function serialise(doc) {
+	return `${JSON.stringify(sortKeys(doc), void 0, 2)}\n`;
 }
 //#endregion
-export { CategoryIdSchema, GraphEdgeSchema, GraphNodeSchema, GraphV1Schema, JsonScalarSchema, KeepsetError, UnknownCategoryError, getCategory, getTarget, kindOf, listCategories };
+export { ArchiveError, ArchiveMemberSchema, ArchiveSchema, CatalogueIndexSchema, CatalogueSyntaxError, CategoryIdSchema, CitationSchema, CrosswalkReferenceSchema, DatasetNotFoundError, EntrySchema, GraphEdgeSchema, GraphNodeSchema, GraphV1Schema, HttpUrlSchema, IdSchema, JsonScalarSchema, KeepsetError, LicenceSchema, LineageSchema, ManifestSchema, NativeParseError, ParserSchema, RESERVED_META_KEYS, RecipeError, RecipeSchema, RelativePathSchema, SemVerSchema, SnapshotSchema, SourceSchema, UnknownCategoryError, WaybackTimestampSchema, countTarget, createCatalogue, getCategory, getDataset, getManifest, getTarget, kindOf, listCategories, listDatasets, parseEntry, parseManifest, serialise, waybackUrl };

package/dist/node/fetch-dataset.d.ts

@@ -0,0 +1,58 @@
+import { B as Entry, C as Dataset, Q as Snapshot, et as Source, i as CategoryTarget } from "../categories-BIZJUKUw.js";
+//#region src/node/fetch-dataset.d.ts
+/** The network port a recipe downloads through: the platform `fetch`, or a stand-in under test. */
+export type FetchPort = (url: string, init: {
+  readonly signal?: AbortSignal;
+}) => Promise<Response>;
+/** What a fetch may be given beyond the dataset: the network port, an abort signal and the clock that dates the provenance record. */
+export interface FetchOptions {
+  readonly fetch?: FetchPort;
+  readonly signal?: AbortSignal;
+  readonly now?: () => Date;
+}
+/** Where a fetched result's bytes came from: enough to say which entry, catalogue and source produced it, which URL was actually requested, when, and exactly which bytes arrived. */
+export interface Provenance {
+  readonly entryId: string;
+  readonly catalogue: Dataset["catalogue"];
+  readonly category: Entry["category"];
+  readonly source: {
+    readonly url: string;
+    readonly status: Source["status"];
+    readonly snapshot?: Snapshot;
+  };
+  readonly fetchedUrl: string;
+  readonly retrievedAt: string;
+  readonly sha256: string;
+  readonly byteLength: number;
+  readonly member?: string;
+}
+/** A fetched dataset: the resolved category target, the provenance of the bytes it came from, and any warnings, such as a count that differs from the one the entry records. */
+export interface FetchedDataset {
+  readonly dataset: Dataset;
+  readonly data: CategoryTarget;
+  readonly provenance: Provenance;
+  readonly warnings: readonly string[];
+}
+/**
+ * The URL a recipe downloads a source from. A dead source is fetched from its pinned snapshot in the Wayback Machine's `id_` form, which serves the archived bytes as they were captured rather than wrapped in the archive's page chrome.
+ * @param source - The source.
+ * @returns The URL to request.
+ */
+export declare function downloadUrl(source: Source): string;
+/**
+ * Runs a dataset's recipe: downloads the source (from its pinned snapshot when the source is dead), unpacks the archive, parses the native shape, applies the twindex crosswalk and validates the result against the category target. The native document the crosswalk receives is the parser's output with `meta` built from the entry.
+ * @param dataset - A dataset from any catalogue.
+ * @param options - The network port, abort signal and clock.
+ * @returns The resolved target with its provenance and any count warnings.
+ * @throws RecipeError naming the entry and the failing step; nothing partial or default-shaped is ever returned.
+ */
+export declare function fetchEntry(dataset: Dataset, options?: FetchOptions): Promise<FetchedDataset>;
+/**
+ * Fetches a dataset of the bundled catalogue by id and resolves it to its category's target shape, so `fetchDataset("cora")` yields graph/v1 `{ meta, nodes, edges }` directly. Node-only: it downloads with the platform `fetch` and unpacks with Node's zlib.
+ * @param id - The entry id.
+ * @param options - The network port, abort signal and clock.
+ * @returns The resolved target with its provenance and any count warnings.
+ * @throws DatasetNotFoundError when no entry has that id, and RecipeError naming the entry and the failing step when the recipe fails.
+ */
+export declare function fetchDataset(id: string, options?: FetchOptions): Promise<FetchedDataset>;
+//#endregion