keepset 0.41.1 → 0.43.0

package/dist/fetch-HjZj2roj.d.ts

@@ -0,0 +1,100 @@
+import { Et as Snapshot, Ot as Source, i as CategoryTarget, o as CollectionTarget, ut as Entry, z as Dataset } from "./categories-BG8lRjy8.js";
+//#region src/fetch.d.ts
+/** The network port a recipe downloads through: `downloadRaw` by default, which returns the bytes exactly as served, or a stand-in under test. */
+type FetchPort = (url: string, init: {
+  readonly signal?: AbortSignal;
+}) => Promise<Response>;
+/** The digest port a provenance record hashes its bytes through: SHA-256 as hex. Node's `crypto` by default, or SubtleCrypto where the platform has no `node:crypto` (the browser entry point's default). */
+type DigestPort = (bytes: Uint8Array) => Promise<string>;
+/** What a fetch takes beyond the dataset: the network port and the digest port every entry point supplies its own of, an abort signal and the clock that dates the provenance record. */
+interface FetchOptions {
+  readonly fetch: FetchPort;
+  readonly digest: DigestPort;
+  readonly signal?: AbortSignal;
+  readonly now?: () => Date;
+}
+/** One download a result's bytes came from: the entry's source, the URL actually requested (the snapshot's for a dead source) and exactly which bytes arrived. */
+interface Download {
+  readonly source: {
+    readonly url: string;
+    readonly status: Source["status"];
+    readonly snapshot?: Snapshot;
+  };
+  readonly fetchedUrl: string;
+  readonly sha256: string;
+  readonly byteLength: number;
+}
+/** What every provenance record says: which entry and catalogue produced the result, its category, and when it was fetched. */
+interface ProvenanceOf {
+  readonly entryId: string;
+  readonly catalogue: Dataset["catalogue"];
+  readonly category: Entry["category"];
+  readonly retrievedAt: string;
+}
+/** The provenance of a result read from one download: the download, and the archive member or members it was read from. */
+interface DownloadProvenance extends ProvenanceOf, Download {
+  /** The archive member a document was read from. */
+  readonly member?: string;
+  /** The archive members a collection, or a document read from several files, was read from, in the order they were read. */
+  readonly members?: readonly string[];
+}
+/** One file of a collection read file by file: its download, the file's path in the collection, and the instances it gave (the path itself, or the path joined to each archive member it unpacked into). */
+interface FileDownload extends Download {
+  readonly path: string;
+  readonly instances: readonly string[];
+}
+/** The provenance of a collection read from files downloaded one by one: each file's download, in the order they were read. */
+interface FilesProvenance extends ProvenanceOf {
+  readonly files: readonly FileDownload[];
+}
+/** Where a fetched result's bytes came from: enough to say which entry, catalogue and source produced it, which URL was actually requested, when, and exactly which bytes arrived, for the one download a recipe reads or for each file of a collection read file by file (told apart by `files`). */
+type Provenance = DownloadProvenance | FilesProvenance;
+/** Whether decoding a dataset's encoded native document gives the native document back, with the provenance of the bytes it was checked on. */
+type RoundTripReport = {
+  readonly holds: true;
+  readonly provenance: Provenance;
+} | {
+  readonly holds: false;
+  readonly path: string;
+  readonly message: string;
+  readonly provenance: Provenance;
+};
+/**
+ * The URL a recipe downloads a source from. A dead source is fetched from its pinned snapshot in the Wayback Machine's `id_` form, which serves the archived bytes as they were captured rather than wrapped in the archive's page chrome.
+ * @param source - The source.
+ * @returns The URL to request.
+ */
+declare function downloadUrl(source: Source): string;
+/** A fetched dataset: the resolved category target (one document, or for a recipe that selects archive members a collection of them), the provenance of the bytes it came from, and any warnings, such as a count that differs from the one the entry records. */
+interface FetchedDataset {
+  readonly dataset: Dataset;
+  readonly data: CategoryTarget | CollectionTarget;
+  readonly provenance: Provenance;
+  readonly warnings: readonly string[];
+}
+/**
+ * Runs a dataset's recipe: downloads the source (from its pinned snapshot when the source is dead), unpacks the archive, parses the native shape, applies the crossmap crosswalk and validates the result against the category target. The native document the crosswalk receives is the parser's output with `meta` built from the entry.
+ * @param dataset - A dataset from any catalogue.
+ * @param options - The network port, abort signal and clock.
+ * @returns The resolved target with its provenance and any count warnings.
+ * @throws RecipeError naming the entry and the failing step, including for a recipe that streams, which `streamEntry` reads; nothing partial or default-shaped is ever returned.
+ */
+declare function fetchEntry(dataset: Dataset, options: FetchOptions): Promise<FetchedDataset>;
+/**
+ * Fetches a dataset of the bundled catalogue by id and resolves it to its category's target shape, so `fetchDataset("cora")` yields graph/v1 `{ meta, nodes, edges }` directly. Node-only: it downloads with Node's http and https clients and unpacks with Node's zlib.
+ * @param id - The entry id.
+ * @param options - The network port, abort signal and clock.
+ * @returns The resolved target with its provenance and any count warnings.
+ * @throws DatasetNotFoundError when no entry has that id, and RecipeError naming the entry and the failing step when the recipe fails.
+ */
+declare function fetchDataset(id: string, options: FetchOptions): Promise<FetchedDataset>;
+/**
+ * Checks a dataset's crosswalk on its real data: runs the recipe to the native document, then crossmap's `assertRoundTrip`, which encodes it, decodes the result and compares the two over every path the crosswalk promises to preserve. A streamed recipe is checked a page at a time, with the round trip holding on the whole table exactly when it holds on every page, so the table is never held.
+ * @param dataset - A dataset from any catalogue.
+ * @param options - The network port, abort signal and clock.
+ * @returns Whether the round trip holds, and where it first differs when it does not.
+ * @throws RecipeError naming the entry and the failing step when the recipe fails before the comparison, or when the crosswalk cannot run.
+ */
+declare function roundTripEntry(dataset: Dataset, options: FetchOptions): Promise<RoundTripReport>;
+//#endregion
+export { FetchPort as a, Provenance as c, fetchDataset as d, fetchEntry as f, FetchOptions as i, RoundTripReport as l, Download as n, FetchedDataset as o, roundTripEntry as p, DownloadProvenance as r, FilesProvenance as s, DigestPort as t, downloadUrl as u };

package/dist/index.js

@@ -1,4 +1,4 @@
-import { $ as TaskSchema, A as SourceRecipeSchema, B as groupsOf, C as LicenceSchema, D as RecipeSchema, E as RESERVED_META_KEYS, G as countedKeys, I as readsCollection, J as getStreamedShape, K as getCategory, L as toIssues, M as SubjectSchema, N as WaybackTimestampSchema, O as RelativePathSchema, P as parseEntry, Q as listCategories, R as waybackUrl, S as IdSchema, T as ParserSchema, U as CategoryIdSchema, V as namesOf, W as countTarget, X as isStreamable, Y as getTarget, Z as kindOf, _ as ColumnNameSchema, a as CatalogueIndexSchema, at as GraphV1Schema, b as FilesRecipeSchema, c as ManifestSchema, ct as CatalogueSyntaxError, d as AliasSchema, dt as KeepsetError, et as TasksMetaSchema, f as ArchiveMemberSchema, ft as NativeParseError, g as CollectionFileSchema, h as CollectionArchiveSchema, i as listNamespaces, it as GraphNodeSchema, j as SourceSchema, k as SnapshotSchema, l as SemVerSchema, lt as DatasetAmbiguousError, m as CitationSchema, mt as UnknownCategoryError, n as getManifest, nt as collectionOf, o as createCatalogue, ot as JsonScalarSchema, p as ArchiveSchema, pt as RecipeError, q as getCollectionTarget, r as listDatasets, rt as GraphEdgeSchema, s as LineageSchema, st as ArchiveError, t as getDataset, tt as TasksV1Schema, u as parseManifest, ut as DatasetNotFoundError, v as CrosswalkReferenceSchema, w as NamespaceSchema, x as HttpUrlSchema, y as EntrySchema, z as NAME_SEPARATOR } from "./bundled-B1094k0B.js";
+import { $ as listCategories, A as SnapshotSchema, B as NAME_SEPARATOR, C as IdSchema, D as RESERVED_META_KEYS, E as ParserSchema, F as parseEntry, G as countTarget, H as namesOf, J as getCollectionTarget, K as countedKeys, L as readsCollection, M as SourceSchema, N as SubjectSchema, O as RecipeSchema, P as WaybackTimestampSchema, Q as kindOf, R as toIssues, S as HttpUrlSchema, T as NamespaceSchema, V as groupsOf, W as CategoryIdSchema, X as getTarget, Y as getStreamedShape, Z as isStreamable, _ as CollectionFileSchema, a as listNamespaces, at as GraphNodeSchema, b as EntrySchema, c as LineageSchema, ct as ArchiveError, d as parseManifest, dt as DatasetNotFoundError, et as TaskSchema, f as AliasSchema, ft as KeepsetError, g as CollectionArchiveSchema, h as CitationSchema, ht as UnknownCategoryError, i as listDatasets, it as GraphEdgeSchema, j as SourceRecipeSchema, k as RelativePathSchema, l as ManifestSchema, lt as CatalogueSyntaxError, m as ArchiveSchema, mt as RecipeError, n as getDataset, nt as TasksV1Schema, o as CatalogueIndexSchema, ot as GraphV1Schema, p as ArchiveMemberSchema, pt as NativeParseError, q as getCategory, r as getManifest, rt as collectionOf, s as createCatalogue, st as JsonScalarSchema, tt as TasksMetaSchema, u as SemVerSchema, ut as DatasetAmbiguousError, v as ColumnNameSchema, w as LicenceSchema, x as FilesRecipeSchema, y as CrosswalkReferenceSchema, z as waybackUrl } from "./bundled-OrF_TPDq.js";
 import { validateCrosswalk } from "crossmap";
 //#region src/serialise.ts
 /**

package/dist/node/fetch-dataset.d.ts

@@ -1,138 +1,92 @@
-import { Et as Snapshot, O as JsonScalar, Ot as Source, c as StreamableId, i as CategoryTarget, l as StreamedNodes, o as CollectionTarget, ut as Entry, z as Dataset } from "../categories-BG8lRjy8.js";
-//#region src/node/fetch-dataset.d.ts
-/** The network port a recipe downloads through: `downloadRaw` by default, which returns the bytes exactly as served, or a stand-in under test. */
-export type FetchPort = (url: string, init: {
-  readonly signal?: AbortSignal;
-}) => Promise<Response>;
-/** What a fetch may be given beyond the dataset: the network port, an abort signal and the clock that dates the provenance record. */
-export interface FetchOptions {
-  readonly fetch?: FetchPort;
-  readonly signal?: AbortSignal;
-  readonly now?: () => Date;
-}
-/** One download a result's bytes came from: the entry's source, the URL actually requested (the snapshot's for a dead source) and exactly which bytes arrived. */
-export interface Download {
-  readonly source: {
-    readonly url: string;
-    readonly status: Source["status"];
-    readonly snapshot?: Snapshot;
-  };
-  readonly fetchedUrl: string;
-  readonly sha256: string;
-  readonly byteLength: number;
-}
-/** What every provenance record says: which entry and catalogue produced the result, its category, and when it was fetched. */
-interface ProvenanceOf {
-  readonly entryId: string;
-  readonly catalogue: Dataset["catalogue"];
-  readonly category: Entry["category"];
-  readonly retrievedAt: string;
-}
-/** The provenance of a result read from one download: the download, and the archive member or members it was read from. */
-export interface DownloadProvenance extends ProvenanceOf, Download {
-  /** The archive member a document was read from. */
-  readonly member?: string;
-  /** The archive members a collection, or a document read from several files, was read from, in the order they were read. */
-  readonly members?: readonly string[];
-}
-/** One file of a collection read file by file: its download, the file's path in the collection, and the instances it gave (the path itself, or the path joined to each archive member it unpacked into). */
-export interface FileDownload extends Download {
-  readonly path: string;
-  readonly instances: readonly string[];
-}
-/** The provenance of a collection read from files downloaded one by one: each file's download, in the order they were read. */
-export interface FilesProvenance extends ProvenanceOf {
-  readonly files: readonly FileDownload[];
-}
-/** Where a fetched result's bytes came from: enough to say which entry, catalogue and source produced it, which URL was actually requested, when, and exactly which bytes arrived, for the one download a recipe reads or for each file of a collection read file by file (told apart by `files`). */
-export type Provenance = DownloadProvenance | FilesProvenance;
-/** Whether decoding a dataset's encoded native document gives the native document back, with the provenance of the bytes it was checked on. */
-export type RoundTripReport = {
-  readonly holds: true;
-  readonly provenance: Provenance;
-} | {
-  readonly holds: false;
-  readonly path: string;
-  readonly message: string;
-  readonly provenance: Provenance;
-};
-/** A node a streamed result hands on: a node of a streamable category's target. */
-export type StreamedNode = StreamedNodes[StreamableId];
+import { O as JsonScalar, c as StreamableId, l as StreamedNodes, z as Dataset } from "../categories-BG8lRjy8.js";
+import { a as FetchPort, c as Provenance, d as fetchDataset$1, f as fetchEntry$1, i as FetchOptions, l as RoundTripReport, n as Download, o as FetchedDataset, p as roundTripEntry$1, r as DownloadProvenance, s as FilesProvenance, t as DigestPort, u as downloadUrl } from "../fetch-HjZj2roj.js";
+import { Crosswalk } from "crossmap";
+//#region src/node/stream-dataset.d.ts
 /** Where a streamed result goes: the meta once, before any node, then the nodes a page at a time, each already checked against the category's node shape and for an id no earlier node had. A sink that returns a promise is awaited before the next page is read, so a slow sink holds the stream back rather than letting pages pile up. */
-export interface NodeSink {
+interface NodeSink {
   readonly meta: (meta: Readonly<Record<string, JsonScalar>>) => void | Promise<void>;
   readonly nodes: (nodes: readonly StreamedNode[]) => void | Promise<void>;
 }
 /** What a streamed fetch ends with, once every node has been handed on: the meta, the provenance of the bytes, how many nodes there were, and any warnings, such as a count that differs from the one the entry records. */
-export interface StreamedDataset {
+interface StreamedDataset {
   readonly dataset: Dataset;
   readonly meta: Readonly<Record<string, JsonScalar>>;
   readonly provenance: DownloadProvenance;
   readonly counts: ReadonlyMap<string, number>;
   readonly warnings: readonly string[];
 }
+/** A node a streamed result hands on: a node of a streamable category's target. */
+type StreamedNode = StreamedNodes[StreamableId];
 /**
- * The URL a recipe downloads a source from. A dead source is fetched from its pinned snapshot in the Wayback Machine's `id_` form, which serves the archived bytes as they were captured rather than wrapped in the archive's page chrome.
- * @param source - The source.
- * @returns The URL to request.
+ * Streams a table, as `streamEntry` describes.
+ * @param dataset - A dataset whose recipe streams.
+ * @param sink - Where the meta and the nodes go.
+ * @param options - The network port, abort signal and clock.
+ * @returns The meta, the provenance, the node count and any count warnings, once every node has been handed on.
+ * @throws RecipeError naming the entry and the failing step, after which the sink has had the nodes of the pages before the failure and no more; a recipe that does not stream, or a crosswalk that cannot run a page at a time, fails before anything is downloaded.
  */
-export declare function downloadUrl(source: Source): string;
-/** A fetched dataset: the resolved category target (one document, or for a recipe that selects archive members a collection of them), the provenance of the bytes it came from, and any warnings, such as a count that differs from the one the entry records. */
-export interface FetchedDataset {
-  readonly dataset: Dataset;
-  readonly data: CategoryTarget | CollectionTarget;
-  readonly provenance: Provenance;
-  readonly warnings: readonly string[];
+export declare function streamTable(dataset: Dataset, sink: NodeSink, options: FetchOptions): Promise<StreamedDataset>;
+//#endregion
+//#region src/node/download.d.ts
+/** How long a connection may send nothing, at any point of a download, before it is given up as dead. A healthy server of the open data portals recipes read sends something every few seconds even while it prepares a large page, and a socket that has gone silent otherwise holds the download for ever, as one did in the middle of a table of millions of rows. */
+export declare const STALL_MILLISECONDS = 120000;
+/** How many times a download is tried when its connection fails, and how long each retry waits for the one before it (the delay times the attempt number), so a network that drops for a few seconds is waited out and not hammered. */
+export declare const DOWNLOAD_ATTEMPTS = 3;
+export declare const RETRY_DELAY_MILLISECONDS = 3000;
+/** Options for one download: the signal that aborts it, how long a silent connection is waited on (`STALL_MILLISECONDS`), and how long a retry waits (`RETRY_DELAY_MILLISECONDS`), the last two only shortened by a test. */
+interface DownloadInit {
+  readonly signal?: AbortSignal;
+  readonly stallMilliseconds?: number;
+  readonly retryDelayMilliseconds?: number;
 }
 /**
- * Runs a dataset's recipe: downloads the source (from its pinned snapshot when the source is dead), unpacks the archive, parses the native shape, applies the crossmap crosswalk and validates the result against the category target. The native document the crosswalk receives is the parser's output with `meta` built from the entry.
- * @param dataset - A dataset from any catalogue.
- * @param options - The network port, abort signal and clock.
- * @returns The resolved target with its provenance and any count warnings.
- * @throws RecipeError naming the entry and the failing step, including for a recipe that streams, which `streamEntry` reads; nothing partial or default-shaped is ever returned.
+ * Downloads a URL's bytes exactly as the server sends them, following redirects. Unlike the platform `fetch`, it never undoes a `Content-Encoding`: servers commonly label a `.gz` file `Content-Encoding: gzip`, and a recipe that unpacks gzip needs the file as published, which is also what its SHA-256 is recorded over. No `Accept-Encoding` is sent, so a server that compresses on the fly has no reason to.
+ * @param url - An http or https URL.
+ * @param init - The request options.
+ * @returns A Response holding the raw bytes, with the final status.
+ * @throws KeepsetError when a redirect has no location, leaves http and https, or passes the redirect limit; the connection's failure, after the tries, when it keeps failing or sends nothing for longer than the stall time.
  */
-export declare function fetchEntry(dataset: Dataset, options?: FetchOptions): Promise<FetchedDataset>;
+export declare function downloadRaw(url: string, init: DownloadInit): Promise<Response>;
+//#endregion
+//#region src/node/fetch-dataset.d.ts
+/** What a fetch may be given beyond the dataset in Node: every port is optional, with Node's own as the default. */
+export type NodeFetchOptions = Omit<FetchOptions, "fetch" | "digest"> & {
+  readonly fetch?: FetchPort;
+  readonly digest?: FetchOptions["digest"];
+  readonly signal?: AbortSignal;
+  readonly now?: () => Date;
+};
 /**
- * Fetches a dataset of the bundled catalogue by id and resolves it to its category's target shape, so `fetchDataset("cora")` yields graph/v1 `{ meta, nodes, edges }` directly. Node-only: it downloads with Node's http and https clients and unpacks with Node's zlib.
+ * Fetches a dataset of the bundled catalogue by id, with Node's own downloader and digest. See `../fetch`'s `fetchEntry` for what a fetch does.
  * @param id - The entry id.
- * @param options - The network port, abort signal and clock.
- * @returns The resolved target with its provenance and any count warnings.
- * @throws DatasetNotFoundError when no entry has that id, and RecipeError naming the entry and the failing step when the recipe fails.
+ * @param options - The ports to override, an abort signal and the clock.
+ * @returns The fetched dataset.
  */
-export declare function fetchDataset(id: string, options?: FetchOptions): Promise<FetchedDataset>;
+export declare function fetchDataset(id: string, options?: NodeFetchOptions): Promise<Awaited<ReturnType<typeof fetchDataset$1>>>;
 /**
- * Checks a dataset's crosswalk on its real data: runs the recipe to the native document, then crossmap's `assertRoundTrip`, which encodes it, decodes the result and compares the two over every path the crosswalk promises to preserve. A streamed recipe is checked a page at a time, with the round trip holding on the whole table exactly when it holds on every page, so the table is never held.
+ * Fetches a dataset object, with Node's own downloader and digest.
  * @param dataset - A dataset from any catalogue.
- * @param options - The network port, abort signal and clock.
- * @returns Whether the round trip holds, and where it first differs when it does not.
- * @throws RecipeError naming the entry and the failing step when the recipe fails before the comparison, or when the crosswalk cannot run.
+ * @param options - The ports to override, an abort signal and the clock.
+ * @returns The fetched dataset.
  */
-export declare function roundTripEntry(dataset: Dataset, options?: FetchOptions): Promise<RoundTripReport>;
+export declare function fetchEntry(dataset: Dataset, options?: NodeFetchOptions): Promise<Awaited<ReturnType<typeof fetchEntry$1>>>;
 /**
- * Streams a dataset of the bundled catalogue by id, as `streamEntry` does: for a table too large to hold as one document, whose recipe streams.
- * @param id - The entry id.
- * @param sink - Where the meta and the nodes go.
- * @param options - The network port, abort signal and clock.
- * @returns The meta, the provenance, the node count and any count warnings, once every node has been handed on.
- * @throws DatasetNotFoundError when no entry has that id, and RecipeError as `streamEntry` does.
- */
-export declare function streamDataset(id: string, sink: NodeSink, options?: FetchOptions): Promise<StreamedDataset>;
-/**
- * Streams a dataset whose table is too large to hold as one document: reads its paged source a page at a time, applies the crosswalk to each page's rows under the document's meta, checks the meta once and each node against the category's node shape and for an id no earlier node had, and hands the nodes on to the sink before reading the next page. What it holds at once is one page's bytes and rows and the ids seen so far (as bytes, not strings), never the table, so the result is the category target `{ meta, nodes }` delivered in order rather than held: the same nodes `fetchEntry` would give, checked by the same rules.
- * @param dataset - A dataset whose recipe streams.
- * @param sink - Where the meta and the nodes go.
- * @param options - The network port, abort signal and clock.
- * @returns The meta, the provenance, the node count and any count warnings, once every node has been handed on.
- * @throws RecipeError naming the entry and the failing step, after which the sink has had the nodes of the pages before the failure and no more; a recipe that does not stream, or a crosswalk that cannot run a page at a time, fails before anything is downloaded.
+ * Checks a dataset's crosswalk on its real data, a streamed table a page at a time.
+ * @param dataset - A dataset from any catalogue.
+ * @param options - The ports to override, an abort signal and the clock.
+ * @returns Whether the round trip holds.
  */
-export declare function streamEntry(dataset: Dataset, sink: NodeSink, options?: FetchOptions): Promise<StreamedDataset>;
+export declare function roundTripEntry(dataset: Dataset, options?: NodeFetchOptions): Promise<Awaited<ReturnType<typeof roundTripEntry$1>>>;
 /**
  * Runs `roundTripEntry` on a dataset of the bundled catalogue by id.
  * @param id - The entry id.
- * @param options - The network port, abort signal and clock.
+ * @param options - The ports to override, an abort signal and the clock.
  * @returns Whether the round trip holds.
- * @throws DatasetNotFoundError when no entry has that id, and RecipeError as `roundTripEntry` does.
  */
-export declare function roundTripDataset(id: string, options?: FetchOptions): Promise<RoundTripReport>;
+export declare function roundTripDataset(id: string, options?: NodeFetchOptions): Promise<Awaited<ReturnType<typeof roundTripEntry$1>>>;
+/** Streams a dataset of the bundled catalogue by id, as `streamEntry` does. */
+export declare function streamDataset(id: string, sink: NodeSink, options?: NodeFetchOptions): Promise<StreamedDataset>;
+/** Streams a dataset whose table is too large to hold as one document, a page at a time, as `streamTable` does. */
+export declare function streamEntry(dataset: Dataset, sink: NodeSink, options?: NodeFetchOptions): Promise<StreamedDataset>;
 //#endregion
-export {};
+export { type DigestPort, type Download, type DownloadInit, type DownloadProvenance, type FetchOptions, type FetchPort, type FetchedDataset, type FilesProvenance, type NodeSink, type Provenance, type RoundTripReport, type StreamedDataset, type StreamedNode, downloadUrl };