keepset 0.39.0 → 0.41.0

package/dist/index.d.ts

@@ -1,4 +1,4 @@
-import { $ as CrosswalkReference, A as Catalogue, B as ManifestSchema, C as JsonScalarSchema, Ct as parseEntry, D as GraphNodeSchema, E as GraphNode, F as NamespaceNode, G as ArchiveMember, H as parseManifest, I as createCatalogue, J as Citation, K as ArchiveMemberSchema, L as Lineage, M as CatalogueIndexSchema, N as Dataset, O as GraphV1, P as DatasetQuery, Q as ColumnNameSchema, R as LineageSchema, S as JsonScalar, St as WaybackTimestampSchema, T as GraphEdgeSchema, U as AliasSchema, V as SemVerSchema, W as Archive, X as CollectionArchive, Y as CitationSchema, Z as CollectionArchiveSchema, _ as TasksV1, _t as Snapshot, a as CategoryTargets, at as IdSchema, b as Instance, bt as SourceSchema, c as countTarget, ct as Namespace, d as getCollectionTarget, dt as ParserSchema, et as CrosswalkReferenceSchema, f as getTarget, ft as PartsArchive, g as TaskSchema, gt as RelativePathSchema, h as Task, ht as RecipeSchema, i as CategoryTarget, it as HttpUrlSchema, j as CatalogueIndex, k as GraphV1Schema, l as countedKeys, lt as NamespaceSchema, m as listCategories, mt as Recipe, n as CategoryId, nt as Entry, o as CollectionTarget, ot as Licence, p as kindOf, pt as RESERVED_META_KEYS, q as ArchiveSchema, r as CategoryIdSchema, rt as EntrySchema, s as CountedKey, st as LicenceSchema, t as Category, tt as DocumentArchive, u as getCategory, ut as Parser, v as TasksV1Schema, vt as SnapshotSchema, w as GraphEdge, wt as waybackUrl, x as collectionOf, xt as SubjectSchema, y as Collection, yt as Source, z as Manifest } from "./categories-DijttH_M.js";
+import { $ as ArchiveSchema, A as GraphEdge, At as SourceRecipeSchema, B as DatasetQuery, C as TasksV1, Ct as Recipe, D as collectionOf, Dt as SnapshotSchema, E as Instance, Et as Snapshot, F as GraphV1Schema, Ft as readsCollection, G as Manifest, H as createCatalogue, I as Catalogue, It as waybackUrl, J as parseManifest, K as ManifestSchema, L as CatalogueIndex, M as GraphNode, Mt as SubjectSchema, N as GraphNodeSchema, Nt as WaybackTimestampSchema, O as JsonScalar, Ot as Source, P as GraphV1, Pt as parseEntry, Q as ArchiveMemberSchema, R as CatalogueIndexSchema, S as TasksMetaSchema, St as RESERVED_META_KEYS, T as Collection, Tt as RelativePathSchema, U as Lineage, V as NamespaceNode, W as LineageSchema, X as Archive, Y as AliasSchema, Z as ArchiveMember, _ as isStreamable, _t as LicenceSchema, a as CategoryTargets, at as CollectionFileSchema, b as Task, bt as Parser, c as StreamableId, ct as CrosswalkReferenceSchema, d as countTarget, dt as EntrySchema, et as Citation, f as countedKeys, ft as FilesRecipe, g as getTarget, gt as Licence, h as getStreamedShape, ht as IdSchema, i as CategoryTarget, it as CollectionFile, j as GraphEdgeSchema, jt as SourceSchema, k as JsonScalarSchema, kt as SourceRecipe, l as StreamedNodes, lt as DocumentArchive, m as getCollectionTarget, mt as HttpUrlSchema, n as CategoryId, nt as CollectionArchive, o as CollectionTarget, ot as ColumnNameSchema, p as getCategory, pt as FilesRecipeSchema, q as SemVerSchema, r as CategoryIdSchema, rt as CollectionArchiveSchema, s as CountedKey, st as CrosswalkReference, t as Category, tt as CitationSchema, u as StreamedShape, ut as Entry, v as kindOf, vt as Namespace, w as TasksV1Schema, wt as RecipeSchema, x as TaskSchema, xt as ParserSchema, y as listCategories, yt as NamespaceSchema, z as Dataset } from "./categories-BbW0rFCf.js";
 import { Crosswalk } from "crossmap";
 //#region src/errors.d.ts
 /** The root of this package's error taxonomy: every error keepset throws is a `KeepsetError`, so callers can catch the family with one check. */
@@ -117,4 +117,4 @@ export declare function serialise(doc: Entry | Manifest | Crosswalk): string;
  */
 export declare function validateDataset(dataset: Dataset): SyntaxIssue[];
 //#endregion
-export { AliasSchema, type Archive, type ArchiveMember, ArchiveMemberSchema, ArchiveSchema, type Catalogue, type CatalogueIndex, CatalogueIndexSchema, type Category, type CategoryId, CategoryIdSchema, type CategoryTarget, type CategoryTargets, type Citation, CitationSchema, type Collection, type CollectionArchive, CollectionArchiveSchema, type CollectionTarget, ColumnNameSchema, type CountedKey, type CrosswalkReference, CrosswalkReferenceSchema, type Dataset, type DatasetQuery, type DocumentArchive, type Entry, type EntryNames, EntrySchema, type GraphEdge, GraphEdgeSchema, type GraphNode, GraphNodeSchema, type GraphV1, GraphV1Schema, HttpUrlSchema, IdSchema, type Instance, type JsonScalar, JsonScalarSchema, type Licence, LicenceSchema, type Lineage, LineageSchema, type Manifest, ManifestSchema, type Namespace, type NamespaceNode, NamespaceSchema, type Parser, ParserSchema, type PartsArchive, RESERVED_META_KEYS, type Recipe, RecipeSchema, type RecipeStep, RelativePathSchema, type SegmentAliases, SemVerSchema, type Snapshot, SnapshotSchema, type Source, SourceSchema, SubjectSchema, type SyntaxIssue, type Task, TaskSchema, type TasksV1, TasksV1Schema, WaybackTimestampSchema, collectionOf, countTarget, countedKeys, createCatalogue, getCategory, getCollectionTarget, getTarget, kindOf, listCategories, parseEntry, parseManifest, waybackUrl };
+export { AliasSchema, type Archive, type ArchiveMember, ArchiveMemberSchema, ArchiveSchema, type Catalogue, type CatalogueIndex, CatalogueIndexSchema, type Category, type CategoryId, CategoryIdSchema, type CategoryTarget, type CategoryTargets, type Citation, CitationSchema, type Collection, type CollectionArchive, CollectionArchiveSchema, type CollectionFile, CollectionFileSchema, type CollectionTarget, ColumnNameSchema, type CountedKey, type CrosswalkReference, CrosswalkReferenceSchema, type Dataset, type DatasetQuery, type DocumentArchive, type Entry, type EntryNames, EntrySchema, type FilesRecipe, FilesRecipeSchema, type GraphEdge, GraphEdgeSchema, type GraphNode, GraphNodeSchema, type GraphV1, GraphV1Schema, HttpUrlSchema, IdSchema, type Instance, type JsonScalar, JsonScalarSchema, type Licence, LicenceSchema, type Lineage, LineageSchema, type Manifest, ManifestSchema, type Namespace, type NamespaceNode, NamespaceSchema, type Parser, ParserSchema, RESERVED_META_KEYS, type Recipe, RecipeSchema, type RecipeStep, RelativePathSchema, type SegmentAliases, SemVerSchema, type Snapshot, SnapshotSchema, type Source, type SourceRecipe, SourceRecipeSchema, SourceSchema, type StreamableId, type StreamedNodes, type StreamedShape, SubjectSchema, type SyntaxIssue, type Task, TaskSchema, TasksMetaSchema, type TasksV1, TasksV1Schema, WaybackTimestampSchema, collectionOf, countTarget, countedKeys, createCatalogue, getCategory, getCollectionTarget, getStreamedShape, getTarget, isStreamable, kindOf, listCategories, parseEntry, parseManifest, readsCollection, waybackUrl };

package/dist/index.js

@@ -1,4 +1,4 @@
-import { $ as ArchiveError, A as WaybackTimestampSchema, B as countedKeys, C as ParserSchema, D as SnapshotSchema, E as RelativePathSchema, F as groupsOf, G as listCategories, H as getCollectionTarget, I as namesOf, J as collectionOf, K as TaskSchema, M as toIssues, N as waybackUrl, O as SourceSchema, P as NAME_SEPARATOR, Q as JsonScalarSchema, R as CategoryIdSchema, S as NamespaceSchema, T as RecipeSchema, U as getTarget, V as getCategory, W as kindOf, X as GraphNodeSchema, Y as GraphEdgeSchema, Z as GraphV1Schema, _ as CrosswalkReferenceSchema, a as CatalogueIndexSchema, at as RecipeError, b as IdSchema, c as ManifestSchema, d as AliasSchema, et as CatalogueSyntaxError, f as ArchiveMemberSchema, g as ColumnNameSchema, h as CollectionArchiveSchema, i as listNamespaces, it as NativeParseError, j as parseEntry, k as SubjectSchema, l as SemVerSchema, m as CitationSchema, n as getManifest, nt as DatasetNotFoundError, o as createCatalogue, ot as UnknownCategoryError, p as ArchiveSchema, q as TasksV1Schema, r as listDatasets, rt as KeepsetError, s as LineageSchema, t as getDataset, tt as DatasetAmbiguousError, u as parseManifest, v as EntrySchema, w as RESERVED_META_KEYS, x as LicenceSchema, y as HttpUrlSchema, z as countTarget } from "./bundled-6dYCPVD-.js";
+import { $ as TaskSchema, A as SourceRecipeSchema, B as groupsOf, C as LicenceSchema, D as RecipeSchema, E as RESERVED_META_KEYS, G as countedKeys, I as readsCollection, J as getStreamedShape, K as getCategory, L as toIssues, M as SubjectSchema, N as WaybackTimestampSchema, O as RelativePathSchema, P as parseEntry, Q as listCategories, R as waybackUrl, S as IdSchema, T as ParserSchema, U as CategoryIdSchema, V as namesOf, W as countTarget, X as isStreamable, Y as getTarget, Z as kindOf, _ as ColumnNameSchema, a as CatalogueIndexSchema, at as GraphV1Schema, b as FilesRecipeSchema, c as ManifestSchema, ct as CatalogueSyntaxError, d as AliasSchema, dt as KeepsetError, et as TasksMetaSchema, f as ArchiveMemberSchema, ft as NativeParseError, g as CollectionFileSchema, h as CollectionArchiveSchema, i as listNamespaces, it as GraphNodeSchema, j as SourceSchema, k as SnapshotSchema, l as SemVerSchema, lt as DatasetAmbiguousError, m as CitationSchema, mt as UnknownCategoryError, n as getManifest, nt as collectionOf, o as createCatalogue, ot as JsonScalarSchema, p as ArchiveSchema, pt as RecipeError, q as getCollectionTarget, r as listDatasets, rt as GraphEdgeSchema, s as LineageSchema, st as ArchiveError, t as getDataset, tt as TasksV1Schema, u as parseManifest, ut as DatasetNotFoundError, v as CrosswalkReferenceSchema, w as NamespaceSchema, x as HttpUrlSchema, y as EntrySchema, z as NAME_SEPARATOR } from "./bundled-B1094k0B.js";
 import { validateCrosswalk } from "crossmap";
 //#region src/serialise.ts
 /**
@@ -57,4 +57,4 @@ function validateDataset(dataset) {
 	return issues;
 }
 //#endregion
-export { AliasSchema, ArchiveError, ArchiveMemberSchema, ArchiveSchema, CatalogueIndexSchema, CatalogueSyntaxError, CategoryIdSchema, CitationSchema, CollectionArchiveSchema, ColumnNameSchema, CrosswalkReferenceSchema, DatasetAmbiguousError, DatasetNotFoundError, EntrySchema, GraphEdgeSchema, GraphNodeSchema, GraphV1Schema, HttpUrlSchema, IdSchema, JsonScalarSchema, KeepsetError, LicenceSchema, LineageSchema, ManifestSchema, NAME_SEPARATOR, NamespaceSchema, NativeParseError, ParserSchema, RESERVED_META_KEYS, RecipeError, RecipeSchema, RelativePathSchema, SemVerSchema, SnapshotSchema, SourceSchema, SubjectSchema, TaskSchema, TasksV1Schema, UnknownCategoryError, WaybackTimestampSchema, collectionOf, countTarget, countedKeys, createCatalogue, getCategory, getCollectionTarget, getDataset, getManifest, getTarget, groupsOf, kindOf, listCategories, listDatasets, listNamespaces, namesOf, parseEntry, parseManifest, serialise, validateDataset, waybackUrl };
+export { AliasSchema, ArchiveError, ArchiveMemberSchema, ArchiveSchema, CatalogueIndexSchema, CatalogueSyntaxError, CategoryIdSchema, CitationSchema, CollectionArchiveSchema, CollectionFileSchema, ColumnNameSchema, CrosswalkReferenceSchema, DatasetAmbiguousError, DatasetNotFoundError, EntrySchema, FilesRecipeSchema, GraphEdgeSchema, GraphNodeSchema, GraphV1Schema, HttpUrlSchema, IdSchema, JsonScalarSchema, KeepsetError, LicenceSchema, LineageSchema, ManifestSchema, NAME_SEPARATOR, NamespaceSchema, NativeParseError, ParserSchema, RESERVED_META_KEYS, RecipeError, RecipeSchema, RelativePathSchema, SemVerSchema, SnapshotSchema, SourceRecipeSchema, SourceSchema, SubjectSchema, TaskSchema, TasksMetaSchema, TasksV1Schema, UnknownCategoryError, WaybackTimestampSchema, collectionOf, countTarget, countedKeys, createCatalogue, getCategory, getCollectionTarget, getDataset, getManifest, getStreamedShape, getTarget, groupsOf, isStreamable, kindOf, listCategories, listDatasets, listNamespaces, namesOf, parseEntry, parseManifest, readsCollection, serialise, validateDataset, waybackUrl };

package/dist/node/fetch-dataset.d.ts

@@ -1,4 +1,4 @@
-import { N as Dataset, _t as Snapshot, i as CategoryTarget, nt as Entry, o as CollectionTarget, yt as Source } from "../categories-DijttH_M.js";
+import { Et as Snapshot, O as JsonScalar, Ot as Source, c as StreamableId, i as CategoryTarget, l as StreamedNodes, o as CollectionTarget, ut as Entry, z as Dataset } from "../categories-BbW0rFCf.js";
 //#region src/node/fetch-dataset.d.ts
 /** The network port a recipe downloads through: `downloadRaw` by default, which returns the bytes exactly as served, or a stand-in under test. */
 export type FetchPort = (url: string, init: {
@@ -10,30 +10,65 @@ export interface FetchOptions {
   readonly signal?: AbortSignal;
   readonly now?: () => Date;
 }
-/** Where a fetched result's bytes came from: enough to say which entry, catalogue and source produced it, which URL was actually requested, when, and exactly which bytes arrived. */
-export interface Provenance {
-  readonly entryId: string;
-  readonly catalogue: Dataset["catalogue"];
-  readonly category: Entry["category"];
+/** One download a result's bytes came from: the entry's source, the URL actually requested (the snapshot's for a dead source) and exactly which bytes arrived. */
+export interface Download {
   readonly source: {
     readonly url: string;
     readonly status: Source["status"];
     readonly snapshot?: Snapshot;
   };
   readonly fetchedUrl: string;
-  readonly retrievedAt: string;
   readonly sha256: string;
   readonly byteLength: number;
+}
+/** What every provenance record says: which entry and catalogue produced the result, its category, and when it was fetched. */
+interface ProvenanceOf {
+  readonly entryId: string;
+  readonly catalogue: Dataset["catalogue"];
+  readonly category: Entry["category"];
+  readonly retrievedAt: string;
+}
+/** The provenance of a result read from one download: the download, and the archive member or members it was read from. */
+export interface DownloadProvenance extends ProvenanceOf, Download {
   /** The archive member a document was read from. */
   readonly member?: string;
   /** The archive members a collection, or a document read from several files, was read from, in the order they were read. */
   readonly members?: readonly string[];
 }
-/** A fetched dataset: the resolved category target (one document, or for a recipe that selects archive members a collection of them), the provenance of the bytes it came from, and any warnings, such as a count that differs from the one the entry records. */
-export interface FetchedDataset {
-  readonly dataset: Dataset;
-  readonly data: CategoryTarget | CollectionTarget;
+/** One file of a collection read file by file: its download, the file's path in the collection, and the instances it gave (the path itself, or the path joined to each archive member it unpacked into). */
+export interface FileDownload extends Download {
+  readonly path: string;
+  readonly instances: readonly string[];
+}
+/** The provenance of a collection read from files downloaded one by one: each file's download, in the order they were read. */
+export interface FilesProvenance extends ProvenanceOf {
+  readonly files: readonly FileDownload[];
+}
+/** Where a fetched result's bytes came from: enough to say which entry, catalogue and source produced it, which URL was actually requested, when, and exactly which bytes arrived, for the one download a recipe reads or for each file of a collection read file by file (told apart by `files`). */
+export type Provenance = DownloadProvenance | FilesProvenance;
+/** Whether decoding a dataset's encoded native document gives the native document back, with the provenance of the bytes it was checked on. */
+export type RoundTripReport = {
+  readonly holds: true;
   readonly provenance: Provenance;
+} | {
+  readonly holds: false;
+  readonly path: string;
+  readonly message: string;
+  readonly provenance: Provenance;
+};
+/** A node a streamed result hands on: a node of a streamable category's target. */
+export type StreamedNode = StreamedNodes[StreamableId];
+/** Where a streamed result goes: the meta once, before any node, then the nodes a page at a time, each already checked against the category's node shape and for an id no earlier node had. A sink that returns a promise is awaited before the next page is read, so a slow sink holds the stream back rather than letting pages pile up. */
+export interface NodeSink {
+  readonly meta: (meta: Readonly<Record<string, JsonScalar>>) => void | Promise<void>;
+  readonly nodes: (nodes: readonly StreamedNode[]) => void | Promise<void>;
+}
+/** What a streamed fetch ends with, once every node has been handed on: the meta, the provenance of the bytes, how many nodes there were, and any warnings, such as a count that differs from the one the entry records. */
+export interface StreamedDataset {
+  readonly dataset: Dataset;
+  readonly meta: Readonly<Record<string, JsonScalar>>;
+  readonly provenance: DownloadProvenance;
+  readonly counts: ReadonlyMap<string, number>;
   readonly warnings: readonly string[];
 }
 /**
@@ -42,12 +77,19 @@ export interface FetchedDataset {
  * @returns The URL to request.
  */
 export declare function downloadUrl(source: Source): string;
+/** A fetched dataset: the resolved category target (one document, or for a recipe that selects archive members a collection of them), the provenance of the bytes it came from, and any warnings, such as a count that differs from the one the entry records. */
+export interface FetchedDataset {
+  readonly dataset: Dataset;
+  readonly data: CategoryTarget | CollectionTarget;
+  readonly provenance: Provenance;
+  readonly warnings: readonly string[];
+}
 /**
  * Runs a dataset's recipe: downloads the source (from its pinned snapshot when the source is dead), unpacks the archive, parses the native shape, applies the crossmap crosswalk and validates the result against the category target. The native document the crosswalk receives is the parser's output with `meta` built from the entry.
  * @param dataset - A dataset from any catalogue.
  * @param options - The network port, abort signal and clock.
  * @returns The resolved target with its provenance and any count warnings.
- * @throws RecipeError naming the entry and the failing step; nothing partial or default-shaped is ever returned.
+ * @throws RecipeError naming the entry and the failing step, including for a recipe that streams, which `streamEntry` reads; nothing partial or default-shaped is ever returned.
  */
 export declare function fetchEntry(dataset: Dataset, options?: FetchOptions): Promise<FetchedDataset>;
 /**
@@ -58,24 +100,32 @@ export declare function fetchEntry(dataset: Dataset, options?: FetchOptions): Pr
  * @throws DatasetNotFoundError when no entry has that id, and RecipeError naming the entry and the failing step when the recipe fails.
  */
 export declare function fetchDataset(id: string, options?: FetchOptions): Promise<FetchedDataset>;
-/** Whether decoding a dataset's encoded native document gives the native document back, with the provenance of the bytes it was checked on. */
-export type RoundTripReport = {
-  readonly holds: true;
-  readonly provenance: Provenance;
-} | {
-  readonly holds: false;
-  readonly path: string;
-  readonly message: string;
-  readonly provenance: Provenance;
-};
 /**
- * Checks a dataset's crosswalk on its real data: runs the recipe to the native document, then crossmap's `assertRoundTrip`, which encodes it, decodes the result and compares the two over every path the crosswalk promises to preserve.
+ * Checks a dataset's crosswalk on its real data: runs the recipe to the native document, then crossmap's `assertRoundTrip`, which encodes it, decodes the result and compares the two over every path the crosswalk promises to preserve. A streamed recipe is checked a page at a time, with the round trip holding on the whole table exactly when it holds on every page, so the table is never held.
  * @param dataset - A dataset from any catalogue.
  * @param options - The network port, abort signal and clock.
  * @returns Whether the round trip holds, and where it first differs when it does not.
  * @throws RecipeError naming the entry and the failing step when the recipe fails before the comparison, or when the crosswalk cannot run.
  */
 export declare function roundTripEntry(dataset: Dataset, options?: FetchOptions): Promise<RoundTripReport>;
+/**
+ * Streams a dataset of the bundled catalogue by id, as `streamEntry` does: for a table too large to hold as one document, whose recipe streams.
+ * @param id - The entry id.
+ * @param sink - Where the meta and the nodes go.
+ * @param options - The network port, abort signal and clock.
+ * @returns The meta, the provenance, the node count and any count warnings, once every node has been handed on.
+ * @throws DatasetNotFoundError when no entry has that id, and RecipeError as `streamEntry` does.
+ */
+export declare function streamDataset(id: string, sink: NodeSink, options?: FetchOptions): Promise<StreamedDataset>;
+/**
+ * Streams a dataset whose table is too large to hold as one document: reads its paged source a page at a time, applies the crosswalk to each page's rows under the document's meta, checks the meta once and each node against the category's node shape and for an id no earlier node had, and hands the nodes on to the sink before reading the next page. What it holds at once is one page's bytes and rows and the ids seen so far (as bytes, not strings), never the table, so the result is the category target `{ meta, nodes }` delivered in order rather than held: the same nodes `fetchEntry` would give, checked by the same rules.
+ * @param dataset - A dataset whose recipe streams.
+ * @param sink - Where the meta and the nodes go.
+ * @param options - The network port, abort signal and clock.
+ * @returns The meta, the provenance, the node count and any count warnings, once every node has been handed on.
+ * @throws RecipeError naming the entry and the failing step, after which the sink has had the nodes of the pages before the failure and no more; a recipe that does not stream, or a crosswalk that cannot run a page at a time, fails before anything is downloaded.
+ */
+export declare function streamEntry(dataset: Dataset, sink: NodeSink, options?: FetchOptions): Promise<StreamedDataset>;
 /**
  * Runs `roundTripEntry` on a dataset of the bundled catalogue by id.
  * @param id - The entry id.
@@ -84,4 +134,5 @@ export declare function roundTripEntry(dataset: Dataset, options?: FetchOptions)
  * @throws DatasetNotFoundError when no entry has that id, and RecipeError as `roundTripEntry` does.
  */
 export declare function roundTripDataset(id: string, options?: FetchOptions): Promise<RoundTripReport>;
-//#endregion
+//#endregion
+export {};