keepset 0.3.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -57,11 +57,11 @@ declare const CitationSchema: z.ZodObject<{
57
57
  type Citation = z.infer<typeof CitationSchema>;
58
58
  /** A relative path inside an archive or a catalogue: `/`-separated segments, none empty, `.` or `..`, so it can never climb out of the directory it is resolved against. */
59
59
  declare const RelativePathSchema: z.ZodString;
60
- /** Which file of an archive to read: a member path, matched against a member's whole path or its trailing segments (so `net.gml` finds `net/net.gml`), or the member whose name ends with an extension. Either must match exactly one member when the recipe runs; macOS resource-fork members (`__MACOSX/` and `._` files) never match. */
60
+ /** Which file of an archive to read: a member path, matched against a member's whole path or its trailing segments (so `net.gml` finds `net/net.gml`), or the member whose name ends with one of a list of extensions (a format can have several, such as Pajek's `.net` and `.paj`). Either must match exactly one member when the recipe runs; macOS resource-fork members (`__MACOSX/` and `._` files) never match. */
61
61
  declare const ArchiveMemberSchema: z.ZodUnion<readonly [z.ZodObject<{
62
62
  path: z.ZodString;
63
63
  }, z.core.$strict>, z.ZodObject<{
64
- extension: z.ZodString;
64
+ extensions: z.ZodArray<z.ZodString>;
65
65
  }, z.core.$strict>]>;
66
66
  type ArchiveMember = z.infer<typeof ArchiveMemberSchema>;
67
67
  /** How the downloaded bytes unpack into the one file the parser reads. */
@@ -74,21 +74,21 @@ declare const ArchiveSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
74
74
  member: z.ZodUnion<readonly [z.ZodObject<{
75
75
  path: z.ZodString;
76
76
  }, z.core.$strict>, z.ZodObject<{
77
- extension: z.ZodString;
77
+ extensions: z.ZodArray<z.ZodString>;
78
78
  }, z.core.$strict>]>;
79
79
  }, z.core.$strict>, z.ZodObject<{
80
80
  kind: z.ZodLiteral<"tar">;
81
81
  member: z.ZodUnion<readonly [z.ZodObject<{
82
82
  path: z.ZodString;
83
83
  }, z.core.$strict>, z.ZodObject<{
84
- extension: z.ZodString;
84
+ extensions: z.ZodArray<z.ZodString>;
85
85
  }, z.core.$strict>]>;
86
86
  }, z.core.$strict>, z.ZodObject<{
87
87
  kind: z.ZodLiteral<"tar.gz">;
88
88
  member: z.ZodUnion<readonly [z.ZodObject<{
89
89
  path: z.ZodString;
90
90
  }, z.core.$strict>, z.ZodObject<{
91
- extension: z.ZodString;
91
+ extensions: z.ZodArray<z.ZodString>;
92
92
  }, z.core.$strict>]>;
93
93
  }, z.core.$strict>], "kind">;
94
94
  type Archive = z.infer<typeof ArchiveSchema>;
@@ -143,21 +143,21 @@ declare const RecipeSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
143
143
  member: z.ZodUnion<readonly [z.ZodObject<{
144
144
  path: z.ZodString;
145
145
  }, z.core.$strict>, z.ZodObject<{
146
- extension: z.ZodString;
146
+ extensions: z.ZodArray<z.ZodString>;
147
147
  }, z.core.$strict>]>;
148
148
  }, z.core.$strict>, z.ZodObject<{
149
149
  kind: z.ZodLiteral<"tar">;
150
150
  member: z.ZodUnion<readonly [z.ZodObject<{
151
151
  path: z.ZodString;
152
152
  }, z.core.$strict>, z.ZodObject<{
153
- extension: z.ZodString;
153
+ extensions: z.ZodArray<z.ZodString>;
154
154
  }, z.core.$strict>]>;
155
155
  }, z.core.$strict>, z.ZodObject<{
156
156
  kind: z.ZodLiteral<"tar.gz">;
157
157
  member: z.ZodUnion<readonly [z.ZodObject<{
158
158
  path: z.ZodString;
159
159
  }, z.core.$strict>, z.ZodObject<{
160
- extension: z.ZodString;
160
+ extensions: z.ZodArray<z.ZodString>;
161
161
  }, z.core.$strict>]>;
162
162
  }, z.core.$strict>], "kind">;
163
163
  parser: z.ZodDiscriminatedUnion<[z.ZodObject<{
@@ -261,21 +261,21 @@ declare const EntrySchema: z.ZodObject<{
261
261
  member: z.ZodUnion<readonly [z.ZodObject<{
262
262
  path: z.ZodString;
263
263
  }, z.core.$strict>, z.ZodObject<{
264
- extension: z.ZodString;
264
+ extensions: z.ZodArray<z.ZodString>;
265
265
  }, z.core.$strict>]>;
266
266
  }, z.core.$strict>, z.ZodObject<{
267
267
  kind: z.ZodLiteral<"tar">;
268
268
  member: z.ZodUnion<readonly [z.ZodObject<{
269
269
  path: z.ZodString;
270
270
  }, z.core.$strict>, z.ZodObject<{
271
- extension: z.ZodString;
271
+ extensions: z.ZodArray<z.ZodString>;
272
272
  }, z.core.$strict>]>;
273
273
  }, z.core.$strict>, z.ZodObject<{
274
274
  kind: z.ZodLiteral<"tar.gz">;
275
275
  member: z.ZodUnion<readonly [z.ZodObject<{
276
276
  path: z.ZodString;
277
277
  }, z.core.$strict>, z.ZodObject<{
278
- extension: z.ZodString;
278
+ extensions: z.ZodArray<z.ZodString>;
279
279
  }, z.core.$strict>]>;
280
280
  }, z.core.$strict>], "kind">;
281
281
  parser: z.ZodDiscriminatedUnion<[z.ZodObject<{
@@ -367,21 +367,21 @@ declare const EntrySchema: z.ZodObject<{
367
367
  member: {
368
368
  path: string;
369
369
  } | {
370
- extension: string;
370
+ extensions: string[];
371
371
  };
372
372
  } | {
373
373
  kind: "tar";
374
374
  member: {
375
375
  path: string;
376
376
  } | {
377
- extension: string;
377
+ extensions: string[];
378
378
  };
379
379
  } | {
380
380
  kind: "tar.gz";
381
381
  member: {
382
382
  path: string;
383
383
  } | {
384
- extension: string;
384
+ extensions: string[];
385
385
  };
386
386
  };
387
387
  parser: {
@@ -552,21 +552,21 @@ declare const CatalogueIndexSchema: z.ZodObject<{
552
552
  member: z.ZodUnion<readonly [z.ZodObject<{
553
553
  path: z.ZodString;
554
554
  }, z.core.$strict>, z.ZodObject<{
555
- extension: z.ZodString;
555
+ extensions: z.ZodArray<z.ZodString>;
556
556
  }, z.core.$strict>]>;
557
557
  }, z.core.$strict>, z.ZodObject<{
558
558
  kind: z.ZodLiteral<"tar">;
559
559
  member: z.ZodUnion<readonly [z.ZodObject<{
560
560
  path: z.ZodString;
561
561
  }, z.core.$strict>, z.ZodObject<{
562
- extension: z.ZodString;
562
+ extensions: z.ZodArray<z.ZodString>;
563
563
  }, z.core.$strict>]>;
564
564
  }, z.core.$strict>, z.ZodObject<{
565
565
  kind: z.ZodLiteral<"tar.gz">;
566
566
  member: z.ZodUnion<readonly [z.ZodObject<{
567
567
  path: z.ZodString;
568
568
  }, z.core.$strict>, z.ZodObject<{
569
- extension: z.ZodString;
569
+ extensions: z.ZodArray<z.ZodString>;
570
570
  }, z.core.$strict>]>;
571
571
  }, z.core.$strict>], "kind">;
572
572
  parser: z.ZodDiscriminatedUnion<[z.ZodObject<{
@@ -658,21 +658,21 @@ declare const CatalogueIndexSchema: z.ZodObject<{
658
658
  member: {
659
659
  path: string;
660
660
  } | {
661
- extension: string;
661
+ extensions: string[];
662
662
  };
663
663
  } | {
664
664
  kind: "tar";
665
665
  member: {
666
666
  path: string;
667
667
  } | {
668
- extension: string;
668
+ extensions: string[];
669
669
  };
670
670
  } | {
671
671
  kind: "tar.gz";
672
672
  member: {
673
673
  path: string;
674
674
  } | {
675
- extension: string;
675
+ extensions: string[];
676
676
  };
677
677
  };
678
678
  parser: {
@@ -762,21 +762,21 @@ declare const CatalogueIndexSchema: z.ZodObject<{
762
762
  member: {
763
763
  path: string;
764
764
  } | {
765
- extension: string;
765
+ extensions: string[];
766
766
  };
767
767
  } | {
768
768
  kind: "tar";
769
769
  member: {
770
770
  path: string;
771
771
  } | {
772
- extension: string;
772
+ extensions: string[];
773
773
  };
774
774
  } | {
775
775
  kind: "tar.gz";
776
776
  member: {
777
777
  path: string;
778
778
  } | {
779
- extension: string;
779
+ extensions: string[];
780
780
  };
781
781
  };
782
782
  parser: {
package/dist/index.d.ts CHANGED
@@ -1,4 +1,4 @@
1
- import { $ as SnapshotSchema, A as SemVerSchema, B as Entry, C as Dataset, D as LineageSchema, E as Lineage, F as ArchiveSchema, G as LicenceSchema, H as HttpUrlSchema, I as Citation, J as RESERVED_META_KEYS, K as Parser, L as CitationSchema, M as Archive, N as ArchiveMember, O as Manifest, P as ArchiveMemberSchema, Q as Snapshot, R as CrosswalkReference, S as CatalogueIndexSchema, T as createCatalogue, U as IdSchema, V as EntrySchema, W as Licence, X as RecipeSchema, Y as Recipe, Z as RelativePathSchema, _ as GraphV1Schema, a as CategoryTargets, b as Catalogue, c as getCategory, d as listCategories, et as Source, f as GraphEdge, g as GraphV1, h as GraphNodeSchema, i as CategoryTarget, it as waybackUrl, j as parseManifest, k as ManifestSchema, l as getTarget, m as GraphNode, n as CategoryId, nt as WaybackTimestampSchema, o as CountedKey, p as GraphEdgeSchema, q as ParserSchema, r as CategoryIdSchema, rt as parseEntry, s as countTarget, t as Category, tt as SourceSchema, u as kindOf, v as JsonScalar, w as DatasetQuery, x as CatalogueIndex, y as JsonScalarSchema, z as CrosswalkReferenceSchema } from "./categories-BIZJUKUw.js";
1
+ import { $ as SnapshotSchema, A as SemVerSchema, B as Entry, C as Dataset, D as LineageSchema, E as Lineage, F as ArchiveSchema, G as LicenceSchema, H as HttpUrlSchema, I as Citation, J as RESERVED_META_KEYS, K as Parser, L as CitationSchema, M as Archive, N as ArchiveMember, O as Manifest, P as ArchiveMemberSchema, Q as Snapshot, R as CrosswalkReference, S as CatalogueIndexSchema, T as createCatalogue, U as IdSchema, V as EntrySchema, W as Licence, X as RecipeSchema, Y as Recipe, Z as RelativePathSchema, _ as GraphV1Schema, a as CategoryTargets, b as Catalogue, c as getCategory, d as listCategories, et as Source, f as GraphEdge, g as GraphV1, h as GraphNodeSchema, i as CategoryTarget, it as waybackUrl, j as parseManifest, k as ManifestSchema, l as getTarget, m as GraphNode, n as CategoryId, nt as WaybackTimestampSchema, o as CountedKey, p as GraphEdgeSchema, q as ParserSchema, r as CategoryIdSchema, rt as parseEntry, s as countTarget, t as Category, tt as SourceSchema, u as kindOf, v as JsonScalar, w as DatasetQuery, x as CatalogueIndex, y as JsonScalarSchema, z as CrosswalkReferenceSchema } from "./categories-CUf1VecC.js";
2
2
  import { Crosswalk } from "twindex";
3
3
  //#region src/errors.d.ts
4
4
  /** The root of this package's error taxonomy: every error keepset throws is a `KeepsetError`, so callers can catch the family with one check. */
@@ -69,4 +69,12 @@ export declare function getManifest(): Manifest;
69
69
  */
70
70
  export declare function serialise(doc: Entry | Manifest | Crosswalk): string;
71
71
  //#endregion
72
+ //#region src/validate.d.ts
73
+ /**
74
+ * Checks a dataset as the catalogue holds it: the entry against the entry schema, and its crosswalk against twindex's structural validation and the entry's category. A catalogue index is checked the same way when it is built, so a bundled dataset is valid unless the package itself is broken; the check is what a contributor runs on an entry they are writing.
75
+ * @param dataset - The dataset.
76
+ * @returns Every issue found, with paths from the dataset's root (`entry.` or `crosswalk.`); empty when the dataset is valid.
77
+ */
78
+ export declare function validateDataset(dataset: Dataset): SyntaxIssue[];
79
+ //#endregion
72
80
  export { type Archive, type ArchiveMember, ArchiveMemberSchema, ArchiveSchema, type Catalogue, type CatalogueIndex, CatalogueIndexSchema, type Category, type CategoryId, CategoryIdSchema, type CategoryTarget, type CategoryTargets, type Citation, CitationSchema, type CountedKey, type CrosswalkReference, CrosswalkReferenceSchema, type Dataset, type DatasetQuery, type Entry, EntrySchema, type GraphEdge, GraphEdgeSchema, type GraphNode, GraphNodeSchema, type GraphV1, GraphV1Schema, HttpUrlSchema, IdSchema, type JsonScalar, JsonScalarSchema, type Licence, LicenceSchema, type Lineage, LineageSchema, type Manifest, ManifestSchema, type Parser, ParserSchema, RESERVED_META_KEYS, type Recipe, RecipeSchema, type RecipeStep, RelativePathSchema, SemVerSchema, type Snapshot, SnapshotSchema, type Source, SourceSchema, type SyntaxIssue, WaybackTimestampSchema, countTarget, createCatalogue, getCategory, getTarget, kindOf, listCategories, parseEntry, parseManifest, waybackUrl };
package/dist/index.js CHANGED
@@ -1,4 +1,5 @@
1
- import { A as getCategory, B as DatasetNotFoundError, C as SnapshotSchema, D as waybackUrl, E as parseEntry, F as GraphNodeSchema, H as NativeParseError, I as GraphV1Schema, L as JsonScalarSchema, M as kindOf, N as listCategories, O as CategoryIdSchema, P as GraphEdgeSchema, R as ArchiveError, S as RelativePathSchema, T as WaybackTimestampSchema, U as RecipeError, V as KeepsetError, W as UnknownCategoryError, _ as IdSchema, a as createCatalogue, b as RESERVED_META_KEYS, c as ManifestSchema, d as ArchiveMemberSchema, f as ArchiveSchema, g as HttpUrlSchema, h as EntrySchema, i as CatalogueIndexSchema, j as getTarget, k as countTarget, l as SemVerSchema, m as CrosswalkReferenceSchema, n as getManifest, p as CitationSchema, r as listDatasets, s as LineageSchema, t as getDataset, u as parseManifest, v as LicenceSchema, w as SourceSchema, x as RecipeSchema, y as ParserSchema, z as CatalogueSyntaxError } from "./bundled--fOitQlP.js";
1
+ import { A as countTarget, B as CatalogueSyntaxError, C as SnapshotSchema, D as toIssues, E as parseEntry, F as GraphEdgeSchema, G as UnknownCategoryError, H as KeepsetError, I as GraphNodeSchema, L as GraphV1Schema, M as getTarget, N as kindOf, O as waybackUrl, P as listCategories, R as JsonScalarSchema, S as RelativePathSchema, T as WaybackTimestampSchema, U as NativeParseError, V as DatasetNotFoundError, W as RecipeError, _ as IdSchema, a as createCatalogue, b as RESERVED_META_KEYS, c as ManifestSchema, d as ArchiveMemberSchema, f as ArchiveSchema, g as HttpUrlSchema, h as EntrySchema, i as CatalogueIndexSchema, j as getCategory, k as CategoryIdSchema, l as SemVerSchema, m as CrosswalkReferenceSchema, n as getManifest, p as CitationSchema, r as listDatasets, s as LineageSchema, t as getDataset, u as parseManifest, v as LicenceSchema, w as SourceSchema, x as RecipeSchema, y as ParserSchema, z as ArchiveError } from "./bundled-P_132GDn.js";
2
+ import { validateCrosswalk } from "twindex";
2
3
  //#region src/serialise.ts
3
4
  /**
4
5
  * Copies a JSON value with every object's keys in ascending code-unit order, the order the house JSON lint (`json/sort-keys`) requires of committed JSON. `Object.fromEntries` defines each key as an own data property, so a key such as `__proto__` stays a key.
@@ -11,12 +12,49 @@ function sortKeys(value) {
11
12
  return value;
12
13
  }
13
14
  /**
15
+ * Serialises any JSON value in the committed form: two-space indented, keys in ascending order, with a trailing newline.
16
+ * @param value - A JSON value.
17
+ * @returns The file text.
18
+ */
19
+ function canonicalJson(value) {
20
+ return `${JSON.stringify(sortKeys(value), void 0, 2)}\n`;
21
+ }
22
+ /**
14
23
  * Serialises a catalogue document (an entry, a manifest or a crosswalk file) as its file holds it: two-space indented JSON with keys in ascending order and a trailing newline. The form depends only on the document's content, so parsing a file written this way and serialising the result reproduces the file byte for byte.
15
24
  * @param doc - A parsed entry, manifest or crosswalk.
16
25
  * @returns The file text.
17
26
  */
18
27
  function serialise(doc) {
19
- return `${JSON.stringify(sortKeys(doc), void 0, 2)}\n`;
28
+ return canonicalJson(doc);
29
+ }
30
+ //#endregion
31
+ //#region src/validate.ts
32
+ /**
33
+ * Checks a dataset as the catalogue holds it: the entry against the entry schema, and its crosswalk against twindex's structural validation and the entry's category. A catalogue index is checked the same way when it is built, so a bundled dataset is valid unless the package itself is broken; the check is what a contributor runs on an entry they are writing.
34
+ * @param dataset - The dataset.
35
+ * @returns Every issue found, with paths from the dataset's root (`entry.` or `crosswalk.`); empty when the dataset is valid.
36
+ */
37
+ function validateDataset(dataset) {
38
+ const issues = [];
39
+ const entry = EntrySchema.safeParse(dataset.entry);
40
+ if (!entry.success) issues.push(...toIssues(entry.error).map((issue) => ({
41
+ path: `entry.${issue.path}`,
42
+ message: issue.message
43
+ })));
44
+ const { rules, oneWay } = dataset.crosswalk;
45
+ for (const issue of validateCrosswalk(dataset.crosswalk)) {
46
+ const { ruleIndex } = issue;
47
+ const path = ruleIndex >= 0 && ruleIndex < rules.length ? `crosswalk.rules.${String(ruleIndex)}` : ruleIndex >= rules.length && ruleIndex < rules.length + oneWay.length ? `crosswalk.oneWay.${String(ruleIndex - rules.length)}` : "crosswalk";
48
+ issues.push({
49
+ path,
50
+ message: issue.message
51
+ });
52
+ }
53
+ if (dataset.crosswalk.target !== dataset.entry.category) issues.push({
54
+ path: "crosswalk.target",
55
+ message: `must be the entry's category, ${dataset.entry.category}`
56
+ });
57
+ return issues;
20
58
  }
21
59
  //#endregion
22
- export { ArchiveError, ArchiveMemberSchema, ArchiveSchema, CatalogueIndexSchema, CatalogueSyntaxError, CategoryIdSchema, CitationSchema, CrosswalkReferenceSchema, DatasetNotFoundError, EntrySchema, GraphEdgeSchema, GraphNodeSchema, GraphV1Schema, HttpUrlSchema, IdSchema, JsonScalarSchema, KeepsetError, LicenceSchema, LineageSchema, ManifestSchema, NativeParseError, ParserSchema, RESERVED_META_KEYS, RecipeError, RecipeSchema, RelativePathSchema, SemVerSchema, SnapshotSchema, SourceSchema, UnknownCategoryError, WaybackTimestampSchema, countTarget, createCatalogue, getCategory, getDataset, getManifest, getTarget, kindOf, listCategories, listDatasets, parseEntry, parseManifest, serialise, waybackUrl };
60
+ export { ArchiveError, ArchiveMemberSchema, ArchiveSchema, CatalogueIndexSchema, CatalogueSyntaxError, CategoryIdSchema, CitationSchema, CrosswalkReferenceSchema, DatasetNotFoundError, EntrySchema, GraphEdgeSchema, GraphNodeSchema, GraphV1Schema, HttpUrlSchema, IdSchema, JsonScalarSchema, KeepsetError, LicenceSchema, LineageSchema, ManifestSchema, NativeParseError, ParserSchema, RESERVED_META_KEYS, RecipeError, RecipeSchema, RelativePathSchema, SemVerSchema, SnapshotSchema, SourceSchema, UnknownCategoryError, WaybackTimestampSchema, countTarget, createCatalogue, getCategory, getDataset, getManifest, getTarget, kindOf, listCategories, listDatasets, parseEntry, parseManifest, serialise, validateDataset, waybackUrl };
@@ -1,6 +1,6 @@
1
- import { B as Entry, C as Dataset, Q as Snapshot, et as Source, i as CategoryTarget } from "../categories-BIZJUKUw.js";
1
+ import { B as Entry, C as Dataset, Q as Snapshot, et as Source, i as CategoryTarget } from "../categories-CUf1VecC.js";
2
2
  //#region src/node/fetch-dataset.d.ts
3
- /** The network port a recipe downloads through: the platform `fetch`, or a stand-in under test. */
3
+ /** The network port a recipe downloads through: `downloadRaw` by default, which returns the bytes exactly as served, or a stand-in under test. */
4
4
  export type FetchPort = (url: string, init: {
5
5
  readonly signal?: AbortSignal;
6
6
  }) => Promise<Response>;
@@ -48,11 +48,37 @@ export declare function downloadUrl(source: Source): string;
48
48
  */
49
49
  export declare function fetchEntry(dataset: Dataset, options?: FetchOptions): Promise<FetchedDataset>;
50
50
  /**
51
- * Fetches a dataset of the bundled catalogue by id and resolves it to its category's target shape, so `fetchDataset("cora")` yields graph/v1 `{ meta, nodes, edges }` directly. Node-only: it downloads with the platform `fetch` and unpacks with Node's zlib.
51
+ * Fetches a dataset of the bundled catalogue by id and resolves it to its category's target shape, so `fetchDataset("cora")` yields graph/v1 `{ meta, nodes, edges }` directly. Node-only: it downloads with Node's http and https clients and unpacks with Node's zlib.
52
52
  * @param id - The entry id.
53
53
  * @param options - The network port, abort signal and clock.
54
54
  * @returns The resolved target with its provenance and any count warnings.
55
55
  * @throws DatasetNotFoundError when no entry has that id, and RecipeError naming the entry and the failing step when the recipe fails.
56
56
  */
57
57
  export declare function fetchDataset(id: string, options?: FetchOptions): Promise<FetchedDataset>;
58
+ /** Whether decoding a dataset's encoded native document gives the native document back, with the provenance of the bytes it was checked on. */
59
+ export type RoundTripReport = {
60
+ readonly holds: true;
61
+ readonly provenance: Provenance;
62
+ } | {
63
+ readonly holds: false;
64
+ readonly path: string;
65
+ readonly message: string;
66
+ readonly provenance: Provenance;
67
+ };
68
+ /**
69
+ * Checks a dataset's crosswalk on its real data: runs the recipe to the native document, then twindex's `assertRoundTrip`, which encodes it, decodes the result and compares the two over every path the crosswalk promises to preserve.
70
+ * @param dataset - A dataset from any catalogue.
71
+ * @param options - The network port, abort signal and clock.
72
+ * @returns Whether the round trip holds, and where it first differs when it does not.
73
+ * @throws RecipeError naming the entry and the failing step when the recipe fails before the comparison, or when the crosswalk cannot run.
74
+ */
75
+ export declare function roundTripEntry(dataset: Dataset, options?: FetchOptions): Promise<RoundTripReport>;
76
+ /**
77
+ * Runs `roundTripEntry` on a dataset of the bundled catalogue by id.
78
+ * @param id - The entry id.
79
+ * @param options - The network port, abort signal and clock.
80
+ * @returns Whether the round trip holds.
81
+ * @throws DatasetNotFoundError when no entry has that id, and RecipeError as `roundTripEntry` does.
82
+ */
83
+ export declare function roundTripDataset(id: string, options?: FetchOptions): Promise<RoundTripReport>;
58
84
  //#endregion
@@ -1,8 +1,10 @@
1
- import { H as NativeParseError, R as ArchiveError, U as RecipeError, V as KeepsetError, j as getTarget, k as countTarget, o as getOwn, t as getDataset } from "../bundled--fOitQlP.js";
2
- import { encode } from "twindex";
1
+ import { A as countTarget, H as KeepsetError, M as getTarget, U as NativeParseError, W as RecipeError, o as getOwn, t as getDataset, z as ArchiveError } from "../bundled-P_132GDn.js";
2
+ import { RoundTripMismatchError, assertRoundTrip, encode } from "twindex";
3
3
  import { createHash } from "node:crypto";
4
4
  import { gunzipSync } from "node:zlib";
5
5
  import { unzipSync } from "fflate";
6
+ import { request } from "node:http";
7
+ import { request as request$1 } from "node:https";
6
8
  //#region src/parsers/native.ts
7
9
  /**
8
10
  * Collects node ids in order of first appearance, so a parser that learns its nodes from its edges emits them deterministically.
@@ -346,8 +348,9 @@ function parseMatrixMarket(text) {
346
348
  }
347
349
  //#endregion
348
350
  //#region src/parsers/pajek.ts
349
- /** Section keywords, lower-cased, mapped to how their lines are read. `*Partition` and `*Vector` hold node attributes this version does not carry, so their lines are skipped, as are the `*Vertices` lines inside them. */
351
+ /** Section keywords, lower-cased, mapped to how their lines are read. `*Network` names the network and starts no section. `*Partition` and `*Vector` hold node attributes this version does not carry, so their lines are skipped, as are the `*Vertices` lines inside them. */
350
352
  const sections = /* @__PURE__ */ new Map([
353
+ ["*network", "network"],
351
354
  ["*vertices", "vertices"],
352
355
  ["*arcs", "pairs"],
353
356
  ["*edges", "pairs"],
@@ -358,6 +361,18 @@ const sections = /* @__PURE__ */ new Map([
358
361
  ["*vector", "attributes"]
359
362
  ]);
360
363
  /**
364
+ * Resolves a section keyword, which Pajek reads case-insensitively and accepts abbreviated (`*net` for `*Network`, `*vert` for `*Vertices`): an exact keyword first, otherwise the shortest keyword it abbreviates, so `*edge` is `*Edges` rather than `*Edgeslist`.
365
+ * @param keyword - The keyword as written.
366
+ * @returns How its section is read, or undefined for a keyword that abbreviates none.
367
+ */
368
+ function sectionOf(keyword) {
369
+ const name = keyword.toLowerCase();
370
+ const exact = sections.get(name);
371
+ if (exact !== void 0) return exact;
372
+ const [shortest] = [...sections.keys()].filter((candidate) => candidate.startsWith(name)).sort((left, right) => left.length - right.length);
373
+ return shortest === void 0 ? void 0 : sections.get(shortest);
374
+ }
375
+ /**
361
376
  * Reads a vertex number, which Pajek numbers from 1 to the declared vertex count.
362
377
  * @param token - The token.
363
378
  * @param count - The declared vertex count.
@@ -388,14 +403,13 @@ function parsePajek(text) {
388
403
  const tokens = content.split(/\s+/);
389
404
  const [keyword] = tokens;
390
405
  if (keyword?.startsWith("*") === true) {
391
- const name = keyword.toLowerCase();
392
- if (name === "*network") {
406
+ const next = sectionOf(keyword);
407
+ if (next === void 0) throw new NativeParseError("pajek", lineNumber, `unknown section ${keyword}`);
408
+ if (next === "network") {
393
409
  networks++;
394
410
  if (networks > 1) throw new NativeParseError("pajek", lineNumber, "a project file holding more than one network is not supported");
395
411
  continue;
396
412
  }
397
- const next = sections.get(name);
398
- if (next === void 0) throw new NativeParseError("pajek", lineNumber, `unknown section ${keyword}`);
399
413
  if (next === "vertices" && section !== "attributes") {
400
414
  if (count !== void 0) throw new NativeParseError("pajek", lineNumber, "a network declares *Vertices twice");
401
415
  const declared = tokens[1];
@@ -616,7 +630,7 @@ function isResourceFork(path) {
616
630
  return path.startsWith("__MACOSX/") || path.split("/").some((segment) => segment.startsWith("._"));
617
631
  }
618
632
  /**
619
- * Whether a member path matches a selector: a path selector matches the whole member path or its trailing segments, an extension selector the end of the member's name.
633
+ * Whether a member path matches a selector: a path selector matches the whole member path or its trailing segments, an extensions selector the end of the member's name, ignoring case.
620
634
  * @param path - The member path.
621
635
  * @param member - The recipe's selector.
622
636
  * @returns True on a match.
@@ -624,7 +638,7 @@ function isResourceFork(path) {
624
638
  function selects(path, member) {
625
639
  if (isResourceFork(path) || path.endsWith("/")) return false;
626
640
  if ("path" in member) return path === member.path || path.endsWith(`/${member.path}`);
627
- return path.toLowerCase().endsWith(member.extension.toLowerCase());
641
+ return member.extensions.some((extension) => path.toLowerCase().endsWith(extension.toLowerCase()));
628
642
  }
629
643
  /**
630
644
  * Picks the one member a selector matches.
@@ -637,7 +651,7 @@ function pick(paths, member) {
637
651
  const matched = paths.filter((path) => selects(path, member));
638
652
  const [only] = matched;
639
653
  if (matched.length !== 1 || only === void 0) {
640
- const wanted = "path" in member ? `member ${member.path}` : `one member ending ${member.extension}`;
654
+ const wanted = "path" in member ? `member ${member.path}` : `one member ending ${member.extensions.join(" or ")}`;
641
655
  throw new ArchiveError(`expected ${wanted}, found ${String(matched.length)} among: ${paths.join(", ")}`);
642
656
  }
643
657
  return only;
@@ -701,6 +715,82 @@ function unzip(bytes, filter) {
701
715
  }
702
716
  }
703
717
  //#endregion
718
+ //#region src/node/download.ts
719
+ /** The most redirects one download follows: the Fetch Standard's own limit (https://fetch.spec.whatwg.org/#http-redirect-fetch, step 5). */
720
+ const MAX_REDIRECTS = 20;
721
+ const REDIRECTS = new Set(Object.values({
722
+ movedPermanently: 301,
723
+ found: 302,
724
+ seeOther: 303,
725
+ temporaryRedirect: 307,
726
+ permanentRedirect: 308
727
+ }));
728
+ /** What the downloader says it is, so a server operator can tell where the traffic comes from. */
729
+ const USER_AGENT = "keepset (+https://github.com/ExaDev/keepset)";
730
+ /**
731
+ * Sends one GET and resolves with the response head, its body still unread.
732
+ * @param url - The URL.
733
+ * @param signal - Aborts the request.
734
+ * @returns The response.
735
+ */
736
+ async function get(url, signal) {
737
+ const send = url.protocol === "https:" ? request$1 : request;
738
+ return new Promise((resolve, reject) => {
739
+ const request = send(url, {
740
+ method: "GET",
741
+ headers: { "user-agent": USER_AGENT },
742
+ ...signal === void 0 ? {} : { signal }
743
+ }, resolve);
744
+ request.on("error", reject);
745
+ request.end();
746
+ });
747
+ }
748
+ /**
749
+ * Reads a response body to the end.
750
+ * @param response - The response.
751
+ * @returns The body's bytes.
752
+ */
753
+ async function readBody(response) {
754
+ const chunks = [];
755
+ for await (const chunk of response) {
756
+ if (!(chunk instanceof Uint8Array)) throw new KeepsetError("the response body yielded a chunk that is not bytes");
757
+ chunks.push(chunk);
758
+ }
759
+ return Buffer.concat(chunks);
760
+ }
761
+ /**
762
+ * Requests one URL, following its redirects by recursion.
763
+ * @param url - The URL to request.
764
+ * @param signal - Aborts the request.
765
+ * @param redirects - How many redirects led here.
766
+ * @returns The final response.
767
+ */
768
+ async function follow(url, signal, redirects) {
769
+ if (url.protocol !== "http:" && url.protocol !== "https:") throw new KeepsetError(`${url.href} is not an http or https URL`);
770
+ const response = await get(url, signal);
771
+ const { statusCode: status, statusMessage } = response;
772
+ if (status === void 0 || statusMessage === void 0) throw new KeepsetError(`${url.href} answered without a status line`);
773
+ if (!REDIRECTS.has(status)) return new Response(await readBody(response), {
774
+ status,
775
+ statusText: statusMessage
776
+ });
777
+ response.resume();
778
+ const { location } = response.headers;
779
+ if (location === void 0) throw new KeepsetError(`${url.href} answered ${String(status)} without a location`);
780
+ if (redirects === MAX_REDIRECTS) throw new KeepsetError(`${url.href} redirected more than ${String(MAX_REDIRECTS)} times`);
781
+ return follow(new URL(location, url), signal, redirects + 1);
782
+ }
783
+ /**
784
+ * Downloads a URL's bytes exactly as the server sends them, following redirects. Unlike the platform `fetch`, it never undoes a `Content-Encoding`: servers commonly label a `.gz` file `Content-Encoding: gzip`, and a recipe that unpacks gzip needs the file as published, which is also what its SHA-256 is recorded over. No `Accept-Encoding` is sent, so a server that compresses on the fly has no reason to.
785
+ * @param url - An http or https URL.
786
+ * @param init - The request options.
787
+ * @returns A Response holding the raw bytes, with the final status.
788
+ * @throws KeepsetError when a redirect has no location, leaves http and https, or passes the redirect limit.
789
+ */
790
+ async function downloadRaw(url, init) {
791
+ return follow(new URL(url), init.signal, 0);
792
+ }
793
+ //#endregion
704
794
  //#region src/node/fetch-dataset.ts
705
795
  /**
706
796
  * The URL a recipe downloads a source from. A dead source is fetched from its pinned snapshot in the Wayback Machine's `id_` form, which serves the archived bytes as they were captured rather than wrapped in the archive's page chrome.
@@ -742,15 +832,14 @@ function metaOf(entry) {
742
832
  };
743
833
  }
744
834
  /**
745
- * Runs a dataset's recipe: downloads the source (from its pinned snapshot when the source is dead), unpacks the archive, parses the native shape, applies the twindex crosswalk and validates the result against the category target. The native document the crosswalk receives is the parser's output with `meta` built from the entry.
746
- * @param dataset - A dataset from any catalogue.
835
+ * Runs a recipe's download, extract and parse steps, and builds the native document the crosswalk receives: the parser's output with `meta` built from the entry.
836
+ * @param dataset - The dataset.
747
837
  * @param options - The network port, abort signal and clock.
748
- * @returns The resolved target with its provenance and any count warnings.
749
- * @throws RecipeError naming the entry and the failing step; nothing partial or default-shaped is ever returned.
838
+ * @returns The native document and the provenance of its bytes.
750
839
  */
751
- async function fetchEntry(dataset, options = {}) {
752
- const { entry, crosswalk } = dataset;
753
- const { fetch: fetchPort = globalThis.fetch, signal, now = () => /* @__PURE__ */ new Date() } = options;
840
+ async function runToNative(dataset, options) {
841
+ const { entry } = dataset;
842
+ const { fetch: fetchPort = downloadRaw, signal, now = () => /* @__PURE__ */ new Date() } = options;
754
843
  const { recipe } = entry;
755
844
  if (recipe.status === "not-fetchable") throw new RecipeError(entry.id, "recipe", `the entry has no fetchable recipe: ${recipe.reason}`);
756
845
  const source = entry.sources[recipe.source];
@@ -763,22 +852,13 @@ async function fetchEntry(dataset, options = {}) {
763
852
  return new Uint8Array(await response.arrayBuffer());
764
853
  });
765
854
  const unpacked = await step(entry.id, "extract", () => unpack(bytes, recipe.archive));
766
- const native = await step(entry.id, "parse", () => parseNative(unpacked.bytes, recipe.parser));
767
- const encoded = await step(entry.id, "crosswalk", () => encode(crosswalk, {
768
- meta: metaOf(entry),
769
- nodes: native.nodes,
770
- edges: native.edges
771
- }));
772
- const data = await step(entry.id, "validate", () => {
773
- const result = getTarget(entry.category).safeParse(encoded);
774
- if (!result.success) throw new KeepsetError(`the result is not ${entry.category}: ${result.error.issues.map((issue) => `${issue.path.map(String).join(".") || "(root)"}: ${issue.message}`).join("; ")}`);
775
- return result.data;
776
- });
777
- const counts = countTarget(entry.category, data);
778
- const warnings = Object.entries(entry.expectedCounts).filter(([key, expected]) => counts.get(key) !== expected).map(([key, expected]) => `${key}: the entry records ${String(expected)}, the fetched result has ${String(counts.get(key))}`);
855
+ const parsed = await step(entry.id, "parse", () => parseNative(unpacked.bytes, recipe.parser));
779
856
  return {
780
- dataset,
781
- data,
857
+ native: {
858
+ meta: metaOf(entry),
859
+ nodes: parsed.nodes,
860
+ edges: parsed.edges
861
+ },
782
862
  provenance: {
783
863
  entryId: entry.id,
784
864
  catalogue: dataset.catalogue,
@@ -793,12 +873,35 @@ async function fetchEntry(dataset, options = {}) {
793
873
  sha256: createHash("sha256").update(bytes).digest("hex"),
794
874
  byteLength: bytes.byteLength,
795
875
  ...unpacked.member === void 0 ? {} : { member: unpacked.member }
796
- },
797
- warnings
876
+ }
798
877
  };
799
878
  }
800
879
  /**
801
- * Fetches a dataset of the bundled catalogue by id and resolves it to its category's target shape, so `fetchDataset("cora")` yields graph/v1 `{ meta, nodes, edges }` directly. Node-only: it downloads with the platform `fetch` and unpacks with Node's zlib.
880
+ * Runs a dataset's recipe: downloads the source (from its pinned snapshot when the source is dead), unpacks the archive, parses the native shape, applies the twindex crosswalk and validates the result against the category target. The native document the crosswalk receives is the parser's output with `meta` built from the entry.
881
+ * @param dataset - A dataset from any catalogue.
882
+ * @param options - The network port, abort signal and clock.
883
+ * @returns The resolved target with its provenance and any count warnings.
884
+ * @throws RecipeError naming the entry and the failing step; nothing partial or default-shaped is ever returned.
885
+ */
886
+ async function fetchEntry(dataset, options = {}) {
887
+ const { entry, crosswalk } = dataset;
888
+ const { native, provenance } = await runToNative(dataset, options);
889
+ const encoded = await step(entry.id, "crosswalk", () => encode(crosswalk, native));
890
+ const data = await step(entry.id, "validate", () => {
891
+ const result = getTarget(entry.category).safeParse(encoded);
892
+ if (!result.success) throw new KeepsetError(`the result is not ${entry.category}: ${result.error.issues.map((issue) => `${issue.path.map(String).join(".") || "(root)"}: ${issue.message}`).join("; ")}`);
893
+ return result.data;
894
+ });
895
+ const counts = countTarget(entry.category, data);
896
+ return {
897
+ dataset,
898
+ data,
899
+ provenance,
900
+ warnings: Object.entries(entry.expectedCounts).filter(([key, expected]) => counts.get(key) !== expected).map(([key, expected]) => `${key}: the entry records ${String(expected)}, the fetched result has ${String(counts.get(key))}`)
901
+ };
902
+ }
903
+ /**
904
+ * Fetches a dataset of the bundled catalogue by id and resolves it to its category's target shape, so `fetchDataset("cora")` yields graph/v1 `{ meta, nodes, edges }` directly. Node-only: it downloads with Node's http and https clients and unpacks with Node's zlib.
802
905
  * @param id - The entry id.
803
906
  * @param options - The network port, abort signal and clock.
804
907
  * @returns The resolved target with its provenance and any count warnings.
@@ -807,5 +910,40 @@ async function fetchEntry(dataset, options = {}) {
807
910
  async function fetchDataset(id, options = {}) {
808
911
  return fetchEntry(getDataset(id), options);
809
912
  }
913
+ /**
914
+ * Checks a dataset's crosswalk on its real data: runs the recipe to the native document, then twindex's `assertRoundTrip`, which encodes it, decodes the result and compares the two over every path the crosswalk promises to preserve.
915
+ * @param dataset - A dataset from any catalogue.
916
+ * @param options - The network port, abort signal and clock.
917
+ * @returns Whether the round trip holds, and where it first differs when it does not.
918
+ * @throws RecipeError naming the entry and the failing step when the recipe fails before the comparison, or when the crosswalk cannot run.
919
+ */
920
+ async function roundTripEntry(dataset, options = {}) {
921
+ const { native, provenance } = await runToNative(dataset, options);
922
+ try {
923
+ assertRoundTrip(dataset.crosswalk, [native]);
924
+ } catch (error) {
925
+ if (error instanceof RoundTripMismatchError) return {
926
+ holds: false,
927
+ path: error.path,
928
+ message: error.message,
929
+ provenance
930
+ };
931
+ throw new RecipeError(dataset.entry.id, "crosswalk", error instanceof Error ? error.message : String(error), { cause: error });
932
+ }
933
+ return {
934
+ holds: true,
935
+ provenance
936
+ };
937
+ }
938
+ /**
939
+ * Runs `roundTripEntry` on a dataset of the bundled catalogue by id.
940
+ * @param id - The entry id.
941
+ * @param options - The network port, abort signal and clock.
942
+ * @returns Whether the round trip holds.
943
+ * @throws DatasetNotFoundError when no entry has that id, and RecipeError as `roundTripEntry` does.
944
+ */
945
+ async function roundTripDataset(id, options = {}) {
946
+ return roundTripEntry(getDataset(id), options);
947
+ }
810
948
  //#endregion
811
- export { downloadUrl, fetchDataset, fetchEntry };
949
+ export { downloadUrl, fetchDataset, fetchEntry, roundTripDataset, roundTripEntry };