keepset 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundled-P_132GDn.js +46192 -0
- package/dist/{categories-BIZJUKUw.d.ts → categories-CUf1VecC.d.ts} +23 -23
- package/dist/index.d.ts +9 -1
- package/dist/index.js +41 -3
- package/dist/node/fetch-dataset.d.ts +29 -3
- package/dist/node/fetch-dataset.js +174 -36
- package/dist-cli/keepset.mjs +246 -0
- package/package.json +7 -1
- package/schemas/entry.schema.json +1 -1
- package/schemas/manifest.schema.json +1 -1
- package/dist/bundled--fOitQlP.js +0 -621
|
@@ -57,11 +57,11 @@ declare const CitationSchema: z.ZodObject<{
|
|
|
57
57
|
type Citation = z.infer<typeof CitationSchema>;
|
|
58
58
|
/** A relative path inside an archive or a catalogue: `/`-separated segments, none empty, `.` or `..`, so it can never climb out of the directory it is resolved against. */
|
|
59
59
|
declare const RelativePathSchema: z.ZodString;
|
|
60
|
-
/** Which file of an archive to read: a member path, matched against a member's whole path or its trailing segments (so `net.gml` finds `net/net.gml`), or the member whose name ends with
|
|
60
|
+
/** Which file of an archive to read: a member path, matched against a member's whole path or its trailing segments (so `net.gml` finds `net/net.gml`), or the member whose name ends with one of a list of extensions (a format can have several, such as Pajek's `.net` and `.paj`). Either must match exactly one member when the recipe runs; macOS resource-fork members (`__MACOSX/` and `._` files) never match. */
|
|
61
61
|
declare const ArchiveMemberSchema: z.ZodUnion<readonly [z.ZodObject<{
|
|
62
62
|
path: z.ZodString;
|
|
63
63
|
}, z.core.$strict>, z.ZodObject<{
|
|
64
|
-
|
|
64
|
+
extensions: z.ZodArray<z.ZodString>;
|
|
65
65
|
}, z.core.$strict>]>;
|
|
66
66
|
type ArchiveMember = z.infer<typeof ArchiveMemberSchema>;
|
|
67
67
|
/** How the downloaded bytes unpack into the one file the parser reads. */
|
|
@@ -74,21 +74,21 @@ declare const ArchiveSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
74
74
|
member: z.ZodUnion<readonly [z.ZodObject<{
|
|
75
75
|
path: z.ZodString;
|
|
76
76
|
}, z.core.$strict>, z.ZodObject<{
|
|
77
|
-
|
|
77
|
+
extensions: z.ZodArray<z.ZodString>;
|
|
78
78
|
}, z.core.$strict>]>;
|
|
79
79
|
}, z.core.$strict>, z.ZodObject<{
|
|
80
80
|
kind: z.ZodLiteral<"tar">;
|
|
81
81
|
member: z.ZodUnion<readonly [z.ZodObject<{
|
|
82
82
|
path: z.ZodString;
|
|
83
83
|
}, z.core.$strict>, z.ZodObject<{
|
|
84
|
-
|
|
84
|
+
extensions: z.ZodArray<z.ZodString>;
|
|
85
85
|
}, z.core.$strict>]>;
|
|
86
86
|
}, z.core.$strict>, z.ZodObject<{
|
|
87
87
|
kind: z.ZodLiteral<"tar.gz">;
|
|
88
88
|
member: z.ZodUnion<readonly [z.ZodObject<{
|
|
89
89
|
path: z.ZodString;
|
|
90
90
|
}, z.core.$strict>, z.ZodObject<{
|
|
91
|
-
|
|
91
|
+
extensions: z.ZodArray<z.ZodString>;
|
|
92
92
|
}, z.core.$strict>]>;
|
|
93
93
|
}, z.core.$strict>], "kind">;
|
|
94
94
|
type Archive = z.infer<typeof ArchiveSchema>;
|
|
@@ -143,21 +143,21 @@ declare const RecipeSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
143
143
|
member: z.ZodUnion<readonly [z.ZodObject<{
|
|
144
144
|
path: z.ZodString;
|
|
145
145
|
}, z.core.$strict>, z.ZodObject<{
|
|
146
|
-
|
|
146
|
+
extensions: z.ZodArray<z.ZodString>;
|
|
147
147
|
}, z.core.$strict>]>;
|
|
148
148
|
}, z.core.$strict>, z.ZodObject<{
|
|
149
149
|
kind: z.ZodLiteral<"tar">;
|
|
150
150
|
member: z.ZodUnion<readonly [z.ZodObject<{
|
|
151
151
|
path: z.ZodString;
|
|
152
152
|
}, z.core.$strict>, z.ZodObject<{
|
|
153
|
-
|
|
153
|
+
extensions: z.ZodArray<z.ZodString>;
|
|
154
154
|
}, z.core.$strict>]>;
|
|
155
155
|
}, z.core.$strict>, z.ZodObject<{
|
|
156
156
|
kind: z.ZodLiteral<"tar.gz">;
|
|
157
157
|
member: z.ZodUnion<readonly [z.ZodObject<{
|
|
158
158
|
path: z.ZodString;
|
|
159
159
|
}, z.core.$strict>, z.ZodObject<{
|
|
160
|
-
|
|
160
|
+
extensions: z.ZodArray<z.ZodString>;
|
|
161
161
|
}, z.core.$strict>]>;
|
|
162
162
|
}, z.core.$strict>], "kind">;
|
|
163
163
|
parser: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
@@ -261,21 +261,21 @@ declare const EntrySchema: z.ZodObject<{
|
|
|
261
261
|
member: z.ZodUnion<readonly [z.ZodObject<{
|
|
262
262
|
path: z.ZodString;
|
|
263
263
|
}, z.core.$strict>, z.ZodObject<{
|
|
264
|
-
|
|
264
|
+
extensions: z.ZodArray<z.ZodString>;
|
|
265
265
|
}, z.core.$strict>]>;
|
|
266
266
|
}, z.core.$strict>, z.ZodObject<{
|
|
267
267
|
kind: z.ZodLiteral<"tar">;
|
|
268
268
|
member: z.ZodUnion<readonly [z.ZodObject<{
|
|
269
269
|
path: z.ZodString;
|
|
270
270
|
}, z.core.$strict>, z.ZodObject<{
|
|
271
|
-
|
|
271
|
+
extensions: z.ZodArray<z.ZodString>;
|
|
272
272
|
}, z.core.$strict>]>;
|
|
273
273
|
}, z.core.$strict>, z.ZodObject<{
|
|
274
274
|
kind: z.ZodLiteral<"tar.gz">;
|
|
275
275
|
member: z.ZodUnion<readonly [z.ZodObject<{
|
|
276
276
|
path: z.ZodString;
|
|
277
277
|
}, z.core.$strict>, z.ZodObject<{
|
|
278
|
-
|
|
278
|
+
extensions: z.ZodArray<z.ZodString>;
|
|
279
279
|
}, z.core.$strict>]>;
|
|
280
280
|
}, z.core.$strict>], "kind">;
|
|
281
281
|
parser: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
@@ -367,21 +367,21 @@ declare const EntrySchema: z.ZodObject<{
|
|
|
367
367
|
member: {
|
|
368
368
|
path: string;
|
|
369
369
|
} | {
|
|
370
|
-
|
|
370
|
+
extensions: string[];
|
|
371
371
|
};
|
|
372
372
|
} | {
|
|
373
373
|
kind: "tar";
|
|
374
374
|
member: {
|
|
375
375
|
path: string;
|
|
376
376
|
} | {
|
|
377
|
-
|
|
377
|
+
extensions: string[];
|
|
378
378
|
};
|
|
379
379
|
} | {
|
|
380
380
|
kind: "tar.gz";
|
|
381
381
|
member: {
|
|
382
382
|
path: string;
|
|
383
383
|
} | {
|
|
384
|
-
|
|
384
|
+
extensions: string[];
|
|
385
385
|
};
|
|
386
386
|
};
|
|
387
387
|
parser: {
|
|
@@ -552,21 +552,21 @@ declare const CatalogueIndexSchema: z.ZodObject<{
|
|
|
552
552
|
member: z.ZodUnion<readonly [z.ZodObject<{
|
|
553
553
|
path: z.ZodString;
|
|
554
554
|
}, z.core.$strict>, z.ZodObject<{
|
|
555
|
-
|
|
555
|
+
extensions: z.ZodArray<z.ZodString>;
|
|
556
556
|
}, z.core.$strict>]>;
|
|
557
557
|
}, z.core.$strict>, z.ZodObject<{
|
|
558
558
|
kind: z.ZodLiteral<"tar">;
|
|
559
559
|
member: z.ZodUnion<readonly [z.ZodObject<{
|
|
560
560
|
path: z.ZodString;
|
|
561
561
|
}, z.core.$strict>, z.ZodObject<{
|
|
562
|
-
|
|
562
|
+
extensions: z.ZodArray<z.ZodString>;
|
|
563
563
|
}, z.core.$strict>]>;
|
|
564
564
|
}, z.core.$strict>, z.ZodObject<{
|
|
565
565
|
kind: z.ZodLiteral<"tar.gz">;
|
|
566
566
|
member: z.ZodUnion<readonly [z.ZodObject<{
|
|
567
567
|
path: z.ZodString;
|
|
568
568
|
}, z.core.$strict>, z.ZodObject<{
|
|
569
|
-
|
|
569
|
+
extensions: z.ZodArray<z.ZodString>;
|
|
570
570
|
}, z.core.$strict>]>;
|
|
571
571
|
}, z.core.$strict>], "kind">;
|
|
572
572
|
parser: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
@@ -658,21 +658,21 @@ declare const CatalogueIndexSchema: z.ZodObject<{
|
|
|
658
658
|
member: {
|
|
659
659
|
path: string;
|
|
660
660
|
} | {
|
|
661
|
-
|
|
661
|
+
extensions: string[];
|
|
662
662
|
};
|
|
663
663
|
} | {
|
|
664
664
|
kind: "tar";
|
|
665
665
|
member: {
|
|
666
666
|
path: string;
|
|
667
667
|
} | {
|
|
668
|
-
|
|
668
|
+
extensions: string[];
|
|
669
669
|
};
|
|
670
670
|
} | {
|
|
671
671
|
kind: "tar.gz";
|
|
672
672
|
member: {
|
|
673
673
|
path: string;
|
|
674
674
|
} | {
|
|
675
|
-
|
|
675
|
+
extensions: string[];
|
|
676
676
|
};
|
|
677
677
|
};
|
|
678
678
|
parser: {
|
|
@@ -762,21 +762,21 @@ declare const CatalogueIndexSchema: z.ZodObject<{
|
|
|
762
762
|
member: {
|
|
763
763
|
path: string;
|
|
764
764
|
} | {
|
|
765
|
-
|
|
765
|
+
extensions: string[];
|
|
766
766
|
};
|
|
767
767
|
} | {
|
|
768
768
|
kind: "tar";
|
|
769
769
|
member: {
|
|
770
770
|
path: string;
|
|
771
771
|
} | {
|
|
772
|
-
|
|
772
|
+
extensions: string[];
|
|
773
773
|
};
|
|
774
774
|
} | {
|
|
775
775
|
kind: "tar.gz";
|
|
776
776
|
member: {
|
|
777
777
|
path: string;
|
|
778
778
|
} | {
|
|
779
|
-
|
|
779
|
+
extensions: string[];
|
|
780
780
|
};
|
|
781
781
|
};
|
|
782
782
|
parser: {
|
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { $ as SnapshotSchema, A as SemVerSchema, B as Entry, C as Dataset, D as LineageSchema, E as Lineage, F as ArchiveSchema, G as LicenceSchema, H as HttpUrlSchema, I as Citation, J as RESERVED_META_KEYS, K as Parser, L as CitationSchema, M as Archive, N as ArchiveMember, O as Manifest, P as ArchiveMemberSchema, Q as Snapshot, R as CrosswalkReference, S as CatalogueIndexSchema, T as createCatalogue, U as IdSchema, V as EntrySchema, W as Licence, X as RecipeSchema, Y as Recipe, Z as RelativePathSchema, _ as GraphV1Schema, a as CategoryTargets, b as Catalogue, c as getCategory, d as listCategories, et as Source, f as GraphEdge, g as GraphV1, h as GraphNodeSchema, i as CategoryTarget, it as waybackUrl, j as parseManifest, k as ManifestSchema, l as getTarget, m as GraphNode, n as CategoryId, nt as WaybackTimestampSchema, o as CountedKey, p as GraphEdgeSchema, q as ParserSchema, r as CategoryIdSchema, rt as parseEntry, s as countTarget, t as Category, tt as SourceSchema, u as kindOf, v as JsonScalar, w as DatasetQuery, x as CatalogueIndex, y as JsonScalarSchema, z as CrosswalkReferenceSchema } from "./categories-
|
|
1
|
+
import { $ as SnapshotSchema, A as SemVerSchema, B as Entry, C as Dataset, D as LineageSchema, E as Lineage, F as ArchiveSchema, G as LicenceSchema, H as HttpUrlSchema, I as Citation, J as RESERVED_META_KEYS, K as Parser, L as CitationSchema, M as Archive, N as ArchiveMember, O as Manifest, P as ArchiveMemberSchema, Q as Snapshot, R as CrosswalkReference, S as CatalogueIndexSchema, T as createCatalogue, U as IdSchema, V as EntrySchema, W as Licence, X as RecipeSchema, Y as Recipe, Z as RelativePathSchema, _ as GraphV1Schema, a as CategoryTargets, b as Catalogue, c as getCategory, d as listCategories, et as Source, f as GraphEdge, g as GraphV1, h as GraphNodeSchema, i as CategoryTarget, it as waybackUrl, j as parseManifest, k as ManifestSchema, l as getTarget, m as GraphNode, n as CategoryId, nt as WaybackTimestampSchema, o as CountedKey, p as GraphEdgeSchema, q as ParserSchema, r as CategoryIdSchema, rt as parseEntry, s as countTarget, t as Category, tt as SourceSchema, u as kindOf, v as JsonScalar, w as DatasetQuery, x as CatalogueIndex, y as JsonScalarSchema, z as CrosswalkReferenceSchema } from "./categories-CUf1VecC.js";
|
|
2
2
|
import { Crosswalk } from "twindex";
|
|
3
3
|
//#region src/errors.d.ts
|
|
4
4
|
/** The root of this package's error taxonomy: every error keepset throws is a `KeepsetError`, so callers can catch the family with one check. */
|
|
@@ -69,4 +69,12 @@ export declare function getManifest(): Manifest;
|
|
|
69
69
|
*/
|
|
70
70
|
export declare function serialise(doc: Entry | Manifest | Crosswalk): string;
|
|
71
71
|
//#endregion
|
|
72
|
+
//#region src/validate.d.ts
|
|
73
|
+
/**
|
|
74
|
+
* Checks a dataset as the catalogue holds it: the entry against the entry schema, and its crosswalk against twindex's structural validation and the entry's category. A catalogue index is checked the same way when it is built, so a bundled dataset is valid unless the package itself is broken; the check is what a contributor runs on an entry they are writing.
|
|
75
|
+
* @param dataset - The dataset.
|
|
76
|
+
* @returns Every issue found, with paths from the dataset's root (`entry.` or `crosswalk.`); empty when the dataset is valid.
|
|
77
|
+
*/
|
|
78
|
+
export declare function validateDataset(dataset: Dataset): SyntaxIssue[];
|
|
79
|
+
//#endregion
|
|
72
80
|
export { type Archive, type ArchiveMember, ArchiveMemberSchema, ArchiveSchema, type Catalogue, type CatalogueIndex, CatalogueIndexSchema, type Category, type CategoryId, CategoryIdSchema, type CategoryTarget, type CategoryTargets, type Citation, CitationSchema, type CountedKey, type CrosswalkReference, CrosswalkReferenceSchema, type Dataset, type DatasetQuery, type Entry, EntrySchema, type GraphEdge, GraphEdgeSchema, type GraphNode, GraphNodeSchema, type GraphV1, GraphV1Schema, HttpUrlSchema, IdSchema, type JsonScalar, JsonScalarSchema, type Licence, LicenceSchema, type Lineage, LineageSchema, type Manifest, ManifestSchema, type Parser, ParserSchema, RESERVED_META_KEYS, type Recipe, RecipeSchema, type RecipeStep, RelativePathSchema, SemVerSchema, type Snapshot, SnapshotSchema, type Source, SourceSchema, type SyntaxIssue, WaybackTimestampSchema, countTarget, createCatalogue, getCategory, getTarget, kindOf, listCategories, parseEntry, parseManifest, waybackUrl };
|
package/dist/index.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import { A as
|
|
1
|
+
import { A as countTarget, B as CatalogueSyntaxError, C as SnapshotSchema, D as toIssues, E as parseEntry, F as GraphEdgeSchema, G as UnknownCategoryError, H as KeepsetError, I as GraphNodeSchema, L as GraphV1Schema, M as getTarget, N as kindOf, O as waybackUrl, P as listCategories, R as JsonScalarSchema, S as RelativePathSchema, T as WaybackTimestampSchema, U as NativeParseError, V as DatasetNotFoundError, W as RecipeError, _ as IdSchema, a as createCatalogue, b as RESERVED_META_KEYS, c as ManifestSchema, d as ArchiveMemberSchema, f as ArchiveSchema, g as HttpUrlSchema, h as EntrySchema, i as CatalogueIndexSchema, j as getCategory, k as CategoryIdSchema, l as SemVerSchema, m as CrosswalkReferenceSchema, n as getManifest, p as CitationSchema, r as listDatasets, s as LineageSchema, t as getDataset, u as parseManifest, v as LicenceSchema, w as SourceSchema, x as RecipeSchema, y as ParserSchema, z as ArchiveError } from "./bundled-P_132GDn.js";
|
|
2
|
+
import { validateCrosswalk } from "twindex";
|
|
2
3
|
//#region src/serialise.ts
|
|
3
4
|
/**
|
|
4
5
|
* Copies a JSON value with every object's keys in ascending code-unit order, the order the house JSON lint (`json/sort-keys`) requires of committed JSON. `Object.fromEntries` defines each key as an own data property, so a key such as `__proto__` stays a key.
|
|
@@ -11,12 +12,49 @@ function sortKeys(value) {
|
|
|
11
12
|
return value;
|
|
12
13
|
}
|
|
13
14
|
/**
|
|
15
|
+
* Serialises any JSON value in the committed form: two-space indented, keys in ascending order, with a trailing newline.
|
|
16
|
+
* @param value - A JSON value.
|
|
17
|
+
* @returns The file text.
|
|
18
|
+
*/
|
|
19
|
+
function canonicalJson(value) {
|
|
20
|
+
return `${JSON.stringify(sortKeys(value), void 0, 2)}\n`;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
14
23
|
* Serialises a catalogue document (an entry, a manifest or a crosswalk file) as its file holds it: two-space indented JSON with keys in ascending order and a trailing newline. The form depends only on the document's content, so parsing a file written this way and serialising the result reproduces the file byte for byte.
|
|
15
24
|
* @param doc - A parsed entry, manifest or crosswalk.
|
|
16
25
|
* @returns The file text.
|
|
17
26
|
*/
|
|
18
27
|
function serialise(doc) {
|
|
19
|
-
return
|
|
28
|
+
return canonicalJson(doc);
|
|
29
|
+
}
|
|
30
|
+
//#endregion
|
|
31
|
+
//#region src/validate.ts
|
|
32
|
+
/**
|
|
33
|
+
* Checks a dataset as the catalogue holds it: the entry against the entry schema, and its crosswalk against twindex's structural validation and the entry's category. A catalogue index is checked the same way when it is built, so a bundled dataset is valid unless the package itself is broken; the check is what a contributor runs on an entry they are writing.
|
|
34
|
+
* @param dataset - The dataset.
|
|
35
|
+
* @returns Every issue found, with paths from the dataset's root (`entry.` or `crosswalk.`); empty when the dataset is valid.
|
|
36
|
+
*/
|
|
37
|
+
function validateDataset(dataset) {
|
|
38
|
+
const issues = [];
|
|
39
|
+
const entry = EntrySchema.safeParse(dataset.entry);
|
|
40
|
+
if (!entry.success) issues.push(...toIssues(entry.error).map((issue) => ({
|
|
41
|
+
path: `entry.${issue.path}`,
|
|
42
|
+
message: issue.message
|
|
43
|
+
})));
|
|
44
|
+
const { rules, oneWay } = dataset.crosswalk;
|
|
45
|
+
for (const issue of validateCrosswalk(dataset.crosswalk)) {
|
|
46
|
+
const { ruleIndex } = issue;
|
|
47
|
+
const path = ruleIndex >= 0 && ruleIndex < rules.length ? `crosswalk.rules.${String(ruleIndex)}` : ruleIndex >= rules.length && ruleIndex < rules.length + oneWay.length ? `crosswalk.oneWay.${String(ruleIndex - rules.length)}` : "crosswalk";
|
|
48
|
+
issues.push({
|
|
49
|
+
path,
|
|
50
|
+
message: issue.message
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
if (dataset.crosswalk.target !== dataset.entry.category) issues.push({
|
|
54
|
+
path: "crosswalk.target",
|
|
55
|
+
message: `must be the entry's category, ${dataset.entry.category}`
|
|
56
|
+
});
|
|
57
|
+
return issues;
|
|
20
58
|
}
|
|
21
59
|
//#endregion
|
|
22
|
-
export { ArchiveError, ArchiveMemberSchema, ArchiveSchema, CatalogueIndexSchema, CatalogueSyntaxError, CategoryIdSchema, CitationSchema, CrosswalkReferenceSchema, DatasetNotFoundError, EntrySchema, GraphEdgeSchema, GraphNodeSchema, GraphV1Schema, HttpUrlSchema, IdSchema, JsonScalarSchema, KeepsetError, LicenceSchema, LineageSchema, ManifestSchema, NativeParseError, ParserSchema, RESERVED_META_KEYS, RecipeError, RecipeSchema, RelativePathSchema, SemVerSchema, SnapshotSchema, SourceSchema, UnknownCategoryError, WaybackTimestampSchema, countTarget, createCatalogue, getCategory, getDataset, getManifest, getTarget, kindOf, listCategories, listDatasets, parseEntry, parseManifest, serialise, waybackUrl };
|
|
60
|
+
export { ArchiveError, ArchiveMemberSchema, ArchiveSchema, CatalogueIndexSchema, CatalogueSyntaxError, CategoryIdSchema, CitationSchema, CrosswalkReferenceSchema, DatasetNotFoundError, EntrySchema, GraphEdgeSchema, GraphNodeSchema, GraphV1Schema, HttpUrlSchema, IdSchema, JsonScalarSchema, KeepsetError, LicenceSchema, LineageSchema, ManifestSchema, NativeParseError, ParserSchema, RESERVED_META_KEYS, RecipeError, RecipeSchema, RelativePathSchema, SemVerSchema, SnapshotSchema, SourceSchema, UnknownCategoryError, WaybackTimestampSchema, countTarget, createCatalogue, getCategory, getDataset, getManifest, getTarget, kindOf, listCategories, listDatasets, parseEntry, parseManifest, serialise, validateDataset, waybackUrl };
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { B as Entry, C as Dataset, Q as Snapshot, et as Source, i as CategoryTarget } from "../categories-
|
|
1
|
+
import { B as Entry, C as Dataset, Q as Snapshot, et as Source, i as CategoryTarget } from "../categories-CUf1VecC.js";
|
|
2
2
|
//#region src/node/fetch-dataset.d.ts
|
|
3
|
-
/** The network port a recipe downloads through: the
|
|
3
|
+
/** The network port a recipe downloads through: `downloadRaw` by default, which returns the bytes exactly as served, or a stand-in under test. */
|
|
4
4
|
export type FetchPort = (url: string, init: {
|
|
5
5
|
readonly signal?: AbortSignal;
|
|
6
6
|
}) => Promise<Response>;
|
|
@@ -48,11 +48,37 @@ export declare function downloadUrl(source: Source): string;
|
|
|
48
48
|
*/
|
|
49
49
|
export declare function fetchEntry(dataset: Dataset, options?: FetchOptions): Promise<FetchedDataset>;
|
|
50
50
|
/**
|
|
51
|
-
* Fetches a dataset of the bundled catalogue by id and resolves it to its category's target shape, so `fetchDataset("cora")` yields graph/v1 `{ meta, nodes, edges }` directly. Node-only: it downloads with
|
|
51
|
+
* Fetches a dataset of the bundled catalogue by id and resolves it to its category's target shape, so `fetchDataset("cora")` yields graph/v1 `{ meta, nodes, edges }` directly. Node-only: it downloads with Node's http and https clients and unpacks with Node's zlib.
|
|
52
52
|
* @param id - The entry id.
|
|
53
53
|
* @param options - The network port, abort signal and clock.
|
|
54
54
|
* @returns The resolved target with its provenance and any count warnings.
|
|
55
55
|
* @throws DatasetNotFoundError when no entry has that id, and RecipeError naming the entry and the failing step when the recipe fails.
|
|
56
56
|
*/
|
|
57
57
|
export declare function fetchDataset(id: string, options?: FetchOptions): Promise<FetchedDataset>;
|
|
58
|
+
/** Whether decoding a dataset's encoded native document gives the native document back, with the provenance of the bytes it was checked on. */
|
|
59
|
+
export type RoundTripReport = {
|
|
60
|
+
readonly holds: true;
|
|
61
|
+
readonly provenance: Provenance;
|
|
62
|
+
} | {
|
|
63
|
+
readonly holds: false;
|
|
64
|
+
readonly path: string;
|
|
65
|
+
readonly message: string;
|
|
66
|
+
readonly provenance: Provenance;
|
|
67
|
+
};
|
|
68
|
+
/**
|
|
69
|
+
* Checks a dataset's crosswalk on its real data: runs the recipe to the native document, then twindex's `assertRoundTrip`, which encodes it, decodes the result and compares the two over every path the crosswalk promises to preserve.
|
|
70
|
+
* @param dataset - A dataset from any catalogue.
|
|
71
|
+
* @param options - The network port, abort signal and clock.
|
|
72
|
+
* @returns Whether the round trip holds, and where it first differs when it does not.
|
|
73
|
+
* @throws RecipeError naming the entry and the failing step when the recipe fails before the comparison, or when the crosswalk cannot run.
|
|
74
|
+
*/
|
|
75
|
+
export declare function roundTripEntry(dataset: Dataset, options?: FetchOptions): Promise<RoundTripReport>;
|
|
76
|
+
/**
|
|
77
|
+
* Runs `roundTripEntry` on a dataset of the bundled catalogue by id.
|
|
78
|
+
* @param id - The entry id.
|
|
79
|
+
* @param options - The network port, abort signal and clock.
|
|
80
|
+
* @returns Whether the round trip holds.
|
|
81
|
+
* @throws DatasetNotFoundError when no entry has that id, and RecipeError as `roundTripEntry` does.
|
|
82
|
+
*/
|
|
83
|
+
export declare function roundTripDataset(id: string, options?: FetchOptions): Promise<RoundTripReport>;
|
|
58
84
|
//#endregion
|
|
@@ -1,8 +1,10 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { encode } from "twindex";
|
|
1
|
+
import { A as countTarget, H as KeepsetError, M as getTarget, U as NativeParseError, W as RecipeError, o as getOwn, t as getDataset, z as ArchiveError } from "../bundled-P_132GDn.js";
|
|
2
|
+
import { RoundTripMismatchError, assertRoundTrip, encode } from "twindex";
|
|
3
3
|
import { createHash } from "node:crypto";
|
|
4
4
|
import { gunzipSync } from "node:zlib";
|
|
5
5
|
import { unzipSync } from "fflate";
|
|
6
|
+
import { request } from "node:http";
|
|
7
|
+
import { request as request$1 } from "node:https";
|
|
6
8
|
//#region src/parsers/native.ts
|
|
7
9
|
/**
|
|
8
10
|
* Collects node ids in order of first appearance, so a parser that learns its nodes from its edges emits them deterministically.
|
|
@@ -346,8 +348,9 @@ function parseMatrixMarket(text) {
|
|
|
346
348
|
}
|
|
347
349
|
//#endregion
|
|
348
350
|
//#region src/parsers/pajek.ts
|
|
349
|
-
/** Section keywords, lower-cased, mapped to how their lines are read. `*Partition` and `*Vector` hold node attributes this version does not carry, so their lines are skipped, as are the `*Vertices` lines inside them. */
|
|
351
|
+
/** Section keywords, lower-cased, mapped to how their lines are read. `*Network` names the network and starts no section. `*Partition` and `*Vector` hold node attributes this version does not carry, so their lines are skipped, as are the `*Vertices` lines inside them. */
|
|
350
352
|
const sections = /* @__PURE__ */ new Map([
|
|
353
|
+
["*network", "network"],
|
|
351
354
|
["*vertices", "vertices"],
|
|
352
355
|
["*arcs", "pairs"],
|
|
353
356
|
["*edges", "pairs"],
|
|
@@ -358,6 +361,18 @@ const sections = /* @__PURE__ */ new Map([
|
|
|
358
361
|
["*vector", "attributes"]
|
|
359
362
|
]);
|
|
360
363
|
/**
|
|
364
|
+
* Resolves a section keyword, which Pajek reads case-insensitively and accepts abbreviated (`*net` for `*Network`, `*vert` for `*Vertices`): an exact keyword first, otherwise the shortest keyword it abbreviates, so `*edge` is `*Edges` rather than `*Edgeslist`.
|
|
365
|
+
* @param keyword - The keyword as written.
|
|
366
|
+
* @returns How its section is read, or undefined for a keyword that abbreviates none.
|
|
367
|
+
*/
|
|
368
|
+
function sectionOf(keyword) {
|
|
369
|
+
const name = keyword.toLowerCase();
|
|
370
|
+
const exact = sections.get(name);
|
|
371
|
+
if (exact !== void 0) return exact;
|
|
372
|
+
const [shortest] = [...sections.keys()].filter((candidate) => candidate.startsWith(name)).sort((left, right) => left.length - right.length);
|
|
373
|
+
return shortest === void 0 ? void 0 : sections.get(shortest);
|
|
374
|
+
}
|
|
375
|
+
/**
|
|
361
376
|
* Reads a vertex number, which Pajek numbers from 1 to the declared vertex count.
|
|
362
377
|
* @param token - The token.
|
|
363
378
|
* @param count - The declared vertex count.
|
|
@@ -388,14 +403,13 @@ function parsePajek(text) {
|
|
|
388
403
|
const tokens = content.split(/\s+/);
|
|
389
404
|
const [keyword] = tokens;
|
|
390
405
|
if (keyword?.startsWith("*") === true) {
|
|
391
|
-
const
|
|
392
|
-
if (
|
|
406
|
+
const next = sectionOf(keyword);
|
|
407
|
+
if (next === void 0) throw new NativeParseError("pajek", lineNumber, `unknown section ${keyword}`);
|
|
408
|
+
if (next === "network") {
|
|
393
409
|
networks++;
|
|
394
410
|
if (networks > 1) throw new NativeParseError("pajek", lineNumber, "a project file holding more than one network is not supported");
|
|
395
411
|
continue;
|
|
396
412
|
}
|
|
397
|
-
const next = sections.get(name);
|
|
398
|
-
if (next === void 0) throw new NativeParseError("pajek", lineNumber, `unknown section ${keyword}`);
|
|
399
413
|
if (next === "vertices" && section !== "attributes") {
|
|
400
414
|
if (count !== void 0) throw new NativeParseError("pajek", lineNumber, "a network declares *Vertices twice");
|
|
401
415
|
const declared = tokens[1];
|
|
@@ -616,7 +630,7 @@ function isResourceFork(path) {
|
|
|
616
630
|
return path.startsWith("__MACOSX/") || path.split("/").some((segment) => segment.startsWith("._"));
|
|
617
631
|
}
|
|
618
632
|
/**
|
|
619
|
-
* Whether a member path matches a selector: a path selector matches the whole member path or its trailing segments, an
|
|
633
|
+
* Whether a member path matches a selector: a path selector matches the whole member path or its trailing segments, an extensions selector the end of the member's name, ignoring case.
|
|
620
634
|
* @param path - The member path.
|
|
621
635
|
* @param member - The recipe's selector.
|
|
622
636
|
* @returns True on a match.
|
|
@@ -624,7 +638,7 @@ function isResourceFork(path) {
|
|
|
624
638
|
function selects(path, member) {
|
|
625
639
|
if (isResourceFork(path) || path.endsWith("/")) return false;
|
|
626
640
|
if ("path" in member) return path === member.path || path.endsWith(`/${member.path}`);
|
|
627
|
-
return path.toLowerCase().endsWith(
|
|
641
|
+
return member.extensions.some((extension) => path.toLowerCase().endsWith(extension.toLowerCase()));
|
|
628
642
|
}
|
|
629
643
|
/**
|
|
630
644
|
* Picks the one member a selector matches.
|
|
@@ -637,7 +651,7 @@ function pick(paths, member) {
|
|
|
637
651
|
const matched = paths.filter((path) => selects(path, member));
|
|
638
652
|
const [only] = matched;
|
|
639
653
|
if (matched.length !== 1 || only === void 0) {
|
|
640
|
-
const wanted = "path" in member ? `member ${member.path}` : `one member ending ${member.
|
|
654
|
+
const wanted = "path" in member ? `member ${member.path}` : `one member ending ${member.extensions.join(" or ")}`;
|
|
641
655
|
throw new ArchiveError(`expected ${wanted}, found ${String(matched.length)} among: ${paths.join(", ")}`);
|
|
642
656
|
}
|
|
643
657
|
return only;
|
|
@@ -701,6 +715,82 @@ function unzip(bytes, filter) {
|
|
|
701
715
|
}
|
|
702
716
|
}
|
|
703
717
|
//#endregion
|
|
718
|
+
//#region src/node/download.ts
|
|
719
|
+
/** The most redirects one download follows: the Fetch Standard's own limit (https://fetch.spec.whatwg.org/#http-redirect-fetch, step 5). */
|
|
720
|
+
const MAX_REDIRECTS = 20;
|
|
721
|
+
const REDIRECTS = new Set(Object.values({
|
|
722
|
+
movedPermanently: 301,
|
|
723
|
+
found: 302,
|
|
724
|
+
seeOther: 303,
|
|
725
|
+
temporaryRedirect: 307,
|
|
726
|
+
permanentRedirect: 308
|
|
727
|
+
}));
|
|
728
|
+
/** What the downloader says it is, so a server operator can tell where the traffic comes from. */
|
|
729
|
+
const USER_AGENT = "keepset (+https://github.com/ExaDev/keepset)";
|
|
730
|
+
/**
|
|
731
|
+
* Sends one GET and resolves with the response head, its body still unread.
|
|
732
|
+
* @param url - The URL.
|
|
733
|
+
* @param signal - Aborts the request.
|
|
734
|
+
* @returns The response.
|
|
735
|
+
*/
|
|
736
|
+
async function get(url, signal) {
|
|
737
|
+
const send = url.protocol === "https:" ? request$1 : request;
|
|
738
|
+
return new Promise((resolve, reject) => {
|
|
739
|
+
const request = send(url, {
|
|
740
|
+
method: "GET",
|
|
741
|
+
headers: { "user-agent": USER_AGENT },
|
|
742
|
+
...signal === void 0 ? {} : { signal }
|
|
743
|
+
}, resolve);
|
|
744
|
+
request.on("error", reject);
|
|
745
|
+
request.end();
|
|
746
|
+
});
|
|
747
|
+
}
|
|
748
|
+
/**
|
|
749
|
+
* Reads a response body to the end.
|
|
750
|
+
* @param response - The response.
|
|
751
|
+
* @returns The body's bytes.
|
|
752
|
+
*/
|
|
753
|
+
async function readBody(response) {
|
|
754
|
+
const chunks = [];
|
|
755
|
+
for await (const chunk of response) {
|
|
756
|
+
if (!(chunk instanceof Uint8Array)) throw new KeepsetError("the response body yielded a chunk that is not bytes");
|
|
757
|
+
chunks.push(chunk);
|
|
758
|
+
}
|
|
759
|
+
return Buffer.concat(chunks);
|
|
760
|
+
}
|
|
761
|
+
/**
|
|
762
|
+
* Requests one URL, following its redirects by recursion.
|
|
763
|
+
* @param url - The URL to request.
|
|
764
|
+
* @param signal - Aborts the request.
|
|
765
|
+
* @param redirects - How many redirects led here.
|
|
766
|
+
* @returns The final response.
|
|
767
|
+
*/
|
|
768
|
+
async function follow(url, signal, redirects) {
|
|
769
|
+
if (url.protocol !== "http:" && url.protocol !== "https:") throw new KeepsetError(`${url.href} is not an http or https URL`);
|
|
770
|
+
const response = await get(url, signal);
|
|
771
|
+
const { statusCode: status, statusMessage } = response;
|
|
772
|
+
if (status === void 0 || statusMessage === void 0) throw new KeepsetError(`${url.href} answered without a status line`);
|
|
773
|
+
if (!REDIRECTS.has(status)) return new Response(await readBody(response), {
|
|
774
|
+
status,
|
|
775
|
+
statusText: statusMessage
|
|
776
|
+
});
|
|
777
|
+
response.resume();
|
|
778
|
+
const { location } = response.headers;
|
|
779
|
+
if (location === void 0) throw new KeepsetError(`${url.href} answered ${String(status)} without a location`);
|
|
780
|
+
if (redirects === MAX_REDIRECTS) throw new KeepsetError(`${url.href} redirected more than ${String(MAX_REDIRECTS)} times`);
|
|
781
|
+
return follow(new URL(location, url), signal, redirects + 1);
|
|
782
|
+
}
|
|
783
|
+
/**
|
|
784
|
+
* Downloads a URL's bytes exactly as the server sends them, following redirects. Unlike the platform `fetch`, it never undoes a `Content-Encoding`: servers commonly label a `.gz` file `Content-Encoding: gzip`, and a recipe that unpacks gzip needs the file as published, which is also what its SHA-256 is recorded over. No `Accept-Encoding` is sent, so a server that compresses on the fly has no reason to.
|
|
785
|
+
* @param url - An http or https URL.
|
|
786
|
+
* @param init - The request options.
|
|
787
|
+
* @returns A Response holding the raw bytes, with the final status.
|
|
788
|
+
* @throws KeepsetError when a redirect has no location, leaves http and https, or passes the redirect limit.
|
|
789
|
+
*/
|
|
790
|
+
async function downloadRaw(url, init) {
|
|
791
|
+
return follow(new URL(url), init.signal, 0);
|
|
792
|
+
}
|
|
793
|
+
//#endregion
|
|
704
794
|
//#region src/node/fetch-dataset.ts
|
|
705
795
|
/**
|
|
706
796
|
* The URL a recipe downloads a source from. A dead source is fetched from its pinned snapshot in the Wayback Machine's `id_` form, which serves the archived bytes as they were captured rather than wrapped in the archive's page chrome.
|
|
@@ -742,15 +832,14 @@ function metaOf(entry) {
|
|
|
742
832
|
};
|
|
743
833
|
}
|
|
744
834
|
/**
|
|
745
|
-
* Runs a
|
|
746
|
-
* @param dataset -
|
|
835
|
+
* Runs a recipe's download, extract and parse steps, and builds the native document the crosswalk receives: the parser's output with `meta` built from the entry.
|
|
836
|
+
* @param dataset - The dataset.
|
|
747
837
|
* @param options - The network port, abort signal and clock.
|
|
748
|
-
* @returns The
|
|
749
|
-
* @throws RecipeError naming the entry and the failing step; nothing partial or default-shaped is ever returned.
|
|
838
|
+
* @returns The native document and the provenance of its bytes.
|
|
750
839
|
*/
|
|
751
|
-
async function
|
|
752
|
-
const { entry
|
|
753
|
-
const { fetch: fetchPort =
|
|
840
|
+
async function runToNative(dataset, options) {
|
|
841
|
+
const { entry } = dataset;
|
|
842
|
+
const { fetch: fetchPort = downloadRaw, signal, now = () => /* @__PURE__ */ new Date() } = options;
|
|
754
843
|
const { recipe } = entry;
|
|
755
844
|
if (recipe.status === "not-fetchable") throw new RecipeError(entry.id, "recipe", `the entry has no fetchable recipe: ${recipe.reason}`);
|
|
756
845
|
const source = entry.sources[recipe.source];
|
|
@@ -763,22 +852,13 @@ async function fetchEntry(dataset, options = {}) {
|
|
|
763
852
|
return new Uint8Array(await response.arrayBuffer());
|
|
764
853
|
});
|
|
765
854
|
const unpacked = await step(entry.id, "extract", () => unpack(bytes, recipe.archive));
|
|
766
|
-
const
|
|
767
|
-
const encoded = await step(entry.id, "crosswalk", () => encode(crosswalk, {
|
|
768
|
-
meta: metaOf(entry),
|
|
769
|
-
nodes: native.nodes,
|
|
770
|
-
edges: native.edges
|
|
771
|
-
}));
|
|
772
|
-
const data = await step(entry.id, "validate", () => {
|
|
773
|
-
const result = getTarget(entry.category).safeParse(encoded);
|
|
774
|
-
if (!result.success) throw new KeepsetError(`the result is not ${entry.category}: ${result.error.issues.map((issue) => `${issue.path.map(String).join(".") || "(root)"}: ${issue.message}`).join("; ")}`);
|
|
775
|
-
return result.data;
|
|
776
|
-
});
|
|
777
|
-
const counts = countTarget(entry.category, data);
|
|
778
|
-
const warnings = Object.entries(entry.expectedCounts).filter(([key, expected]) => counts.get(key) !== expected).map(([key, expected]) => `${key}: the entry records ${String(expected)}, the fetched result has ${String(counts.get(key))}`);
|
|
855
|
+
const parsed = await step(entry.id, "parse", () => parseNative(unpacked.bytes, recipe.parser));
|
|
779
856
|
return {
|
|
780
|
-
|
|
781
|
-
|
|
857
|
+
native: {
|
|
858
|
+
meta: metaOf(entry),
|
|
859
|
+
nodes: parsed.nodes,
|
|
860
|
+
edges: parsed.edges
|
|
861
|
+
},
|
|
782
862
|
provenance: {
|
|
783
863
|
entryId: entry.id,
|
|
784
864
|
catalogue: dataset.catalogue,
|
|
@@ -793,12 +873,35 @@ async function fetchEntry(dataset, options = {}) {
|
|
|
793
873
|
sha256: createHash("sha256").update(bytes).digest("hex"),
|
|
794
874
|
byteLength: bytes.byteLength,
|
|
795
875
|
...unpacked.member === void 0 ? {} : { member: unpacked.member }
|
|
796
|
-
}
|
|
797
|
-
warnings
|
|
876
|
+
}
|
|
798
877
|
};
|
|
799
878
|
}
|
|
800
879
|
/**
|
|
801
|
-
*
|
|
880
|
+
* Runs a dataset's recipe: downloads the source (from its pinned snapshot when the source is dead), unpacks the archive, parses the native shape, applies the twindex crosswalk and validates the result against the category target. The native document the crosswalk receives is the parser's output with `meta` built from the entry.
|
|
881
|
+
* @param dataset - A dataset from any catalogue.
|
|
882
|
+
* @param options - The network port, abort signal and clock.
|
|
883
|
+
* @returns The resolved target with its provenance and any count warnings.
|
|
884
|
+
* @throws RecipeError naming the entry and the failing step; nothing partial or default-shaped is ever returned.
|
|
885
|
+
*/
|
|
886
|
+
async function fetchEntry(dataset, options = {}) {
|
|
887
|
+
const { entry, crosswalk } = dataset;
|
|
888
|
+
const { native, provenance } = await runToNative(dataset, options);
|
|
889
|
+
const encoded = await step(entry.id, "crosswalk", () => encode(crosswalk, native));
|
|
890
|
+
const data = await step(entry.id, "validate", () => {
|
|
891
|
+
const result = getTarget(entry.category).safeParse(encoded);
|
|
892
|
+
if (!result.success) throw new KeepsetError(`the result is not ${entry.category}: ${result.error.issues.map((issue) => `${issue.path.map(String).join(".") || "(root)"}: ${issue.message}`).join("; ")}`);
|
|
893
|
+
return result.data;
|
|
894
|
+
});
|
|
895
|
+
const counts = countTarget(entry.category, data);
|
|
896
|
+
return {
|
|
897
|
+
dataset,
|
|
898
|
+
data,
|
|
899
|
+
provenance,
|
|
900
|
+
warnings: Object.entries(entry.expectedCounts).filter(([key, expected]) => counts.get(key) !== expected).map(([key, expected]) => `${key}: the entry records ${String(expected)}, the fetched result has ${String(counts.get(key))}`)
|
|
901
|
+
};
|
|
902
|
+
}
|
|
903
|
+
/**
|
|
904
|
+
* Fetches a dataset of the bundled catalogue by id and resolves it to its category's target shape, so `fetchDataset("cora")` yields graph/v1 `{ meta, nodes, edges }` directly. Node-only: it downloads with Node's http and https clients and unpacks with Node's zlib.
|
|
802
905
|
* @param id - The entry id.
|
|
803
906
|
* @param options - The network port, abort signal and clock.
|
|
804
907
|
* @returns The resolved target with its provenance and any count warnings.
|
|
@@ -807,5 +910,40 @@ async function fetchEntry(dataset, options = {}) {
|
|
|
807
910
|
async function fetchDataset(id, options = {}) {
|
|
808
911
|
return fetchEntry(getDataset(id), options);
|
|
809
912
|
}
|
|
913
|
+
/**
|
|
914
|
+
* Checks a dataset's crosswalk on its real data: runs the recipe to the native document, then twindex's `assertRoundTrip`, which encodes it, decodes the result and compares the two over every path the crosswalk promises to preserve.
|
|
915
|
+
* @param dataset - A dataset from any catalogue.
|
|
916
|
+
* @param options - The network port, abort signal and clock.
|
|
917
|
+
* @returns Whether the round trip holds, and where it first differs when it does not.
|
|
918
|
+
* @throws RecipeError naming the entry and the failing step when the recipe fails before the comparison, or when the crosswalk cannot run.
|
|
919
|
+
*/
|
|
920
|
+
async function roundTripEntry(dataset, options = {}) {
|
|
921
|
+
const { native, provenance } = await runToNative(dataset, options);
|
|
922
|
+
try {
|
|
923
|
+
assertRoundTrip(dataset.crosswalk, [native]);
|
|
924
|
+
} catch (error) {
|
|
925
|
+
if (error instanceof RoundTripMismatchError) return {
|
|
926
|
+
holds: false,
|
|
927
|
+
path: error.path,
|
|
928
|
+
message: error.message,
|
|
929
|
+
provenance
|
|
930
|
+
};
|
|
931
|
+
throw new RecipeError(dataset.entry.id, "crosswalk", error instanceof Error ? error.message : String(error), { cause: error });
|
|
932
|
+
}
|
|
933
|
+
return {
|
|
934
|
+
holds: true,
|
|
935
|
+
provenance
|
|
936
|
+
};
|
|
937
|
+
}
|
|
938
|
+
/**
|
|
939
|
+
* Runs `roundTripEntry` on a dataset of the bundled catalogue by id.
|
|
940
|
+
* @param id - The entry id.
|
|
941
|
+
* @param options - The network port, abort signal and clock.
|
|
942
|
+
* @returns Whether the round trip holds.
|
|
943
|
+
* @throws DatasetNotFoundError when no entry has that id, and RecipeError as `roundTripEntry` does.
|
|
944
|
+
*/
|
|
945
|
+
async function roundTripDataset(id, options = {}) {
|
|
946
|
+
return roundTripEntry(getDataset(id), options);
|
|
947
|
+
}
|
|
810
948
|
//#endregion
|
|
811
|
-
export { downloadUrl, fetchDataset, fetchEntry };
|
|
949
|
+
export { downloadUrl, fetchDataset, fetchEntry, roundTripDataset, roundTripEntry };
|