keepset 0.1.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,621 @@
1
+ import { CrosswalkSchema, validateCrosswalk } from "twindex";
2
+ import { z } from "zod";
3
+ //#region src/errors.ts
4
+ /** The root of this package's error taxonomy: every error keepset throws is a `KeepsetError`, so callers can catch the family with one check. */
5
+ var KeepsetError = class extends Error {
6
+ constructor(message, options) {
7
+ super(message, options);
8
+ this.name = new.target.name;
9
+ }
10
+ };
11
+ /** Thrown when a category id names no category in the registry. Carries the ids that do exist, so the message says what would have been accepted. */
12
+ var UnknownCategoryError = class extends KeepsetError {
13
+ categoryId;
14
+ knownIds;
15
+ constructor(categoryId, knownIds) {
16
+ super(`unknown category ${JSON.stringify(categoryId)}: known categories are ${knownIds.join(", ")}`);
17
+ this.categoryId = categoryId;
18
+ this.knownIds = knownIds;
19
+ }
20
+ };
21
+ /** Thrown when an entry, a catalogue manifest or a catalogue index fails its schema. Carries every issue found, not just the first, so one failed parse reports the whole document. */
22
+ var CatalogueSyntaxError = class extends KeepsetError {
23
+ document;
24
+ issues;
25
+ constructor(document, issues) {
26
+ super(`${document} syntax error: ${issues.map((issue) => `${issue.path}: ${issue.message}`).join("; ")}`);
27
+ this.document = document;
28
+ this.issues = issues;
29
+ }
30
+ };
31
+ /** Thrown by a parser when a file does not have the format its recipe names. `line` is 1-based, counting every line of the file, so it points at the offending line in an editor. */
32
+ var NativeParseError = class extends KeepsetError {
33
+ format;
34
+ line;
35
+ constructor(format, line, message) {
36
+ super(`${format} line ${String(line)}: ${message}`);
37
+ this.format = format;
38
+ this.line = line;
39
+ }
40
+ };
41
+ /** Thrown when an archive does not hold exactly one member the recipe's member selector matches, or is not a well-formed archive of its stated kind. */
42
+ var ArchiveError = class extends KeepsetError {};
43
+ /** Thrown when no entry in the catalogue has the requested id. */
44
+ var DatasetNotFoundError = class extends KeepsetError {
45
+ datasetId;
46
+ constructor(datasetId) {
47
+ super(`no dataset ${JSON.stringify(datasetId)} in the catalogue`);
48
+ this.datasetId = datasetId;
49
+ }
50
+ };
51
+ /** Thrown when running an entry's recipe fails. Names the entry and the failing step, and carries the underlying error as `cause`; a recipe never resolves to a partial or default-shaped result. */
52
+ var RecipeError = class extends KeepsetError {
53
+ entryId;
54
+ step;
55
+ constructor(entryId, step, message, options) {
56
+ super(`${entryId}: ${step} failed: ${message}`, options);
57
+ this.entryId = entryId;
58
+ this.step = step;
59
+ }
60
+ };
61
+ //#endregion
62
+ //#region src/define.ts
63
+ /**
64
+ * Attaches a type guard to a Zod schema (the schema-attached guard pattern: validation, inference and narrowing share one definition, so the three cannot drift). This module is isolated because it is the one place in the package that augments an object in place with Object.assign, which the house lint rule that eslint.config.ts turns off for this file alone would otherwise reject: object spread is no alternative, as it copies own properties only and drops the Zod prototype's parse/refine methods.
65
+ * @param schema - The schema to augment.
66
+ * @returns The same schema object, now also carrying `is`.
67
+ */
68
+ function define(schema) {
69
+ return Object.assign(schema, { is(value) {
70
+ return schema.safeParse(value).success;
71
+ } });
72
+ }
73
+ //#endregion
74
+ //#region src/record.ts
75
+ /**
76
+ * A record whose every key must satisfy `key`. Zod's own record silently drops an own `__proto__` key (which `JSON.parse` creates as an ordinary property) before the key schema sees it, so this checks the raw input first and rejects such a key loudly instead of losing the value.
77
+ * @param key - The key schema.
78
+ * @param value - The value schema.
79
+ * @returns The record schema.
80
+ */
81
+ function recordOf(key, value) {
82
+ return z.preprocess((input, ctx) => {
83
+ if (typeof input === "object" && input !== null && Object.hasOwn(input, "__proto__")) ctx.issues.push({
84
+ code: "custom",
85
+ input,
86
+ message: "__proto__ is not a valid key"
87
+ });
88
+ return input;
89
+ }, z.record(key, value));
90
+ }
91
+ //#endregion
92
+ //#region src/targets/graph-v1.ts
93
+ /** A JSON scalar: the only values graph/v1 metadata may hold, so `meta` stays a flat record any consumer can tabulate. `z.number()` already rejects NaN and the infinities, which JSON cannot carry. */
94
+ const JsonScalarSchema = z.union([
95
+ z.string(),
96
+ z.number(),
97
+ z.boolean(),
98
+ z.null()
99
+ ]);
100
+ /** A node: an id, an optional type, and whatever else the dataset carries. The target fixes only what every graph consumer needs, so extra fields pass through untouched. */
101
+ const GraphNodeSchema = z.looseObject({
102
+ id: z.string().min(1),
103
+ type: z.string().min(1).optional()
104
+ });
105
+ /** An edge: source and target node ids, an optional type, and whatever else the dataset carries. Direction is the dataset's own: graph/v1 records it in `meta`, never by duplicating edges. */
106
+ const GraphEdgeSchema = z.looseObject({
107
+ source: z.string().min(1),
108
+ target: z.string().min(1),
109
+ type: z.string().min(1).optional()
110
+ });
111
+ /**
112
+ * The graph/v1 normalisation target: `{ meta, nodes, edges }`, the shape the thesis pipeline's benchmark cache already uses. It describes the result `fetchDataset` resolves a graph entry to, not what a source file may contain. Beyond the field shapes it checks the two properties every consumer relies on: node ids are unique, and every edge endpoint names a node.
113
+ */
114
+ const GraphV1Schema = define(z.object({
115
+ meta: recordOf(z.string().min(1), JsonScalarSchema),
116
+ nodes: z.array(GraphNodeSchema),
117
+ edges: z.array(GraphEdgeSchema)
118
+ }).strict().superRefine((graph, ctx) => {
119
+ const ids = /* @__PURE__ */ new Set();
120
+ for (const [index, node] of graph.nodes.entries()) {
121
+ if (ids.has(node.id)) ctx.addIssue({
122
+ code: "custom",
123
+ path: [
124
+ "nodes",
125
+ index,
126
+ "id"
127
+ ],
128
+ message: `duplicate node id ${JSON.stringify(node.id)}`
129
+ });
130
+ ids.add(node.id);
131
+ }
132
+ for (const [index, edge] of graph.edges.entries()) for (const end of ["source", "target"]) if (!ids.has(edge[end])) ctx.addIssue({
133
+ code: "custom",
134
+ path: [
135
+ "edges",
136
+ index,
137
+ end
138
+ ],
139
+ message: `edge ${end} ${JSON.stringify(edge[end])} names no node`
140
+ });
141
+ }));
142
+ //#endregion
143
+ //#region src/categories.ts
144
+ /** Every category id the registry knows. A category id is `<kind>/v<version>`: the kind of data, and the version of its normalisation target, so a breaking change to a target is a new category rather than an edit to an old one. */
145
+ const CategoryIdSchema = z.enum(["graph/v1"]);
146
+ /** The registry. Adding a category is adding its id to `CategoryIdSchema` and its entry here, with its target schema beside graph/v1's in src/targets/: the mapped type makes the compiler reject a registry that misses an id or has one too many. */
147
+ const registry = { "graph/v1": {
148
+ id: "graph/v1",
149
+ description: "A graph as { meta, nodes, edges }: a flat record of scalar metadata, nodes carrying at least an id, and edges carrying at least source and target node ids, with optional node and edge types.",
150
+ target: GraphV1Schema,
151
+ counted: ["nodes", "edges"]
152
+ } };
153
+ function getCategory(id) {
154
+ const parsed = CategoryIdSchema.safeParse(id);
155
+ if (!parsed.success) throw new UnknownCategoryError(id, CategoryIdSchema.options);
156
+ return registry[parsed.data];
157
+ }
158
+ function getTarget(id) {
159
+ return getCategory(id).target;
160
+ }
161
+ /**
162
+ * The kind of data a category holds: the part of its id before the version.
163
+ * @param id - A category id.
164
+ * @returns The kind, such as `graph`.
165
+ */
166
+ function kindOf(id) {
167
+ return id.slice(0, id.indexOf("/"));
168
+ }
169
+ /** @returns Every registered category, in id order. */
170
+ function listCategories() {
171
+ return CategoryIdSchema.options.map((id) => getCategory(id));
172
+ }
173
+ /**
174
+ * Counts the collections an entry may record expected counts for, in a resolved target.
175
+ * @param id - The target's category.
176
+ * @param target - A resolved target of that category.
177
+ * @returns The length of each counted array, by key.
178
+ */
179
+ function countTarget(id, target) {
180
+ const category = getCategory(id);
181
+ return new Map(category.counted.map((key) => [key, target[key].length]));
182
+ }
183
+ //#endregion
184
+ //#region src/entry.ts
185
+ /** A stable identifier: an entry id, a catalogue id. It starts with a letter or digit, so no id can be `__proto__` or any other name that begins with an underscore. Entry ids stay the same across forks, which is what lets two catalogues be compared entry by entry. */
186
+ const IdSchema = z.string().regex(/^[A-Za-z0-9][A-Za-z0-9._-]*$/, { message: "must start with a letter or digit and contain only letters, digits, '.', '_' and '-'" });
187
+ /** A key of an entry's `properties` or `expectedCounts` record: an identifier that starts with a letter, so it can never be `__proto__` or another name Object.prototype reacts to on assignment. */
188
+ const RecordKeySchema = z.string().regex(/^[A-Za-z][A-Za-z0-9]*$/, { message: "must be a letter followed by letters or digits" });
189
+ /** A calendar date, `YYYY-MM-DD`. */
190
+ const IsoDateSchema = z.iso.date();
191
+ /** An http or https URL: the only schemes a recipe may download from. Any host is allowed, an address or a single-label name included, so a fork can point a recipe at a server on its own network. */
192
+ const HttpUrlSchema = z.url({ protocol: /^https?$/ });
193
+ /** A Wayback Machine timestamp, `YYYYMMDD` or `YYYYMMDDhhmmss`: the moment the snapshot is pinned to. */
194
+ const WaybackTimestampSchema = z.string().regex(/^\d{8}(?:\d{6})?$/, { message: "must be a Wayback timestamp, YYYYMMDD or YYYYMMDDhhmmss" });
195
+ /**
196
+ * The Wayback Machine URL of a snapshot of `url` at `timestamp`: the page form a person follows. The raw bytes a recipe downloads come from the `id_` form, which `fetchDataset` derives from this one.
197
+ * @param timestamp - A Wayback timestamp.
198
+ * @param url - The original URL.
199
+ * @returns The snapshot URL.
200
+ */
201
+ function waybackUrl(timestamp, url) {
202
+ return `https://web.archive.org/web/${timestamp}/${url}`;
203
+ }
204
+ /** A pinned Wayback Machine snapshot of a source. The timestamp is part of the recipe's reproducibility, not incidental metadata, so it travels with the URL and the two are checked against each other. */
205
+ const SnapshotSchema = z.object({
206
+ url: HttpUrlSchema,
207
+ timestamp: WaybackTimestampSchema
208
+ }).strict();
209
+ /**
210
+ * Where a dataset's bytes come from. A `dead` source no longer serves the bytes the recipe was written against and must carry the snapshot the recipe fetches instead; a `live` source may also carry one, as the pinned record of what it served. The snapshot URL must be exactly `waybackUrl(timestamp, url)`.
211
+ */
212
+ const SourceSchema = z.object({
213
+ url: HttpUrlSchema,
214
+ retrieved: IsoDateSchema,
215
+ status: z.enum(["live", "dead"]),
216
+ snapshot: SnapshotSchema.optional()
217
+ }).strict().superRefine((source, ctx) => {
218
+ if (source.status === "dead" && source.snapshot === void 0) ctx.addIssue({
219
+ code: "custom",
220
+ path: ["snapshot"],
221
+ message: "a dead source must pin the Wayback snapshot its recipe fetches"
222
+ });
223
+ if (source.snapshot !== void 0 && source.snapshot.url !== waybackUrl(source.snapshot.timestamp, source.url)) ctx.addIssue({
224
+ code: "custom",
225
+ path: ["snapshot", "url"],
226
+ message: `must be ${waybackUrl(source.snapshot.timestamp, source.url)}, the snapshot of this source's url at this timestamp`
227
+ });
228
+ });
229
+ /** The licence a dataset is published under: an SPDX identifier, a named licence with the URL of its terms, or a statement that the source records none. A licence the source does not state is recorded as unrecorded rather than guessed. */
230
+ const LicenceSchema = z.discriminatedUnion("kind", [
231
+ z.object({
232
+ kind: z.literal("spdx"),
233
+ id: z.string().regex(/^[A-Za-z0-9][A-Za-z0-9.+-]*$/, { message: "must be an SPDX licence identifier" })
234
+ }).strict(),
235
+ z.object({
236
+ kind: z.literal("custom"),
237
+ name: z.string().min(1),
238
+ url: HttpUrlSchema
239
+ }).strict(),
240
+ z.object({ kind: z.literal("unrecorded") }).strict()
241
+ ]);
242
+ /** A work to cite for the dataset: a formatted reference, with its DOI or URL where the source records one. */
243
+ const CitationSchema = z.object({
244
+ text: z.string().min(1),
245
+ doi: z.string().regex(/^10\.\d{4,9}\/\S+$/, { message: "must be a bare DOI, 10.<registrant>/<suffix>" }).optional(),
246
+ url: HttpUrlSchema.optional()
247
+ }).strict();
248
+ /** A relative path inside an archive or a catalogue: `/`-separated segments, none empty, `.` or `..`, so it can never climb out of the directory it is resolved against. */
249
+ const RelativePathSchema = z.string().refine((path) => path.split("/").every((segment) => segment !== "" && segment !== "." && segment !== ".."), { message: "must be a relative path of non-empty segments with no '.' or '..'" });
250
+ /** Which file of an archive to read: a member path, matched against a member's whole path or its trailing segments (so `net.gml` finds `net/net.gml`), or the member whose name ends with an extension. Either must match exactly one member when the recipe runs; macOS resource-fork members (`__MACOSX/` and `._` files) never match. */
251
+ const ArchiveMemberSchema = z.union([z.object({ path: RelativePathSchema }).strict(), z.object({ extension: z.string().regex(/^\.[A-Za-z0-9][A-Za-z0-9.]*$/, { message: "must be a file extension such as .mtx" }) }).strict()]);
252
+ /** How the downloaded bytes unpack into the one file the parser reads. */
253
+ const ArchiveSchema = z.discriminatedUnion("kind", [
254
+ z.object({ kind: z.literal("none") }).strict(),
255
+ z.object({ kind: z.literal("gzip") }).strict(),
256
+ z.object({
257
+ kind: z.literal("zip"),
258
+ member: ArchiveMemberSchema
259
+ }).strict(),
260
+ z.object({
261
+ kind: z.literal("tar"),
262
+ member: ArchiveMemberSchema
263
+ }).strict(),
264
+ z.object({
265
+ kind: z.literal("tar.gz"),
266
+ member: ArchiveMemberSchema
267
+ }).strict()
268
+ ]);
269
+ const ColumnSchema = z.int().min(0);
270
+ /** The text encoding of the unpacked file. Decoding is strict: bytes that are not valid in the stated encoding fail the recipe rather than turning into replacement characters. */
271
+ const EncodingSchema = z.enum(["utf-8", "latin1"]);
272
+ /** The parser that reads the unpacked file into the dataset's native JSON shape, with the options that format needs. Every option is stated in the entry, so a recipe reads the same way wherever it runs. */
273
+ const ParserSchema = z.discriminatedUnion("format", [
274
+ z.object({
275
+ format: z.literal("edge-list"),
276
+ encoding: EncodingSchema,
277
+ delimiter: z.enum([
278
+ "detect",
279
+ "whitespace",
280
+ "tab",
281
+ "comma",
282
+ "space"
283
+ ]),
284
+ skipLines: z.int().min(0),
285
+ commentPrefixes: z.array(z.string().min(1)),
286
+ sourceColumn: ColumnSchema,
287
+ targetColumn: ColumnSchema
288
+ }).strict().refine((parser) => parser.sourceColumn !== parser.targetColumn, {
289
+ message: "sourceColumn and targetColumn must differ",
290
+ path: ["targetColumn"]
291
+ }),
292
+ z.object({
293
+ format: z.literal("matrix-market"),
294
+ encoding: EncodingSchema
295
+ }).strict(),
296
+ z.object({
297
+ format: z.literal("gml"),
298
+ encoding: EncodingSchema
299
+ }).strict(),
300
+ z.object({
301
+ format: z.literal("pajek"),
302
+ encoding: EncodingSchema
303
+ }).strict()
304
+ ]);
305
+ /** How to obtain the dataset: a fetchable recipe naming the source (by index into `sources`), the archive layout and the parser, or a statement that no declarative recipe exists yet and why. */
306
+ const RecipeSchema = z.discriminatedUnion("status", [z.object({
307
+ status: z.literal("fetchable"),
308
+ source: z.int().min(0),
309
+ archive: ArchiveSchema,
310
+ parser: ParserSchema
311
+ }).strict(), z.object({
312
+ status: z.literal("not-fetchable"),
313
+ reason: z.string().min(1)
314
+ }).strict()]);
315
+ /** The twindex crosswalk from the native shape to the category target: the document inline, or a `.json` file path relative to the catalogue root, so entries that share a native shape share one crosswalk. */
316
+ const CrosswalkReferenceSchema = z.union([CrosswalkSchema, z.object({ file: RelativePathSchema.refine((path) => path.endsWith(".json"), { message: "must name a .json file" }) }).strict()]);
317
+ /** The meta keys a fetched result takes from the entry's own descriptor, which its `properties` therefore may not also set. */
318
+ const RESERVED_META_KEYS = ["id", "name"];
319
+ /**
320
+ * One catalogue entry: the descriptor (id, name, description, domain, tags), the category whose target the entry resolves to, where the bytes come from, the licence and citations, the recipe, the crosswalk, the counts a fetched result is expected to have and the dataset's recorded properties. The kind of data is the category's (`kindOf(category)`), never a second field that could disagree with it.
321
+ */
322
+ const EntrySchema = define(z.object({
323
+ id: IdSchema,
324
+ name: z.string().min(1),
325
+ description: z.string().min(1).optional(),
326
+ domain: z.string().min(1).optional(),
327
+ tags: z.array(z.string().min(1)).refine((tags) => new Set(tags).size === tags.length, { message: "tags must be unique" }),
328
+ category: CategoryIdSchema,
329
+ sources: z.array(SourceSchema).min(1),
330
+ licence: LicenceSchema,
331
+ citations: z.array(CitationSchema),
332
+ recipe: RecipeSchema,
333
+ crosswalk: CrosswalkReferenceSchema,
334
+ expectedCounts: recordOf(RecordKeySchema, z.int().min(0)),
335
+ properties: recordOf(RecordKeySchema, JsonScalarSchema)
336
+ }).strict().superRefine((entry, ctx) => {
337
+ if (entry.recipe.status === "fetchable" && entry.recipe.source >= entry.sources.length) ctx.addIssue({
338
+ code: "custom",
339
+ path: ["recipe", "source"],
340
+ message: `names source ${String(entry.recipe.source)} but the entry has ${String(entry.sources.length)}`
341
+ });
342
+ if ("target" in entry.crosswalk && entry.crosswalk.target !== entry.category) ctx.addIssue({
343
+ code: "custom",
344
+ path: ["crosswalk", "target"],
345
+ message: `must be the entry's category, ${entry.category}`
346
+ });
347
+ const counted = getCategory(entry.category).counted;
348
+ for (const key of Object.keys(entry.expectedCounts)) if (!counted.includes(key)) ctx.addIssue({
349
+ code: "custom",
350
+ path: ["expectedCounts", key],
351
+ message: `${entry.category} counts only ${counted.join(", ")}`
352
+ });
353
+ for (const key of RESERVED_META_KEYS) if (Object.hasOwn(entry.properties, key)) ctx.addIssue({
354
+ code: "custom",
355
+ path: ["properties", key],
356
+ message: `${key} is set from the entry itself in a fetched result's meta, so it cannot be a property`
357
+ });
358
+ }));
359
+ /**
360
+ * Collapses Zod issues into the package's own issue shape, so the public error surface stays stable across Zod versions.
361
+ * @param error - A Zod error.
362
+ * @returns One issue per Zod issue, with its path written as dotted text.
363
+ */
364
+ function toIssues(error) {
365
+ return error.issues.map((issue) => ({
366
+ path: issue.path.map(String).join(".") || "(root)",
367
+ message: issue.message
368
+ }));
369
+ }
370
+ /**
371
+ * Parses an unknown value as a catalogue entry.
372
+ * @param doc - The candidate entry document.
373
+ * @returns The validated entry.
374
+ * @throws CatalogueSyntaxError carrying every issue found.
375
+ */
376
+ function parseEntry(doc) {
377
+ const result = EntrySchema.safeParse(doc);
378
+ if (!result.success) throw new CatalogueSyntaxError("entry", toIssues(result.error));
379
+ return result.data;
380
+ }
381
+ //#endregion
382
+ //#region src/manifest.ts
383
+ /** A semantic version, `MAJOR.MINOR.PATCH` with optional pre-release and build parts (https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string). */
384
+ const SemVerSchema = z.string().regex(/^(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)(?:-[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?$/, { message: "must be a semantic version, MAJOR.MINOR.PATCH" });
385
+ /** Where a forked catalogue came from: the catalogue it was forked from, the version it forked at and the repository that holds it, which together are what lets a fork's entries be compared with, and brought back to, their origin. */
386
+ const LineageSchema = z.object({
387
+ catalogue: IdSchema,
388
+ version: SemVerSchema,
389
+ repository: HttpUrlSchema
390
+ }).strict();
391
+ /**
392
+ * A catalogue manifest: the catalogue's own id and version, where it lives, and, for a fork, its lineage. A catalogue is a self-contained set of entry documents beside its manifest, and a fork is its own catalogue with its own id, so lineage may never name the catalogue itself.
393
+ */
394
+ const ManifestSchema = define(z.object({
395
+ id: IdSchema,
396
+ name: z.string().min(1),
397
+ description: z.string().min(1),
398
+ version: SemVerSchema,
399
+ repository: HttpUrlSchema,
400
+ lineage: LineageSchema.optional()
401
+ }).strict().superRefine((manifest, ctx) => {
402
+ if (manifest.lineage?.catalogue === manifest.id) ctx.addIssue({
403
+ code: "custom",
404
+ path: ["lineage", "catalogue"],
405
+ message: "a fork is its own catalogue: lineage must name the catalogue it was forked from, not itself"
406
+ });
407
+ }));
408
+ /**
409
+ * Parses an unknown value as a catalogue manifest.
410
+ * @param doc - The candidate manifest document.
411
+ * @returns The validated manifest.
412
+ * @throws CatalogueSyntaxError carrying every issue found.
413
+ */
414
+ function parseManifest(doc) {
415
+ const result = ManifestSchema.safeParse(doc);
416
+ if (!result.success) throw new CatalogueSyntaxError("manifest", toIssues(result.error));
417
+ return result.data;
418
+ }
419
+ //#endregion
420
+ //#region src/own.ts
421
+ /**
422
+ * Reads an own property of a record keyed by strings that come from data. A plain `record[key]` read follows the prototype chain, so a key such as `constructor` would find an inherited function instead of reporting absence.
423
+ * @param record - The record to read.
424
+ * @param key - Any string.
425
+ * @returns The own property's value, or `undefined` when the record has no such own property.
426
+ */
427
+ function getOwn(record, key) {
428
+ return Object.hasOwn(record, key) ? record[key] : void 0;
429
+ }
430
+ //#endregion
431
+ //#region src/catalogue.ts
432
+ /**
433
+ * A catalogue index: the manifest, every entry, and every crosswalk file of the catalogue, keyed by its catalogue-relative path. The build assembles it from a catalogue directory; a consumer holding its own catalogue (a fork, say) can pass one to `createCatalogue`. Entry ids are unique, every crosswalk passes twindex's structural validation, every crosswalk file reference resolves, and every crosswalk's target is the category of each entry that uses it.
434
+ */
435
+ const CatalogueIndexSchema = define(z.object({
436
+ manifest: ManifestSchema,
437
+ crosswalks: recordOf(RelativePathSchema, CrosswalkSchema),
438
+ entries: z.array(EntrySchema)
439
+ }).strict().superRefine((index, ctx) => {
440
+ for (const [file, crosswalk] of Object.entries(index.crosswalks)) for (const issue of validateCrosswalk(crosswalk)) ctx.addIssue({
441
+ code: "custom",
442
+ path: ["crosswalks", file],
443
+ message: `rule ${String(issue.ruleIndex)}: ${issue.message}`
444
+ });
445
+ const seen = /* @__PURE__ */ new Set();
446
+ for (const [position, entry] of index.entries.entries()) {
447
+ if (seen.has(entry.id)) ctx.addIssue({
448
+ code: "custom",
449
+ path: [
450
+ "entries",
451
+ position,
452
+ "id"
453
+ ],
454
+ message: `duplicate entry id ${entry.id}`
455
+ });
456
+ seen.add(entry.id);
457
+ if (!("file" in entry.crosswalk)) for (const issue of validateCrosswalk(entry.crosswalk)) ctx.addIssue({
458
+ code: "custom",
459
+ path: [
460
+ "entries",
461
+ position,
462
+ "crosswalk"
463
+ ],
464
+ message: `rule ${String(issue.ruleIndex)}: ${issue.message}`
465
+ });
466
+ else {
467
+ const crosswalk = getOwn(index.crosswalks, entry.crosswalk.file);
468
+ if (crosswalk === void 0) ctx.addIssue({
469
+ code: "custom",
470
+ path: [
471
+ "entries",
472
+ position,
473
+ "crosswalk",
474
+ "file"
475
+ ],
476
+ message: `no crosswalk file ${entry.crosswalk.file} in the catalogue`
477
+ });
478
+ else if (crosswalk.target !== entry.category) ctx.addIssue({
479
+ code: "custom",
480
+ path: [
481
+ "entries",
482
+ position,
483
+ "crosswalk",
484
+ "file"
485
+ ],
486
+ message: `${entry.crosswalk.file} targets ${crosswalk.target}, not the entry's category ${entry.category}`
487
+ });
488
+ }
489
+ }
490
+ }));
491
+ /**
492
+ * Whether a dataset matches a query.
493
+ * @param dataset - The dataset.
494
+ * @param query - The filters.
495
+ * @returns True when every given filter matches.
496
+ */
497
+ function matches(dataset, query) {
498
+ const { entry } = dataset;
499
+ if (query.kind !== void 0 && dataset.kind !== query.kind) return false;
500
+ if (query.category !== void 0 && entry.category !== query.category) return false;
501
+ if (query.domain !== void 0 && entry.domain !== query.domain) return false;
502
+ if (query.text !== void 0) {
503
+ const needle = query.text.toLowerCase();
504
+ return [
505
+ entry.id,
506
+ entry.name,
507
+ entry.description,
508
+ entry.domain,
509
+ ...entry.tags
510
+ ].filter((field) => field !== void 0).some((field) => field.toLowerCase().includes(needle));
511
+ }
512
+ return true;
513
+ }
514
+ /**
515
+ * Builds a queryable catalogue from an index. Pure and synchronous: no filesystem, no network, so it runs the same in a browser as in Node.
516
+ * @param index - A candidate catalogue index.
517
+ * @returns The catalogue.
518
+ * @throws CatalogueSyntaxError carrying every issue when the index is not a valid catalogue.
519
+ */
520
+ function createCatalogue(index) {
521
+ const parsed = CatalogueIndexSchema.safeParse(index);
522
+ if (!parsed.success) throw new CatalogueSyntaxError("catalogue", toIssues(parsed.error));
523
+ const { manifest, crosswalks, entries } = parsed.data;
524
+ const catalogue = {
525
+ id: manifest.id,
526
+ version: manifest.version
527
+ };
528
+ const datasets = /* @__PURE__ */ new Map();
529
+ for (const entry of [...entries].sort((left, right) => left.id < right.id ? -1 : 1)) {
530
+ const crosswalk = "file" in entry.crosswalk ? getOwn(crosswalks, entry.crosswalk.file) : entry.crosswalk;
531
+ if (crosswalk === void 0) throw new CatalogueSyntaxError("catalogue", [{
532
+ path: `entries.${entry.id}.crosswalk.file`,
533
+ message: "the crosswalk file does not resolve"
534
+ }]);
535
+ datasets.set(entry.id, {
536
+ entry,
537
+ kind: kindOf(entry.category),
538
+ crosswalk,
539
+ catalogue
540
+ });
541
+ }
542
+ return {
543
+ manifest,
544
+ listDatasets: (query = {}) => [...datasets.values()].filter((dataset) => matches(dataset, query)),
545
+ getDataset: (id) => {
546
+ const dataset = datasets.get(id);
547
+ if (dataset === void 0) throw new DatasetNotFoundError(id);
548
+ return dataset;
549
+ }
550
+ };
551
+ }
552
+ //#endregion
553
+ //#region src/generated/catalogue.json
554
+ var catalogue_default = {
555
+ manifest: {
556
+ "id": "keepset",
557
+ "name": "keepset",
558
+ "description": "The keepset catalogue: datasets with machine-readable recipes that fetch each one and normalise it into its category's target shape.",
559
+ "version": "0.1.0",
560
+ "repository": "https://github.com/ExaDev/keepset"
561
+ },
562
+ crosswalks: { "crosswalks/graph-v1/topology.json": {
563
+ "source": "keepset/topology",
564
+ "target": "graph/v1",
565
+ "rules": [
566
+ {
567
+ "op": "move",
568
+ "from": "meta",
569
+ "to": "meta"
570
+ },
571
+ {
572
+ "op": "move",
573
+ "from": "nodes[].id",
574
+ "to": "nodes[].id"
575
+ },
576
+ {
577
+ "op": "move",
578
+ "from": "edges[].source",
579
+ "to": "edges[].source"
580
+ },
581
+ {
582
+ "op": "move",
583
+ "from": "edges[].target",
584
+ "to": "edges[].target"
585
+ }
586
+ ],
587
+ "oneWay": []
588
+ } },
589
+ entries: []
590
+ };
591
+ //#endregion
592
+ //#region src/bundled.ts
593
+ let bundled;
594
+ /** The catalogue shipped in this package, assembled from catalogue/ by the build and parsed on first use, so importing the package costs nothing until a query runs. */
595
+ function catalogue() {
596
+ bundled ??= createCatalogue(catalogue_default);
597
+ return bundled;
598
+ }
599
+ /**
600
+ * Lists the datasets in the bundled catalogue that match a query. Browser-safe: a pure read over the index bundled with the package.
601
+ * @param query - The filters; all must match, and none given lists everything.
602
+ * @returns The matching datasets, in entry id order.
603
+ */
604
+ function listDatasets(query) {
605
+ return catalogue().listDatasets(query);
606
+ }
607
+ /**
608
+ * Returns one dataset of the bundled catalogue: its entry (metadata and recipe) and its resolved crosswalk. Browser-safe.
609
+ * @param id - The entry id.
610
+ * @returns The dataset.
611
+ * @throws DatasetNotFoundError when no entry has that id.
612
+ */
613
+ function getDataset(id) {
614
+ return catalogue().getDataset(id);
615
+ }
616
+ /** @returns The bundled catalogue's manifest: its id, version and repository. */
617
+ function getManifest() {
618
+ return catalogue().manifest;
619
+ }
620
+ //#endregion
621
+ export { getCategory as A, DatasetNotFoundError as B, SnapshotSchema as C, waybackUrl as D, parseEntry as E, GraphNodeSchema as F, NativeParseError as H, GraphV1Schema as I, JsonScalarSchema as L, kindOf as M, listCategories as N, CategoryIdSchema as O, GraphEdgeSchema as P, ArchiveError as R, RelativePathSchema as S, WaybackTimestampSchema as T, RecipeError as U, KeepsetError as V, UnknownCategoryError as W, IdSchema as _, createCatalogue as a, RESERVED_META_KEYS as b, ManifestSchema as c, ArchiveMemberSchema as d, ArchiveSchema as f, HttpUrlSchema as g, EntrySchema as h, CatalogueIndexSchema as i, getTarget as j, countTarget as k, SemVerSchema as l, CrosswalkReferenceSchema as m, getManifest as n, getOwn as o, CitationSchema as p, listDatasets as r, LineageSchema as s, getDataset as t, parseManifest as u, LicenceSchema as v, SourceSchema as w, RecipeSchema as x, ParserSchema as y, CatalogueSyntaxError as z };