keepset 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -1
- package/dist/{bundled-BIBLzetw.js → bundled-DaXJpk0C.js} +1492 -9
- package/dist/{categories-CUf1VecC.d.ts → categories-BSJj6C-y.d.ts} +25 -8
- package/dist/index.d.ts +26 -3
- package/dist/index.js +2 -2
- package/dist/node/fetch-dataset.d.ts +1 -1
- package/dist/node/fetch-dataset.js +1 -1
- package/dist-cli/keepset.mjs +28 -14
- package/package.json +1 -1
- package/schemas/entry.schema.json +1 -1
- package/schemas/manifest.schema.json +1 -1
|
@@ -3,6 +3,10 @@ import { z } from "zod";
|
|
|
3
3
|
//#region src/entry.d.ts
|
|
4
4
|
/** A stable identifier: an entry id, a catalogue id. It starts with a letter or digit, so no id can be `__proto__` or any other name that begins with an underscore. Entry ids stay the same across forks, which is what lets two catalogues be compared entry by entry. */
|
|
5
5
|
declare const IdSchema: z.ZodString;
|
|
6
|
+
/** A key of an entry's `properties` or `expectedCounts` record: an identifier that starts with a letter, so it can never be `__proto__` or another name Object.prototype reacts to on assignment. */
|
|
7
|
+
/** The path a dataset sits under in a collection, outermost segment first, such as `["suitesparse", "gset"]`. Each segment is an id, so a segment can never contain the `/` that joins a namespace to a name. */
|
|
8
|
+
declare const NamespaceSchema: z.ZodArray<z.ZodString>;
|
|
9
|
+
type Namespace = z.infer<typeof NamespaceSchema>;
|
|
6
10
|
/** An http or https URL: the only schemes a recipe may download from. Any host is allowed, an address or a single-label name included, so a fork can point a recipe at a server on its own network. */
|
|
7
11
|
declare const HttpUrlSchema: z.ZodURL;
|
|
8
12
|
/** A Wayback Machine timestamp, `YYYYMMDD` or `YYYYMMDDhhmmss`: the moment the snapshot is pinned to. */
|
|
@@ -211,10 +215,12 @@ type CrosswalkReference = z.infer<typeof CrosswalkReferenceSchema>;
|
|
|
211
215
|
/** The meta keys a fetched result takes from the entry's own descriptor, which its `properties` therefore may not also set. */
|
|
212
216
|
declare const RESERVED_META_KEYS: readonly ["id", "name"];
|
|
213
217
|
/**
|
|
214
|
-
* One catalogue entry: the descriptor (id, name, description, domain, tags), the category whose target the entry resolves to, where the bytes come from, the licence and citations, the recipe, the crosswalk, the counts a fetched result is expected to have and the dataset's recorded properties. The kind of data is the category's (`kindOf(category)`), never a second field that could disagree with it.
|
|
218
|
+
* One catalogue entry: the descriptor (id, optional namespace and aliases, name, description, domain, tags), the category whose target the entry resolves to, where the bytes come from, the licence and citations, the recipe, the crosswalk, the counts a fetched result is expected to have and the dataset's recorded properties. The kind of data is the category's (`kindOf(category)`), never a second field that could disagree with it.
|
|
215
219
|
*/
|
|
216
220
|
declare const EntrySchema: z.ZodObject<{
|
|
217
221
|
id: z.ZodString;
|
|
222
|
+
namespace: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
223
|
+
aliases: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
218
224
|
name: z.ZodString;
|
|
219
225
|
description: z.ZodOptional<z.ZodString>;
|
|
220
226
|
domain: z.ZodOptional<z.ZodString>;
|
|
@@ -411,6 +417,8 @@ declare const EntrySchema: z.ZodObject<{
|
|
|
411
417
|
};
|
|
412
418
|
expectedCounts: Record<string, number>;
|
|
413
419
|
properties: Record<string, string | number | boolean | null>;
|
|
420
|
+
namespace?: string[] | undefined;
|
|
421
|
+
aliases?: string[] | undefined;
|
|
414
422
|
description?: string | undefined;
|
|
415
423
|
domain?: string | undefined;
|
|
416
424
|
};
|
|
@@ -473,7 +481,7 @@ declare function parseManifest(doc: unknown): Manifest;
|
|
|
473
481
|
//#endregion
|
|
474
482
|
//#region src/catalogue.d.ts
|
|
475
483
|
/**
|
|
476
|
-
* A catalogue index: the manifest, every entry, and every crosswalk file of the catalogue, keyed by its catalogue-relative path. The build assembles it from a catalogue directory; a consumer holding its own catalogue (a fork, say) can pass one to `createCatalogue`. Entry ids are unique, every crosswalk passes twindex's structural validation, every crosswalk file reference resolves, and every crosswalk's target is the category of each entry that uses it.
|
|
484
|
+
* A catalogue index: the manifest, every entry, and every crosswalk file of the catalogue, keyed by its catalogue-relative path. The build assembles it from a catalogue directory; a consumer holding its own catalogue (a fork, say) can pass one to `createCatalogue`. Entry ids are unique, every fully qualified name (the whole namespace and an id or alias) identifies one entry and is not another entry's id, every crosswalk passes twindex's structural validation, every crosswalk file reference resolves, and every crosswalk's target is the category of each entry that uses it.
|
|
477
485
|
*/
|
|
478
486
|
declare const CatalogueIndexSchema: z.ZodObject<{
|
|
479
487
|
manifest: z.ZodObject<{
|
|
@@ -506,6 +514,8 @@ declare const CatalogueIndexSchema: z.ZodObject<{
|
|
|
506
514
|
}>, unknown>;
|
|
507
515
|
entries: z.ZodArray<z.ZodObject<{
|
|
508
516
|
id: z.ZodString;
|
|
517
|
+
namespace: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
518
|
+
aliases: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
509
519
|
name: z.ZodString;
|
|
510
520
|
description: z.ZodOptional<z.ZodString>;
|
|
511
521
|
domain: z.ZodOptional<z.ZodString>;
|
|
@@ -702,6 +712,8 @@ declare const CatalogueIndexSchema: z.ZodObject<{
|
|
|
702
712
|
};
|
|
703
713
|
expectedCounts: Record<string, number>;
|
|
704
714
|
properties: Record<string, string | number | boolean | null>;
|
|
715
|
+
namespace?: string[] | undefined;
|
|
716
|
+
aliases?: string[] | undefined;
|
|
705
717
|
description?: string | undefined;
|
|
706
718
|
domain?: string | undefined;
|
|
707
719
|
};
|
|
@@ -806,6 +818,8 @@ declare const CatalogueIndexSchema: z.ZodObject<{
|
|
|
806
818
|
};
|
|
807
819
|
expectedCounts: Record<string, number>;
|
|
808
820
|
properties: Record<string, string | number | boolean | null>;
|
|
821
|
+
namespace?: string[] | undefined;
|
|
822
|
+
aliases?: string[] | undefined;
|
|
809
823
|
description?: string | undefined;
|
|
810
824
|
domain?: string | undefined;
|
|
811
825
|
}[];
|
|
@@ -827,7 +841,9 @@ interface DatasetQuery {
|
|
|
827
841
|
readonly category?: string;
|
|
828
842
|
/** The entry's domain, matched exactly. */
|
|
829
843
|
readonly domain?: string;
|
|
830
|
-
/**
|
|
844
|
+
/** A namespace path such as `suitesparse` or `suitesparse/gset`: matches the entries in that namespace and every namespace beneath it. */
|
|
845
|
+
readonly namespace?: string;
|
|
846
|
+
/** Text that must appear, ignoring case, in the entry's id, an alias, its namespace, name, description, domain or one of its tags. */
|
|
831
847
|
readonly text?: string;
|
|
832
848
|
}
|
|
833
849
|
/** A queryable catalogue: the surface `listDatasets` and `getDataset` are bound to. */
|
|
@@ -840,12 +856,13 @@ interface Catalogue {
|
|
|
840
856
|
*/
|
|
841
857
|
readonly listDatasets: (query?: DatasetQuery) => Dataset[];
|
|
842
858
|
/**
|
|
843
|
-
* Returns one dataset by entry id.
|
|
844
|
-
* @param
|
|
859
|
+
* Returns one dataset by name. An entry's own id always resolves to it, ahead of any alias or shorter form; otherwise the name is an alias, or an id or alias with any trailing run of the entry's namespace before it (`b/c` or `a/b/c` for id `c` in namespace `a/b`), and resolves only when exactly one entry answers to it.
|
|
860
|
+
* @param name - An entry id, alias or qualified name; untrusted, so lookup never reaches inherited properties.
|
|
845
861
|
* @returns The dataset.
|
|
846
|
-
* @throws
|
|
862
|
+
* @throws DatasetAmbiguousError when more than one entry answers to the name and none has it as its id.
|
|
863
|
+
* @throws DatasetNotFoundError when no entry answers to the name.
|
|
847
864
|
*/
|
|
848
|
-
readonly getDataset: (
|
|
865
|
+
readonly getDataset: (name: string) => Dataset;
|
|
849
866
|
}
|
|
850
867
|
/**
|
|
851
868
|
* Builds a queryable catalogue from an index. Pure and synchronous: no filesystem, no network, so it runs the same in a browser as in Node.
|
|
@@ -957,4 +974,4 @@ declare function listCategories(): readonly Category[];
|
|
|
957
974
|
*/
|
|
958
975
|
declare function countTarget(id: CategoryId, target: CategoryTarget): ReadonlyMap<string, number>;
|
|
959
976
|
//#endregion
|
|
960
|
-
export {
|
|
977
|
+
export { RelativePathSchema as $, SemVerSchema as A, Entry as B, Dataset as C, LineageSchema as D, Lineage as E, ArchiveSchema as F, LicenceSchema as G, HttpUrlSchema as H, Citation as I, Parser as J, Namespace as K, CitationSchema as L, Archive as M, ArchiveMember as N, Manifest as O, ArchiveMemberSchema as P, RecipeSchema as Q, CrosswalkReference as R, CatalogueIndexSchema as S, createCatalogue as T, IdSchema as U, EntrySchema as V, Licence as W, RESERVED_META_KEYS as X, ParserSchema as Y, Recipe as Z, GraphV1Schema as _, CategoryTargets as a, parseEntry as at, Catalogue as b, getCategory as c, listCategories as d, Snapshot as et, GraphEdge as f, GraphV1 as g, GraphNodeSchema as h, CategoryTarget as i, WaybackTimestampSchema as it, parseManifest as j, ManifestSchema as k, getTarget as l, GraphNode as m, CategoryId as n, Source as nt, CountedKey as o, waybackUrl as ot, GraphEdgeSchema as p, NamespaceSchema as q, CategoryIdSchema as r, SourceSchema as rt, countTarget as s, Category as t, SnapshotSchema as tt, kindOf as u, JsonScalar as v, DatasetQuery as w, CatalogueIndex as x, JsonScalarSchema as y, CrosswalkReferenceSchema as z };
|
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { $ as
|
|
1
|
+
import { $ as RelativePathSchema, A as SemVerSchema, B as Entry, C as Dataset, D as LineageSchema, E as Lineage, F as ArchiveSchema, G as LicenceSchema, H as HttpUrlSchema, I as Citation, J as Parser, K as Namespace, L as CitationSchema, M as Archive, N as ArchiveMember, O as Manifest, P as ArchiveMemberSchema, Q as RecipeSchema, R as CrosswalkReference, S as CatalogueIndexSchema, T as createCatalogue, U as IdSchema, V as EntrySchema, W as Licence, X as RESERVED_META_KEYS, Y as ParserSchema, Z as Recipe, _ as GraphV1Schema, a as CategoryTargets, at as parseEntry, b as Catalogue, c as getCategory, d as listCategories, et as Snapshot, f as GraphEdge, g as GraphV1, h as GraphNodeSchema, i as CategoryTarget, it as WaybackTimestampSchema, j as parseManifest, k as ManifestSchema, l as getTarget, m as GraphNode, n as CategoryId, nt as Source, o as CountedKey, ot as waybackUrl, p as GraphEdgeSchema, q as NamespaceSchema, r as CategoryIdSchema, rt as SourceSchema, s as countTarget, t as Category, tt as SnapshotSchema, u as kindOf, v as JsonScalar, w as DatasetQuery, x as CatalogueIndex, y as JsonScalarSchema, z as CrosswalkReferenceSchema } from "./categories-BSJj6C-y.js";
|
|
2
2
|
import { Crosswalk } from "twindex";
|
|
3
3
|
//#region src/errors.d.ts
|
|
4
4
|
/** The root of this package's error taxonomy: every error keepset throws is a `KeepsetError`, so callers can catch the family with one check. */
|
|
@@ -30,11 +30,17 @@ export declare class NativeParseError extends KeepsetError {
|
|
|
30
30
|
}
|
|
31
31
|
/** Thrown when an archive does not hold exactly one member the recipe's member selector matches, or is not a well-formed archive of its stated kind. */
|
|
32
32
|
export declare class ArchiveError extends KeepsetError {}
|
|
33
|
-
/** Thrown when no entry in the catalogue
|
|
33
|
+
/** Thrown when no entry in the catalogue answers to the requested id, alias or qualified name. */
|
|
34
34
|
export declare class DatasetNotFoundError extends KeepsetError {
|
|
35
35
|
readonly datasetId: string;
|
|
36
36
|
constructor(datasetId: string);
|
|
37
37
|
}
|
|
38
|
+
/** Thrown when a name that is not an entry's own id is a valid way to address more than one entry. Carries the ids of the entries it could mean, so the message says how to be specific. */
|
|
39
|
+
export declare class DatasetAmbiguousError extends KeepsetError {
|
|
40
|
+
readonly requested: string;
|
|
41
|
+
readonly candidateIds: readonly string[];
|
|
42
|
+
constructor(name: string, candidateIds: readonly string[]);
|
|
43
|
+
}
|
|
38
44
|
/** The step of a recipe that failed: the recipe itself (not fetchable), downloading the source, unpacking the archive, parsing the native shape, applying the crosswalk, or validating the result against the category target. */
|
|
39
45
|
type RecipeStep = "recipe" | "download" | "extract" | "parse" | "crosswalk" | "validate";
|
|
40
46
|
/** Thrown when running an entry's recipe fails. Names the entry and the failing step, and carries the underlying error as `cause`; a recipe never resolves to a partial or default-shaped result. */
|
|
@@ -61,6 +67,23 @@ export declare function getDataset(id: string): Dataset;
|
|
|
61
67
|
/** @returns The bundled catalogue's manifest: its id, version and repository. */
|
|
62
68
|
export declare function getManifest(): Manifest;
|
|
63
69
|
//#endregion
|
|
70
|
+
//#region src/names.d.ts
|
|
71
|
+
/** The character that joins a namespace's segments to each other and to a name. It is not in an id's alphabet, so a name containing it is always a path and never a literal id. */
|
|
72
|
+
export declare const NAME_SEPARATOR = "/";
|
|
73
|
+
/** The names an entry answers to: its fully qualified names, which must each identify it alone, and every name that resolves to it, which also includes the shorter forms. */
|
|
74
|
+
interface EntryNames {
|
|
75
|
+
/** The whole namespace joined to each leaf, or the bare leaf for an entry with no namespace. */
|
|
76
|
+
readonly full: readonly string[];
|
|
77
|
+
/** Every suffix of the namespace joined to each leaf, the fully qualified name and the bare leaf among them. */
|
|
78
|
+
readonly all: readonly string[];
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Works out the names an entry answers to. A leaf is the entry's id or one of its aliases; a name is a leaf with any trailing run of the entry's namespace segments before it, so an entry in namespace `a/b` with id `c` answers to `c`, `b/c` and `a/b/c`.
|
|
82
|
+
* @param entry - The entry: only its id, namespace and aliases are read.
|
|
83
|
+
* @returns The fully qualified names and every name that resolves to the entry.
|
|
84
|
+
*/
|
|
85
|
+
export declare function namesOf(entry: Pick<Entry, "id" | "namespace" | "aliases">): EntryNames;
|
|
86
|
+
//#endregion
|
|
64
87
|
//#region src/serialise.d.ts
|
|
65
88
|
/**
|
|
66
89
|
* Serialises a catalogue document (an entry, a manifest or a crosswalk file) as its file holds it: two-space indented JSON with keys in ascending order and a trailing newline. The form depends only on the document's content, so parsing a file written this way and serialising the result reproduces the file byte for byte.
|
|
@@ -77,4 +100,4 @@ export declare function serialise(doc: Entry | Manifest | Crosswalk): string;
|
|
|
77
100
|
*/
|
|
78
101
|
export declare function validateDataset(dataset: Dataset): SyntaxIssue[];
|
|
79
102
|
//#endregion
|
|
80
|
-
export { type Archive, type ArchiveMember, ArchiveMemberSchema, ArchiveSchema, type Catalogue, type CatalogueIndex, CatalogueIndexSchema, type Category, type CategoryId, CategoryIdSchema, type CategoryTarget, type CategoryTargets, type Citation, CitationSchema, type CountedKey, type CrosswalkReference, CrosswalkReferenceSchema, type Dataset, type DatasetQuery, type Entry, EntrySchema, type GraphEdge, GraphEdgeSchema, type GraphNode, GraphNodeSchema, type GraphV1, GraphV1Schema, HttpUrlSchema, IdSchema, type JsonScalar, JsonScalarSchema, type Licence, LicenceSchema, type Lineage, LineageSchema, type Manifest, ManifestSchema, type Parser, ParserSchema, RESERVED_META_KEYS, type Recipe, RecipeSchema, type RecipeStep, RelativePathSchema, SemVerSchema, type Snapshot, SnapshotSchema, type Source, SourceSchema, type SyntaxIssue, WaybackTimestampSchema, countTarget, createCatalogue, getCategory, getTarget, kindOf, listCategories, parseEntry, parseManifest, waybackUrl };
|
|
103
|
+
export { type Archive, type ArchiveMember, ArchiveMemberSchema, ArchiveSchema, type Catalogue, type CatalogueIndex, CatalogueIndexSchema, type Category, type CategoryId, CategoryIdSchema, type CategoryTarget, type CategoryTargets, type Citation, CitationSchema, type CountedKey, type CrosswalkReference, CrosswalkReferenceSchema, type Dataset, type DatasetQuery, type Entry, type EntryNames, EntrySchema, type GraphEdge, GraphEdgeSchema, type GraphNode, GraphNodeSchema, type GraphV1, GraphV1Schema, HttpUrlSchema, IdSchema, type JsonScalar, JsonScalarSchema, type Licence, LicenceSchema, type Lineage, LineageSchema, type Manifest, ManifestSchema, type Namespace, NamespaceSchema, type Parser, ParserSchema, RESERVED_META_KEYS, type Recipe, RecipeSchema, type RecipeStep, RelativePathSchema, SemVerSchema, type Snapshot, SnapshotSchema, type Source, SourceSchema, type SyntaxIssue, WaybackTimestampSchema, countTarget, createCatalogue, getCategory, getTarget, kindOf, listCategories, parseEntry, parseManifest, waybackUrl };
|
package/dist/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { A as
|
|
1
|
+
import { A as toIssues, B as GraphV1Schema, C as RESERVED_META_KEYS, D as SourceSchema, E as SnapshotSchema, F as getTarget, G as DatasetNotFoundError, H as ArchiveError, I as kindOf, J as RecipeError, K as KeepsetError, L as listCategories, M as CategoryIdSchema, N as countTarget, O as WaybackTimestampSchema, P as getCategory, R as GraphEdgeSchema, S as ParserSchema, T as RelativePathSchema, U as CatalogueSyntaxError, V as JsonScalarSchema, W as DatasetAmbiguousError, Y as UnknownCategoryError, _ as EntrySchema, a as createCatalogue, b as LicenceSchema, c as namesOf, d as SemVerSchema, f as parseManifest, g as CrosswalkReferenceSchema, h as CitationSchema, i as CatalogueIndexSchema, j as waybackUrl, k as parseEntry, l as LineageSchema, m as ArchiveSchema, n as getManifest, p as ArchiveMemberSchema, q as NativeParseError, r as listDatasets, s as NAME_SEPARATOR, t as getDataset, u as ManifestSchema, v as HttpUrlSchema, w as RecipeSchema, x as NamespaceSchema, y as IdSchema, z as GraphNodeSchema } from "./bundled-DaXJpk0C.js";
|
|
2
2
|
import { validateCrosswalk } from "twindex";
|
|
3
3
|
//#region src/serialise.ts
|
|
4
4
|
/**
|
|
@@ -57,4 +57,4 @@ function validateDataset(dataset) {
|
|
|
57
57
|
return issues;
|
|
58
58
|
}
|
|
59
59
|
//#endregion
|
|
60
|
-
export { ArchiveError, ArchiveMemberSchema, ArchiveSchema, CatalogueIndexSchema, CatalogueSyntaxError, CategoryIdSchema, CitationSchema, CrosswalkReferenceSchema, DatasetNotFoundError, EntrySchema, GraphEdgeSchema, GraphNodeSchema, GraphV1Schema, HttpUrlSchema, IdSchema, JsonScalarSchema, KeepsetError, LicenceSchema, LineageSchema, ManifestSchema, NativeParseError, ParserSchema, RESERVED_META_KEYS, RecipeError, RecipeSchema, RelativePathSchema, SemVerSchema, SnapshotSchema, SourceSchema, UnknownCategoryError, WaybackTimestampSchema, countTarget, createCatalogue, getCategory, getDataset, getManifest, getTarget, kindOf, listCategories, listDatasets, parseEntry, parseManifest, serialise, validateDataset, waybackUrl };
|
|
60
|
+
export { ArchiveError, ArchiveMemberSchema, ArchiveSchema, CatalogueIndexSchema, CatalogueSyntaxError, CategoryIdSchema, CitationSchema, CrosswalkReferenceSchema, DatasetAmbiguousError, DatasetNotFoundError, EntrySchema, GraphEdgeSchema, GraphNodeSchema, GraphV1Schema, HttpUrlSchema, IdSchema, JsonScalarSchema, KeepsetError, LicenceSchema, LineageSchema, ManifestSchema, NAME_SEPARATOR, NamespaceSchema, NativeParseError, ParserSchema, RESERVED_META_KEYS, RecipeError, RecipeSchema, RelativePathSchema, SemVerSchema, SnapshotSchema, SourceSchema, UnknownCategoryError, WaybackTimestampSchema, countTarget, createCatalogue, getCategory, getDataset, getManifest, getTarget, kindOf, listCategories, listDatasets, namesOf, parseEntry, parseManifest, serialise, validateDataset, waybackUrl };
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { B as Entry, C as Dataset,
|
|
1
|
+
import { B as Entry, C as Dataset, et as Snapshot, i as CategoryTarget, nt as Source } from "../categories-BSJj6C-y.js";
|
|
2
2
|
//#region src/node/fetch-dataset.d.ts
|
|
3
3
|
/** The network port a recipe downloads through: `downloadRaw` by default, which returns the bytes exactly as served, or a stand-in under test. */
|
|
4
4
|
export type FetchPort = (url: string, init: {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { F as getTarget, H as ArchiveError, J as RecipeError, K as KeepsetError, N as countTarget, o as getOwn, q as NativeParseError, t as getDataset } from "../bundled-DaXJpk0C.js";
|
|
2
2
|
import { RoundTripMismatchError, assertRoundTrip, encode } from "twindex";
|
|
3
3
|
import { createHash } from "node:crypto";
|
|
4
4
|
import { gunzipSync } from "node:zlib";
|
package/dist-cli/keepset.mjs
CHANGED
|
@@ -1,33 +1,42 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { writeFileSync } from "node:fs";
|
|
3
|
-
import { KeepsetError, countTarget, getDataset, listDatasets, validateDataset } from "keepset";
|
|
3
|
+
import { KeepsetError, NAME_SEPARATOR, countTarget, getDataset, listDatasets, validateDataset } from "keepset";
|
|
4
4
|
import { fetchDataset, roundTripDataset } from "keepset/node";
|
|
5
5
|
import { parseArgs } from "node:util";
|
|
6
6
|
//#region package.json
|
|
7
|
-
var version = "0.
|
|
7
|
+
var version = "0.11.0";
|
|
8
8
|
const usage = `Usage: keepset <command> [options]
|
|
9
9
|
|
|
10
10
|
Commands:
|
|
11
|
-
search [text] [--kind <kind>] [--domain <domain>] [--
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
11
|
+
search [text] [--kind <kind>] [--domain <domain>] [--namespace <path>]
|
|
12
|
+
[--category <id>] [--json]
|
|
13
|
+
list the datasets whose id, alias, namespace, name,
|
|
14
|
+
description, domain or tags contain the text, filtered by
|
|
15
|
+
kind, domain, namespace (and everything beneath it) or
|
|
16
|
+
category
|
|
17
|
+
show <dataset> [--json]
|
|
18
|
+
print a dataset's metadata, recipe and crosswalk
|
|
19
|
+
fetch <dataset> [--out <file>]
|
|
16
20
|
run the dataset's recipe and write the normalised result
|
|
17
21
|
to <file> (default <id>.json), with its provenance beside
|
|
18
22
|
it in <file without .json>.provenance.json
|
|
19
|
-
validate <
|
|
23
|
+
validate <dataset> check the entry against the entry schema and its
|
|
20
24
|
crosswalk against twindex's validation
|
|
21
|
-
roundtrip <
|
|
25
|
+
roundtrip <dataset> fetch the dataset and check that decoding the encoded
|
|
22
26
|
native document gives it back
|
|
23
27
|
|
|
28
|
+
A <dataset> is an entry id, an alias, or a qualified name such as
|
|
29
|
+
suitesparse/gset/g53, written with / between the namespace and the name. Any
|
|
30
|
+
leading part of the namespace may be dropped (gset/g53, g53) while the name
|
|
31
|
+
still means one dataset; an entry's own id always means that entry.
|
|
32
|
+
|
|
24
33
|
Options:
|
|
25
34
|
--help, -h print this help
|
|
26
35
|
--version, -v print the version
|
|
27
36
|
|
|
28
37
|
Exit status: 0 for a result, a valid entry or a round trip that holds; 1 for an
|
|
29
38
|
invalid entry or a round trip that does not hold; 2 when there is no answer:
|
|
30
|
-
bad usage, an unknown dataset or a recipe that failed.
|
|
39
|
+
bad usage, an unknown or ambiguous dataset or a recipe that failed.
|
|
31
40
|
`;
|
|
32
41
|
function parseCommandLine(args) {
|
|
33
42
|
return parseArgs({
|
|
@@ -37,6 +46,7 @@ function parseCommandLine(args) {
|
|
|
37
46
|
options: {
|
|
38
47
|
kind: { type: "string" },
|
|
39
48
|
domain: { type: "string" },
|
|
49
|
+
namespace: { type: "string" },
|
|
40
50
|
category: { type: "string" },
|
|
41
51
|
json: { type: "boolean" },
|
|
42
52
|
out: { type: "string" },
|
|
@@ -77,6 +87,8 @@ function formatShow(dataset) {
|
|
|
77
87
|
const lines = [`${entry.name} (${entry.id})`];
|
|
78
88
|
if (entry.description !== void 0) lines.push("", entry.description);
|
|
79
89
|
lines.push("", `category: ${entry.category} (kind ${dataset.kind})`, `catalogue: ${dataset.catalogue.id} ${dataset.catalogue.version}`);
|
|
90
|
+
if (entry.namespace !== void 0) lines.push(`namespace: ${entry.namespace.join(NAME_SEPARATOR)}`);
|
|
91
|
+
if (entry.aliases !== void 0) lines.push(`aliases: ${entry.aliases.join(", ")}`);
|
|
80
92
|
if (entry.domain !== void 0) lines.push(`domain: ${entry.domain}`);
|
|
81
93
|
if (entry.tags.length > 0) lines.push(`tags: ${entry.tags.join(", ")}`);
|
|
82
94
|
const { licence } = entry;
|
|
@@ -164,13 +176,14 @@ async function run(args, library, io) {
|
|
|
164
176
|
...text === void 0 ? {} : { text },
|
|
165
177
|
...values.kind === void 0 ? {} : { kind: values.kind },
|
|
166
178
|
...values.domain === void 0 ? {} : { domain: values.domain },
|
|
179
|
+
...values.namespace === void 0 ? {} : { namespace: values.namespace },
|
|
167
180
|
...values.category === void 0 ? {} : { category: values.category }
|
|
168
181
|
};
|
|
169
182
|
const datasets = library.listDatasets(query);
|
|
170
183
|
io.writeStdout(values.json === true ? json(datasets.map((dataset) => dataset.entry)) : formatSearch(datasets));
|
|
171
184
|
return 0;
|
|
172
185
|
}
|
|
173
|
-
const [
|
|
186
|
+
const [name, ...extra] = rest;
|
|
174
187
|
if (command === void 0) return reject("expected a command");
|
|
175
188
|
if (![
|
|
176
189
|
"show",
|
|
@@ -178,14 +191,15 @@ async function run(args, library, io) {
|
|
|
178
191
|
"validate",
|
|
179
192
|
"roundtrip"
|
|
180
193
|
].includes(command)) return reject(`unknown command ${command}`);
|
|
181
|
-
if (
|
|
194
|
+
if (name === void 0 || extra.length > 0) return reject(`${command} takes exactly one dataset`);
|
|
195
|
+
const dataset = library.getDataset(name);
|
|
196
|
+
const { id } = dataset.entry;
|
|
182
197
|
if (command === "show") {
|
|
183
|
-
const dataset = library.getDataset(id);
|
|
184
198
|
io.writeStdout(values.json === true ? json(dataset) : formatShow(dataset));
|
|
185
199
|
return 0;
|
|
186
200
|
}
|
|
187
201
|
if (command === "validate") {
|
|
188
|
-
const issues = library.validateDataset(
|
|
202
|
+
const issues = library.validateDataset(dataset);
|
|
189
203
|
if (issues.length > 0) {
|
|
190
204
|
io.writeStdout(formatIssues(issues));
|
|
191
205
|
return 1;
|
package/package.json
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"$defs":{"Crosswalk":{"$ref":"https://cdn.jsdelivr.net/npm/twindex@0.1.1/schemas/twindex.schema.json"},"Entry":{"additionalProperties":false,"properties":{"category":{"enum":["graph/v1"],"type":"string"},"citations":{"items":{"additionalProperties":false,"properties":{"doi":{"pattern":"^10\\.\\d{4,9}\\/\\S+$","type":"string"},"text":{"minLength":1,"type":"string"},"url":{"format":"uri","type":"string"}},"required":["text"],"type":"object"},"type":"array"},"crosswalk":{"anyOf":[{"$ref":"#/$defs/Crosswalk"},{"additionalProperties":false,"properties":{"file":{"type":"string"}},"required":["file"],"type":"object"}]},"description":{"minLength":1,"type":"string"},"domain":{"minLength":1,"type":"string"},"expectedCounts":{"additionalProperties":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"propertyNames":{"pattern":"^[A-Za-z][A-Za-z0-9]*$","type":"string"},"type":"object"},"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"licence":{"oneOf":[{"additionalProperties":false,"properties":{"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9.+-]*$","type":"string"},"kind":{"const":"spdx","type":"string"}},"required":["kind","id"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"custom","type":"string"},"name":{"minLength":1,"type":"string"},"url":{"format":"uri","type":"string"}},"required":["kind","name","url"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"unrecorded","type":"string"}},"required":["kind"],"type":"object"}]},"name":{"minLength":1,"type":"string"},"properties":{"additionalProperties":{"type":["string","number","boolean","null"]},"propertyNames":{"pattern":"^[A-Za-z][A-Za-z0-9]*$","type":"string"},"type":"object"},"recipe":{"oneOf":[{"additionalProperties":false,"properties":{"archive":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"none","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"gzip","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"zip","type":"string"},"member":{"anyOf":[{"additionalProperties":false,"properties":{"path":{"type":"string"}},"required":["path"],"type":"object"},{"additionalProperties":false,"properties":{"extensions":{"items":{"pattern":"^\\.[A-Za-z0-9][A-Za-z0-9.]*$","type":"string"},"minItems":1,"type":"array"}},"required":["extensions"],"type":"object"}]}},"required":["kind","member"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"tar","type":"string"},"member":{"anyOf":[{"additionalProperties":false,"properties":{"path":{"type":"string"}},"required":["path"],"type":"object"},{"additionalProperties":false,"properties":{"extensions":{"items":{"pattern":"^\\.[A-Za-z0-9][A-Za-z0-9.]*$","type":"string"},"minItems":1,"type":"array"}},"required":["extensions"],"type":"object"}]}},"required":["kind","member"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"tar.gz","type":"string"},"member":{"anyOf":[{"additionalProperties":false,"properties":{"path":{"type":"string"}},"required":["path"],"type":"object"},{"additionalProperties":false,"properties":{"extensions":{"items":{"pattern":"^\\.[A-Za-z0-9][A-Za-z0-9.]*$","type":"string"},"minItems":1,"type":"array"}},"required":["extensions"],"type":"object"}]}},"required":["kind","member"],"type":"object"}]},"parser":{"oneOf":[{"additionalProperties":false,"properties":{"commentPrefixes":{"items":{"minLength":1,"type":"string"},"type":"array"},"delimiter":{"enum":["detect","whitespace","tab","comma","space"],"type":"string"},"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"edge-list","type":"string"},"skipLines":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"sourceColumn":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"targetColumn":{"maximum":9007199254740991,"minimum":0,"type":"integer"}},"required":["format","encoding","delimiter","skipLines","commentPrefixes","sourceColumn","targetColumn"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"matrix-market","type":"string"}},"required":["format","encoding"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"gml","type":"string"}},"required":["format","encoding"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"pajek","type":"string"}},"required":["format","encoding"],"type":"object"}]},"source":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"status":{"const":"fetchable","type":"string"}},"required":["status","source","archive","parser"],"type":"object"},{"additionalProperties":false,"properties":{"reason":{"minLength":1,"type":"string"},"status":{"const":"not-fetchable","type":"string"}},"required":["status","reason"],"type":"object"}]},"sources":{"items":{"additionalProperties":false,"properties":{"retrieved":{"format":"date","pattern":"^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))$","type":"string"},"snapshot":{"additionalProperties":false,"properties":{"timestamp":{"pattern":"^\\d{8}(?:\\d{6})?$","type":"string"},"url":{"format":"uri","type":"string"}},"required":["url","timestamp"],"type":"object"},"status":{"enum":["live","dead"],"type":"string"},"url":{"format":"uri","type":"string"}},"required":["url","retrieved","status"],"type":"object"},"minItems":1,"type":"array"},"tags":{"items":{"minLength":1,"type":"string"},"type":"array"}},"required":["id","name","tags","category","sources","licence","citations","recipe","crosswalk","expectedCounts","properties"],"type":"object"}},"$id":"https://cdn.jsdelivr.net/npm/keepset@0.
|
|
1
|
+
{"$defs":{"Crosswalk":{"$ref":"https://cdn.jsdelivr.net/npm/twindex@0.1.1/schemas/twindex.schema.json"},"Entry":{"additionalProperties":false,"properties":{"aliases":{"items":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"minItems":1,"type":"array"},"category":{"enum":["graph/v1"],"type":"string"},"citations":{"items":{"additionalProperties":false,"properties":{"doi":{"pattern":"^10\\.\\d{4,9}\\/\\S+$","type":"string"},"text":{"minLength":1,"type":"string"},"url":{"format":"uri","type":"string"}},"required":["text"],"type":"object"},"type":"array"},"crosswalk":{"anyOf":[{"$ref":"#/$defs/Crosswalk"},{"additionalProperties":false,"properties":{"file":{"type":"string"}},"required":["file"],"type":"object"}]},"description":{"minLength":1,"type":"string"},"domain":{"minLength":1,"type":"string"},"expectedCounts":{"additionalProperties":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"propertyNames":{"pattern":"^[A-Za-z][A-Za-z0-9]*$","type":"string"},"type":"object"},"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"licence":{"oneOf":[{"additionalProperties":false,"properties":{"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9.+-]*$","type":"string"},"kind":{"const":"spdx","type":"string"}},"required":["kind","id"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"custom","type":"string"},"name":{"minLength":1,"type":"string"},"url":{"format":"uri","type":"string"}},"required":["kind","name","url"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"unrecorded","type":"string"}},"required":["kind"],"type":"object"}]},"name":{"minLength":1,"type":"string"},"namespace":{"items":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"minItems":1,"type":"array"},"properties":{"additionalProperties":{"type":["string","number","boolean","null"]},"propertyNames":{"pattern":"^[A-Za-z][A-Za-z0-9]*$","type":"string"},"type":"object"},"recipe":{"oneOf":[{"additionalProperties":false,"properties":{"archive":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"none","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"gzip","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"zip","type":"string"},"member":{"anyOf":[{"additionalProperties":false,"properties":{"path":{"type":"string"}},"required":["path"],"type":"object"},{"additionalProperties":false,"properties":{"extensions":{"items":{"pattern":"^\\.[A-Za-z0-9][A-Za-z0-9.]*$","type":"string"},"minItems":1,"type":"array"}},"required":["extensions"],"type":"object"}]}},"required":["kind","member"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"tar","type":"string"},"member":{"anyOf":[{"additionalProperties":false,"properties":{"path":{"type":"string"}},"required":["path"],"type":"object"},{"additionalProperties":false,"properties":{"extensions":{"items":{"pattern":"^\\.[A-Za-z0-9][A-Za-z0-9.]*$","type":"string"},"minItems":1,"type":"array"}},"required":["extensions"],"type":"object"}]}},"required":["kind","member"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"tar.gz","type":"string"},"member":{"anyOf":[{"additionalProperties":false,"properties":{"path":{"type":"string"}},"required":["path"],"type":"object"},{"additionalProperties":false,"properties":{"extensions":{"items":{"pattern":"^\\.[A-Za-z0-9][A-Za-z0-9.]*$","type":"string"},"minItems":1,"type":"array"}},"required":["extensions"],"type":"object"}]}},"required":["kind","member"],"type":"object"}]},"parser":{"oneOf":[{"additionalProperties":false,"properties":{"commentPrefixes":{"items":{"minLength":1,"type":"string"},"type":"array"},"delimiter":{"enum":["detect","whitespace","tab","comma","space"],"type":"string"},"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"edge-list","type":"string"},"skipLines":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"sourceColumn":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"targetColumn":{"maximum":9007199254740991,"minimum":0,"type":"integer"}},"required":["format","encoding","delimiter","skipLines","commentPrefixes","sourceColumn","targetColumn"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"matrix-market","type":"string"}},"required":["format","encoding"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"gml","type":"string"}},"required":["format","encoding"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"pajek","type":"string"}},"required":["format","encoding"],"type":"object"}]},"source":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"status":{"const":"fetchable","type":"string"}},"required":["status","source","archive","parser"],"type":"object"},{"additionalProperties":false,"properties":{"reason":{"minLength":1,"type":"string"},"status":{"const":"not-fetchable","type":"string"}},"required":["status","reason"],"type":"object"}]},"sources":{"items":{"additionalProperties":false,"properties":{"retrieved":{"format":"date","pattern":"^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))$","type":"string"},"snapshot":{"additionalProperties":false,"properties":{"timestamp":{"pattern":"^\\d{8}(?:\\d{6})?$","type":"string"},"url":{"format":"uri","type":"string"}},"required":["url","timestamp"],"type":"object"},"status":{"enum":["live","dead"],"type":"string"},"url":{"format":"uri","type":"string"}},"required":["url","retrieved","status"],"type":"object"},"minItems":1,"type":"array"},"tags":{"items":{"minLength":1,"type":"string"},"type":"array"}},"required":["id","name","tags","category","sources","licence","citations","recipe","crosswalk","expectedCounts","properties"],"type":"object"}},"$id":"https://cdn.jsdelivr.net/npm/keepset@0.11.0/schemas/entry.schema.json","$ref":"#/$defs/Entry","$schema":"https://json-schema.org/draft/2020-12/schema"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"$defs":{"Manifest":{"additionalProperties":false,"properties":{"description":{"minLength":1,"type":"string"},"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"lineage":{"additionalProperties":false,"properties":{"catalogue":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"repository":{"format":"uri","type":"string"},"version":{"pattern":"^(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)(?:-[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?(?:\\+[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?$","type":"string"}},"required":["catalogue","version","repository"],"type":"object"},"name":{"minLength":1,"type":"string"},"repository":{"format":"uri","type":"string"},"version":{"pattern":"^(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)(?:-[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?(?:\\+[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?$","type":"string"}},"required":["id","name","description","version","repository"],"type":"object"}},"$id":"https://cdn.jsdelivr.net/npm/keepset@0.
|
|
1
|
+
{"$defs":{"Manifest":{"additionalProperties":false,"properties":{"description":{"minLength":1,"type":"string"},"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"lineage":{"additionalProperties":false,"properties":{"catalogue":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"repository":{"format":"uri","type":"string"},"version":{"pattern":"^(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)(?:-[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?(?:\\+[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?$","type":"string"}},"required":["catalogue","version","repository"],"type":"object"},"name":{"minLength":1,"type":"string"},"repository":{"format":"uri","type":"string"},"version":{"pattern":"^(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)(?:-[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?(?:\\+[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?$","type":"string"}},"required":["id","name","description","version","repository"],"type":"object"}},"$id":"https://cdn.jsdelivr.net/npm/keepset@0.11.0/schemas/manifest.schema.json","$ref":"#/$defs/Manifest","$schema":"https://json-schema.org/draft/2020-12/schema"}
|