keepset 0.18.0 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{bundled-BiuG_xLd.js → bundled-8oHQPxcd.js} +1070 -53
- package/dist/{categories-CiJbssql.d.ts → categories-tH53cHmu.d.ts} +94 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/node/fetch-dataset.d.ts +1 -1
- package/dist/node/fetch-dataset.js +191 -1
- package/dist-cli/keepset.mjs +1 -1
- package/package.json +1 -1
- package/schemas/entry.schema.json +1 -1
- package/schemas/manifest.schema.json +1 -1
|
@@ -154,12 +154,26 @@ declare const ParserSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
154
154
|
factor: z.ZodNumber;
|
|
155
155
|
}, z.core.$strict>>;
|
|
156
156
|
columns: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
157
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
158
|
+
format: z.ZodLiteral<"geojson">;
|
|
159
|
+
encoding: z.ZodLiteral<"utf-8">;
|
|
160
|
+
idColumn: z.ZodNullable<z.ZodString>;
|
|
161
|
+
numericColumns: z.ZodArray<z.ZodObject<{
|
|
162
|
+
column: z.ZodString;
|
|
163
|
+
factor: z.ZodNumber;
|
|
164
|
+
}, z.core.$strict>>;
|
|
165
|
+
dateColumns: z.ZodArray<z.ZodString>;
|
|
166
|
+
columns: z.ZodArray<z.ZodString>;
|
|
157
167
|
}, z.core.$strict>], "format">;
|
|
158
168
|
type Parser = z.infer<typeof ParserSchema>;
|
|
159
169
|
/** How to obtain the dataset: a fetchable recipe naming the source (by index into `sources`), the archive layout and the parser, or a statement that no declarative recipe exists yet and why. */
|
|
160
170
|
declare const RecipeSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
161
171
|
status: z.ZodLiteral<"fetchable">;
|
|
162
172
|
source: z.ZodInt;
|
|
173
|
+
paging: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
174
|
+
kind: z.ZodLiteral<"arcgis-query">;
|
|
175
|
+
orderBy: z.ZodString;
|
|
176
|
+
}, z.core.$strict>], "kind">>;
|
|
163
177
|
archive: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
164
178
|
kind: z.ZodLiteral<"none">;
|
|
165
179
|
}, z.core.$strict>, z.ZodObject<{
|
|
@@ -239,6 +253,16 @@ declare const RecipeSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
239
253
|
factor: z.ZodNumber;
|
|
240
254
|
}, z.core.$strict>>;
|
|
241
255
|
columns: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
256
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
257
|
+
format: z.ZodLiteral<"geojson">;
|
|
258
|
+
encoding: z.ZodLiteral<"utf-8">;
|
|
259
|
+
idColumn: z.ZodNullable<z.ZodString>;
|
|
260
|
+
numericColumns: z.ZodArray<z.ZodObject<{
|
|
261
|
+
column: z.ZodString;
|
|
262
|
+
factor: z.ZodNumber;
|
|
263
|
+
}, z.core.$strict>>;
|
|
264
|
+
dateColumns: z.ZodArray<z.ZodString>;
|
|
265
|
+
columns: z.ZodArray<z.ZodString>;
|
|
242
266
|
}, z.core.$strict>], "format">;
|
|
243
267
|
}, z.core.$strict>, z.ZodObject<{
|
|
244
268
|
status: z.ZodLiteral<"not-fetchable">;
|
|
@@ -299,6 +323,10 @@ declare const EntrySchema: z.ZodObject<{
|
|
|
299
323
|
recipe: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
300
324
|
status: z.ZodLiteral<"fetchable">;
|
|
301
325
|
source: z.ZodInt;
|
|
326
|
+
paging: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
327
|
+
kind: z.ZodLiteral<"arcgis-query">;
|
|
328
|
+
orderBy: z.ZodString;
|
|
329
|
+
}, z.core.$strict>], "kind">>;
|
|
302
330
|
archive: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
303
331
|
kind: z.ZodLiteral<"none">;
|
|
304
332
|
}, z.core.$strict>, z.ZodObject<{
|
|
@@ -378,6 +406,16 @@ declare const EntrySchema: z.ZodObject<{
|
|
|
378
406
|
factor: z.ZodNumber;
|
|
379
407
|
}, z.core.$strict>>;
|
|
380
408
|
columns: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
409
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
410
|
+
format: z.ZodLiteral<"geojson">;
|
|
411
|
+
encoding: z.ZodLiteral<"utf-8">;
|
|
412
|
+
idColumn: z.ZodNullable<z.ZodString>;
|
|
413
|
+
numericColumns: z.ZodArray<z.ZodObject<{
|
|
414
|
+
column: z.ZodString;
|
|
415
|
+
factor: z.ZodNumber;
|
|
416
|
+
}, z.core.$strict>>;
|
|
417
|
+
dateColumns: z.ZodArray<z.ZodString>;
|
|
418
|
+
columns: z.ZodArray<z.ZodString>;
|
|
381
419
|
}, z.core.$strict>], "format">;
|
|
382
420
|
}, z.core.$strict>, z.ZodObject<{
|
|
383
421
|
status: z.ZodLiteral<"not-fetchable">;
|
|
@@ -468,6 +506,16 @@ declare const EntrySchema: z.ZodObject<{
|
|
|
468
506
|
factor: number;
|
|
469
507
|
}[];
|
|
470
508
|
columns?: string[] | undefined;
|
|
509
|
+
} | {
|
|
510
|
+
format: "geojson";
|
|
511
|
+
encoding: "utf-8";
|
|
512
|
+
idColumn: string | null;
|
|
513
|
+
numericColumns: {
|
|
514
|
+
column: string;
|
|
515
|
+
factor: number;
|
|
516
|
+
}[];
|
|
517
|
+
dateColumns: string[];
|
|
518
|
+
columns: string[];
|
|
471
519
|
} | {
|
|
472
520
|
format: "matrix-market";
|
|
473
521
|
encoding: "utf-8" | "latin1";
|
|
@@ -478,6 +526,10 @@ declare const EntrySchema: z.ZodObject<{
|
|
|
478
526
|
format: "pajek";
|
|
479
527
|
encoding: "utf-8" | "latin1";
|
|
480
528
|
};
|
|
529
|
+
paging?: {
|
|
530
|
+
kind: "arcgis-query";
|
|
531
|
+
orderBy: string;
|
|
532
|
+
} | undefined;
|
|
481
533
|
} | {
|
|
482
534
|
status: "not-fetchable";
|
|
483
535
|
reason: string;
|
|
@@ -628,6 +680,10 @@ declare const CatalogueIndexSchema: z.ZodObject<{
|
|
|
628
680
|
recipe: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
629
681
|
status: z.ZodLiteral<"fetchable">;
|
|
630
682
|
source: z.ZodInt;
|
|
683
|
+
paging: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
684
|
+
kind: z.ZodLiteral<"arcgis-query">;
|
|
685
|
+
orderBy: z.ZodString;
|
|
686
|
+
}, z.core.$strict>], "kind">>;
|
|
631
687
|
archive: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
632
688
|
kind: z.ZodLiteral<"none">;
|
|
633
689
|
}, z.core.$strict>, z.ZodObject<{
|
|
@@ -707,6 +763,16 @@ declare const CatalogueIndexSchema: z.ZodObject<{
|
|
|
707
763
|
factor: z.ZodNumber;
|
|
708
764
|
}, z.core.$strict>>;
|
|
709
765
|
columns: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
766
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
767
|
+
format: z.ZodLiteral<"geojson">;
|
|
768
|
+
encoding: z.ZodLiteral<"utf-8">;
|
|
769
|
+
idColumn: z.ZodNullable<z.ZodString>;
|
|
770
|
+
numericColumns: z.ZodArray<z.ZodObject<{
|
|
771
|
+
column: z.ZodString;
|
|
772
|
+
factor: z.ZodNumber;
|
|
773
|
+
}, z.core.$strict>>;
|
|
774
|
+
dateColumns: z.ZodArray<z.ZodString>;
|
|
775
|
+
columns: z.ZodArray<z.ZodString>;
|
|
710
776
|
}, z.core.$strict>], "format">;
|
|
711
777
|
}, z.core.$strict>, z.ZodObject<{
|
|
712
778
|
status: z.ZodLiteral<"not-fetchable">;
|
|
@@ -797,6 +863,16 @@ declare const CatalogueIndexSchema: z.ZodObject<{
|
|
|
797
863
|
factor: number;
|
|
798
864
|
}[];
|
|
799
865
|
columns?: string[] | undefined;
|
|
866
|
+
} | {
|
|
867
|
+
format: "geojson";
|
|
868
|
+
encoding: "utf-8";
|
|
869
|
+
idColumn: string | null;
|
|
870
|
+
numericColumns: {
|
|
871
|
+
column: string;
|
|
872
|
+
factor: number;
|
|
873
|
+
}[];
|
|
874
|
+
dateColumns: string[];
|
|
875
|
+
columns: string[];
|
|
800
876
|
} | {
|
|
801
877
|
format: "matrix-market";
|
|
802
878
|
encoding: "utf-8" | "latin1";
|
|
@@ -807,6 +883,10 @@ declare const CatalogueIndexSchema: z.ZodObject<{
|
|
|
807
883
|
format: "pajek";
|
|
808
884
|
encoding: "utf-8" | "latin1";
|
|
809
885
|
};
|
|
886
|
+
paging?: {
|
|
887
|
+
kind: "arcgis-query";
|
|
888
|
+
orderBy: string;
|
|
889
|
+
} | undefined;
|
|
810
890
|
} | {
|
|
811
891
|
status: "not-fetchable";
|
|
812
892
|
reason: string;
|
|
@@ -915,6 +995,16 @@ declare const CatalogueIndexSchema: z.ZodObject<{
|
|
|
915
995
|
factor: number;
|
|
916
996
|
}[];
|
|
917
997
|
columns?: string[] | undefined;
|
|
998
|
+
} | {
|
|
999
|
+
format: "geojson";
|
|
1000
|
+
encoding: "utf-8";
|
|
1001
|
+
idColumn: string | null;
|
|
1002
|
+
numericColumns: {
|
|
1003
|
+
column: string;
|
|
1004
|
+
factor: number;
|
|
1005
|
+
}[];
|
|
1006
|
+
dateColumns: string[];
|
|
1007
|
+
columns: string[];
|
|
918
1008
|
} | {
|
|
919
1009
|
format: "matrix-market";
|
|
920
1010
|
encoding: "utf-8" | "latin1";
|
|
@@ -925,6 +1015,10 @@ declare const CatalogueIndexSchema: z.ZodObject<{
|
|
|
925
1015
|
format: "pajek";
|
|
926
1016
|
encoding: "utf-8" | "latin1";
|
|
927
1017
|
};
|
|
1018
|
+
paging?: {
|
|
1019
|
+
kind: "arcgis-query";
|
|
1020
|
+
orderBy: string;
|
|
1021
|
+
} | undefined;
|
|
928
1022
|
} | {
|
|
929
1023
|
status: "not-fetchable";
|
|
930
1024
|
reason: string;
|
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { $ as Parser, A as LineageSchema, B as Citation, C as CatalogueIndex, D as NamespaceNode, E as DatasetQuery, F as AliasSchema, G as Entry, H as ColumnNameSchema, I as Archive, J as IdSchema, K as EntrySchema, L as ArchiveMember, M as ManifestSchema, N as SemVerSchema, O as createCatalogue, P as parseManifest, Q as NamespaceSchema, R as ArchiveMemberSchema, S as Catalogue, T as Dataset, U as CrosswalkReference, V as CitationSchema, W as CrosswalkReferenceSchema, X as LicenceSchema, Y as Licence, Z as Namespace, _ as GraphEdgeSchema, a as CategoryTargets, at as Snapshot, b as GraphV1, c as getCategory, ct as SourceSchema, d as listCategories, dt as waybackUrl, et as ParserSchema, f as OrNode, g as GraphEdge, h as OrV1Schema, i as CategoryTarget, it as RelativePathSchema, j as Manifest, k as Lineage, l as getTarget, lt as WaybackTimestampSchema, m as OrV1, n as CategoryId, nt as Recipe, o as CountedKey, ot as SnapshotSchema, p as OrNodeSchema, q as HttpUrlSchema, r as CategoryIdSchema, rt as RecipeSchema, s as countTarget, st as Source, t as Category, tt as RESERVED_META_KEYS, u as kindOf, ut as parseEntry, v as GraphNode, w as CatalogueIndexSchema, x as GraphV1Schema, y as GraphNodeSchema, z as ArchiveSchema } from "./categories-
|
|
1
|
+
import { $ as Parser, A as LineageSchema, B as Citation, C as CatalogueIndex, D as NamespaceNode, E as DatasetQuery, F as AliasSchema, G as Entry, H as ColumnNameSchema, I as Archive, J as IdSchema, K as EntrySchema, L as ArchiveMember, M as ManifestSchema, N as SemVerSchema, O as createCatalogue, P as parseManifest, Q as NamespaceSchema, R as ArchiveMemberSchema, S as Catalogue, T as Dataset, U as CrosswalkReference, V as CitationSchema, W as CrosswalkReferenceSchema, X as LicenceSchema, Y as Licence, Z as Namespace, _ as GraphEdgeSchema, a as CategoryTargets, at as Snapshot, b as GraphV1, c as getCategory, ct as SourceSchema, d as listCategories, dt as waybackUrl, et as ParserSchema, f as OrNode, g as GraphEdge, h as OrV1Schema, i as CategoryTarget, it as RelativePathSchema, j as Manifest, k as Lineage, l as getTarget, lt as WaybackTimestampSchema, m as OrV1, n as CategoryId, nt as Recipe, o as CountedKey, ot as SnapshotSchema, p as OrNodeSchema, q as HttpUrlSchema, r as CategoryIdSchema, rt as RecipeSchema, s as countTarget, st as Source, t as Category, tt as RESERVED_META_KEYS, u as kindOf, ut as parseEntry, v as GraphNode, w as CatalogueIndexSchema, x as GraphV1Schema, y as GraphNodeSchema, z as ArchiveSchema } from "./categories-tH53cHmu.js";
|
|
2
2
|
import { Crosswalk } from "twindex";
|
|
3
3
|
import { z } from "zod";
|
|
4
4
|
//#region src/errors.d.ts
|
package/dist/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { $ as NativeParseError, A as toIssues, B as kindOf, C as RESERVED_META_KEYS, D as SourceSchema, E as SnapshotSchema, G as GraphNodeSchema, H as OrNodeSchema, I as CategoryIdSchema, J as ArchiveError, K as GraphV1Schema, L as countTarget, M as NAME_SEPARATOR, N as groupsOf, O as WaybackTimestampSchema, P as namesOf, Q as KeepsetError, R as getCategory, S as ParserSchema, T as RelativePathSchema, U as OrV1Schema, V as listCategories, W as GraphEdgeSchema, X as DatasetAmbiguousError, Y as CatalogueSyntaxError, Z as DatasetNotFoundError, _ as EntrySchema, a as CatalogueIndexSchema, b as LicenceSchema, c as ManifestSchema, d as AliasSchema, et as RecipeError, f as ArchiveMemberSchema, g as CrosswalkReferenceSchema, h as ColumnNameSchema, i as listNamespaces, j as waybackUrl, k as parseEntry, l as SemVerSchema, m as CitationSchema, n as getManifest, o as createCatalogue, p as ArchiveSchema, q as JsonScalarSchema, r as listDatasets, s as LineageSchema, t as getDataset, tt as UnknownCategoryError, u as parseManifest, v as HttpUrlSchema, w as RecipeSchema, x as NamespaceSchema, y as IdSchema, z as getTarget } from "./bundled-
|
|
1
|
+
import { $ as NativeParseError, A as toIssues, B as kindOf, C as RESERVED_META_KEYS, D as SourceSchema, E as SnapshotSchema, G as GraphNodeSchema, H as OrNodeSchema, I as CategoryIdSchema, J as ArchiveError, K as GraphV1Schema, L as countTarget, M as NAME_SEPARATOR, N as groupsOf, O as WaybackTimestampSchema, P as namesOf, Q as KeepsetError, R as getCategory, S as ParserSchema, T as RelativePathSchema, U as OrV1Schema, V as listCategories, W as GraphEdgeSchema, X as DatasetAmbiguousError, Y as CatalogueSyntaxError, Z as DatasetNotFoundError, _ as EntrySchema, a as CatalogueIndexSchema, b as LicenceSchema, c as ManifestSchema, d as AliasSchema, et as RecipeError, f as ArchiveMemberSchema, g as CrosswalkReferenceSchema, h as ColumnNameSchema, i as listNamespaces, j as waybackUrl, k as parseEntry, l as SemVerSchema, m as CitationSchema, n as getManifest, o as createCatalogue, p as ArchiveSchema, q as JsonScalarSchema, r as listDatasets, s as LineageSchema, t as getDataset, tt as UnknownCategoryError, u as parseManifest, v as HttpUrlSchema, w as RecipeSchema, x as NamespaceSchema, y as IdSchema, z as getTarget } from "./bundled-8oHQPxcd.js";
|
|
2
2
|
import { validateCrosswalk } from "twindex";
|
|
3
3
|
//#region src/serialise.ts
|
|
4
4
|
/**
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { G as Entry, T as Dataset, at as Snapshot, i as CategoryTarget, st as Source } from "../categories-
|
|
1
|
+
import { G as Entry, T as Dataset, at as Snapshot, i as CategoryTarget, st as Source } from "../categories-tH53cHmu.js";
|
|
2
2
|
//#region src/node/fetch-dataset.d.ts
|
|
3
3
|
/** The network port a recipe downloads through: `downloadRaw` by default, which returns the bytes exactly as served, or a stand-in under test. */
|
|
4
4
|
export type FetchPort = (url: string, init: {
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import { $ as NativeParseError, F as getOwn, J as ArchiveError, L as countTarget, Q as KeepsetError, et as RecipeError, t as getDataset, z as getTarget } from "../bundled-
|
|
1
|
+
import { $ as NativeParseError, F as getOwn, J as ArchiveError, L as countTarget, Q as KeepsetError, et as RecipeError, t as getDataset, z as getTarget } from "../bundled-8oHQPxcd.js";
|
|
2
2
|
import { RoundTripMismatchError, assertRoundTrip, encode } from "twindex";
|
|
3
|
+
import { z } from "zod";
|
|
3
4
|
import { createHash } from "node:crypto";
|
|
4
5
|
import { gunzipSync } from "node:zlib";
|
|
5
6
|
import { unzipSync } from "fflate";
|
|
@@ -617,6 +618,106 @@ function parseCsvTable(text, options) {
|
|
|
617
618
|
return { nodes };
|
|
618
619
|
}
|
|
619
620
|
//#endregion
|
|
621
|
+
//#region src/parsers/geojson.ts
|
|
622
|
+
/** The columns a point feature's position fills, which no property may share a name with: the latitude and the longitude of a `Point` geometry, and null for a feature with any other geometry or none. */
|
|
623
|
+
const POINT_COLUMNS = {
|
|
624
|
+
latitude: "geometry_latitude",
|
|
625
|
+
longitude: "geometry_longitude"
|
|
626
|
+
};
|
|
627
|
+
const FeatureCollectionSchema = z.object({
|
|
628
|
+
type: z.literal("FeatureCollection"),
|
|
629
|
+
features: z.array(z.object({
|
|
630
|
+
geometry: z.object({
|
|
631
|
+
type: z.string(),
|
|
632
|
+
coordinates: z.unknown()
|
|
633
|
+
}).nullable(),
|
|
634
|
+
properties: z.record(z.string(), z.unknown()).nullable()
|
|
635
|
+
}))
|
|
636
|
+
});
|
|
637
|
+
/**
|
|
638
|
+
* The position of a point geometry.
|
|
639
|
+
* @param geometry - The feature's geometry.
|
|
640
|
+
* @returns The longitude and latitude of a `Point`, and undefined for any other geometry or none.
|
|
641
|
+
*/
|
|
642
|
+
function pointOf(geometry) {
|
|
643
|
+
const coordinates = z.array(z.unknown()).safeParse(geometry?.type === "Point" ? geometry.coordinates : void 0);
|
|
644
|
+
if (!coordinates.success) return;
|
|
645
|
+
const [longitude, latitude] = coordinates.data;
|
|
646
|
+
return typeof longitude === "number" && typeof latitude === "number" ? {
|
|
647
|
+
longitude,
|
|
648
|
+
latitude
|
|
649
|
+
} : void 0;
|
|
650
|
+
}
|
|
651
|
+
/**
|
|
652
|
+
* One cell of a feature, in the form its column is declared to take.
|
|
653
|
+
* @param value - The property's value, or undefined when the feature lacks it.
|
|
654
|
+
* @param column - The column, for the error.
|
|
655
|
+
* @param feature - The 1-based feature number, for the error.
|
|
656
|
+
* @param kind - How the column is read: as a number times a factor, as an epoch-millisecond date written in ISO 8601, or as text.
|
|
657
|
+
* @returns The cell: null for a missing, null or empty value.
|
|
658
|
+
* @throws NativeParseError when the value is not the kind the recipe declares.
|
|
659
|
+
*/
|
|
660
|
+
function cellOf(value, column, feature, kind) {
|
|
661
|
+
if (value === void 0 || value === null) return null;
|
|
662
|
+
if (kind === "date") {
|
|
663
|
+
if (typeof value !== "number" || !Number.isFinite(value)) throw new NativeParseError("geojson", feature, `column ${column} is declared a date but holds ${JSON.stringify(value)}`);
|
|
664
|
+
return new Date(value).toISOString();
|
|
665
|
+
}
|
|
666
|
+
if (kind !== "text") {
|
|
667
|
+
if (typeof value !== "number" || !Number.isFinite(value)) throw new NativeParseError("geojson", feature, `column ${column} is declared numeric but holds ${JSON.stringify(value)}`);
|
|
668
|
+
return value * kind.factor;
|
|
669
|
+
}
|
|
670
|
+
if (typeof value === "string") return value.trim() === "" ? null : value.trim();
|
|
671
|
+
if (typeof value === "number" || typeof value === "boolean") return String(value);
|
|
672
|
+
throw new NativeParseError("geojson", feature, `column ${column} holds a ${Array.isArray(value) ? "list" : "nested object"}, which a column cannot`);
|
|
673
|
+
}
|
|
674
|
+
/**
|
|
675
|
+
* Parses a GeoJSON feature collection into rows, one per feature in file order: the properties the recipe lists become columns, a point's position fills `geometry_latitude` and `geometry_longitude` when they are listed, and an id comes from the recipe's id column (not also a column) or the feature's 1-based number. A column declared numeric is a number times its factor, one declared a date is an epoch-millisecond number written as an ISO 8601 text, and any other is text (a number or flag is written as its text), an empty or missing value being null so every row carries every column.
|
|
676
|
+
* @param text - The file text.
|
|
677
|
+
* @param options - The recipe's parser.
|
|
678
|
+
* @returns The rows.
|
|
679
|
+
* @throws NativeParseError when the text is not a GeoJSON feature collection, a value is not the kind its column is declared, a listed column is also a point column of another name, or an id is empty.
|
|
680
|
+
*/
|
|
681
|
+
function parseGeojson(text, options) {
|
|
682
|
+
let document;
|
|
683
|
+
try {
|
|
684
|
+
document = JSON.parse(text);
|
|
685
|
+
} catch (error) {
|
|
686
|
+
throw new NativeParseError("geojson", 1, `the file is not JSON (${error instanceof Error ? error.message : String(error)})`);
|
|
687
|
+
}
|
|
688
|
+
const collection = FeatureCollectionSchema.safeParse(document);
|
|
689
|
+
if (!collection.success) throw new NativeParseError("geojson", 1, `the file is not a GeoJSON feature collection (${collection.error.issues.map((issue) => `${issue.path.map(String).join(".") || "(root)"}: ${issue.message}`).join("; ")})`);
|
|
690
|
+
const factors = new Map(options.numericColumns.map((numeric) => [numeric.column, numeric.factor]));
|
|
691
|
+
const dates = new Set(options.dateColumns);
|
|
692
|
+
const kinds = options.columns.map((column) => {
|
|
693
|
+
const factor = factors.get(column);
|
|
694
|
+
return {
|
|
695
|
+
column,
|
|
696
|
+
kind: factor === void 0 ? dates.has(column) ? "date" : "text" : { factor }
|
|
697
|
+
};
|
|
698
|
+
});
|
|
699
|
+
return { nodes: collection.data.features.map((feature, index) => {
|
|
700
|
+
const number = index + 1;
|
|
701
|
+
const properties = feature.properties ?? {};
|
|
702
|
+
const point = pointOf(feature.geometry);
|
|
703
|
+
const own = (column) => Object.hasOwn(properties, column) ? properties[column] : void 0;
|
|
704
|
+
const id = options.idColumn === null ? String(number) : cellOf(own(options.idColumn), options.idColumn, number, "text");
|
|
705
|
+
if (id === null) throw new NativeParseError("geojson", number, `the id column ${options.idColumn ?? ""} is empty`);
|
|
706
|
+
const cells = kinds.map(({ column, kind }) => {
|
|
707
|
+
if (column === POINT_COLUMNS.latitude || column === POINT_COLUMNS.longitude) {
|
|
708
|
+
if (Object.hasOwn(properties, column)) throw new NativeParseError("geojson", number, `a property is named ${column}, which is the column of the point's position`);
|
|
709
|
+
const position = column === POINT_COLUMNS.latitude ? point?.latitude : point?.longitude;
|
|
710
|
+
return [column, position === void 0 ? null : position * (typeof kind === "object" ? kind.factor : 1)];
|
|
711
|
+
}
|
|
712
|
+
return [column, cellOf(own(column), column, number, kind)];
|
|
713
|
+
});
|
|
714
|
+
return {
|
|
715
|
+
...Object.fromEntries(cells),
|
|
716
|
+
id: String(id)
|
|
717
|
+
};
|
|
718
|
+
}) };
|
|
719
|
+
}
|
|
720
|
+
//#endregion
|
|
620
721
|
//#region src/parsers/pajek.ts
|
|
621
722
|
/** Section keywords, lower-cased, mapped to how their lines are read. `*Network` names the network and starts no section. `*Partition` and `*Vector` hold node attributes this version does not carry, so their lines are skipped, as are the `*Vertices` lines inside them. */
|
|
622
723
|
const sections = /* @__PURE__ */ new Map([
|
|
@@ -780,6 +881,7 @@ function* decodePieces(bytes, parser, pieceBytes = DECODE_PIECE_BYTES) {
|
|
|
780
881
|
function parseNative(bytes, parser) {
|
|
781
882
|
if (parser.format === "csv-table") return parseCsvTable(decodePieces(bytes, parser), parser);
|
|
782
883
|
const text = decode(bytes, parser);
|
|
884
|
+
if (parser.format === "geojson") return parseGeojson(text, parser);
|
|
783
885
|
if (parser.format === "edge-list") return parseEdgeList(text, parser);
|
|
784
886
|
if (parser.format === "matrix-market") return parseMatrixMarket(text);
|
|
785
887
|
if (parser.format === "gml") return parseGml(text);
|
|
@@ -1082,6 +1184,93 @@ async function downloadRaw(url, init) {
|
|
|
1082
1184
|
return follow(new URL(url), init.signal, 0);
|
|
1083
1185
|
}
|
|
1084
1186
|
//#endregion
|
|
1187
|
+
//#region src/node/paged.ts
|
|
1188
|
+
const countResponse = (value) => typeof value === "object" && value !== null && "count" in value && typeof value.count === "number" && Number.isInteger(value.count) && value.count >= 0 ? value.count : void 0;
|
|
1189
|
+
/**
|
|
1190
|
+
* The error message an ArcGIS server returns in a 200 response, if the body is one.
|
|
1191
|
+
* @param value - A parsed response body.
|
|
1192
|
+
* @returns The message, or undefined when the body is not an error.
|
|
1193
|
+
*/
|
|
1194
|
+
function arcgisError(value) {
|
|
1195
|
+
if (typeof value !== "object" || value === null || !("error" in value)) return;
|
|
1196
|
+
const { error } = value;
|
|
1197
|
+
return typeof error === "object" && error !== null && "message" in error && typeof error.message === "string" ? error.message : "the server returned an error";
|
|
1198
|
+
}
|
|
1199
|
+
/**
|
|
1200
|
+
* Reads one JSON response of the layer's query endpoint.
|
|
1201
|
+
* @param port - The network port.
|
|
1202
|
+
* @param url - The request URL.
|
|
1203
|
+
* @param signal - Aborts the request.
|
|
1204
|
+
* @returns The parsed body.
|
|
1205
|
+
* @throws KeepsetError when the server does not answer successfully, the body is not JSON, or it is an ArcGIS error.
|
|
1206
|
+
*/
|
|
1207
|
+
async function getJson(port, url, signal) {
|
|
1208
|
+
const response = await port(url, signal === void 0 ? {} : { signal });
|
|
1209
|
+
if (!response.ok) throw new KeepsetError(`${url} answered ${String(response.status)} ${response.statusText}`);
|
|
1210
|
+
let body;
|
|
1211
|
+
try {
|
|
1212
|
+
body = JSON.parse(new TextDecoder("utf-8").decode(new Uint8Array(await response.arrayBuffer())));
|
|
1213
|
+
} catch (error) {
|
|
1214
|
+
throw new KeepsetError(`${url} did not answer JSON (${error instanceof Error ? error.message : String(error)})`);
|
|
1215
|
+
}
|
|
1216
|
+
const message = arcgisError(body);
|
|
1217
|
+
if (message !== void 0) throw new KeepsetError(`${url} answered an error: ${message}`);
|
|
1218
|
+
return body;
|
|
1219
|
+
}
|
|
1220
|
+
const FeatureListSchema = z.array(z.unknown());
|
|
1221
|
+
/**
|
|
1222
|
+
* Reads a layer's features page by page, each request starting where the features held so far end.
|
|
1223
|
+
* @param layer - The layer read.
|
|
1224
|
+
* @param held - The features read so far.
|
|
1225
|
+
* @returns Every feature of the layer.
|
|
1226
|
+
* @throws KeepsetError when a page is empty before the count is reached or the server answers an error.
|
|
1227
|
+
*/
|
|
1228
|
+
async function readPages(layer, held) {
|
|
1229
|
+
if (held.length >= layer.count) return held;
|
|
1230
|
+
const url = `${layer.query}?where=1%3D1&outFields=*&orderByFields=${encodeURIComponent(layer.orderBy)}&resultOffset=${String(held.length)}&f=geojson`;
|
|
1231
|
+
const page = await getJson(layer.port, url, layer.signal);
|
|
1232
|
+
const features = FeatureListSchema.safeParse(typeof page === "object" && page !== null && "features" in page ? page.features : void 0);
|
|
1233
|
+
if (!features.success || features.data.length === 0) throw new KeepsetError(`${layer.query} gave no features at offset ${String(held.length)}, before the ${String(layer.count)} it reports, so it does not page by offset`);
|
|
1234
|
+
return readPages(layer, [...held, ...features.data]);
|
|
1235
|
+
}
|
|
1236
|
+
/**
|
|
1237
|
+
* Downloads a whole ArcGIS feature layer as one GeoJSON feature collection: the layer's feature count is read first, then its pages in `orderBy` order by offset until every feature is held, so a server that ignores the offset is an error and not an endless loop.
|
|
1238
|
+
* @param layerUrl - The layer's URL, such as `.../FeatureServer/0`.
|
|
1239
|
+
* @param orderBy - The field the pages are ordered by.
|
|
1240
|
+
* @param port - The network port.
|
|
1241
|
+
* @param signal - Aborts the download.
|
|
1242
|
+
* @returns The bytes of one `FeatureCollection`, which is what the parser reads and what the provenance hashes.
|
|
1243
|
+
* @throws KeepsetError when the layer does not report a count, a page is empty before the count is reached, or the server answers an error.
|
|
1244
|
+
*/
|
|
1245
|
+
async function downloadArcgisLayer(layerUrl, orderBy, port, signal) {
|
|
1246
|
+
const query = `${layerUrl.replace(/\/+$/, "")}/query`;
|
|
1247
|
+
const count = countResponse(await getJson(port, `${query}?where=1%3D1&returnCountOnly=true&f=json`, signal));
|
|
1248
|
+
if (count === void 0) throw new KeepsetError(`${layerUrl} did not report a feature count`);
|
|
1249
|
+
const pages = await readPages({
|
|
1250
|
+
query,
|
|
1251
|
+
orderBy,
|
|
1252
|
+
count,
|
|
1253
|
+
port,
|
|
1254
|
+
signal
|
|
1255
|
+
}, []);
|
|
1256
|
+
return new TextEncoder().encode(JSON.stringify({
|
|
1257
|
+
type: "FeatureCollection",
|
|
1258
|
+
features: pages
|
|
1259
|
+
}));
|
|
1260
|
+
}
|
|
1261
|
+
/**
|
|
1262
|
+
* Downloads a paged source as the one file its pages make, in the way its recipe's paging says.
|
|
1263
|
+
* @param url - The source's URL.
|
|
1264
|
+
* @param paging - How the source is paged.
|
|
1265
|
+
* @param port - The network port.
|
|
1266
|
+
* @param signal - Aborts the download.
|
|
1267
|
+
* @returns The bytes the parser reads.
|
|
1268
|
+
* @throws KeepsetError when a page cannot be read or the pages do not add up to what the source reports.
|
|
1269
|
+
*/
|
|
1270
|
+
async function downloadPaged(url, paging, port, signal) {
|
|
1271
|
+
return downloadArcgisLayer(url, paging.orderBy, port, signal);
|
|
1272
|
+
}
|
|
1273
|
+
//#endregion
|
|
1085
1274
|
//#region src/node/fetch-dataset.ts
|
|
1086
1275
|
/**
|
|
1087
1276
|
* The URL a recipe downloads a source from. A dead source is fetched from its pinned snapshot in the Wayback Machine's `id_` form, which serves the archived bytes as they were captured rather than wrapped in the archive's page chrome.
|
|
@@ -1138,6 +1327,7 @@ async function runToNative(dataset, options) {
|
|
|
1138
1327
|
const fetchedUrl = downloadUrl(source);
|
|
1139
1328
|
const retrievedAt = now().toISOString();
|
|
1140
1329
|
const bytes = await step(entry.id, "download", async () => {
|
|
1330
|
+
if (recipe.paging !== void 0) return downloadPaged(fetchedUrl, recipe.paging, fetchPort, signal);
|
|
1141
1331
|
const response = await fetchPort(fetchedUrl, signal === void 0 ? {} : { signal });
|
|
1142
1332
|
if (!response.ok) throw new KeepsetError(`${fetchedUrl} answered ${String(response.status)} ${response.statusText}`);
|
|
1143
1333
|
return new Uint8Array(await response.arrayBuffer());
|
package/dist-cli/keepset.mjs
CHANGED
|
@@ -4,7 +4,7 @@ import { KeepsetError, NAME_SEPARATOR, countTarget, getDataset, listDatasets, li
|
|
|
4
4
|
import { fetchDataset, roundTripDataset } from "keepset/node";
|
|
5
5
|
import { parseArgs } from "node:util";
|
|
6
6
|
//#region package.json
|
|
7
|
-
var version = "0.
|
|
7
|
+
var version = "0.19.0";
|
|
8
8
|
const usage = `Usage: keepset <command> [options]
|
|
9
9
|
|
|
10
10
|
Commands:
|
package/package.json
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"$defs":{"Crosswalk":{"$ref":"https://cdn.jsdelivr.net/npm/twindex@0.1.4/schemas/twindex.schema.json"},"Entry":{"additionalProperties":false,"properties":{"aliases":{"items":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*(?:\\/[A-Za-z0-9][A-Za-z0-9._-]*)*$","type":"string"},"minItems":1,"type":"array"},"category":{"enum":["graph/v1","or/v1"],"type":"string"},"citations":{"items":{"additionalProperties":false,"properties":{"doi":{"pattern":"^10\\.\\d{4,9}\\/\\S+$","type":"string"},"text":{"minLength":1,"type":"string"},"url":{"format":"uri","type":"string"}},"required":["text"],"type":"object"},"type":"array"},"crosswalk":{"anyOf":[{"$ref":"#/$defs/Crosswalk"},{"additionalProperties":false,"properties":{"file":{"type":"string"}},"required":["file"],"type":"object"}]},"description":{"minLength":1,"type":"string"},"domain":{"minLength":1,"type":"string"},"expectedCounts":{"additionalProperties":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"propertyNames":{"pattern":"^[A-Za-z][A-Za-z0-9]*$","type":"string"},"type":"object"},"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"licence":{"oneOf":[{"additionalProperties":false,"properties":{"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9.+-]*$","type":"string"},"kind":{"const":"spdx","type":"string"}},"required":["kind","id"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"custom","type":"string"},"name":{"minLength":1,"type":"string"},"url":{"format":"uri","type":"string"}},"required":["kind","name","url"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"unrecorded","type":"string"}},"required":["kind"],"type":"object"}]},"name":{"minLength":1,"type":"string"},"namespace":{"items":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"minItems":1,"type":"array"},"properties":{"additionalProperties":{"type":["string","number","boolean","null"]},"propertyNames":{"pattern":"^[A-Za-z][A-Za-z0-9]*$","type":"string"},"type":"object"},"recipe":{"oneOf":[{"additionalProperties":false,"properties":{"archive":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"none","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"gzip","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"zip","type":"string"},"member":{"anyOf":[{"additionalProperties":false,"properties":{"path":{"type":"string"}},"required":["path"],"type":"object"},{"additionalProperties":false,"properties":{"extensions":{"items":{"pattern":"^\\.[A-Za-z0-9][A-Za-z0-9.]*$","type":"string"},"minItems":1,"type":"array"}},"required":["extensions"],"type":"object"}]}},"required":["kind","member"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"tar","type":"string"},"member":{"anyOf":[{"additionalProperties":false,"properties":{"path":{"type":"string"}},"required":["path"],"type":"object"},{"additionalProperties":false,"properties":{"extensions":{"items":{"pattern":"^\\.[A-Za-z0-9][A-Za-z0-9.]*$","type":"string"},"minItems":1,"type":"array"}},"required":["extensions"],"type":"object"}]}},"required":["kind","member"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"tar.gz","type":"string"},"member":{"anyOf":[{"additionalProperties":false,"properties":{"path":{"type":"string"}},"required":["path"],"type":"object"},{"additionalProperties":false,"properties":{"extensions":{"items":{"pattern":"^\\.[A-Za-z0-9][A-Za-z0-9.]*$","type":"string"},"minItems":1,"type":"array"}},"required":["extensions"],"type":"object"}]}},"required":["kind","member"],"type":"object"}]},"parser":{"oneOf":[{"additionalProperties":false,"properties":{"commentPrefixes":{"items":{"minLength":1,"type":"string"},"type":"array"},"delimiter":{"enum":["detect","whitespace","tab","comma","space"],"type":"string"},"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"edge-list","type":"string"},"skipLines":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"sourceColumn":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"targetColumn":{"maximum":9007199254740991,"minimum":0,"type":"integer"}},"required":["format","encoding","delimiter","skipLines","commentPrefixes","sourceColumn","targetColumn"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"matrix-market","type":"string"}},"required":["format","encoding"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"gml","type":"string"}},"required":["format","encoding"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"pajek","type":"string"}},"required":["format","encoding"],"type":"object"},{"additionalProperties":false,"properties":{"columns":{"items":{"pattern":"^[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?$","type":"string"},"type":"array"},"delimiter":{"enum":["comma","tab","semicolon"],"type":"string"},"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"csv-table","type":"string"},"idColumn":{"anyOf":[{"pattern":"^[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?$","type":"string"},{"type":"null"}]},"numericColumns":{"items":{"additionalProperties":false,"properties":{"column":{"pattern":"^[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?$","type":"string"},"factor":{"exclusiveMinimum":0,"type":"number"}},"required":["column","factor"],"type":"object"},"type":"array"},"skipLines":{"maximum":9007199254740991,"minimum":0,"type":"integer"}},"required":["format","encoding","delimiter","skipLines","idColumn","numericColumns"],"type":"object"}]},"source":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"status":{"const":"fetchable","type":"string"}},"required":["status","source","archive","parser"],"type":"object"},{"additionalProperties":false,"properties":{"reason":{"minLength":1,"type":"string"},"status":{"const":"not-fetchable","type":"string"}},"required":["status","reason"],"type":"object"}]},"sources":{"items":{"additionalProperties":false,"properties":{"retrieved":{"format":"date","pattern":"^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))$","type":"string"},"snapshot":{"additionalProperties":false,"properties":{"timestamp":{"pattern":"^\\d{8}(?:\\d{6})?$","type":"string"},"url":{"format":"uri","type":"string"}},"required":["url","timestamp"],"type":"object"},"status":{"enum":["live","dead"],"type":"string"},"url":{"format":"uri","type":"string"}},"required":["url","retrieved","status"],"type":"object"},"minItems":1,"type":"array"},"tags":{"items":{"minLength":1,"type":"string"},"type":"array"}},"required":["id","name","tags","category","sources","licence","citations","recipe","crosswalk","expectedCounts","properties"],"type":"object"}},"$id":"https://cdn.jsdelivr.net/npm/keepset@0.
|
|
1
|
+
{"$defs":{"Crosswalk":{"$ref":"https://cdn.jsdelivr.net/npm/twindex@0.1.4/schemas/twindex.schema.json"},"Entry":{"additionalProperties":false,"properties":{"aliases":{"items":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*(?:\\/[A-Za-z0-9][A-Za-z0-9._-]*)*$","type":"string"},"minItems":1,"type":"array"},"category":{"enum":["graph/v1","or/v1"],"type":"string"},"citations":{"items":{"additionalProperties":false,"properties":{"doi":{"pattern":"^10\\.\\d{4,9}\\/\\S+$","type":"string"},"text":{"minLength":1,"type":"string"},"url":{"format":"uri","type":"string"}},"required":["text"],"type":"object"},"type":"array"},"crosswalk":{"anyOf":[{"$ref":"#/$defs/Crosswalk"},{"additionalProperties":false,"properties":{"file":{"type":"string"}},"required":["file"],"type":"object"}]},"description":{"minLength":1,"type":"string"},"domain":{"minLength":1,"type":"string"},"expectedCounts":{"additionalProperties":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"propertyNames":{"pattern":"^[A-Za-z][A-Za-z0-9]*$","type":"string"},"type":"object"},"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"licence":{"oneOf":[{"additionalProperties":false,"properties":{"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9.+-]*$","type":"string"},"kind":{"const":"spdx","type":"string"}},"required":["kind","id"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"custom","type":"string"},"name":{"minLength":1,"type":"string"},"url":{"format":"uri","type":"string"}},"required":["kind","name","url"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"unrecorded","type":"string"}},"required":["kind"],"type":"object"}]},"name":{"minLength":1,"type":"string"},"namespace":{"items":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"minItems":1,"type":"array"},"properties":{"additionalProperties":{"type":["string","number","boolean","null"]},"propertyNames":{"pattern":"^[A-Za-z][A-Za-z0-9]*$","type":"string"},"type":"object"},"recipe":{"oneOf":[{"additionalProperties":false,"properties":{"archive":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"none","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"gzip","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"zip","type":"string"},"member":{"anyOf":[{"additionalProperties":false,"properties":{"path":{"type":"string"}},"required":["path"],"type":"object"},{"additionalProperties":false,"properties":{"extensions":{"items":{"pattern":"^\\.[A-Za-z0-9][A-Za-z0-9.]*$","type":"string"},"minItems":1,"type":"array"}},"required":["extensions"],"type":"object"}]}},"required":["kind","member"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"tar","type":"string"},"member":{"anyOf":[{"additionalProperties":false,"properties":{"path":{"type":"string"}},"required":["path"],"type":"object"},{"additionalProperties":false,"properties":{"extensions":{"items":{"pattern":"^\\.[A-Za-z0-9][A-Za-z0-9.]*$","type":"string"},"minItems":1,"type":"array"}},"required":["extensions"],"type":"object"}]}},"required":["kind","member"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"tar.gz","type":"string"},"member":{"anyOf":[{"additionalProperties":false,"properties":{"path":{"type":"string"}},"required":["path"],"type":"object"},{"additionalProperties":false,"properties":{"extensions":{"items":{"pattern":"^\\.[A-Za-z0-9][A-Za-z0-9.]*$","type":"string"},"minItems":1,"type":"array"}},"required":["extensions"],"type":"object"}]}},"required":["kind","member"],"type":"object"}]},"paging":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"arcgis-query","type":"string"},"orderBy":{"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"}},"required":["kind","orderBy"],"type":"object"}]},"parser":{"oneOf":[{"additionalProperties":false,"properties":{"commentPrefixes":{"items":{"minLength":1,"type":"string"},"type":"array"},"delimiter":{"enum":["detect","whitespace","tab","comma","space"],"type":"string"},"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"edge-list","type":"string"},"skipLines":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"sourceColumn":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"targetColumn":{"maximum":9007199254740991,"minimum":0,"type":"integer"}},"required":["format","encoding","delimiter","skipLines","commentPrefixes","sourceColumn","targetColumn"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"matrix-market","type":"string"}},"required":["format","encoding"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"gml","type":"string"}},"required":["format","encoding"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"pajek","type":"string"}},"required":["format","encoding"],"type":"object"},{"additionalProperties":false,"properties":{"columns":{"items":{"pattern":"^[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?$","type":"string"},"type":"array"},"delimiter":{"enum":["comma","tab","semicolon"],"type":"string"},"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"csv-table","type":"string"},"idColumn":{"anyOf":[{"pattern":"^[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?$","type":"string"},{"type":"null"}]},"numericColumns":{"items":{"additionalProperties":false,"properties":{"column":{"pattern":"^[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?$","type":"string"},"factor":{"exclusiveMinimum":0,"type":"number"}},"required":["column","factor"],"type":"object"},"type":"array"},"skipLines":{"maximum":9007199254740991,"minimum":0,"type":"integer"}},"required":["format","encoding","delimiter","skipLines","idColumn","numericColumns"],"type":"object"},{"additionalProperties":false,"properties":{"columns":{"items":{"pattern":"^[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?$","type":"string"},"minItems":1,"type":"array"},"dateColumns":{"items":{"pattern":"^[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?$","type":"string"},"type":"array"},"encoding":{"const":"utf-8","type":"string"},"format":{"const":"geojson","type":"string"},"idColumn":{"anyOf":[{"pattern":"^[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?$","type":"string"},{"type":"null"}]},"numericColumns":{"items":{"additionalProperties":false,"properties":{"column":{"pattern":"^[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?$","type":"string"},"factor":{"exclusiveMinimum":0,"type":"number"}},"required":["column","factor"],"type":"object"},"type":"array"}},"required":["format","encoding","idColumn","numericColumns","dateColumns","columns"],"type":"object"}]},"source":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"status":{"const":"fetchable","type":"string"}},"required":["status","source","archive","parser"],"type":"object"},{"additionalProperties":false,"properties":{"reason":{"minLength":1,"type":"string"},"status":{"const":"not-fetchable","type":"string"}},"required":["status","reason"],"type":"object"}]},"sources":{"items":{"additionalProperties":false,"properties":{"retrieved":{"format":"date","pattern":"^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))$","type":"string"},"snapshot":{"additionalProperties":false,"properties":{"timestamp":{"pattern":"^\\d{8}(?:\\d{6})?$","type":"string"},"url":{"format":"uri","type":"string"}},"required":["url","timestamp"],"type":"object"},"status":{"enum":["live","dead"],"type":"string"},"url":{"format":"uri","type":"string"}},"required":["url","retrieved","status"],"type":"object"},"minItems":1,"type":"array"},"tags":{"items":{"minLength":1,"type":"string"},"type":"array"}},"required":["id","name","tags","category","sources","licence","citations","recipe","crosswalk","expectedCounts","properties"],"type":"object"}},"$id":"https://cdn.jsdelivr.net/npm/keepset@0.19.0/schemas/entry.schema.json","$ref":"#/$defs/Entry","$schema":"https://json-schema.org/draft/2020-12/schema"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"$defs":{"Manifest":{"additionalProperties":false,"properties":{"description":{"minLength":1,"type":"string"},"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"lineage":{"additionalProperties":false,"properties":{"catalogue":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"repository":{"format":"uri","type":"string"},"version":{"pattern":"^(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)(?:-[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?(?:\\+[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?$","type":"string"}},"required":["catalogue","version","repository"],"type":"object"},"name":{"minLength":1,"type":"string"},"namespaceAliases":{"additionalProperties":{"items":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"minItems":1,"type":"array"},"propertyNames":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"type":"object"},"repository":{"format":"uri","type":"string"},"version":{"pattern":"^(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)(?:-[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?(?:\\+[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?$","type":"string"}},"required":["id","name","description","version","repository"],"type":"object"}},"$id":"https://cdn.jsdelivr.net/npm/keepset@0.
|
|
1
|
+
{"$defs":{"Manifest":{"additionalProperties":false,"properties":{"description":{"minLength":1,"type":"string"},"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"lineage":{"additionalProperties":false,"properties":{"catalogue":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"repository":{"format":"uri","type":"string"},"version":{"pattern":"^(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)(?:-[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?(?:\\+[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?$","type":"string"}},"required":["catalogue","version","repository"],"type":"object"},"name":{"minLength":1,"type":"string"},"namespaceAliases":{"additionalProperties":{"items":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"minItems":1,"type":"array"},"propertyNames":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"type":"object"},"repository":{"format":"uri","type":"string"},"version":{"pattern":"^(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)(?:-[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?(?:\\+[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?$","type":"string"}},"required":["id","name","description","version","repository"],"type":"object"}},"$id":"https://cdn.jsdelivr.net/npm/keepset@0.19.0/schemas/manifest.schema.json","$ref":"#/$defs/Manifest","$schema":"https://json-schema.org/draft/2020-12/schema"}
|