keepset 0.17.0 → 0.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,5 +1,6 @@
1
- import { $ as NativeParseError, F as getOwn, J as ArchiveError, L as countTarget, Q as KeepsetError, et as RecipeError, t as getDataset, z as getTarget } from "../bundled-Ca5unUGI.js";
1
+ import { $ as NativeParseError, F as getOwn, J as ArchiveError, L as countTarget, Q as KeepsetError, et as RecipeError, t as getDataset, z as getTarget } from "../bundled-8oHQPxcd.js";
2
2
  import { RoundTripMismatchError, assertRoundTrip, encode } from "twindex";
3
+ import { z } from "zod";
3
4
  import { createHash } from "node:crypto";
4
5
  import { gunzipSync } from "node:zlib";
5
6
  import { unzipSync } from "fflate";
@@ -388,76 +389,164 @@ function columnNames(cells, idColumn, line) {
388
389
  }
389
390
  return names;
390
391
  }
392
+ /** The code unit of each character the scanner stops at. */
393
+ const QUOTE = "\"".charCodeAt(0);
394
+ const LINE_FEED = "\n".charCodeAt(0);
395
+ const CARRIAGE_RETURN = "\r".charCodeAt(0);
396
+ /** Scans a table file's text as it arrives in pieces, holding what a record in progress needs between them: the fields and field so far, whether a quoted field is open, and the line the record began on and the line the scan has reached. */
397
+ var RecordScanner = class {
398
+ delimiterCode;
399
+ fields = [];
400
+ field = "";
401
+ quoted = false;
402
+ started;
403
+ line;
404
+ constructor(delimiterCode, firstLine) {
405
+ this.delimiterCode = delimiterCode;
406
+ this.started = firstLine;
407
+ this.line = firstLine;
408
+ }
409
+ /**
410
+ * Scans text, returning the records it completes.
411
+ * @param text - The unscanned text.
412
+ * @param final - Whether no more text follows, so a character that needs the next one to be understood can be taken as it stands.
413
+ * @returns The records the text completes, and the unscanned end of it, which starts at a character that needs the next one when more text may follow.
414
+ * @throws NativeParseError when a quote opens inside an unquoted field.
415
+ */
416
+ scan(text, final) {
417
+ const completed = [];
418
+ let index = 0;
419
+ while (index < text.length) {
420
+ if (this.quoted) {
421
+ const close = text.indexOf("\"", index);
422
+ const end = close === -1 ? text.length : close;
423
+ const run = text.slice(index, end);
424
+ for (let at = run.indexOf("\n"); at !== -1; at = run.indexOf("\n", at + 1)) this.line += 1;
425
+ this.field += run;
426
+ index = end;
427
+ if (close === -1) break;
428
+ if (index + 1 >= text.length && !final) return {
429
+ completed,
430
+ carry: text.slice(index)
431
+ };
432
+ if (text.charCodeAt(index + 1) === QUOTE) {
433
+ this.field += "\"";
434
+ index += 2;
435
+ } else {
436
+ this.quoted = false;
437
+ index += 1;
438
+ }
439
+ continue;
440
+ }
441
+ const code = text.charCodeAt(index);
442
+ if (code === QUOTE) {
443
+ if (this.field !== "") throw new NativeParseError("csv-table", this.line, "a quote opens inside an unquoted field");
444
+ this.quoted = true;
445
+ index += 1;
446
+ } else if (code === this.delimiterCode) {
447
+ this.fields.push(this.field);
448
+ this.field = "";
449
+ index += 1;
450
+ } else if (code === LINE_FEED || code === CARRIAGE_RETURN) {
451
+ if (code === CARRIAGE_RETURN && index + 1 >= text.length && !final) return {
452
+ completed,
453
+ carry: text.slice(index)
454
+ };
455
+ index += code === CARRIAGE_RETURN && text.charCodeAt(index + 1) === LINE_FEED ? 2 : 1;
456
+ this.line += 1;
457
+ completed.push(this.endRecord());
458
+ } else {
459
+ let next = index + 1;
460
+ while (next < text.length) {
461
+ const following = text.charCodeAt(next);
462
+ if (following === QUOTE || following === this.delimiterCode || following === LINE_FEED || following === CARRIAGE_RETURN) break;
463
+ next += 1;
464
+ }
465
+ this.field += text.slice(index, next);
466
+ index = next;
467
+ }
468
+ }
469
+ return {
470
+ completed,
471
+ carry: ""
472
+ };
473
+ }
474
+ /**
475
+ * The record left unfinished when the text ends without a line break, if any.
476
+ * @returns The record, or undefined when the text ended on a line break.
477
+ * @throws NativeParseError when a quoted field is never closed.
478
+ */
479
+ finish() {
480
+ if (this.quoted) throw new NativeParseError("csv-table", this.started, "a quoted field is never closed");
481
+ return this.field !== "" || this.fields.length > 0 ? this.endRecord() : void 0;
482
+ }
483
+ endRecord() {
484
+ this.fields.push(this.field);
485
+ const record = {
486
+ line: this.started,
487
+ fields: this.fields
488
+ };
489
+ this.fields = [];
490
+ this.field = "";
491
+ this.started = this.line;
492
+ return record;
493
+ }
494
+ };
391
495
  /**
392
- * Splits text into records and fields by RFC 4180: a field may be quoted, a quoted field may hold the delimiter, line breaks and doubled quotes, and a record ends at an unquoted LF, CRLF or CR.
393
- * @param text - The file text, after the lines the recipe skips.
496
+ * Splits text into records and fields by RFC 4180 as the text arrives in pieces, one piece's records at a time, so neither the whole file nor every cell is held at once: a field may be quoted, a quoted field may hold the delimiter, line breaks and doubled quotes, and a record ends at an unquoted LF, CRLF or CR. A piece may end anywhere, even between the two quotes of a doubled quote or between a CR and its LF.
497
+ * @param pieces - The file text in order, after the lines the recipe skips.
394
498
  * @param delimiter - The field delimiter.
395
499
  * @param firstLine - The 1-based line of the file the text starts on.
396
- * @returns The records, blank lines included as a record of one empty field.
500
+ * @returns The records in order, blank lines included as a record of one empty field.
397
501
  * @throws NativeParseError when a quoted field is never closed or a quote opens inside an unquoted field.
398
502
  */
399
- function records(text, delimiter, firstLine) {
400
- const out = [];
401
- let fields = [];
402
- let field = "";
403
- let quoted = false;
404
- let started = firstLine;
405
- let line = firstLine;
406
- let index = 0;
407
- const endRecord = () => {
408
- fields.push(field);
409
- out.push({
410
- line: started,
411
- fields
412
- });
413
- fields = [];
414
- field = "";
415
- started = line;
416
- };
417
- while (index < text.length) {
418
- const char = text.charAt(index);
419
- if (quoted) {
420
- if (char === "\"") {
421
- if (text.charAt(index + 1) === "\"") {
422
- field += "\"";
423
- index += 1;
424
- } else quoted = false;
425
- } else {
426
- if (char === "\n") line += 1;
427
- field += char;
428
- }
429
- } else if (char === "\"") {
430
- if (field !== "") throw new NativeParseError("csv-table", line, "a quote opens inside an unquoted field");
431
- quoted = true;
432
- } else if (char === delimiter) {
433
- fields.push(field);
434
- field = "";
435
- } else if (char === "\n" || char === "\r") {
436
- if (char === "\r" && text.charAt(index + 1) === "\n") index += 1;
437
- line += 1;
438
- endRecord();
439
- } else field += char;
440
- index += 1;
503
+ function* records(pieces, delimiter, firstLine) {
504
+ const scanner = new RecordScanner(delimiter.charCodeAt(0), firstLine);
505
+ let carry = "";
506
+ for (const piece of pieces) {
507
+ const scanned = scanner.scan(carry + piece, false);
508
+ carry = scanned.carry;
509
+ yield* scanned.completed;
441
510
  }
442
- if (quoted) throw new NativeParseError("csv-table", started, "a quoted field is never closed");
443
- if (field !== "" || fields.length > 0) endRecord();
444
- return out;
511
+ yield* scanner.scan(carry, true).completed;
512
+ const last = scanner.finish();
513
+ if (last !== void 0) yield last;
445
514
  }
446
515
  /**
447
- * Drops the first lines of a file.
448
- * @param text - The file text.
516
+ * Drops the first physical lines of a file given in pieces.
517
+ * @param pieces - The file text in order.
449
518
  * @param count - How many physical lines to drop.
450
- * @returns The rest of the text.
519
+ * @returns The rest of the text, in pieces.
451
520
  * @throws NativeParseError when the file has fewer lines than that.
452
521
  */
453
- function afterLines(text, count) {
454
- let index = 0;
455
- for (let dropped = 0; dropped < count; dropped += 1) {
456
- const next = text.slice(index).search(/\n/);
457
- if (next === -1) throw new NativeParseError("csv-table", dropped + 1, `the file has fewer than ${String(count)} lines to skip before the header`);
458
- index += next + 1;
522
+ function* afterLines(pieces, count) {
523
+ let remaining = count;
524
+ for (const piece of pieces) {
525
+ let rest = piece;
526
+ while (remaining > 0) {
527
+ const next = rest.indexOf("\n");
528
+ if (next === -1) {
529
+ rest = "";
530
+ break;
531
+ }
532
+ rest = rest.slice(next + 1);
533
+ remaining -= 1;
534
+ }
535
+ if (remaining === 0 && rest !== "") yield rest;
536
+ }
537
+ if (remaining > 0) throw new NativeParseError("csv-table", count - remaining + 1, `the file has fewer than ${String(count)} lines to skip before the header`);
538
+ }
539
+ /**
540
+ * The text of a file as its pieces, with a leading byte order mark removed.
541
+ * @param text - The file text, or its pieces in order.
542
+ * @returns The pieces.
543
+ */
544
+ function* withoutByteOrderMark(text) {
545
+ let first = true;
546
+ for (const piece of typeof text === "string" ? [text] : text) {
547
+ yield first && piece.startsWith(BYTE_ORDER_MARK) ? piece.slice(1) : piece;
548
+ first = false;
459
549
  }
460
- return text.slice(index);
461
550
  }
462
551
  /**
463
552
  * Reads a numeric cell.
@@ -474,42 +563,157 @@ function numeric$1(cell, factor, column, line) {
474
563
  return Number(cell) * factor;
475
564
  }
476
565
  /**
477
- * The records of a table file after the lines the recipe skips and without its blank lines: the header first, then the rows.
478
- * @param text - The file text, which may begin with a byte order mark.
566
+ * The records of a table file after the lines the recipe skips and without its blank lines: the header first, then the rows, one at a time.
567
+ * @param text - The file text, or its pieces in order, which may begin with a byte order mark.
479
568
  * @param options - The recipe's table parser, of which the delimiter and the lines to skip are read.
480
- * @returns The records.
569
+ * @returns Each record.
481
570
  * @throws NativeParseError when a quote is never closed or opens inside an unquoted field, or the file has fewer lines than the recipe skips.
482
571
  */
483
- function tableRecords(text, options) {
484
- return records(afterLines(text.startsWith(BYTE_ORDER_MARK) ? text.slice(1) : text, options.skipLines), DELIMITERS[options.delimiter], options.skipLines + 1).filter((record) => record.fields.length > 1 || record.fields[0]?.trim() !== "");
572
+ function* tableRecords(text, options) {
573
+ const body = afterLines(withoutByteOrderMark(text), options.skipLines);
574
+ for (const record of records(body, DELIMITERS[options.delimiter], options.skipLines + 1)) if (record.fields.length > 1 || record.fields[0]?.trim() !== "") yield record;
485
575
  }
486
576
  /**
487
- * Parses a delimited table into rows: the first record after `skipLines` lines is the header, each later record is a row, and every row carries every column but the id column, an empty cell being null. Cells are trimmed. A column the recipe declares numeric is read as a decimal number times its factor and any other column stays a string. `id` is the row's id, so a column of that name which is not the id column is named by its position. A row's id is the cell of the recipe's id column, which is then not also a column, or its 1-based row number when the recipe names none. Blank lines are skipped; a row with a different number of fields than the header, two columns with the same name, and an id cell that is empty are errors.
488
- * @param text - The file text.
577
+ * Parses a delimited table into rows: the first record after `skipLines` lines is the header, each later record is a row, and every row carries every column but the id column (or, when the recipe lists `columns`, exactly those, so a wide table is not held cell for cell), an empty cell being null. Cells are trimmed. A column the recipe declares numeric is read as a decimal number times its factor and any other column stays a string. `id` is the row's id, so a column of that name which is not the id column is named by its position. A row's id is the cell of the recipe's id column, which is then not also a column, or its 1-based row number when the recipe names none. Blank lines are skipped; a row with a different number of fields than the header, two columns with the same name, and an id cell that is empty are errors.
578
+ * @param text - The file text, or its pieces in order.
489
579
  * @param options - The recipe's table parser.
490
580
  * @returns The rows, in file order.
491
- * @throws NativeParseError naming the line of the first problem.
581
+ * @throws NativeParseError naming the line of the first problem, including a listed or numeric column the header lacks or a numeric column that is not listed.
492
582
  */
493
583
  function parseCsvTable(text, options) {
494
- const [header, ...rows] = tableRecords(text, options);
584
+ const table = tableRecords(text, options);
585
+ const { value: header } = table.next();
495
586
  if (header === void 0) throw new NativeParseError("csv-table", options.skipLines + 1, "the file has no header row");
496
587
  const names = columnNames(header.fields, options.idColumn, header.line);
497
- const { idColumn, numericColumns } = options;
498
- for (const wanted of [...idColumn === null ? [] : [idColumn], ...numericColumns.map((entry) => entry.column)]) if (!names.includes(wanted)) throw new NativeParseError("csv-table", header.line, `the recipe names the column ${wanted}, which the header does not have`);
588
+ const { idColumn, numericColumns, columns } = options;
589
+ const wanted = new Set(columns ?? names.filter((name) => name !== idColumn));
590
+ for (const named of [
591
+ ...idColumn === null ? [] : [idColumn],
592
+ ...numericColumns.map((entry) => entry.column),
593
+ ...columns ?? []
594
+ ]) if (!names.includes(named)) throw new NativeParseError("csv-table", header.line, `the recipe names the column ${named}, which the header does not have`);
595
+ if (idColumn !== null && wanted.has(idColumn)) throw new NativeParseError("csv-table", header.line, `the id column ${idColumn} is the row's id, so it is not listed among the columns`);
596
+ for (const { column } of numericColumns) if (!wanted.has(column)) throw new NativeParseError("csv-table", header.line, `the column ${column} is declared numeric but is not among the columns kept`);
499
597
  const factors = new Map(numericColumns.map((entry) => [entry.column, entry.factor]));
500
- return { nodes: rows.map((record, rowIndex) => {
598
+ const idPosition = idColumn === null ? -1 : names.indexOf(idColumn);
599
+ const kept = names.flatMap((name, position) => name !== idColumn && wanted.has(name) ? [{
600
+ name,
601
+ position,
602
+ factor: factors.get(name)
603
+ }] : []);
604
+ const nodes = [];
605
+ for (const record of table) {
501
606
  if (record.fields.length !== names.length) throw new NativeParseError("csv-table", record.line, `the row has ${String(record.fields.length)} fields, the header has ${String(names.length)}`);
502
- const cells = new Map(names.map((name, position) => [name, (record.fields[position] ?? "").trim()]));
503
- const id = idColumn === null ? String(rowIndex + 1) : cells.get(idColumn) ?? "";
607
+ const id = idColumn === null ? String(nodes.length + 1) : (record.fields[idPosition] ?? "").trim();
504
608
  if (id === "") throw new NativeParseError("csv-table", record.line, `the id column ${idColumn ?? ""} is empty`);
505
- const columns = names.filter((name) => name !== idColumn).map((name) => {
506
- const cell = cells.get(name) ?? "";
507
- const factor = factors.get(name);
609
+ const cells = kept.map(({ name, position, factor }) => {
610
+ const cell = (record.fields[position] ?? "").trim();
508
611
  return [name, factor === void 0 ? cell === "" ? null : cell : numeric$1(cell, factor, name, record.line)];
509
612
  });
510
- return {
511
- ...Object.fromEntries(columns),
613
+ nodes.push({
614
+ ...Object.fromEntries(cells),
512
615
  id
616
+ });
617
+ }
618
+ return { nodes };
619
+ }
620
+ //#endregion
621
+ //#region src/parsers/geojson.ts
622
+ /** The columns a point feature's position fills, which no property may share a name with: the latitude and the longitude of a `Point` geometry, and null for a feature with any other geometry or none. */
623
+ const POINT_COLUMNS = {
624
+ latitude: "geometry_latitude",
625
+ longitude: "geometry_longitude"
626
+ };
627
+ const FeatureCollectionSchema = z.object({
628
+ type: z.literal("FeatureCollection"),
629
+ features: z.array(z.object({
630
+ geometry: z.object({
631
+ type: z.string(),
632
+ coordinates: z.unknown()
633
+ }).nullable(),
634
+ properties: z.record(z.string(), z.unknown()).nullable()
635
+ }))
636
+ });
637
+ /**
638
+ * The position of a point geometry.
639
+ * @param geometry - The feature's geometry.
640
+ * @returns The longitude and latitude of a `Point`, and undefined for any other geometry or none.
641
+ */
642
+ function pointOf(geometry) {
643
+ const coordinates = z.array(z.unknown()).safeParse(geometry?.type === "Point" ? geometry.coordinates : void 0);
644
+ if (!coordinates.success) return;
645
+ const [longitude, latitude] = coordinates.data;
646
+ return typeof longitude === "number" && typeof latitude === "number" ? {
647
+ longitude,
648
+ latitude
649
+ } : void 0;
650
+ }
651
+ /**
652
+ * One cell of a feature, in the form its column is declared to take.
653
+ * @param value - The property's value, or undefined when the feature lacks it.
654
+ * @param column - The column, for the error.
655
+ * @param feature - The 1-based feature number, for the error.
656
+ * @param kind - How the column is read: as a number times a factor, as an epoch-millisecond date written in ISO 8601, or as text.
657
+ * @returns The cell: null for a missing, null or empty value.
658
+ * @throws NativeParseError when the value is not the kind the recipe declares.
659
+ */
660
+ function cellOf(value, column, feature, kind) {
661
+ if (value === void 0 || value === null) return null;
662
+ if (kind === "date") {
663
+ if (typeof value !== "number" || !Number.isFinite(value)) throw new NativeParseError("geojson", feature, `column ${column} is declared a date but holds ${JSON.stringify(value)}`);
664
+ return new Date(value).toISOString();
665
+ }
666
+ if (kind !== "text") {
667
+ if (typeof value !== "number" || !Number.isFinite(value)) throw new NativeParseError("geojson", feature, `column ${column} is declared numeric but holds ${JSON.stringify(value)}`);
668
+ return value * kind.factor;
669
+ }
670
+ if (typeof value === "string") return value.trim() === "" ? null : value.trim();
671
+ if (typeof value === "number" || typeof value === "boolean") return String(value);
672
+ throw new NativeParseError("geojson", feature, `column ${column} holds a ${Array.isArray(value) ? "list" : "nested object"}, which a column cannot`);
673
+ }
674
+ /**
675
+ * Parses a GeoJSON feature collection into rows, one per feature in file order: the properties the recipe lists become columns, a point's position fills `geometry_latitude` and `geometry_longitude` when they are listed, and an id comes from the recipe's id column (not also a column) or the feature's 1-based number. A column declared numeric is a number times its factor, one declared a date is an epoch-millisecond number written as an ISO 8601 text, and any other is text (a number or flag is written as its text), an empty or missing value being null so every row carries every column.
676
+ * @param text - The file text.
677
+ * @param options - The recipe's parser.
678
+ * @returns The rows.
679
+ * @throws NativeParseError when the text is not a GeoJSON feature collection, a value is not the kind its column is declared, a listed column is also a point column of another name, or an id is empty.
680
+ */
681
+ function parseGeojson(text, options) {
682
+ let document;
683
+ try {
684
+ document = JSON.parse(text);
685
+ } catch (error) {
686
+ throw new NativeParseError("geojson", 1, `the file is not JSON (${error instanceof Error ? error.message : String(error)})`);
687
+ }
688
+ const collection = FeatureCollectionSchema.safeParse(document);
689
+ if (!collection.success) throw new NativeParseError("geojson", 1, `the file is not a GeoJSON feature collection (${collection.error.issues.map((issue) => `${issue.path.map(String).join(".") || "(root)"}: ${issue.message}`).join("; ")})`);
690
+ const factors = new Map(options.numericColumns.map((numeric) => [numeric.column, numeric.factor]));
691
+ const dates = new Set(options.dateColumns);
692
+ const kinds = options.columns.map((column) => {
693
+ const factor = factors.get(column);
694
+ return {
695
+ column,
696
+ kind: factor === void 0 ? dates.has(column) ? "date" : "text" : { factor }
697
+ };
698
+ });
699
+ return { nodes: collection.data.features.map((feature, index) => {
700
+ const number = index + 1;
701
+ const properties = feature.properties ?? {};
702
+ const point = pointOf(feature.geometry);
703
+ const own = (column) => Object.hasOwn(properties, column) ? properties[column] : void 0;
704
+ const id = options.idColumn === null ? String(number) : cellOf(own(options.idColumn), options.idColumn, number, "text");
705
+ if (id === null) throw new NativeParseError("geojson", number, `the id column ${options.idColumn ?? ""} is empty`);
706
+ const cells = kinds.map(({ column, kind }) => {
707
+ if (column === POINT_COLUMNS.latitude || column === POINT_COLUMNS.longitude) {
708
+ if (Object.hasOwn(properties, column)) throw new NativeParseError("geojson", number, `a property is named ${column}, which is the column of the point's position`);
709
+ const position = column === POINT_COLUMNS.latitude ? point?.latitude : point?.longitude;
710
+ return [column, position === void 0 ? null : position * (typeof kind === "object" ? kind.factor : 1)];
711
+ }
712
+ return [column, cellOf(own(column), column, number, kind)];
713
+ });
714
+ return {
715
+ ...Object.fromEntries(cells),
716
+ id: String(id)
513
717
  };
514
718
  }) };
515
719
  }
@@ -647,12 +851,40 @@ function decode(bytes, parser) {
647
851
  throw new NativeParseError(parser.format, 1, `the file is not valid utf-8 (${error instanceof Error ? error.message : String(error)}); a recipe for a latin1 file states encoding latin1`);
648
852
  }
649
853
  }
854
+ const KIBIBYTE = 1024;
855
+ /** Bytes decoded at a time for a format that is read in pieces. The decoder holds only a piece's text at once, so this bounds what a large table costs beyond its bytes; it is large enough that the per-piece work is small beside the piece. */
856
+ const DECODE_PIECE_BYTES = 8 * KIBIBYTE * KIBIBYTE;
857
+ /**
858
+ * Decodes a file's bytes in the recipe's encoding a piece at a time, as strictly as `decode`, so a file whose text would not fit in one string can still be read.
859
+ * @param bytes - The unpacked file.
860
+ * @param parser - The recipe's parser, which names the encoding.
861
+ * @param pieceBytes - How many bytes to decode at a time.
862
+ * @returns The text of each piece, in order; a character that spans two pieces is yielded whole with the later one.
863
+ * @throws NativeParseError when the bytes are not valid in the stated encoding.
864
+ */
865
+ function* decodePieces(bytes, parser, pieceBytes = DECODE_PIECE_BYTES) {
866
+ const decoder = parser.encoding === "latin1" ? void 0 : new TextDecoder("utf-8", { fatal: true });
867
+ try {
868
+ for (let start = 0; start < bytes.length; start += pieceBytes) {
869
+ const piece = bytes.subarray(start, start + pieceBytes);
870
+ yield decoder === void 0 ? decode(piece, parser) : decoder.decode(piece, { stream: true });
871
+ }
872
+ if (decoder !== void 0) {
873
+ const rest = decoder.decode();
874
+ if (rest !== "") yield rest;
875
+ }
876
+ } catch (error) {
877
+ if (error instanceof NativeParseError) throw error;
878
+ throw new NativeParseError(parser.format, 1, `the file is not valid utf-8 (${error instanceof Error ? error.message : String(error)}); a recipe for a latin1 file states encoding latin1`);
879
+ }
880
+ }
650
881
  function parseNative(bytes, parser) {
882
+ if (parser.format === "csv-table") return parseCsvTable(decodePieces(bytes, parser), parser);
651
883
  const text = decode(bytes, parser);
884
+ if (parser.format === "geojson") return parseGeojson(text, parser);
652
885
  if (parser.format === "edge-list") return parseEdgeList(text, parser);
653
886
  if (parser.format === "matrix-market") return parseMatrixMarket(text);
654
887
  if (parser.format === "gml") return parseGml(text);
655
- if (parser.format === "csv-table") return parseCsvTable(text, parser);
656
888
  return parsePajek(text);
657
889
  }
658
890
  /** The ustar header layout (POSIX.1-1988, pax(1) "ustar Interchange Format"): each field's byte offset and length. */
@@ -952,6 +1184,93 @@ async function downloadRaw(url, init) {
952
1184
  return follow(new URL(url), init.signal, 0);
953
1185
  }
954
1186
  //#endregion
1187
+ //#region src/node/paged.ts
1188
+ const countResponse = (value) => typeof value === "object" && value !== null && "count" in value && typeof value.count === "number" && Number.isInteger(value.count) && value.count >= 0 ? value.count : void 0;
1189
+ /**
1190
+ * The error message an ArcGIS server returns in a 200 response, if the body is one.
1191
+ * @param value - A parsed response body.
1192
+ * @returns The message, or undefined when the body is not an error.
1193
+ */
1194
+ function arcgisError(value) {
1195
+ if (typeof value !== "object" || value === null || !("error" in value)) return;
1196
+ const { error } = value;
1197
+ return typeof error === "object" && error !== null && "message" in error && typeof error.message === "string" ? error.message : "the server returned an error";
1198
+ }
1199
+ /**
1200
+ * Reads one JSON response of the layer's query endpoint.
1201
+ * @param port - The network port.
1202
+ * @param url - The request URL.
1203
+ * @param signal - Aborts the request.
1204
+ * @returns The parsed body.
1205
+ * @throws KeepsetError when the server does not answer successfully, the body is not JSON, or it is an ArcGIS error.
1206
+ */
1207
+ async function getJson(port, url, signal) {
1208
+ const response = await port(url, signal === void 0 ? {} : { signal });
1209
+ if (!response.ok) throw new KeepsetError(`${url} answered ${String(response.status)} ${response.statusText}`);
1210
+ let body;
1211
+ try {
1212
+ body = JSON.parse(new TextDecoder("utf-8").decode(new Uint8Array(await response.arrayBuffer())));
1213
+ } catch (error) {
1214
+ throw new KeepsetError(`${url} did not answer JSON (${error instanceof Error ? error.message : String(error)})`);
1215
+ }
1216
+ const message = arcgisError(body);
1217
+ if (message !== void 0) throw new KeepsetError(`${url} answered an error: ${message}`);
1218
+ return body;
1219
+ }
1220
+ const FeatureListSchema = z.array(z.unknown());
1221
+ /**
1222
+ * Reads a layer's features page by page, each request starting where the features held so far end.
1223
+ * @param layer - The layer read.
1224
+ * @param held - The features read so far.
1225
+ * @returns Every feature of the layer.
1226
+ * @throws KeepsetError when a page is empty before the count is reached or the server answers an error.
1227
+ */
1228
+ async function readPages(layer, held) {
1229
+ if (held.length >= layer.count) return held;
1230
+ const url = `${layer.query}?where=1%3D1&outFields=*&orderByFields=${encodeURIComponent(layer.orderBy)}&resultOffset=${String(held.length)}&f=geojson`;
1231
+ const page = await getJson(layer.port, url, layer.signal);
1232
+ const features = FeatureListSchema.safeParse(typeof page === "object" && page !== null && "features" in page ? page.features : void 0);
1233
+ if (!features.success || features.data.length === 0) throw new KeepsetError(`${layer.query} gave no features at offset ${String(held.length)}, before the ${String(layer.count)} it reports, so it does not page by offset`);
1234
+ return readPages(layer, [...held, ...features.data]);
1235
+ }
1236
+ /**
1237
+ * Downloads a whole ArcGIS feature layer as one GeoJSON feature collection: the layer's feature count is read first, then its pages in `orderBy` order by offset until every feature is held, so a server that ignores the offset is an error and not an endless loop.
1238
+ * @param layerUrl - The layer's URL, such as `.../FeatureServer/0`.
1239
+ * @param orderBy - The field the pages are ordered by.
1240
+ * @param port - The network port.
1241
+ * @param signal - Aborts the download.
1242
+ * @returns The bytes of one `FeatureCollection`, which is what the parser reads and what the provenance hashes.
1243
+ * @throws KeepsetError when the layer does not report a count, a page is empty before the count is reached, or the server answers an error.
1244
+ */
1245
+ async function downloadArcgisLayer(layerUrl, orderBy, port, signal) {
1246
+ const query = `${layerUrl.replace(/\/+$/, "")}/query`;
1247
+ const count = countResponse(await getJson(port, `${query}?where=1%3D1&returnCountOnly=true&f=json`, signal));
1248
+ if (count === void 0) throw new KeepsetError(`${layerUrl} did not report a feature count`);
1249
+ const pages = await readPages({
1250
+ query,
1251
+ orderBy,
1252
+ count,
1253
+ port,
1254
+ signal
1255
+ }, []);
1256
+ return new TextEncoder().encode(JSON.stringify({
1257
+ type: "FeatureCollection",
1258
+ features: pages
1259
+ }));
1260
+ }
1261
+ /**
1262
+ * Downloads a paged source as the one file its pages make, in the way its recipe's paging says.
1263
+ * @param url - The source's URL.
1264
+ * @param paging - How the source is paged.
1265
+ * @param port - The network port.
1266
+ * @param signal - Aborts the download.
1267
+ * @returns The bytes the parser reads.
1268
+ * @throws KeepsetError when a page cannot be read or the pages do not add up to what the source reports.
1269
+ */
1270
+ async function downloadPaged(url, paging, port, signal) {
1271
+ return downloadArcgisLayer(url, paging.orderBy, port, signal);
1272
+ }
1273
+ //#endregion
955
1274
  //#region src/node/fetch-dataset.ts
956
1275
  /**
957
1276
  * The URL a recipe downloads a source from. A dead source is fetched from its pinned snapshot in the Wayback Machine's `id_` form, which serves the archived bytes as they were captured rather than wrapped in the archive's page chrome.
@@ -1008,6 +1327,7 @@ async function runToNative(dataset, options) {
1008
1327
  const fetchedUrl = downloadUrl(source);
1009
1328
  const retrievedAt = now().toISOString();
1010
1329
  const bytes = await step(entry.id, "download", async () => {
1330
+ if (recipe.paging !== void 0) return downloadPaged(fetchedUrl, recipe.paging, fetchPort, signal);
1011
1331
  const response = await fetchPort(fetchedUrl, signal === void 0 ? {} : { signal });
1012
1332
  if (!response.ok) throw new KeepsetError(`${fetchedUrl} answered ${String(response.status)} ${response.statusText}`);
1013
1333
  return new Uint8Array(await response.arrayBuffer());
@@ -4,7 +4,7 @@ import { KeepsetError, NAME_SEPARATOR, countTarget, getDataset, listDatasets, li
4
4
  import { fetchDataset, roundTripDataset } from "keepset/node";
5
5
  import { parseArgs } from "node:util";
6
6
  //#region package.json
7
- var version = "0.17.0";
7
+ var version = "0.19.0";
8
8
  const usage = `Usage: keepset <command> [options]
9
9
 
10
10
  Commands:
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "keepset",
3
3
  "description": "Library, CLI and statically generated catalogue for dataset fetch-and-normalise recipes",
4
- "version": "0.17.0",
4
+ "version": "0.19.0",
5
5
  "bin": {
6
6
  "keepset": "./dist-cli/keepset.mjs"
7
7
  },
@@ -1 +1 @@
1
- {"$defs":{"Crosswalk":{"$ref":"https://cdn.jsdelivr.net/npm/twindex@0.1.4/schemas/twindex.schema.json"},"Entry":{"additionalProperties":false,"properties":{"aliases":{"items":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*(?:\\/[A-Za-z0-9][A-Za-z0-9._-]*)*$","type":"string"},"minItems":1,"type":"array"},"category":{"enum":["graph/v1","or/v1"],"type":"string"},"citations":{"items":{"additionalProperties":false,"properties":{"doi":{"pattern":"^10\\.\\d{4,9}\\/\\S+$","type":"string"},"text":{"minLength":1,"type":"string"},"url":{"format":"uri","type":"string"}},"required":["text"],"type":"object"},"type":"array"},"crosswalk":{"anyOf":[{"$ref":"#/$defs/Crosswalk"},{"additionalProperties":false,"properties":{"file":{"type":"string"}},"required":["file"],"type":"object"}]},"description":{"minLength":1,"type":"string"},"domain":{"minLength":1,"type":"string"},"expectedCounts":{"additionalProperties":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"propertyNames":{"pattern":"^[A-Za-z][A-Za-z0-9]*$","type":"string"},"type":"object"},"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"licence":{"oneOf":[{"additionalProperties":false,"properties":{"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9.+-]*$","type":"string"},"kind":{"const":"spdx","type":"string"}},"required":["kind","id"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"custom","type":"string"},"name":{"minLength":1,"type":"string"},"url":{"format":"uri","type":"string"}},"required":["kind","name","url"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"unrecorded","type":"string"}},"required":["kind"],"type":"object"}]},"name":{"minLength":1,"type":"string"},"namespace":{"items":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"minItems":1,"type":"array"},"properties":{"additionalProperties":{"type":["string","number","boolean","null"]},"propertyNames":{"pattern":"^[A-Za-z][A-Za-z0-9]*$","type":"string"},"type":"object"},"recipe":{"oneOf":[{"additionalProperties":false,"properties":{"archive":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"none","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"gzip","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"zip","type":"string"},"member":{"anyOf":[{"additionalProperties":false,"properties":{"path":{"type":"string"}},"required":["path"],"type":"object"},{"additionalProperties":false,"properties":{"extensions":{"items":{"pattern":"^\\.[A-Za-z0-9][A-Za-z0-9.]*$","type":"string"},"minItems":1,"type":"array"}},"required":["extensions"],"type":"object"}]}},"required":["kind","member"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"tar","type":"string"},"member":{"anyOf":[{"additionalProperties":false,"properties":{"path":{"type":"string"}},"required":["path"],"type":"object"},{"additionalProperties":false,"properties":{"extensions":{"items":{"pattern":"^\\.[A-Za-z0-9][A-Za-z0-9.]*$","type":"string"},"minItems":1,"type":"array"}},"required":["extensions"],"type":"object"}]}},"required":["kind","member"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"tar.gz","type":"string"},"member":{"anyOf":[{"additionalProperties":false,"properties":{"path":{"type":"string"}},"required":["path"],"type":"object"},{"additionalProperties":false,"properties":{"extensions":{"items":{"pattern":"^\\.[A-Za-z0-9][A-Za-z0-9.]*$","type":"string"},"minItems":1,"type":"array"}},"required":["extensions"],"type":"object"}]}},"required":["kind","member"],"type":"object"}]},"parser":{"oneOf":[{"additionalProperties":false,"properties":{"commentPrefixes":{"items":{"minLength":1,"type":"string"},"type":"array"},"delimiter":{"enum":["detect","whitespace","tab","comma","space"],"type":"string"},"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"edge-list","type":"string"},"skipLines":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"sourceColumn":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"targetColumn":{"maximum":9007199254740991,"minimum":0,"type":"integer"}},"required":["format","encoding","delimiter","skipLines","commentPrefixes","sourceColumn","targetColumn"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"matrix-market","type":"string"}},"required":["format","encoding"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"gml","type":"string"}},"required":["format","encoding"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"pajek","type":"string"}},"required":["format","encoding"],"type":"object"},{"additionalProperties":false,"properties":{"delimiter":{"enum":["comma","tab","semicolon"],"type":"string"},"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"csv-table","type":"string"},"idColumn":{"anyOf":[{"pattern":"^[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?$","type":"string"},{"type":"null"}]},"numericColumns":{"items":{"additionalProperties":false,"properties":{"column":{"pattern":"^[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?$","type":"string"},"factor":{"exclusiveMinimum":0,"type":"number"}},"required":["column","factor"],"type":"object"},"type":"array"},"skipLines":{"maximum":9007199254740991,"minimum":0,"type":"integer"}},"required":["format","encoding","delimiter","skipLines","idColumn","numericColumns"],"type":"object"}]},"source":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"status":{"const":"fetchable","type":"string"}},"required":["status","source","archive","parser"],"type":"object"},{"additionalProperties":false,"properties":{"reason":{"minLength":1,"type":"string"},"status":{"const":"not-fetchable","type":"string"}},"required":["status","reason"],"type":"object"}]},"sources":{"items":{"additionalProperties":false,"properties":{"retrieved":{"format":"date","pattern":"^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))$","type":"string"},"snapshot":{"additionalProperties":false,"properties":{"timestamp":{"pattern":"^\\d{8}(?:\\d{6})?$","type":"string"},"url":{"format":"uri","type":"string"}},"required":["url","timestamp"],"type":"object"},"status":{"enum":["live","dead"],"type":"string"},"url":{"format":"uri","type":"string"}},"required":["url","retrieved","status"],"type":"object"},"minItems":1,"type":"array"},"tags":{"items":{"minLength":1,"type":"string"},"type":"array"}},"required":["id","name","tags","category","sources","licence","citations","recipe","crosswalk","expectedCounts","properties"],"type":"object"}},"$id":"https://cdn.jsdelivr.net/npm/keepset@0.17.0/schemas/entry.schema.json","$ref":"#/$defs/Entry","$schema":"https://json-schema.org/draft/2020-12/schema"}
1
+ {"$defs":{"Crosswalk":{"$ref":"https://cdn.jsdelivr.net/npm/twindex@0.1.4/schemas/twindex.schema.json"},"Entry":{"additionalProperties":false,"properties":{"aliases":{"items":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*(?:\\/[A-Za-z0-9][A-Za-z0-9._-]*)*$","type":"string"},"minItems":1,"type":"array"},"category":{"enum":["graph/v1","or/v1"],"type":"string"},"citations":{"items":{"additionalProperties":false,"properties":{"doi":{"pattern":"^10\\.\\d{4,9}\\/\\S+$","type":"string"},"text":{"minLength":1,"type":"string"},"url":{"format":"uri","type":"string"}},"required":["text"],"type":"object"},"type":"array"},"crosswalk":{"anyOf":[{"$ref":"#/$defs/Crosswalk"},{"additionalProperties":false,"properties":{"file":{"type":"string"}},"required":["file"],"type":"object"}]},"description":{"minLength":1,"type":"string"},"domain":{"minLength":1,"type":"string"},"expectedCounts":{"additionalProperties":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"propertyNames":{"pattern":"^[A-Za-z][A-Za-z0-9]*$","type":"string"},"type":"object"},"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"licence":{"oneOf":[{"additionalProperties":false,"properties":{"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9.+-]*$","type":"string"},"kind":{"const":"spdx","type":"string"}},"required":["kind","id"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"custom","type":"string"},"name":{"minLength":1,"type":"string"},"url":{"format":"uri","type":"string"}},"required":["kind","name","url"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"unrecorded","type":"string"}},"required":["kind"],"type":"object"}]},"name":{"minLength":1,"type":"string"},"namespace":{"items":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"minItems":1,"type":"array"},"properties":{"additionalProperties":{"type":["string","number","boolean","null"]},"propertyNames":{"pattern":"^[A-Za-z][A-Za-z0-9]*$","type":"string"},"type":"object"},"recipe":{"oneOf":[{"additionalProperties":false,"properties":{"archive":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"none","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"gzip","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"zip","type":"string"},"member":{"anyOf":[{"additionalProperties":false,"properties":{"path":{"type":"string"}},"required":["path"],"type":"object"},{"additionalProperties":false,"properties":{"extensions":{"items":{"pattern":"^\\.[A-Za-z0-9][A-Za-z0-9.]*$","type":"string"},"minItems":1,"type":"array"}},"required":["extensions"],"type":"object"}]}},"required":["kind","member"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"tar","type":"string"},"member":{"anyOf":[{"additionalProperties":false,"properties":{"path":{"type":"string"}},"required":["path"],"type":"object"},{"additionalProperties":false,"properties":{"extensions":{"items":{"pattern":"^\\.[A-Za-z0-9][A-Za-z0-9.]*$","type":"string"},"minItems":1,"type":"array"}},"required":["extensions"],"type":"object"}]}},"required":["kind","member"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"tar.gz","type":"string"},"member":{"anyOf":[{"additionalProperties":false,"properties":{"path":{"type":"string"}},"required":["path"],"type":"object"},{"additionalProperties":false,"properties":{"extensions":{"items":{"pattern":"^\\.[A-Za-z0-9][A-Za-z0-9.]*$","type":"string"},"minItems":1,"type":"array"}},"required":["extensions"],"type":"object"}]}},"required":["kind","member"],"type":"object"}]},"paging":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"arcgis-query","type":"string"},"orderBy":{"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"}},"required":["kind","orderBy"],"type":"object"}]},"parser":{"oneOf":[{"additionalProperties":false,"properties":{"commentPrefixes":{"items":{"minLength":1,"type":"string"},"type":"array"},"delimiter":{"enum":["detect","whitespace","tab","comma","space"],"type":"string"},"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"edge-list","type":"string"},"skipLines":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"sourceColumn":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"targetColumn":{"maximum":9007199254740991,"minimum":0,"type":"integer"}},"required":["format","encoding","delimiter","skipLines","commentPrefixes","sourceColumn","targetColumn"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"matrix-market","type":"string"}},"required":["format","encoding"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"gml","type":"string"}},"required":["format","encoding"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"pajek","type":"string"}},"required":["format","encoding"],"type":"object"},{"additionalProperties":false,"properties":{"columns":{"items":{"pattern":"^[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?$","type":"string"},"type":"array"},"delimiter":{"enum":["comma","tab","semicolon"],"type":"string"},"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"csv-table","type":"string"},"idColumn":{"anyOf":[{"pattern":"^[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?$","type":"string"},{"type":"null"}]},"numericColumns":{"items":{"additionalProperties":false,"properties":{"column":{"pattern":"^[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?$","type":"string"},"factor":{"exclusiveMinimum":0,"type":"number"}},"required":["column","factor"],"type":"object"},"type":"array"},"skipLines":{"maximum":9007199254740991,"minimum":0,"type":"integer"}},"required":["format","encoding","delimiter","skipLines","idColumn","numericColumns"],"type":"object"},{"additionalProperties":false,"properties":{"columns":{"items":{"pattern":"^[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?$","type":"string"},"minItems":1,"type":"array"},"dateColumns":{"items":{"pattern":"^[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?$","type":"string"},"type":"array"},"encoding":{"const":"utf-8","type":"string"},"format":{"const":"geojson","type":"string"},"idColumn":{"anyOf":[{"pattern":"^[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?$","type":"string"},{"type":"null"}]},"numericColumns":{"items":{"additionalProperties":false,"properties":{"column":{"pattern":"^[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?$","type":"string"},"factor":{"exclusiveMinimum":0,"type":"number"}},"required":["column","factor"],"type":"object"},"type":"array"}},"required":["format","encoding","idColumn","numericColumns","dateColumns","columns"],"type":"object"}]},"source":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"status":{"const":"fetchable","type":"string"}},"required":["status","source","archive","parser"],"type":"object"},{"additionalProperties":false,"properties":{"reason":{"minLength":1,"type":"string"},"status":{"const":"not-fetchable","type":"string"}},"required":["status","reason"],"type":"object"}]},"sources":{"items":{"additionalProperties":false,"properties":{"retrieved":{"format":"date","pattern":"^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))$","type":"string"},"snapshot":{"additionalProperties":false,"properties":{"timestamp":{"pattern":"^\\d{8}(?:\\d{6})?$","type":"string"},"url":{"format":"uri","type":"string"}},"required":["url","timestamp"],"type":"object"},"status":{"enum":["live","dead"],"type":"string"},"url":{"format":"uri","type":"string"}},"required":["url","retrieved","status"],"type":"object"},"minItems":1,"type":"array"},"tags":{"items":{"minLength":1,"type":"string"},"type":"array"}},"required":["id","name","tags","category","sources","licence","citations","recipe","crosswalk","expectedCounts","properties"],"type":"object"}},"$id":"https://cdn.jsdelivr.net/npm/keepset@0.19.0/schemas/entry.schema.json","$ref":"#/$defs/Entry","$schema":"https://json-schema.org/draft/2020-12/schema"}
@@ -1 +1 @@
1
- {"$defs":{"Manifest":{"additionalProperties":false,"properties":{"description":{"minLength":1,"type":"string"},"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"lineage":{"additionalProperties":false,"properties":{"catalogue":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"repository":{"format":"uri","type":"string"},"version":{"pattern":"^(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)(?:-[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?(?:\\+[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?$","type":"string"}},"required":["catalogue","version","repository"],"type":"object"},"name":{"minLength":1,"type":"string"},"namespaceAliases":{"additionalProperties":{"items":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"minItems":1,"type":"array"},"propertyNames":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"type":"object"},"repository":{"format":"uri","type":"string"},"version":{"pattern":"^(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)(?:-[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?(?:\\+[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?$","type":"string"}},"required":["id","name","description","version","repository"],"type":"object"}},"$id":"https://cdn.jsdelivr.net/npm/keepset@0.17.0/schemas/manifest.schema.json","$ref":"#/$defs/Manifest","$schema":"https://json-schema.org/draft/2020-12/schema"}
1
+ {"$defs":{"Manifest":{"additionalProperties":false,"properties":{"description":{"minLength":1,"type":"string"},"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"lineage":{"additionalProperties":false,"properties":{"catalogue":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"repository":{"format":"uri","type":"string"},"version":{"pattern":"^(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)(?:-[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?(?:\\+[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?$","type":"string"}},"required":["catalogue","version","repository"],"type":"object"},"name":{"minLength":1,"type":"string"},"namespaceAliases":{"additionalProperties":{"items":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"minItems":1,"type":"array"},"propertyNames":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"type":"object"},"repository":{"format":"uri","type":"string"},"version":{"pattern":"^(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)(?:-[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?(?:\\+[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?$","type":"string"}},"required":["id","name","description","version","repository"],"type":"object"}},"$id":"https://cdn.jsdelivr.net/npm/keepset@0.19.0/schemas/manifest.schema.json","$ref":"#/$defs/Manifest","$schema":"https://json-schema.org/draft/2020-12/schema"}