@loaders.gl/flatgeobuf 4.1.0-alpha.6 → 4.1.0-alpha.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. package/dist/LICENSE +9 -0
  2. package/dist/flatgeobuf/3.27.2/LICENSE +29 -0
  3. package/dist/flatgeobuf/3.27.2/README.md +45 -0
  4. package/dist/flatgeobuf/3.27.2/column-meta.d.ts +14 -0
  5. package/dist/flatgeobuf/3.27.2/column-meta.d.ts.map +1 -0
  6. package/dist/flatgeobuf/3.27.2/column-meta.js +2 -0
  7. package/dist/flatgeobuf/3.27.2/column-meta.js.map +1 -0
  8. package/dist/flatgeobuf/3.27.2/config.d.ts +10 -0
  9. package/dist/flatgeobuf/3.27.2/config.d.ts.map +1 -0
  10. package/dist/flatgeobuf/3.27.2/config.js +16 -0
  11. package/dist/flatgeobuf/3.27.2/config.js.map +1 -0
  12. package/dist/flatgeobuf/3.27.2/constants.d.ts +3 -0
  13. package/dist/flatgeobuf/3.27.2/constants.d.ts.map +1 -0
  14. package/dist/flatgeobuf/3.27.2/constants.js +3 -0
  15. package/dist/flatgeobuf/3.27.2/constants.js.map +1 -0
  16. package/dist/flatgeobuf/3.27.2/crs-meta.d.ts +9 -0
  17. package/dist/flatgeobuf/3.27.2/crs-meta.d.ts.map +1 -0
  18. package/dist/flatgeobuf/3.27.2/crs-meta.js +2 -0
  19. package/dist/flatgeobuf/3.27.2/crs-meta.js.map +1 -0
  20. package/dist/flatgeobuf/3.27.2/dumptree.d.ts +2 -0
  21. package/dist/flatgeobuf/3.27.2/dumptree.d.ts.map +1 -0
  22. package/dist/flatgeobuf/3.27.2/dumptree.js +61 -0
  23. package/dist/flatgeobuf/3.27.2/dumptree.js.map +1 -0
  24. package/dist/flatgeobuf/3.27.2/feature.d.ts +2 -0
  25. package/dist/flatgeobuf/3.27.2/feature.d.ts.map +1 -0
  26. package/dist/flatgeobuf/3.27.2/feature.js +3 -0
  27. package/dist/flatgeobuf/3.27.2/feature.js.map +1 -0
  28. package/dist/flatgeobuf/3.27.2/flat-geobuf/column-type.d.ts +18 -0
  29. package/dist/flatgeobuf/3.27.2/flat-geobuf/column-type.d.ts.map +1 -0
  30. package/dist/flatgeobuf/3.27.2/flat-geobuf/column-type.js +19 -0
  31. package/dist/flatgeobuf/3.27.2/flat-geobuf/column-type.js.map +1 -0
  32. package/dist/flatgeobuf/3.27.2/flat-geobuf/column.d.ts +39 -0
  33. package/dist/flatgeobuf/3.27.2/flat-geobuf/column.d.ts.map +1 -0
  34. package/dist/flatgeobuf/3.27.2/flat-geobuf/column.js +121 -0
  35. package/dist/flatgeobuf/3.27.2/flat-geobuf/column.js.map +1 -0
  36. package/dist/flatgeobuf/3.27.2/flat-geobuf/crs.d.ts +29 -0
  37. package/dist/flatgeobuf/3.27.2/flat-geobuf/crs.d.ts.map +1 -0
  38. package/dist/flatgeobuf/3.27.2/flat-geobuf/crs.js +79 -0
  39. package/dist/flatgeobuf/3.27.2/flat-geobuf/crs.js.map +1 -0
  40. package/dist/flatgeobuf/3.27.2/flat-geobuf/feature.d.ts +29 -0
  41. package/dist/flatgeobuf/3.27.2/flat-geobuf/feature.d.ts.map +1 -0
  42. package/dist/flatgeobuf/3.27.2/flat-geobuf/feature.js +95 -0
  43. package/dist/flatgeobuf/3.27.2/flat-geobuf/feature.js.map +1 -0
  44. package/dist/flatgeobuf/3.27.2/flat-geobuf/geometry-type.d.ts +21 -0
  45. package/dist/flatgeobuf/3.27.2/flat-geobuf/geometry-type.d.ts.map +1 -0
  46. package/dist/flatgeobuf/3.27.2/flat-geobuf/geometry-type.js +22 -0
  47. package/dist/flatgeobuf/3.27.2/flat-geobuf/geometry-type.js.map +1 -0
  48. package/dist/flatgeobuf/3.27.2/flat-geobuf/geometry.d.ts +75 -0
  49. package/dist/flatgeobuf/3.27.2/flat-geobuf/geometry.d.ts.map +1 -0
  50. package/dist/flatgeobuf/3.27.2/flat-geobuf/geometry.js +214 -0
  51. package/dist/flatgeobuf/3.27.2/flat-geobuf/geometry.js.map +1 -0
  52. package/dist/flatgeobuf/3.27.2/flat-geobuf/header.d.ts +59 -0
  53. package/dist/flatgeobuf/3.27.2/flat-geobuf/header.d.ts.map +1 -0
  54. package/dist/flatgeobuf/3.27.2/flat-geobuf/header.js +166 -0
  55. package/dist/flatgeobuf/3.27.2/flat-geobuf/header.js.map +1 -0
  56. package/dist/flatgeobuf/3.27.2/flat-geobuf.d.ts +6 -0
  57. package/dist/flatgeobuf/3.27.2/flat-geobuf.d.ts.map +1 -0
  58. package/dist/flatgeobuf/3.27.2/flat-geobuf.js +6 -0
  59. package/dist/flatgeobuf/3.27.2/flat-geobuf.js.map +1 -0
  60. package/dist/flatgeobuf/3.27.2/generic/feature.d.ts +19 -0
  61. package/dist/flatgeobuf/3.27.2/generic/feature.d.ts.map +1 -0
  62. package/dist/flatgeobuf/3.27.2/generic/feature.js +220 -0
  63. package/dist/flatgeobuf/3.27.2/generic/feature.js.map +1 -0
  64. package/dist/flatgeobuf/3.27.2/generic/featurecollection.d.ts +18 -0
  65. package/dist/flatgeobuf/3.27.2/generic/featurecollection.d.ts.map +1 -0
  66. package/dist/flatgeobuf/3.27.2/generic/featurecollection.js +162 -0
  67. package/dist/flatgeobuf/3.27.2/generic/featurecollection.js.map +1 -0
  68. package/dist/flatgeobuf/3.27.2/generic/geometry.d.ts +33 -0
  69. package/dist/flatgeobuf/3.27.2/generic/geometry.d.ts.map +1 -0
  70. package/dist/flatgeobuf/3.27.2/generic/geometry.js +81 -0
  71. package/dist/flatgeobuf/3.27.2/generic/geometry.js.map +1 -0
  72. package/dist/flatgeobuf/3.27.2/generic/header.d.ts +5 -0
  73. package/dist/flatgeobuf/3.27.2/generic/header.d.ts.map +1 -0
  74. package/dist/flatgeobuf/3.27.2/generic/header.js +27 -0
  75. package/dist/flatgeobuf/3.27.2/generic/header.js.map +1 -0
  76. package/dist/flatgeobuf/3.27.2/generic.d.ts +16 -0
  77. package/dist/flatgeobuf/3.27.2/generic.d.ts.map +1 -0
  78. package/dist/flatgeobuf/3.27.2/generic.js +9 -0
  79. package/dist/flatgeobuf/3.27.2/generic.js.map +1 -0
  80. package/dist/flatgeobuf/3.27.2/geojson/feature.d.ts +8 -0
  81. package/dist/flatgeobuf/3.27.2/geojson/feature.d.ts.map +1 -0
  82. package/dist/flatgeobuf/3.27.2/geojson/feature.js +13 -0
  83. package/dist/flatgeobuf/3.27.2/geojson/feature.js.map +1 -0
  84. package/dist/flatgeobuf/3.27.2/geojson/featurecollection.d.ts +8 -0
  85. package/dist/flatgeobuf/3.27.2/geojson/featurecollection.d.ts.map +1 -0
  86. package/dist/flatgeobuf/3.27.2/geojson/featurecollection.js +54 -0
  87. package/dist/flatgeobuf/3.27.2/geojson/featurecollection.js.map +1 -0
  88. package/dist/flatgeobuf/3.27.2/geojson/geometry.d.ts +13 -0
  89. package/dist/flatgeobuf/3.27.2/geojson/geometry.d.ts.map +1 -0
  90. package/dist/flatgeobuf/3.27.2/geojson/geometry.js +118 -0
  91. package/dist/flatgeobuf/3.27.2/geojson/geometry.js.map +1 -0
  92. package/dist/flatgeobuf/3.27.2/geojson.d.ts +17 -0
  93. package/dist/flatgeobuf/3.27.2/geojson.d.ts.map +1 -0
  94. package/dist/flatgeobuf/3.27.2/geojson.js +10 -0
  95. package/dist/flatgeobuf/3.27.2/geojson.js.map +1 -0
  96. package/dist/flatgeobuf/3.27.2/header-meta.d.ts +17 -0
  97. package/dist/flatgeobuf/3.27.2/header-meta.d.ts.map +1 -0
  98. package/dist/flatgeobuf/3.27.2/header-meta.js +46 -0
  99. package/dist/flatgeobuf/3.27.2/header-meta.js.map +1 -0
  100. package/dist/flatgeobuf/3.27.2/header.d.ts +2 -0
  101. package/dist/flatgeobuf/3.27.2/header.d.ts.map +1 -0
  102. package/dist/flatgeobuf/3.27.2/header.js +3 -0
  103. package/dist/flatgeobuf/3.27.2/header.js.map +1 -0
  104. package/dist/flatgeobuf/3.27.2/http-reader.d.ts +41 -0
  105. package/dist/flatgeobuf/3.27.2/http-reader.d.ts.map +1 -0
  106. package/dist/flatgeobuf/3.27.2/http-reader.js +193 -0
  107. package/dist/flatgeobuf/3.27.2/http-reader.js.map +1 -0
  108. package/dist/flatgeobuf/3.27.2/index.d.ts +14 -0
  109. package/dist/flatgeobuf/3.27.2/index.d.ts.map +1 -0
  110. package/dist/flatgeobuf/3.27.2/index.js +16 -0
  111. package/dist/flatgeobuf/3.27.2/index.js.map +1 -0
  112. package/dist/flatgeobuf/3.27.2/logger.d.ts +15 -0
  113. package/dist/flatgeobuf/3.27.2/logger.d.ts.map +1 -0
  114. package/dist/flatgeobuf/3.27.2/logger.js +65 -0
  115. package/dist/flatgeobuf/3.27.2/logger.js.map +1 -0
  116. package/dist/flatgeobuf/3.27.2/packedrtree.d.ts +43 -0
  117. package/dist/flatgeobuf/3.27.2/packedrtree.d.ts.map +1 -0
  118. package/dist/flatgeobuf/3.27.2/packedrtree.js +138 -0
  119. package/dist/flatgeobuf/3.27.2/packedrtree.js.map +1 -0
  120. package/dist/flatgeobuf/3.27.2/streams/utils.d.ts +7 -0
  121. package/dist/flatgeobuf/3.27.2/streams/utils.d.ts.map +1 -0
  122. package/dist/flatgeobuf/3.27.2/streams/utils.js +67 -0
  123. package/dist/flatgeobuf/3.27.2/streams/utils.js.map +1 -0
  124. package/dist/flatgeobuf/3.27.2/types/jsts.d.ts +23 -0
  125. package/dist/flatgeobuf/3.27.2/types/slice.d.ts +16 -0
  126. package/dist/flatgeobuf-loader.js +1 -1
  127. package/dist/index.cjs +1547 -6
  128. package/dist/lib/binary-geometries.d.ts +1 -1
  129. package/dist/lib/binary-geometries.d.ts.map +1 -1
  130. package/dist/lib/binary-geometries.js.map +1 -1
  131. package/dist/lib/get-schema-from-fgb-header.d.ts +1 -1
  132. package/dist/lib/get-schema-from-fgb-header.d.ts.map +1 -1
  133. package/dist/lib/get-schema-from-fgb-header.js.map +1 -1
  134. package/dist/lib/parse-flatgeobuf.d.ts +1 -1
  135. package/dist/lib/parse-flatgeobuf.d.ts.map +1 -1
  136. package/dist/lib/parse-flatgeobuf.js +3 -3
  137. package/dist/lib/parse-flatgeobuf.js.map +1 -1
  138. package/package.json +8 -5
  139. package/src/LICENSE +9 -0
  140. package/src/flatgeobuf/3.27.2/LICENSE +29 -0
  141. package/src/flatgeobuf/3.27.2/README.md +45 -0
  142. package/src/flatgeobuf/3.27.2/column-meta.ts +14 -0
  143. package/src/flatgeobuf/3.27.2/config.ts +19 -0
  144. package/src/flatgeobuf/3.27.2/constants.ts +4 -0
  145. package/src/flatgeobuf/3.27.2/crs-meta.ts +8 -0
  146. package/src/flatgeobuf/3.27.2/dumptree.ts +82 -0
  147. package/src/flatgeobuf/3.27.2/feature.ts +3 -0
  148. package/src/flatgeobuf/3.27.2/flat-geobuf/column-type.ts +19 -0
  149. package/src/flatgeobuf/3.27.2/flat-geobuf/column.ts +170 -0
  150. package/src/flatgeobuf/3.27.2/flat-geobuf/crs.ts +114 -0
  151. package/src/flatgeobuf/3.27.2/flat-geobuf/feature.ts +140 -0
  152. package/src/flatgeobuf/3.27.2/flat-geobuf/geometry-type.ts +22 -0
  153. package/src/flatgeobuf/3.27.2/flat-geobuf/geometry.ts +382 -0
  154. package/src/flatgeobuf/3.27.2/flat-geobuf/header.ts +246 -0
  155. package/src/flatgeobuf/3.27.2/flat-geobuf.ts +7 -0
  156. package/src/flatgeobuf/3.27.2/generic/feature.ts +243 -0
  157. package/src/flatgeobuf/3.27.2/generic/featurecollection.ts +227 -0
  158. package/src/flatgeobuf/3.27.2/generic/geometry.ts +120 -0
  159. package/src/flatgeobuf/3.27.2/generic/header.ts +32 -0
  160. package/src/flatgeobuf/3.27.2/generic.ts +33 -0
  161. package/src/flatgeobuf/3.27.2/geojson/feature.ts +20 -0
  162. package/src/flatgeobuf/3.27.2/geojson/featurecollection.ts +101 -0
  163. package/src/flatgeobuf/3.27.2/geojson/geometry.ts +142 -0
  164. package/src/flatgeobuf/3.27.2/geojson.ts +37 -0
  165. package/src/flatgeobuf/3.27.2/header-meta.ts +66 -0
  166. package/src/flatgeobuf/3.27.2/header.ts +3 -0
  167. package/src/flatgeobuf/3.27.2/http-reader.ts +351 -0
  168. package/src/flatgeobuf/3.27.2/index.ts +19 -0
  169. package/src/flatgeobuf/3.27.2/logger.ts +51 -0
  170. package/src/flatgeobuf/3.27.2/packedrtree.ts +255 -0
  171. package/src/flatgeobuf/3.27.2/streams/utils.ts +80 -0
  172. package/src/flatgeobuf/3.27.2/types/jsts.d.ts +23 -0
  173. package/src/flatgeobuf/3.27.2/types/slice.d.ts +16 -0
  174. package/src/lib/binary-geometries.ts +2 -2
  175. package/src/lib/get-schema-from-fgb-header.ts +1 -1
  176. package/src/lib/parse-flatgeobuf.ts +4 -4
@@ -0,0 +1,101 @@
1
+ import ColumnMeta from '../column-meta.js';
2
+ import HeaderMeta from '../header-meta.js';
3
+
4
+ import {fromFeature} from './feature.js';
5
+ import {parseGeometry, parseGC} from './geometry.js';
6
+ import {
7
+ buildHeader,
8
+ deserialize as genericDeserialize,
9
+ deserializeStream as genericDeserializeStream,
10
+ deserializeFiltered as genericDeserializeFiltered,
11
+ mapColumn
12
+ } from '../generic/featurecollection.js';
13
+ import {Rect} from '../packedrtree.js';
14
+ import {buildFeature, IProperties} from '../generic/feature.js';
15
+ import {HeaderMetaFn} from '../generic.js';
16
+ import {magicbytes} from '../constants.js';
17
+ import {inferGeometryType} from '../generic/header.js';
18
+
19
+ import {
20
+ FeatureCollection as GeoJsonFeatureCollection,
21
+ // Point,
22
+ // MultiPoint,
23
+ // LineString,
24
+ // MultiLineString,
25
+ // Polygon,
26
+ // MultiPolygon,
27
+ GeometryCollection
28
+ } from 'geojson';
29
+
30
+ export function serialize(featurecollection: GeoJsonFeatureCollection): Uint8Array {
31
+ const headerMeta = introspectHeaderMeta(featurecollection);
32
+ const header = buildHeader(headerMeta);
33
+ const features: Uint8Array[] = featurecollection.features.map((f) =>
34
+ buildFeature(
35
+ f.geometry.type === 'GeometryCollection'
36
+ ? parseGC(f.geometry as GeometryCollection)
37
+ : parseGeometry(f.geometry),
38
+ f.properties as IProperties,
39
+ headerMeta
40
+ )
41
+ );
42
+ const featuresLength = features.map((f) => f.length).reduce((a, b) => a + b);
43
+ const uint8 = new Uint8Array(magicbytes.length + header.length + featuresLength);
44
+ uint8.set(header, magicbytes.length);
45
+ let offset = magicbytes.length + header.length;
46
+ for (const feature of features) {
47
+ uint8.set(feature, offset);
48
+ offset += feature.length;
49
+ }
50
+ uint8.set(magicbytes);
51
+ return uint8;
52
+ }
53
+
54
+ export function deserialize(
55
+ bytes: Uint8Array,
56
+ headerMetaFn?: HeaderMetaFn
57
+ ): GeoJsonFeatureCollection {
58
+ const features = genericDeserialize(bytes, fromFeature, headerMetaFn);
59
+ return {
60
+ type: 'FeatureCollection',
61
+ features
62
+ } as GeoJsonFeatureCollection;
63
+ }
64
+
65
+ export function deserializeStream(
66
+ stream: ReadableStream,
67
+ headerMetaFn?: HeaderMetaFn
68
+ ): AsyncGenerator<any, void, unknown> {
69
+ return genericDeserializeStream(stream, fromFeature, headerMetaFn);
70
+ }
71
+
72
+ export function deserializeFiltered(
73
+ url: string,
74
+ rect: Rect,
75
+ headerMetaFn?: HeaderMetaFn
76
+ ): AsyncGenerator<any, void, unknown> {
77
+ return genericDeserializeFiltered(url, rect, fromFeature, headerMetaFn);
78
+ }
79
+
80
+ function introspectHeaderMeta(featurecollection: GeoJsonFeatureCollection): HeaderMeta {
81
+ const feature = featurecollection.features[0];
82
+ const properties = feature.properties;
83
+
84
+ let columns: ColumnMeta[] | null = null;
85
+ if (properties) columns = Object.keys(properties).map((k) => mapColumn(properties, k));
86
+
87
+ const geometryType = inferGeometryType(featurecollection.features);
88
+ const headerMeta: HeaderMeta = {
89
+ geometryType,
90
+ columns,
91
+ envelope: null,
92
+ featuresCount: featurecollection.features.length,
93
+ indexNodeSize: 0,
94
+ crs: null,
95
+ title: null,
96
+ description: null,
97
+ metadata: null
98
+ };
99
+
100
+ return headerMeta;
101
+ }
@@ -0,0 +1,142 @@
1
+ import {GeometryType} from '../flat-geobuf/geometry-type.js';
2
+ import {Geometry} from '../flat-geobuf/geometry.js';
3
+
4
+ import {
5
+ Geometry as GeoJsonGeometry,
6
+ Point,
7
+ MultiPoint,
8
+ LineString,
9
+ MultiLineString,
10
+ Polygon,
11
+ MultiPolygon,
12
+ GeometryCollection
13
+ } from 'geojson';
14
+
15
+ import {IParsedGeometry, flat, pairFlatCoordinates, toGeometryType} from '../generic/geometry.js';
16
+
17
+ export interface IGeoJsonGeometry {
18
+ type: string;
19
+ coordinates: number[] | number[][] | number[][][] | number[][][][];
20
+ geometries?: IGeoJsonGeometry[];
21
+ }
22
+
23
+ export function parseGeometry(
24
+ geometry: Point | MultiPoint | LineString | MultiLineString | Polygon | MultiPolygon
25
+ ): IParsedGeometry {
26
+ const cs = geometry.coordinates;
27
+ const xy: number[] = [];
28
+ const z: number[] = [];
29
+ let ends: number[] | undefined;
30
+ let parts: IParsedGeometry[] | undefined;
31
+ const type: GeometryType = toGeometryType(geometry.type);
32
+ let end = 0;
33
+ switch (geometry.type) {
34
+ case 'Point':
35
+ flat(cs, xy, z);
36
+ break;
37
+ case 'MultiPoint':
38
+ case 'LineString':
39
+ flat(cs as number[][], xy, z);
40
+ break;
41
+ case 'MultiLineString':
42
+ case 'Polygon': {
43
+ const css = cs as number[][][];
44
+ flat(css, xy, z);
45
+ if (css.length > 1) ends = css.map((c) => (end += c.length));
46
+ break;
47
+ }
48
+ case 'MultiPolygon': {
49
+ const csss = cs as number[][][][];
50
+ const geometries = csss.map((coordinates) => ({
51
+ type: 'Polygon',
52
+ coordinates
53
+ })) as Polygon[];
54
+ parts = geometries.map(parseGeometry);
55
+ break;
56
+ }
57
+ }
58
+ return {
59
+ xy,
60
+ z: z.length > 0 ? z : undefined,
61
+ ends,
62
+ type,
63
+ parts
64
+ } as IParsedGeometry;
65
+ }
66
+
67
+ export function parseGC(geometry: GeometryCollection): IParsedGeometry {
68
+ const type: GeometryType = toGeometryType(geometry.type);
69
+ const parts: IParsedGeometry[] = [];
70
+ for (let i = 0; i < geometry.geometries.length; i++) {
71
+ const g = geometry.geometries[i];
72
+ if (g.type === 'GeometryCollection') parts.push(parseGC(g));
73
+ else parts.push(parseGeometry(g));
74
+ }
75
+ return {
76
+ type,
77
+ parts
78
+ } as IParsedGeometry;
79
+ }
80
+
81
+ function extractParts(xy: Float64Array, z: Float64Array, ends: Uint32Array) {
82
+ if (!ends || ends.length === 0) return [pairFlatCoordinates(xy, z)];
83
+ let s = 0;
84
+ const xySlices = Array.from(ends).map((e) => xy.slice(s, (s = e << 1)));
85
+ let zSlices: Float64Array[];
86
+ if (z) {
87
+ s = 0;
88
+ zSlices = Array.from(ends).map((e) => z.slice(s, (s = e)));
89
+ }
90
+ return xySlices.map((xy, i) => pairFlatCoordinates(xy, zSlices ? zSlices[i] : undefined));
91
+ }
92
+
93
+ function toGeoJsonCoordinates(geometry: Geometry, type: GeometryType) {
94
+ const xy = geometry.xyArray() as Float64Array;
95
+ const z = geometry.zArray() as Float64Array;
96
+ switch (type) {
97
+ case GeometryType.Point: {
98
+ const a = Array.from(xy);
99
+ if (z) a.push(z[0]);
100
+ return a;
101
+ }
102
+ case GeometryType.MultiPoint:
103
+ case GeometryType.LineString:
104
+ return pairFlatCoordinates(xy, z);
105
+ case GeometryType.MultiLineString:
106
+ return extractParts(xy, z, geometry.endsArray() as Uint32Array);
107
+ case GeometryType.Polygon:
108
+ return extractParts(xy, z, geometry.endsArray() as Uint32Array);
109
+ }
110
+ }
111
+
112
+ export function fromGeometry(geometry: Geometry, headerType: GeometryType): GeoJsonGeometry {
113
+ let type = headerType;
114
+ if (type === GeometryType.Unknown) {
115
+ type = geometry.type();
116
+ }
117
+ if (type === GeometryType.GeometryCollection) {
118
+ const geometries: GeoJsonGeometry[] = [];
119
+ for (let i = 0; i < geometry.partsLength(); i++) {
120
+ const part = geometry.parts(i) as Geometry;
121
+ const partType = part.type() as GeometryType;
122
+ geometries.push(fromGeometry(part, partType));
123
+ }
124
+ return {
125
+ type: GeometryType[type],
126
+ geometries
127
+ } as GeoJsonGeometry;
128
+ } else if (type === GeometryType.MultiPolygon) {
129
+ const geometries: GeoJsonGeometry[] = [];
130
+ for (let i = 0; i < geometry.partsLength(); i++)
131
+ geometries.push(fromGeometry(geometry.parts(i) as Geometry, GeometryType.Polygon));
132
+ return {
133
+ type: GeometryType[type],
134
+ coordinates: geometries.map((g) => (g as Polygon).coordinates)
135
+ } as GeoJsonGeometry;
136
+ }
137
+ const coordinates = toGeoJsonCoordinates(geometry, type);
138
+ return {
139
+ type: GeometryType[type],
140
+ coordinates
141
+ } as GeoJsonGeometry;
142
+ }
@@ -0,0 +1,37 @@
1
+ import {
2
+ deserialize as fcDeserialize,
3
+ deserializeStream as fcDeserializeStream,
4
+ deserializeFiltered as fcDeserializeFiltered,
5
+ serialize as fcSerialize
6
+ } from './geojson/featurecollection.js';
7
+
8
+ import {FeatureCollection as GeoJsonFeatureCollection} from 'geojson';
9
+
10
+ import {Rect} from './packedrtree.js';
11
+ import {IGeoJsonFeature} from './geojson/feature.js';
12
+ import {HeaderMetaFn} from './generic.js';
13
+
14
+ /**
15
+ * Serialize GeoJSON to FlatGeobuf
16
+ * @param geojson GeoJSON object to serialize
17
+ */
18
+ export function serialize(geojson: GeoJsonFeatureCollection): Uint8Array {
19
+ const bytes = fcSerialize(geojson);
20
+ return bytes;
21
+ }
22
+
23
+ /**
24
+ * Deserialize FlatGeobuf into GeoJSON
25
+ * @param input Input byte array, stream or string
26
+ * @param rect Filter rectangle
27
+ * @param headerMetaFn Callback that will recieve header metadata when available
28
+ */
29
+ export function deserialize(
30
+ input: Uint8Array | ReadableStream | string,
31
+ rect?: Rect,
32
+ headerMetaFn?: HeaderMetaFn
33
+ ): GeoJsonFeatureCollection | AsyncGenerator<IGeoJsonFeature> {
34
+ if (input instanceof Uint8Array) return fcDeserialize(input, headerMetaFn);
35
+ else if (input instanceof ReadableStream) return fcDeserializeStream(input, headerMetaFn);
36
+ return fcDeserializeFiltered(input, rect as Rect, headerMetaFn);
37
+ }
@@ -0,0 +1,66 @@
1
+ import * as flatbuffers from 'flatbuffers';
2
+
3
+ import ColumnMeta from './column-meta.js';
4
+ import CrsMeta from './crs-meta.js';
5
+ import {GeometryType} from './flat-geobuf/geometry-type.js';
6
+ import {Header} from './flat-geobuf/header.js';
7
+
8
+ export default interface HeaderMeta {
9
+ geometryType: GeometryType;
10
+ columns: ColumnMeta[] | null;
11
+ envelope: Float64Array | null;
12
+ featuresCount: number;
13
+ indexNodeSize: number;
14
+ crs: CrsMeta | null;
15
+ title: string | null;
16
+ description: string | null;
17
+ metadata: string | null;
18
+ }
19
+
20
+ export function fromByteBuffer(bb: flatbuffers.ByteBuffer): HeaderMeta {
21
+ const header = Header.getRootAsHeader(bb);
22
+ const featuresCount = header.featuresCount();
23
+ const indexNodeSize = header.indexNodeSize();
24
+
25
+ const columns: ColumnMeta[] = [];
26
+ for (let j = 0; j < header.columnsLength(); j++) {
27
+ const column = header.columns(j);
28
+ if (!column) throw new Error('Column unexpectedly missing');
29
+ if (!column.name()) throw new Error('Column name unexpectedly missing');
30
+ columns.push({
31
+ name: column.name() as string,
32
+ type: column.type(),
33
+ title: column.title(),
34
+ description: column.description(),
35
+ width: column.width(),
36
+ precision: column.precision(),
37
+ scale: column.scale(),
38
+ nullable: column.nullable(),
39
+ unique: column.unique(),
40
+ primary_key: column.primaryKey()
41
+ });
42
+ }
43
+ const crs = header.crs();
44
+ const crsMeta: CrsMeta | null = crs
45
+ ? {
46
+ org: crs.org(),
47
+ code: crs.code(),
48
+ name: crs.name(),
49
+ description: crs.description(),
50
+ wkt: crs.wkt(),
51
+ code_string: crs.codeString()
52
+ }
53
+ : null;
54
+ const headerMeta: HeaderMeta = {
55
+ geometryType: header.geometryType(),
56
+ columns,
57
+ envelope: null,
58
+ featuresCount: Number(featuresCount),
59
+ indexNodeSize,
60
+ crs: crsMeta,
61
+ title: header.title(),
62
+ description: header.description(),
63
+ metadata: header.metadata()
64
+ };
65
+ return headerMeta;
66
+ }
@@ -0,0 +1,3 @@
1
+ // automatically generated by the FlatBuffers compiler, do not modify
2
+
3
+ export * as FlatGeobuf from './flat-geobuf.js';
@@ -0,0 +1,351 @@
1
+ import * as flatbuffers from 'flatbuffers';
2
+ import {Repeater} from '@repeaterjs/repeater';
3
+
4
+ import {
5
+ Rect,
6
+ calcTreeSize,
7
+ DEFAULT_NODE_SIZE,
8
+ NODE_ITEM_BYTE_LEN,
9
+ streamSearch
10
+ } from './packedrtree.js';
11
+ import {magicbytes, SIZE_PREFIX_LEN} from './constants.js';
12
+ import Config from './config.js';
13
+ import Logger from './logger.js';
14
+ import HeaderMeta, {fromByteBuffer} from './header-meta.js';
15
+ import {Feature} from './flat-geobuf/feature.js';
16
+
17
+ export class HttpReader {
18
+ private headerClient: BufferedHttpRangeClient;
19
+ public header: HeaderMeta;
20
+ private headerLength: number;
21
+ private indexLength: number;
22
+
23
+ constructor(
24
+ headerClient: BufferedHttpRangeClient,
25
+ header: HeaderMeta,
26
+ headerLength: number,
27
+ indexLength: number
28
+ ) {
29
+ this.headerClient = headerClient;
30
+ this.header = header;
31
+ this.headerLength = headerLength;
32
+ this.indexLength = indexLength;
33
+ }
34
+
35
+ // Fetch the header, preparing the reader to read Feature data.
36
+ //
37
+ // and potentially some opportunistic fetching of the index.
38
+ static async open(url: string): Promise<HttpReader> {
39
+ // In reality, the header is probably less than half this size, but
40
+ // better to overshoot and fetch an extra kb rather than have to issue
41
+ // a second request.
42
+ const assumedHeaderLength = 2024;
43
+
44
+ const headerClient = new BufferedHttpRangeClient(url);
45
+
46
+ // Immediately following the header is the optional spatial index, we deliberately fetch
47
+ // a small part of that to skip subsequent requests.
48
+ const assumedIndexLength = (() => {
49
+ // The actual branching factor will be in the header, but since we
50
+ // don't have the header yet, we just guess. The consequence of
51
+ // getting this wrong isn't terminal, it only means we may be
52
+ // fetching slightly more than we need or that we need to make an
53
+ // extra request later.
54
+ const assumedBranchingFactor = DEFAULT_NODE_SIZE;
55
+
56
+ // NOTE: each layer is exponentially larger
57
+ const prefetchedLayers = 3;
58
+
59
+ let result = 0;
60
+ let i: number;
61
+ for (i = 0; i < prefetchedLayers; i++) {
62
+ const layer_width = assumedBranchingFactor ** i * NODE_ITEM_BYTE_LEN;
63
+ result += layer_width;
64
+ }
65
+ return result;
66
+ })();
67
+
68
+ const minReqLength = assumedHeaderLength + assumedIndexLength;
69
+ Logger.debug(
70
+ `fetching header. minReqLength: ${minReqLength} (assumedHeaderLength: ${assumedHeaderLength}, assumedIndexLength: ${assumedIndexLength})`
71
+ );
72
+
73
+ {
74
+ const bytes = new Uint8Array(await headerClient.getRange(0, 8, minReqLength, 'header'));
75
+ if (!bytes.subarray(0, 3).every((v, i) => magicbytes[i] === v)) {
76
+ Logger.error(`bytes: ${bytes} != ${magicbytes}`);
77
+ throw new Error('Not a FlatGeobuf file');
78
+ }
79
+ Logger.debug('magic bytes look good');
80
+ }
81
+
82
+ let headerLength: number;
83
+ {
84
+ const bytes = await headerClient.getRange(8, 4, minReqLength, 'header');
85
+ headerLength = new DataView(bytes).getUint32(0, true);
86
+ const HEADER_MAX_BUFFER_SIZE = 1048576 * 10;
87
+ if (headerLength > HEADER_MAX_BUFFER_SIZE || headerLength < 8) {
88
+ // minimum size check avoids panic in FlatBuffers header decoding
89
+ throw new Error('Invalid header size');
90
+ }
91
+ Logger.debug(`headerLength: ${headerLength}`);
92
+ }
93
+
94
+ const bytes = await headerClient.getRange(12, headerLength, minReqLength, 'header');
95
+ const bb = new flatbuffers.ByteBuffer(new Uint8Array(bytes));
96
+ const header = fromByteBuffer(bb);
97
+
98
+ const indexLength = calcTreeSize(header.featuresCount, header.indexNodeSize);
99
+
100
+ Logger.debug('completed: opening http reader');
101
+ return new HttpReader(headerClient, header, headerLength, indexLength);
102
+ }
103
+
104
+ async *selectBbox(rect: Rect): AsyncGenerator<Feature, void, unknown> {
105
+ // Read R-Tree index and build filter for features within bbox
106
+ const lengthBeforeTree = this.lengthBeforeTree();
107
+
108
+ const bufferedClient = this.headerClient;
109
+ const readNode = async function (offsetIntoTree: number, size: number): Promise<ArrayBuffer> {
110
+ const minReqLength = 0;
111
+ return bufferedClient.getRange(
112
+ lengthBeforeTree + offsetIntoTree,
113
+ size,
114
+ minReqLength,
115
+ 'index'
116
+ );
117
+ };
118
+
119
+ const batches: [number, number][][] = [];
120
+ let currentBatch: [number, number][] = [];
121
+ for await (const searchResult of streamSearch(
122
+ this.header.featuresCount,
123
+ this.header.indexNodeSize,
124
+ rect,
125
+ readNode
126
+ )) {
127
+ const [featureOffset, ,] = searchResult;
128
+ let [, , featureLength] = searchResult;
129
+ if (!featureLength) {
130
+ Logger.info('final feature');
131
+ // Normally we get the feature length by subtracting between
132
+ // adjacent nodes from the index, which we can't do for the
133
+ // _very_ last feature in a dataset.
134
+ //
135
+ // We could *guess* the size, but we'd risk overshooting the length,
136
+ // which will cause some webservers to return HTTP 416: Unsatisfiable range
137
+ //
138
+ // So instead we fetch only the final features byte length, stored in the
139
+ // first 4 bytes.
140
+ featureLength = 4;
141
+ }
142
+
143
+ if (currentBatch.length == 0) {
144
+ currentBatch.push([featureOffset, featureLength]);
145
+ continue;
146
+ }
147
+
148
+ const prevFeature = currentBatch[currentBatch.length - 1];
149
+ const gap = featureOffset - (prevFeature[0] + prevFeature[1]);
150
+ if (gap > Config.global.extraRequestThreshold()) {
151
+ Logger.info(`Pushing new feature batch, since gap ${gap} was too large`);
152
+ batches.push(currentBatch);
153
+ currentBatch = [];
154
+ }
155
+
156
+ currentBatch.push([featureOffset, featureLength]);
157
+ }
158
+ this.headerClient.logUsage('header+index');
159
+ if (currentBatch.length > 0) {
160
+ batches.push(currentBatch);
161
+ }
162
+
163
+ const promises: AsyncGenerator<Feature, any, any>[] = batches.flatMap(
164
+ (batch: [number, number][]) => this.readFeatureBatch(batch)
165
+ );
166
+
167
+ // Fetch all batches concurrently, yielding features as they become
168
+ // available, meaning the results may be intermixed.
169
+ yield* Repeater.merge(promises);
170
+ }
171
+
172
+ lengthBeforeTree(): number {
173
+ // FGB Layout is: [magicbytes (fixed), headerLength (i32), header (variable), Tree (variable), Features (variable)]
174
+ return magicbytes.length + SIZE_PREFIX_LEN + this.headerLength;
175
+ }
176
+
177
+ lengthBeforeFeatures(): number {
178
+ return this.lengthBeforeTree() + this.indexLength;
179
+ }
180
+
181
+ buildFeatureClient(): BufferedHttpRangeClient {
182
+ return new BufferedHttpRangeClient(this.headerClient.httpClient);
183
+ }
184
+
185
+ /**
186
+ * Fetch a batch of features in a single request, yielding each Feature
187
+ *
188
+ * `batch`: [offset, length] of features in the batch
189
+ */
190
+ async *readFeatureBatch(batch: [number, number][]): AsyncGenerator<Feature, void, unknown> {
191
+ const [firstFeatureOffset] = batch[0];
192
+ const [lastFeatureOffset, lastFeatureLength] = batch[batch.length - 1];
193
+
194
+ const batchStart = firstFeatureOffset;
195
+ const batchEnd = lastFeatureOffset + lastFeatureLength;
196
+ const batchSize = batchEnd - batchStart;
197
+
198
+ // A new feature client is needed for each batch to own the underlying buffer as features are yielded.
199
+ const featureClient = this.buildFeatureClient();
200
+
201
+ let minFeatureReqLength = batchSize;
202
+ for (const [featureOffset] of batch) {
203
+ yield await this.readFeature(featureClient, featureOffset, minFeatureReqLength);
204
+ // Only set minFeatureReqLength for the first request.
205
+ //
206
+ // This should only affect a batch that contains the final feature, otherwise
207
+ // we've calculated `batchSize` to get all the data we need for the batch.
208
+ // For the very final feature in a dataset, we don't know it's length, so we
209
+ // will end up executing an extra request for that batch.
210
+ minFeatureReqLength = 0;
211
+ }
212
+ featureClient.logUsage('feature');
213
+ }
214
+
215
+ async readFeature(
216
+ featureClient: BufferedHttpRangeClient,
217
+ featureOffset: number,
218
+ minFeatureReqLength: number
219
+ ): Promise<Feature> {
220
+ const offset = featureOffset + this.lengthBeforeFeatures();
221
+
222
+ let featureLength: number;
223
+ {
224
+ const bytes = await featureClient.getRange(offset, 4, minFeatureReqLength, 'feature length');
225
+ featureLength = new DataView(bytes).getUint32(0, true);
226
+ }
227
+
228
+ const byteBuffer = await featureClient.getRange(
229
+ offset + 4,
230
+ featureLength,
231
+ minFeatureReqLength,
232
+ 'feature data'
233
+ );
234
+ const bytes = new Uint8Array(byteBuffer);
235
+ const bytesAligned = new Uint8Array(featureLength + SIZE_PREFIX_LEN);
236
+ bytesAligned.set(bytes, SIZE_PREFIX_LEN);
237
+ const bb = new flatbuffers.ByteBuffer(bytesAligned);
238
+ bb.setPosition(SIZE_PREFIX_LEN);
239
+ return Feature.getRootAsFeature(bb);
240
+ }
241
+ }
242
+
243
+ class BufferedHttpRangeClient {
244
+ httpClient: HttpRangeClient;
245
+ bytesEverUsed = 0;
246
+ bytesEverFetched = 0;
247
+
248
+ private buffer: ArrayBuffer = new ArrayBuffer(0);
249
+
250
+ // Byte offset of `buffer` with respect to the beginning of the file being
251
+ // buffered
252
+ private head = 0;
253
+
254
+ constructor(source: string | HttpRangeClient) {
255
+ if (typeof source === 'string') {
256
+ this.httpClient = new HttpRangeClient(source);
257
+ } else if (source instanceof HttpRangeClient) {
258
+ this.httpClient = source;
259
+ } else {
260
+ throw new Error('Unknown source ');
261
+ }
262
+ }
263
+
264
+ async getRange(
265
+ start: number,
266
+ length: number,
267
+ minReqLength: number,
268
+ purpose: string
269
+ ): Promise<ArrayBuffer> {
270
+ this.bytesEverUsed += length;
271
+
272
+ const start_i = start - this.head;
273
+ const end_i = start_i + length;
274
+ if (start_i >= 0 && end_i <= this.buffer.byteLength) {
275
+ return this.buffer.slice(start_i, end_i);
276
+ }
277
+
278
+ const lengthToFetch = Math.max(length, minReqLength);
279
+
280
+ this.bytesEverFetched += lengthToFetch;
281
+ Logger.debug(`requesting for new Range: ${start}-${start + lengthToFetch - 1}`);
282
+ this.buffer = await this.httpClient.getRange(start, lengthToFetch, purpose);
283
+ this.head = start;
284
+
285
+ return this.buffer.slice(0, length);
286
+ }
287
+
288
+ logUsage(purpose: string): void {
289
+ const category = purpose.split(' ')[0];
290
+ const used = this.bytesEverUsed;
291
+ const requested = this.bytesEverFetched;
292
+ const efficiency = ((100.0 * used) / requested).toFixed(2);
293
+
294
+ Logger.info(`${category} bytes used/requested: ${used} / ${requested} = ${efficiency}%`);
295
+ }
296
+ }
297
+
298
+ class HttpRangeClient {
299
+ url: string;
300
+ requestsEverMade = 0;
301
+ bytesEverRequested = 0;
302
+
303
+ constructor(url: string) {
304
+ this.url = url;
305
+ }
306
+
307
+ async getRange(begin: number, length: number, purpose: string): Promise<ArrayBuffer> {
308
+ this.requestsEverMade += 1;
309
+ this.bytesEverRequested += length;
310
+
311
+ const range = `bytes=${begin}-${begin + length - 1}`;
312
+ Logger.info(
313
+ `request: #${this.requestsEverMade}, purpose: ${purpose}), bytes: (this_request: ${length}, ever: ${this.bytesEverRequested}), Range: ${range}`
314
+ );
315
+
316
+ const response = await fetch(this.url, {
317
+ headers: {
318
+ Range: range
319
+ // TODO: better parallelize requests on Chrome
320
+ //
321
+ // Safari and Firefox have no issue performing Range requests
322
+ // for a resource in parallel, but Chrome will stall a
323
+ // subsequent request to the resource until it's received the
324
+ // response headers of the prior request. So, it still allows
325
+ // some limited parallelization, but it's not ideal.
326
+ //
327
+ // This is seemingly an artifact of how Chrome manages caching
328
+ // and it might differ between platforms. We could work around it
329
+ // by setting the request header:
330
+ //
331
+ // 'Cache-Control': 'no-cache, no-store'
332
+ //
333
+ // This allows requests to be fully parallelized in Chrome, but
334
+ // then Chrome won't cache the response, so it seems not a
335
+ // great trade-off.
336
+ //
337
+ // Another work around would be to make each Range request for
338
+ // a separate URL by appending something like
339
+ // `?cache_buster=<range>` to the URL, but then Chrome will
340
+ // require an additional CORS preflight OPTIONS requests per
341
+ // Range, which is also not a great trade-off.
342
+ //
343
+ // See:
344
+ // https://bugs.chromium.org/p/chromium/issues/detail?id=969828&q=concurrent%20range%20requests&can=2
345
+ // https://stackoverflow.com/questions/27513994/chrome-stalls-when-making-multiple-requests-to-same-resource
346
+ }
347
+ });
348
+
349
+ return response.arrayBuffer();
350
+ }
351
+ }