@loaders.gl/flatgeobuf 4.1.0-alpha.1 → 4.1.0-alpha.11

package/src/flatgeobuf/3.27.2/generic/feature.ts

@@ -0,0 +1,243 @@
+import * as flatbuffers from 'flatbuffers';
+
+import ColumnMeta from '../column-meta';
+import {ColumnType} from '../flat-geobuf/column-type';
+import {Feature} from '../flat-geobuf/feature';
+import HeaderMeta from '../header-meta';
+import {buildGeometry, ISimpleGeometry, ICreateGeometry, IParsedGeometry} from './geometry';
+
+const textEncoder = new TextEncoder();
+const textDecoder = new TextDecoder();
+
+export interface IFeature {
+  getGeometry?(): ISimpleGeometry;
+  getProperties?(): any;
+  setProperties?(properties: Record<string, unknown>): any;
+}
+
+export interface ICreateFeature {
+  (geometry?: ISimpleGeometry, properties?: Record<string, unknown>): IFeature;
+}
+
+export interface IProperties {
+  [key: string]: boolean | number | string | any;
+}
+
+export function fromFeature(
+  feature: Feature,
+  header: HeaderMeta,
+  createGeometry: ICreateGeometry,
+  createFeature: ICreateFeature
+): IFeature {
+  const columns = header.columns;
+  const geometry = feature.geometry();
+  const simpleGeometry = createGeometry(geometry, header.geometryType);
+  const properties = parseProperties(feature, columns as ColumnMeta[]);
+  return createFeature(simpleGeometry, properties);
+}
+
+export function buildFeature(
+  geometry: IParsedGeometry,
+  properties: IProperties,
+  header: HeaderMeta
+): Uint8Array {
+  const columns = header.columns;
+  const builder = new flatbuffers.Builder();
+
+  let offset = 0;
+  let capacity = 1024;
+  let bytes = new Uint8Array(capacity);
+  let view = new DataView(bytes.buffer);
+
+  const prep = function (size: number) {
+    if (offset + size < capacity) return;
+    capacity = Math.max(capacity + size, capacity * 2);
+    const newBytes = new Uint8Array(capacity);
+    newBytes.set(bytes);
+    bytes = newBytes;
+    view = new DataView(bytes.buffer);
+  };
+
+  if (columns) {
+    for (let i = 0; i < columns.length; i++) {
+      const column = columns[i];
+      const value = properties[column.name];
+      if (value === null) continue;
+      prep(2);
+      view.setUint16(offset, i, true);
+      offset += 2;
+      switch (column.type) {
+        case ColumnType.Bool:
+          prep(1);
+          view.setUint8(offset, value as number);
+          offset += 1;
+          break;
+        case ColumnType.Short:
+          prep(2);
+          view.setInt16(offset, value as number, true);
+          offset += 2;
+          break;
+        case ColumnType.UShort:
+          prep(2);
+          view.setUint16(offset, value as number, true);
+          offset += 2;
+          break;
+        case ColumnType.Int:
+          prep(4);
+          view.setInt32(offset, value as number, true);
+          offset += 4;
+          break;
+        case ColumnType.UInt:
+          prep(4);
+          view.setUint32(offset, value as number, true);
+          offset += 4;
+          break;
+        case ColumnType.Long:
+          prep(8);
+          view.setBigInt64(offset, BigInt(value), true);
+          offset += 8;
+          break;
+        case ColumnType.Float:
+          prep(4);
+          view.setFloat32(offset, value as number, true);
+          offset += 4;
+          break;
+        case ColumnType.Double:
+          prep(8);
+          view.setFloat64(offset, value as number, true);
+          offset += 8;
+          break;
+        case ColumnType.DateTime:
+        case ColumnType.String: {
+          const str = textEncoder.encode(value);
+          prep(4);
+          view.setUint32(offset, str.length, true);
+          offset += 4;
+          prep(str.length);
+          bytes.set(str, offset);
+          offset += str.length;
+          break;
+        }
+        case ColumnType.Json: {
+          const str = textEncoder.encode(JSON.stringify(value));
+          prep(4);
+          view.setUint32(offset, str.length, true);
+          offset += 4;
+          prep(str.length);
+          bytes.set(str, offset);
+          offset += str.length;
+          break;
+        }
+        default:
+          throw new Error(`Unknown type ${column.type}`);
+      }
+    }
+  }
+
+  let propertiesOffset: number | null = null;
+  if (offset > 0)
+    propertiesOffset = Feature.createPropertiesVector(builder, bytes.slice(0, offset));
+
+  const geometryOffset = buildGeometry(builder, geometry);
+  Feature.startFeature(builder);
+  Feature.addGeometry(builder, geometryOffset);
+  if (propertiesOffset) Feature.addProperties(builder, propertiesOffset);
+  const featureOffset = Feature.endFeature(builder);
+  builder.finishSizePrefixed(featureOffset);
+  return builder.asUint8Array();
+}
+
+export function parseProperties(
+  feature: Feature,
+  columns?: ColumnMeta[] | null
+): Record<string, unknown> {
+  const properties: Record<string, unknown> = {};
+  if (!columns || columns.length === 0) return properties;
+  const array = feature.propertiesArray();
+  if (!array) return properties;
+  const view = new DataView(array.buffer, array.byteOffset);
+  const length = feature.propertiesLength();
+  let offset = 0;
+  while (offset < length) {
+    const i = view.getUint16(offset, true);
+    offset += 2;
+    const column = columns[i];
+    const name = column.name;
+    switch (column.type) {
+      case ColumnType.Bool: {
+        properties[name] = Boolean(view.getUint8(offset));
+        offset += 1;
+        break;
+      }
+      case ColumnType.Byte: {
+        properties[name] = view.getInt8(offset);
+        offset += 1;
+        break;
+      }
+      case ColumnType.UByte: {
+        properties[name] = view.getUint8(offset);
+        offset += 1;
+        break;
+      }
+      case ColumnType.Short: {
+        properties[name] = view.getInt16(offset, true);
+        offset += 2;
+        break;
+      }
+      case ColumnType.UShort: {
+        properties[name] = view.getUint16(offset, true);
+        offset += 2;
+        break;
+      }
+      case ColumnType.Int: {
+        properties[name] = view.getInt32(offset, true);
+        offset += 4;
+        break;
+      }
+      case ColumnType.UInt: {
+        properties[name] = view.getUint32(offset, true);
+        offset += 4;
+        break;
+      }
+      case ColumnType.Long: {
+        properties[name] = Number(view.getBigInt64(offset, true));
+        offset += 8;
+        break;
+      }
+      case ColumnType.ULong: {
+        properties[name] = Number(view.getBigUint64(offset, true));
+        offset += 8;
+        break;
+      }
+      case ColumnType.Float: {
+        properties[name] = view.getFloat32(offset, true);
+        offset += 4;
+        break;
+      }
+      case ColumnType.Double: {
+        properties[name] = view.getFloat64(offset, true);
+        offset += 8;
+        break;
+      }
+      case ColumnType.DateTime:
+      case ColumnType.String: {
+        const length = view.getUint32(offset, true);
+        offset += 4;
+        properties[name] = textDecoder.decode(array.subarray(offset, offset + length));
+        offset += length;
+        break;
+      }
+      case ColumnType.Json: {
+        const length = view.getUint32(offset, true);
+        offset += 4;
+        const str = textDecoder.decode(array.subarray(offset, offset + length));
+        properties[name] = JSON.parse(str);
+        offset += length;
+        break;
+      }
+      default:
+        throw new Error(`Unknown type ${column.type}`);
+    }
+  }
+  return properties;
+}

package/src/flatgeobuf/3.27.2/generic/featurecollection.ts

@@ -0,0 +1,227 @@
+import * as flatbuffers from 'flatbuffers';
+import slice from 'slice-source';
+
+import ColumnMeta from '../column-meta';
+
+import {Header} from '../flat-geobuf/header';
+
+import {Column} from '../flat-geobuf/column';
+import {ColumnType} from '../flat-geobuf/column-type';
+import {Feature} from '../flat-geobuf/feature';
+import HeaderMeta, {fromByteBuffer} from '../header-meta';
+
+import {buildFeature, IFeature} from './feature';
+import {HttpReader} from '../http-reader';
+import Logger from '../logger';
+import {Rect, calcTreeSize} from '../packedrtree';
+import {parseGeometry} from './geometry';
+import {HeaderMetaFn} from '../generic';
+import {magicbytes, SIZE_PREFIX_LEN} from '../constants';
+import {inferGeometryType} from './header';
+
+export type FromFeatureFn = (feature: Feature, header: HeaderMeta) => IFeature;
+type ReadFn = (size: number, purpose: string) => Promise<ArrayBuffer>;
+
+/**
+ * Serialize generic features to FlatGeobuf
+ * @param features
+ */
+export function serialize(features: IFeature[]): Uint8Array {
+  const headerMeta = introspectHeaderMeta(features);
+  const header = buildHeader(headerMeta);
+  const featureBuffers: Uint8Array[] = features.map((f) => {
+    if (!f.getGeometry) throw new Error('Missing getGeometry implementation');
+    if (!f.getProperties) throw new Error('Missing getProperties implementation');
+    return buildFeature(
+      parseGeometry(f.getGeometry(), headerMeta.geometryType),
+      f.getProperties(),
+      headerMeta
+    );
+  });
+  const featuresLength = featureBuffers.map((f) => f.length).reduce((a, b) => a + b);
+  const uint8 = new Uint8Array(magicbytes.length + header.length + featuresLength);
+  uint8.set(header, magicbytes.length);
+  let offset = magicbytes.length + header.length;
+  for (const feature of featureBuffers) {
+    uint8.set(feature, offset);
+    offset += feature.length;
+  }
+  uint8.set(magicbytes);
+  return uint8;
+}
+
+export function deserialize(
+  bytes: Uint8Array,
+  fromFeature: FromFeatureFn,
+  headerMetaFn?: HeaderMetaFn
+): IFeature[] {
+  if (!bytes.subarray(0, 3).every((v, i) => magicbytes[i] === v))
+    throw new Error('Not a FlatGeobuf file');
+
+  const bb = new flatbuffers.ByteBuffer(bytes);
+  const headerLength = bb.readUint32(magicbytes.length);
+  bb.setPosition(magicbytes.length + SIZE_PREFIX_LEN);
+
+  const headerMeta = fromByteBuffer(bb);
+  if (headerMetaFn) headerMetaFn(headerMeta);
+
+  let offset = magicbytes.length + SIZE_PREFIX_LEN + headerLength;
+
+  const {indexNodeSize, featuresCount} = headerMeta;
+  if (indexNodeSize > 0) offset += calcTreeSize(featuresCount, indexNodeSize);
+
+  const features: IFeature[] = [];
+  while (offset < bb.capacity()) {
+    const featureLength = bb.readUint32(offset);
+    bb.setPosition(offset + SIZE_PREFIX_LEN);
+    const feature = Feature.getRootAsFeature(bb);
+    features.push(fromFeature(feature, headerMeta));
+    offset += SIZE_PREFIX_LEN + featureLength;
+  }
+
+  return features;
+}
+
+export async function* deserializeStream(
+  stream: ReadableStream,
+  fromFeature: FromFeatureFn,
+  headerMetaFn?: HeaderMetaFn
+): AsyncGenerator<IFeature> {
+  const reader = slice(stream);
+  const read: ReadFn = async (size) => await reader.slice(size);
+
+  let bytes = new Uint8Array(await read(8, 'magic bytes'));
+  if (!bytes.subarray(0, 3).every((v, i) => magicbytes[i] === v))
+    throw new Error('Not a FlatGeobuf file');
+  bytes = new Uint8Array(await read(4, 'header length'));
+  let bb = new flatbuffers.ByteBuffer(bytes);
+  const headerLength = bb.readUint32(0);
+  bytes = new Uint8Array(await read(headerLength, 'header data'));
+  bb = new flatbuffers.ByteBuffer(bytes);
+
+  const headerMeta = fromByteBuffer(bb);
+  if (headerMetaFn) headerMetaFn(headerMeta);
+
+  const {indexNodeSize, featuresCount} = headerMeta;
+  if (indexNodeSize > 0) {
+    const treeSize = calcTreeSize(featuresCount, indexNodeSize);
+    await read(treeSize, 'entire index, w/o rect');
+  }
+  let feature: IFeature | undefined;
+  while ((feature = await readFeature(read, headerMeta, fromFeature))) yield feature;
+}
+
+export async function* deserializeFiltered(
+  url: string,
+  rect: Rect,
+  fromFeature: FromFeatureFn,
+  headerMetaFn?: HeaderMetaFn
+): AsyncGenerator<IFeature> {
+  const reader = await HttpReader.open(url);
+  Logger.debug('opened reader');
+  if (headerMetaFn) headerMetaFn(reader.header);
+
+  for await (const feature of reader.selectBbox(rect)) {
+    yield fromFeature(feature, reader.header);
+  }
+}
+
+async function readFeature(
+  read: ReadFn,
+  headerMeta: HeaderMeta,
+  fromFeature: FromFeatureFn
+): Promise<IFeature | undefined> {
+  let bytes = new Uint8Array(await read(4, 'feature length'));
+  if (bytes.byteLength === 0) return;
+  let bb = new flatbuffers.ByteBuffer(bytes);
+  const featureLength = bb.readUint32(0);
+  bytes = new Uint8Array(await read(featureLength, 'feature data'));
+  const bytesAligned = new Uint8Array(featureLength + 4);
+  bytesAligned.set(bytes, 4);
+  bb = new flatbuffers.ByteBuffer(bytesAligned);
+  bb.setPosition(SIZE_PREFIX_LEN);
+  const feature = Feature.getRootAsFeature(bb);
+  return fromFeature(feature, headerMeta);
+}
+
+function buildColumn(builder: flatbuffers.Builder, column: ColumnMeta): number {
+  const nameOffset = builder.createString(column.name);
+  Column.startColumn(builder);
+  Column.addName(builder, nameOffset);
+  Column.addType(builder, column.type);
+  return Column.endColumn(builder);
+}
+
+export function buildHeader(header: HeaderMeta): Uint8Array {
+  const builder = new flatbuffers.Builder();
+
+  let columnOffsets: number | null = null;
+  if (header.columns)
+    columnOffsets = Header.createColumnsVector(
+      builder,
+      header.columns.map((c) => buildColumn(builder, c))
+    );
+
+  const nameOffset = builder.createString('L1');
+
+  Header.startHeader(builder);
+  Header.addFeaturesCount(builder, BigInt(header.featuresCount));
+  Header.addGeometryType(builder, header.geometryType);
+  Header.addIndexNodeSize(builder, 0);
+  if (columnOffsets) Header.addColumns(builder, columnOffsets);
+  Header.addName(builder, nameOffset);
+  const offset = Header.endHeader(builder);
+  builder.finishSizePrefixed(offset);
+  return builder.asUint8Array();
+}
+
+function valueToType(value: boolean | number | string): ColumnType {
+  if (typeof value === 'boolean') return ColumnType.Bool;
+  else if (typeof value === 'number')
+    if (value % 1 === 0) return ColumnType.Int;
+    else return ColumnType.Double;
+  else if (typeof value === 'string') return ColumnType.String;
+  else if (value === null) return ColumnType.String;
+  else if (typeof value === 'object') return ColumnType.Json;
+  throw new Error(`Unknown type (value '${value}')`);
+}
+
+export function mapColumn(properties: any, k: string): ColumnMeta {
+  return {
+    name: k,
+    type: valueToType(properties[k]),
+    title: null,
+    description: null,
+    width: -1,
+    precision: -1,
+    scale: -1,
+    nullable: true,
+    unique: false,
+    primary_key: false
+  };
+}
+
+function introspectHeaderMeta(features: IFeature[]): HeaderMeta {
+  const sampleFeature = features[0];
+  const properties = sampleFeature.getProperties ? sampleFeature.getProperties() : {};
+
+  let columns: ColumnMeta[] | null = null;
+  if (properties)
+    columns = Object.keys(properties)
+      .filter((key) => key !== 'geometry')
+      .map((k) => mapColumn(properties, k));
+
+  const geometryType = inferGeometryType(features);
+  const headerMeta: HeaderMeta = {
+    geometryType,
+    columns,
+    envelope: null,
+    featuresCount: features.length,
+    indexNodeSize: 0,
+    crs: null,
+    title: null,
+    description: null,
+    metadata: null
+  };
+  return headerMeta;
+}

package/src/flatgeobuf/3.27.2/generic/geometry.ts

@@ -0,0 +1,120 @@
+import * as flatbuffers from 'flatbuffers';
+import {GeometryType} from '../flat-geobuf/geometry-type';
+import {Geometry} from '../flat-geobuf/geometry';
+
+export interface IParsedGeometry {
+  xy: number[];
+  z: number[];
+  ends: number[];
+  parts: IParsedGeometry[];
+  type: GeometryType;
+}
+
+export interface ISimpleGeometry {
+  getFlatCoordinates?(): number[];
+  getType(): string;
+}
+
+export interface IPolygon extends ISimpleGeometry {
+  getEnds(): number[];
+}
+
+export interface IMultiLineString extends ISimpleGeometry {
+  getEnds(): number[];
+}
+
+export interface IMultiPolygon extends ISimpleGeometry {
+  getEndss(): number[][];
+  getPolygons(): IPolygon[];
+}
+
+export interface ICreateGeometry {
+  (geometry: Geometry | null, type: GeometryType): ISimpleGeometry | undefined;
+}
+
+export function buildGeometry(builder: flatbuffers.Builder, parsedGeometry: IParsedGeometry): any {
+  const {xy, z, ends, parts, type} = parsedGeometry;
+
+  if (parts) {
+    const partOffsets = parts.map((part) => buildGeometry(builder, part));
+    const partsOffset = Geometry.createPartsVector(builder, partOffsets);
+    Geometry.startGeometry(builder);
+    Geometry.addParts(builder, partsOffset);
+    Geometry.addType(builder, type);
+    return Geometry.endGeometry(builder);
+  }
+
+  const xyOffset = Geometry.createXyVector(builder, xy);
+  let zOffset: number | undefined;
+  if (z) zOffset = Geometry.createZVector(builder, z);
+
+  let endsOffset: number | undefined;
+  if (ends) endsOffset = Geometry.createEndsVector(builder, ends);
+
+  Geometry.startGeometry(builder);
+  if (endsOffset) Geometry.addEnds(builder, endsOffset);
+  Geometry.addXy(builder, xyOffset);
+  if (zOffset) Geometry.addZ(builder, zOffset);
+  Geometry.addType(builder, type);
+  return Geometry.endGeometry(builder);
+}
+
+export function flat(a: any[], xy: number[], z: number[]): number[] | undefined {
+  if (a.length === 0) return;
+  if (Array.isArray(a[0])) {
+    for (const sa of a) flat(sa, xy, z);
+  } else if (a.length === 2) xy.push(...a);
+  else {
+    xy.push(a[0], a[1]);
+    z.push(a[2]);
+  }
+}
+
+export function parseGeometry(
+  geometry: ISimpleGeometry,
+  headerGeomType: GeometryType
+): IParsedGeometry {
+  let xy: number[] | undefined;
+  let ends: number[] | undefined;
+  let parts: IParsedGeometry[] | undefined;
+
+  let type = headerGeomType;
+  if (type === GeometryType.Unknown) {
+    type = toGeometryType(geometry.getType());
+  }
+
+  if (type === GeometryType.MultiLineString) {
+    if (geometry.getFlatCoordinates) xy = geometry.getFlatCoordinates();
+    const mlsEnds = (geometry as IMultiLineString).getEnds();
+    if (mlsEnds.length > 1) ends = mlsEnds.map((e) => e >> 1);
+  } else if (type === GeometryType.Polygon) {
+    if (geometry.getFlatCoordinates) xy = geometry.getFlatCoordinates();
+    const pEnds = (geometry as IPolygon).getEnds();
+    if (pEnds.length > 1) ends = pEnds.map((e) => e >> 1);
+  } else if (type === GeometryType.MultiPolygon) {
+    const mp = geometry as IMultiPolygon;
+    parts = mp.getPolygons().map((p) => parseGeometry(p, GeometryType.Polygon));
+  } else if (geometry.getFlatCoordinates) xy = geometry.getFlatCoordinates();
+  return {
+    xy,
+    ends,
+    type,
+    parts
+  } as IParsedGeometry;
+}
+
+export function pairFlatCoordinates(xy: Float64Array, z?: Float64Array): number[][] {
+  const newArray: number[][] = [];
+  for (let i = 0; i < xy.length; i += 2) {
+    const a = [xy[i], xy[i + 1]];
+    if (z) a.push(z[i >> 1]);
+    newArray.push(a);
+  }
+  return newArray;
+}
+
+export function toGeometryType(name?: string): GeometryType {
+  if (!name) return GeometryType.Unknown;
+  const type: GeometryType = (GeometryType as any)[name];
+  return type;
+}

package/src/flatgeobuf/3.27.2/generic/header.ts

@@ -0,0 +1,32 @@
+import {GeometryType} from '../flat-geobuf/geometry-type';
+import {toGeometryType} from '../generic/geometry';
+import {IFeature} from './feature';
+import {IGeoJsonFeature} from '../geojson/feature';
+
+function featureGeomType(feature: IFeature | IGeoJsonFeature): GeometryType {
+  if (feature.getGeometry) {
+    return toGeometryType(feature.getGeometry().getType());
+  }
+  return toGeometryType((feature as IGeoJsonFeature).geometry.type);
+}
+
+export function inferGeometryType(features: (IFeature | IGeoJsonFeature)[]): GeometryType {
+  let geometryType: GeometryType | undefined;
+
+  for (const f of features) {
+    if (geometryType === GeometryType.Unknown) {
+      break;
+    }
+
+    const gtype = featureGeomType(f);
+    if (geometryType === undefined) {
+      geometryType = gtype;
+    } else if (geometryType !== gtype) {
+      geometryType = GeometryType.Unknown;
+    }
+  }
+  if (geometryType === undefined) {
+    throw new Error('Could not infer geometry type for collection of features.');
+  }
+  return geometryType;
+}

package/src/flatgeobuf/3.27.2/generic.ts

@@ -0,0 +1,33 @@
+import {
+  deserialize as deserializeArray,
+  deserializeStream,
+  deserializeFiltered,
+  FromFeatureFn
+} from './generic/featurecollection';
+
+import {Rect} from './packedrtree';
+import {IFeature} from './generic/feature';
+import HeaderMeta from './header-meta';
+
+export type HeaderMetaFn = (headerMeta: HeaderMeta) => void;
+
+/**
+ * Deserialize FlatGeobuf into generic features
+ * @param input Input byte array, stream or string
+ * @param fromFeature Callback that recieves generic features
+ * @param rect Filter rectangle
+ */
+export function deserialize(
+  input: Uint8Array | ReadableStream | string,
+  fromFeature: FromFeatureFn,
+  rect?: Rect
+): any[] | AsyncGenerator<IFeature> {
+  if (input instanceof Uint8Array) return deserializeArray(input, fromFeature);
+  else if (input instanceof ReadableStream) return deserializeStream(input, fromFeature);
+  return deserializeFiltered(input, rect as Rect, fromFeature);
+}
+
+export {serialize} from './generic/featurecollection';
+
+export {GeometryType} from './flat-geobuf/geometry-type';
+export {ColumnType} from './flat-geobuf/column-type';

package/src/flatgeobuf/3.27.2/geojson/feature.ts

@@ -0,0 +1,20 @@
+import {Feature} from '../flat-geobuf/feature';
+import {Geometry} from '../flat-geobuf/geometry';
+import HeaderMeta from '../header-meta';
+import {fromGeometry} from './geometry';
+import {IFeature, parseProperties} from '../generic/feature';
+
+import {Feature as GeoJsonFeature} from 'geojson';
+
+export interface IGeoJsonFeature extends IFeature, GeoJsonFeature {}
+
+export function fromFeature(feature: Feature, header: HeaderMeta): IGeoJsonFeature {
+  const columns = header.columns;
+  const geometry = fromGeometry(feature.geometry() as Geometry, header.geometryType);
+  const geoJsonfeature: GeoJsonFeature = {
+    type: 'Feature',
+    geometry,
+    properties: parseProperties(feature, columns)
+  };
+  return geoJsonfeature;
+}