flatsql 0.4.0 → 0.4.3

package/README.md

@@ -250,6 +250,106 @@ FlatSQL ingests size-prefixed FlatBuffer streams:
 
 The 4-byte file identifier in each FlatBuffer determines which table receives the record.
 
+## Spatial Extensions
+
+FlatSQL includes a full spatial computation engine with SpatiaLite-compatible functions, polygon boolean operations, Voronoi/Delaunay tessellation, and coordinate transforms.
+
+### Spatial SQL Functions
+
+| Category | Functions |
+|----------|-----------|
+| **Point ops** | `geo_distance`, `geo_bearing`, `geo_destination`, `geo_midpoint`, `geo_area_bbox`, `geo_within_radius`, `geo_bbox_contains` |
+| **Geohash** | `geo_geohash_encode`, `geo_geohash_decode_lat`, `geo_geohash_decode_lon` |
+| **Geometry I/O** | `geo_from_text` (WKT→blob), `geo_as_text` (blob→WKT), `geo_from_geojson`, `geo_as_geojson` |
+| **Predicates** | `geo_contains` (point-in-polygon, ray casting with hole support) |
+| **Boolean ops** | `geo_intersection`, `geo_union`, `geo_difference`, `geo_sym_difference`, `geo_buffer` |
+| **Analysis** | `geo_area_geom`, `geo_centroid`, `geo_length_geom`, `geo_envelope`, `geo_convex_hull` |
+| **Voronoi/Delaunay** | `geo_voronoi`, `geo_delaunay` |
+| **Coord transforms** | `geo_to_ecef` (WGS84→ECEF), `geo_from_ecef` (ECEF→WGS84) |
+
+### Examples
+
+```sql
+-- Distance between NYC and DC (~328 km)
+SELECT geo_distance(40.7128, -74.0060, 38.9072, -77.0369);
+
+-- Bearing from NYC to LA
+SELECT geo_bearing(40.7128, -74.0060, 34.0522, -118.2437);
+
+-- Geohash encode/decode
+SELECT geo_geohash_encode(42.6, -5.6, 5);  -- 'ezs42'
+
+-- Point-in-polygon
+SELECT geo_contains('POLYGON((0 0, 10 0, 10 10, 0 10, 0 0))', 5, 5);  -- 1
+
+-- Polygon intersection (accepts WKT or blob)
+SELECT geo_as_text(geo_intersection(
+  'POLYGON((0 0, 10 0, 10 10, 0 10, 0 0))',
+  'POLYGON((5 5, 15 5, 15 15, 5 15, 5 5))'
+));
+
+-- WGS84 to ECEF coordinate transform
+SELECT geo_to_ecef(48.8566, 2.3522, 35);  -- '4200952.xxx,172458.xxx,4780111.xxx'
+
+-- Voronoi diagram
+SELECT geo_as_text(geo_voronoi(
+  'MULTIPOINT((0 0), (10 0), (10 10), (0 10))',
+  'POLYGON((-5 -5, 15 -5, 15 15, -5 15, -5 -5))'
+));
+```
+
+### R-Tree Spatial Indexing
+
+FlatSQL automatically creates R-Tree shadow tables for spatial columns during ingest. Spatial columns are detected by:
+- **Convention**: columns named `latitude`/`lat` + `longitude`/`lon`/`lng`
+- **Schema attribute**: fields marked with `(spatial)` in FlatBuffers IDL
+
+```sql
+-- R-Tree pre-filter + precise radius check (72x faster than full scan)
+SELECT p.* FROM points p
+INNER JOIN _rtree_points r ON p.rowid = r.id
+WHERE r.minLat >= 36.2 AND r.maxLat <= 45.2
+  AND r.minLon >= -80 AND r.maxLon <= -68
+  AND geo_within_radius(40.7, -74.0, p.lat, p.lon, 500) = 1;
+```
+
+### Spatial Performance
+
+| Operation | Throughput | Latency |
+|-----------|-----------|---------|
+| Point operations (distance, bearing) | 733K–1.07M ops/sec | ~1 µs |
+| Geohash encode/decode | 631K–746K ops/sec | ~1.4 µs |
+| Point-in-polygon | 127K–255K ops/sec | ~4 µs |
+| Polygon intersection | 81K ops/sec | ~12 µs |
+| Voronoi (4 sites) | 34K ops/sec | ~29 µs |
+| Coordinate transforms | 556K–609K ops/sec | ~1.7 µs |
+| R-Tree bbox query (10K records) | 77K ops/sec | ~13 µs |
+| R-Tree vs full scan speedup | **58–72x** | |
+
+### Space Data Module (SDM)
+
+FlatSQL Spatial is also available as a standalone WASM module compliant with the [Space Data Module SDK](https://github.com/DigitalArsenal/space-data-module-sdk):
+
+```javascript
+import { initSpatialSDM } from 'flatsql/sdm';
+
+const sdm = await initSpatialSDM();
+
+// Read the embedded manifest
+console.log(sdm.getManifest().pluginId);
+// → 'com.digitalarsenal.flatsql.spatial'
+
+// Compute distance
+const km = sdm.computeDistance(40.7128, -74.006, 38.9072, -77.0369);
+
+// Point-in-polygon
+const inside = sdm.pointInPolygon('POLYGON((0 0, 10 0, 10 10, 0 10, 0 0))', 5, 5);
+
+// Coordinate transform
+const ecef = sdm.toECEF(48.8566, 2.3522, 35);
+const back = sdm.fromECEF(ecef.x, ecef.y, ecef.z);
+```
+
 ## SQL Support
 
 ### Supported

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "flatsql",
-  "version": "0.4.0",
+  "version": "0.4.3",
   "description": "SQL queries over raw FlatBuffer storage - zero conversion, real-time indexing, pure streaming",
   "type": "module",
   "main": "dist/index.js",
@@ -18,13 +18,22 @@
       "types": "./wasm/wasi.d.ts",
       "import": "./wasm/wasi.js"
     },
-    "./wasi.wasm": "./wasm/flatsql-wasi.wasm"
+    "./wasi.wasm": "./wasm/flatsql-wasi.wasm",
+    "./sdm": {
+      "types": "./sdm/index.d.ts",
+      "import": "./sdm/index.js"
+    },
+    "./sdm.wasm": "./sdm/flatsql-spatial.wasm"
   },
   "files": [
     "dist",
     "wasm/*.js",
     "wasm/*.wasm",
     "wasm/*.d.ts",
+    "sdm/*.js",
+    "sdm/*.wasm",
+    "sdm/*.d.ts",
+    "sdm/*.json",
     "README.md",
     "LICENSE"
   ],
@@ -64,20 +73,26 @@
     "wasm",
     "webassembly",
     "query",
-    "virtual-table"
+    "virtual-table",
+    "spatial",
+    "geospatial",
+    "voronoi",
+    "delaunay",
+    "rtree",
+    "space-data-module"
   ],
   "author": "Digital Arsenal <tj@digitalarsenal.io>",
   "license": "Apache-2.0",
   "dependencies": {
-    "flatc-wasm": "^26.1.15",
+    "flatc-wasm": "^26.1.32",
     "sql.js": "^1.10.0"
   },
   "devDependencies": {
+    "@types/jest": "^29.5.0",
     "@types/node": "^20.0.0",
-    "typescript": "^5.3.0",
     "jest": "^29.7.0",
-    "@types/jest": "^29.5.0",
     "ts-jest": "^29.1.0",
+    "typescript": "^5.3.0",
     "vite": "^5.0.0"
   },
   "engines": {

package/sdm/index.d.ts

@@ -0,0 +1,88 @@
+/**
+ * FlatSQL Spatial Engine — Space Data Module (SDM)
+ * TypeScript type definitions
+ *
+ * @see https://github.com/DigitalArsenal/space-data-module-sdk
+ */
+
+export interface ECEFPoint {
+  x: number;
+  y: number;
+  z: number;
+}
+
+export interface GeodeticPoint {
+  lat: number;
+  lon: number;
+  alt: number;
+}
+
+export interface MethodResult {
+  status: number;
+  output: string;
+}
+
+export interface PluginManifest {
+  pluginId: string;
+  name: string;
+  version: string;
+  pluginFamily: string;
+  description: string;
+  capabilities: string[];
+  externalInterfaces: string[];
+  methods: Array<{
+    methodId: string;
+    description: string;
+    inputPorts: Array<{
+      portId: string;
+      schemaNames: string[];
+      maxItems: number;
+    }>;
+    outputPorts: Array<{
+      portId: string;
+      schemaNames: string[];
+      maxItems: number;
+    }>;
+    maxBatch: number;
+    drainPolicy: string;
+  }>;
+  runtimeTargets: string[];
+}
+
+export interface FlatSQLSpatialSDM {
+  /** Get the embedded plugin manifest */
+  getManifest(): PluginManifest;
+
+  /** Compute haversine distance between two WGS84 points (km) */
+  computeDistance(lat1: number, lon1: number, lat2: number, lon2: number): number;
+
+  /** Compute forward azimuth between two WGS84 points (degrees 0-360) */
+  computeBearing(lat1: number, lon1: number, lat2: number, lon2: number): number;
+
+  /** Test if a WGS84 point is inside a WKT polygon */
+  pointInPolygon(wkt: string, lat: number, lon: number): boolean;
+
+  /** Compute intersection of two WKT polygons */
+  polygonIntersection(wkt1: string, wkt2: string): string;
+
+  /** Compute union of two WKT polygons */
+  polygonUnion(wkt1: string, wkt2: string): string;
+
+  /** Compute Voronoi diagram from multipoint WKT within bounds */
+  computeVoronoi(multipointWkt: string, boundsWkt: string): string;
+
+  /** Compute Delaunay triangulation from multipoint WKT */
+  computeDelaunay(multipointWkt: string): string;
+
+  /** Transform WGS84 geodetic to ECEF */
+  toECEF(lat: number, lon: number, alt: number): ECEFPoint;
+
+  /** Transform ECEF to WGS84 geodetic */
+  fromECEF(x: number, y: number, z: number): GeodeticPoint;
+
+  /** Low-level method call */
+  callMethod(methodName: string, input: string): MethodResult;
+}
+
+/** Initialize the FlatSQL Spatial SDM module */
+export function initSpatialSDM(): Promise<FlatSQLSpatialSDM>;

package/sdm/index.js

@@ -0,0 +1,253 @@
+/**
+ * FlatSQL Spatial Engine — Space Data Module (SDM)
+ *
+ * Standalone WASM module compliant with space-data-module-sdk.
+ * Provides spatial computation functions without requiring full FlatSQL.
+ *
+ * @see https://github.com/DigitalArsenal/space-data-module-sdk
+ */
+
+import { readFile } from "node:fs/promises";
+import { fileURLToPath } from "node:url";
+import { dirname, join } from "node:path";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+/**
+ * Initialize the FlatSQL Spatial SDM module.
+ * @returns {Promise<FlatSQLSpatialSDM>}
+ */
+export async function initSpatialSDM() {
+  const wasmPath = join(__dirname, "flatsql-spatial.wasm");
+  const wasmBytes = await readFile(wasmPath);
+  let exports;
+  const invoke = () => (fnPtr, ...args) => {
+    const table = exports?.__indirect_function_table;
+    const fn = table?.get(fnPtr);
+    if (typeof fn !== "function") {
+      throw new Error(`Missing indirect WASM function: ${fnPtr}`);
+    }
+    return fn(...args);
+  };
+  const { instance } = await WebAssembly.instantiate(wasmBytes, {
+    env: {
+      invoke_v: invoke("v"),
+      invoke_i: invoke("i"),
+      invoke_ii: invoke("ii"),
+      invoke_iii: invoke("iii"),
+      invoke_iiii: invoke("iiii"),
+      invoke_iiiii: invoke("iiiii"),
+      invoke_iiiiii: invoke("iiiiii"),
+      invoke_iiiiiii: invoke("iiiiiii"),
+      invoke_iiiiiiii: invoke("iiiiiiii"),
+      invoke_iiiiiiiiiiii: invoke("iiiiiiiiiiii"),
+      invoke_iiiiid: invoke("iiiiid"),
+      invoke_di: invoke("di"),
+      invoke_dii: invoke("dii"),
+      invoke_diii: invoke("diii"),
+      invoke_fiii: invoke("fiii"),
+      invoke_iid: invoke("iid"),
+      invoke_jiiii: invoke("jiiii"),
+      invoke_vi: invoke("vi"),
+      invoke_vid: invoke("vid"),
+      invoke_vii: invoke("vii"),
+      invoke_viii: invoke("viii"),
+      invoke_viiii: invoke("viiii"),
+      invoke_viiiii: invoke("viiiii"),
+      invoke_viiiiiii: invoke("viiiiiii"),
+      invoke_viiiiiiiiii: invoke("viiiiiiiiii"),
+      invoke_viiiiiiiiiiiiiii: invoke("viiiiiiiiiiiiiii"),
+      __cxa_begin_catch: (ptr) => ptr,
+      __cxa_end_catch: () => {},
+      __cxa_find_matching_catch_2: () => 0,
+      __cxa_find_matching_catch_3: () => 0,
+      __cxa_rethrow: () => {},
+      __cxa_throw: () => {},
+      __cxa_uncaught_exceptions: () => 0,
+      __resumeException: (ptr) => {
+        throw new Error(`Unhandled WASM exception: ${ptr}`);
+      },
+      emscripten_notify_memory_growth: () => {},
+    },
+    wasi_snapshot_preview1: {
+      environ_get: () => 0,
+      environ_sizes_get: (environCount, environBufSize) => {
+        const memory = exports?.memory;
+        if (memory) {
+          const view = new DataView(memory.buffer);
+          view.setUint32(environCount, 0, true);
+          view.setUint32(environBufSize, 0, true);
+        }
+        return 0;
+      },
+      proc_exit: () => {},
+      fd_write: () => 0,
+      fd_seek: () => 0,
+      fd_close: () => 0,
+    },
+  });
+  exports = instance.exports;
+  const memory = /** @type {WebAssembly.Memory} */ (exports.memory);
+
+  /**
+   * Write a string to the input buffer.
+   * @param {string} str
+   */
+  function writeInput(str) {
+    const inputPtr = /** @type {Function} */ (exports.sdm_get_input_buffer)();
+    const encoder = new TextEncoder();
+    const bytes = encoder.encode(str);
+    const view = new Uint8Array(memory.buffer, inputPtr, bytes.length);
+    view.set(bytes);
+    /** @type {Function} */ (exports.sdm_set_input_length)(bytes.length);
+  }
+
+  /**
+   * Read the output buffer as a string.
+   * @returns {string}
+   */
+  function readOutput() {
+    const outputLen = /** @type {Function} */ (exports.sdm_get_output_length)();
+    const outputPtr = /** @type {Function} */ (exports.sdm_get_output_buffer)();
+    const view = new Uint8Array(memory.buffer, outputPtr, outputLen);
+    return new TextDecoder().decode(view);
+  }
+
+  /**
+   * Call a method with string input, return string output.
+   * @param {string} methodName
+   * @param {string} input
+   * @returns {{ status: number, output: string }}
+   */
+  function callMethod(methodName, input) {
+    writeInput(input);
+    const fn = exports[methodName];
+    if (!fn) throw new Error(`Unknown method: ${methodName}`);
+    const status = /** @type {Function} */ (fn)();
+    return { status, output: readOutput() };
+  }
+
+  /**
+   * Get the embedded plugin manifest.
+   * @returns {object}
+   */
+  function getManifest() {
+    const ptr = /** @type {Function} */ (exports.plugin_get_manifest)();
+    const size = /** @type {Function} */ (exports.plugin_get_manifest_size)();
+    const view = new Uint8Array(memory.buffer, ptr, size);
+    const text = new TextDecoder().decode(view);
+    return JSON.parse(text);
+  }
+
+  return {
+    /** Get the plugin manifest */
+    getManifest,
+
+    /**
+     * Compute haversine distance between two points.
+     * @param {number} lat1 @param {number} lon1
+     * @param {number} lat2 @param {number} lon2
+     * @returns {number} distance in km
+     */
+    computeDistance(lat1, lon1, lat2, lon2) {
+      const { status, output } = callMethod("compute_distance", `${lat1},${lon1},${lat2},${lon2}`);
+      if (status !== 0) throw new Error(output);
+      return parseFloat(output);
+    },
+
+    /**
+     * Compute bearing from point 1 to point 2.
+     * @param {number} lat1 @param {number} lon1
+     * @param {number} lat2 @param {number} lon2
+     * @returns {number} bearing in degrees (0-360)
+     */
+    computeBearing(lat1, lon1, lat2, lon2) {
+      const { status, output } = callMethod("compute_bearing", `${lat1},${lon1},${lat2},${lon2}`);
+      if (status !== 0) throw new Error(output);
+      return parseFloat(output);
+    },
+
+    /**
+     * Test if a point is inside a polygon.
+     * @param {string} wkt - WKT polygon
+     * @param {number} lat @param {number} lon
+     * @returns {boolean}
+     */
+    pointInPolygon(wkt, lat, lon) {
+      const { status, output } = callMethod("point_in_polygon", `${wkt}\n${lat},${lon}`);
+      if (status !== 0) throw new Error(output);
+      return output === "1";
+    },
+
+    /**
+     * Compute polygon intersection.
+     * @param {string} wkt1 @param {string} wkt2
+     * @returns {string} WKT of intersection
+     */
+    polygonIntersection(wkt1, wkt2) {
+      const { status, output } = callMethod("polygon_intersection", `${wkt1}\n${wkt2}`);
+      if (status !== 0) throw new Error(output);
+      return output;
+    },
+
+    /**
+     * Compute polygon union.
+     * @param {string} wkt1 @param {string} wkt2
+     * @returns {string} WKT of union
+     */
+    polygonUnion(wkt1, wkt2) {
+      const { status, output } = callMethod("polygon_union", `${wkt1}\n${wkt2}`);
+      if (status !== 0) throw new Error(output);
+      return output;
+    },
+
+    /**
+     * Compute Voronoi diagram.
+     * @param {string} multipointWkt @param {string} boundsWkt
+     * @returns {string} MULTIPOLYGON WKT
+     */
+    computeVoronoi(multipointWkt, boundsWkt) {
+      const { status, output } = callMethod("compute_voronoi", `${multipointWkt}\n${boundsWkt}`);
+      if (status !== 0) throw new Error(output);
+      return output;
+    },
+
+    /**
+     * Compute Delaunay triangulation.
+     * @param {string} multipointWkt
+     * @returns {string} MULTIPOLYGON WKT
+     */
+    computeDelaunay(multipointWkt) {
+      const { status, output } = callMethod("compute_delaunay", multipointWkt);
+      if (status !== 0) throw new Error(output);
+      return output;
+    },
+
+    /**
+     * Transform WGS84 to ECEF.
+     * @param {number} lat @param {number} lon @param {number} alt
+     * @returns {{ x: number, y: number, z: number }}
+     */
+    toECEF(lat, lon, alt) {
+      const { status, output } = callMethod("transform_to_ecef", `${lat},${lon},${alt}`);
+      if (status !== 0) throw new Error(output);
+      const [x, y, z] = output.split(",").map(Number);
+      return { x, y, z };
+    },
+
+    /**
+     * Transform ECEF to WGS84.
+     * @param {number} x @param {number} y @param {number} z
+     * @returns {{ lat: number, lon: number, alt: number }}
+     */
+    fromECEF(x, y, z) {
+      const { status, output } = callMethod("transform_from_ecef", `${x},${y},${z}`);
+      if (status !== 0) throw new Error(output);
+      const [lat, lon, alt] = output.split(",").map(Number);
+      return { lat, lon, alt };
+    },
+
+    /** Low-level method call */
+    callMethod,
+  };
+}