@reearth/core 0.0.1

package/src/mantle/data/shapefile.ts

@@ -0,0 +1,232 @@
+import type { GeoJSON } from "geojson";
+import JSZip from "jszip";
+
+import type { Data, DataRange, Feature } from "../types";
+
+import { processGeoJSON } from "./geojson";
+import { f, FetchOptions } from "./utils";
+
+export async function fetchShapefile(
+  data: Data,
+  range?: DataRange,
+  options?: FetchOptions,
+): Promise<Feature[] | void> {
+  const arrayBuffer = data.url ? await (await f(data.url, options)).arrayBuffer() : data.value;
+  const zip = await JSZip.loadAsync(new Uint8Array(arrayBuffer));
+
+  let shpFileArrayBuffer: ArrayBuffer | undefined;
+  let dbfFileArrayBuffer: ArrayBuffer | undefined;
+
+  // Access the files inside the ZIP archive
+  const zipEntries = Object.values(zip.files);
+  for (const entry of zipEntries) {
+    const filename = entry.name;
+    if (filename.endsWith(".shp")) {
+      shpFileArrayBuffer = await entry.async("arraybuffer");
+    } else if (filename.endsWith(".dbf")) {
+      dbfFileArrayBuffer = await entry.async("arraybuffer");
+    }
+  }
+
+  if (shpFileArrayBuffer && dbfFileArrayBuffer) {
+    return processGeoJSON(await parseShapefiles(shpFileArrayBuffer, dbfFileArrayBuffer), range);
+  } else {
+    throw new Error(`Zip archive does not contain .shp and .dbf files`);
+  }
+}
+
+export const parseShapefiles = async (
+  shpFile: ArrayBuffer,
+  dbfFile: ArrayBuffer,
+  configuration?: Configuration,
+): Promise<GeoJSON> => {
+  return new ShapefileParser(shpFile, dbfFile, configuration).parse();
+};
+
+interface Configuration {
+  trim?: boolean;
+}
+
+class ShapefileParser {
+  #shp: ArrayBuffer;
+  #dbf: ArrayBuffer;
+  #configuration?: Configuration;
+  #features: any[] = [];
+  #propertiesArray: any[] = [];
+
+  constructor(shp: ArrayBuffer, dbf: ArrayBuffer, configuration?: Configuration) {
+    this.#shp = shp;
+    this.#dbf = dbf;
+    this.#configuration = configuration;
+  }
+
+  #parseShp() {
+    const dataView = new DataView(this.#shp);
+    let idx = 0;
+    const wordLength = dataView.getInt32((idx += 6 * 4), false);
+    const byteLength = wordLength * 2;
+    idx += 4; //version
+    idx += 4; //shapeType
+    idx += 4; //minX, minY
+    idx += 8 * 8; //min(Y, Z, M),max(X, Y, Z, M)
+
+    const features: any[] = [];
+    while (idx < byteLength) {
+      const feature: any = {};
+      const length: number = dataView.getInt32((idx += 4), false);
+
+      const type: number = dataView.getInt32((idx += 4), true);
+      let idxFeature: number = idx + 4;
+      let numberOfParts: number, nbpoints: number, numberOfPoints: number, nbpartsPoint: number[];
+      switch (type) {
+        case 1:
+        case 11:
+        case 21:
+          feature.type = "Point";
+          feature.coordinates = [
+            dataView.getFloat64(idxFeature, true),
+            dataView.getFloat64(idxFeature + 8, true),
+          ];
+          break;
+        case 3:
+        case 13:
+        case 23:
+        case 5:
+        case 15:
+        case 25:
+          if (type === 3 || type === 13 || type === 23) {
+            feature.type = "MultiLineString";
+          } else if (type === 5 || type === 15 || type === 25) {
+            feature.type = "Polygon";
+          }
+          numberOfParts = dataView.getInt32(idxFeature + 32, true);
+          nbpoints = dataView.getInt32(idxFeature + 36, true);
+          idxFeature += 40;
+          nbpartsPoint = new Array(numberOfParts).fill(0).map(() => {
+            const result = dataView.getInt32(idxFeature, true);
+            idxFeature += 4;
+            return result;
+          });
+
+          feature.coordinates = new Array(numberOfParts).fill(0).map((_, i) => {
+            const idstart = nbpartsPoint[i];
+            const idend = (i < numberOfParts - 1 ? nbpartsPoint[i + 1] : nbpoints) - 1;
+            const part = [];
+            for (let j = idstart; j <= idend; j++) {
+              part.push([
+                dataView.getFloat64(idxFeature, true),
+                dataView.getFloat64(idxFeature + 8, true),
+              ]);
+              idxFeature += 16;
+            }
+            return part;
+          });
+          break;
+        case 8:
+        case 18:
+        case 28:
+          feature.type = "MultiPoint";
+          numberOfPoints = dataView.getInt32(idxFeature + 32, true);
+          idxFeature += 36;
+          feature.coordinates = new Array(numberOfPoints).fill(0).map(() => {
+            const result = [
+              dataView.getFloat64(idxFeature, true),
+              dataView.getFloat64(idxFeature + 8, true),
+            ];
+            idxFeature += 16;
+            return result;
+          });
+          break;
+      }
+
+      idx += length * 2;
+      features.push(feature);
+    }
+    this.#features = features;
+  }
+
+  #parseDbf() {
+    const dataView = new DataView(new Uint8Array(this.#dbf).buffer);
+    let idx = 4;
+    const numberOfRecords: number = dataView.getInt32(idx, true);
+    idx += 28;
+    let end = false;
+    const fields = [];
+    while (!end) {
+      const field: any = {};
+      const nameArray: string[] = [];
+      for (let i = 0; i < 10; i++) {
+        const letter = dataView.getUint8(idx);
+        if (letter !== 0) {
+          nameArray.push(String.fromCharCode(letter));
+        }
+        idx += 1;
+      }
+      field.name = nameArray.join("");
+      idx += 1;
+      field.type = String.fromCharCode(dataView.getUint8(idx));
+      idx += 5;
+      field.fieldLength = dataView.getUint8(idx);
+      idx += 16;
+      fields.push(field);
+      if (dataView.getUint8(idx) === 0x0d) {
+        break;
+      }
+    }
+    idx += 1;
+    const propertiesArray = [];
+    for (let i = 0; i < numberOfRecords; i++) {
+      const properties: any = {};
+      if (!end) {
+        try {
+          idx += 1;
+          for (let j = 0; j < fields.length; j++) {
+            let str = "";
+            const charString = [];
+            for (let h = 0; h < fields[j].fieldLength; h++) {
+              charString.push(String.fromCharCode(dataView.getUint8(idx)));
+              idx += 1;
+            }
+            str = charString.join("");
+            // }
+            if (this.#configuration?.trim !== false) {
+              str = str.trim();
+            }
+            const number = parseFloat(str);
+            if (isNaN(number)) {
+              properties[fields[j].name] = str;
+            } else {
+              properties[fields[j].name] = number;
+            }
+          }
+        } catch (err) {
+          end = true;
+        }
+      }
+      propertiesArray.push(properties);
+    }
+    this.#propertiesArray = propertiesArray;
+  }
+
+  #geoJSON() {
+    const geojson: any = {
+      type: "FeatureCollection",
+      features: [],
+    };
+    for (let i = 0; i < Math.min(this.#features.length, this.#propertiesArray.length); i++) {
+      geojson.features.push({
+        type: "Feature",
+        geometry: this.#features[i],
+        properties: this.#propertiesArray[i],
+      });
+    }
+    return geojson;
+  }
+
+  parse(): GeoJSON {
+    this.#parseShp();
+    this.#parseDbf();
+
+    return this.#geoJSON();
+  }
+}

package/src/mantle/data/utils.ts

@@ -0,0 +1,21 @@
+export { generateRandomString } from "../../utils";
+
+export type FetchOptions = {
+  signal?: AbortSignal;
+};
+
+export async function f(url: string, options?: FetchOptions): Promise<Response> {
+  const res = await fetch(url, options);
+  if (res.status !== 200) {
+    throw new Error(`fetched ${url} but status code was ${res.status}`);
+  }
+  return res;
+}
+
+export function isDefined<T>(value: T | undefined | null): value is T {
+  return value !== undefined && value !== null;
+}
+
+export function convertToCoordinates(coordinates: string) {
+  return coordinates.split(" ").map(coord => parseFloat(coord));
+}

package/src/mantle/evaluator/index.ts

@@ -0,0 +1,40 @@
+import {
+  AppearanceTypes,
+  ComputedFeature,
+  Data,
+  DataRange,
+  Feature,
+  Layer,
+  LayerSimple,
+} from "../types";
+
+import { evalSimpleLayerFeature, evalSimpleLayer } from "./simple";
+
+export { clearAllExpressionCaches, evalExpression } from "./simple";
+
+export type EvalContext = {
+  getFeatures: (d: Data, r?: DataRange) => Promise<Feature[] | undefined>;
+  getAllFeatures: (d: Data) => Promise<Feature[] | undefined>;
+};
+
+export type EvalResult = {
+  features?: ComputedFeature[];
+  layer?: Partial<AppearanceTypes>;
+};
+
+export async function evalLayer(
+  layer: LayerSimple,
+  ctx: EvalContext,
+): Promise<EvalResult | undefined> {
+  if (layer.type === "simple") {
+    return evalSimpleLayer(layer, ctx);
+  }
+  return;
+}
+
+export function evalFeature(layer: Layer, feature: Feature): ComputedFeature | undefined {
+  if (layer.type === "simple") {
+    return evalSimpleLayerFeature(layer, feature);
+  }
+  return;
+}

package/src/mantle/evaluator/simple/conditionalExpression/index.ts

@@ -0,0 +1,60 @@
+import { Feature, ConditionsExpression } from "../../../types";
+import { Expression } from "../expression";
+
+export class ConditionalExpression {
+  private _conditions: [string, string][];
+  private _runtimeConditions: Statement[];
+  private _feature?: Feature;
+  private _memoizedResult: any;
+
+  constructor(conditionsExpression: ConditionsExpression, feature?: Feature, defines?: any) {
+    this._conditions = conditionsExpression.conditions;
+    this._runtimeConditions = [];
+    this._feature = feature;
+    this._memoizedResult = undefined;
+
+    this.setRuntime(defines);
+  }
+
+  setRuntime(defines: any) {
+    const runtimeConditions = this._conditions?.map(statement => {
+      const cond = String(statement[0]);
+      const condExpression = String(statement[1]);
+      return new Statement(
+        new Expression(cond, this._feature, defines),
+        new Expression(condExpression, this._feature, defines),
+      );
+    });
+    this._runtimeConditions = runtimeConditions ?? [];
+    this._memoizedResult = undefined;
+  }
+
+  evaluate() {
+    if (typeof this._memoizedResult !== "undefined") {
+      return this._memoizedResult;
+    }
+
+    const conditions = this._runtimeConditions;
+    const length = conditions.length;
+    for (let i = 0; i < length; i++) {
+      const statement = conditions[i];
+      if (statement.condition.evaluate()) {
+        const result = statement.expression.evaluate();
+        this._memoizedResult = result;
+        return result;
+      }
+    }
+    const undefinedResult = undefined;
+    this._memoizedResult = undefinedResult;
+    return undefinedResult;
+  }
+}
+
+class Statement {
+  condition: Expression;
+  expression: Expression;
+  constructor(condition: Expression, expression: Expression) {
+    this.condition = condition;
+    this.expression = expression;
+  }
+}

package/src/mantle/evaluator/simple/expression/constants.ts

@@ -0,0 +1,204 @@
+export const enum ExpressionNodeType {
+  VARIABLE,
+  UNARY,
+  BINARY,
+  TERNARY,
+  CONDITIONAL,
+  MEMBER,
+  FUNCTION_CALL,
+  ARRAY,
+  REGEX,
+  VARIABLE_IN_STRING,
+  LITERAL_NULL,
+  LITERAL_BOOLEAN,
+  LITERAL_NUMBER,
+  LITERAL_STRING,
+  LITERAL_COLOR,
+  LITERAL_VECTOR,
+  LITERAL_REGEX,
+  LITERAL_UNDEFINED,
+  BUILTIN_VARIABLE,
+}
+
+export const unaryOperators = ["!", "-", "+"];
+export const binaryOperators = [
+  "+",
+  "-",
+  "*",
+  "/",
+  "%",
+  "===",
+  "!==",
+  ">",
+  ">=",
+  "<",
+  "<=",
+  "&&",
+  "||",
+  "!~",
+  "=~",
+  "==",
+  "!=",
+];
+
+export const backslashRegex = /\\/g;
+export const backslashReplacement = "@#%";
+export const replacementRegex = /@#%/g;
+
+//#rgba
+export const rgbaMatcher = /^#([0-9a-f])([0-9a-f])([0-9a-f])([0-9a-f])?$/i;
+//#rrggbbaa
+export const rrggbbaaMatcher = /^#([0-9a-f]{2})([0-9a-f]{2})([0-9a-f]{2})([0-9a-f]{2})?$/i;
+
+type ColorMap = {
+  [key: string]: string;
+};
+
+export const builtInColor: ColorMap = {
+  ALICEBLUE: "#F0F8FF",
+  ANTIQUEWHITE: "#FAEBD7",
+  AQUA: "#00FFFF",
+  AQUAMARINE: "#7FFFD4",
+  AZURE: "#F0FFFF",
+  BEIGE: "#F5F5DC",
+  BISQUE: "#FFE4C4",
+  BLACK: "#000000",
+  BLANCHEDALMOND: "#FFEBCD",
+  BLUE: "#0000FF",
+  BLUEVIOLET: "#8A2BE2",
+  BROWN: "#A52A2A",
+  BURLYWOOD: "#DEB887",
+  CADETBLUE: "#5F9EA0",
+  CHARTREUSE: "#7FFF00",
+  CHOCOLATE: "#D2691E",
+  CORAL: "#FF7F50",
+  CORNFLOWERBLUE: "#6495ED",
+  CORNSILK: "#FFF8DC",
+  CRIMSON: "#DC143C",
+  CYAN: "#00FFFF",
+  DARKBLUE: "#00008B",
+  DARKCYAN: "#008B8B",
+  DARKGOLDENROD: "#B8860B",
+  DARKGRAY: "#A9A9A9",
+  DARKGREEN: "#006400",
+  DARKGREY: "#A9A9A9",
+  DARKKHAKI: "#BDB76B",
+  DARKMAGENTA: "#8B008B",
+  DARKOLIVEGREEN: "#556B2F",
+  DARKORANGE: "#FF8C00",
+  DARKORCHID: "#9932CC",
+  DARKRED: "#8B0000",
+  DARKSALMON: "#E9967A",
+  DARKSEAGREEN: "#8FBC8F",
+  DARKSLATEBLUE: "#483D8B",
+  DARKSLATEGRAY: "#2F4F4F",
+  DARKSLATEGREY: "#2F4F4F",
+  DARKTURQUOISE: "#00CED1",
+  DARKVIOLET: "#9400D3",
+  DEEPPINK: "#FF1493",
+  DEEPSKYBLUE: "#00BFFF",
+  DIMGRAY: "#696969",
+  DIMGREY: "#696969",
+  DODGERBLUE: "#1E90FF",
+  FIREBRICK: "#B22222",
+  FLORALWHITE: "#FFFAF0",
+  FORESTGREEN: "#228B22",
+  FUCHSIA: "#FF00FF",
+  GAINSBORO: "#DCDCDC",
+  GHOSTWHITE: "#F8F8FF",
+  GOLD: "#FFD700",
+  GOLDENROD: "#DAA520",
+  GRAY: "#808080",
+  GREEN: "#008000",
+  GREENYELLOW: "#ADFF2F",
+  GREY: "#808080",
+  HONEYDEW: "#F0FFF0",
+  HOTPINK: "#FF69B4",
+  INDIANRED: "#CD5C5C",
+  INDIGO: "#4B0082",
+  IVORY: "#FFFFF0",
+  KHAKI: "#F0E68C",
+  LAVENDER: "#E6E6FA",
+  LAVENDAR_BLUSH: "#FFF0F5",
+  LAWNGREEN: "#7CFC00",
+  LEMONCHIFFON: "#FFFACD",
+  LIGHTBLUE: "#ADD8E6",
+  LIGHTCORAL: "#F08080",
+  LIGHTCYAN: "#E0FFFF",
+  LIGHTGOLDENRODYELLOW: "#FAFAD2",
+  LIGHTGRAY: "#D3D3D3",
+  LIGHTGREEN: "#90EE90",
+  LIGHTGREY: "#D3D3D3",
+  LIGHTPINK: "#FFB6C1",
+  LIGHTSEAGREEN: "#20B2AA",
+  LIGHTSKYBLUE: "#87CEFA",
+  LIGHTSLATEGRAY: "#778899",
+  LIGHTSLATEGREY: "#778899",
+  LIGHTSTEELBLUE: "#B0C4DE",
+  LIGHTYELLOW: "#FFFFE0",
+  LIME: "#00FF00",
+  LIMEGREEN: "#32CD32",
+  LINEN: "#FAF0E6",
+  MAGENTA: "#FF00FF",
+  MAROON: "#800000",
+  MEDIUMAQUAMARINE: "#66CDAA",
+  MEDIUMBLUE: "#0000CD",
+  MEDIUMORCHID: "#BA55D3",
+  MEDIUMPURPLE: "#9370DB",
+  MEDIUMSEAGREEN: "#3CB371",
+  MEDIUMSLATEBLUE: "#7B68EE",
+  MEDIUMSPRINGGREEN: "#00FA9A",
+  MEDIUMTURQUOISE: "#48D1CC",
+  MEDIUMVIOLETRED: "#C71585",
+  MIDNIGHTBLUE: "#191970",
+  MINTCREAM: "#F5FFFA",
+  MISTYROSE: "#FFE4E1",
+  MOCCASIN: "#FFE4B5",
+  NAVAJOWHITE: "#FFDEAD",
+  NAVY: "#000080",
+  OLDLACE: "#FDF5E6",
+  OLIVE: "#808000",
+  OLIVEDRAB: "#6B8E23",
+  ORANGE: "#FFA500",
+  ORANGERED: "#FF4500",
+  ORCHID: "#DA70D6",
+  PALEGOLDENROD: "#EEE8AA",
+  PALEGREEN: "#98FB98",
+  PALETURQUOISE: "#AFEEEE",
+  PALEVIOLETRED: "#DB7093",
+  PAPAYAWHIP: "#FFEFD5",
+  PEACHPUFF: "#FFDAB9",
+  PERU: "#CD853F",
+  PINK: "#FFC0CB",
+  PLUM: "#DDA0DD",
+  POWDERBLUE: "#B0E0E6",
+  PURPLE: "#800080",
+  RED: "#FF0000",
+  ROSYBROWN: "#BC8F8F",
+  ROYALBLUE: "#4169E1",
+  SADDLEBROWN: "#8B4513",
+  SALMON: "#FA8072",
+  SANDYBROWN: "#F4A460",
+  SEAGREEN: "#2E8B57",
+  SEASHELL: "#FFF5EE",
+  SIENNA: "#A0522D",
+  SILVER: "#C0C0C0",
+  SKYBLUE: "#87CEEB",
+  SLATEBLUE: "#6A5ACD",
+  SLATEGRAY: "#708090",
+  SLATEGREY: "#708090",
+  SNOW: "#FFFAFA",
+  SPRINGGREEN: "#00FF7F",
+  STEELBLUE: "#4682B4",
+  TAN: "#D2B48C",
+  TEAL: "#008080",
+  THISTLE: "#D8BFD8",
+  TOMATO: "#FF6347",
+  TURQUOISE: "#40E0D0",
+  VIOLET: "#EE82EE",
+  WHEAT: "#F5DEB3",
+  WHITE: "#FFFFFF",
+  WHITESMOKE: "#F5F5F5",
+  YELLOW: "#FFFF00",
+  YELLOWGREEN: "#9ACD32",
+};

package/src/mantle/evaluator/simple/expression/expression.test.ts

@@ -0,0 +1,63 @@
+import jsep from "jsep";
+import { expect, test, describe, beforeEach } from "vitest";
+
+import { Feature } from "../../../types";
+
+import {
+  replaceDefines,
+  removeBackslashes,
+  Expression,
+  EXPRESSION_CACHES,
+  clearExpressionCaches,
+  REPLACED_VARIABLES_CACHE,
+} from "./expression";
+import { createRuntimeAst } from "./runtime";
+
+describe("replaceDefines", () => {
+  test("should replace defined placeholders with the corresponding values in the expression string", () => {
+    const result = replaceDefines("${key}", { key: "value" });
+    expect(result).toBe("(value)");
+  });
+
+  test("should handle multiple defined placeholders in the expression string", () => {
+    const result = replaceDefines("${key1} + ${key2}", {
+      key1: "value1",
+      key2: "value2",
+    });
+    expect(result).toBe("(value1) + (value2)");
+  });
+});
+
+describe("removeBackslashes", () => {
+  test("should remove all backslashes from the expression string", () => {
+    const result = removeBackslashes("\\");
+    expect(result).toBe("@#%");
+  });
+
+  test("should handle multiple backslashes in the expression string", () => {
+    const result = removeBackslashes("\\\\");
+    expect(result).toBe("@#%@#%");
+  });
+});
+
+describe("expression caches", () => {
+  beforeEach(() => {
+    EXPRESSION_CACHES.clear();
+  });
+
+  test("should remove caches", () => {
+    const originalExpression = "${HEIGHT} > 2 ? color('red') : color('blue')";
+    const replacedExpression = "czm_HEIGHT > 2 ? color('red') : color('blue')";
+    const feature = { properties: { HEIGHT: 1 } } as Feature;
+    const expression = new Expression(originalExpression, feature);
+    expect(REPLACED_VARIABLES_CACHE.get(originalExpression)).toEqual([replacedExpression, []]);
+    expect(EXPRESSION_CACHES.get(replacedExpression)).toEqual(
+      createRuntimeAst(expression, jsep(replacedExpression)),
+    );
+
+    clearExpressionCaches(originalExpression, feature, undefined);
+
+    expect(REPLACED_VARIABLES_CACHE.get(originalExpression)).toBeUndefined();
+    expect(EXPRESSION_CACHES.get(replacedExpression)).toBeUndefined();
+  });
+});