@reearth/core 0.0.1

package/src/mantle/data/csv.test.ts

@@ -0,0 +1,478 @@
+import { expect, test, vi } from "vitest";
+
+import { fetchCSV } from "./csv";
+import * as Utils from "./utils";
+
+test("with header", async () => {
+  const fetchDataMock = vi.spyOn(Utils, "f");
+  fetchDataMock.mockImplementation(async () => {
+    return {
+      text: async () => `id,country,lng,,lat,height
+1,Japan,0,,1,10
+2,US,2,,3,
+3,UK,4,,5,30
+`,
+    } as Response;
+  });
+
+  const features = await fetchCSV({
+    type: "csv",
+    url: "http://example.com",
+    csv: {
+      idColumn: "id",
+      latColumn: "lat",
+      lngColumn: "lng",
+      heightColumn: "height",
+      noHeader: false,
+    },
+  });
+
+  expect(features).toEqual([
+    {
+      type: "feature",
+      id: "1",
+      geometry: {
+        type: "Point",
+        coordinates: [0, 1, 10],
+      },
+      properties: {
+        country: "Japan",
+      },
+      range: undefined,
+    },
+    {
+      type: "feature",
+      id: "2",
+      geometry: {
+        type: "Point",
+        coordinates: [2, 3],
+      },
+      properties: {
+        country: "US",
+      },
+      range: undefined,
+    },
+    {
+      type: "feature",
+      id: "3",
+      geometry: {
+        type: "Point",
+        coordinates: [4, 5, 30],
+      },
+      properties: {
+        country: "UK",
+      },
+      range: undefined,
+    },
+  ]);
+});
+
+test("specify csv directly", async () => {
+  const features = await fetchCSV({
+    type: "csv",
+    value: `id,lng,lat,height,country\n1,0,0,0,Japan`,
+    csv: {
+      idColumn: "id",
+      latColumn: "lat",
+      lngColumn: "lng",
+      heightColumn: "height",
+      noHeader: false,
+    },
+  });
+
+  expect(features).toEqual([
+    {
+      type: "feature",
+      id: "1",
+      geometry: {
+        type: "Point",
+        coordinates: [0, 0, 0],
+      },
+      properties: {
+        country: "Japan",
+      },
+      range: undefined,
+    },
+  ]);
+});
+
+test("has header but set index", async () => {
+  const fetchDataMock = vi.spyOn(Utils, "f");
+  fetchDataMock.mockImplementation(async () => {
+    return {
+      // lat has no header name
+      text: async () => `id,country,,,lat
+1,Japan,0,,1
+2,US,2,,3
+3,UK,4,,5
+`,
+    } as Response;
+  });
+
+  const features = await fetchCSV({
+    type: "csv",
+    url: "http://example.com",
+    csv: {
+      idColumn: 0,
+      lngColumn: 2,
+      latColumn: 100, // This should not found
+      noHeader: false,
+    },
+  });
+
+  expect(features).toEqual([
+    {
+      type: "feature",
+      id: "1",
+      geometry: undefined,
+      properties: {
+        country: "Japan",
+        lat: 1,
+      },
+      range: undefined,
+    },
+    {
+      type: "feature",
+      id: "2",
+      geometry: undefined,
+      properties: {
+        country: "US",
+        lat: 3,
+      },
+      range: undefined,
+    },
+    {
+      type: "feature",
+      id: "3",
+      geometry: undefined,
+      properties: {
+        country: "UK",
+        lat: 5,
+      },
+      range: undefined,
+    },
+  ]);
+});
+
+test("has multiline field", async () => {
+  const fetchDataMock = vi.spyOn(Utils, "f");
+  fetchDataMock.mockImplementation(async () => {
+    return {
+      // lat has no header name
+      text: async () => `id,country,lng,text,lat
+1,Japan,0,"Hello
+World",1
+2,US,2,"This is list
+- a
+- b
+- c
+",3
+3,UK,4,"Hey,
+I'm CSV",5
+`,
+    } as Response;
+  });
+
+  const features = await fetchCSV({
+    type: "csv",
+    url: "http://example.com",
+    csv: {
+      idColumn: 0,
+      lngColumn: 2,
+      latColumn: 4, // This should not fount.
+      noHeader: false,
+    },
+  });
+
+  expect(features).toEqual([
+    {
+      type: "feature",
+      id: "1",
+      geometry: {
+        type: "Point",
+        coordinates: [0, 1],
+      },
+      properties: {
+        country: "Japan",
+        text: `Hello
+World`,
+      },
+      range: undefined,
+    },
+    {
+      type: "feature",
+      id: "2",
+      geometry: {
+        type: "Point",
+        coordinates: [2, 3],
+      },
+      properties: {
+        country: "US",
+        text: `This is list
+- a
+- b
+- c
+`,
+      },
+      range: undefined,
+    },
+    {
+      type: "feature",
+      id: "3",
+      geometry: {
+        type: "Point",
+        coordinates: [4, 5],
+      },
+      properties: {
+        country: "UK",
+        text: `Hey,
+I'm CSV`,
+      },
+      range: undefined,
+    },
+  ]);
+});
+
+test("without header", async () => {
+  const fetchDataMock = vi.spyOn(Utils, "f");
+  fetchDataMock.mockImplementation(async () => {
+    return {
+      text: async () => `1,Japan,0,,1
+2,US,2,,3
+3,UK,4,,5
+`,
+    } as Response;
+  });
+
+  const features = await fetchCSV({
+    type: "csv",
+    url: "http://example.com",
+    csv: {
+      idColumn: 0,
+      lngColumn: 2,
+      latColumn: 4,
+      noHeader: true,
+    },
+  });
+
+  expect(features).toEqual([
+    {
+      type: "feature",
+      id: "1",
+      geometry: {
+        type: "Point",
+        coordinates: [0, 1],
+      },
+      properties: {},
+      range: undefined,
+    },
+    {
+      type: "feature",
+      id: "2",
+      geometry: {
+        type: "Point",
+        coordinates: [2, 3],
+      },
+      properties: {},
+      range: undefined,
+    },
+    {
+      type: "feature",
+      id: "3",
+      geometry: {
+        type: "Point",
+        coordinates: [4, 5],
+      },
+      properties: {},
+      range: undefined,
+    },
+  ]);
+});
+
+test("some delimiter", async () => {
+  const fetchDataMock = vi.spyOn(Utils, "f");
+  fetchDataMock.mockImplementation(async () => {
+    return {
+      text: async () => `1,Japan,0,,1
+2;US;2;;3
+3\tUK\t4\t\t5
+`,
+    } as Response;
+  });
+
+  const features = await fetchCSV({
+    type: "csv",
+    url: "http://example.com",
+    csv: {
+      idColumn: 0,
+      lngColumn: 2,
+      latColumn: 4,
+      noHeader: true,
+    },
+  });
+
+  expect(features).toEqual([
+    {
+      type: "feature",
+      id: "1",
+      geometry: {
+        type: "Point",
+        coordinates: [0, 1],
+      },
+      properties: {},
+      range: undefined,
+    },
+    {
+      type: "feature",
+      id: "2",
+      geometry: {
+        type: "Point",
+        coordinates: [2, 3],
+      },
+      properties: {},
+      range: undefined,
+    },
+    {
+      type: "feature",
+      id: "3",
+      geometry: {
+        type: "Point",
+        coordinates: [4, 5],
+      },
+      properties: {},
+      range: undefined,
+    },
+  ]);
+});
+
+test("invalid parameters", async () => {
+  const fetchDataMock = vi.spyOn(Utils, "f");
+  const generateRandomStringMock = vi.spyOn(Utils, "generateRandomString");
+  fetchDataMock.mockImplementation(async () => {
+    return {
+      text: async () => `id,lng,lat,height
+1,abc,1,10
+,2,,20
+3,,,30
+4,100,100,abc
+5,200,200,50
+`,
+    } as Response;
+  });
+  generateRandomStringMock.mockImplementation(() => "random");
+
+  const features = await fetchCSV({
+    type: "csv",
+    url: "http://example.com",
+    csv: {
+      idColumn: "id",
+      latColumn: "lat",
+      lngColumn: "lng",
+      heightColumn: "height",
+      noHeader: false,
+    },
+  });
+
+  expect(features).toEqual([
+    {
+      type: "feature",
+      id: "1",
+      geometry: undefined,
+      properties: {},
+      range: undefined,
+    },
+    {
+      type: "feature",
+      id: "random",
+      geometry: undefined,
+      properties: {},
+      range: undefined,
+    },
+    {
+      type: "feature",
+      id: "3",
+      geometry: undefined,
+      properties: {},
+      range: undefined,
+    },
+    {
+      type: "feature",
+      id: "4",
+      geometry: {
+        type: "Point",
+        coordinates: [100, 100],
+      },
+      properties: {},
+      range: undefined,
+    },
+    {
+      type: "feature",
+      id: "5",
+      geometry: {
+        type: "Point",
+        coordinates: [200, 200, 50],
+      },
+      properties: {},
+      range: undefined,
+    },
+  ]);
+});
+
+test("has header but set index with disableTypeConversion", async () => {
+  const fetchDataMock = vi.spyOn(Utils, "f");
+  fetchDataMock.mockImplementation(async () => {
+    return {
+      // lat has no header name
+      text: async () => `id,country,,,lat
+1,Japan,0,,1
+2,US,2,,3
+3,UK,4,,5
+`,
+    } as Response;
+  });
+
+  const features = await fetchCSV({
+    type: "csv",
+    url: "http://example.com",
+    csv: {
+      idColumn: 0,
+      lngColumn: 2,
+      latColumn: 100, // This should not found
+      noHeader: false,
+      disableTypeConversion: true,
+    },
+  });
+
+  expect(features).toEqual([
+    {
+      type: "feature",
+      id: "1",
+      geometry: undefined,
+      properties: {
+        country: "Japan",
+        lat: "1",
+      },
+      range: undefined,
+    },
+    {
+      type: "feature",
+      id: "2",
+      geometry: undefined,
+      properties: {
+        country: "US",
+        lat: "3",
+      },
+      range: undefined,
+    },
+    {
+      type: "feature",
+      id: "3",
+      geometry: undefined,
+      properties: {
+        country: "UK",
+        lat: "5",
+      },
+      range: undefined,
+    },
+  ]);
+});

package/src/mantle/data/csv.ts

@@ -0,0 +1,194 @@
+import type { Feature as GeoJSONFeature, Point } from "geojson";
+
+import type { Data, DataRange, Feature } from "../types";
+
+import { f, FetchOptions, generateRandomString } from "./utils";
+
+export async function fetchCSV(
+  data: Data,
+  range?: DataRange,
+  options?: FetchOptions,
+): Promise<Feature[] | void> {
+  if (!data.url) {
+    const value = data.value;
+    if (typeof value !== "string") {
+      const feature = makeFeature(value, range);
+      return feature ? [feature] : undefined;
+    }
+    return await parseCSV(value, data.csv, range);
+  }
+  const csvText = await (await f(data.url, options)).text();
+  return await parseCSV(csvText, data.csv, range);
+}
+
+const parseCSV = async (
+  text: string,
+  options: Data["csv"],
+  range: DataRange | undefined,
+): Promise<Feature[] | void> => {
+  const { parse } = await import("csv-parse");
+  return new Promise((resolve, reject) => {
+    const result: Feature[] = [];
+
+    const parser = parse({
+      // Referring this specification https://csv-spec.org/.
+      delimiter: [",", ";", "\t"],
+    });
+
+    let headerKeys: string[] | undefined;
+
+    // Use the readable stream api to consume records
+    parser.on("readable", () => {
+      let record;
+      while ((record = parser.read()) !== null) {
+        if (!headerKeys) {
+          if (!options?.noHeader) {
+            headerKeys = record;
+            continue;
+          }
+          headerKeys = [];
+        }
+        const value = makeGeoJSONFromArray(headerKeys, record, options);
+        const feature = makeFeature(value, range);
+        if (feature) {
+          result.push(feature);
+        }
+      }
+    });
+
+    // Catch any error
+    parser.on("error", err => {
+      reject(`${err.name} is occurred while parsing CSV: ${err.message}`);
+    });
+
+    parser.on("end", () => {
+      resolve(result);
+    });
+
+    parser.write(text);
+
+    parser.end();
+  });
+};
+
+const SUPPORTED_COORDINATES = {
+  lng: 0,
+  lat: 1,
+  height: 2,
+} as const;
+
+const setCoordinatesToPointGeometry = (
+  geometry: Point | null,
+  value: any,
+  coordIdx: (typeof SUPPORTED_COORDINATES)[keyof typeof SUPPORTED_COORDINATES],
+  optional?: boolean,
+): Point | null => {
+  if (geometry === null) {
+    return null;
+  }
+  const coordinate = Number(value);
+  if (isNaN(coordinate) || (typeof value !== "number" && !value)) {
+    return optional ? geometry : null;
+  }
+  geometry.coordinates[coordIdx] = coordinate;
+  return geometry;
+};
+
+type CSVGeoJSONFeature = GeoJSONFeature<Point | null>;
+
+const makeGeoJSONFromArray = (
+  headers: string[],
+  values: string[],
+  options: Data["csv"],
+): CSVGeoJSONFeature => {
+  const result = values.reduce(
+    (result: CSVGeoJSONFeature, element, idx) => {
+      let value: string | number = element;
+      if (!options?.disableTypeConversion) value = filterNumericString(element);
+      if (options?.idColumn !== undefined && [headers[idx], idx].includes(options.idColumn)) {
+        return {
+          ...result,
+          id: value,
+        };
+      }
+
+      if (options?.lngColumn !== undefined && [headers[idx], idx].includes(options.lngColumn)) {
+        return {
+          ...result,
+          geometry: setCoordinatesToPointGeometry(
+            result.geometry,
+            value,
+            SUPPORTED_COORDINATES.lng,
+          ),
+        };
+      }
+      if (options?.latColumn !== undefined && [headers[idx], idx].includes(options.latColumn)) {
+        return {
+          ...result,
+          geometry: setCoordinatesToPointGeometry(
+            result.geometry,
+            value,
+            SUPPORTED_COORDINATES.lat,
+          ),
+        };
+      }
+      if (
+        options?.heightColumn !== undefined &&
+        [headers[idx], idx].includes(options.heightColumn)
+      ) {
+        return {
+          ...result,
+          geometry: setCoordinatesToPointGeometry(
+            result.geometry,
+            value,
+            SUPPORTED_COORDINATES.height,
+            true,
+          ),
+        };
+      }
+
+      return {
+        ...result,
+        properties: {
+          ...result.properties,
+          ...(headers[idx] ? { [headers[idx]]: value } : {}),
+        },
+      };
+    },
+    {
+      type: "Feature",
+      geometry: {
+        type: "Point",
+        coordinates: [],
+      },
+      properties: {},
+    },
+  );
+  if ((result.geometry?.coordinates.length || 0) < 2) {
+    result.geometry = null;
+  }
+  return result;
+};
+
+const makeFeature = (value: CSVGeoJSONFeature, range: DataRange | undefined): Feature | void => {
+  if (value.type !== "Feature") {
+    return;
+  }
+
+  const geo = value.geometry;
+  return {
+    type: "feature",
+    id: (value.id && String(value.id)) || generateRandomString(12),
+    geometry: geo ?? undefined,
+    properties: value.properties,
+    range,
+  };
+};
+
+const filterNumericString = (input: string): number | string => {
+  const parsed = Number(input);
+  if (isNaN(parsed) || input === "") {
+    return input;
+  }
+  return parsed;
+};