@reearth/core 0.0.1

package/src/mantle/atoms/compute.ts

@@ -0,0 +1,344 @@
+import { atom } from "jotai";
+import { merge, pick, isEqual } from "lodash-es";
+
+import { processGeoJSON } from "../data/geojson";
+import { evalLayer, EvalResult } from "../evaluator";
+import type {
+  ComputedFeature,
+  ComputedLayer,
+  ComputedLayerStatus,
+  Data,
+  DataRange,
+  DataType,
+  Feature,
+  Layer,
+  LayerSimple,
+} from "../types";
+import { appearanceKeys } from "../types";
+
+import { dataAtom, DATA_CACHE_KEYS, globalDataFeaturesCache } from "./data";
+
+export type Atom = ReturnType<typeof computeAtom>;
+
+export type Command =
+  | { type: "setLayer"; layer?: Layer }
+  | { type: "writeLayer"; value: Partial<Pick<LayerSimple, "properties">> }
+  | { type: "requestFetch"; range: DataRange }
+  | { type: "writeFeatures"; features: Feature[] }
+  | {
+      type: "writeComputedFeatures";
+      value: { feature: Feature[]; computed: ComputedFeature[]; needComputingLayer?: boolean };
+    }
+  | { type: "deleteFeatures"; features: string[] }
+  | { type: "deleteComputedFeatures"; features: string[] }
+  | { type: "override"; overrides?: Record<string, any> }
+  | { type: "updateDelegatedDataTypes"; delegatedDataTypes: DataType[] }
+  | { type: "forceUpdateFeatures" };
+
+export function computeAtom(cache?: typeof globalDataFeaturesCache) {
+  const delegatedDataTypes = atom<DataType[]>([]);
+  const updateDelegatedDataTypes = atom(null, async (_, set, value: DataType[]) => {
+    set(delegatedDataTypes, value);
+    await set(compute, undefined);
+  });
+
+  const layer = atom<Layer | undefined>(undefined);
+  const overrides = atom<Record<string, any> | undefined, Record<string, any> | undefined>(
+    undefined,
+    (_, set, value) => {
+      set(overrides, pick(value, appearanceKeys));
+    },
+  );
+
+  const computedResult = atom<EvalResult | undefined>(undefined);
+  const finalFeatures = atom(get =>
+    get(computedResult)?.features?.map((f): ComputedFeature => merge({ ...f }, get(overrides))),
+  );
+  const layerStatus = atom<ComputedLayerStatus>("fetching");
+  const dataAtoms = dataAtom(cache);
+
+  const get = atom((get): ComputedLayer | undefined => {
+    const currentLayer = get(layer);
+    if (!currentLayer) return;
+
+    return {
+      id: currentLayer.id,
+      layer: currentLayer,
+      status: get(layerStatus),
+      features: get(finalFeatures) ?? [],
+      originalFeatures:
+        currentLayer.type === "simple" && currentLayer.data
+          ? get(dataAtoms.getAll)(currentLayer.id, currentLayer.data)?.flat() ?? []
+          : [],
+      properties: currentLayer.type === "simple" ? currentLayer.properties : undefined,
+      ...get(computedResult)?.layer,
+    };
+  });
+
+  const compute = atom(null, async (get, set, value: { forceUpdateData: boolean } | undefined) => {
+    const currentLayer = get(layer);
+    if (!currentLayer) return;
+
+    if (currentLayer.type !== "simple" || !currentLayer.data) {
+      set(layerStatus, "ready");
+      return;
+    }
+
+    const layerId = currentLayer.id;
+
+    // To write feature with delegated data, you should use `writeComputedFeature` atom.
+    if (get(delegatedDataTypes).includes(currentLayer.data.type)) {
+      set(layerStatus, "ready");
+      return;
+    }
+
+    set(layerStatus, "fetching");
+
+    const shouldFetch = (prevFeatures: Feature[] | undefined) => {
+      if (currentLayer.type !== "simple") {
+        return false;
+      }
+
+      // `data.url` should be cached on dataAtoms, so we can return true in this line.
+      if (currentLayer.data?.url || !currentLayer.data?.value) {
+        return true;
+      }
+
+      // Only support geojson for specifying direct feature.
+      if (currentLayer.data?.type !== "geojson") {
+        return true;
+      }
+
+      const curFeatures = processGeoJSON(currentLayer.data.value);
+      if (curFeatures.length === prevFeatures?.length) {
+        return !curFeatures.every((cur, i) => {
+          const prev = prevFeatures[i];
+          return isEqual({ ...cur, id: "" }, { ...prev, id: "" });
+        });
+      }
+
+      return true;
+    };
+
+    // Used for a simple layer.
+    // It retrieves all features for the layer stored in the cache,
+    // but attempts to retrieve data from the network if the main feature is not yet in the cache.
+    const getAllFeatures = async (data: Data) => {
+      const getterAll = get(dataAtoms.getAll);
+      const allFeatures = getterAll(layerId, data);
+
+      const flattenAllFeatures = allFeatures?.flat();
+
+      // Ignore cache if data is embedded
+      if (
+        !value?.forceUpdateData &&
+        allFeatures &&
+        // Check if data has cache key
+        DATA_CACHE_KEYS.every(k => !!data[k])
+      ) {
+        return flattenAllFeatures;
+      }
+
+      if (!value?.forceUpdateData && !shouldFetch(flattenAllFeatures)) {
+        return flattenAllFeatures;
+      }
+
+      await set(dataAtoms.fetch, { data, layerId, forceUpdateData: value?.forceUpdateData });
+      return getterAll(layerId, data)?.flat() ?? [];
+    };
+
+    // Used for a flow layer.
+    // Retrieves and returns only a specific range of features from the cache.
+    // If it is not stored in the cache, it attempts to retrieve the data from the network.
+    const getFeatures = async (data: Data, range?: DataRange) => {
+      const getter = get(dataAtoms.get);
+      const c = getter(layerId, data, range);
+
+      // Ignore cache if data is embedded
+      if (c) return c;
+
+      if (!shouldFetch(c)) {
+        return c;
+      }
+
+      await set(dataAtoms.fetch, { data, range, layerId });
+      return getter(layerId, data, range);
+    };
+
+    const result = await evalLayer(currentLayer, { getFeatures, getAllFeatures });
+    set(layerStatus, "ready");
+    set(computedResult, result);
+  });
+
+  const set = atom(null, async (get, set, value: Layer | undefined) => {
+    set(layer, l =>
+      value
+        ? {
+            ...value,
+            ...("properties" in value
+              ? { properties: value.properties }
+              : l && "properties" in l
+              ? { properties: l.properties }
+              : {}),
+          }
+        : undefined,
+    );
+    await set(compute, undefined);
+  });
+
+  const requestFetch = atom(null, async (get, set, value: DataRange) => {
+    const currentLayer = get(layer);
+    if (currentLayer?.type !== "simple" || !currentLayer.data) return;
+
+    await set(dataAtoms.fetch, { data: currentLayer.data, range: value, layerId: currentLayer.id });
+  });
+
+  // For delegated data
+  const writeLayer = atom(null, (get, set, value: Partial<Pick<LayerSimple, "properties">>) => {
+    const currentLayer = get(layer);
+    if (currentLayer?.type !== "simple" || !currentLayer.data) return;
+    set(layer, { ...currentLayer, ...value });
+  });
+
+  const writeFeatures = atom(null, async (get, set, value: Feature[]) => {
+    const currentLayer = get(layer);
+    if (currentLayer?.type !== "simple" || !currentLayer.data) return;
+
+    set(dataAtoms.set, {
+      data: currentLayer.data,
+      features: value,
+      layerId: currentLayer.id,
+    });
+    await set(compute, undefined);
+  });
+
+  const writeComputedFeatures = atom(
+    null,
+    async (
+      get,
+      set,
+      value: { feature: Feature[]; computed: ComputedFeature[]; needComputingLayer?: boolean },
+    ) => {
+      const currentLayer = get(layer);
+      if (currentLayer?.type !== "simple" || !currentLayer.data) return;
+
+      if (import.meta.env.DEV) {
+        if (!get(delegatedDataTypes).includes(currentLayer.data.type)) {
+          throw new Error("writeComputedFeature can be called with delegated data");
+        }
+      }
+
+      set(layerStatus, "fetching");
+
+      set(dataAtoms.set, {
+        data: currentLayer.data,
+        features: value.feature,
+        layerId: currentLayer.id,
+      });
+
+      const computedLayer = value.needComputingLayer
+        ? await evalLayer(currentLayer, {
+            getAllFeatures: async () => undefined,
+            getFeatures: async () => undefined,
+          })
+        : undefined;
+
+      set(layerStatus, "ready");
+
+      const prevResult = get(computedResult);
+
+      const result = {
+        layer: computedLayer?.layer,
+        features: [...(prevResult?.features || []), ...value.computed],
+      };
+
+      set(computedResult, result);
+    },
+  );
+
+  const deleteFeatures = atom(null, async (get, set, value: string[]) => {
+    const currentLayer = get(layer);
+    if (currentLayer?.type !== "simple" || !currentLayer?.data) return;
+
+    set(dataAtoms.deleteAll, {
+      data: currentLayer.data,
+      features: value,
+      layerId: currentLayer.id,
+    });
+    await set(compute, undefined);
+  });
+
+  const deleteComputedFeatures = atom(null, async (get, set, value: string[]) => {
+    const currentLayer = get(layer);
+    if (currentLayer?.type !== "simple" || !currentLayer?.data) return;
+
+    if (import.meta.env.DEV) {
+      if (!get(delegatedDataTypes).includes(currentLayer.data.type)) {
+        throw new Error("deleteComputedFeatures can be called with delegated data");
+      }
+    }
+
+    set(dataAtoms.deleteAll, {
+      data: currentLayer.data,
+      features: value,
+      layerId: currentLayer.id,
+    });
+
+    const prevResult = get(computedResult);
+
+    if (!prevResult?.layer) {
+      return;
+    }
+
+    const result = {
+      layer: prevResult?.layer,
+      features: prevResult?.features?.filter(f => !value.includes(f.id)) || [],
+    };
+
+    set(computedResult, result);
+  });
+
+  const forceUpdateFeatures = atom(null, async (get, set, _: any) => {
+    const currentLayer = get(layer);
+    if (currentLayer?.type !== "simple" || !currentLayer?.data) return;
+    await set(compute, { forceUpdateData: true });
+  });
+
+  return atom<ComputedLayer | undefined, Command>(
+    g => g(get),
+    async (_, s, value) => {
+      switch (value.type) {
+        case "setLayer":
+          await s(set, value.layer);
+          break;
+        case "writeLayer":
+          await s(writeLayer, value.value);
+          break;
+        case "requestFetch":
+          await s(requestFetch, value.range);
+          break;
+        case "writeFeatures":
+          await s(writeFeatures, value.features);
+          break;
+        case "writeComputedFeatures":
+          await s(writeComputedFeatures, value.value);
+          break;
+        case "deleteFeatures":
+          await s(deleteFeatures, value.features);
+          break;
+        case "deleteComputedFeatures":
+          await s(deleteComputedFeatures, value.features);
+          break;
+        case "override":
+          await s(overrides, value.overrides);
+          break;
+        case "updateDelegatedDataTypes":
+          await s(updateDelegatedDataTypes, value.delegatedDataTypes);
+          break;
+        case "forceUpdateFeatures":
+          await s(forceUpdateFeatures, undefined);
+          break;
+      }
+    },
+  );
+}

package/src/mantle/atoms/data.test.ts

@@ -0,0 +1,182 @@
+import { renderHook, act, waitFor } from "@testing-library/react";
+import { useAtomValue, useSetAtom } from "jotai";
+import { useMemo } from "react";
+import { test, expect, vi } from "vitest";
+
+import { fetchData } from "../data";
+import type { Data, DataRange, Feature } from "../types";
+
+import { doubleKeyCacheAtom } from "./cache";
+import { dataAtom } from "./data";
+
+const data: Data = { type: "geojson", url: "https://example.com/example.geojson" };
+const range: DataRange = { x: 0, y: 0, z: 0 };
+const features: Feature[] = [
+  { type: "feature", id: "a", geometry: { type: "Point", coordinates: [0, 0] } },
+];
+const features2: Feature[] = [
+  { type: "feature", id: "a", geometry: { type: "Point", coordinates: [0, 0] }, range },
+];
+const features3: Feature[] = [
+  { type: "feature", id: "a", geometry: { type: "Point", coordinates: [0, 0] }, range },
+  { type: "feature", id: "b", geometry: { type: "Point", coordinates: [0, 0] }, range },
+  { type: "feature", id: "c", geometry: { type: "Point", coordinates: [0, 0] }, range },
+];
+
+test("dataAtom set", () => {
+  const layerId = "x";
+
+  const { result } = renderHook(() => {
+    const atoms = useMemo(() => dataAtom(doubleKeyCacheAtom<string, string, Feature[]>()), []);
+    const get = useAtomValue(atoms.get);
+    const getAll = useAtomValue(atoms.getAll);
+    const set = useSetAtom(atoms.set);
+    return { get, set, getAll };
+  });
+
+  expect(result.current.get(layerId, data)).toBeUndefined();
+
+  act(() => {
+    result.current.set({ layerId, data, features });
+  });
+
+  expect(result.current.get(layerId, data)).toEqual(features);
+  expect(result.current.getAll(layerId, data)).toEqual([features]);
+  expect(result.current.get(layerId, data, range)).toBeUndefined();
+
+  act(() => {
+    result.current.set({ layerId, data, features: features2 });
+  });
+
+  expect(result.current.get(layerId, data, range)).toEqual(features2);
+  expect(result.current.getAll(layerId, data)).toEqual([features, features2]);
+});
+
+test("dataAtom fetch", async () => {
+  const layerId = "x";
+
+  const { result } = renderHook(() => {
+    const atoms = useMemo(() => dataAtom(doubleKeyCacheAtom<string, string, Feature[]>()), []);
+    const get = useAtomValue(atoms.get);
+    const getAll = useAtomValue(atoms.getAll);
+    const fetch = useSetAtom(atoms.fetch);
+    return { get, fetch, getAll };
+  });
+
+  expect(result.current.get(layerId, data)).toBeUndefined();
+  expect(result.current.getAll(layerId, data)).toBeUndefined();
+
+  act(() => {
+    result.current.fetch({ layerId, data });
+  });
+
+  await waitFor(() => expect(result.current.get(layerId, data)).toEqual(features));
+  expect(result.current.get(layerId, data, range)).toBeUndefined();
+  expect(result.current.getAll(layerId, data)).toEqual([features]);
+});
+
+test("dataAtom deleteAll", async () => {
+  const layerId = "x";
+
+  const { result } = renderHook(() => {
+    const atoms = useMemo(() => dataAtom(doubleKeyCacheAtom<string, string, Feature[]>()), []);
+    const get = useAtomValue(atoms.get);
+    const getAll = useAtomValue(atoms.getAll);
+    const set = useSetAtom(atoms.set);
+    const deleteAll = useSetAtom(atoms.deleteAll);
+    return { get, getAll, set, deleteAll };
+  });
+
+  act(() => {
+    result.current.set({ layerId, data, features: features3 });
+  });
+
+  await waitFor(() => expect(result.current.get(layerId, data, range)).toEqual(features3));
+
+  act(() => {
+    result.current.deleteAll({ layerId, data, features: ["b"] });
+  });
+
+  await waitFor(() =>
+    expect(result.current.get(layerId, data, range)).toEqual([features3[0], features3[2]]),
+  );
+});
+
+test("dataAtom double fetch", async () => {
+  const layerId = "x";
+
+  const { result } = renderHook(() => {
+    const atoms1 = useMemo(() => dataAtom(doubleKeyCacheAtom<string, string, Feature[]>()), []);
+    const atoms2 = useMemo(() => dataAtom(doubleKeyCacheAtom<string, string, Feature[]>()), []);
+    const getAll1 = useAtomValue(atoms1.getAll);
+    const fetch1 = useSetAtom(atoms1.fetch);
+    const getAll2 = useAtomValue(atoms2.getAll);
+    const fetch2 = useSetAtom(atoms2.fetch);
+    return { fetch1, getAll1, fetch2, getAll2 };
+  });
+
+  const { fetchData } = await import("../data");
+
+  expect(result.current.getAll1(layerId, data)).toBeUndefined();
+  expect(result.current.getAll2(layerId, data)).toBeUndefined();
+  expect(fetchData).toBeCalledTimes(1);
+
+  act(() => {
+    result.current.fetch1({ layerId, data });
+    result.current.fetch2({ layerId, data });
+  });
+
+  await waitFor(() => expect(result.current.getAll1(layerId, data)).toEqual([features]));
+  await waitFor(() => expect(result.current.getAll2(layerId, data)).toEqual([features]));
+  expect(fetchData).toBeCalledTimes(3);
+});
+
+test("data.value is present", async () => {
+  const layerId = "x";
+  const data1: Data = {
+    type: "geojson",
+    value: { id: "f", type: "feature", geometry: { type: "Point", coordinates: [1, 2] } },
+  };
+  const data2: Data = {
+    type: "geojson",
+    value: { id: "g", type: "feature", geometry: { type: "Point", coordinates: [2, 3] } },
+  };
+
+  const { result } = renderHook(() => {
+    const atoms = useMemo(() => dataAtom(doubleKeyCacheAtom<string, string, Feature[]>()), []);
+    const getAll = useAtomValue(atoms.getAll);
+    const fetch = useSetAtom(atoms.fetch);
+    return { fetch, getAll };
+  });
+
+  expect(result.current.getAll(layerId, data1)).toBeUndefined();
+
+  act(() => {
+    result.current.fetch({ layerId, data: data1 });
+  });
+
+  await waitFor(() =>
+    expect(result.current.getAll(layerId, data1)).toEqual([
+      [{ type: "feature", id: "f", geometry: { type: "Point", coordinates: [1, 2] } }],
+    ]),
+  );
+
+  act(() => {
+    console.log("ACT");
+    result.current.fetch({ layerId, data: data2 });
+  });
+
+  await waitFor(() =>
+    expect(result.current.getAll(layerId, data1)).toEqual([
+      [{ type: "feature", id: "g", geometry: { type: "Point", coordinates: [2, 3] } }],
+    ]),
+  );
+});
+
+vi.mock("../data", (): { fetchData: typeof fetchData } => ({
+  fetchData: vi.fn(async data =>
+    data.value
+      ? [{ type: "feature", id: data.value.id, geometry: data.value.geometry } as Feature]
+      : features,
+  ),
+}));

package/src/mantle/atoms/data.ts

@@ -0,0 +1,114 @@
+import { atom } from "jotai";
+import { groupBy } from "lodash-es";
+
+import { fetchData } from "../data";
+import type { Feature, Data, DataRange } from "../types";
+
+import { doubleKeyCacheAtom } from "./cache";
+
+export const globalDataFeaturesCache = doubleKeyCacheAtom<string, string, Feature[]>();
+
+export function dataAtom(cacheAtoms = globalDataFeaturesCache) {
+  const fetching = atom<[string, string, AbortController][]>([]);
+
+  const get = atom(
+    get => (layerId: string, data: Data, range?: DataRange) =>
+      get(cacheAtoms.get)(dataKey(layerId, data), rangeKey(range)),
+  );
+
+  const getAll = atom(
+    get => (layerId: string, data: Data) => get(cacheAtoms.getAll)(dataKey(layerId, data)),
+  );
+
+  const set = atom(
+    null,
+    async (_get, set, value: { data: Data; features: Feature[]; layerId: string }) => {
+      Object.entries(groupBy(value.features, f => rangeKey(f.range))).forEach(([k, v]) => {
+        set(cacheAtoms.set, {
+          key: dataKey(value.layerId, value.data),
+          key2: k,
+          value: v,
+        });
+      });
+    },
+  );
+
+  const deleteAll = atom(
+    null,
+    async (get, set, value: { data: Data; features: string[]; layerId: string }) => {
+      const d = dataKey(value.layerId, value.data);
+      Object.entries(
+        groupBy(
+          get(getAll)(value.layerId, value.data)
+            ?.filter(f => f.length)
+            .map(
+              f =>
+                [rangeKey(f[0].range), f, f.filter(g => !value.features.includes(g.id))] as const,
+            )
+            .filter(f => f[1].length !== f[2].length),
+          g => g[0],
+        ),
+      ).forEach(([k, f]) => {
+        set(cacheAtoms.set, {
+          key: d,
+          key2: k,
+          value: f.flatMap(g => g[2]),
+        });
+      });
+    },
+  );
+
+  const fetch = atom(
+    null,
+    async (
+      get,
+      set,
+      value: { data: Data; range?: DataRange; layerId: string; forceUpdateData?: boolean },
+    ) => {
+      const k = dataKey(value.layerId, value.data);
+      if (!k) return;
+
+      const rk = rangeKey(value.range);
+      const cached =
+        !value.forceUpdateData && !value.data.value && value.data.url
+          ? get(cacheAtoms.get)(k, rk)
+          : undefined;
+      if (cached) return;
+      const status = get(fetching).find(e => e[0] === k && e[1] === rk);
+      if (status) {
+        status[2].abort();
+      }
+
+      try {
+        const abort = new AbortController();
+        set(fetching, f => [...f, [k, rk, abort]]);
+        const features = await fetchData(value.data, value.range, { signal: abort.signal });
+        if (features) {
+          set(cacheAtoms.set, { key: k, key2: rk, value: features });
+        }
+      } catch (e) {
+        // Do nothing
+      } finally {
+        set(fetching, f => f.filter(e => e[0] !== k || e[1] !== rk));
+      }
+    },
+  );
+
+  return {
+    get,
+    getAll,
+    set,
+    fetch,
+    deleteAll,
+  };
+}
+
+export const DATA_CACHE_KEYS: (keyof Pick<Data, "type" | "url">)[] = ["type", "url"];
+
+function dataKey(layerId: string, data: Data): string {
+  return !data.value && data.url ? DATA_CACHE_KEYS.map(k => data[k]).join(":") : `layer:${layerId}`;
+}
+
+function rangeKey(range?: DataRange): string {
+  return range ? `${range.x}:${range.y}:${range.z}` : "";
+}

package/src/mantle/atoms/index.ts

@@ -0,0 +1 @@
+export { computeAtom, type Atom } from "./compute";