@evolu/common 1.0.0

package/src/Sql.ts

@@ -0,0 +1,114 @@
+// TODO: This file should be generated from a script via Kysely.
+// The reason for not using Kysely directly is bundle size.
+// [Playground Link](https://kyse.link/?p=b&i=haFkpnNxbGl0ZaF2pjAuMjYuMKFz2gFoaW1wb3J0IHsgR2VuZXJhdGVkIH0gZnJvbSAna3lzZWx5JwoKZGVjbGFyZSBnbG9iYWwgewogIGludGVyZmFjZSBEQiB7CiAgICBldm9sdV9tZXNzYWdlOiB7CiAgICAgIHRpbWVzdGFtcDogc3RyaW5nLAogICAgICB0YWJsZTogc3RyaW5nLAogICAgICByb3c6IHN0cmluZywKICAgICAgY29sdW1uOiBzdHJpbmcsCiAgICAgIHZhbHVlOiB1bmtub3duCiAgICB9LAoKICAgIGV2b2x1X293bmVyOiB7CiAgICAgIGlkOiBzdHJpbmcKICAgICAgbW5lbW9uaWM6IHN0cmluZwogICAgICBlbmNyeXB0aW9uS2V5OiBVaW50OEFycmF5LAogICAgICB0aW1lc3RhbXA6IHN0cmluZywKICAgICAgbWVya2xlVHJlZTogc3RyaW5nCiAgICB9CiAgfQp9oXHaBnhhd2FpdCBreXNlbHkuc2VsZWN0RnJvbSgiZXZvbHVfb3duZXIiKQogIC5zZWxlY3QoWyJpZCIsICJtbmVtb25pYyIsICJlbmNyeXB0aW9uS2V5Il0pCiAgLmV4ZWN1dGUoKQoKYXdhaXQga3lzZWx5LnNjaGVtYQogIC5jcmVhdGVUYWJsZSgnZXZvbHVfbWVzc2FnZScpCiAgLmFkZENvbHVtbigndGltZXN0YW1wJywgJ2Jsb2InLCBjb2wgPT4gY29sLnByaW1hcnlLZXkoKSkKICAuYWRkQ29sdW1uKCd0YWJsZScsICdibG9iJykKICAuYWRkQ29sdW1uKCdyb3cnLCAnYmxvYicpCiAgLmFkZENvbHVtbignY29sdW1uJywgJ2Jsb2InKQogIC5hZGRDb2x1bW4oJ3ZhbHVlJywgJ2Jsb2InKQogIC5leGVjdXRlKCkKCmF3YWl0IGt5c2VseS5zY2hlbWEKICAuY3JlYXRlSW5kZXgoImluZGV4X2V2b2x1X21lc3NhZ2UiKQogIC5vbigiZXZvbHVfbWVzc2FnZSIpCiAgLmNvbHVtbnMoWyJ0YWJsZSIsICJyb3ciLCAiY29sdW1uIiwgInRpbWVzdGFtcCJdKQogIC5leGVjdXRlKCkKCmF3YWl0IGt5c2VseS5zY2hlbWEKICAuY3JlYXRlVGFibGUoJ2V2b2x1X19vd25lcicpCiAgLmFkZENvbHVtbignaWQnLCAnYmxvYicpCiAgLmFkZENvbHVtbignbW5lbW9uaWMnLCAnYmxvYicpCiAgLmFkZENvbHVtbignZW5jcnlwdGlvbktleScsICdibG9iJykKICAuYWRkQ29sdW1uKCd0aW1lc3RhbXAnLCAnYmxvYicpCiAgLmFkZENvbHVtbignbWVya2xlVHJlZScsICdibG9iJykKICAuZXhlY3V0ZSgpCgphd2FpdCBreXNlbHkuaW5zZXJ0SW50bygiZXZvbHVfb3duZXIiKQogIC52YWx1ZXMoewogICAgImlkIjogImIiLAogICAgIm1uZW1vbmljIjogImEiLAogICAgImVuY3J5cHRpb25LZXkiOiBuZXcgVWludDhBcnJheSgpLAogICAgInRpbWVzdGFtcCI6ICJhIiwKICAgICJtZXJrbGVUcmVlIjogImIiCiAgfSkKICAuZXhlY3V0ZSgpCgphd2FpdCBreXNlbHkuc2VsZWN0RnJvbSgiZXZvbHVfb3duZXIiKQogIC5zZWxlY3QoWyJ0aW1lc3RhbXAiLCAibWVya2xlVHJlZSJdKQogIC5leGVjdXRlKCkKCmF3YWl0IGt5c2VseS5zZWxlY3RGcm9tKCJldm9sdV9tZXNzYWdlIikKICAuc2VsZWN0KCJ0aW1lc3RhbXAiKQogIC53aGVyZSgndGFibGUnLCAnPScsICcxJykKICAud2hlcmUoJ3JvdycsICc9JywgJzInKQogIC53aGVyZSgnY29sdW1uJywgJz0nLCAnMycpCiAgLm9yZGVyQnkoInRpbWVzdGFtcCIsICJkZXNjIikKICAubGltaXQoMSkKICAuZXhlY3V0ZVRha2VGaXJzdCgpCgphd2FpdCBreXNlbHkuaW5zZXJ0SW50bygiZXZvbHVfbWVzc2FnZSIpCiAgLnZhbHVlcyh7CiAgICAidGltZXN0YW1wIjogJzEnLAogICAgInRhYmxlIjogJzInLAogICAgInJvdyI6ICczJywKICAgICJjb2x1bW4iOiAnNCcsCiAgICAidmFsdWUiOiAnNScsCiAgfSkKICAub25Db25mbGljdChvYyA9PiBvYy5kb05vdGhpbmcoKSkKICAuZXhlY3V0ZSgpCgphd2FpdCBreXNlbHkudXBkYXRlVGFibGUoImV2b2x1X293bmVyIikKICAuc2V0KHsKICAgICJtZXJrbGVUcmVlIjogJzEnLAogICAgInRpbWVzdGFtcCI6ICIyIgogIH0pCiAgLmV4ZWN1dGUoKQoKYXdhaXQga3lzZWx5LnNlbGVjdEZyb20oImV2b2x1X21lc3NhZ2UiKQogIC5zZWxlY3RBbGwoKQogIC53aGVyZSgidGltZXN0YW1wIiwgIj49IiwgJzEnKQogIC5vcmRlckJ5KCJ0aW1lc3RhbXAiKQogIC5leGVjdXRlKCmhY8M=)
+
+export const selectOwner = `
+SELECT
+  "id",
+  "mnemonic",
+  "encryptionKey"
+FROM
+  "evolu_owner"
+`;
+
+export const createMessageTable = `
+CREATE TABLE
+  "evolu_message" (
+    "timestamp" blob PRIMARY KEY,
+    "table" blob,
+    "row" blob,
+    "column" blob,
+    "value" blob
+  );
+`;
+
+export const createMessageTableIndex = `
+CREATE INDEX "index_evolu_message" ON "evolu_message" (
+  "table", "row", "column", "timestamp"
+);
+`;
+
+export const createOwnerTable = `
+CREATE TABLE
+  "evolu_owner" (
+    "id" blob,
+    "mnemonic" blob,
+    "encryptionKey" blob,
+    "timestamp" blob,
+    "merkleTree" blob
+  );
+`;
+
+export const insertOwner = `
+INSERT INTO
+  "evolu_owner" (
+    "id",
+    "mnemonic",
+    "encryptionKey",
+    "timestamp",
+    "merkleTree"
+  )
+VALUES
+  (?, ?, ?, ?, ?);
+`;
+
+export const selectOwnerTimestampAndMerkleTree = `
+SELECT
+  "timestamp",
+  "merkleTree"
+FROM
+  "evolu_owner"
+`;
+
+export const selectLastTimestampForTableRowColumn = `
+SELECT
+  "timestamp"
+FROM
+  "evolu_message"
+WHERE
+  "table" = ?
+  AND "row" = ?
+  AND "column" = ?
+ORDER BY
+  "timestamp" DESC
+LIMIT
+  1
+`;
+
+export const insertValueIntoTableRowColumn = (
+  table: string,
+  column: string,
+): string => `
+INSERT INTO
+  "${table}" ("id", "${column}")
+VALUES
+  (?, ?)
+ON CONFLICT DO UPDATE SET
+  "${column}" = ?
+`;
+
+export const insertIntoMessagesIfNew = `
+INSERT INTO
+  "evolu_message" ("timestamp", "table", "row", "column", "value")
+VALUES
+  (?, ?, ?, ?, ?)
+ON CONFLICT DO NOTHING
+`;
+
+export const updateOwnerTimestampAndMerkleTree = `
+UPDATE "evolu_owner"
+SET
+  "timestamp" = ?,
+  "merkleTree" = ?
+`;
+
+export const selectMessagesToSync = `
+SELECT
+  *
+FROM
+  "evolu_message"
+WHERE
+  "timestamp" >= ?
+ORDER BY
+  "timestamp"
+`;

package/src/Sqlite.ts

@@ -0,0 +1,92 @@
+import { Brand, Context, Effect, Predicate, ReadonlyRecord } from "effect";
+
+export interface Sqlite {
+  readonly exec: (
+    arg: string | QueryObject,
+  ) => Effect.Effect<never, never, ExecResult>;
+}
+
+export const Sqlite = Context.Tag<Sqlite>("evolu/Sqlite");
+
+export interface QueryObject {
+  readonly sql: string;
+  readonly parameters: ReadonlyArray<Value>;
+}
+
+export type Value = SqliteValue | JsonObjectOrArray;
+
+export type SqliteValue = null | string | number | Uint8Array;
+export type SqliteRow = Record<string, SqliteValue>;
+
+export type JsonObjectOrArray = JsonObject | JsonArray;
+
+type JsonObject = ReadonlyRecord.ReadonlyRecord<Json>;
+type JsonArray = ReadonlyArray<Json>;
+type Json = string | number | boolean | null | JsonObject | JsonArray;
+
+interface ExecResult {
+  readonly rows: ReadonlyArray<Row>;
+  readonly changes: number;
+}
+
+export type Row = ReadonlyRecord.ReadonlyRecord<
+  Value | Row | ReadonlyArray<Row>
+>;
+
+export type Query = string & Brand.Brand<"Query">;
+
+export const isJsonObjectOrArray: Predicate.Refinement<
+  Value,
+  JsonObjectOrArray
+> = (value): value is JsonObjectOrArray =>
+  value !== null && typeof value === "object" && !(value instanceof Uint8Array);
+
+export const valuesToSqliteValues = (
+  values: ReadonlyArray<Value>,
+): SqliteValue[] =>
+  values.map((value) =>
+    isJsonObjectOrArray(value) ? JSON.stringify(value) : value,
+  );
+
+export const queryObjectToQuery = ({ sql, parameters }: QueryObject): Query =>
+  JSON.stringify({ sql, parameters }) as Query;
+
+export const queryObjectFromQuery = (s: Query): QueryObject =>
+  JSON.parse(s) as QueryObject;
+
+export const parseJsonResults = (rows: SqliteRow[]): void => {
+  parseArray(rows);
+};
+
+const parseArray = <T>(a: T[]): T[] => {
+  for (let i = 0; i < a.length; ++i) a[i] = parse(a[i]) as T;
+  return a;
+};
+
+const parse = (o: unknown): unknown => {
+  if (Predicate.isString(o)) return parseString(o);
+  // eslint-disable-next-line @typescript-eslint/no-unsafe-return
+  if (Array.isArray(o)) return parseArray(o);
+  // Predicate.isReadonlyRecord
+  if (typeof o === "object" && o !== null && !Predicate.isUint8Array(o))
+    return parseObject(o as Record<string, unknown>);
+  return o;
+};
+
+const parseString = (s: string): unknown => {
+  if (maybeJson(s))
+    try {
+      return parse(JSON.parse(s));
+    } catch (err) {
+      // Nothing to do.
+    }
+  return s;
+};
+
+export const maybeJson: Predicate.Predicate<string> = (value) =>
+  value.match(/^[[{]/) != null;
+
+const parseObject = (o: Record<string, unknown>): Record<string, unknown> => {
+  for (const key in o) o[key] = parse(o[key]);
+  return o;
+};

package/src/Store.ts

@@ -0,0 +1,32 @@
+export type StoreListener = () => void;
+
+export type StoreUnsubscribe = () => void;
+
+export interface Store<T> {
+  readonly subscribe: (listener: StoreListener) => StoreUnsubscribe;
+  readonly setState: (state: T) => void;
+  readonly getState: () => T;
+}
+
+export const makeStore = <T>(initialState: T): Store<T> => {
+  let currentState = initialState;
+
+  const listeners = new Set<StoreListener>();
+
+  const subscribe: Store<T>["subscribe"] = (listener) => {
+    listeners.add(listener);
+    return () => {
+      listeners.delete(listener);
+    };
+  };
+
+  const setState: Store<T>["setState"] = (state: T) => {
+    if (state === currentState) return;
+    currentState = state;
+    listeners.forEach((listener) => listener());
+  };
+
+  const getState: Store<T>["getState"] = () => currentState;
+
+  return { subscribe, setState, getState };
+};

package/src/SyncWorker.ts

@@ -0,0 +1,344 @@
+import { concatBytes } from "@noble/ciphers/utils";
+import { BinaryReader, BinaryWriter } from "@protobuf-ts/runtime";
+import {
+  Context,
+  Effect,
+  Function,
+  Layer,
+  Match,
+  Option,
+  Predicate,
+  ReadonlyArray,
+  absurd,
+  identity,
+} from "effect";
+import { SecretBox } from "./Crypto.js";
+import { Owner } from "./Db.js";
+import { UnexpectedError, makeUnexpectedError } from "./Errors.js";
+import {
+  MerkleTree,
+  merkleTreeToString,
+  unsafeMerkleTreeFromString,
+} from "./MerkleTree.js";
+import { Id } from "./Model.js";
+import { Fetch, SyncLock } from "./Platform.js";
+import {
+  EncryptedMessage,
+  MessageContent,
+  SyncRequest,
+  SyncResponse,
+} from "./Protobuf.js";
+import { JsonObjectOrArray, Value } from "./Sqlite.js";
+import { Millis, Timestamp, TimestampString } from "./Timestamp.js";
+
+export interface SyncWorker {
+  readonly postMessage: (input: SyncWorkerInput) => void;
+  onMessage: (output: SyncWorkerOutput) => void;
+}
+
+export const SyncWorker = Context.Tag<SyncWorker>("evolu/SyncWorker");
+
+export type SyncWorkerPostMessage = SyncWorker["postMessage"];
+
+export const SyncWorkerPostMessage = Context.Tag<SyncWorkerPostMessage>(
+  "evolu/SyncWorkerPostMessage",
+);
+
+export type SyncWorkerInput =
+  | SyncWorkerInputSync
+  | SyncWorkerInputSyncCompleted;
+
+interface SyncWorkerInputSync {
+  readonly _tag: "sync";
+  readonly syncUrl: string;
+  readonly messages: ReadonlyArray<Message>;
+  readonly merkleTree: MerkleTree;
+  readonly timestamp: Timestamp;
+  readonly owner: Owner;
+  readonly syncLoopCount: number;
+}
+
+export interface NewMessage {
+  readonly table: string;
+  readonly row: Id;
+  readonly column: string;
+  readonly value: Value;
+}
+
+export interface Message extends NewMessage {
+  readonly timestamp: TimestampString;
+}
+
+interface SyncWorkerInputSyncCompleted {
+  readonly _tag: "syncCompleted";
+}
+
+type SyncWorkerOnMessage = SyncWorker["onMessage"];
+
+const SyncWorkerOnMessage = Context.Tag<SyncWorkerOnMessage>(
+  "evolu/SyncWorkerOnMessage",
+);
+
+export type SyncWorkerOutput =
+  | UnexpectedError
+  | SyncWorkerOutputSyncResponse
+  | SyncStateIsNotSyncedError
+  | SyncStateIsSyncing;
+
+export type SyncWorkerOutputSyncResponse = {
+  readonly _tag: "SyncWorkerOutputSyncResponse";
+  readonly messages: ReadonlyArray<Message>;
+  readonly merkleTree: MerkleTree;
+  readonly syncLoopCount: number;
+};
+
+export type SyncState =
+  | SyncStateInitial
+  | SyncStateIsSyncing
+  | SyncStateIsSynced
+  | SyncStateIsNotSyncedError;
+
+export interface SyncStateInitial {
+  readonly _tag: "SyncStateInitial";
+}
+
+export interface SyncStateIsSyncing {
+  readonly _tag: "SyncStateIsSyncing";
+}
+
+export interface SyncStateIsSynced {
+  readonly _tag: "SyncStateIsSynced";
+  readonly time: Millis;
+}
+
+export interface SyncStateIsNotSyncedError {
+  readonly _tag: "SyncStateIsNotSyncedError";
+  readonly error:
+    | SyncStateNetworkError
+    | SyncStateServerError
+    | SyncStatePaymentRequiredError;
+}
+
+export interface SyncStateNetworkError {
+  readonly _tag: "NetworkError";
+}
+
+export interface SyncStateServerError {
+  readonly _tag: "ServerError";
+  readonly status: number;
+}
+
+export interface SyncStatePaymentRequiredError {
+  readonly _tag: "PaymentRequiredError";
+}
+
+const version1 = new Uint8Array([0, 1]);
+
+const valueToProtobuf = (value: Value): MessageContent["value"] => {
+  switch (typeof value) {
+    case "string":
+      return { oneofKind: "stringValue", stringValue: value };
+    case "number":
+      return { oneofKind: "numberValue", numberValue: value };
+  }
+  if (value == null) return { oneofKind: undefined };
+  if (Predicate.isUint8Array(value))
+    return { oneofKind: "bytesValue", bytesValue: value };
+  return { oneofKind: "jsonValue", jsonValue: JSON.stringify(value) };
+};
+
+const valueFromProtobuf = (value: MessageContent["value"]): Value => {
+  switch (value.oneofKind) {
+    case "numberValue":
+      return value.numberValue;
+    case "stringValue":
+      return value.stringValue;
+    case "bytesValue":
+      return value.bytesValue;
+    case "jsonValue":
+      return JSON.parse(value.jsonValue) as JsonObjectOrArray;
+    case undefined:
+      return null;
+    default:
+      return absurd(value);
+  }
+};
+
+const newMessageToBinary = ({ value, ...rest }: NewMessage): Uint8Array =>
+  concatBytes(
+    version1,
+    MessageContent.toBinary(
+      { value: valueToProtobuf(value), ...rest },
+      binaryWriteOptions,
+    ),
+  );
+
+const startsWithArray = (array: Uint8Array, prefix: Uint8Array): boolean => {
+  if (prefix.length > array.length) return false;
+  for (let i = 0; i < prefix.length; i++) {
+    if (array[i] !== prefix[i]) return false;
+  }
+  return true;
+};
+
+const newMessageFromBinary = (
+  binary: Uint8Array,
+): Option.Option<NewMessage> => {
+  if (!startsWithArray(binary, version1)) return Option.none();
+  const { value, ...content } = MessageContent.fromBinary(
+    binary.slice(version1.length),
+    binaryReadOptions,
+  );
+  return Option.some({ value: valueFromProtobuf(value), ...content });
+};
+
+// The 'protobuf-ts' uses TextEncoder, but polyfill fast-text-encoding
+// doesn't support the fatal option.
+// https://github.com/timostamm/protobuf-ts/issues/184#issuecomment-1658443836
+const binaryWriteOptions = {
+  writerFactory: (): BinaryWriter =>
+    new BinaryWriter({
+      encode: (input: string): Uint8Array => new TextEncoder().encode(input),
+    }),
+};
+const binaryReadOptions = {
+  readerFactory: (bytes: Uint8Array): BinaryReader =>
+    new BinaryReader(bytes, {
+      decode: (input: Uint8Array): string => new TextDecoder().decode(input),
+    }),
+};
+
+const sync = (
+  input: SyncWorkerInputSync,
+): Effect.Effect<
+  SyncLock | SyncWorkerOnMessage | Fetch | SecretBox,
+  never,
+  void
+> =>
+  Effect.gen(function* (_) {
+    const syncLock = yield* _(SyncLock);
+    const syncWorkerOnMessage = yield* _(SyncWorkerOnMessage);
+    const fetch = yield* _(Fetch);
+    const secretBox = yield* _(SecretBox);
+
+    if (input.syncLoopCount === 0) {
+      if (!(yield* _(syncLock.acquire))) return;
+      syncWorkerOnMessage({ _tag: "SyncStateIsSyncing" });
+    }
+
+    yield* _(
+      Effect.forEach(input.messages, ({ timestamp, ...newMessage }) =>
+        secretBox
+          .seal(input.owner.encryptionKey, newMessageToBinary(newMessage))
+          .pipe(
+            Effect.map((content): EncryptedMessage => ({ timestamp, content })),
+          ),
+      ),
+      Effect.map((messages) =>
+        SyncRequest.toBinary(
+          {
+            messages,
+            userId: input.owner.id,
+            nodeId: input.timestamp.node,
+            merkleTree: merkleTreeToString(input.merkleTree),
+          },
+          binaryWriteOptions,
+        ),
+      ),
+      Effect.flatMap((body) => fetch(input.syncUrl, body)),
+      Effect.catchTag("FetchError", () =>
+        Effect.fail<SyncStateIsNotSyncedError>({
+          _tag: "SyncStateIsNotSyncedError",
+          error: { _tag: "NetworkError" },
+        }),
+      ),
+      Effect.flatMap((response) => {
+        switch (response.status) {
+          case 402:
+            return Effect.fail<SyncStateIsNotSyncedError>({
+              _tag: "SyncStateIsNotSyncedError",
+              error: { _tag: "PaymentRequiredError" },
+            });
+          case 200:
+            return Effect.promise(() =>
+              response
+                .arrayBuffer()
+                .then((buffer) => new Uint8Array(buffer))
+                .then((array) =>
+                  SyncResponse.fromBinary(array, binaryReadOptions),
+                ),
+            );
+          default:
+            return Effect.fail<SyncStateIsNotSyncedError>({
+              _tag: "SyncStateIsNotSyncedError",
+              error: { _tag: "ServerError", status: response.status },
+            });
+        }
+      }),
+      Effect.flatMap((syncResponse) =>
+        Effect.forEach(syncResponse.messages, ({ timestamp, content }) =>
+          secretBox
+            .open(input.owner.encryptionKey, content)
+            .pipe(
+              Effect.map(newMessageFromBinary),
+              Effect.map(
+                Option.map(
+                  (newMessage): Message => ({ timestamp, ...newMessage }),
+                ),
+              ),
+            ),
+        ).pipe(
+          Effect.map(
+            (messages): SyncWorkerOutputSyncResponse => ({
+              _tag: "SyncWorkerOutputSyncResponse",
+              messages: ReadonlyArray.filterMap(messages, identity),
+              merkleTree: unsafeMerkleTreeFromString(syncResponse.merkleTree),
+              syncLoopCount: input.syncLoopCount,
+            }),
+          ),
+        ),
+      ),
+      Effect.tapError(() => syncLock.release),
+      Effect.merge,
+      Effect.map(syncWorkerOnMessage),
+    );
+  });
+
+export const SyncWorkerLive = Layer.effect(
+  SyncWorker,
+  Effect.gen(function* (_) {
+    const syncLock = yield* _(SyncLock);
+    const fetch = yield* _(Fetch);
+    const secretBox = yield* _(SecretBox);
+
+    const onError = (
+      error: UnexpectedError,
+    ): Effect.Effect<never, never, void> =>
+      Effect.sync(() => {
+        syncWorker.onMessage(error);
+      });
+
+    const postMessage: SyncWorker["postMessage"] = (input) => {
+      void Match.value(input).pipe(
+        Match.tagsExhaustive({
+          sync,
+          syncCompleted: () => syncLock.release,
+        }),
+        Effect.catchAllDefect(makeUnexpectedError),
+        Effect.catchAll(onError),
+        Effect.provideService(SyncLock, syncLock),
+        Effect.provideService(Fetch, fetch),
+        Effect.provideService(SecretBox, secretBox),
+        Effect.provideService(SyncWorkerOnMessage, syncWorker.onMessage),
+        Effect.runPromise,
+      );
+    };
+
+    const syncWorker: SyncWorker = {
+      postMessage,
+      onMessage: Function.constVoid,
+    };
+
+    return syncWorker;
+  }),
+);