@evolu/common 4.1.1 → 5.0.1

package/src/Sync.ts

@@ -0,0 +1,306 @@
+import * as Http from "@effect/platform/HttpClient";
+import * as S from "@effect/schema/Schema";
+import { concatBytes } from "@noble/ciphers/utils";
+import { BinaryReader, BinaryWriter } from "@protobuf-ts/runtime";
+import * as Arr from "effect/Array";
+import * as Context from "effect/Context";
+import * as Deferred from "effect/Deferred";
+import * as Effect from "effect/Effect";
+import { absurd } from "effect/Function";
+import * as Option from "effect/Option";
+import * as Predicate from "effect/Predicate";
+import { Config } from "./Config.js";
+import {
+  MerkleTree,
+  Millis,
+  Timestamp,
+  TimestampString,
+  merkleTreeToString,
+  unsafeMerkleTreeFromString,
+} from "./Crdt.js";
+import { SecretBox } from "./Crypto.js";
+import { Id } from "./Model.js";
+import { Owner } from "./Owner.js";
+import * as Protobuf from "./Protobuf.js";
+import { JsonObjectOrArray, Value } from "./Sqlite.js";
+
+export interface Sync {
+  readonly init: (owner: Owner) => Effect.Effect<void>;
+  readonly sync: (
+    syncData: SyncData,
+  ) => Effect.Effect<SyncResult, SyncStateIsNotSynced, Config>;
+}
+
+export const Sync = Context.GenericTag<Sync>("Sync");
+
+export interface SyncData {
+  merkleTree: MerkleTree;
+  timestamp: Timestamp;
+  messages?: ReadonlyArray<Message>;
+}
+
+export interface Message extends NewMessage {
+  readonly timestamp: TimestampString;
+}
+
+export interface NewMessage {
+  readonly table: string;
+  readonly row: Id;
+  readonly column: string;
+  readonly value: Value;
+}
+
+export interface SyncResult {
+  readonly messages: ReadonlyArray<Message>;
+  readonly merkleTree: MerkleTree;
+}
+
+/**
+ * The SyncState type represents the various states that a synchronization
+ * process can be in.
+ */
+export type SyncState =
+  | SyncStateInitial
+  | SyncStateIsSyncing
+  | SyncStateIsSynced
+  | SyncStateIsNotSynced;
+
+/** On app start, we need to find out whether the state is synced. */
+export interface SyncStateInitial {
+  readonly _tag: "SyncStateInitial";
+}
+
+// To preserve identity.
+export const initialSyncState: SyncStateInitial = { _tag: "SyncStateInitial" };
+
+export interface SyncStateIsSyncing {
+  readonly _tag: "SyncStateIsSyncing";
+}
+
+export interface SyncStateIsSynced {
+  readonly _tag: "SyncStateIsSynced";
+  readonly time: Millis;
+}
+
+export interface SyncStateIsNotSynced {
+  readonly _tag: "SyncStateIsNotSynced";
+  readonly error: NetworkError | ServerError | PaymentRequiredError;
+}
+
+export interface NetworkError {
+  readonly _tag: "NetworkError";
+}
+
+export interface ServerError {
+  readonly _tag: "ServerError";
+  readonly status: number;
+}
+
+export interface PaymentRequiredError {
+  readonly _tag: "PaymentRequiredError";
+}
+
+export class SyncFactory extends Context.Tag("SyncFactory")<
+  SyncFactory,
+  {
+    readonly createSync: Effect.Effect<Sync, never, never>;
+  }
+>() {}
+
+export const createSync = Effect.gen(function* () {
+  const initContext = Context.empty().pipe(
+    Context.add(SecretBox, yield* SecretBox),
+  );
+
+  const afterInitContext =
+    yield* Deferred.make<Context.Context<SecretBox | Owner>>();
+
+  return Sync.of({
+    init: (owner) =>
+      Effect.logDebug(["Sync init", { owner }]).pipe(
+        Effect.tap(
+          Deferred.succeed(
+            afterInitContext,
+            initContext.pipe(Context.add(Owner, owner)),
+          ),
+        ),
+      ),
+    sync: ({ merkleTree, timestamp, messages }) =>
+      Effect.gen(function* () {
+        yield* Effect.logDebug([
+          "Sync request",
+          { merkleTree, timestamp, messages },
+        ]);
+        const secretBox = yield* SecretBox;
+        const owner = yield* Owner;
+        const config = yield* Config;
+
+        return yield* Effect.forEach(
+          messages || [],
+          ({ timestamp, ...newMessage }) =>
+            Effect.map(
+              secretBox.seal(
+                owner.encryptionKey,
+                newMessageToBinary(newMessage),
+              ),
+              (content): Protobuf.EncryptedMessage => ({ timestamp, content }),
+            ),
+        ).pipe(
+          Effect.map((encrypedMessages) =>
+            Protobuf.SyncRequest.toBinary(
+              {
+                messages: encrypedMessages,
+                userId: owner.id,
+                nodeId: timestamp.node,
+                merkleTree: merkleTreeToString(merkleTree),
+              },
+              binaryWriteOptions,
+            ),
+          ),
+          Effect.flatMap((body) =>
+            Http.request
+              .post(config.syncUrl)
+              .pipe(
+                Http.request.uint8ArrayBody(body, "application/x-protobuf"),
+                Http.client.fetchOk,
+                Http.response.arrayBuffer,
+              ),
+          ),
+          Effect.map((buffer) =>
+            Protobuf.SyncResponse.fromBinary(
+              new Uint8Array(buffer),
+              binaryReadOptions,
+            ),
+          ),
+          Effect.flatMap((syncResponse) =>
+            Effect.forEach(syncResponse.messages, (encrypedMessage) =>
+              Effect.map(
+                secretBox.open(owner.encryptionKey, encrypedMessage.content),
+                (binary) => [binary, encrypedMessage.timestamp] as const,
+              ),
+            ).pipe(
+              Effect.map(
+                Arr.filterMap(([binary, timestamp]) =>
+                  Option.map(
+                    newMessageFromBinary(binary),
+                    (newMessage): Message => ({ ...newMessage, timestamp }),
+                  ),
+                ),
+              ),
+              Effect.map((messages) => ({
+                messages,
+                merkleTree: unsafeMerkleTreeFromString(syncResponse.merkleTree),
+              })),
+              Effect.tap((response) =>
+                Effect.logDebug(["Sync response", response]),
+              ),
+            ),
+          ),
+          Effect.catchTag("RequestError", () =>
+            Effect.fail<SyncStateIsNotSynced>({
+              _tag: "SyncStateIsNotSynced",
+              error: { _tag: "NetworkError" },
+            }),
+          ),
+          Effect.catchTag("ResponseError", ({ response: { status } }) => {
+            switch (status) {
+              case 402:
+                return Effect.fail<SyncStateIsNotSynced>({
+                  _tag: "SyncStateIsNotSynced",
+                  error: { _tag: "PaymentRequiredError" },
+                });
+              default:
+                return Effect.fail<SyncStateIsNotSynced>({
+                  _tag: "SyncStateIsNotSynced",
+                  error: { _tag: "ServerError", status },
+                });
+            }
+          }),
+        );
+      }).pipe((effect) =>
+        Effect.flatMap(Deferred.await(afterInitContext), (context) =>
+          Effect.provide(effect, context),
+        ),
+      ),
+  });
+});
+
+const newMessageToBinary = ({ value, ...rest }: NewMessage): Uint8Array =>
+  concatBytes(
+    version1,
+    Protobuf.MessageContent.toBinary(
+      { value: valueToProtobuf(value), ...rest },
+      binaryWriteOptions,
+    ),
+  );
+
+const version1 = new Uint8Array([0, 1]);
+
+const valueToProtobuf = (value: Value): Protobuf.MessageContent["value"] => {
+  switch (typeof value) {
+    case "string":
+      return { oneofKind: "stringValue", stringValue: value };
+    case "number":
+      return {
+        oneofKind: "numberValue",
+        numberValue: S.encodeSync(S.NumberFromString)(value),
+      };
+  }
+  if (value == null) return { oneofKind: undefined };
+  if (Predicate.isUint8Array(value))
+    return { oneofKind: "bytesValue", bytesValue: value };
+  return { oneofKind: "jsonValue", jsonValue: JSON.stringify(value) };
+};
+
+// The 'protobuf-ts' uses TextEncoder, but polyfill fast-text-encoding
+// doesn't support the fatal option.
+// https://github.com/timostamm/protobuf-ts/issues/184#issuecomment-1658443836
+const binaryWriteOptions = {
+  writerFactory: (): BinaryWriter =>
+    new BinaryWriter({
+      encode: (input: string): Uint8Array => new TextEncoder().encode(input),
+    }),
+};
+
+const binaryReadOptions = {
+  readerFactory: (bytes: Uint8Array): BinaryReader =>
+    new BinaryReader(bytes, {
+      decode: (input: Uint8Array): string => new TextDecoder().decode(input),
+    }),
+};
+
+const newMessageFromBinary = (
+  binary: Uint8Array,
+): Option.Option<NewMessage> => {
+  if (!startsWithArray(binary, version1)) return Option.none();
+  const { value, ...content } = Protobuf.MessageContent.fromBinary(
+    binary.slice(version1.length),
+    binaryReadOptions,
+  );
+  return Option.some({ value: valueFromProtobuf(value), ...content });
+};
+
+const startsWithArray = (array: Uint8Array, prefix: Uint8Array): boolean => {
+  if (prefix.length > array.length) return false;
+  for (let i = 0; i < prefix.length; i++) {
+    if (array[i] !== prefix[i]) return false;
+  }
+  return true;
+};
+
+const valueFromProtobuf = (value: Protobuf.MessageContent["value"]): Value => {
+  switch (value.oneofKind) {
+    case "numberValue":
+      return S.decodeSync(S.NumberFromString)(value.numberValue);
+    case "stringValue":
+      return value.stringValue;
+    case "bytesValue":
+      return value.bytesValue;
+    case "jsonValue":
+      return JSON.parse(value.jsonValue) as JsonObjectOrArray;
+    case undefined:
+      return null;
+    default:
+      return absurd(value);
+  }
+};

package/src/index.ts

@@ -2,17 +2,14 @@ export * from "./Config.js";
 export * from "./Crdt.js";
 export * from "./Crypto.js";
 export * from "./Db.js";
-export * from "./DbWorker.js";
 export * from "./Diff.js";
-export * from "./ErrorStore.js";
+export * from "./Error.js";
 export * from "./Evolu.js";
 export * from "./Model.js";
 export * from "./Murmurhash.js";
-export * from "./OnCompletes.js";
 export * from "./Owner.js";
 export * from "./Platform.js";
 export * from "./Protobuf.js";
-export * from "./Public.js";
 export * from "./Sqlite.js";
 export * from "./Store.js";
-export * from "./SyncWorker.js";
+export * from "./Sync.js";

package/dist/src/DbWorker.d.ts

@@ -1,82 +0,0 @@
-import * as Context from "effect/Context";
-import * as Effect from "effect/Effect";
-import * as Layer from "effect/Layer";
-import * as ReadonlyArray from "effect/ReadonlyArray";
-import * as ReadonlyRecord from "effect/ReadonlyRecord";
-import { Config } from "./Config.js";
-import { Millis } from "./Crdt.js";
-import { Bip39, Mnemonic, NanoIdGenerator } from "./Crypto.js";
-import { Queries, Query } from "./Db.js";
-import { QueryPatches } from "./Diff.js";
-import { EvoluError } from "./ErrorStore.js";
-import { Id } from "./Model.js";
-import { OnCompleteId } from "./OnCompletes.js";
-import { Owner } from "./Owner.js";
-import { DbWorkerLock } from "./Platform.js";
-import { Sqlite, SqliteSchema, Value } from "./Sqlite.js";
-import { NewMessage, SyncState, SyncWorker, SyncWorkerOutputSyncResponse } from "./SyncWorker.js";
-import { Messaging } from "./Types.js";
-export interface DbWorker extends Messaging<DbWorkerInput, DbWorkerOutput> {
-}
-export declare const DbWorker: Context.Tag<DbWorker, DbWorker>;
-export type DbWorkerInput = DbWorkerInputInit | DbWorkerInputQuery | DbWorkerInputMutate | DbWorkerInputSync | DbWorkerInputReset | DbWorkerInputEnsureSchema | SyncWorkerOutputSyncResponse;
-interface DbWorkerInputInit {
-    readonly _tag: "init";
-    readonly config: Config;
-}
-interface DbWorkerInputQuery {
-    readonly _tag: "query";
-    readonly queries: ReadonlyArray.NonEmptyReadonlyArray<Query>;
-}
-interface DbWorkerInputMutate {
-    readonly _tag: "mutate";
-    readonly mutations: ReadonlyArray.NonEmptyReadonlyArray<Mutation>;
-    readonly queries: Queries;
-}
-interface DbWorkerInputSync {
-    readonly _tag: "sync";
-    readonly queries: Queries;
-}
-interface DbWorkerInputReset {
-    readonly _tag: "reset";
-    readonly mnemonic?: Mnemonic;
-}
-interface DbWorkerInputEnsureSchema extends SqliteSchema {
-    readonly _tag: "ensureSchema";
-}
-export type DbWorkerOutput = DbWorkerOutputOnError | DbWorkerOutputOnOwner | DbWorkerOutputOnQuery | DbWorkerOutputOnReceive | DbWorkerOutputOnResetOrRestore | DbWorkerOutputOnSyncState;
-export interface DbWorkerOutputOnError {
-    readonly _tag: "onError";
-    readonly error: EvoluError;
-}
-export interface DbWorkerOutputOnOwner {
-    readonly _tag: "onOwner";
-    readonly owner: Owner;
-}
-export interface DbWorkerOutputOnQuery {
-    readonly _tag: "onQuery";
-    readonly queriesPatches: ReadonlyArray<QueryPatches>;
-    readonly onCompleteIds: ReadonlyArray<OnCompleteId>;
-}
-interface DbWorkerOutputOnReceive {
-    readonly _tag: "onReceive";
-}
-interface DbWorkerOutputOnResetOrRestore {
-    readonly _tag: "onResetOrRestore";
-}
-interface DbWorkerOutputOnSyncState {
-    readonly _tag: "onSyncState";
-    readonly state: SyncState;
-}
-export interface Mutation {
-    readonly table: string;
-    readonly id: Id;
-    readonly values: ReadonlyRecord.ReadonlyRecord<string, Value | Date | boolean | undefined>;
-    readonly isInsert: boolean;
-    readonly onCompleteId: OnCompleteId | null;
-}
-export declare const mutationsToNewMessages: (mutations: ReadonlyArray<Mutation>) => ReadonlyArray<NewMessage>;
-export declare const upsertValueIntoTableRowColumn: (message: NewMessage, messages: ReadonlyArray<NewMessage>, millis: Millis) => Effect.Effect<void, never, Sqlite>;
-export declare const DbWorkerCommonLive: Layer.Layer<DbWorker, never, Bip39 | NanoIdGenerator | Sqlite | DbWorkerLock | SyncWorker>;
-export {};
-//# sourceMappingURL=DbWorker.d.ts.map

package/dist/src/DbWorker.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"DbWorker.d.ts","sourceRoot":"","sources":["../../src/DbWorker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,gBAAgB,CAAC;AAC1C,OAAO,KAAK,MAAM,MAAM,eAAe,CAAC;AAGxC,OAAO,KAAK,KAAK,MAAM,cAAc,CAAC;AAGtC,OAAO,KAAK,aAAa,MAAM,sBAAsB,CAAC;AACtD,OAAO,KAAK,cAAc,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,MAAM,EAAc,MAAM,aAAa,CAAC;AACjD,OAAO,EAEL,MAAM,EAiBP,MAAM,WAAW,CAAC;AACnB,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC/D,OAAO,EACL,OAAO,EACP,KAAK,EASN,MAAM,SAAS,CAAC;AACjB,OAAO,EAAE,YAAY,EAAe,MAAM,WAAW,CAAC;AACtD,OAAO,EAAE,UAAU,EAAuB,MAAM,iBAAiB,CAAC;AAClE,OAAO,EAAE,EAAE,EAAQ,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAChD,OAAO,EAAE,KAAK,EAAW,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAE7C,OAAO,EACL,MAAM,EAEN,YAAY,EAEZ,KAAK,EAEN,MAAM,aAAa,CAAC;AACrB,OAAO,EAEL,UAAU,EAEV,SAAS,EACT,UAAU,EACV,4BAA4B,EAE7B,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAEvC,MAAM,WAAW,QAAS,SAAQ,SAAS,CAAC,aAAa,EAAE,cAAc,CAAC;CAAG;AAC7E,eAAO,MAAM,QAAQ,iCAAqD,CAAC;AAE3E,MAAM,MAAM,aAAa,GACrB,iBAAiB,GACjB,kBAAkB,GAClB,mBAAmB,GACnB,iBAAiB,GACjB,kBAAkB,GAClB,yBAAyB,GACzB,4BAA4B,CAAC;AAEjC,UAAU,iBAAiB;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,UAAU,kBAAkB;IAC1B,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC,qBAAqB,CAAC,KAAK,CAAC,CAAC;CAC9D;AAED,UAAU,mBAAmB;IAC3B,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IACxB,QAAQ,CAAC,SAAS,EAAE,aAAa,CAAC,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IAClE,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;CAC3B;AAED,UAAU,iBAAiB;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;CAC3B;AAED,UAAU,kBAAkB;IAC1B,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IACvB,QAAQ,CAAC,QAAQ,CAAC,EAAE,QAAQ,CAAC;CAC9B;AAED,UAAU,yBAA0B,SAAQ,YAAY;IACtD,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC;CAC/B;AAQD,MAAM,MAAM,cAAc,GACtB,qBAAqB,GACrB,qBAAqB,GACrB,qBAAqB,GACrB,uBAAuB,GACvB,8BAA8B,GAC9B,yBAAyB,CAAC;AAE9B,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IACzB,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAC;CAC5B;AAED,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IACzB,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC;CACvB;AAED,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IACzB,QAAQ,CAAC,cAAc,EAAE,aAAa,CAAC,YAAY,CAAC,CAAC;IACrD,QAAQ,CAAC,aAAa,EAAE,aAAa,CAAC,YAAY,CAAC,CAAC;CACrD;AAED,UAAU,uBAAuB;IAC/B,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC;CAC5B;AAED,UAAU,8BAA8B;IACtC,QAAQ,CAAC,IAAI,EAAE,kBAAkB,CAAC;CACnC;AAED,UAAU,yBAAyB;IACjC,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC;IAC7B,QAAQ,CAAC,KAAK,EAAE,SAAS,CAAC;CAC3B;AAED,MAAM,WAAW,QAAQ;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,EAAE,EAAE,EAAE,CAAC;IAChB,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC,cAAc,CAC5C,MAAM,EACN,KAAK,GAAG,IAAI,GAAG,OAAO,GAAG,SAAS,CACnC,CAAC;IACF,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;IAC3B,QAAQ,CAAC,YAAY,EAAE,YAAY,GAAG,IAAI,CAAC;CAC5C;AAyFD,eAAO,MAAM,sBAAsB,cACtB,cAAc,QAAQ,CAAC,KACjC,cAAc,UAAU,CAsCxB,CAAC;AA0BJ,eAAO,MAAM,6BAA6B,YAC/B,UAAU,YACT,cAAc,UAAU,CAAC,UAC3B,MAAM,KACb,aAAa,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,CAgChC,CAAC;AA2PL,eAAO,MAAM,kBAAkB,4FAmH9B,CAAC"}