@evolu/common 8.9.0 → 8.11.0

package/src/local-first/Protocol.ts

@@ -182,6 +182,7 @@ import {
 } from "../Array.ts";
 import {
   assert,
+  assertNonEmptyArray,
   assertNonNullable,
   assertNotUndefined,
   assertSame,
@@ -204,7 +205,7 @@ import {
   encodeNonNegativeInt,
   encodeNumber,
   encodeString,
-} from "../Binary.ts";
+} from "../Bytes.ts";
 import type { ConsoleDep } from "../Console.ts";
 import {
   createPadmePadding,
@@ -247,6 +248,7 @@ import {
   zeroNonNegativeInt,
 } from "../Type.ts";
 import type { Predicate } from "../Types.ts";
+import type { Evolu } from "./Evolu.ts";
 import {
   type Owner,
   type OwnerError,
@@ -271,6 +273,7 @@ import {
   type RangeUpperBound,
   type SkipRange,
   type StorageDep,
+  type StorageWriteMessagesError,
   type TimestampsRange,
 } from "./Storage.ts";
 import {
@@ -367,18 +370,27 @@ export const MessageType = {
 
 export type MessageType = (typeof MessageType)[keyof typeof MessageType];
 
-/** Parsed protocol header for supported protocol messages. */
+/**
+ * The routing prefix of a {@link ProtocolMessage}.
+ *
+ * The version and the {@link OwnerId} begin every message of every protocol
+ * version. The message type follows them only in messages of this peer's
+ * version; the layout after the prefix is unknown for any other version.
+ */
 export interface ProtocolHeader extends Typed<"ProtocolHeader"> {
-  readonly version: 1;
+  readonly version: NonNegativeInt;
   readonly ownerId: OwnerId;
-  readonly messageType: MessageType;
+  /** Absent when `version` differs from {@link protocolVersion}. */
+  readonly messageType?: MessageType;
 }
 
 /**
- * Parses the protocol header needed for routing.
+ * Parses the {@link ProtocolHeader} a transport needs to route a message.
  *
- * Every protocol message begins with `protocolVersion`, `ownerId`, and
- * `messageType`.
+ * Parsing succeeds for versions this peer cannot process, because a
+ * non-initiator answers a version mismatch with only its version and the owner
+ * ID. Rejecting that reply here would drop it before
+ * {@link applyProtocolMessageAsClient} can report {@link ProtocolVersionError}.
  */
 export const parseProtocolHeader = (
   inputMessage: Uint8Array,
@@ -468,8 +480,9 @@ export interface ProtocolWriteError
  * still sync normally.
  *
  * Clients should prompt the user to contact the relay provider or upgrade their
- * plan. Quota monitoring and management is the relay provider's
- * responsibility.
+ * plan. Quota monitoring and management is the relay provider's responsibility.
+ * After additional quota is available, call {@link Evolu.requestSync} with the
+ * affected owner's ID to retry locally stored changes.
  */
 export interface ProtocolQuotaError
   extends OwnerError, Typed<"ProtocolQuotaError"> {}
@@ -558,9 +571,87 @@ export const createProtocolMessageFromCrdtMessages =
     return buffer.unwrap();
   };
 
+/**
+ * Creates size-limited broadcast {@link ProtocolMessage}s containing every
+ * supplied {@link CrdtMessage}.
+ *
+ * Broadcasts contain no synchronization ranges or write key. Unlike
+ * {@link createProtocolMessageFromCrdtMessages}, this function splits all
+ * messages into complete frames rather than relying on later synchronization
+ * rounds to deliver messages that do not fit. Each individual message must fit
+ * the configured size limit.
+ *
+ * ### Example
+ *
+ * ```ts
+ * import {
+ *   assertSame,
+ *   createId,
+ *   getOrThrow,
+ *   testCreateDeps,
+ * } from "@evolu/common";
+ * import {
+ *   createProtocolBroadcastMessagesFromCrdtMessages,
+ *   MessageType,
+ *   parseProtocolHeader,
+ *   testAppOwner,
+ *   testCreateCrdtMessage,
+ * } from "@evolu/common/local-first";
+ *
+ * const deps = testCreateDeps();
+ * const broadcasts = createProtocolBroadcastMessagesFromCrdtMessages(deps)(
+ *   testAppOwner,
+ *   [testCreateCrdtMessage(createId(deps), 1, "Ada")],
+ * );
+ * assertSame(broadcasts.length, 1);
+ * assertSame(
+ *   getOrThrow(parseProtocolHeader(broadcasts[0])).messageType,
+ *   MessageType.Broadcast,
+ * );
+ * ```
+ */
+export const createProtocolBroadcastMessagesFromCrdtMessages =
+  (deps: RandomBytesDep) =>
+  (
+    owner: Owner,
+    messages: NonEmptyReadonlyArray<CrdtMessage>,
+    maxSize: ProtocolMessageMaxSize = defaultProtocolMessageMaxSize,
+  ): NonEmptyReadonlyArray<ProtocolMessage> => {
+    const broadcasts: Array<ProtocolMessage> = [];
+    let buffer = createProtocolMessageBuffer(owner.id, {
+      messageType: MessageType.Broadcast,
+      totalMaxSize: maxSize,
+    });
+    for (const message of messages) {
+      const encryptedMessage = {
+        timestamp: message.timestamp,
+        change: encodeAndEncryptDbChange(deps)(message, owner.encryptionKey),
+      };
+
+      if (!buffer.canAddMessage(encryptedMessage)) {
+        const nextBuffer = createProtocolMessageBuffer(owner.id, {
+          messageType: MessageType.Broadcast,
+          totalMaxSize: maxSize,
+        });
+        assert(
+          nextBuffer.canAddMessage(encryptedMessage),
+          "the message is too big",
+        );
+        broadcasts.push(buffer.unwrap());
+        buffer = nextBuffer;
+      }
+
+      buffer.addMessage(encryptedMessage);
+    }
+
+    broadcasts.push(buffer.unwrap());
+    assertNonEmptyArray(broadcasts);
+    return broadcasts;
+  };
+
 /** Creates a {@link ProtocolMessage} for sync. */
 export const createProtocolMessageForSync =
-  (deps: StorageDep & ConsoleDep) =>
+  (deps: StorageDep) =>
   (ownerId: OwnerId, subscriptionFlag?: SubscriptionFlag): ProtocolMessage => {
     const buffer = createProtocolMessageBuffer(ownerId, {
       messageType: MessageType.Request,
@@ -902,21 +993,53 @@ export interface ApplyProtocolMessageAsClientOptions {
 }
 
 /**
- * Result type for {@link applyProtocolMessageAsClient} that distinguishes
- * between responses to client requests and broadcast messages.
+ * Result of {@link applyProtocolMessageAsClient}: a continuation for the
+ * non-initiator.
  */
 export interface ApplyProtocolMessageAsClientResponse extends Typed<"Response"> {
   readonly message: ProtocolMessage;
+  /** The uploaded messages, if any, for delivery to other local databases. */
+  readonly broadcast?: ProtocolMessage;
 }
 
-export interface ApplyProtocolMessageAsClientNoResponse extends Typed<"NoResponse"> {}
+/**
+ * Result of {@link applyProtocolMessageAsClient}: the response needed nothing
+ * more, because its ranges all matched or it carried none.
+ */
+export interface ApplyProtocolMessageAsClientConverged extends Typed<"Converged"> {}
 
+/**
+ * Result of {@link applyProtocolMessageAsClient}: a Broadcast message whose
+ * messages were written.
+ */
 export interface ApplyProtocolMessageAsClientBroadcast extends Typed<"Broadcast"> {}
 
+/**
+ * Result of {@link applyProtocolMessageAsClient}: the messages were written, but
+ * without a write key no ranges are reconciled.
+ */
+export interface ApplyProtocolMessageAsClientReadonly extends Typed<"Readonly"> {}
+
+/**
+ * Result of {@link applyProtocolMessageAsClient}: the protocol logged an
+ * exception thrown by calling the storage's `writeMessages` (`Write`) or a
+ * failed range reconciliation (`Sync`) to the console. An exception while the
+ * returned Task runs, as the built-in storages throw, is a defect that aborts
+ * the Run instead. Expected write rejections return the original
+ * {@link StorageWriteMessagesError} through {@link Result}. Reconciliation can
+ * fail after messages have been committed; that failure does not roll back the
+ * write.
+ */
+export interface ApplyProtocolMessageAsClientFailed extends Typed<"Failed"> {
+  readonly cause: "Write" | "Sync";
+}
+
 export type ApplyProtocolMessageAsClientResult =
   | ApplyProtocolMessageAsClientResponse
-  | ApplyProtocolMessageAsClientNoResponse
-  | ApplyProtocolMessageAsClientBroadcast;
+  | ApplyProtocolMessageAsClientConverged
+  | ApplyProtocolMessageAsClientBroadcast
+  | ApplyProtocolMessageAsClientReadonly
+  | ApplyProtocolMessageAsClientFailed;
 
 export const applyProtocolMessageAsClient =
   (
@@ -924,12 +1047,7 @@ export const applyProtocolMessageAsClient =
     options: ApplyProtocolMessageAsClientOptions = {},
   ): Task<
     ApplyProtocolMessageAsClientResult,
-    | ProtocolInvalidDataError
-    | ProtocolSyncError
-    | ProtocolVersionError
-    | ProtocolWriteError
-    | ProtocolWriteKeyError
-    | ProtocolQuotaError,
+    ProtocolError | StorageWriteMessagesError,
     StorageDep
   > =>
   async (run) => {
@@ -995,15 +1113,18 @@ export const applyProtocolMessageAsClient =
           const result = await run(
             storage.writeMessages(ownerIdBytes, messages),
           );
-          // Quota errors are handled by Storage; protocol just stops syncing.
-          if (!result.ok) return ok({ type: "NoResponse" });
+          if (!result.ok) return result;
         } catch (error) {
           if (AbortError.is(error)) throw error;
           run.deps.console.error(error);
-          return ok({ type: "NoResponse" });
+          return ok({ type: "Failed", cause: "Write" });
         }
       }
 
+      if (messageType === MessageType.Broadcast) {
+        return ok({ type: "Broadcast" });
+      }
+
       // Now: No writeKey, no sync.
       // TODO: Allow to sync SharedReadonlyOwner
       // Without local changes, writeKey will not be required.
@@ -1011,17 +1132,13 @@ export const applyProtocolMessageAsClient =
       // the sync will stop.
       const writeKey = options.writeKey;
       if (writeKey == null) {
-        return ok({ type: "NoResponse" });
-      }
-
-      if (messageType === MessageType.Broadcast) {
-        return ok({ type: "Broadcast" });
+        return ok({ type: "Readonly" });
       }
 
       const ranges = decodeRanges(input);
 
       if (!isNonEmptyArray(ranges)) {
-        return ok({ type: "NoResponse" });
+        return ok({ type: "Converged" });
       }
 
       const output = createProtocolMessageBuffer(ownerId, {
@@ -1030,14 +1147,23 @@ export const applyProtocolMessageAsClient =
         rangesMaxSize: options.rangesMaxSize,
       });
 
-      const result = sync(run.deps)(ranges, output, ownerIdBytes);
+      let broadcast: ProtocolMessageBuffer | undefined;
+      const result = sync(run.deps)(ranges, output, ownerIdBytes, (message) => {
+        broadcast ??= createProtocolMessageBuffer(ownerId, {
+          messageType: MessageType.Broadcast,
+        });
+        broadcast.addMessage(message);
+      });
 
-      // Client sync error (handled via Storage) or no changes.
-      if (!result.ok || !result.value) {
-        return ok({ type: "NoResponse" });
-      }
+      // A failure was logged by sync.
+      if (!result.ok) return ok({ type: "Failed", cause: "Sync" });
+      if (!result.value) return ok({ type: "Converged" });
 
-      return ok({ type: "Response", message: output.unwrap() });
+      return ok({
+        type: "Response",
+        message: output.unwrap(),
+        ...(broadcast && { broadcast: broadcast.unwrap() }),
+      });
     } catch (error) {
       if (AbortError.is(error)) throw error;
       return err<ProtocolInvalidDataError>({
@@ -1159,9 +1285,13 @@ export const applyProtocolMessageAsRelay =
           );
 
           if (!result.ok) {
+            const isQuotaError = result.error.type === "StorageQuotaError";
+            if (!isQuotaError) run.deps.console.error(result.error);
             const message = createProtocolMessageBuffer(ownerId, {
               messageType: MessageType.Response,
-              errorCode: ProtocolErrorCode.QuotaError,
+              errorCode: isQuotaError
+                ? ProtocolErrorCode.QuotaError
+                : ProtocolErrorCode.WriteError,
             }).unwrap();
             return ok({ type: "Response", message });
           }
@@ -1256,7 +1386,7 @@ const parseProtocolHeaderFromBuffer = (input: Buffer): ProtocolHeader => {
   const [version, ownerId] = decodeVersionAndOwner(input);
 
   if (version !== protocolVersion) {
-    throw new ProtocolDecodeError(`Unsupported protocol version: ${version}`);
+    return { type: "ProtocolHeader", version, ownerId };
   }
 
   const messageTypeValue = input.shift();
@@ -1276,12 +1406,7 @@ const parseProtocolHeaderFromBuffer = (input: Buffer): ProtocolHeader => {
       throw new ProtocolDecodeError("Invalid MessageType");
   }
 
-  return {
-    type: "ProtocolHeader",
-    version: 1,
-    ownerId,
-    messageType,
-  };
+  return { type: "ProtocolHeader", version, ownerId, messageType };
 };
 
 /**
@@ -1319,6 +1444,7 @@ const sync =
     ranges: NonEmptyReadonlyArray<Range>,
     output: ProtocolMessageBuffer,
     ownerIdBytes: OwnerIdBytes,
+    onMessage?: (message: EncryptedCrdtMessage) => void,
   ): Result<boolean, typeof ProtocolErrorCode.SyncError> => {
     const outputInitialSize = output.getSize();
     let storageSize: NonNegativeInt;
@@ -1427,13 +1553,19 @@ const sync =
             skipRange(range);
           } else if (output.canSplitRange()) {
             coalesceSkipsBeforeAdd();
-            splitRange(deps)(
-              ownerIdBytes,
-              lower,
-              upper,
-              currentUpperBound,
-              output,
-            );
+            try {
+              splitRange(deps)(
+                ownerIdBytes,
+                lower,
+                upper,
+                currentUpperBound,
+                output,
+              );
+            } catch (error) {
+              if (AbortError.is(error)) throw error;
+              deps.console.error(error);
+              return err(ProtocolErrorCode.SyncError);
+            }
           } else {
             return addFingerprintForRemainingRange(upper)
               ? ok(true)
@@ -1495,7 +1627,10 @@ const sync =
                 }
 
                 ourTimestamps.add(timestampBinary);
-                if (message) output.addMessage(message);
+                if (message) {
+                  output.addMessage(message);
+                  onMessage?.(message);
+                }
                 return true;
               },
             );
@@ -1547,7 +1682,7 @@ const sync =
   };
 
 const splitRange =
-  (deps: StorageDep & ConsoleDep) =>
+  (deps: StorageDep) =>
   (
     ownerId: OwnerIdBytes,
     lower: NonNegativeInt,
@@ -1565,15 +1700,10 @@ const splitRange =
         timestamps: createTimestampsBuffer(),
       };
 
-      deps.storage.iterate(
-        ownerId,
-        zeroNonNegativeInt,
-        itemCount,
-        (timestamp) => {
-          range.timestamps.add(timestampBytesToTimestamp(timestamp));
-          return true;
-        },
-      );
+      deps.storage.iterate(ownerId, lower, upper, (timestamp) => {
+        range.timestamps.add(timestampBytesToTimestamp(timestamp));
+        return true;
+      });
 
       buffer.addRange(range);
       return;
@@ -1588,17 +1718,11 @@ const splitRange =
             ...buckets.value.map((b) => NonNegativeInt.orThrow(b + lower)),
           ];
 
-    let fingerprintRanges: ReadonlyArray<FingerprintRange>;
-    try {
-      fingerprintRanges = deps.storage.fingerprintRanges(
-        ownerId,
-        fingerprintRangesBuckets,
-        upperBound,
-      );
-    } catch (error) {
-      deps.console.error(error);
-      return;
-    }
+    const fingerprintRanges = deps.storage.fingerprintRanges(
+      ownerId,
+      fingerprintRangesBuckets,
+      upperBound,
+    );
 
     const rangesToUse =
       lower > 0 ? fingerprintRanges.slice(1) : fingerprintRanges;

package/src/local-first/Query.ts

@@ -43,25 +43,22 @@ export type { NotNull as KyselyNotNull } from "kysely";
  * import {
  *   assertType,
  *   createQueryBuilder,
- *   id,
+ *   testEvoluSchema,
+ *   type TestEvoluSchema,
  *   NonEmptyTrimmedString100,
  *   type Query,
+ *   type TestTodoId,
  * } from "@evolu/common";
  *
- * const TodoId = id("Todo");
- * type TodoId = typeof TodoId.Output;
- * const Schema = {
- *   todo: { id: TodoId, title: NonEmptyTrimmedString100 },
- * };
- * const createQuery = createQueryBuilder(Schema);
+ * const createQuery = createQueryBuilder(testEvoluSchema);
  * const allTodos = createQuery((db) => db.selectFrom("todo").selectAll());
  *
  * type AllTodosRow = typeof allTodos.Row;
  * assertType<
- *   typeof allTodos extends Query<typeof Schema> ? true : false,
+ *   typeof allTodos extends Query<TestEvoluSchema> ? true : false,
  *   true
  * >();
- * assertType<AllTodosRow["id"], TodoId>();
+ * assertType<AllTodosRow["id"], TestTodoId>();
  * assertType<AllTodosRow["title"], NonEmptyTrimmedString100 | null>();
  * ```
  */
@@ -82,15 +79,11 @@ export type Query<
      * import {
      *   assertType,
      *   createQueryBuilder,
-     *   id,
+     *   testEvoluSchema,
      *   type InferRow,
-     *   NonEmptyTrimmedString100,
      * } from "@evolu/common";
      *
-     * const Schema = {
-     *   todo: { id: id("Todo"), title: NonEmptyTrimmedString100 },
-     * };
-     * const createQuery = createQueryBuilder(Schema);
+     * const createQuery = createQueryBuilder(testEvoluSchema);
      * const allTodos = createQuery((db) =>
      *   db.selectFrom("todo").selectAll(),
      * );

package/src/local-first/Schema.test.ts

@@ -0,0 +1,143 @@
+import { test } from "node:test";
+import {
+  assertEqual,
+  assertErr,
+  assertOk,
+  assertSame,
+  assertType,
+  createEvolu,
+  createQueryBuilder,
+  type Evolu,
+  NonEmptyTrimmedString100,
+  object,
+  type OwnerIdBytes,
+  ownerIdToOwnerIdBytes,
+  sqliteQueryStringToSqliteQuery,
+  type Task,
+  testAppName,
+  testAppOwner,
+  type TestEvoluSchema,
+  testEvoluSchema,
+  testLocalOnlyEvoluSchema,
+  testOwnerSecret,
+  TestProjectId,
+  testProjectId,
+  TestTodoId,
+  testTodoId,
+  type TimestampBytes,
+} from "../index.ts";
+import type { EvoluPlatformDeps } from "./Evolu.ts";
+import type { ValidateSchema } from "./Schema.ts";
+
+test("history queries select and filter by ownerId bytes", () => {
+  const createQuery = createQueryBuilder(testEvoluSchema);
+  const query = createQuery((db) =>
+    db
+      .selectFrom("evolu_history")
+      .select(["ownerId", "timestamp"])
+      .where("ownerId", "=", ownerIdToOwnerIdBytes(testAppOwner.id)),
+  );
+
+  assertType<
+    typeof query.Row,
+    { ownerId: OwnerIdBytes; timestamp: TimestampBytes }
+  >();
+});
+
+test("testEvoluSchema supports project-linked and independent todos", () => {
+  assertType<TestEvoluSchema, typeof testEvoluSchema>();
+  assertType<ValidateSchema<TestEvoluSchema>, TestEvoluSchema>();
+  assertType<typeof testTodoId, TestTodoId>();
+  assertType<typeof testProjectId, TestProjectId>();
+  assertSame(testEvoluSchema.todo.id, TestTodoId);
+  assertSame(testEvoluSchema.project.id, TestProjectId);
+  assertEqual(new Set<string>([testTodoId, testProjectId]).size, 2);
+  const Todo = object(testEvoluSchema.todo);
+  const todo = {
+    id: testTodoId,
+    title: "Write documentation",
+    isCompleted: null,
+    projectId: testProjectId,
+  };
+  assertOk(Todo.fromUnknown(todo), todo);
+  const independentTodo = { ...todo, projectId: null };
+  assertOk(Todo.fromUnknown(independentTodo), independentTodo);
+  assertOk(testEvoluSchema.todo.projectId.from(testProjectId), testProjectId);
+  void (() => {
+    // @ts-expect-error A todo ID is not a project ID.
+    testEvoluSchema.todo.projectId.from(testTodoId);
+  });
+
+  const createQuery = createQueryBuilder(testEvoluSchema);
+  const query = createQuery((db) =>
+    db
+      .selectFrom("todo")
+      .leftJoin("project", "project.id", "todo.projectId")
+      .select(["todo.title", "project.name as projectName"]),
+  );
+  assertType<
+    typeof query.Row,
+    {
+      title: NonEmptyTrimmedString100 | null;
+      projectName: NonEmptyTrimmedString100 | null;
+    }
+  >();
+  assertEqual(
+    sqliteQueryStringToSqliteQuery(query).sql,
+    'select "todo"."title", "project"."name" as "projectName" from "todo" left join "project" on "project"."id" = "todo"."projectId"',
+  );
+
+  const createTodos = createEvolu(testEvoluSchema, {
+    appName: testAppName,
+    appOwner: testAppOwner,
+    transports: [],
+  });
+  assertType<
+    typeof createTodos,
+    Task<Evolu<TestEvoluSchema>, never, EvoluPlatformDeps>
+  >();
+});
+
+test("testLocalOnlyEvoluSchema supports keys with or without recovery material", () => {
+  assertType<
+    ValidateSchema<typeof testLocalOnlyEvoluSchema>,
+    typeof testLocalOnlyEvoluSchema
+  >();
+  const AppOwnerRow = object(testLocalOnlyEvoluSchema._appOwner);
+  const recoverableOwner = {
+    id: testAppOwner.id,
+    encryptionKey: testAppOwner.encryptionKey,
+    writeKey: testAppOwner.writeKey,
+    secret: testOwnerSecret,
+    name: "Personal",
+  };
+  assertOk(AppOwnerRow.fromUnknown(recoverableOwner), recoverableOwner);
+  const keyOnlyOwner = {
+    ...recoverableOwner,
+    secret: null,
+    name: null,
+  };
+  assertOk(AppOwnerRow.fromUnknown(keyOnlyOwner), keyOnlyOwner);
+  assertErr(
+    AppOwnerRow.fromUnknown({
+      ...keyOnlyOwner,
+      encryptionKey: new Uint8Array(1),
+    }),
+  );
+  assertErr(
+    AppOwnerRow.fromUnknown({
+      ...recoverableOwner,
+      secret: new Uint8Array(1),
+    }),
+  );
+
+  const createAccounts = createEvolu(testLocalOnlyEvoluSchema, {
+    appName: testAppName,
+    appOwner: testAppOwner,
+    transports: [],
+  });
+  assertType<
+    typeof createAccounts,
+    Task<Evolu<typeof testLocalOnlyEvoluSchema>, never, EvoluPlatformDeps>
+  >();
+});