@evolu/common 6.0.1-preview.33 → 6.0.1-preview.35

package/src/Array.ts

@@ -6,19 +6,19 @@
  * Prepared for TC39 Hack pipes:
  *
  * ```ts
- * // Problem: nested calls are hard to follow
+ * // Problem: nested functions can be hard to follow
  * const result = firstInArray(
  *   mapArray(dedupeArray(appendToArray(value, 2)), (x) => x * 2),
  * );
  *
- * // Ideal: TC39 Hack pipes (when available)
+ * // Ideal solution: TC39 Hack pipes (when available)
  * // const result = value
  * //   |> appendToArray(%, 2)
  * //   |> dedupeArray(%)
  * //   |> mapArray(%, (x) => x * 2)
  * //   |> firstInArray(%);
  *
- * // Pragmatic: name each step (or use p1, p2 if lazy)
+ * // Current solution: name each step (or use p1, p2 if lazy)
  * const p1 = appendToArray(value, 2);
  * const p2 = dedupeArray(p1);
  * const p3 = mapArray(p2, (x) => x * 2);

package/src/Crypto.ts

@@ -172,10 +172,13 @@ export const createSymmetricCrypto = (
  * Returns the PADMÉ padded length for a given input length.
  *
  * PADMÉ limits information leakage about the length of the plain-text for a
- * wide range of encrypted data sizes. See the PURBs paper for details:
- * https://bford.info/pub/sec/purb.pdf
+ * wide range of encrypted data sizes.
+ *
+ * See the PURBs paper for details: https://bford.info/pub/sec/purb.pdf
  */
-export const padmePaddedLength = (length: NonNegativeInt): NonNegativeInt => {
+export const createPadmePaddedLength = (
+  length: NonNegativeInt,
+): NonNegativeInt => {
   if (length <= 0) return NonNegativeInt.orThrow(0);
   const e = 31 - Math.clz32(length >>> 0);
   const s = 32 - Math.clz32(e >>> 0);
@@ -184,12 +187,11 @@ export const padmePaddedLength = (length: NonNegativeInt): NonNegativeInt => {
   return NonNegativeInt.orThrow((length + mask) & ~mask);
 };
 
-/**
- * Returns the PADMÉ padding length for a given input length. Uses
- * {@link padmePaddedLength}.
- */
-export const padmePaddingLength = (length: NonNegativeInt): NonNegativeInt => {
-  return NonNegativeInt.orThrow(padmePaddedLength(length) - length);
+/** Creates a PADMÉ padding array of zeros for the given input length. */
+export const createPadmePadding = (length: NonNegativeInt): Uint8Array => {
+  const paddedLength = createPadmePaddedLength(length);
+  const paddingLength = NonNegativeInt.orThrow(paddedLength - length);
+  return new globalThis.Uint8Array(paddingLength);
 };
 
 /**

package/src/Evolu/Db.ts

@@ -63,11 +63,11 @@ import {
 } from "./Sync.js";
 import {
   Timestamp,
+  TimestampBytes,
+  timestampBytesToTimestamp,
   TimestampConfig,
   TimestampError,
-  TimestampString,
-  timestampStringToTimestamp,
-  timestampToTimestampString,
+  timestampToTimestampBytes,
 } from "./Timestamp.js";
 
 export interface DbConfig extends ConsoleConfig, TimestampConfig {
@@ -387,7 +387,7 @@ const createDbWorkerDeps =
         // }
 
         const configResult = sqlite.exec<{
-          clock: TimestampString;
+          clock: TimestampBytes;
           appOwnerId: OwnerId;
           appOwnerEncryptionKey: OwnerEncryptionKey;
           appOwnerWriteKey: OwnerWriteKey;
@@ -415,7 +415,7 @@ const createDbWorkerDeps =
         };
 
         clock = createClock({ ...platformDeps, sqlite })(
-          timestampStringToTimestamp(config.clock),
+          timestampBytesToTimestamp(config.clock),
         );
       } else {
         appOwner =
@@ -507,7 +507,7 @@ const initializeDb =
 
       sql`
         create table evolu_config (
-          "clock" text not null,
+          "clock" blob not null,
           "appOwnerId" text not null,
           "appOwnerEncryptionKey" blob not null,
           "appOwnerWriteKey" blob not null,
@@ -527,7 +527,7 @@ const initializeDb =
           )
         values
           (
-            ${timestampToTimestampString(initialClock)},
+            ${timestampToTimestampBytes(initialClock)},
             ${initialAppOwner.id},
             ${initialAppOwner.encryptionKey},
             ${initialAppOwner.writeKey},

package/src/Evolu/Evolu.ts

@@ -4,7 +4,7 @@ import {
   isNonEmptyArray,
   isNonEmptyReadonlyArray,
 } from "../Array.js";
-import { assert, assertNonEmptyReadonlyArray } from "../Assert.js";
+import { assertNonEmptyReadonlyArray } from "../Assert.js";
 import { createCallbacks } from "../Callbacks.js";
 import { ConsoleDep } from "../Console.js";
 import { RandomBytesDep, SymmetricCryptoDecryptError } from "../Crypto.js";
@@ -13,7 +13,13 @@ import { TransferableError } from "../Error.js";
 import { exhaustiveCheck } from "../Function.js";
 import { createInstances, Instances } from "../Instances.js";
 import { err, ok, Result } from "../Result.js";
-import { isSqlMutation, SafeSql, SqliteError, SqliteQuery } from "../Sqlite.js";
+import {
+  isSqlMutation,
+  SafeSql,
+  SqliteBoolean,
+  SqliteError,
+  SqliteQuery,
+} from "../Sqlite.js";
 import { createStore, StoreSubscribe } from "../Store.js";
 import { TimeDep } from "../Time.js";
 import {
@@ -48,7 +54,6 @@ import {
 } from "./Query.js";
 import {
   CreateQuery,
-  DefaultColumns,
   EvoluSchema,
   evoluSchemaToDbSchema,
   IndexesConfig,
@@ -59,6 +64,7 @@ import {
   MutationKind,
   MutationMapping,
   MutationOptions,
+  SystemColumns,
   updateable,
   upsertable,
   ValidateSchema,
@@ -239,7 +245,7 @@ export interface Evolu<S extends EvoluSchema = EvoluSchema> extends Disposable {
    *
    * Evolu does not use SQL for mutations to ensure data can be safely and
    * predictably merged without conflicts. Explicit mutations also allow Evolu
-   * to automatically add and update {@link DefaultColumns}.
+   * to automatically update {@link SystemColumns}.
    *
    * ### Example
    *
@@ -284,7 +290,7 @@ export interface Evolu<S extends EvoluSchema = EvoluSchema> extends Disposable {
    *
    * Evolu does not use SQL for mutations to ensure data can be safely and
    * predictably merged without conflicts. Explicit mutations also allow Evolu
-   * to automatically add and update {@link DefaultColumns}.
+   * to automatically update {@link SystemColumns}.
    *
    * ### Example
    *
@@ -337,7 +343,7 @@ export interface Evolu<S extends EvoluSchema = EvoluSchema> extends Disposable {
    *
    * Evolu does not use SQL for mutations to ensure data can be safely and
    * predictably merged without conflicts. Explicit mutations also allow Evolu
-   * to automatically add and update {@link DefaultColumns}.
+   * to automatically update {@link SystemColumns}.
    *
    * ### Example
    *
@@ -408,7 +414,12 @@ export interface Evolu<S extends EvoluSchema = EvoluSchema> extends Disposable {
    */
   readonly reloadApp: () => void;
 
-  /** Export SQLite database file as Uint8Array. */
+  /**
+   * Export SQLite database file as Uint8Array.
+   *
+   * In the future, it will be possible to import a database and export/import
+   * history for 1:1 migrations across owners.
+   */
   readonly exportDatabase: () => Promise<Uint8Array<ArrayBuffer>>;
 
   /**
@@ -713,24 +724,20 @@ const createEvoluInstance =
             // Mark the transaction as invalid by pushing null
             mutateMicrotaskQueue.push([null, undefined]);
           } else {
-            const values = { ...result.value };
-            delete values.id;
-
-            if (kind === "insert" || kind === "upsert") {
-              // Only set createdAt if not provided by user
-              if (!("createdAt" in values)) {
-                values.createdAt = new Date(deps.time.now()).toISOString();
-              }
-            }
-
-            const dbChange = { table, id, values };
-            assert(
-              DbChange.is(dbChange),
-              `Failed to create DbChange for table "${dbChange.table}"`,
-            );
+            const { id: _, isDeleted, ...values } = result.value;
+
+            const dbChange = DbChange.orThrow({
+              table,
+              id,
+              values,
+              isInsert: kind === "insert" || kind === "upsert",
+              isDelete: SqliteBoolean.is(isDeleted) ? Boolean(isDeleted) : null,
+            });
 
-            const mutationChange = { ...dbChange, ownerId: options?.ownerId };
-            mutateMicrotaskQueue.push([mutationChange, options?.onComplete]);
+            mutateMicrotaskQueue.push([
+              { ...dbChange, ownerId: options?.ownerId },
+              options?.onComplete,
+            ]);
           }
 
           if (mutateMicrotaskQueue.length === 1) {

package/src/Evolu/Protocol.ts

@@ -170,7 +170,11 @@
  *   initiator/non-initiator terminology instead, and consolidate into a single
  *   `applyProtocolMessage` function with conditional arguments to reduce code
  *   duplication.
- * - ProtocolQuotaError should return storedBytes and actual quota.
+ * - Replace try-catch with Result + new Error (to preserve stacktraces). Measure
+ *   Result overhead, it should be super small.
+ * - Allow clients to broadcast messages that are not persisted by relays. This
+ *   would enable real-time ephemeral data (like cursor positions, typing
+ *   indicators) to be forwarded by relays without storage overhead.
  */
 
 import { Packr } from "msgpackr";
@@ -186,8 +190,8 @@ import {
   utf8ToBytes,
 } from "../Buffer.js";
 import {
+  createPadmePadding,
   EncryptionKey,
-  padmePaddingLength,
   RandomBytesDep,
   SymmetricCryptoDecryptError,
   SymmetricCryptoDep,
@@ -414,7 +418,9 @@ export interface ProtocolWriteError extends BaseOwnerError {
  * excess local data is affected. Other devices that haven't exceeded quota can
  * still sync normally.
  *
- * Clients should prompt the user to upgrade their plan.
+ * Clients should prompt the user to contact the relay provider or upgrade their
+ * plan. Quota monitoring and management is the relay provider's
+ * responsibility.
  */
 export interface ProtocolQuotaError extends BaseOwnerError {
   readonly type: "ProtocolQuotaError";
@@ -436,7 +442,7 @@ export interface ProtocolSyncError extends BaseOwnerError {
 export interface ProtocolTimestampMismatchError {
   readonly type: "ProtocolTimestampMismatchError";
   readonly expected: Timestamp;
-  readonly embedded: Timestamp;
+  readonly timestamp: Timestamp;
 }
 
 /**
@@ -909,8 +915,6 @@ export const applyProtocolMessageAsClient =
       | ProtocolQuotaError
     >
   > => {
-    // try-catch instead of Result for performance and stacktraces
-    // DEV: Measure it again, I think we should use Result with new Error.
     try {
       const input = createBuffer(inputMessage);
       const [requestedVersion, ownerId] = decodeVersionAndOwner(input);
@@ -1057,8 +1061,6 @@ export const applyProtocolMessageAsRelay =
   ): Promise<
     Result<ApplyProtocolMessageAsRelayResult, ProtocolInvalidDataError>
   > => {
-    // try-catch instead of Result for performance and stacktraces
-    // DEV: Measure it again, I think we should use Result with new Error.
     try {
       const input = createBuffer(inputMessage);
       const [requestedVersion, ownerId] = decodeVersionAndOwner(input);
@@ -1664,6 +1666,52 @@ export const decodeNumber = (buffer: Buffer): number => {
   return numberResult.value;
 };
 
+/**
+ * Encodes an array of boolean flags into a single byte.
+ *
+ * Each element in the array corresponds to a bit (0-7). Array can have 0-8
+ * elements.
+ *
+ * ### Example
+ *
+ * ```ts
+ * encodeFlags(buffer, [true, false, true]); // Encodes bits 0, 1, 2
+ * ```
+ */
+export const encodeFlags = (
+  buffer: Buffer,
+  flags: ReadonlyArray<boolean>,
+): void => {
+  let byte = 0;
+  for (let i = 0; i < flags.length && i < 8; i++) {
+    if (flags[i]) {
+      byte |= 1 << i;
+    }
+  }
+  buffer.extend([byte]);
+};
+
+/**
+ * Decodes a byte into an array of boolean flags.
+ *
+ * ### Example
+ *
+ * ```ts
+ * const flags = decodeFlags(buffer, 3); // Decode 3 flags
+ * ```
+ */
+export const decodeFlags = (
+  buffer: Buffer,
+  count: PositiveInt,
+): ReadonlyArray<boolean> => {
+  const byte = buffer.shift();
+  const flags: Array<boolean> = [];
+  for (let i = 0; i < count && i < 8; i++) {
+    flags.push((byte & (1 << i)) !== 0);
+  }
+  return flags;
+};
+
 /**
  * Encodes and encrypts a {@link DbChange} using the provided owner's encryption
  * key. Returns an encrypted binary representation as {@link EncryptedDbChange}.
@@ -1675,36 +1723,33 @@ export const decodeNumber = (buffer: Buffer): number => {
 export const encodeAndEncryptDbChange =
   (deps: SymmetricCryptoDep) =>
   (message: CrdtMessage, key: EncryptionKey): EncryptedDbChange => {
-    const change = message.change;
     const buffer = createBuffer();
 
-    // Encode protocol version first for backward compatibility
     encodeNonNegativeInt(buffer, protocolVersion);
 
-    // Encode the timestamp (after version) for tamper verification
-    const timestampBytes = timestampToTimestampBytes(message.timestamp);
-    buffer.extend(timestampBytes);
+    // Encode the timestamp to prevent tampering (e.g., a malicious relay
+    // assigning this EncryptedDbChange to a different EncryptedCrdtMessage)
+    buffer.extend(timestampToTimestampBytes(message.timestamp));
 
-    encodeString(buffer, change.table);
+    encodeFlags(buffer, [
+      message.change.isInsert,
+      message.change.isDelete != null,
+      message.change.isDelete ?? false,
+    ]);
 
-    buffer.extend(idToIdBytes(change.id));
+    encodeString(buffer, message.change.table);
+    buffer.extend(idToIdBytes(message.change.id));
 
-    const entries = objectToEntries(change.values).map(
-      ([column, value]): [string, SqliteValue] => {
-        return [column, value];
-      },
-    );
+    const entries = objectToEntries(message.change.values);
 
     encodeLength(buffer, entries);
-
     for (const [column, value] of entries) {
       encodeString(buffer, column);
       encodeSqliteValue(buffer, value);
     }
 
-    const paddingLength = padmePaddingLength(buffer.getLength());
-    // Add zero bytes as PADMÉ padding - these will be ignored during decoding.
-    buffer.extend(new Uint8Array(paddingLength));
+    // Add PADMÉ padding (ignored during decoding)
+    buffer.extend(createPadmePadding(buffer.getLength()));
 
     const { nonce, ciphertext } = deps.symmetricCrypto.encrypt(
       buffer.unwrap(),
@@ -1735,13 +1780,11 @@ export const decryptAndDecodeDbChange =
     | ProtocolInvalidDataError
     | ProtocolTimestampMismatchError
   > => {
-    // try-catch instead of Result for performance and stacktraces
     try {
       const buffer = createBuffer(message.change);
-      const nonce = buffer.shiftN(deps.symmetricCrypto.nonceLength);
 
-      const ciphertextLength = decodeLength(buffer);
-      const ciphertext = buffer.shiftN(ciphertextLength);
+      const nonce = buffer.shiftN(deps.symmetricCrypto.nonceLength);
+      const ciphertext = buffer.shiftN(decodeLength(buffer));
 
       const plaintextBytes = deps.symmetricCrypto.decrypt(
         ciphertext,
@@ -1753,24 +1796,22 @@ export const decryptAndDecodeDbChange =
       buffer.reset();
       buffer.extend(plaintextBytes.value);
 
-      // Decode version (for future compatibility, no validation needed for now)
+      // Decode version (for future compatibility, not need yet)
       decodeNonNegativeInt(buffer);
 
-      // Decode and verify the embedded timestamp
-      const embeddedTimestampBytes = buffer.shiftN(timestampBytesLength);
-      const embeddedTimestamp = timestampBytesToTimestamp(
-        embeddedTimestampBytes as TimestampBytes,
+      const timestamp = timestampBytesToTimestamp(
+        buffer.shiftN(timestampBytesLength) as TimestampBytes,
       );
 
-      // Verify timestamp integrity
-      if (!eqTimestamp(embeddedTimestamp, message.timestamp)) {
+      if (!eqTimestamp(timestamp, message.timestamp)) {
         return err<ProtocolTimestampMismatchError>({
           type: "ProtocolTimestampMismatchError",
           expected: message.timestamp,
-          embedded: embeddedTimestamp,
+          timestamp,
         });
       }
 
+      const flags = decodeFlags(buffer, PositiveInt.orThrow(3));
       const table = decodeString(buffer);
       const id = decodeId(buffer);
 
@@ -1783,7 +1824,13 @@ export const decryptAndDecodeDbChange =
         values[column] = value;
       }
 
-      const dbChange = { table, id, values };
+      const dbChange = DbChange.orThrow({
+        table,
+        id,
+        values,
+        isInsert: flags[0],
+        isDelete: flags[1] ? flags[2] : null,
+      });
 
       return ok(dbChange);
     } catch (error) {

package/src/Evolu/PublicKysely.ts

@@ -209,8 +209,7 @@ export function getJsonObjectArgs(
   table: string,
 ): Array<Expression<unknown> | string> {
   const args: Array<Expression<unknown> | string> = [];
-  args.push(kyselyJsonIdentifier, kyselyJsonIdentifier);
-
+  
   for (const { selection: s } of node.selections ?? []) {
     if (ReferenceNode.is(s) && ColumnNode.is(s.column)) {
       args.push(

package/src/Evolu/Query.ts

@@ -1,3 +1,4 @@
+import { Brand } from "../Brand.js";
 import { bytesToHex, hexToBytes } from "../Buffer.js";
 import { objectToEntries } from "../Object.js";
 import {
@@ -9,7 +10,6 @@ import {
 } from "../Sqlite.js";
 import { Store, StoreSubscribe } from "../Store.js";
 import { Simplify } from "../Types.js";
-import { Brand } from "../Brand.js";
 
 /**
  * A type-safe SQL query.
@@ -52,7 +52,7 @@ export const serializeQuery = <R extends Row>(query: SqliteQuery): Query<R> => {
   );
 
   const options = query.options
-    ? objectToEntries(query.options).sort(([a], [b]) => a.localeCompare(b))
+    ? objectToEntries(query.options).toSorted(([a], [b]) => a.localeCompare(b))
     : [];
 
   return JSON.stringify([query.sql, params, options]) as Query<R>;