effect 4.0.6 → 4.1.0

package/src/eventlog/EventLogSessionAuth.ts

@@ -9,8 +9,13 @@
  * @stability unstable
  * @since 4.0.0
  */
+import * as KeyFormat from "../crypto/KeyFormat.ts"
+import * as SecureRandom from "../crypto/SecureRandom.ts"
+import * as Signature from "../crypto/Signature.ts"
+import { WebCrypto } from "../crypto/WebCrypto.ts"
 import * as Data from "../Data.ts"
 import * as Effect from "../Effect.ts"
+import * as Layer from "../Layer.ts"
 
 const textEncoder = new TextEncoder()
 const textDecoder = new TextDecoder("utf-8", { fatal: true })
@@ -127,12 +132,6 @@ export class EventLogSessionAuthError extends Data.TaggedError("EventLogSessionA
   readonly cause?: unknown
 }> {}
 
-const toArrayBuffer = (data: Uint8Array): ArrayBuffer => {
-  const copy = new Uint8Array(data.byteLength)
-  copy.set(data)
-  return copy.buffer
-}
-
 const decodeUtf8 = (bytes: Uint8Array) =>
   Effect.try({
     try: () => textDecoder.decode(bytes),
@@ -165,31 +164,22 @@ const assertSignatureLength = (signature: Uint8Array): Effect.Effect<void, Event
   )
 }
 
-const getSubtle = Effect.suspend(() => {
-  const subtle = globalThis.crypto?.subtle
-  if (subtle === undefined) {
-    return Effect.fail(
-      new EventLogSessionAuthError({
-        reason: "CryptoUnavailable",
-        message: "globalThis.crypto.subtle is not available"
-      })
-    )
-  }
-  return Effect.succeed(subtle)
-})
+// The Web Crypto layers read the `WebCrypto` reference, so a runtime without
+// Web Crypto fails here rather than with a defect inside the layer.
+const requireWebCrypto = (needsSubtle: boolean): Effect.Effect<void, EventLogSessionAuthError> =>
+  WebCrypto.use((crypto) => {
+    if (crypto === undefined || (needsSubtle && crypto.subtle === undefined)) {
+      return Effect.fail(
+        new EventLogSessionAuthError({
+          reason: "CryptoUnavailable",
+          message: needsSubtle ? "crypto.subtle is not available" : "crypto is not available"
+        })
+      )
+    }
+    return Effect.void
+  })
 
-const getCrypto = Effect.suspend(() => {
-  const crypto = globalThis.crypto
-  if (crypto === undefined) {
-    return Effect.fail(
-      new EventLogSessionAuthError({
-        reason: "CryptoUnavailable",
-        message: "globalThis.crypto is not available"
-      })
-    )
-  }
-  return Effect.succeed(crypto)
-})
+const layerSignature = Signature.layerWebCrypto.pipe(Layer.provide(SecureRandom.layerWebCrypto))
 
 const writeLength = (
   target: Uint8Array,
@@ -359,7 +349,8 @@ export const decodeSessionAuthPayload = Effect.fnUntraced(
  *
  * **Details**
  *
- * The private key must be PKCS#8-encoded bytes importable by `SubtleCrypto`.
+ * The private key must be PKCS#8-encoded Ed25519 bytes. Signing runs on
+ * `Signature.layerWebCrypto` with the `WebCrypto` reference.
  *
  * @stability unstable
  * @category signing
@@ -370,35 +361,29 @@ export const signSessionAuthPayloadBytes = Effect.fnUntraced(function*(options:
   readonly signingPrivateKey: Uint8Array
 }): Effect.fn.Return<Uint8Array<ArrayBuffer>, EventLogSessionAuthError> {
   yield* decodeSessionAuthPayload(options.payload)
+  yield* requireWebCrypto(true)
 
-  const subtle = yield* getSubtle
-  let privateKey = yield* Effect.tryPromise({
-    try: () =>
-      subtle.importKey(
-        "pkcs8",
-        toArrayBuffer(options.signingPrivateKey),
-        "Ed25519",
-        false,
-        ["sign"]
-      ),
-    catch: (cause) =>
+  const privateKey = yield* KeyFormat.decodePrivate(KeyFormat.pkcs8, Signature.ed25519, options.signingPrivateKey).pipe(
+    Effect.mapError((cause) =>
       new EventLogSessionAuthError({
         reason: "InvalidSigningPrivateKey",
         message: "Failed to import Ed25519 signing private key (expected PKCS#8 bytes)",
         cause
       })
-  })
+    )
+  )
 
-  const signature = yield* Effect.tryPromise({
-    try: () => subtle.sign("Ed25519", privateKey, toArrayBuffer(options.payload)),
-    catch: (cause) =>
+  const signature = yield* Signature.Signature.use((signature) => signature.sign(privateKey, options.payload)).pipe(
+    Effect.provide(layerSignature),
+    Effect.mapError((cause) =>
       new EventLogSessionAuthError({
         reason: "CryptoFailure",
         message: "Failed to sign canonical session auth payload",
         cause
       })
-  })
-  return new Uint8Array(signature)
+    )
+  )
+  return signature as Uint8Array<ArrayBuffer>
 })
 
 /**
@@ -408,7 +393,7 @@ export const signSessionAuthPayloadBytes = Effect.fnUntraced(function*(options:
  * **Details**
  *
  * The payload, signing public key, and signature lengths are validated before
- * calling `SubtleCrypto.verify`.
+ * verifying on `Signature.layerWebCrypto` with the `WebCrypto` reference.
  *
  * @stability unstable
  * @category verification
@@ -422,27 +407,29 @@ export const verifySessionAuthPayloadBytes = Effect.fnUntraced(function*(options
   yield* decodeSessionAuthPayload(options.payload)
   yield* assertSigningPublicKeyLength(options.signingPublicKey)
   yield* assertSignatureLength(options.signature)
+  yield* requireWebCrypto(true)
 
-  const subtle = yield* getSubtle
-  const publicKey = yield* Effect.tryPromise({
-    try: () => subtle.importKey("raw", toArrayBuffer(options.signingPublicKey), "Ed25519", false, ["verify"]),
-    catch: (cause) =>
+  const publicKey = yield* Signature.publicKey(Signature.ed25519, options.signingPublicKey).pipe(
+    Effect.mapError((cause) =>
       new EventLogSessionAuthError({
         reason: "InvalidSigningPublicKeyLength",
         message: "Failed to import Ed25519 signing public key",
         cause
       })
-  })
+    )
+  )
 
-  return yield* Effect.tryPromise({
-    try: () => subtle.verify("Ed25519", publicKey, toArrayBuffer(options.signature), toArrayBuffer(options.payload)),
-    catch: (cause) =>
-      new EventLogSessionAuthError({
-        reason: "CryptoFailure",
-        message: "Failed to verify canonical session auth payload signature",
-        cause
-      })
-  })
+  return yield* Signature.Signature.use((signature) => signature.verify(publicKey, options.signature, options.payload))
+    .pipe(
+      Effect.provide(layerSignature),
+      Effect.mapError((cause) =>
+        new EventLogSessionAuthError({
+          reason: "CryptoFailure",
+          message: "Failed to verify canonical session auth payload signature",
+          cause
+        })
+      )
+    )
 })
 
 /**
@@ -491,7 +478,8 @@ export const verifySessionAuthPayload = (
   )
 
 /**
- * Generates a random session authentication challenge using `globalThis.crypto`.
+ * Generates a random session authentication challenge from `SecureRandom` on
+ * the `WebCrypto` reference.
  *
  * @stability unstable
  * @category constructors
@@ -500,12 +488,10 @@ export const verifySessionAuthPayload = (
 export const makeSessionAuthChallenge: Effect.Effect<
   Uint8Array<ArrayBuffer>,
   EventLogSessionAuthError
-> = Effect.gen(function*() {
-  const crypto = yield* getCrypto
-  const challenge = new Uint8Array(SessionAuthChallengeLength)
-  crypto.getRandomValues(challenge)
-  return challenge
-})
+> = requireWebCrypto(false).pipe(
+  Effect.andThen(SecureRandom.SecureRandom.use((random) => random.randomBytes(SessionAuthChallengeLength))),
+  Effect.provide(SecureRandom.layerWebCrypto)
+) as Effect.Effect<Uint8Array<ArrayBuffer>, EventLogSessionAuthError>
 
 /**
  * Verifies an authentication request by requiring the `Ed25519` algorithm and

package/src/eventlog/EventRow.ts

@@ -524,6 +524,10 @@ export const handler = <
  * patches would apply to a different row, or to none. Entries that fail to
  * decode are kept.
  *
+ * The compactor opts into remote compaction (see the `remote` option of
+ * `EventLog.groupCompaction`). There it also keeps entries with fields the row
+ * schema does not know.
+ *
  * Leaves other events' compactors unchanged. Register one compactor per row
  * event.
  *
@@ -545,14 +549,19 @@ export const compaction = <
   Layer.effectDiscard(Effect.gen(function*() {
     const registry = yield* EventLog.Registry
     const services = yield* Effect.context<DecodingServices<Fields, Key, KeySchema>>()
-    const decode = Schema.decodeUnknownEffect(group.events[row.tag].payloadSchemaBinary)
+    const event = group.events[row.tag]
+    const decode = Schema.decodeUnknownEffect(event.payloadSchemaBinary)
+    // A remote compaction treats entries with unknown fields as undecodable.
+    const decodeStrict = Schema.decodeUnknownEffect(SchemaBinary.toCodecStrict(event.payload))
     yield* registry.registerCompaction({
       events: [row.tag],
-      effect: Effect.fnUntraced(function*({ entries, write }) {
+      remote: true,
+      effect: Effect.fnUntraced(function*({ entries, remote, write }) {
+        const decodePayload = remote ? decodeStrict : decode
         const ops = new Map<Entry, Op>()
         for (const entry of entries) {
           const payload = yield* Effect.result(
-            decode(entry.payload).pipe(Effect.updateContext((input) => Context.merge(services, input)))
+            decodePayload(entry.payload).pipe(Effect.updateContext((input) => Context.merge(services, input)))
           ) as Effect.Effect<Result.Result<Op, unknown>>
           if (Result.isSuccess(payload)) ops.set(entry, payload.success)
         }

package/src/eventlog/SqlEventJournal.ts

@@ -16,7 +16,7 @@ import * as SqlClient from "../sql/SqlClient.ts"
 import * as SqlError from "../sql/SqlError.ts"
 import * as SqlSchema from "../sql/SqlSchema.ts"
 import * as EventJournal from "./EventJournal.ts"
-import { ConflictIndex } from "./internal/conflicts.ts"
+import { type ConflictGroup, ConflictIndex } from "./internal/conflicts.ts"
 
 type WriteFromRemoteOptions = Parameters<EventJournal.EventJournal["Service"]["writeFromRemote"]>[0]
 
@@ -315,12 +315,18 @@ export const make = (options?: {
 
     const insertEntries = SqlSchema.void({
       Request: EntryRowArray,
-      execute: (entries) => sql`INSERT INTO ${entryTableSql} ${sql.insert(entries)} ON CONFLICT DO NOTHING`
+      execute: (entries) =>
+        forEachChunk(
+          entries,
+          entryColumns,
+          (chunk) => sql`INSERT INTO ${entryTableSql} ${sql.insert(chunk)} ON CONFLICT DO NOTHING`
+        )
     })
     const insertSources = (entries: ReadonlyArray<EventJournal.Entry>, remoteId: EventJournal.RemoteId) =>
-      sql`INSERT INTO ${sourcesTableSql} ${
-        sql.insert(entries.map((entry) => ({ entry_id: entry.id, remote_id: remoteId })))
-      } ON CONFLICT DO NOTHING`
+      forEachChunk(entries, 2, (chunk) => {
+        const rows = Arr.map(chunk, (entry) => ({ entry_id: entry.id, remote_id: remoteId }))
+        return sql`INSERT INTO ${sourcesTableSql} ${sql.insert(rows)} ON CONFLICT DO NOTHING`
+      })
     // Serialize writers so checkpoints cannot skip out-of-order commits.
     // Lock before callbacks in both paths to avoid projection-row deadlocks.
     const lockWriters = sql.onDialectOrElse({
@@ -328,9 +334,10 @@ export const make = (options?: {
       orElse: () => Effect.void
     })
     const insertSequence = (entries: ReadonlyArray<EventJournal.Entry>) =>
-      sql`INSERT INTO ${sequenceTableSql} ${
-        sql.insert(entries.map((entry) => ({ entry_id: entry.id })))
-      } ON CONFLICT DO NOTHING`
+      forEachChunk(entries, 1, (chunk) => {
+        const rows = Arr.map(chunk, (entry) => ({ entry_id: entry.id }))
+        return sql`INSERT INTO ${sequenceTableSql} ${sql.insert(rows)} ON CONFLICT DO NOTHING`
+      })
 
     const decodeCursorRows = Schema.decodeUnknownEffect(CursorRowArray)
 
@@ -361,24 +368,12 @@ export const make = (options?: {
 
       yield* lockWriters
 
-      const existingIds = new Set<string>()
-      if (entries.length > 0) {
-        yield* sql`SELECT id FROM ${entryTableSql} WHERE ${
-          sql.in(
-            "id",
-            entries.map((entry) => entry.id)
-          )
-        }`.pipe(
-          Effect.flatMap(decodeEntryIdRows),
-          Effect.tap((rows) =>
-            Effect.sync(() => {
-              for (const row of rows) {
-                existingIds.add(Uuid.stringify(row.id))
-              }
-            })
-          )
-        )
-      }
+      const existingRows = yield* forEachChunk(
+        entries,
+        1,
+        (chunk) => sql`SELECT id FROM ${entryTableSql} WHERE ${sql.in("id", chunk.map((entry) => entry.id))}`
+      ).pipe(Effect.flatMap(decodeEntryIdRows))
+      const existingIds = new Set(existingRows.map((row) => Uuid.stringify(row.id)))
       // Overlapping batches can repeat an entry; the first copy wins.
       const incomingIds = new Set<string>()
       const uncommitted = options.entries.filter((entry) => {
@@ -396,27 +391,38 @@ export const make = (options?: {
 
       if (Arr.isReadonlyArrayNonEmpty(compacted)) {
         const index = new ConflictIndex(compacted)
-        const primaryKeys = Array.from(new Set(compacted.map((entry) => entry.primaryKey)))
-        // Match the hash index; exact predicates below rule out hash collisions.
-        const hashed = sql.onDialectOrElse({
-          pg: () => [
-            sql`md5(event) IN (${sql.csv(index.events.map((event) => sql`md5(${event})`))})`,
-            sql`md5(primary_key) IN (${sql.csv(primaryKeys.map((primaryKey) => sql`md5(${primaryKey})`))})`
-          ],
-          orElse: () => []
-        })
-        const conflicts = sql.and([
-          ...hashed,
-          sql.in("event", index.events),
-          sql.in("primary_key", primaryKeys),
-          // A coarse filter: the index compares ids exactly.
-          sql`timestamp >= ${index.oldestMillis}`
-        ])
-        const stored = yield* sql`SELECT * FROM ${entryTableSql} WHERE ${conflicts}`.pipe(
-          Effect.flatMap(decodeEntryRows),
+        const conflicts = (groups: ReadonlyArray<ConflictGroup>) => {
+          const events = Array.from(new Set(groups.map((group) => group.event)))
+          const primaryKeys = Array.from(new Set(groups.map((group) => group.primaryKey)))
+          return sql`SELECT * FROM ${entryTableSql} WHERE ${
+            sql.and([
+              // Match the hash index; exact predicates below rule out hash collisions.
+              ...sql.onDialectOrElse({
+                pg: () => [
+                  sql`md5(event) IN (${sql.csv(events.map((event) => sql`md5(${event})`))})`,
+                  sql`md5(primary_key) IN (${sql.csv(primaryKeys.map((primaryKey) => sql`md5(${primaryKey})`))})`
+                ],
+                orElse: () => []
+              }),
+              sql.in("event", events),
+              sql.in("primary_key", primaryKeys),
+              // A coarse filter: the index compares ids exactly.
+              sql`timestamp >= ${index.oldestMillis}`
+            ])
+          }`
+        }
+        // Each group binds its event and key, twice on PostgreSQL, and one
+        // parameter is left for the timestamp.
+        const groupsPerChunk = Math.floor((maxParameters - 1) / sql.onDialectOrElse({ pg: () => 4, orElse: () => 2 }))
+        const stored = yield* Effect.forEach(Arr.chunksOf(index.groups(), groupsPerChunk), conflicts).pipe(
+          Effect.flatMap((rows) => decodeEntryRows(rows.flat())),
           Effect.map(toEntries)
         )
+        // Chunks can share an event and a key, so a row can be returned twice.
+        const added = new Set<string>()
         for (const entry of stored) {
+          if (added.has(entry.idString)) continue
+          added.add(entry.idString)
           index.add(entry)
         }
         yield* options.effect(index.toBatch())
@@ -485,7 +491,7 @@ export const make = (options?: {
         Effect.mapError((cause) => new EventJournal.EventJournalError({ cause, method: "entries" }))
       ),
       entriesById: (ids) =>
-        ids.length === 0 ? Effect.succeed([]) : sql`SELECT * FROM ${entryTableSql} WHERE ${sql.in("id", ids)}`.pipe(
+        forEachChunk(ids, 1, (chunk) => sql`SELECT * FROM ${entryTableSql} WHERE ${sql.in("id", chunk)}`).pipe(
           withTracerDisabled,
           Effect.flatMap(decodeEntryRows),
           Effect.map((rows) => {
@@ -641,6 +647,7 @@ const EntryRow = Schema.Struct({
 })
 
 const EntryRowArray = Schema.Array(EntryRow)
+const entryColumns = Object.keys(EntryRow.fields).length
 const decodeUploadBounds = Schema.decodeUnknownEffect(Schema.Tuple([
   Schema.Struct({ checkpoint: SqlNatural, latest: SqlNatural })
 ]))
@@ -668,6 +675,22 @@ const CursorRowArray = Schema.Array(Schema.Struct({ next_sequence: SqlNatural })
 
 const withTracerDisabled = Effect.withTracerEnabled(false)
 
+// Statements bind at most this many parameters: Cloudflare Durable Object
+// SQLite's per-statement limit, the lowest of the supported databases.
+const maxParameters = 100
+
+// Runs `f` in order over chunks of `values`, each binding `parametersPerValue`
+// parameters per value, and concatenates the rows.
+const forEachChunk = <A, B, E, R>(
+  values: ReadonlyArray<A>,
+  parametersPerValue: number,
+  f: (chunk: Arr.NonEmptyReadonlyArray<A>) => Effect.Effect<ReadonlyArray<B>, E, R>
+): Effect.Effect<Array<B>, E, R> =>
+  Effect.map(
+    Effect.forEach(Arr.chunksOf(values, Math.floor(maxParameters / parametersPerValue)), f),
+    (rows) => rows.flat()
+  )
+
 // Preserve callback errors without wrapping them as journal SQL errors.
 class CallbackFailure<E> {
   readonly error: E