effect 4.0.7 → 4.1.1

package/src/eventlog/EventLogEncryption.ts

@@ -11,6 +11,10 @@
  */
 import type { NonEmptyArray } from "../Array.ts"
 import * as Context from "../Context.ts"
+import * as Aead from "../crypto/Aead.ts"
+import * as Digest from "../crypto/Digest.ts"
+import * as SecureRandom from "../crypto/SecureRandom.ts"
+import * as WebCrypto from "../crypto/WebCrypto.ts"
 import * as Effect from "../Effect.ts"
 import * as SchemaBinary from "../encoding/SchemaBinary.ts"
 import * as Layer from "../Layer.ts"
@@ -19,7 +23,8 @@ import * as Schema from "../Schema.ts"
 import * as Transferable from "../workers/Transferable.ts"
 import { type Entry, EntryFields, entryFromFields, EntryId, RemoteEntry } from "./EventJournal.ts"
 import type { Identity } from "./EventLog.ts"
-import { makeGetIdentityRootSecretMaterial } from "./internal/identityRootSecretDerivation.ts"
+import { toBatches } from "./internal/batches.ts"
+import { cachedByIdentity, deriveEncryptionKey } from "./internal/identityRootSecretDerivation.ts"
 
 /**
  * Schema for a batch of journal entries encrypted as one AES-GCM message.
@@ -77,33 +82,10 @@ export const EncryptedRemoteBatch = Schema.Struct({
   ...EncryptedBatch.fields
 })
 
-const maxBatchEntries = 100
-const maxBatchPayloadBytes = 256 * 1024
-
-// A batch holds at most `maxBatchEntries` entries or `maxBatchPayloadBytes` of
-// payload, whichever comes first. A single larger entry is a batch on its own.
-const toBatches = (entries: ReadonlyArray<Entry>): Array<NonEmptyArray<Entry>> => {
-  const batches: Array<NonEmptyArray<Entry>> = []
-  let current: Array<Entry> = []
-  let bytes = 0
-  for (const entry of entries) {
-    const size = entry.payload.byteLength
-    if (current.length > 0 && (current.length === maxBatchEntries || bytes + size > maxBatchPayloadBytes)) {
-      batches.push(current as NonEmptyArray<Entry>)
-      current = []
-      bytes = 0
-    }
-    current.push(entry)
-    bytes += size
-  }
-  if (current.length > 0) batches.push(current as NonEmptyArray<Entry>)
-  return batches
-}
-
 const textEncoder = new TextEncoder()
 
 // The store id is length-prefixed, followed by the 16-byte entry ids.
-const additionalData = (storeId: string, entryIds: ReadonlyArray<EntryId>): Uint8Array<ArrayBuffer> => {
+const additionalData = (storeId: string, entryIds: ReadonlyArray<EntryId>): Uint8Array => {
   const store = textEncoder.encode(storeId)
   const data = new Uint8Array(4 + store.byteLength + entryIds.length * 16)
   new DataView(data.buffer).setUint32(0, store.byteLength)
@@ -128,12 +110,6 @@ const BatchFromSchemaBinary = SchemaBinary.toCodec(Schema.NonEmptyArray(EntryFie
 const encodeBatch = Schema.encodeEffect(BatchFromSchemaBinary)
 const decodeBatch = Schema.decodeEffect(BatchFromSchemaBinary)
 
-const toArrayBuffer = (data: Uint8Array): ArrayBuffer => {
-  const buffer = new ArrayBuffer(data.byteLength)
-  new Uint8Array(buffer).set(data)
-  return buffer
-}
-
 /**
  * Service that provides identity generation, entry
  * encryption and decryption, and SHA-256 hashing for event-log replication.
@@ -158,103 +134,135 @@ export class EventLogEncryption extends Context.Service<EventLogEncryption, {
     storeId: string,
     batches: ReadonlyArray<EncryptedRemoteBatch>
   ) => Effect.Effect<Array<RemoteEntry>>
-  readonly sha256String: (data: Uint8Array) => Effect.Effect<string>
   readonly sha256: (data: Uint8Array) => Effect.Effect<Uint8Array>
   readonly generateIdentity: Effect.Effect<Identity["Service"]>
 }>()("effect/eventlog/EventLogEncryption") {}
 
 /**
- * Creates an `EventLogEncryption` service backed by the Web Crypto `SubtleCrypto`
- * APIs from the supplied `Crypto` implementation.
+ * Creates an `EventLogEncryption` service from the `effect/crypto` services.
+ *
+ * **Details**
+ *
+ * Each identity's AES-256-GCM key is derived from its root secret with
+ * `Digest` and cached per identity. Batches are encrypted with `Aead` under a
+ * random 12-byte nonce from `SecureRandom`, which also generates identities.
+ * A batch that fails authentication, or whose entries do not match its entry
+ * ids, is a defect.
  *
  * @stability unstable
- * @category encryption
+ * @category constructors
  * @since 4.0.0
  */
-export const makeEncryptionSubtle = (crypto: Crypto): Effect.Effect<EventLogEncryption["Service"]> =>
-  Effect.sync(() => {
-    const getIdentityRootSecretMaterial = makeGetIdentityRootSecretMaterial(crypto)
+export const make: Effect.Effect<
+  EventLogEncryption["Service"],
+  never,
+  Aead.Aead | Digest.Digest | SecureRandom.SecureRandom
+> = Effect.gen(function*() {
+  const aead = yield* Aead.Aead
+  const digest = yield* Digest.Digest
+  const random = yield* SecureRandom.SecureRandom
+  const getEncryptionKey = cachedByIdentity((rootSecret) =>
+    deriveEncryptionKey(rootSecret).pipe(
+      Effect.map(({ encryptionKey }) => encryptionKey),
+      Effect.provideService(Digest.Digest, digest)
+    )
+  )
 
-    return EventLogEncryption.of({
-      encrypt: Effect.fnUntraced(function*(identity, storeId, entries) {
-        const key = (yield* getIdentityRootSecretMaterial(identity)).encryptionKey
-        const batches = toBatches(entries)
-        const data = yield* Effect.orDie(Effect.forEach(batches, (batch) => encodeBatch(batch)))
-        return yield* Effect.promise(() =>
-          Promise.all(batches.map((batch, i) => {
-            const entryIds = batch.map((entry) => entry.id) as NonEmptyArray<EntryId>
-            const iv = crypto.getRandomValues(new Uint8Array(12))
-            return crypto.subtle.encrypt(
-              { name: "AES-GCM", iv, additionalData: additionalData(storeId, entryIds), tagLength: 128 },
-              key,
-              data[i]
-            ).then((encryptedBatch) => ({ entryIds, iv, encryptedBatch: new Uint8Array(encryptedBatch) }))
-          }))
-        )
-      }),
-      decrypt: Effect.fnUntraced(function*(identity, storeId, batches) {
-        const key = (yield* getIdentityRootSecretMaterial(identity)).encryptionKey
-        const decrypted = yield* Effect.promise(() =>
-          Promise.all(batches.map((batch) =>
-            crypto.subtle.decrypt(
-              {
-                name: "AES-GCM",
-                iv: batch.iv,
-                additionalData: additionalData(storeId, batch.entryIds),
-                tagLength: 128
-              },
-              key,
-              batch.encryptedBatch
-            )
+  return EventLogEncryption.of({
+    encrypt: Effect.fnUntraced(function*(identity, storeId, entries) {
+      const key = yield* getEncryptionKey(identity)
+      const batches = toBatches(entries)
+      const data = yield* Effect.orDie(Effect.forEach(batches, (batch) => encodeBatch(batch)))
+      return yield* Effect.forEach(batches, (batch, i) => {
+        const entryIds = batch.map((entry) => entry.id) as NonEmptyArray<EntryId>
+        return Effect.flatMap(random.randomBytes(12), (iv) =>
+          Effect.map(
+            aead.encrypt(key, data[i], { nonce: iv, aad: additionalData(storeId, entryIds) }),
+            (encryptedBatch) => ({
+              entryIds,
+              iv: iv as Uint8Array<ArrayBuffer>,
+              encryptedBatch: encryptedBatch as Uint8Array<ArrayBuffer>
+            })
           ))
-        )
-        const result: Array<RemoteEntry> = []
-        for (let i = 0; i < batches.length; i++) {
-          const batch = batches[i]
-          const fields = yield* Effect.orDie(decodeBatch(new Uint8Array(decrypted[i])))
-          const entries = fields.map(entryFromFields)
-          if (
-            entries.length !== batch.entryIds.length ||
-            entries.some((entry, j) => !equalBytes(entry.id, batch.entryIds[j]))
-          ) {
-            return yield* Effect.die(new Error("Encrypted batch entries do not match its entry ids"))
-          }
-          for (let j = 0; j < entries.length; j++) {
-            result.push(new RemoteEntry({ remoteSequence: batch.sequence + j, entry: entries[j] }))
-          }
+      }, { concurrency: "unbounded" }).pipe(Effect.orDie)
+    }),
+    decrypt: Effect.fnUntraced(function*(identity, storeId, batches) {
+      const key = yield* getEncryptionKey(identity)
+      const decrypted = yield* Effect.forEach(
+        batches,
+        (batch) =>
+          aead.decrypt(key, batch.encryptedBatch, { nonce: batch.iv, aad: additionalData(storeId, batch.entryIds) }),
+        { concurrency: "unbounded" }
+      ).pipe(Effect.orDie)
+      const result: Array<RemoteEntry> = []
+      for (let i = 0; i < batches.length; i++) {
+        const batch = batches[i]
+        const fields = yield* Effect.orDie(decodeBatch(decrypted[i] as Uint8Array<ArrayBuffer>))
+        const entries = fields.map(entryFromFields)
+        if (
+          entries.length !== batch.entryIds.length ||
+          entries.some((entry, j) => !equalBytes(entry.id, batch.entryIds[j]))
+        ) {
+          return yield* Effect.die(new Error("Encrypted batch entries do not match its entry ids"))
+        }
+        for (let j = 0; j < entries.length; j++) {
+          result.push(new RemoteEntry({ remoteSequence: batch.sequence + j, entry: entries[j] }))
         }
-        return result
-      }),
-      sha256: (data) =>
-        Effect.promise(() => crypto.subtle.digest("SHA-256", toArrayBuffer(data))).pipe(
-          Effect.map((hash) => new Uint8Array(hash))
-        ),
-      sha256String: (data) =>
-        Effect.map(
-          Effect.promise(() => crypto.subtle.digest("SHA-256", toArrayBuffer(data))),
-          (hash) => {
-            const hashArray = Array.from(new Uint8Array(hash))
-            const hashHex = hashArray
-              .map((bytes) => bytes.toString(16).padStart(2, "0"))
-              .join("")
-            return hashHex
-          }
-        ),
-      generateIdentity: Effect.sync(() => ({
-        publicKey: crypto.randomUUID(),
-        privateKey: Redacted.make(crypto.getRandomValues(new Uint8Array(32)))
-      }))
-    })
+      }
+      return result
+    }),
+    sha256: (data) => digest.digest(Digest.sha256, data),
+    generateIdentity: Effect.zipWith(
+      random.uuidV4(),
+      random.randomBytes(32),
+      (publicKey, privateKey) => ({ publicKey, privateKey: Redacted.make(privateKey as Uint8Array<ArrayBuffer>) })
+    )
   })
+})
 
 /**
- * Provides `EventLogEncryption` using `globalThis.crypto`.
+ * Provides `EventLogEncryption` from the `Aead`, `Digest` and `SecureRandom`
+ * services.
+ *
+ * **When to use**
+ *
+ * Use to choose the crypto layers yourself, such as the `NodeCrypto` layers.
  *
  * @stability unstable
  * @category layers
  * @since 4.0.0
  */
-export const layerSubtle: Layer.Layer<EventLogEncryption> = Layer.effect(
-  EventLogEncryption,
-  makeEncryptionSubtle(globalThis.crypto)
+export const layer: Layer.Layer<EventLogEncryption, never, Aead.Aead | Digest.Digest | SecureRandom.SecureRandom> =
+  Layer.effect(EventLogEncryption, make)
+
+const layerWebCrypto = Layer.mergeAll(Aead.layerWebCrypto, Digest.layerWebCrypto).pipe(
+  Layer.provideMerge(SecureRandom.layerWebCrypto)
 )
+
+/**
+ * Creates an `EventLogEncryption` service on the Web Crypto layers of
+ * `Aead`, `Digest` and `SecureRandom`, using the supplied `Crypto` object.
+ *
+ * @stability unstable
+ * @category encryption
+ * @since 4.0.0
+ */
+export const makeEncryptionSubtle = (crypto: Crypto): Effect.Effect<EventLogEncryption["Service"]> =>
+  // `local` keeps enclosing Web Crypto layers, built on another `Crypto`, from
+  // being reused.
+  Effect.provide(make, Layer.provide(layerWebCrypto, WebCrypto.layer(crypto)), { local: true })
+
+/**
+ * Provides `EventLogEncryption` on the Web Crypto layers of `Aead`, `Digest`
+ * and `SecureRandom`.
+ *
+ * **Details**
+ *
+ * The layers use the `Crypto` object of the `WebCrypto` reference, which
+ * defaults to `globalThis.crypto`.
+ *
+ * @stability unstable
+ * @category layers
+ * @since 4.0.0
+ */
+export const layerSubtle: Layer.Layer<EventLogEncryption> = Layer.provide(layer, layerWebCrypto)

package/src/eventlog/EventLogMessage.ts

@@ -113,6 +113,7 @@ export class EventLogAuthentication extends RpcMiddleware.Service<EventLogAuthen
  *
  * Contains the remote id, a challenge for the client to sign, and the lowest
  * assigned sequence (default 0) for initializing acknowledgement cursors.
+ * `compaction` is `true` when the server serves `EventLogCompactionRpcs`.
  *
  * @stability unstable
  * @category protocols
@@ -121,7 +122,8 @@ export class EventLogAuthentication extends RpcMiddleware.Service<EventLogAuthen
 export class HelloResponse extends Schema.Class<HelloResponse>("effect/eventlog/EventLogRemote/HelloResponse")({
   remoteId: RemoteId,
   challenge: Transferable.Uint8Array,
-  firstSequence: Schema.optional(Schema.Natural)
+  firstSequence: Schema.optional(Schema.Natural),
+  compaction: Schema.optional(Schema.Literal(true))
 }) {}
 
 /**
@@ -441,3 +443,273 @@ export class EventLogRemoteRpcs extends RpcGroup.make(
   WriteSingleRpc,
   ChangesRpc
 ) {}
+
+const compactionScope = {
+  publicKey: Schema.String,
+  storeId: StoreId
+}
+
+const compactionStage = {
+  ...compactionScope,
+  stageId: Schema.String
+}
+
+/**
+ * The compaction state of a store, as reported to clients.
+ *
+ * **Details**
+ *
+ * `wanted` is `true` when the server would open a compaction stage. `head` is
+ * the highest stored sequence.
+ *
+ * @stability unstable
+ * @category protocols
+ * @since 4.0.0
+ */
+export class CompactionStatus
+  extends Schema.Class<CompactionStatus>("effect/eventlog/EventLogRemote/CompactionStatus")({
+    wanted: Schema.Boolean,
+    head: Schema.Int
+  })
+{}
+
+/**
+ * RPC that reports a store's compaction state.
+ *
+ * @stability unstable
+ * @category protocols
+ * @since 4.0.0
+ */
+export class CompactionStatusRpc extends Rpc.make("EventLog.CompactionStatus", {
+  payload: compactionScope,
+  success: CompactionStatus,
+  error: EventLogProtocolError
+}).middleware(EventLogAuthentication) {}
+
+/**
+ * Schema for the result of opening a compaction stage.
+ *
+ * **Details**
+ *
+ * `Started` pins the stage to the active generation and to the last input
+ * sequence. `Busy` means another stage is open on the store. `NotWanted` means
+ * the store doesn't need compacting.
+ *
+ * @stability unstable
+ * @category protocols
+ * @since 4.0.0
+ */
+export const CompactionBeginResult = Schema.Union([
+  Schema.TaggedStruct("Started", {
+    stageId: Schema.String
+  }),
+  Schema.TaggedStruct("Busy", {}),
+  Schema.TaggedStruct("NotWanted", {})
+])
+
+/**
+ * The result of opening a compaction stage.
+ *
+ * @stability unstable
+ * @category protocols
+ * @since 4.0.0
+ */
+export type CompactionBeginResult = typeof CompactionBeginResult.Type
+
+/**
+ * RPC that opens a compaction stage.
+ *
+ * @stability unstable
+ * @category protocols
+ * @since 4.0.0
+ */
+export class CompactionBeginRpc extends Rpc.make("EventLog.CompactionBegin", {
+  payload: compactionScope,
+  success: CompactionBeginResult,
+  error: EventLogProtocolError
+}).middleware(EventLogAuthentication) {}
+
+/**
+ * Schema for the header of one compaction input batch.
+ *
+ * **Details**
+ *
+ * An entry's input sequence is the batch `sequence` plus its position in
+ * `entryIds`.
+ *
+ * @stability unstable
+ * @category protocols
+ * @since 4.0.0
+ */
+export const CompactionInputHeader = Schema.Struct({
+  sequence: Schema.Natural,
+  entryIds: Schema.NonEmptyArray(EntryId)
+})
+
+/**
+ * The header of one compaction input batch.
+ *
+ * @stability unstable
+ * @category protocols
+ * @since 4.0.0
+ */
+export type CompactionInputHeader = typeof CompactionInputHeader.Type
+
+/**
+ * RPC that streams a stage's input batch headers in input order.
+ *
+ * **Details**
+ *
+ * The input is the pinned snapshot, then the originals up to the stage's
+ * `through`. Headers are framed as `SingleMessage` values or `ChunkedMessage`
+ * parts, as in `ChangesRpc`. It fails with a `NotFound` error once the stage
+ * or its pin is gone.
+ *
+ * @stability unstable
+ * @category protocols
+ * @since 4.0.0
+ */
+export class CompactionInputRpc extends Rpc.make("EventLog.CompactionInput", {
+  payload: compactionStage,
+  success: Schema.Union([SingleMessage, ChunkedMessage]),
+  error: EventLogProtocolError,
+  stream: true
+}).middleware(EventLogAuthentication) {
+  static HeadersFromSchemaBinary = SchemaBinary.toCodec(Schema.NonEmptyArray(CompactionInputHeader))
+  static encodeHeaders = Schema.encodeEffect(CompactionInputRpc.HeadersFromSchemaBinary)
+  static decodeHeaders = Schema.decodeEffect(CompactionInputRpc.HeadersFromSchemaBinary)
+}
+
+/**
+ * RPC that streams the encrypted input batches at the given input sequences.
+ *
+ * **Details**
+ *
+ * `batches` holds the input sequence of each batch to fetch. Batches are framed
+ * and encoded as in `ChangesRpc`. It fails with a `NotFound` error once the
+ * stage or its pin is gone.
+ *
+ * @stability unstable
+ * @category protocols
+ * @since 4.0.0
+ */
+export class CompactionFetchRpc extends Rpc.make("EventLog.CompactionFetch", {
+  payload: {
+    ...compactionStage,
+    batches: Schema.Array(Schema.Natural)
+  },
+  success: Schema.Union([SingleMessage, ChunkedMessage]),
+  error: EventLogProtocolError,
+  stream: true
+}).middleware(EventLogAuthentication) {}
+
+/**
+ * RPC that uploads one piece of a compaction snapshot segment.
+ *
+ * **Details**
+ *
+ * A segment is an encoded non-empty array of encrypted batches, indexed by
+ * `chunk.id`, and is split into pieces when larger than
+ * `ChunkedMessage.chunkSize`. Re-uploading a stored segment is acknowledged
+ * without storing it again. A segment that fails validation aborts the stage
+ * with an `InvalidRequest` error.
+ *
+ * @stability unstable
+ * @category protocols
+ * @since 4.0.0
+ */
+export class CompactionPartRpc extends Rpc.make("EventLog.CompactionPart", {
+  payload: {
+    ...compactionStage,
+    chunk: ChunkedMessage
+  },
+  error: EventLogProtocolError
+}).middleware(EventLogAuthentication) {
+  static SegmentFromSchemaBinary = SchemaBinary.toCodec(Schema.NonEmptyArray(EncryptedBatch))
+  static encodeSegment = Schema.encodeEffect(CompactionPartRpc.SegmentFromSchemaBinary)
+  static decodeSegment = Schema.decodeEffect(CompactionPartRpc.SegmentFromSchemaBinary)
+}
+
+/**
+ * RPC that releases an open compaction stage. A late abort is a no-op.
+ *
+ * @stability unstable
+ * @category protocols
+ * @since 4.0.0
+ */
+export class CompactionAbortRpc extends Rpc.make("EventLog.CompactionAbort", {
+  payload: compactionStage,
+  error: EventLogProtocolError
+}).middleware(EventLogAuthentication) {}
+
+/**
+ * Schema for the outcome of a compaction run reported at commit.
+ *
+ * **Details**
+ *
+ * `Snapshot` activates the `segmentCount` uploaded segments. `NoOp` reports a
+ * run that dropped nothing.
+ *
+ * @stability unstable
+ * @category protocols
+ * @since 4.0.0
+ */
+export const CompactionResult = Schema.Union([
+  Schema.TaggedStruct("Snapshot", { segmentCount: Schema.Natural }),
+  Schema.TaggedStruct("NoOp", {})
+])
+
+/**
+ * The outcome of a compaction run reported at commit.
+ *
+ * @stability unstable
+ * @category protocols
+ * @since 4.0.0
+ */
+export type CompactionResult = typeof CompactionResult.Type
+
+/**
+ * RPC that commits a compaction stage.
+ *
+ * **Details**
+ *
+ * Answers `Superseded` when the stage's pinned generation is no longer active.
+ * Fails with `InvalidRequest`, aborting the stage, when the segments are
+ * incomplete or the kept ids are not a non-empty, repeat-free subsequence of
+ * the pinned input. Fails with `NotFound` when the stage expired, was aborted
+ * or committed, or is unknown, so a retried commit fails too.
+ *
+ * @stability unstable
+ * @category protocols
+ * @since 4.0.0
+ */
+export class CompactionCommitRpc extends Rpc.make("EventLog.CompactionCommit", {
+  payload: {
+    ...compactionStage,
+    result: CompactionResult
+  },
+  success: Schema.Literals(["Committed", "Superseded"]),
+  error: EventLogProtocolError
+}).middleware(EventLogAuthentication) {}
+
+/**
+ * RPC group for encrypted remote compaction.
+ *
+ * **Details**
+ *
+ * Served by the encrypted server alongside `EventLogRemoteRpcs`. Clients call
+ * it only when `HelloResponse.compaction` is set.
+ *
+ * @stability unstable
+ * @category protocols
+ * @since 4.0.0
+ */
+export class EventLogCompactionRpcs extends RpcGroup.make(
+  CompactionStatusRpc,
+  CompactionBeginRpc,
+  CompactionInputRpc,
+  CompactionFetchRpc,
+  CompactionPartRpc,
+  CompactionAbortRpc,
+  CompactionCommitRpc
+) {}