effect 4.0.6 → 4.1.0

package/src/eventlog/SqlEventLogServerUnencrypted.ts

@@ -521,12 +521,12 @@ export const makeStorage = Effect.fnUntraced(function*(options?: StorageOptions)
             primary_key: entry.primaryKey,
             payload: entry.payload
           })
-          if (rowsForInsert.length >= insertBatchSize) {
+          if (rowsForInsert.length >= insertBatchSize && Arr.isArrayNonEmpty(rowsForInsert)) {
             yield* sql`INSERT INTO ${entriesTableSql} ${sql.insert(rowsForInsert)}`
             rowsForInsert = []
           }
         }
-        if (rowsForInsert.length > 0) {
+        if (Arr.isArrayNonEmpty(rowsForInsert)) {
           yield* sql`INSERT INTO ${entriesTableSql} ${sql.insert(rowsForInsert)}`
         }
         const nextSequence = currentNextSequence + entries.length

package/src/eventlog/internal/batches.ts

@@ -0,0 +1,30 @@
+import type { NonEmptyArray } from "../../Array.ts"
+import type { Entry } from "../EventJournal.ts"
+
+/** The most entries a remote batch holds. */
+export const maxBatchEntries = 100
+
+/** The most payload bytes a remote batch holds, unless it is a single entry. */
+export const maxBatchPayloadBytes = 256 * 1024
+
+/**
+ * Packs entries into batches of at most `maxBatchEntries` entries and
+ * `maxBatchPayloadBytes` of payload. A larger entry is a batch on its own.
+ */
+export const toBatches = (entries: ReadonlyArray<Entry>): Array<NonEmptyArray<Entry>> => {
+  const batches: Array<NonEmptyArray<Entry>> = []
+  let current: Array<Entry> = []
+  let bytes = 0
+  for (const entry of entries) {
+    const size = entry.payload.byteLength
+    if (current.length > 0 && (current.length === maxBatchEntries || bytes + size > maxBatchPayloadBytes)) {
+      batches.push(current as NonEmptyArray<Entry>)
+      current = []
+      bytes = 0
+    }
+    current.push(entry)
+    bytes += size
+  }
+  if (current.length > 0) batches.push(current as NonEmptyArray<Entry>)
+  return batches
+}

package/src/eventlog/internal/compaction.ts

@@ -0,0 +1,64 @@
+import * as Effect from "../../Effect.ts"
+import type { StoreId } from "../EventLogMessage.ts"
+
+/** Rules and audit logs shared by the memory and SQL compaction storages. */
+
+export const hourMillis = 60 * 60 * 1000
+
+/** The newest storage-time hour that ended at least `lagMillis` ago. */
+export const lagHour = (now: number, lagMillis: number): number => Math.floor((now - lagMillis) / hourMillis) - 1
+
+/** A staged batch's position packs its segment index and its index in the segment. */
+export const positionsPerSegment = 2 ** 20
+export const maxSegments = Math.floor(Number.MAX_SAFE_INTEGER / positionsPerSegment)
+
+const service = "EventLogServerEncrypted"
+
+export const logCommit = (options: {
+  readonly publicKey: string
+  readonly storeId: StoreId
+  readonly result: "Committed" | "Superseded"
+  readonly through: number
+  readonly keptCount: number
+  readonly activationId: string | undefined
+}): Effect.Effect<void> =>
+  Effect.logInfo(
+    options.result === "Committed" ? "Committed compaction snapshot" : "Compaction snapshot superseded"
+  ).pipe(
+    Effect.annotateLogs({
+      service,
+      publicKey: options.publicKey,
+      storeId: options.storeId,
+      keptCount: options.keptCount,
+      through: options.through,
+      activationId: options.activationId
+    })
+  )
+
+export const logNoOp = (options: {
+  readonly publicKey: string
+  readonly storeId: StoreId
+  readonly stageId: string
+  readonly through: number
+}): Effect.Effect<void> =>
+  Effect.logInfo("Committed compaction no-op").pipe(Effect.annotateLogs({ service, ...options }))
+
+export const logRollback = (options: {
+  readonly publicKey: string
+  readonly storeId: StoreId
+  readonly activation: { readonly id: string; readonly base: number; readonly keptCount: number | undefined }
+  readonly withdrawn: { readonly id: string; readonly base: number; readonly keptCount: number | undefined }
+}): Effect.Effect<void> =>
+  Effect.logInfo("Rolled back compaction snapshot").pipe(
+    Effect.annotateLogs({
+      service,
+      publicKey: options.publicKey,
+      storeId: options.storeId,
+      keptCount: options.activation.keptCount,
+      base: options.activation.base,
+      activationId: options.activation.id,
+      withdrawnActivationId: options.withdrawn.id,
+      withdrawnBase: options.withdrawn.base,
+      withdrawnKeptCount: options.withdrawn.keptCount
+    })
+  )

package/src/eventlog/internal/conflicts.ts

@@ -82,10 +82,6 @@ export class ConflictIndex {
     }
   }
 
-  get events(): Array<string> {
-    return Array.from(this.stored.keys())
-  }
-
   /** The `(event, primaryKey)` groups in the batch. */
   *groups(): Iterable<ConflictGroup> {
     for (const byKey of this.stored.values()) {

package/src/eventlog/internal/identityRootSecretDerivation.ts

@@ -1,30 +1,16 @@
+import * as Aead from "../../crypto/Aead.ts"
+import * as Digest from "../../crypto/Digest.ts"
+import * as KeyFormat from "../../crypto/KeyFormat.ts"
+import * as SecureRandom from "../../crypto/SecureRandom.ts"
+import * as Signature from "../../crypto/Signature.ts"
+import * as WebCrypto from "../../crypto/WebCrypto.ts"
 import * as Effect from "../../Effect.ts"
+import * as Layer from "../../Layer.ts"
 import * as Redacted from "../../Redacted.ts"
 import type { Identity } from "../EventLog.ts"
 
 const textEncoder = new TextEncoder()
 
-const Ed25519PublicKeyLength = 32
-
-const Ed25519Pkcs8SeedPrefix = Uint8Array.from([
-  0x30,
-  0x2e,
-  0x02,
-  0x01,
-  0x00,
-  0x30,
-  0x05,
-  0x06,
-  0x03,
-  0x2b,
-  0x65,
-  0x70,
-  0x04,
-  0x22,
-  0x04,
-  0x20
-])
-
 /**
  * @internal
  */
@@ -40,124 +26,108 @@ export const SigningDerivationLabelV1 = "effect/eventlog/identity/v1/signing"
  */
 export interface IdentityRootSecretMaterial {
   readonly encryptionKeyMaterial: Uint8Array<ArrayBuffer>
-  readonly encryptionKey: CryptoKey
+  readonly encryptionKey: Aead.Key<Aead.AesGcm>
   readonly signingPublicKey: Uint8Array<ArrayBuffer>
+  /**
+   * The PKCS#8 encoding of the Ed25519 signing key.
+   */
   readonly signingPrivateKey: Redacted.Redacted<Uint8Array<ArrayBuffer>>
 }
 
-const toArrayBuffer = (data: Uint8Array): ArrayBuffer => {
-  const copy = new Uint8Array(data.byteLength)
-  copy.set(data)
-  return copy.buffer
-}
-
-const decodeBase64Url = (value: string): Uint8Array<ArrayBuffer> => {
-  const normalized = value.replaceAll("-", "+").replaceAll("_", "/")
-  const remainder = normalized.length % 4
-  const padded = remainder === 0 ? normalized : `${normalized}${"=".repeat(4 - remainder)}`
-  const decoded = atob(padded)
-  const bytes = new Uint8Array(decoded.length)
-  for (let i = 0; i < decoded.length; i++) {
-    bytes[i] = decoded.charCodeAt(i)
-  }
-  return bytes
-}
-
-const makeEd25519Pkcs8FromSeed = (seed: Uint8Array): Uint8Array<ArrayBuffer> => {
-  const key = new Uint8Array(Ed25519Pkcs8SeedPrefix.byteLength + seed.byteLength)
-  key.set(Ed25519Pkcs8SeedPrefix, 0)
-  key.set(seed, Ed25519Pkcs8SeedPrefix.byteLength)
-  return key
-}
-
-const deriveSecretBytes = Effect.fnUntraced(function*(options: {
-  readonly crypto: Crypto
-  readonly rootSecret: Uint8Array
-  readonly label: string
-}) {
-  const labelBytes = textEncoder.encode(options.label)
-  const derivationInput = new Uint8Array(labelBytes.byteLength + 1 + options.rootSecret.byteLength)
-  derivationInput.set(labelBytes, 0)
-  derivationInput[labelBytes.byteLength] = 0
-  derivationInput.set(options.rootSecret, labelBytes.byteLength + 1)
-
-  const digest = yield* Effect.promise(() => options.crypto.subtle.digest("SHA-256", toArrayBuffer(derivationInput)))
+// The crypto services return fresh arrays over an `ArrayBuffer`.
+const toArrayBufferBytes = (bytes: Uint8Array) => bytes as Uint8Array<ArrayBuffer>
+
+// Each secret is SHA-256(label || 0x00 || root secret).
+const deriveSecretBytes = (rootSecret: Uint8Array, label: string): Effect.Effect<Uint8Array, never, Digest.Digest> =>
+  Digest.Digest.use((digest) => {
+    const labelBytes = textEncoder.encode(label)
+    const derivationInput = new Uint8Array(labelBytes.byteLength + 1 + rootSecret.byteLength)
+    derivationInput.set(labelBytes, 0)
+    derivationInput.set(rootSecret, labelBytes.byteLength + 1)
+    return digest.digest(Digest.sha256, derivationInput)
+  })
 
-  return new Uint8Array(digest)
+/**
+ * Derives the AES-256-GCM key that encrypts an identity's entries.
+ *
+ * @internal
+ */
+export const deriveEncryptionKey = Effect.fnUntraced(function*(rootSecret: Uint8Array) {
+  const encryptionKeyMaterial = yield* deriveSecretBytes(rootSecret, EncryptionDerivationLabelV1)
+  const encryptionKey = yield* Effect.orDie(Aead.key(Aead.aes256Gcm, encryptionKeyMaterial))
+  return { encryptionKeyMaterial: toArrayBufferBytes(encryptionKeyMaterial), encryptionKey }
 })
 
 /**
+ * Derives the Ed25519 key an identity signs session authentication payloads
+ * with.
+ *
  * @internal
  */
-export const deriveIdentityRootSecretMaterial = Effect.fnUntraced(function*(options: {
-  readonly crypto: Crypto
-  readonly rootSecret: Uint8Array
-}) {
-  const encryptionKeyMaterial = yield* deriveSecretBytes({
-    crypto: options.crypto,
-    rootSecret: options.rootSecret,
-    label: EncryptionDerivationLabelV1
-  })
-  const signingSeed = yield* deriveSecretBytes({
-    crypto: options.crypto,
-    rootSecret: options.rootSecret,
-    label: SigningDerivationLabelV1
-  })
-
-  const signingPrivateKeyBytes = makeEd25519Pkcs8FromSeed(signingSeed)
-
-  const encryptionKey = yield* Effect.promise(() =>
-    options.crypto.subtle.importKey("raw", toArrayBuffer(encryptionKeyMaterial), "AES-GCM", true, [
-      "encrypt",
-      "decrypt"
-    ])
-  )
-  const signingPrivateKey = yield* Effect.promise(() =>
-    options.crypto.subtle.importKey("pkcs8", toArrayBuffer(signingPrivateKeyBytes), "Ed25519", true, ["sign"])
-  )
-  const signingJwk = yield* Effect.promise(() => options.crypto.subtle.exportKey("jwk", signingPrivateKey))
-
-  if (typeof signingJwk.x !== "string") {
-    return yield* Effect.die(new Error("Unable to export deterministic Ed25519 public key"))
-  }
-
-  const signingPublicKey = decodeBase64Url(signingJwk.x)
-
-  if (signingPublicKey.byteLength !== Ed25519PublicKeyLength) {
-    return yield* Effect.die(
-      new Error(`Expected derived signing public key to be ${Ed25519PublicKeyLength} bytes`)
-    )
-  }
-
+const deriveSigningKey = Effect.fnUntraced(function*(rootSecret: Uint8Array) {
+  const signature = yield* Signature.Signature
+  const seed = yield* deriveSecretBytes(rootSecret, SigningDerivationLabelV1)
+  const privateKey = yield* Signature.privateKey(Signature.ed25519, seed)
+  const signingPublicKey = yield* Effect.flatMap(signature.publicKeyOf(privateKey), Signature.toBytes)
+  const signingPrivateKey = yield* KeyFormat.encode(KeyFormat.pkcs8, privateKey)
   return {
-    encryptionKeyMaterial,
-    encryptionKey,
-    signingPublicKey,
-    signingPrivateKey: Redacted.make(signingPrivateKeyBytes)
-  } satisfies IdentityRootSecretMaterial
-})
+    signingPublicKey: toArrayBufferBytes(signingPublicKey),
+    signingPrivateKey: Redacted.make(toArrayBufferBytes(signingPrivateKey))
+  }
+}, Effect.orDie)
 
 /**
  * @internal
  */
-export const makeGetIdentityRootSecretMaterial = (crypto: Crypto) => {
-  const cache = new WeakMap<Identity["Service"], IdentityRootSecretMaterial>()
+export const deriveIdentityRootSecretMaterial = (
+  rootSecret: Uint8Array
+): Effect.Effect<IdentityRootSecretMaterial, never, Digest.Digest | Signature.Signature> =>
+  Effect.zipWith(deriveEncryptionKey(rootSecret), deriveSigningKey(rootSecret), (encryption, signing) => ({
+    ...encryption,
+    ...signing
+  }))
 
-  return Effect.fnUntraced(function*(identity: Identity["Service"]) {
-    const cached = cache.get(identity)
-    if (cached !== undefined) {
-      return cached
-    }
-
-    const derived = yield* deriveIdentityRootSecretMaterial({
-      crypto,
-      rootSecret: Redacted.value(identity.privateKey)
+/**
+ * Memoizes a derivation from an identity's root secret per identity instance.
+ *
+ * @internal
+ */
+export const cachedByIdentity = <A, R>(
+  derive: (rootSecret: Uint8Array) => Effect.Effect<A, never, R>
+) => {
+  const cache = new WeakMap<Identity["Service"], A>()
+  return (identity: Identity["Service"]): Effect.Effect<A, never, R> =>
+    Effect.suspend(() => {
+      const cached = cache.get(identity)
+      if (cached !== undefined) {
+        return Effect.succeed(cached)
+      }
+      return Effect.tap(derive(Redacted.value(identity.privateKey)), (derived) =>
+        Effect.sync(() => {
+          cache.set(identity, derived)
+        }))
     })
+}
 
-    yield* Effect.sync(() => {
-      cache.set(identity, derived)
-    })
+const layerWebCrypto = Layer.mergeAll(Digest.layerWebCrypto, Signature.layerWebCrypto).pipe(
+  Layer.provide(SecureRandom.layerWebCrypto)
+)
 
-    return derived
-  })
-}
+/**
+ * Derives identity material on the Web Crypto layers, using `crypto` when it
+ * is given and the `WebCrypto` reference otherwise.
+ *
+ * @internal
+ */
+export const makeGetIdentityRootSecretMaterial = (crypto?: Crypto) =>
+  cachedByIdentity((rootSecret) =>
+    crypto === undefined
+      ? Effect.provide(deriveIdentityRootSecretMaterial(rootSecret), layerWebCrypto)
+      // `local` keeps enclosing Web Crypto layers, built on another `Crypto`,
+      // from being reused.
+      : Effect.provide(
+        deriveIdentityRootSecretMaterial(rootSecret),
+        Layer.provideMerge(layerWebCrypto, WebCrypto.layer(crypto)),
+        { local: true }
+      )
+  )

package/src/eventlog/internal/remoteCompaction.ts

@@ -0,0 +1,92 @@
+import * as Data from "../../Data.ts"
+import * as Effect from "../../Effect.ts"
+import * as Stream from "../../Stream.ts"
+import type { Entry } from "../EventJournal.ts"
+import type { Registry } from "../EventLog.ts"
+
+/** A second pass over the kept set dropped `entries`, so a compactor does not compose. */
+export class IdempotenceError extends Data.TaggedError("IdempotenceError")<{
+  readonly entries: ReadonlyArray<Entry>
+}> {}
+
+type Compactor = Registry["Service"]["compactors"] extends ReadonlyMap<string, infer A> ? A : never
+
+/**
+ * Folds the opted-in compactors over `entries`, which arrive in input order
+ * without repeated ids, and returns the kept entries in input order.
+ *
+ * Each step folds a chunk into the kept set. The chunk starts at `chunkSize`
+ * and grows to the size of the kept set, so a run stays linear when little is
+ * dropped and memory stays at about twice the kept set. A compactor that
+ * composes keeps the same entries at any chunking.
+ *
+ * Fails with `IdempotenceError` when a second pass over the kept set drops
+ * more. A compactor defect fails the run.
+ */
+export const run = <E, R>(options: {
+  readonly compactors: Registry["Service"]["compactors"]
+  readonly entries: Stream.Stream<Entry, E, R>
+  readonly chunkSize: number
+}): Effect.Effect<ReadonlyArray<Entry>, E | IdempotenceError, R> => {
+  const drop = Effect.fnUntraced(function*(entries: ReadonlyArray<Entry>) {
+    const kept = new Set<Entry>()
+    const calls = new Map<Compactor, Array<Entry>>()
+    for (const entry of entries) {
+      const compactor = options.compactors.get(entry.event)
+      if (compactor === undefined || !compactor.remote) {
+        kept.add(entry)
+        continue
+      }
+      const input = calls.get(compactor)
+      if (input === undefined) calls.set(compactor, [entry])
+      else input.push(entry)
+    }
+    for (const [compactor, input] of calls) {
+      const inputs = new Set(input)
+      const forwarded = new Set<Entry>()
+      let replaced = false
+      yield* compactor.effect({
+        entries: input,
+        remote: true,
+        write: (entry) =>
+          Effect.sync(() => {
+            if (inputs.has(entry)) forwarded.add(entry)
+            else replaced = true
+          })
+      })
+      // Only forwarded input entries can be kept verbatim, so a call that
+      // outputs anything else keeps its whole input.
+      if (replaced) {
+        yield* Effect.logWarning("A remote compactor wrote an entry it was not given, so its input is kept").pipe(
+          Effect.annotateLogs({ events: [...compactor.events] })
+        )
+      }
+      for (const entry of replaced ? input : forwarded) kept.add(entry)
+    }
+    return entries.filter((entry) => kept.has(entry))
+  })
+
+  return Effect.gen(function*() {
+    let kept: ReadonlyArray<Entry> = []
+    let pending: Array<Entry> = []
+    yield* options.entries.pipe(
+      Stream.grouped(options.chunkSize),
+      Stream.runForEach((group) => {
+        for (const entry of group) pending.push(entry)
+        if (pending.length < Math.max(options.chunkSize, kept.length)) return Effect.void
+        const input = kept.concat(pending)
+        pending = []
+        return Effect.map(drop(input), (next) => {
+          kept = next
+        })
+      })
+    )
+    if (pending.length > 0) kept = yield* drop(kept.concat(pending))
+    const again = yield* drop(kept)
+    if (again.length !== kept.length) {
+      const survivors = new Set(again)
+      return yield* new IdempotenceError({ entries: kept.filter((entry) => !survivors.has(entry)) })
+    }
+    return kept
+  })
+}