effect 4.0.5 → 4.0.7

package/src/eventlog/EventLogServerUnencrypted.ts

@@ -39,6 +39,7 @@ import {
   EventLogProtocolError,
   EventLogRemoteRpcs,
   type StoreId,
+  type WriteAck,
   WriteEntriesUnencrypted
 } from "./EventLogMessage.ts"
 import * as EventLogServer from "./EventLogServer.ts"
@@ -104,8 +105,9 @@ export const makeWrite = <Groups extends EventGroup.Any>(
  * **Details**
  *
  * Incoming plaintext entries are authorized, mapped to a server store, checked
- * for conflicts, run through registered handlers, and persisted; change streams
- * include compacted backlog entries when compactors are registered.
+ * for conflicts, run through registered handlers, and persisted. Acknowledgements
+ * include duplicates with their original sequences. Change streams apply registered
+ * compactors to backlog entries and exclude this connection's writes.
  *
  * @stability unstable
  * @category layers
@@ -129,12 +131,12 @@ export const layerRpcHandlers: Layer.Layer<
     readonly entries: Arr.NonEmptyReadonlyArray<Entry>
   }) {
     const entries = Arr.sort(options.entries, Entry.Order)
-    let history = yield* storage.entriesAfter(options.storeId, entries[0])
+    const stored = yield* storage.entriesAfter(options.storeId, entries[0], entries)
+    const index = new HistoryIndex(stored.map((remoteEntry) => remoteEntry.entry))
     const persistedEntries = Arr.empty<Entry>()
     for (const entry of entries) {
-      const [duplicate, conflicts, newHistory] = toConflicts(history, entry)
-      if (duplicate) continue
-      history = newHistory
+      if (index.has(entry)) continue
+      const conflicts = index.conflicts(entry)
       yield* handler({
         publicKey: options.publicKey,
         storeId: options.storeId,
@@ -143,11 +145,26 @@ export const layerRpcHandlers: Layer.Layer<
       })
       persistedEntries.push(entry)
     }
-    yield* storage.write(options.storeId, persistedEntries)
+    const committed = yield* storage.write(options.storeId, persistedEntries)
+
+    const sequences = new Map<string, number>()
+    for (const remoteEntry of stored) {
+      sequences.set(remoteEntry.entry.idString, remoteEntry.remoteSequence)
+    }
+    for (const remoteEntry of committed) {
+      sequences.set(remoteEntry.entry.idString, remoteEntry.remoteSequence)
+    }
+    const acks: Array<WriteAck> = []
+    for (const entry of options.entries) {
+      const sequence = sequences.get(entry.idString)
+      if (sequence !== undefined) acks.push({ entryId: entry.id, sequence })
+    }
+    return acks
   }, storage.withTransaction)
 
   return EventLogServer.layerRpcHandlers({
     remoteId,
+    firstSequence: storage.firstSequence,
     authorizeIdentity: (publicKey) =>
       auth.authorizeIdentity({ publicKey }).pipe(
         Effect.mapError((error) =>
@@ -161,7 +178,7 @@ export const layerRpcHandlers: Layer.Layer<
       ),
     getOrCreateSessionAuthBinding: (publicKey, signingPublicKey) =>
       storage.getOrCreateSessionAuthBinding(publicKey, signingPublicKey),
-    onWrite: Effect.fnUntraced(function*(data, authenticatedPublicKeys) {
+    onWrite: Effect.fnUntraced(function*(data, authenticatedPublicKeys, writes) {
       const request = yield* WriteEntriesUnencrypted.decode(data).pipe(
         Effect.mapError((_) =>
           new EventLogProtocolError({
@@ -180,7 +197,7 @@ export const layerRpcHandlers: Layer.Layer<
           message: "Identity is not authenticated"
         })
       }
-      if (!Arr.isReadonlyArrayNonEmpty(request.entries)) return
+      if (!Arr.isReadonlyArrayNonEmpty(request.entries)) return []
 
       const resolvedStoreId = yield* mapping.resolve({
         publicKey: request.publicKey,
@@ -211,11 +228,17 @@ export const layerRpcHandlers: Layer.Layer<
           })
         )
       )
-      yield* processEntries({
+      return yield* processEntries({
         publicKey: request.publicKey,
         storeId: resolvedStoreId,
         entries: request.entries
       }).pipe(
+        (write) =>
+          writes.track({
+            publicKey: request.publicKey,
+            storeId: request.storeId,
+            entryIds: request.entries.map((entry) => entry.id)
+          }, write),
         Effect.catchCause((_) =>
           Effect.fail(
             new EventLogProtocolError({
@@ -238,11 +261,19 @@ export const layerRpcHandlers: Layer.Layer<
         publicKey: request.publicKey,
         storeId
       })
-      return storage.changes({
-        storeId,
-        startSequence: request.startSequence,
-        compactors: registry.compactors
-      }).pipe(
+      return request.writes.filter(
+        {
+          publicKey: request.publicKey,
+          storeId: request.storeId,
+          entryIds: (remoteEntry) => [remoteEntry.entry.id],
+          sequence: (remoteEntry) => remoteEntry.remoteSequence
+        },
+        storage.changes({
+          storeId,
+          startSequence: request.startSequence,
+          compactors: registry.compactors
+        })
+      ).pipe(
         Stream.mapArrayEffect((entries) => Effect.map(ChangesRpc.encodeUnencrypted(entries), Arr.of))
       )
     }, Stream.unwrap)
@@ -378,7 +409,8 @@ export const layerStoreMappingStatic = (options: {
  *
  * It provides the server remote id, stores session authentication bindings,
  * allocates remote sequence numbers, persists entries, streams changes, and
- * exposes a transaction boundary.
+ * exposes a transaction boundary. `entriesAfter` returns stored entries with
+ * their sequences, so that duplicate writes can be acknowledged.
  *
  * @stability unstable
  * @category services
@@ -386,11 +418,26 @@ export const layerStoreMappingStatic = (options: {
  */
 export class Storage extends Context.Service<Storage, {
   readonly getId: Effect.Effect<RemoteId>
+  /**
+   * Lowest assigned sequence (default 0). Overestimating skips entries.
+   */
+  readonly firstSequence?: number | undefined
   readonly getOrCreateSessionAuthBinding: (
     publicKey: string,
     signingPublicKey: Uint8Array<ArrayBuffer>
   ) => Effect.Effect<Uint8Array<ArrayBuffer>>
-  readonly entriesAfter: (storeId: StoreId, entry: Entry) => Effect.Effect<Array<Entry>>
+  /**
+   * Returns the stored entries at or after `entry`, in sequence order.
+   *
+   * The server also passes the incoming `batch`, sorted and starting with
+   * `entry`. Implementations may use it to return only the entries that share
+   * an id, or an event and primary key, with a batch entry.
+   */
+  readonly entriesAfter: (
+    storeId: StoreId,
+    entry: Entry,
+    batch?: Arr.NonEmptyReadonlyArray<Entry>
+  ) => Effect.Effect<Array<RemoteEntry>>
   readonly write: (
     storeId: StoreId,
     entries: ReadonlyArray<Entry>
@@ -411,43 +458,83 @@ const constEmptyPrivateKey = Redacted.make(new Uint8Array(32))
 
 const makeServerWriteIdentityPublicKey = (storeId: StoreId): string => `effect-eventlog-server-write:${storeId}`
 
-const entriesAfter = (journal: Array<RemoteEntry>, startSequence: number): ReadonlyArray<RemoteEntry> =>
-  journal.filter((entry) => entry.remoteSequence >= startSequence)
+// The memory journal is append-only with strictly increasing sequences.
+const entriesAfter = (journal: Array<RemoteEntry>, startSequence: number): ReadonlyArray<RemoteEntry> => {
+  let low = 0
+  let high = journal.length
+  while (low < high) {
+    const mid = Math.floor((low + high) / 2)
+    if (journal[mid].remoteSequence >= startSequence) {
+      high = mid
+    } else {
+      low = mid + 1
+    }
+  }
+  return journal.slice(low)
+}
 
-const toConflicts = (
-  history: ReadonlyArray<Entry>,
-  originEntry: Entry
-): [duplicate: boolean, conflicts: Array<Entry>, newHistory: Array<Entry>] => {
-  let duplicate = false
+class KeyIndex<A> {
+  private readonly byEvent = new Map<string, Map<string, Array<A>>>()
 
-  for (let i = 0; i < history.length; i++) {
-    const entry = history[i]
-    if (entry.createdAtMillis < originEntry.createdAtMillis) {
-      continue
-    } else if (entry.idString === originEntry.idString) {
-      duplicate = true
-      continue
+  add(event: string, primaryKey: string, value: A): void {
+    let byKey = this.byEvent.get(event)
+    if (byKey === undefined) {
+      byKey = new Map()
+      this.byEvent.set(event, byKey)
+    }
+    const values = byKey.get(primaryKey)
+    if (values === undefined) {
+      byKey.set(primaryKey, [value])
+    } else {
+      values.push(value)
     }
+  }
 
-    const newHistory = history.slice(i)
-    let conflicts: Array<Entry> = []
-    for (let j = 0; j < newHistory.length; j++) {
-      const scannedEntry = newHistory[j]!
-      if (scannedEntry.event === originEntry.event && scannedEntry.primaryKey === originEntry.primaryKey) {
-        conflicts.push(scannedEntry)
-      }
+  get(event: string, primaryKey: string): ReadonlyArray<A> | undefined {
+    return this.byEvent.get(event)?.get(primaryKey)
+  }
+}
+
+/**
+ * Indexes the history returned by `Storage.entriesAfter` for one batch. A
+ * conflict is a stored entry with the same event and primary key and a
+ * strictly greater id, reported in id order. Batch entries are never added, so
+ * they do not conflict with each other.
+ */
+class HistoryIndex {
+  private readonly ids = new Set<string>()
+  private readonly byKey = new KeyIndex<Entry>()
+
+  constructor(history: ReadonlyArray<Entry>) {
+    // Storage returns history in sequence order; conflicts are reported in id order.
+    for (const entry of Arr.sort(history, Entry.Order)) {
+      this.ids.add(entry.idString)
+      this.byKey.add(entry.event, entry.primaryKey, entry)
     }
-    return [duplicate, conflicts, newHistory]
   }
 
-  return [duplicate, [], []]
+  has(entry: Entry): boolean {
+    return this.ids.has(entry.idString)
+  }
+
+  conflicts(entry: Entry): Array<Entry> {
+    const stored = this.byKey.get(entry.event, entry.primaryKey)
+    if (stored === undefined) return []
+    return stored.filter((candidate) => Entry.Order(candidate, entry) > 0)
+  }
+}
+
+type MemoryStore = {
+  readonly journal: Array<RemoteEntry>
+  readonly byId: Map<string, RemoteEntry>
+  readonly byKey: KeyIndex<RemoteEntry>
 }
 
 type RegisteredCompactor = {
   readonly events: ReadonlySet<string>
   readonly effect: (options: {
     readonly entries: ReadonlyArray<Entry>
-    readonly write: (entry: Entry) => Effect.Effect<void>
+    readonly write: (entry: Entry, source?: Entry) => Effect.Effect<void>
   }) => Effect.Effect<void>
 }
 
@@ -513,6 +600,12 @@ const toCompactedRemoteEntries = (options: {
  * entries when the replacement count can be mapped back to increasing remote
  * sequence numbers; otherwise the original entries are kept.
  *
+ * Outputs are ordered by the position of `source ?? entry` within the run, as in
+ * client replay. Unknown positions go to the start; ties keep write order.
+ *
+ * Client replay groups the whole batch per compactor; the server calls each
+ * compactor per contiguous run, so batch boundaries can differ.
+ *
  * @stability unstable
  * @category compaction
  * @since 4.0.0
@@ -539,7 +632,6 @@ export const compactBacklog = Effect.fnUntraced(function*(options: {
 
     const entries: Array<Entry> = [remoteEntry.entry]
     const remoteGroup: Array<RemoteEntry> = [remoteEntry]
-    const compacted: Array<Entry> = []
     index++
 
     while (index < options.remoteEntries.length) {
@@ -553,25 +645,29 @@ export const compactBacklog = Effect.fnUntraced(function*(options: {
       index++
     }
 
+    const positions = new Map<string, number>()
+    const compacted: Array<Array<Entry>> = []
+    for (let i = 0; i < entries.length; i++) {
+      positions.set(entries[i].idString, i)
+      compacted.push([])
+    }
     yield* compactor.effect({
       entries,
-      write(entry) {
+      write(entry, source) {
         return Effect.sync(() => {
-          compacted.push(entry)
+          compacted[positions.get((source ?? entry).idString) ?? 0].push(entry)
         })
       }
     }).pipe(Effect.orDie)
 
     const projected = toCompactedRemoteEntries({
-      compacted,
+      compacted: compacted.flat(),
       remoteEntries: remoteGroup
     })
 
-    if (projected === undefined) {
-      compactedRemoteEntries.push(...remoteGroup)
-      continue
+    for (const entry of projected ?? remoteGroup) {
+      compactedRemoteEntries.push(entry)
     }
-    compactedRemoteEntries.push(...projected)
   }
 
   return compactedRemoteEntries
@@ -591,25 +687,16 @@ export const compactBacklog = Effect.fnUntraced(function*(options: {
  * @since 4.0.0
  */
 export const makeStorageMemory: Effect.Effect<Storage["Service"], never, Scope.Scope> = Effect.gen(function*() {
-  const knownIds = new Map<string, Map<string, number>>()
-  const journals = new Map<string, Array<RemoteEntry>>()
+  const stores = new Map<string, MemoryStore>()
   const sessionAuthBindings = new Map<string, Uint8Array<ArrayBuffer>>()
   const remoteId = makeRemoteIdUnsafe()
 
-  const ensureKnownIds = (storeId: StoreId): Map<string, number> => {
-    let storeKnownIds = knownIds.get(storeId)
-    if (storeKnownIds) return storeKnownIds
-    storeKnownIds = new Map()
-    knownIds.set(storeId, storeKnownIds)
-    return storeKnownIds
-  }
-
-  const ensureJournal = (storeId: StoreId): Array<RemoteEntry> => {
-    let journal = journals.get(storeId)
-    if (journal) return journal
-    journal = []
-    journals.set(storeId, journal)
-    return journal
+  const ensureStore = (storeId: StoreId): MemoryStore => {
+    let store = stores.get(storeId)
+    if (store) return store
+    store = { journal: [], byId: new Map(), byKey: new KeyIndex() }
+    stores.set(storeId, store)
+    return store
   }
 
   const pubsubs = yield* RcMap.make({
@@ -622,15 +709,13 @@ export const makeStorageMemory: Effect.Effect<Storage["Service"], never, Scope.S
   })
 
   const write = Effect.fnUntraced(function*(storeId: StoreId, entries: ReadonlyArray<Entry>) {
-    const sequenceNumbers: Array<number> = []
     const committed: Array<RemoteEntry> = []
-    const storeKnownIds = ensureKnownIds(storeId)
-    const journal = ensureJournal(storeId)
-    let lastSequenceNumber = Arr.last(journal).pipe(
+    const store = ensureStore(storeId)
+    let lastSequenceNumber = Arr.last(store.journal).pipe(
       Option.map((entry) => entry.remoteSequence),
       Option.getOrElse(() => 0)
     )
-    if (entries.some((entry) => storeKnownIds.has(entry.idString))) {
+    if (entries.some((entry) => store.byId.has(entry.idString))) {
       return yield* Effect.die("Duplicate entries")
     }
 
@@ -640,10 +725,10 @@ export const makeStorageMemory: Effect.Effect<Storage["Service"], never, Scope.S
         entry
       }, { disableChecks: true })
 
-      sequenceNumbers.push(remoteEntry.remoteSequence)
       committed.push(remoteEntry)
-      journal.push(remoteEntry)
-      storeKnownIds.set(entry.idString, remoteEntry.remoteSequence)
+      store.journal.push(remoteEntry)
+      store.byId.set(entry.idString, remoteEntry)
+      store.byKey.add(entry.event, entry.primaryKey, remoteEntry)
     }
 
     const pubsub = yield* RcMap.get(pubsubs, storeId)
@@ -656,6 +741,7 @@ export const makeStorageMemory: Effect.Effect<Storage["Service"], never, Scope.S
 
   return Storage.of({
     getId: Effect.succeed(remoteId),
+    firstSequence: 1,
     getOrCreateSessionAuthBinding: (publicKey, signingPublicKey) =>
       Effect.sync(() => {
         const existing = sessionAuthBindings.get(publicKey)
@@ -663,10 +749,27 @@ export const makeStorageMemory: Effect.Effect<Storage["Service"], never, Scope.S
         sessionAuthBindings.set(publicKey, signingPublicKey)
         return signingPublicKey
       }),
-    entriesAfter: (storeId, entry) =>
+    entriesAfter: (storeId, oldest, batch) =>
       Effect.sync(() => {
-        const journal = ensureJournal(storeId)
-        return journal.filter((e) => Entry.Order(e.entry, entry) >= 0).map((e) => e.entry)
+        const store = ensureStore(storeId)
+        if (batch === undefined) {
+          return store.journal
+            .filter((remoteEntry) => Entry.Order(remoteEntry.entry, oldest) >= 0)
+        }
+        const found = new Set<RemoteEntry>()
+        const scanned = new Set<ReadonlyArray<RemoteEntry>>()
+        for (const entry of batch) {
+          const known = store.byId.get(entry.idString)
+          if (known !== undefined) found.add(known)
+          const sameKey = store.byKey.get(entry.event, entry.primaryKey)
+          if (sameKey === undefined || scanned.has(sameKey)) continue
+          scanned.add(sameKey)
+          for (const remoteEntry of sameKey) {
+            if (Entry.Order(remoteEntry.entry, oldest) >= 0) found.add(remoteEntry)
+          }
+        }
+        return Array.from(found)
+          .sort((a, b) => a.remoteSequence - b.remoteSequence)
       }),
     write,
     changes: Effect.fnUntraced(function*({ storeId, startSequence, compactors }) {
@@ -674,7 +777,7 @@ export const makeStorageMemory: Effect.Effect<Storage["Service"], never, Scope.S
       const subscription = yield* PubSub.subscribe(pubsub)
 
       const backlog = yield* compactBacklog({
-        remoteEntries: entriesAfter(ensureJournal(storeId), startSequence),
+        remoteEntries: entriesAfter(ensureStore(storeId).journal, startSequence),
         compactors
       })
       const replayedUpTo = backlog.length > 0 ? backlog[backlog.length - 1].remoteSequence : startSequence - 1