effect 4.0.5 → 4.0.6

package/src/eventlog/SqlEventLogServerUnencrypted.ts

@@ -11,7 +11,6 @@
  * @since 4.0.0
  */
 import * as Arr from "../Array.ts"
-import * as Context from "../Context.ts"
 import * as Effect from "../Effect.ts"
 import * as Layer from "../Layer.ts"
 import * as PubSub from "../PubSub.ts"
@@ -26,8 +25,9 @@ import * as Stream from "../Stream.ts"
 import { Entry, EntryId, makeRemoteIdUnsafe, RemoteEntry, type RemoteId } from "./EventJournal.ts"
 import * as EventLogServerUnencrypted from "./EventLogServerUnencrypted.ts"
 
-let storageIdCounter = 0
 const withTracerDisabled = Effect.withTracerEnabled(false)
+// Keeps each history lookup within dialect parameter limits.
+const lookupBatchSize = 200
 
 type StorageOptions = {
   readonly entryTablePrefix?: string
@@ -42,7 +42,13 @@ type StorageOptions = {
  *
  * The implementation creates tables for the server remote id, store sequences,
  * entries, and session authentication bindings, then persists and streams
- * plaintext remote entries.
+ * plaintext remote entries. Writes reach `changes` subscribers once the
+ * outermost SQL transaction commits, including caller-owned transactions, and are
+ * dropped if it rolls back.
+ *
+ * **Gotchas**
+ *
+ * Concurrent caller-owned transactions can publish writes out of sequence order.
  *
  * @stability unstable
  * @category constructors
@@ -144,6 +150,40 @@ export const makeStorage = Effect.fnUntraced(function*(options?: StorageOptions)
           )`
   })
 
+  // Serves the batch lookup in `entriesAfter`. Postgres indexes hashes to stay
+  // under its B-tree row limit, MySQL indexes prefixes, and MSSQL has no
+  // bounded index yet.
+  const eventKeyIndex = `${entriesTable}_event_key_idx`
+  yield* sql.onDialectOrElse({
+    pg: () =>
+      sql`
+          CREATE INDEX IF NOT EXISTS ${sql(eventKeyIndex)}
+          ON ${entriesTableSql} (store_id, hashtextextended(event, 0), hashtextextended(primary_key, 0), entry_id)
+        `,
+    mysql: () => {
+      const indexExists = sql<{ readonly count: number }>`
+          SELECT COUNT(*) AS count FROM information_schema.statistics
+          WHERE table_schema = DATABASE()
+            AND table_name = ${entriesTable}
+            AND index_name = ${eventKeyIndex}
+        `.pipe(
+        Effect.map((rows) => Number(rows[0].count) > 0)
+      )
+      return sql`
+          CREATE INDEX ${sql(eventKeyIndex)}
+          ON ${entriesTableSql} (store_id, event(191), primary_key(191), entry_id)
+        `.pipe(
+        Effect.catch((error) => Effect.flatMap(indexExists, (exists) => exists ? Effect.void : Effect.fail(error)))
+      )
+    },
+    mssql: () => Effect.void,
+    orElse: () =>
+      sql`
+          CREATE INDEX IF NOT EXISTS ${sql(eventKeyIndex)}
+          ON ${entriesTableSql} (store_id, event, primary_key, entry_id)
+        `
+  })
+
   yield* sql.onDialectOrElse({
     pg: () =>
       sql`
@@ -237,49 +277,22 @@ export const makeStorage = Effect.fnUntraced(function*(options?: StorageOptions)
   })
 
   const transactionSemaphore = yield* Semaphore.make(1)
-  type PendingWrites = Array<readonly [storeId: string, entries: Array<RemoteEntry>]>
-  const pendingWrites = Context.Service<PendingWrites>(
-    `effect/eventlog/SqlEventLogServerUnencrypted/PendingWrites/${storageIdCounter++}`
-  )
-  const publish = Effect.fnUntraced(function*([storeId, entries]: PendingWrites[number]) {
-    const pubsub = yield* RcMap.get(pubsubs, storeId)
-    return yield* PubSub.publishAll(pubsub, entries)
-  }, Effect.scoped)
-  // Merge nested buffers on success; publish only after the outer commit.
-  // Hold the outer permit through publication to prevent later commits from overtaking it.
-  const withTransaction = Effect.fnUntraced(function*<A, E, R>(
-    effect: Effect.Effect<A, E, R>
-  ): Effect.fn.Return<A, E | SqlError.SqlError, Exclude<R, PendingWrites>> {
-    const parent = yield* Effect.serviceOption(pendingWrites)
-    // Allow body interruption, but mask commit through publication.
-    const transaction = Effect.uninterruptibleMask(Effect.fnUntraced(function*(restore) {
-      const writes: PendingWrites = []
-      const result = yield* sql.withTransaction(
-        Effect.provideService(restore(effect), pendingWrites, writes)
-      )
-      if (parent._tag === "Some") {
-        parent.value.push(...writes)
-      } else {
-        const stores = new Map<string, Array<RemoteEntry>>()
-        for (const [storeId, entries] of writes) {
-          const existing = stores.get(storeId)
-          if (existing === undefined) {
-            stores.set(storeId, [...entries])
-          } else {
-            existing.push(...entries)
-          }
-        }
-        // Sibling buffers may merge out of sequence order.
-        for (const entries of stores.values()) {
-          entries.sort((a, b) => a.remoteSequence - b.remoteSequence)
-        }
-        yield* Effect.forEach(stores, publish, { discard: true })
-      }
-      return result
-    }))
-    // Wait interruptibly; nested transactions reuse the outer permit.
-    return yield* parent._tag === "Some" ? transaction : transactionSemaphore.withPermit(transaction)
-  })
+  const publish = (storeId: string, entries: ReadonlyArray<RemoteEntry>) =>
+    RcMap.get(pubsubs, storeId).pipe(
+      Effect.flatMap((pubsub) => PubSub.publishAll(pubsub, entries)),
+      Effect.asVoid,
+      Effect.scoped
+    )
+  // Hold the permit through commit and publication to preserve commit order
+  // for storage-owned transactions.
+  const withTransaction = <A, E, R>(effect: Effect.Effect<A, E, R>): Effect.Effect<A, E | SqlError.SqlError, R> =>
+    Effect.flatMap(
+      Effect.serviceOption(sql.transactionService),
+      (transaction) =>
+        transaction._tag === "Some"
+          ? sql.withTransaction(effect)
+          : transactionSemaphore.withPermit(sql.withTransaction(effect))
+    )
 
   const ensureStore = (storeId: string) =>
     sql.onDialectOrElse({
@@ -362,6 +375,77 @@ export const makeStorage = Effect.fnUntraced(function*(options?: StorageOptions)
       Effect.flatMap(decodeRemoteEntries)
     )
 
+  // Postgres matches on the indexed hashes, then on the values to rule out
+  // collisions.
+  const matchesKey = sql.onDialectOrElse({
+    pg: () => (event: string, primaryKeys: ReadonlyArray<string>) =>
+      sql`
+          hashtextextended(event, 0) = hashtextextended(${event}, 0) AND event = ${event}
+          AND hashtextextended(primary_key, 0) IN (${
+        sql.csv(primaryKeys.map((primaryKey) => sql`hashtextextended(${primaryKey}, 0)`))
+      })
+          AND ${sql.in("primary_key", primaryKeys)}
+        `,
+    orElse: () => (event: string, primaryKeys: ReadonlyArray<string>) =>
+      sql`event = ${event} AND ${sql.in("primary_key", primaryKeys)}`
+  })
+
+  // One SELECT per index lookup, joined with UNION ALL: MySQL scans the whole
+  // store for the equivalent OR.
+  const selectChunkHistory = (storeId: string, oldest: Entry, entries: ReadonlyArray<Entry>) => {
+    const primaryKeysByEvent = new Map<string, Set<string>>()
+    for (const entry of entries) {
+      const primaryKeys = primaryKeysByEvent.get(entry.event)
+      if (primaryKeys === undefined) {
+        primaryKeysByEvent.set(entry.event, new Set([entry.primaryKey]))
+      } else {
+        primaryKeys.add(entry.primaryKey)
+      }
+    }
+    let statement = sql`
+        SELECT sequence, entry_id, event, primary_key, payload
+        FROM ${entriesTableSql}
+        WHERE store_id = ${storeId} AND ${sql.in("entry_id", entries.map((entry) => entry.id))}
+      `
+    for (const [event, primaryKeys] of primaryKeysByEvent) {
+      statement = sql`
+          ${statement}
+          UNION ALL
+          SELECT sequence, entry_id, event, primary_key, payload
+          FROM ${entriesTableSql}
+          WHERE store_id = ${storeId}
+            AND ${matchesKey(event, Array.from(primaryKeys))}
+            AND entry_id >= ${oldest.id}
+        `
+    }
+    return Effect.flatMap(statement, decodeRemoteEntries)
+  }
+
+  // Chunks and the UNION ALL branches can return the same row more than once.
+  const selectBatchHistory = (storeId: string, oldest: Entry, batch: Arr.NonEmptyReadonlyArray<Entry>) =>
+    Effect.forEach(
+      Arr.chunksOf(batch, lookupBatchSize),
+      (chunk) => selectChunkHistory(storeId, oldest, chunk)
+    ).pipe(
+      Effect.map((chunks) => {
+        const bySequence = new Map<number, RemoteEntry>()
+        for (const remoteEntry of chunks.flat()) {
+          bySequence.set(remoteEntry.remoteSequence, remoteEntry)
+        }
+        return Array.from(bySequence.values()).sort((a, b) => a.remoteSequence - b.remoteSequence)
+      })
+    )
+
+  const selectEntriesFrom = (storeId: string, entry: Entry) =>
+    sql`
+        SELECT sequence, entry_id, event, primary_key, payload
+        FROM ${entriesTableSql}
+        WHERE store_id = ${storeId} AND entry_id >= ${entry.id}
+        ORDER BY sequence ASC
+      `.pipe(
+      Effect.flatMap(decodeRemoteEntries)
+    )
+
   const getSessionAuthBinding = (publicKey: string) =>
     sql`
         SELECT public_key, signing_public_key
@@ -378,6 +462,7 @@ export const makeStorage = Effect.fnUntraced(function*(options?: StorageOptions)
 
   return EventLogServerUnencrypted.Storage.of({
     getId: Effect.succeed(remoteId),
+    firstSequence: 1,
     getOrCreateSessionAuthBinding: Effect.fnUntraced(
       function*(publicKey, signingPublicKey) {
         const existing = yield* getSessionAuthBinding(publicKey)
@@ -395,15 +480,8 @@ export const makeStorage = Effect.fnUntraced(function*(options?: StorageOptions)
       Effect.orDie,
       withTracerDisabled
     ),
-    entriesAfter: (storeId, entry) =>
-      sql`
-          SELECT sequence, entry_id, event, primary_key, payload
-          FROM ${entriesTableSql}
-          WHERE store_id = ${storeId} AND entry_id >= ${entry.id}
-          ORDER BY sequence ASC
-        `.pipe(
-        Effect.flatMap(decodeRemoteEntries),
-        Effect.map(Arr.map((r) => r.entry)),
+    entriesAfter: (storeId, entry, batch) =>
+      (batch === undefined ? selectEntriesFrom(storeId, entry) : selectBatchHistory(storeId, entry, batch)).pipe(
         Effect.orDie,
         withTracerDisabled
       ),
@@ -454,8 +532,7 @@ export const makeStorage = Effect.fnUntraced(function*(options?: StorageOptions)
         const nextSequence = currentNextSequence + entries.length
         yield* setNextSequence(storeId, nextSequence)
 
-        const writes = yield* pendingWrites
-        writes.push([storeId, committed])
+        yield* sql.afterCommit(publish(storeId, committed))
 
         return committed
       },
@@ -467,11 +544,13 @@ export const makeStorage = Effect.fnUntraced(function*(options?: StorageOptions)
       function*({ storeId, startSequence, compactors }) {
         const pubsub = yield* RcMap.get(pubsubs, storeId)
         const subscription = yield* PubSub.subscribe(pubsub)
-        const backlog = yield* EventLogServerUnencrypted.compactBacklog({
-          compactors,
-          remoteEntries: yield* selectEntriesAfter(storeId, startSequence)
-        })
-        let watermark = backlog.length > 0 ? backlog[backlog.length - 1]!.remoteSequence : startSequence - 1
+        const remoteEntries = yield* selectEntriesAfter(storeId, startSequence)
+        const backlog = yield* EventLogServerUnencrypted.compactBacklog({ compactors, remoteEntries })
+        // Compaction can drop trailing entries, so skip live entries up to the
+        // last uncompacted one.
+        const watermark = remoteEntries.length > 0
+          ? remoteEntries[remoteEntries.length - 1]!.remoteSequence
+          : startSequence - 1
 
         return Stream.fromArray(backlog).pipe(
           Stream.concat(

package/src/eventlog/index.ts

@@ -71,6 +71,12 @@ export * as EventLogServerUnencrypted from "./EventLogServerUnencrypted.ts"
  */
 export * as EventLogSessionAuth from "./EventLogSessionAuth.ts"
 
+/**
+ * @stability unstable
+ * @since 4.0.6
+ */
+export * as EventRow from "./EventRow.ts"
+
 /**
  * @stability unstable
  * @since 4.0.0

package/src/eventlog/internal/conflicts.ts

@@ -0,0 +1,123 @@
+import type * as Arr from "../../Array.ts"
+import * as Order from "../../Order.ts"
+import type { Entry } from "../EventJournal.ts"
+
+/** Orders entry ids by their raw UUID bytes. */
+export const entryIdOrder = Order.make<Uint8Array>((a, b) => {
+  for (let i = 0; i < 16; i++) {
+    if (a[i] !== b[i]) {
+      return a[i] < b[i] ? -1 : 1
+    }
+  }
+  return 0
+})
+
+/** Orders entries by id. */
+const entryOrder = Order.make<Entry>((a, b) => entryIdOrder(a.id, b.id))
+
+/**
+ * The source entry a compaction output stands for, and the output's position
+ * among the outputs written for that source.
+ */
+interface Anchor {
+  readonly entry: Entry
+  readonly index: number
+}
+
+// Set during client replay only, so it never leaves the process.
+const anchors = new WeakMap<Entry, Anchor>()
+
+/**
+ * Records that `entry` was written by a compactor for `source`, as its
+ * `index`th output for that source. Untagged entries are their own anchor.
+ */
+export const setAnchor = (entry: Entry, source: Entry, index: number): void => {
+  anchors.set(entry, { entry: source, index })
+}
+
+const anchorEntry = (entry: Entry): Entry => anchors.get(entry)?.entry ?? entry
+const anchorIndex = (entry: Entry): number => anchors.get(entry)?.index ?? 0
+
+// Sorts by anchor id, then by write order among one anchor's outputs.
+const compareAnchors = (a: Entry, b: Entry): number =>
+  entryOrder(anchorEntry(a), anchorEntry(b)) || anchorIndex(a) - anchorIndex(b)
+
+/**
+ * Pairs a batch of incoming entries with the stored entries that conflict with
+ * them: same event and primary key, and an id strictly greater than the
+ * incoming entry's anchor.
+ *
+ * The batch is sorted by anchor id, then by write order, so it is applied in
+ * the same order on every replica and a compaction output takes its source's
+ * place and precedence. Add the stored entries that may conflict, then call
+ * `toBatch` to sort and filter them.
+ */
+export class ConflictIndex {
+  /** The incoming entries in anchor order. */
+  readonly batch: ReadonlyArray<Entry>
+  /** The anchor of the first incoming entry. Only stored entries with a greater id can conflict. */
+  readonly oldest: Entry
+  /** Creation time of `oldest`, for coarse filters. */
+  readonly oldestMillis: number
+  private readonly stored = new Map<string, Map<string, ConflictGroup>>()
+
+  constructor(batch: Arr.NonEmptyReadonlyArray<Entry>) {
+    this.batch = batch.slice().sort(compareAnchors)
+    this.oldest = anchorEntry(this.batch[0])
+    this.oldestMillis = this.oldest.createdAtMillis
+    for (const entry of this.batch) {
+      let byKey = this.stored.get(entry.event)
+      if (byKey === undefined) {
+        byKey = new Map()
+        this.stored.set(entry.event, byKey)
+      }
+      if (!byKey.has(entry.primaryKey)) {
+        byKey.set(entry.primaryKey, {
+          event: entry.event,
+          primaryKey: entry.primaryKey,
+          oldest: anchorEntry(entry),
+          entries: []
+        })
+      }
+    }
+  }
+
+  get events(): Array<string> {
+    return Array.from(this.stored.keys())
+  }
+
+  /** The `(event, primaryKey)` groups in the batch. */
+  *groups(): Iterable<ConflictGroup> {
+    for (const byKey of this.stored.values()) {
+      yield* byKey.values()
+    }
+  }
+
+  add(entry: Entry): void {
+    this.stored.get(entry.event)?.get(entry.primaryKey)?.entries.push(entry)
+  }
+
+  toBatch(): Array<{ readonly entry: Entry; readonly conflicts: ReadonlyArray<Entry> }> {
+    for (const group of this.groups()) {
+      group.entries.sort(entryOrder)
+    }
+    return this.batch.map((entry) => {
+      const anchor = anchorEntry(entry)
+      return {
+        entry,
+        conflicts: this.stored.get(entry.event)!.get(entry.primaryKey)!.entries.filter((stored) =>
+          entryOrder(stored, anchor) > 0
+        )
+      }
+    })
+  }
+}
+
+/** An event and primary key in the batch, with the stored entries added for it. */
+export interface ConflictGroup {
+  readonly event: string
+  readonly primaryKey: string
+  /** The lowest anchor among the group's incoming entries. Only stored entries with a greater id can conflict. */
+  readonly oldest: Entry
+  readonly entries: Array<Entry>
+}

package/src/graphql/GraphQLClientError.ts

@@ -5,11 +5,12 @@
  * @stability experimental
  * @since 4.0.0
  */
-import * as Duration from "../Duration.ts"
-import type * as Headers from "../http/Headers.ts"
+import type * as Duration from "../Duration.ts"
+import * as Headers from "../http/Headers.ts"
 import type * as HttpClientError from "../http/HttpClientError.ts"
 import { HeadersSchema } from "../internal/graphqlHeaders.ts"
 import { hasProperty } from "../Predicate.ts"
+import * as Result from "../Result.ts"
 import * as Schema from "../Schema.ts"
 
 const TypeId = "~effect/graphql/GraphQLClientError"
@@ -226,16 +227,8 @@ const isRetryableStatus = (status: number): boolean => status === 429 || (status
  */
 const parseRetryAfter = (value: string | undefined, now: number): Duration.Duration | undefined => {
   if (value === undefined) return undefined
-  const trimmed = value.trim()
-  if (/^\d+$/.test(trimmed)) {
-    return Duration.seconds(Number(trimmed))
-  }
-  // Every HTTP date names a day or month; `Date.parse` alone also accepts
-  // bare numbers such as `1.5` or `-1`.
-  if (!/[a-z]/i.test(trimmed)) return undefined
-  const date = Date.parse(trimmed)
-  if (Number.isNaN(date)) return undefined
-  return Duration.millis(Math.max(0, date - now))
+  const retryAfter = Headers.parseRetryAfter(value)
+  return Result.isSuccess(retryAfter) ? Headers.retryAfterDelay(retryAfter.success, now) : undefined
 }
 
 /**

package/src/graphql/GraphQLProtocol.ts

@@ -547,9 +547,10 @@ export const makeHttp: {
  *
  * **Gotchas**
  *
- * `GraphQLResolver` resolves each lookup's URL once, before middleware run and
- * before the batched request exists, then sends each batch to the URL it was
- * grouped by. A single call resolves it in the transport, after middleware.
+ * `GraphQLResolver` resolves each lookup's URL once, in the calling fiber,
+ * before middleware run and before the batched request exists, then sends
+ * each batch to the URL it was grouped by. A single call resolves it in the
+ * transport, after middleware.
  * So the URL must depend only on the context and the operation, and
  * middleware must not change `kind` or `operationName` or provide services
  * the URL reads.
@@ -762,9 +763,10 @@ export const layerHttp: {
    *
    * **Gotchas**
    *
-   * `GraphQLResolver` resolves each lookup's URL once, before middleware run and
-   * before the batched request exists, then sends each batch to the URL it was
-   * grouped by. A single call resolves it in the transport, after middleware.
+   * `GraphQLResolver` resolves each lookup's URL once, in the calling fiber,
+   * before middleware run and before the batched request exists, then sends
+   * each batch to the URL it was grouped by. A single call resolves it in the
+   * transport, after middleware.
    * So the URL must depend only on the context and the operation, and
    * middleware must not change `kind` or `operationName` or provide services
    * the URL reads.
@@ -992,9 +994,10 @@ export const layerHttp: {
    *
    * **Gotchas**
    *
-   * `GraphQLResolver` resolves each lookup's URL once, before middleware run and
-   * before the batched request exists, then sends each batch to the URL it was
-   * grouped by. A single call resolves it in the transport, after middleware.
+   * `GraphQLResolver` resolves each lookup's URL once, in the calling fiber,
+   * before middleware run and before the batched request exists, then sends
+   * each batch to the URL it was grouped by. A single call resolves it in the
+   * transport, after middleware.
    * So the URL must depend only on the context and the operation, and
    * middleware must not change `kind` or `operationName` or provide services
    * the URL reads.