@sixb/core 0.1.9 → 0.1.10

package/src/broker/in-memory.ts

@@ -1,11 +1,14 @@
 import { getInvalidJsonValueReason, type JsonValue } from "../json"
 import { BrokerCursorExpiredError, BrokerError } from "./errors"
+import { createStreamRetentionResolver } from "./retention"
+import { waitForSubscriber } from "./subscriber"
 import type {
   Broker,
   BrokerPage,
   BrokerRecord,
   BrokerRecordInput,
   BrokerStreamDefinition,
+  BrokerStreamRetention,
 } from "./types"
 
 // Payloads are validated by assertBrokerPayload above the call site, so we can
@@ -34,18 +37,30 @@ interface Subscription {
   readonly streamId: string
   readonly names?: readonly string[]
   readonly keys?: readonly string[]
-  readonly handler: (records: readonly BrokerRecord[]) => void
+  readonly handler: (records: readonly BrokerRecord[]) => unknown
+  cursor: string
+  pumping: boolean
+  readonly controller: AbortController
+}
+
+export interface InMemoryBrokerOptions {
+  readonly streamRetention?: BrokerStreamRetention
 }
 
 export class InMemoryBroker implements Broker {
   readonly scope = "process" as const
   private readonly streams = new Map<string, StoredStream>()
   private readonly subscriptions = new Set<Subscription>()
+  private readonly resolveStream: ReturnType<typeof createStreamRetentionResolver>
+
+  constructor(options: InMemoryBrokerOptions = {}) {
+    this.resolveStream = createStreamRetentionResolver(options.streamRetention)
+  }
 
   async ensureStream(params: { projectId: string; stream: BrokerStreamDefinition }): Promise<void> {
     assertProjectId(params.projectId)
     assertStream(params.stream)
-    this.getOrCreateStream(params.projectId, params.stream)
+    this.getOrCreateStream(params.projectId, this.resolveStream(params.stream))
   }
 
   async append(params: {
@@ -90,7 +105,7 @@ export class InMemoryBroker implements Broker {
     storedStream.retainedBytes += stored.reduce((total, record) => total + record.byteSize, 0)
     this.applyRetention(storedStream)
     const records = stored.map(toBrokerRecord)
-    this.notify(params.projectId, params.streamId, stored)
+    this.notify(params.projectId, params.streamId)
     return records
   }
 
@@ -173,7 +188,7 @@ export class InMemoryBroker implements Broker {
       names?: readonly string[]
       keys?: readonly string[]
     },
-    handler: (records: readonly BrokerRecord[]) => void
+    handler: (records: readonly BrokerRecord[]) => unknown
   ): Promise<() => void> {
     assertProjectId(params.projectId)
     assertStreamId(params.streamId)
@@ -188,27 +203,31 @@ export class InMemoryBroker implements Broker {
     this.applyRetention(storedStream)
     this.assertCursorInRetainedRange(storedStream, params.afterCursor)
 
+    const startMode = params.afterCursor !== undefined ? undefined : (params.from ?? "latest")
     const subscription: Subscription = {
       projectId: params.projectId,
       streamId: params.streamId,
       names: params.names,
       keys: params.keys,
       handler,
+      cursor:
+        params.afterCursor ??
+        (startMode === "earliest"
+          ? BigInt(storedStream.records[0]?.cursor ?? storedStream.nextSequence) - 1n
+          : storedStream.nextSequence - 1n
+        ).toString(),
+      pumping: false,
+      controller: new AbortController(),
     }
     this.subscriptions.add(subscription)
 
-    const startMode = params.afterCursor !== undefined ? undefined : (params.from ?? "latest")
     if (params.afterCursor !== undefined || startMode === "earliest") {
-      const initial = this.readForward(storedStream.records, {
-        afterCursor: params.afterCursor,
-        names: params.names,
-        keys: params.keys,
-      }).records
-      this.deliver(subscription, initial)
+      this.deliver(subscription)
     }
 
     return () => {
       this.subscriptions.delete(subscription)
+      subscription.controller.abort()
     }
   }
 
@@ -335,30 +354,50 @@ export class InMemoryBroker implements Broker {
     }
   }
 
-  private notify(projectId: string, streamId: string, records: readonly StoredRecord[]): void {
+  private notify(projectId: string, streamId: string): void {
     for (const subscription of this.subscriptions) {
       if (subscription.projectId !== projectId || subscription.streamId !== streamId) {
         continue
       }
-      this.deliver(
-        subscription,
-        this.readForward(records, {
-          names: subscription.names,
-          keys: subscription.keys,
-        }).records
-      )
+      this.deliver(subscription)
     }
   }
 
-  private deliver(subscription: Subscription, records: readonly BrokerRecord[]): void {
-    if (records.length === 0) {
-      return
-    }
+  private deliver(subscription: Subscription): void {
+    if (subscription.pumping) return
+    subscription.pumping = true
+    // Only the retained stream holds the backlog, never a second queue of cloned batches.
+    void this.pump(subscription).catch((error) => {
+      this.subscriptions.delete(subscription)
+      subscription.controller.abort()
+      console.error("[InMemoryBroker] Subscription stopped:", error)
+    })
+  }
 
-    try {
-      subscription.handler(records)
-    } catch {
-      // Broker subscriptions are live fan-out; handler failures should not break publishers.
+  private async pump(subscription: Subscription): Promise<void> {
+    while (this.subscriptions.has(subscription)) {
+      const stream = this.getEnsuredStream(subscription.projectId, subscription.streamId)
+      this.applyRetention(stream)
+      this.assertCursorInRetainedRange(stream, subscription.cursor)
+      const page = this.readForward(stream.records, {
+        afterCursor: subscription.cursor,
+        names: subscription.names,
+        keys: subscription.keys,
+        limit: 100,
+      })
+      if (page.records.length === 0) {
+        subscription.cursor = page.cursor ?? subscription.cursor
+        // Clear synchronously: an append must not miss the wake-up between return and finally.
+        subscription.pumping = false
+        return
+      }
+      try {
+        await waitForSubscriber(subscription.handler(page.records), subscription.controller.signal)
+      } catch {
+        // Preserve observer error isolation, including rejected async callbacks.
+      }
+      if (!this.subscriptions.has(subscription)) return
+      subscription.cursor = page.cursor ?? subscription.cursor
     }
   }
 
@@ -378,11 +417,9 @@ export class InMemoryBroker implements Broker {
     let cursor = filters.afterCursor
     let stoppedAt = records.length
 
-    for (let index = 0; index < records.length; index += 1) {
+    const start = afterCursor === undefined ? 0 : cursorBoundary(records, afterCursor, true)
+    for (let index = start; index < records.length; index += 1) {
       const record = records[index]!
-      if (afterCursor !== undefined && BigInt(record.cursor) <= afterCursor) {
-        continue
-      }
       cursor = record.cursor
       if (!matchesFilters(record, names, keys)) {
         continue
@@ -397,7 +434,7 @@ export class InMemoryBroker implements Broker {
     return {
       records: result,
       cursor,
-      hasMore: records.slice(stoppedAt).some((record) => matchesFilters(record, names, keys)),
+      hasMore: hasMatchingRecord(records, stoppedAt, records.length, names, keys),
     }
   }
 
@@ -417,11 +454,12 @@ export class InMemoryBroker implements Broker {
     let cursor = filters.beforeCursor
     let stoppedAt = -1
 
-    for (let index = records.length - 1; index >= 0; index -= 1) {
+    const start =
+      beforeCursor === undefined
+        ? records.length - 1
+        : cursorBoundary(records, beforeCursor, false) - 1
+    for (let index = start; index >= 0; index -= 1) {
       const record = records[index]!
-      if (beforeCursor !== undefined && BigInt(record.cursor) >= beforeCursor) {
-        continue
-      }
       cursor = record.cursor
       if (!matchesFilters(record, names, keys)) {
         continue
@@ -436,9 +474,7 @@ export class InMemoryBroker implements Broker {
     return {
       records: reversed.reverse(),
       cursor,
-      hasMore:
-        stoppedAt >= 0 &&
-        records.slice(0, stoppedAt + 1).some((record) => matchesFilters(record, names, keys)),
+      hasMore: stoppedAt >= 0 && hasMatchingRecord(records, 0, stoppedAt + 1, names, keys),
     }
   }
 }
@@ -516,3 +552,32 @@ function assertCursor(cursor: string | undefined): void {
     })
   }
 }
+
+function cursorBoundary(records: readonly StoredRecord[], cursor: bigint, after: boolean): number {
+  // Retained cursors remain sorted even after retention removes older records.
+  let low = 0
+  let high = records.length
+  while (low < high) {
+    const middle = Math.floor((low + high) / 2)
+    const value = BigInt(records[middle]!.cursor)
+    if (value < cursor || (after && value === cursor)) {
+      low = middle + 1
+    } else {
+      high = middle
+    }
+  }
+  return low
+}
+
+function hasMatchingRecord(
+  records: readonly StoredRecord[],
+  start: number,
+  end: number,
+  names?: ReadonlySet<string>,
+  keys?: ReadonlySet<string>
+): boolean {
+  for (let i = start; i < end; i++) {
+    if (matchesFilters(records[i]!, names, keys)) return true
+  }
+  return false
+}

package/src/broker/index.ts

@@ -1,6 +1,8 @@
 export type { ProviderScope } from "../provider-scope"
 export { BrokerCursorExpiredError, BrokerError } from "./errors"
-export { InMemoryBroker } from "./in-memory"
+export { InMemoryBroker, type InMemoryBrokerOptions } from "./in-memory"
+export { createStreamRetentionResolver } from "./retention"
+export { waitForSubscriber } from "./subscriber"
 export type {
   Broker,
   BrokerCursor,
@@ -9,4 +11,5 @@ export type {
   BrokerRecordInput,
   BrokerRetention,
   BrokerStreamDefinition,
+  BrokerStreamRetention,
 } from "./types"

package/src/broker/retention.ts

@@ -0,0 +1,28 @@
+import { BrokerError } from "./errors"
+import type { BrokerRetention, BrokerStreamDefinition, BrokerStreamRetention } from "./types"
+
+/** Provider utility: snapshot and validate configuration before opening any connections. */
+export function createStreamRetentionResolver(
+  configuration: BrokerStreamRetention = {}
+): (stream: BrokerStreamDefinition) => BrokerStreamDefinition {
+  const overrides = new Map<string, BrokerRetention>()
+  for (const [id, retention] of Object.entries(configuration)) {
+    if (!id.trim()) throw new BrokerError("Retention stream id must not be empty")
+    const limits: Partial<Record<keyof BrokerRetention, number>> = {}
+    for (const field of ["maxAgeMs", "maxRecords", "maxBytes"] as const) {
+      const value = retention[field]
+      if (value === undefined) continue
+      if (!Number.isSafeInteger(value) || value <= 0) {
+        throw new BrokerError(
+          `Retention ${field} for stream '${id}' must be a positive safe integer`
+        )
+      }
+      limits[field] = value
+    }
+    overrides.set(id, limits)
+  }
+  return (stream) => {
+    const retention = overrides.get(stream.id)
+    return retention ? { ...stream, retention: { ...stream.retention, ...retention } } : stream
+  }
+}

package/src/broker/subscriber.ts

@@ -0,0 +1,21 @@
+/**
+ * Await one observer delivery, or detach it on unsubscribe. The observer's owner
+ * remains responsible for cancellation/draining. Rejections are observed even
+ * after detaching; each completed delivery removes its abort listener.
+ *
+ * Do not race every batch against one permanent stop promise: that retains a
+ * promise reaction per batch for the subscription's entire lifetime.
+ */
+export async function waitForSubscriber(result: unknown, signal: AbortSignal): Promise<void> {
+  let onAbort: (() => void) | undefined
+  try {
+    await new Promise<void>((resolve, reject) => {
+      onAbort = resolve
+      signal.addEventListener("abort", onAbort, { once: true })
+      Promise.resolve(result).then(() => resolve(), reject)
+      if (signal.aborted) resolve()
+    })
+  } finally {
+    if (onAbort) signal.removeEventListener("abort", onAbort)
+  }
+}

package/src/broker/types.ts

@@ -21,6 +21,9 @@ export interface BrokerStreamDefinition {
   readonly retention?: BrokerRetention
 }
 
+/** Per-stream overrides at creation; omitted limits keep the stream's own defaults. */
+export type BrokerStreamRetention = Readonly<Record<string, BrokerRetention>>
+
 export interface BrokerRecordInput {
   readonly name?: string
   readonly key?: string
@@ -102,6 +105,17 @@ export interface Broker {
    * range rather than silently advancing to the first retained record, so a
    * resuming subscriber learns the gap exists instead of mistaking a truncated
    * stream for a complete one.
+   *
+   * A returned handler promise is awaited before the next delivery. Providers
+   * bound read-ahead instead of accumulating pending handler calls. Return the
+   * processing promise to apply backpressure; detached work cannot be bounded.
+   * Handler errors remain isolated from publishers and other subscribers; they
+   * are not a negative acknowledgement. Reliable consumers must retry within
+   * their handler before returning. No progress survives a process restart.
+   *
+   * Unsubscribe stops further delivery, not an already running callback. Its
+   * owner must cancel/drain that work; provider close must not wait indefinitely
+   * for an uncooperative handler. Return values other than promises are ignored.
    */
   subscribe(
     params: {
@@ -112,7 +126,7 @@ export interface Broker {
       names?: readonly string[]
       keys?: readonly string[]
     },
-    handler: (records: readonly BrokerRecord[]) => void
+    handler: (records: readonly BrokerRecord[]) => unknown
   ): Promise<() => void>
 
   /**

package/src/connectors/builders.ts

@@ -52,12 +52,6 @@ export function defineConnector<TId extends string, TAdapter extends AnyConnecto
         "OAuth connector adapters must define oauth2 authentication and implement authorizationUrl, exchangeCode, refresh, and discoverAccounts."
       )
     }
-    if (adapter.webhooks !== undefined) {
-      throw createConnectorCodedError(
-        "connector.configuration_invalid",
-        "OAuth connector adapters cannot register webhooks until connection routing is defined."
-      )
-    }
   }
 
   return {

package/src/connectors/connections/authorizations.ts

@@ -160,7 +160,8 @@ export class DefaultConnectorAuthorizationLifecycle implements ConnectorAuthoriz
       input.definition,
       input.code,
       input.codeVerifier,
-      input.redirectUri
+      input.redirectUri,
+      input.callbackParameters
     )
     const credentialsEnvelope = await this.credentials.seal(
       input.definition.id,
@@ -482,7 +483,11 @@ export class DefaultConnectorAuthorizationLifecycle implements ConnectorAuthoriz
           const credentials = validateCredentials(
             await input.definition.adapter.authentication.exchangeCode(
               this.authorizationAdapterContext(input.definition.id, input.redirectUri, signal),
-              { code: input.code, codeVerifier: input.codeVerifier }
+              {
+                code: input.code,
+                ...(input.codeVerifier === undefined ? {} : { codeVerifier: input.codeVerifier }),
+                callbackParameters: input.callbackParameters,
+              }
             )
           )
           assertCredentialMutationOperationActive(signal)
@@ -544,15 +549,16 @@ export class DefaultConnectorAuthorizationLifecycle implements ConnectorAuthoriz
   private async exchangeAuthorizationCode(
     definition: OAuthConnectorDefinition,
     code: string,
-    codeVerifier: string,
-    redirectUri: string
+    codeVerifier: string | undefined,
+    redirectUri: string,
+    callbackParameters: Readonly<Record<string, string>> | undefined
   ): Promise<ConnectorOAuthCredentials> {
     try {
       return validateCredentials(
         await this.mutations.withBoundedProviderSignal((signal) =>
           definition.adapter.authentication.exchangeCode(
             this.authorizationAdapterContext(definition.id, redirectUri, signal),
-            { code, codeVerifier }
+            { code, ...(codeVerifier === undefined ? {} : { codeVerifier }), callbackParameters }
           )
         )
       )

package/src/connectors/connections/contracts.ts

@@ -46,6 +46,8 @@ export interface CompleteConnectorAuthorizationInput {
   readonly state: string
   readonly code: string
   readonly redirectUri: string
+  /** Raw provider callback values; duplicate values are rejected for allowlisted parameters. */
+  readonly callbackParameters?: Readonly<Record<string, string | readonly string[]>>
 }
 
 export type CompleteConnectorConnectionRunInput =
@@ -54,6 +56,7 @@ export type CompleteConnectorConnectionRunInput =
       readonly code: string
       readonly redirectUri: string
       readonly callbackBinding: string
+      readonly callbackParameters?: Readonly<Record<string, string | readonly string[]>>
     }
   | {
       readonly state: string

package/src/connectors/connections/request.ts

@@ -34,11 +34,14 @@ import type {
 import { runBoundedConnectorProviderOperation } from "./provider-operation"
 import {
   assertAuthorizationUrlParameters,
+  connectorPkceEnabled,
   hashSecret,
   nonblank,
   normalizedHttpUrl,
   parseAttemptId,
   positiveDuration,
+  selectCallbackParameters,
+  validateCallbackParameterNames,
 } from "./validation"
 import { assertConnectorConnectionSelector, connectorConnectionView } from "./views"
 
@@ -64,8 +67,9 @@ export interface CompleteNewConnectorAuthorizationInput {
   readonly definition: OAuthConnectorDefinition
   readonly principal: AuthorizablePrincipal
   readonly code: string
-  readonly codeVerifier: string
+  readonly codeVerifier?: string
   readonly redirectUri: string
+  readonly callbackParameters?: Readonly<Record<string, string>>
   /** Called immediately after the encrypted grant is durable, before account discovery. */
   readonly onAuthorizationPersisted?: (authorization: ConnectorAuthorizationRecord) => Promise<void>
 }
@@ -172,6 +176,7 @@ export class ConnectorAuthorizationRequestHandler {
     const definition = this.resolveDefinition(nonblank(connectorId, "connector id"))
     const actor = requireConnectorConnectionCommandActor(command.execution, this.projectId)
     assertConnectorConnectionSelector(input)
+    validateCallbackParameterNames(definition.adapter.authentication.callbackParameters)
     const redirectUri = normalizedHttpUrl(input.redirectUri, "OAuth callback URL")
     const reauthorization = await this.prepareReauthorization(
       definition,
@@ -181,8 +186,13 @@ export class ConnectorAuthorizationRequestHandler {
 
     const attemptId = `cat_${randomUUID()}`
     const state = `${attemptId}.${randomBytes(32).toString("base64url")}`
-    const codeVerifier = randomBytes(32).toString("base64url")
-    const codeChallenge = createHash("sha256").update(codeVerifier).digest("base64url")
+    const codeVerifier = connectorPkceEnabled(definition.adapter.authentication.pkce)
+      ? randomBytes(32).toString("base64url")
+      : undefined
+    const codeChallenge =
+      codeVerifier === undefined
+        ? undefined
+        : createHash("sha256").update(codeVerifier).digest("base64url")
     let authorizationUrlInput: string | URL
     try {
       authorizationUrlInput = await runBoundedConnectorProviderOperation(
@@ -198,7 +208,12 @@ export class ConnectorAuthorizationRequestHandler {
               redirectUri,
               signal,
             },
-            { state, codeChallenge, codeChallengeMethod: "S256" }
+            {
+              state,
+              ...(codeChallenge === undefined
+                ? {}
+                : { codeChallenge, codeChallengeMethod: "S256" }),
+            }
           ),
         () =>
           createConnectorCodedError(
@@ -220,15 +235,15 @@ export class ConnectorAuthorizationRequestHandler {
       "connector.adapter_invalid"
     )
     assertAuthorizationUrlParameters(authorizationUrl, { state, codeChallenge })
-    const codeVerifierEnvelope = await this.credentialProtector.seal(
-      textEncoder.encode(codeVerifier),
-      {
-        projectId: this.projectId,
-        connectorId: definition.id,
-        recordId: attemptId,
-        purpose: "pkce-verifier",
-      }
-    )
+    const codeVerifierEnvelope =
+      codeVerifier === undefined
+        ? undefined
+        : await this.credentialProtector.seal(textEncoder.encode(codeVerifier), {
+            projectId: this.projectId,
+            connectorId: definition.id,
+            recordId: attemptId,
+            purpose: "pkce-verifier",
+          })
 
     return {
       definition,
@@ -243,7 +258,7 @@ export class ConnectorAuthorizationRequestHandler {
         slot: input.slot,
         initiatedByExecutionId: command.execution.id,
         stateHash: hashSecret(state),
-        codeVerifier: codeVerifierEnvelope,
+        ...(codeVerifierEnvelope === undefined ? {} : { codeVerifier: codeVerifierEnvelope }),
         redirectUri,
         ...(reauthorization === undefined
           ? {}
@@ -275,6 +290,7 @@ export class ConnectorAuthorizationRequestHandler {
       principal: actor.principal,
       code,
       redirectUri,
+      callbackParameters: input.callbackParameters,
     })
 
     return {
@@ -291,19 +307,34 @@ export class ConnectorAuthorizationRequestHandler {
     readonly principal: AuthorizablePrincipal
     readonly code: string
     readonly redirectUri: string
+    readonly callbackParameters?: Readonly<Record<string, string | readonly string[]>>
     readonly onAuthorizationPersisted?: (
       authorization: ConnectorAuthorizationRecord
     ) => Promise<void>
   }): Promise<ConnectorAuthorizationRecord> {
-    const verifierBytes = await this.credentialProtector.open(input.attempt.codeVerifier, {
-      projectId: this.projectId,
-      connectorId: input.definition.id,
-      recordId: input.attempt.id,
-      purpose: "pkce-verifier",
-    })
+    const enabled = connectorPkceEnabled(input.definition.adapter.authentication.pkce)
+    if (enabled !== (input.attempt.codeVerifier !== undefined)) {
+      throw createConnectorCodedError(
+        "connector.authorization_invalid",
+        "OAuth connector PKCE configuration changed; restart authorization."
+      )
+    }
+    const verifierBytes =
+      input.attempt.codeVerifier === undefined
+        ? undefined
+        : await this.credentialProtector.open(input.attempt.codeVerifier, {
+            projectId: this.projectId,
+            connectorId: input.definition.id,
+            recordId: input.attempt.id,
+            purpose: "pkce-verifier",
+          })
     return this.completeGrant({
       ...input,
-      codeVerifier: textDecoder.decode(verifierBytes),
+      callbackParameters: selectCallbackParameters(
+        input.definition.adapter.authentication.callbackParameters,
+        input.callbackParameters
+      ),
+      ...(verifierBytes === undefined ? {} : { codeVerifier: textDecoder.decode(verifierBytes) }),
     })
   }
 
@@ -399,8 +430,9 @@ export class ConnectorAuthorizationRequestHandler {
     readonly attempt: ConnectorAuthorizationAttemptRecord
     readonly principal: AuthorizablePrincipal
     readonly code: string
-    readonly codeVerifier: string
+    readonly codeVerifier?: string
     readonly redirectUri: string
+    readonly callbackParameters: Readonly<Record<string, string>>
     readonly onAuthorizationPersisted?: (
       authorization: ConnectorAuthorizationRecord
     ) => Promise<void>
@@ -411,6 +443,7 @@ export class ConnectorAuthorizationRequestHandler {
         principal: input.principal,
         code: input.code,
         codeVerifier: input.codeVerifier,
+        callbackParameters: input.callbackParameters,
         redirectUri: input.redirectUri,
         ...(input.onAuthorizationPersisted === undefined
           ? {}
@@ -433,6 +466,7 @@ export class ConnectorAuthorizationRequestHandler {
       principal: input.principal,
       code: input.code,
       codeVerifier: input.codeVerifier,
+      callbackParameters: input.callbackParameters,
       redirectUri: input.redirectUri,
     })
   }

package/src/connectors/connections/runs.ts

@@ -207,6 +207,7 @@ export class ConnectorConnectionRunService {
         attempt,
         principal,
         code: nonblank(input.code, "OAuth authorization code", "connector.authorization_invalid"),
+        callbackParameters: input.callbackParameters,
         redirectUri,
         onAuthorizationPersisted: async (persisted) => {
           authorizationId = persisted.id

package/src/connectors/connections/tokens.ts

@@ -169,6 +169,10 @@ export class ConnectorTokenAccess {
                 ? { tokenType: current.tokenType }
                 : {}),
               scopes: refreshed.scopes ?? current.scopes ?? authorization.scopes,
+              ...(refreshed.authorizationContext === undefined &&
+              current.authorizationContext !== undefined
+                ? { authorizationContext: current.authorizationContext }
+                : {}),
             } satisfies ConnectorOAuthCredentials
             const envelope = await this.credentials.seal(
               definition.id,