@trigger.dev/redis-worker 4.7.3 → 4.7.5

package/dist/index.d.cts

@@ -1,4 +1,4 @@
-import { Callback, Redis, RedisOptions, Result } from "@internal/redis";
+import { Callback, RedisClient, RedisClusterClientOptions, RedisOptions, Result } from "@internal/redis";
 import { Logger } from "@trigger.dev/core/logger";
 import { AnyZodSchema, inferZodSchemaOutput } from "@trigger.dev/core/v3";
 import { Attributes, Context, Counter, Histogram, Meter, ObservableGauge, Span, SpanKind, Tracer } from "@internal/tracing";
@@ -51,6 +51,22 @@ declare class SimpleQueue<TMessageCatalog extends MessageCatalogSchema> {
     visibilityTimeoutMs: number;
     cancellationKey?: string;
   }): Promise<void>;
+  /**
+   * Re-enqueues an item for a retry, unless since it was dequeued a newer enqueue replaced it (its
+   * stored deduplication key no longer matches) or it was removed by an ack or cancel. Returns
+   * false when it skipped, so a failed attempt never overwrites the newer item or revives a removed
+   * one. A newer enqueue starts its own retry budget: an item a handler re-enqueues before failing
+   * restarts at attempt 0 rather than moving towards the DLQ.
+   */
+  requeue({ id, job, item, attempt, availableAt, visibilityTimeoutMs, deduplicationKey }: {
+    id: string;
+    job: MessageCatalogKey<TMessageCatalog>;
+    item: MessageCatalogValue<TMessageCatalog, MessageCatalogKey<TMessageCatalog>>;
+    attempt: number;
+    availableAt: Date;
+    visibilityTimeoutMs: number;
+    deduplicationKey?: string;
+  }): Promise<boolean>;
   enqueueOnce({ id, job, item, attempt, availableAt, visibilityTimeoutMs }: {
     id: string;
     job: MessageCatalogKey<TMessageCatalog>;
@@ -82,7 +98,11 @@ declare class SimpleQueue<TMessageCatalog extends MessageCatalogSchema> {
    */
   oldestMessageAge(): Promise<number>;
   getJob(id: string): Promise<QueueItem<TMessageCatalog> | null>;
-  moveToDeadLetterQueue(id: string, errorMessage: string): Promise<void>;
+  /**
+   * Moves an item to the dead letter queue. With a deduplication key, an item a newer enqueue has
+   * replaced is left in the queue and false is returned.
+   */
+  moveToDeadLetterQueue(id: string, errorMessage: string, deduplicationKey?: string): Promise<boolean>;
   sizeOfDeadLetterQueue(): Promise<number>;
   redriveFromDeadLetterQueue(id: string): Promise<void>;
   close(): Promise<void>;
@@ -94,7 +114,8 @@ declare module "@internal/redis" {
     dequeueItems(queue: string, items: string, now: number, count: number, callback?: Callback<Array<[string, string, string]>>): Result<Array<[string, string, string]>, Context>;
     ackItem(queue: string, items: string, id: string, deduplicationKey: string, callback?: Callback<number>): Result<number, Context>;
     redriveFromDeadLetterQueue(queue: string, items: string, dlq: string, dlqItems: string, id: string, callback?: Callback<number>): Result<number, Context>;
-    moveToDeadLetterQueue(queue: string, items: string, dlq: string, dlqItems: string, id: string, errorMessage: string, callback?: Callback<number>): Result<number, Context>;
+    moveToDeadLetterQueue(queue: string, items: string, dlq: string, dlqItems: string, id: string, errorMessage: string, deduplicationKey: string, callback?: Callback<number>): Result<number, Context>;
+    requeueItem(queue: string, items: string, id: string, score: number, serializedItem: string, deduplicationKey: string, callback?: Callback<number>): Result<number, Context>;
     enqueueItemOnce(queue: string, items: string, id: string, score: number, serializedItem: string, callback?: Callback<number>): Result<number, Context>;
     getJob(queue: string, items: string, id: string, callback?: Callback<[string, string, string] | null>): Result<[string, string, string] | null, Context>;
     getOldestDueScore(queue: string, items: string, now: number, callback?: Callback<string | number>): Result<string | number, Context>;
@@ -262,9 +283,6 @@ declare class Worker<TCatalog extends WorkerCatalog> {
    * Processes a batch of items for a batch-enabled job type.
    */
   private processBatch;
-  /**
-   * Processes a single item.
-   */
   private processItem;
   private withHistogram;
   private static delay;
@@ -290,6 +308,26 @@ declare class Worker<TCatalog extends WorkerCatalog> {
  */
 declare function isAbortError(error: unknown): boolean;
 //#endregion
+//#region src/fair-queue/connection.d.ts
+/**
+ * Where a fair queue keeps its state: one Redis node, or a cluster. A cluster only works with a key
+ * producer that keeps every key a script touches in one slot, such as
+ * `TenantShardedFairQueueKeyProducer`. The cluster's key prefix goes in `clusterOptions.keyPrefix`.
+ */
+type FairQueueRedisOptions = RedisOptions | {
+  cluster: RedisClusterClientOptions;
+};
+declare function isClusterConnection(options: FairQueueRedisOptions): options is {
+  cluster: RedisClusterClientOptions;
+};
+declare function createFairQueueRedisClient(options: FairQueueRedisOptions): RedisClient;
+declare function keyPrefixOf(options: FairQueueRedisOptions): string;
+/**
+ * Every key matching `pattern` (which must include the key prefix), page by page. A cluster doesn't
+ * fan SCAN out to its nodes, so this scans each primary in turn.
+ */
+declare function scanKeys(client: RedisClient, pattern: string, count?: number): AsyncGenerator<string[]>;
+//#endregion
 //#region src/fair-queue/retry.d.ts
 /**
  * RetryStrategy interface for pluggable retry logic.
@@ -642,6 +680,21 @@ interface FairQueueKeyProducer {
   extractTenantId(queueId: string): string;
   /** Extract a specific group ID from a queue ID */
   extractGroupId(groupName: string, queueId: string): string;
+  /**
+   * The shard a tenant's dispatch entry, queues and in-flight messages live on. When set, every
+   * component shards by tenant instead of by queue, so one shard holds all of a tenant's keys.
+   */
+  tenantShard?(tenantId: string): number;
+  /** The number of shards `tenantShard` spreads tenants over. Required with `tenantShard`. */
+  readonly shardCount?: number;
+  /** The key holding one message's payload, for `separatePayloads`. */
+  payloadKey?(queueId: string, messageId: string): string;
+  /** The counter of a tenant's queued and in-flight payload bytes, for `separatePayloads`. */
+  tenantPayloadBytesKey?(tenantId: string): string;
+  /** The SCAN pattern matching one shard's concurrency sets for a group. */
+  concurrencyKeyPattern?(groupName: string, shardId: number): string;
+  /** The DRR scheduler's deficit hash. */
+  drrDeficitKey?(): string;
 }
 /**
  * Worker queue configuration options.
@@ -684,11 +737,25 @@ interface CooloffOptions {
  */
 interface FairQueueOptions<TPayloadSchema extends AnyZodSchema = z.ZodUnknown> {
   /** Redis connection options */
-  redis: RedisOptions;
+  redis: FairQueueRedisOptions;
   /** Key producer for Redis keys */
   keys: FairQueueKeyProducer;
   /** Scheduler for fair queue selection */
   scheduler: FairScheduler;
+  /**
+   * Keep each payload in its own key (`keys.payloadKey`), written once at enqueue and deleted at
+   * completion. Queue items and in-flight entries then hold only the message envelope, so claiming,
+   * releasing and retrying a message never rewrite its payload. `resolveWorkerQueue` receives the
+   * envelope without its payload. Enqueueing an id that is already queued or in flight is a no-op
+   * rather than a replace. Each tenant's payload bytes are counted while queued or in flight.
+   * Requires a key producer with `payloadKey` and `tenantPayloadBytesKey`.
+   */
+  separatePayloads?: boolean;
+  /**
+   * The most payload bytes a tenant may have queued or in flight, with `separatePayloads`. An
+   * enqueue past it throws `TenantQueueFullError`. Absent, or a result of 0, means no limit.
+   */
+  tenantPayloadBytesLimit?: (tenantId: string) => Promise<number> | number;
   /** Zod schema for message payload validation */
   payloadSchema?: TPayloadSchema;
   /** Whether to validate payloads on enqueue (default: false) */
@@ -821,6 +888,11 @@ interface EnqueueOptions<TPayload = unknown> {
   timestamp?: number;
   /** Optional metadata for concurrency group extraction */
   metadata?: Record<string, string>;
+  /**
+   * Enqueue past `tenantPayloadBytesLimit`. For a system's own follow-up work, which must not be
+   * refused because its tenant's incoming work filled the budget. Its bytes are still counted.
+   */
+  ignorePayloadBytesLimit?: boolean;
 }
 /**
  * Options for enqueueing multiple messages.
@@ -850,7 +922,7 @@ interface DRRSchedulerConfig {
   /** Maximum queues to fetch from master queue (default: 1000) */
   masterQueueLimit?: number;
   /** Redis options for state storage */
-  redis: RedisOptions;
+  redis: FairQueueRedisOptions;
   /** Key producer */
   keys: FairQueueKeyProducer;
   /** Optional logger */
@@ -886,7 +958,7 @@ interface WeightedSchedulerBiases {
  */
 interface WeightedSchedulerConfig {
   /** Redis options */
-  redis: RedisOptions;
+  redis: FairQueueRedisOptions;
   /** Key producer */
   keys: FairQueueKeyProducer;
   /** Default tenant concurrency limit */
@@ -907,7 +979,7 @@ interface WeightedSchedulerConfig {
 //#endregion
 //#region src/fair-queue/concurrency.d.ts
 interface ConcurrencyManagerOptions {
-  redis: RedisOptions;
+  redis: FairQueueRedisOptions;
   keys: FairQueueKeyProducer;
   groups: ConcurrencyGroupConfig[];
   logger?: {
@@ -965,7 +1037,10 @@ declare class ConcurrencyManager {
    * record can only be a leak; if the message is about to be re-claimed, reserve simply
    * re-adds the member.
    *
-   * @param inflightDataKeys - The in-flight data hash keys for every shard. The sweep
+   * With a tenant-sharded key producer, each shard's sets are checked against that shard's
+   * in-flight data only, so the check stays in one cluster slot.
+   *
+   * @param inflightDataKeys - The in-flight data hash keys for every shard, in shard order. The sweep
    *   refuses to run when this is empty, since with nowhere to look for running messages
    *   every member would look orphaned and all concurrency accounting would be erased.
    * @returns The message ids that were removed, and how many sets were checked
@@ -1089,10 +1164,62 @@ declare class CallbackFairQueueKeyProducer extends DefaultFairQueueKeyProducer {
   extractTenantId(queueId: string): string;
   extractGroupId(groupName: string, queueId: string): string;
 }
+/**
+ * Key producer for running a fair queue on a Redis cluster. Each tenant belongs to one shard,
+ * `jumpHash(tenantId, shardCount)`, and every key of a shard carries the hash tag
+ * `{prefix:shard}`, so a shard's queues, items, payloads, tenant indexes, dispatch index,
+ * in-flight set and concurrency sets share a slot and every script stays single-slot.
+ *
+ * Concurrency groups must be keyed by tenant id. The shard count can't change without moving keys.
+ *
+ * Key structure (S = the tenant's shard):
+ * - Dispatch index: {prefix:S}:dispatch
+ * - Tenant queue index: {prefix:S}:tenantq:{tenantId}
+ * - Queue: {prefix:S}:queue:{queueId}, items {prefix:S}:queue:{queueId}:items
+ * - Payload: {prefix:S}:payload:{queueId}:{messageId}
+ * - Payload bytes per tenant: {prefix:S}:payloadbytes:{tenantId}
+ * - Concurrency: {prefix:S}:concurrency:{groupName}:{tenantId}
+ * - In-flight: {prefix:S}:inflight, data {prefix:S}:inflight:data
+ * - Dead letter queue: {prefix:S}:dlq:{tenantId}, data {prefix:S}:dlq:{tenantId}:data
+ * - Legacy master queue: {prefix:S}:master
+ * - Worker queue: {prefix}:worker:{consumerId} (its own slot)
+ * - DRR deficits: {prefix}:drr:deficit (its own slot)
+ */
+declare class TenantShardedFairQueueKeyProducer implements FairQueueKeyProducer {
+  #private;
+  readonly shardCount: number;
+  private readonly prefix;
+  private readonly tenantExtractor;
+  private readonly groupExtractor;
+  constructor(options: {
+    prefix: string;
+    shardCount: number;
+    extractTenantId: (queueId: string) => string;
+    extractGroupId: (groupName: string, queueId: string) => string;
+  });
+  tenantShard(tenantId: string): number;
+  masterQueueKey(shardId: number): string;
+  queueKey(queueId: string): string;
+  queueItemsKey(queueId: string): string;
+  payloadKey(queueId: string, messageId: string): string;
+  tenantPayloadBytesKey(tenantId: string): string;
+  concurrencyKey(groupName: string, groupId: string): string;
+  concurrencyKeyPattern(groupName: string, shardId: number): string;
+  inflightKey(shardId: number): string;
+  inflightDataKey(shardId: number): string;
+  workerQueueKey(consumerId: string): string;
+  deadLetterQueueKey(tenantId: string): string;
+  deadLetterQueueDataKey(tenantId: string): string;
+  dispatchKey(shardId: number): string;
+  tenantQueueIndexKey(tenantId: string): string;
+  drrDeficitKey(): string;
+  extractTenantId(queueId: string): string;
+  extractGroupId(groupName: string, queueId: string): string;
+}
 //#endregion
 //#region src/fair-queue/masterQueue.d.ts
 interface MasterQueueOptions {
-  redis: RedisOptions;
+  redis: FairQueueRedisOptions;
   keys: FairQueueKeyProducer;
   shardCount: number;
 }
@@ -1355,7 +1482,7 @@ declare class WeightedScheduler extends BaseScheduler {
 //#endregion
 //#region src/fair-queue/schedulers/roundRobin.d.ts
 interface RoundRobinSchedulerConfig {
-  redis: RedisOptions;
+  redis: FairQueueRedisOptions;
   keys: FairQueueKeyProducer;
   /** Maximum queues to fetch from master queue per iteration */
   masterQueueLimit?: number;
@@ -1423,6 +1550,7 @@ interface FairQueueMetrics {
   dispatchLength: ObservableGauge;
   inflightCount: ObservableGauge;
   dlqLength: ObservableGauge;
+  oldestMessageAge: ObservableGauge;
 }
 /**
  * Options for creating FairQueue telemetry.
@@ -1509,6 +1637,7 @@ declare class FairQueueTelemetry {
     getDispatchLength?: (shardId: number) => Promise<number>;
     getInflightCount?: (shardId: number) => Promise<number>;
     getDLQLength?: (tenantId: string) => Promise<number>;
+    getOldestMessageAge?: (shardId: number) => Promise<number>;
     shardCount?: number;
     observedQueues?: string[];
     observedTenants?: string[];
@@ -1642,7 +1771,7 @@ declare const noopTelemetry: FairQueueTelemetry;
 //#endregion
 //#region src/fair-queue/tenantDispatch.d.ts
 interface TenantDispatchOptions {
-  redis: RedisOptions;
+  redis: FairQueueRedisOptions;
   keys: FairQueueKeyProducer;
   shardCount: number;
 }
@@ -1720,7 +1849,7 @@ declare class TenantDispatch {
 //#endregion
 //#region src/fair-queue/visibility.d.ts
 interface VisibilityManagerOptions {
-  redis: RedisOptions;
+  redis: FairQueueRedisOptions;
   keys: FairQueueKeyProducer;
   shardCount: number;
   defaultTimeoutMs: number;
@@ -1875,7 +2004,7 @@ declare module "@internal/redis" {
 //#endregion
 //#region src/fair-queue/workerQueue.d.ts
 interface WorkerQueueManagerOptions {
-  redis: RedisOptions;
+  redis: FairQueueRedisOptions;
   keys: FairQueueKeyProducer;
   logger?: {
     debug: (message: string, context?: Record<string, unknown>) => void;
@@ -1967,7 +2096,7 @@ declare class WorkerQueueManager {
    * Register custom commands on an external Redis client.
    * Use this when initializing FairQueue with worker queues.
    */
-  registerCommands(redis: Redis): void;
+  registerCommands(redis: RedisClient): void;
 }
 declare module "@internal/redis" {
   interface RedisCommander<Context> {
@@ -1976,6 +2105,12 @@ declare module "@internal/redis" {
 }
 //#endregion
 //#region src/fair-queue/index.d.ts
+/** An enqueue that would take a tenant past `tenantPayloadBytesLimit`. */
+declare class TenantQueueFullError extends Error {
+  readonly tenantId: string;
+  readonly limitBytes: number;
+  constructor(tenantId: string, limitBytes: number);
+}
 /**
  * FairQueue is the main orchestrator for fair queue message routing.
  *
@@ -2019,6 +2154,7 @@ declare class FairQueue<TPayloadSchema extends AnyZodSchema = z.ZodUnknown> {
   private reconcileIntervalMs;
   private workerQueueResolver;
   private batchClaimSize;
+  private separatePayloads;
   private cooloffEnabled;
   private cooloffThreshold;
   private cooloffPeriodMs;
@@ -2116,6 +2252,11 @@ declare class FairQueue<TPayloadSchema extends AnyZodSchema = z.ZodUnknown> {
    * Get total tenant count across dispatch shards plus any legacy queues still draining.
    */
   getTotalQueueCount(): Promise<number>;
+  /**
+   * Get the age in ms of the oldest due message not yet completed in a shard, or 0 if none.
+   * Claims don't refresh the index scores, so claimed messages count until their queue next completes or empties.
+   */
+  getOldestMessageAge(shardId: number): Promise<number>;
   /**
    * Get total in-flight message count.
    */
@@ -2167,6 +2308,8 @@ declare class FairQueue<TPayloadSchema extends AnyZodSchema = z.ZodUnknown> {
    * @param error - Optional error that caused the failure
    */
   failMessage(messageId: string, queueId: string, error?: Error): Promise<void>;
+  /** A tenant's queued and in-flight payload bytes, with `separatePayloads`. */
+  getTenantPayloadBytes(tenantId: string): Promise<number>;
 }
 declare module "@internal/redis" {
   interface RedisCommander<Context> {
@@ -2176,6 +2319,8 @@ declare module "@internal/redis" {
     enqueueMessageAtomicV2(queueKey: string, queueItemsKey: string, tenantQueueIndexKey: string, dispatchKey: string, queueId: string, messageId: string, timestamp: string, payload: string, tenantId: string): Promise<number>;
     enqueueBatchAtomicV2(queueKey: string, queueItemsKey: string, tenantQueueIndexKey: string, dispatchKey: string, queueId: string, tenantId: string, ...args: string[]): Promise<number>;
     updateDispatchIndexes(queueKey: string, tenantQueueIndexKey: string, dispatchKey: string, queueId: string, tenantId: string): Promise<number>;
+    enqueueMessageSeparatePayload(queueKey: string, queueItemsKey: string, tenantQueueIndexKey: string, dispatchKey: string, payloadKey: string, inflightKey: string, bytesKey: string, queueId: string, messageId: string, timestamp: string, envelope: string, tenantId: string, payload: string, inflightMember: string, limit: string): Promise<number>;
+    completeSeparatePayload(inflightKey: string, inflightDataKey: string, payloadKey: string, bytesKey: string, inflightMember: string, messageId: string): Promise<number>;
   }
 }
 //#endregion
@@ -2432,4 +2577,4 @@ declare class MollifierDrainer<TPayload = unknown> {
   private processEntry;
 }
 //#endregion
-export { type AcceptResult, AnyMessageCatalog, AnyQueueItem, BaseScheduler, BatchConfig, BatchedSpanManager, BatchedSpanManagerOptions, type BufferEntry, BufferEntryError, BufferEntrySchema, BufferEntryStatus, CallbackFairQueueKeyProducer, type CasSetMetadataResult, ClaimResult, ConcurrencyCheckResult, ConcurrencyGroupConfig, ConcurrencyManager, ConcurrencyManagerOptions, ConcurrencyState, ConsumerLoopState, CooloffOptions, CronSchema, CustomRetry, DRRScheduler, DRRSchedulerConfig, DeadLetterMessage, DefaultFairQueueKeyProducer, DispatchSchedulerContext, type DrainResult, EnqueueBatchOptions, EnqueueOptions, ExponentialBackoffRetry, FairQueue, FairQueueAttributes, FairQueueKeyProducer, FairQueueMetrics, FairQueueOptions, FairQueueTelemetry, FairScheduler, FixedDelayRetry, GlobalRateLimiter, type IdempotencyClaimResult, type IdempotencyLookupInput, ImmediateRetry, InFlightMessage, JobHandler, JobHandlerParams, LinearBackoffRetry, MasterQueue, MasterQueueOptions, MessageCatalogKey, MessageCatalogSchema, MessageCatalogValue, MessageHandler, MessageHandlerContext, MessagingAttributes, MollifierBuffer, type MollifierBufferOptions, MollifierDrainer, type MollifierDrainerHandler, type MollifierDrainerOptions, type MollifierDrainerTerminalFailureCause, type MollifierDrainerTerminalFailureHandler, type MutateSnapshotResult, NoRetry, NoopScheduler, QueueCooloffState, QueueDescriptor, QueueItem, QueueMessage, QueueWithScore, ReclaimedMessageInfo, RetryOptions, RetryStrategy, RoundRobinScheduler, SchedulerContext, SimpleQueue, type SnapshotPatch, StoredMessage, TelemetryOptions, TenantDispatch, TenantDispatchOptions, TenantQueues, TenantWithScore, VisibilityManager, VisibilityManagerOptions, WeightedScheduler, WeightedSchedulerBiases, WeightedSchedulerConfig, Worker, WorkerCatalog, WorkerConcurrencyOptions, WorkerQueueManager, WorkerQueueManagerOptions, WorkerQueueOptions, createDefaultRetryStrategy, defaultRetryOptions, deserialiseSnapshot, idempotencyLookupKeyFor, isAbortError, makeIdempotencyClaimKey, noopTelemetry, serialiseSnapshot };
+export { type AcceptResult, AnyMessageCatalog, AnyQueueItem, BaseScheduler, BatchConfig, BatchedSpanManager, BatchedSpanManagerOptions, type BufferEntry, BufferEntryError, BufferEntrySchema, BufferEntryStatus, CallbackFairQueueKeyProducer, type CasSetMetadataResult, ClaimResult, ConcurrencyCheckResult, ConcurrencyGroupConfig, ConcurrencyManager, ConcurrencyManagerOptions, ConcurrencyState, ConsumerLoopState, CooloffOptions, CronSchema, CustomRetry, DRRScheduler, DRRSchedulerConfig, DeadLetterMessage, DefaultFairQueueKeyProducer, DispatchSchedulerContext, type DrainResult, EnqueueBatchOptions, EnqueueOptions, ExponentialBackoffRetry, FairQueue, FairQueueAttributes, FairQueueKeyProducer, FairQueueMetrics, FairQueueOptions, FairQueueRedisOptions, FairQueueTelemetry, FairScheduler, FixedDelayRetry, GlobalRateLimiter, type IdempotencyClaimResult, type IdempotencyLookupInput, ImmediateRetry, InFlightMessage, JobHandler, JobHandlerParams, LinearBackoffRetry, MasterQueue, MasterQueueOptions, MessageCatalogKey, MessageCatalogSchema, MessageCatalogValue, MessageHandler, MessageHandlerContext, MessagingAttributes, MollifierBuffer, type MollifierBufferOptions, MollifierDrainer, type MollifierDrainerHandler, type MollifierDrainerOptions, type MollifierDrainerTerminalFailureCause, type MollifierDrainerTerminalFailureHandler, type MutateSnapshotResult, NoRetry, NoopScheduler, QueueCooloffState, QueueDescriptor, QueueItem, QueueMessage, QueueWithScore, ReclaimedMessageInfo, RetryOptions, RetryStrategy, RoundRobinScheduler, SchedulerContext, SimpleQueue, type SnapshotPatch, StoredMessage, TelemetryOptions, TenantDispatch, TenantDispatchOptions, TenantQueueFullError, TenantQueues, TenantShardedFairQueueKeyProducer, TenantWithScore, VisibilityManager, VisibilityManagerOptions, WeightedScheduler, WeightedSchedulerBiases, WeightedSchedulerConfig, Worker, WorkerCatalog, WorkerConcurrencyOptions, WorkerQueueManager, WorkerQueueManagerOptions, WorkerQueueOptions, createDefaultRetryStrategy, createFairQueueRedisClient, defaultRetryOptions, deserialiseSnapshot, idempotencyLookupKeyFor, isAbortError, isClusterConnection, keyPrefixOf, makeIdempotencyClaimKey, noopTelemetry, scanKeys, serialiseSnapshot };