@hatchet-dev/typescript-sdk 1.35.0 → 1.36.0

package/clients/dispatcher/action-listener.js

@@ -225,8 +225,6 @@ class ActionListener {
             }
         });
     }
-    // Leaves the heartbeat running: tasks that are already in flight still need
-    // the engine to see this worker as alive until they finish.
     stopStream() {
         this.done = true;
         // Abort the gRPC stream to immediately cancel the generator

package/clients/rest/generated/Api.d.ts

@@ -1,4 +1,4 @@
-import { APIErrors, APIMeta, AcceptInviteRequest, BulkCreateEventRequest, CancelEventRequest, CreateAPITokenRequest, CreateAPITokenResponse, CreateCronWorkflowTriggerRequest, CreateEventRequest, CreateSNSIntegrationRequest, CreateTenantAlertEmailGroupRequest, CreateTenantInviteRequest, CreateTenantRequest, CronWorkflows, CronWorkflowsList, CronWorkflowsOrderByField, Event, EventData, EventKey, EventKeyList, EventList, EventOrderByDirection, EventOrderByField, EventSearch, Events, FeatureFlagEvaluationResult, FeatureFlagId, ListAPIMetaIntegration, ListAPITokensResponse, ListSNSIntegrations, ListSlackWebhooks, OtelSpanList, RateLimitList, RateLimitOrderByDirection, RateLimitOrderByField, RejectInviteRequest, ReplayEventRequest, ReplayWorkflowRunsRequest, ReplayWorkflowRunsResponse, RerunStepRunRequest, SNSIntegration, ScheduleWorkflowRunRequest, ScheduledRunStatus, ScheduledWorkflows, ScheduledWorkflowsBulkDeleteRequest, ScheduledWorkflowsBulkDeleteResponse, ScheduledWorkflowsBulkUpdateRequest, ScheduledWorkflowsBulkUpdateResponse, ScheduledWorkflowsList, ScheduledWorkflowsOrderByField, StepRun, StepRunArchiveList, StepRunEventList, TaskStats, Tenant, TenantAlertEmailGroup, TenantAlertEmailGroupList, TenantAlertingSettings, TenantInvite, TenantInviteList, TenantMember, TenantMemberList, TenantQueueMetrics, TenantResourcePolicy, TenantStepRunQueueMetrics, TriggerRunResult, TriggerWorkflowRunRequest, UpdateCronWorkflowTriggerRequest, UpdateScheduledWorkflowRunRequest, UpdateTenantAlertEmailGroupRequest, UpdateTenantInviteRequest, UpdateTenantMemberRequest, UpdateTenantRequest, UpdateWorkerRequest, User, UserChangePasswordRequest, UserLoginRequest, UserRegisterRequest, UserTenantMembershipsList, V1AdditionalMetadataOperator, V1BranchDurableTaskRequest, V1BranchDurableTaskResponse, V1CELDebugRequest, V1CELDebugResponse, V1CancelTaskRequest, V1CancelledTasks, V1CreateFilterRequest, V1CreateWebhookRequest, V1DagChildren, V1DurableEventLogList, V1Event, V1EventList, V1Filter, V1FilterList, V1LogLineLevel, V1LogLineList, V1LogLineOrderByDirection, V1LogsPointMetrics, V1ReplayTaskRequest, V1ReplayedTasks, V1RestoreTaskResponse, V1RunningFilter, V1TaskEventList, V1TaskPointMetrics, V1TaskRunMetrics, V1TaskStatus, V1TaskSummary, V1TaskSummaryList, V1TaskTimingList, V1TriggerWorkflowRunRequest, V1UpdateFilterRequest, V1UpdateWebhookRequest, V1Webhook, V1WebhookList, V1WebhookResponse, V1WebhookSourceName, V1WorkflowRunDetails, V1WorkflowRunDisplayNameList, V1WorkflowRunExternalIdList, WebhookWorkerCreateRequest, WebhookWorkerCreated, WebhookWorkerListResponse, WebhookWorkerRequestListResponse, Worker, WorkerList, WorkerStatus, Workflow, WorkflowID, WorkflowKindList, WorkflowList, WorkflowMetrics, WorkflowRun, WorkflowRunList, WorkflowRunOrderByDirection, WorkflowRunOrderByField, WorkflowRunShape, WorkflowRunStatus, WorkflowRunStatusList, WorkflowRunsCancelRequest, WorkflowRunsMetrics, WorkflowUpdateRequest, WorkflowVersion, WorkflowWorkersCount } from './data-contracts';
+import { APIErrors, APIMeta, AcceptInviteRequest, BulkCreateEventRequest, CancelEventRequest, CreateAPITokenRequest, CreateAPITokenResponse, CreateCronWorkflowTriggerRequest, CreateEventRequest, CreateSNSIntegrationRequest, CreateTenantAlertEmailGroupRequest, CreateTenantInviteRequest, CreateTenantRequest, CronWorkflows, CronWorkflowsList, CronWorkflowsOrderByField, Event, EventData, EventKey, EventKeyList, EventList, EventOrderByDirection, EventOrderByField, EventSearch, Events, FeatureFlagEvaluationResult, FeatureFlagId, ListAPIMetaIntegration, ListAPITokensResponse, ListSNSIntegrations, ListSlackWebhooks, OtelSpanList, RateLimitList, RateLimitOrderByDirection, RateLimitOrderByField, RejectInviteRequest, ReplayEventRequest, ReplayWorkflowRunsRequest, ReplayWorkflowRunsResponse, RerunStepRunRequest, SNSIntegration, ScheduleWorkflowRunRequest, ScheduledRunStatus, ScheduledWorkflows, ScheduledWorkflowsBulkDeleteRequest, ScheduledWorkflowsBulkDeleteResponse, ScheduledWorkflowsBulkUpdateRequest, ScheduledWorkflowsBulkUpdateResponse, ScheduledWorkflowsList, ScheduledWorkflowsOrderByField, StepRun, StepRunArchiveList, StepRunEventList, TaskStats, Tenant, TenantAlertEmailGroup, TenantAlertEmailGroupList, TenantAlertingSettings, TenantInvite, TenantInviteList, TenantMember, TenantMemberList, TenantQueueMetrics, TenantResourcePolicy, TenantStepRunQueueMetrics, TriggerRunResult, TriggerWorkflowRunRequest, UpdateCronWorkflowTriggerRequest, UpdateScheduledWorkflowRunRequest, UpdateTenantAlertEmailGroupRequest, UpdateTenantInviteRequest, UpdateTenantMemberRequest, UpdateTenantRequest, UpdateWorkerRequest, User, UserChangePasswordRequest, UserLoginRequest, UserRegisterRequest, UserTenantMembershipsList, V1AdditionalMetadataOperator, V1BranchDurableTaskRequest, V1BranchDurableTaskResponse, V1CELDebugRequest, V1CELDebugResponse, V1CancelTaskRequest, V1CancelledTasks, V1CreateFilterRequest, V1CreateWebhookRequest, V1DagChildren, V1DurableEventLogList, V1Event, V1EventList, V1Filter, V1FilterList, V1LogLineLevel, V1LogLineList, V1LogLineOrderByDirection, V1LogsPointMetrics, V1ReplayTaskRequest, V1ReplayedTasks, V1RestoreTaskResponse, V1RunningFilter, V1StreamPayloadRef, V1TaskEventList, V1TaskPointMetrics, V1TaskRunMetrics, V1TaskStatus, V1TaskSummary, V1TaskSummaryList, V1TaskTimingList, V1TriggerWorkflowRunRequest, V1UpdateFilterRequest, V1UpdateWebhookRequest, V1Webhook, V1WebhookList, V1WebhookResponse, V1WebhookSourceName, V1WorkflowRunDetails, V1WorkflowRunDisplayNameList, V1WorkflowRunExternalIdList, WebhookWorkerCreateRequest, WebhookWorkerCreated, WebhookWorkerListResponse, WebhookWorkerRequestListResponse, Worker, WorkerList, WorkerStatus, Workflow, WorkflowID, WorkflowKindList, WorkflowList, WorkflowMetrics, WorkflowRun, WorkflowRunList, WorkflowRunOrderByDirection, WorkflowRunOrderByField, WorkflowRunShape, WorkflowRunStatus, WorkflowRunStatusList, WorkflowRunsCancelRequest, WorkflowRunsMetrics, WorkflowUpdateRequest, WorkflowVersion, WorkflowWorkersCount } from './data-contracts';
 import { HttpClient, RequestParams } from './http-client';
 export declare class Api<SecurityDataType = unknown> extends HttpClient<SecurityDataType> {
     /**
@@ -341,6 +341,32 @@ export declare class Api<SecurityDataType = unknown> extends HttpClient<Security
          */
         limit?: number;
     }, params?: RequestParams) => Promise<import("axios").AxiosResponse<V1DurableEventLogList, any, {}>>;
+    /**
+     * @description Get a durable stream payload uploaded ahead of its publish, by the payload_ref of the message that references it.
+     *
+     * @tags Streams
+     * @name V1StreamPayloadGet
+     * @summary Get stream payload
+     * @request GET:/api/v1/stable/tenants/{tenant}/streams/payloads
+     * @secure
+     */
+    v1StreamPayloadGet: (tenant: string, query: {
+        /** The payload_ref of the stream message */
+        ref: string;
+    }, params?: RequestParams) => Promise<import("axios").AxiosResponse<File, any, {}>>;
+    /**
+     * @description Upload a payload too large for a gRPC publish, then publish the message with the returned ref as its payload_ref.
+     *
+     * @tags Streams
+     * @name V1StreamPayloadUpload
+     * @summary Upload stream payload
+     * @request POST:/api/v1/stable/tenants/{tenant}/streams/payloads
+     * @secure
+     */
+    v1StreamPayloadUpload: (tenant: string, data: {
+        /** @format binary */
+        payload: File;
+    }, params?: RequestParams) => Promise<import("axios").AxiosResponse<V1StreamPayloadRef, any, {}>>;
     /**
      * @description Get a workflow run and its metadata to display on the "detail" page
      *

package/clients/rest/generated/Api.js

@@ -166,6 +166,26 @@ class Api extends http_client_1.HttpClient {
          * @secure
          */
         this.v1DurableTaskEventLogList = (tenant, durableTask, query, params = {}) => this.request(Object.assign({ path: `/api/v1/stable/tenants/${tenant}/durable-tasks/${durableTask}`, method: 'GET', query: query, secure: true, format: 'json' }, params));
+        /**
+         * @description Get a durable stream payload uploaded ahead of its publish, by the payload_ref of the message that references it.
+         *
+         * @tags Streams
+         * @name V1StreamPayloadGet
+         * @summary Get stream payload
+         * @request GET:/api/v1/stable/tenants/{tenant}/streams/payloads
+         * @secure
+         */
+        this.v1StreamPayloadGet = (tenant, query, params = {}) => this.request(Object.assign({ path: `/api/v1/stable/tenants/${tenant}/streams/payloads`, method: 'GET', query: query, secure: true }, params));
+        /**
+         * @description Upload a payload too large for a gRPC publish, then publish the message with the returned ref as its payload_ref.
+         *
+         * @tags Streams
+         * @name V1StreamPayloadUpload
+         * @summary Upload stream payload
+         * @request POST:/api/v1/stable/tenants/{tenant}/streams/payloads
+         * @secure
+         */
+        this.v1StreamPayloadUpload = (tenant, data, params = {}) => this.request(Object.assign({ path: `/api/v1/stable/tenants/${tenant}/streams/payloads`, method: 'POST', body: data, secure: true, type: http_client_1.ContentType.FormData, format: 'json' }, params));
         /**
          * @description Get a workflow run and its metadata to display on the "detail" page
          *

package/clients/rest/generated/data-contracts.d.ts

@@ -766,6 +766,10 @@ export interface V1DurableEventLogEntry {
     taskDisplayName: string;
 }
 export type V1DurableEventLogList = V1DurableEventLogEntry[];
+export interface V1StreamPayloadRef {
+    /** Publish the message with this as its payload_ref */
+    ref: string;
+}
 export interface OtelSpan {
     traceId: string;
     spanId: string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hatchet-dev/typescript-sdk",
-  "version": "1.35.0",
+  "version": "1.36.0",
   "engines": {
     "node": ">=20"
   },

package/protoc/v1/streams.d.ts

@@ -7,6 +7,11 @@ export interface PublishStreamMessageRequest {
     payload: Uint8Array;
     producerId: string;
     producerSeq: number;
+    /**
+     * instead of payload: a ref returned by the stream payload upload endpoint,
+     * for payloads too large for one gRPC message
+     */
+    payloadRef: string;
 }
 export interface PublishStreamMessageResponse {
 }
@@ -24,12 +29,31 @@ export interface StreamEntry {
     payload: Uint8Array;
     cursor: string;
     createdAt: Date | undefined;
+    /** instead of payload: fetch it from the stream payload endpoint by this ref */
+    payloadRef: string;
+}
+export interface GetStreamTopicMetadataRequest {
+    namespace: string;
+    topic: string;
+}
+/** Counts and the latest message cover only what's within the tenant's retention. */
+export interface StreamTopicMetadata {
+    namespace: string;
+    topic: string;
+    tenantId: string;
+    /** resumes a subscription after the newest retained message; unset when none is retained */
+    latestCursor?: string | undefined;
+    messageCount: number;
+    /** when the newest retained message was stored; unset when none is retained */
+    lastPublishedAt: Date | undefined;
 }
 export declare const PublishStreamMessageRequest: MessageFns<PublishStreamMessageRequest>;
 export declare const PublishStreamMessageResponse: MessageFns<PublishStreamMessageResponse>;
 export declare const SubscribeStreamRequest: MessageFns<SubscribeStreamRequest>;
 export declare const StreamMessage: MessageFns<StreamMessage>;
 export declare const StreamEntry: MessageFns<StreamEntry>;
+export declare const GetStreamTopicMetadataRequest: MessageFns<GetStreamTopicMetadataRequest>;
+export declare const StreamTopicMetadata: MessageFns<StreamTopicMetadata>;
 export type V1StreamsDefinition = typeof V1StreamsDefinition;
 export declare const V1StreamsDefinition: {
     readonly name: "V1Streams";
@@ -51,15 +75,25 @@ export declare const V1StreamsDefinition: {
             readonly responseStream: true;
             readonly options: {};
         };
+        readonly getTopicMetadata: {
+            readonly name: "GetTopicMetadata";
+            readonly requestType: typeof GetStreamTopicMetadataRequest;
+            readonly requestStream: false;
+            readonly responseType: typeof StreamTopicMetadata;
+            readonly responseStream: false;
+            readonly options: {};
+        };
     };
 };
 export interface V1StreamsServiceImplementation<CallContextExt = {}> {
     publish(request: PublishStreamMessageRequest, context: CallContext & CallContextExt): Promise<DeepPartial<PublishStreamMessageResponse>>;
     subscribe(request: SubscribeStreamRequest, context: CallContext & CallContextExt): ServerStreamingMethodResult<DeepPartial<StreamMessage>>;
+    getTopicMetadata(request: GetStreamTopicMetadataRequest, context: CallContext & CallContextExt): Promise<DeepPartial<StreamTopicMetadata>>;
 }
 export interface V1StreamsClient<CallOptionsExt = {}> {
     publish(request: DeepPartial<PublishStreamMessageRequest>, options?: CallOptions & CallOptionsExt): Promise<PublishStreamMessageResponse>;
     subscribe(request: DeepPartial<SubscribeStreamRequest>, options?: CallOptions & CallOptionsExt): AsyncIterable<StreamMessage>;
+    getTopicMetadata(request: DeepPartial<GetStreamTopicMetadataRequest>, options?: CallOptions & CallOptionsExt): Promise<StreamTopicMetadata>;
 }
 type Builtin = Date | Function | Uint8Array | string | number | boolean | undefined;
 export type DeepPartial<T> = T extends Builtin ? T : T extends globalThis.Array<infer U> ? globalThis.Array<DeepPartial<U>> : T extends ReadonlyArray<infer U> ? ReadonlyArray<DeepPartial<U>> : T extends {} ? {

package/protoc/v1/streams.js

@@ -5,13 +5,20 @@
 //   protoc               v3.19.1
 // source: v1/streams.proto
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.V1StreamsDefinition = exports.StreamEntry = exports.StreamMessage = exports.SubscribeStreamRequest = exports.PublishStreamMessageResponse = exports.PublishStreamMessageRequest = exports.protobufPackage = void 0;
+exports.V1StreamsDefinition = exports.StreamTopicMetadata = exports.GetStreamTopicMetadataRequest = exports.StreamEntry = exports.StreamMessage = exports.SubscribeStreamRequest = exports.PublishStreamMessageResponse = exports.PublishStreamMessageRequest = exports.protobufPackage = void 0;
 /* eslint-disable */
 const wire_1 = require("@bufbuild/protobuf/wire");
 const timestamp_1 = require("../google/protobuf/timestamp");
 exports.protobufPackage = 'v1';
 function createBasePublishStreamMessageRequest() {
-    return { namespace: '', topic: '', payload: new Uint8Array(0), producerId: '', producerSeq: 0 };
+    return {
+        namespace: '',
+        topic: '',
+        payload: new Uint8Array(0),
+        producerId: '',
+        producerSeq: 0,
+        payloadRef: '',
+    };
 }
 exports.PublishStreamMessageRequest = {
     encode(message, writer = new wire_1.BinaryWriter()) {
@@ -30,6 +37,9 @@ exports.PublishStreamMessageRequest = {
         if (message.producerSeq !== 0) {
             writer.uint32(40).int64(message.producerSeq);
         }
+        if (message.payloadRef !== '') {
+            writer.uint32(50).string(message.payloadRef);
+        }
         return writer;
     },
     decode(input, length) {
@@ -74,6 +84,13 @@ exports.PublishStreamMessageRequest = {
                     message.producerSeq = longToNumber(reader.int64());
                     continue;
                 }
+                case 6: {
+                    if (tag !== 50) {
+                        break;
+                    }
+                    message.payloadRef = reader.string();
+                    continue;
+                }
             }
             if ((tag & 7) === 4 || tag === 0) {
                 break;
@@ -97,6 +114,11 @@ exports.PublishStreamMessageRequest = {
                 : isSet(object.producer_seq)
                     ? globalThis.Number(object.producer_seq)
                     : 0,
+            payloadRef: isSet(object.payloadRef)
+                ? globalThis.String(object.payloadRef)
+                : isSet(object.payload_ref)
+                    ? globalThis.String(object.payload_ref)
+                    : '',
         };
     },
     toJSON(message) {
@@ -116,19 +138,23 @@ exports.PublishStreamMessageRequest = {
         if (message.producerSeq !== 0) {
             obj.producerSeq = Math.round(message.producerSeq);
         }
+        if (message.payloadRef !== '') {
+            obj.payloadRef = message.payloadRef;
+        }
         return obj;
     },
     create(base) {
         return exports.PublishStreamMessageRequest.fromPartial(base !== null && base !== void 0 ? base : {});
     },
     fromPartial(object) {
-        var _a, _b, _c, _d, _e;
+        var _a, _b, _c, _d, _e, _f;
         const message = createBasePublishStreamMessageRequest();
         message.namespace = (_a = object.namespace) !== null && _a !== void 0 ? _a : '';
         message.topic = (_b = object.topic) !== null && _b !== void 0 ? _b : '';
         message.payload = (_c = object.payload) !== null && _c !== void 0 ? _c : new Uint8Array(0);
         message.producerId = (_d = object.producerId) !== null && _d !== void 0 ? _d : '';
         message.producerSeq = (_e = object.producerSeq) !== null && _e !== void 0 ? _e : 0;
+        message.payloadRef = (_f = object.payloadRef) !== null && _f !== void 0 ? _f : '';
         return message;
     },
 };
@@ -341,7 +367,7 @@ exports.StreamMessage = {
     },
 };
 function createBaseStreamEntry() {
-    return { payload: new Uint8Array(0), cursor: '', createdAt: undefined };
+    return { payload: new Uint8Array(0), cursor: '', createdAt: undefined, payloadRef: '' };
 }
 exports.StreamEntry = {
     encode(message, writer = new wire_1.BinaryWriter()) {
@@ -354,6 +380,9 @@ exports.StreamEntry = {
         if (message.createdAt !== undefined) {
             timestamp_1.Timestamp.encode(toTimestamp(message.createdAt), writer.uint32(26).fork()).join();
         }
+        if (message.payloadRef !== '') {
+            writer.uint32(34).string(message.payloadRef);
+        }
         return writer;
     },
     decode(input, length) {
@@ -384,6 +413,13 @@ exports.StreamEntry = {
                     message.createdAt = fromTimestamp(timestamp_1.Timestamp.decode(reader, reader.uint32()));
                     continue;
                 }
+                case 4: {
+                    if (tag !== 34) {
+                        break;
+                    }
+                    message.payloadRef = reader.string();
+                    continue;
+                }
             }
             if ((tag & 7) === 4 || tag === 0) {
                 break;
@@ -401,6 +437,11 @@ exports.StreamEntry = {
                 : isSet(object.created_at)
                     ? fromJsonTimestamp(object.created_at)
                     : undefined,
+            payloadRef: isSet(object.payloadRef)
+                ? globalThis.String(object.payloadRef)
+                : isSet(object.payload_ref)
+                    ? globalThis.String(object.payload_ref)
+                    : '',
         };
     },
     toJSON(message) {
@@ -414,17 +455,242 @@ exports.StreamEntry = {
         if (message.createdAt !== undefined) {
             obj.createdAt = message.createdAt.toISOString();
         }
+        if (message.payloadRef !== '') {
+            obj.payloadRef = message.payloadRef;
+        }
         return obj;
     },
     create(base) {
         return exports.StreamEntry.fromPartial(base !== null && base !== void 0 ? base : {});
     },
     fromPartial(object) {
-        var _a, _b, _c;
+        var _a, _b, _c, _d;
         const message = createBaseStreamEntry();
         message.payload = (_a = object.payload) !== null && _a !== void 0 ? _a : new Uint8Array(0);
         message.cursor = (_b = object.cursor) !== null && _b !== void 0 ? _b : '';
         message.createdAt = (_c = object.createdAt) !== null && _c !== void 0 ? _c : undefined;
+        message.payloadRef = (_d = object.payloadRef) !== null && _d !== void 0 ? _d : '';
+        return message;
+    },
+};
+function createBaseGetStreamTopicMetadataRequest() {
+    return { namespace: '', topic: '' };
+}
+exports.GetStreamTopicMetadataRequest = {
+    encode(message, writer = new wire_1.BinaryWriter()) {
+        if (message.namespace !== '') {
+            writer.uint32(10).string(message.namespace);
+        }
+        if (message.topic !== '') {
+            writer.uint32(18).string(message.topic);
+        }
+        return writer;
+    },
+    decode(input, length) {
+        const reader = input instanceof wire_1.BinaryReader ? input : new wire_1.BinaryReader(input);
+        const end = length === undefined ? reader.len : reader.pos + length;
+        const message = createBaseGetStreamTopicMetadataRequest();
+        while (reader.pos < end) {
+            const tag = reader.uint32();
+            switch (tag >>> 3) {
+                case 1: {
+                    if (tag !== 10) {
+                        break;
+                    }
+                    message.namespace = reader.string();
+                    continue;
+                }
+                case 2: {
+                    if (tag !== 18) {
+                        break;
+                    }
+                    message.topic = reader.string();
+                    continue;
+                }
+            }
+            if ((tag & 7) === 4 || tag === 0) {
+                break;
+            }
+            reader.skip(tag & 7);
+        }
+        return message;
+    },
+    fromJSON(object) {
+        return {
+            namespace: isSet(object.namespace) ? globalThis.String(object.namespace) : '',
+            topic: isSet(object.topic) ? globalThis.String(object.topic) : '',
+        };
+    },
+    toJSON(message) {
+        const obj = {};
+        if (message.namespace !== '') {
+            obj.namespace = message.namespace;
+        }
+        if (message.topic !== '') {
+            obj.topic = message.topic;
+        }
+        return obj;
+    },
+    create(base) {
+        return exports.GetStreamTopicMetadataRequest.fromPartial(base !== null && base !== void 0 ? base : {});
+    },
+    fromPartial(object) {
+        var _a, _b;
+        const message = createBaseGetStreamTopicMetadataRequest();
+        message.namespace = (_a = object.namespace) !== null && _a !== void 0 ? _a : '';
+        message.topic = (_b = object.topic) !== null && _b !== void 0 ? _b : '';
+        return message;
+    },
+};
+function createBaseStreamTopicMetadata() {
+    return {
+        namespace: '',
+        topic: '',
+        tenantId: '',
+        latestCursor: undefined,
+        messageCount: 0,
+        lastPublishedAt: undefined,
+    };
+}
+exports.StreamTopicMetadata = {
+    encode(message, writer = new wire_1.BinaryWriter()) {
+        if (message.namespace !== '') {
+            writer.uint32(10).string(message.namespace);
+        }
+        if (message.topic !== '') {
+            writer.uint32(18).string(message.topic);
+        }
+        if (message.tenantId !== '') {
+            writer.uint32(26).string(message.tenantId);
+        }
+        if (message.latestCursor !== undefined) {
+            writer.uint32(34).string(message.latestCursor);
+        }
+        if (message.messageCount !== 0) {
+            writer.uint32(40).int64(message.messageCount);
+        }
+        if (message.lastPublishedAt !== undefined) {
+            timestamp_1.Timestamp.encode(toTimestamp(message.lastPublishedAt), writer.uint32(50).fork()).join();
+        }
+        return writer;
+    },
+    decode(input, length) {
+        const reader = input instanceof wire_1.BinaryReader ? input : new wire_1.BinaryReader(input);
+        const end = length === undefined ? reader.len : reader.pos + length;
+        const message = createBaseStreamTopicMetadata();
+        while (reader.pos < end) {
+            const tag = reader.uint32();
+            switch (tag >>> 3) {
+                case 1: {
+                    if (tag !== 10) {
+                        break;
+                    }
+                    message.namespace = reader.string();
+                    continue;
+                }
+                case 2: {
+                    if (tag !== 18) {
+                        break;
+                    }
+                    message.topic = reader.string();
+                    continue;
+                }
+                case 3: {
+                    if (tag !== 26) {
+                        break;
+                    }
+                    message.tenantId = reader.string();
+                    continue;
+                }
+                case 4: {
+                    if (tag !== 34) {
+                        break;
+                    }
+                    message.latestCursor = reader.string();
+                    continue;
+                }
+                case 5: {
+                    if (tag !== 40) {
+                        break;
+                    }
+                    message.messageCount = longToNumber(reader.int64());
+                    continue;
+                }
+                case 6: {
+                    if (tag !== 50) {
+                        break;
+                    }
+                    message.lastPublishedAt = fromTimestamp(timestamp_1.Timestamp.decode(reader, reader.uint32()));
+                    continue;
+                }
+            }
+            if ((tag & 7) === 4 || tag === 0) {
+                break;
+            }
+            reader.skip(tag & 7);
+        }
+        return message;
+    },
+    fromJSON(object) {
+        return {
+            namespace: isSet(object.namespace) ? globalThis.String(object.namespace) : '',
+            topic: isSet(object.topic) ? globalThis.String(object.topic) : '',
+            tenantId: isSet(object.tenantId)
+                ? globalThis.String(object.tenantId)
+                : isSet(object.tenant_id)
+                    ? globalThis.String(object.tenant_id)
+                    : '',
+            latestCursor: isSet(object.latestCursor)
+                ? globalThis.String(object.latestCursor)
+                : isSet(object.latest_cursor)
+                    ? globalThis.String(object.latest_cursor)
+                    : undefined,
+            messageCount: isSet(object.messageCount)
+                ? globalThis.Number(object.messageCount)
+                : isSet(object.message_count)
+                    ? globalThis.Number(object.message_count)
+                    : 0,
+            lastPublishedAt: isSet(object.lastPublishedAt)
+                ? fromJsonTimestamp(object.lastPublishedAt)
+                : isSet(object.last_published_at)
+                    ? fromJsonTimestamp(object.last_published_at)
+                    : undefined,
+        };
+    },
+    toJSON(message) {
+        const obj = {};
+        if (message.namespace !== '') {
+            obj.namespace = message.namespace;
+        }
+        if (message.topic !== '') {
+            obj.topic = message.topic;
+        }
+        if (message.tenantId !== '') {
+            obj.tenantId = message.tenantId;
+        }
+        if (message.latestCursor !== undefined) {
+            obj.latestCursor = message.latestCursor;
+        }
+        if (message.messageCount !== 0) {
+            obj.messageCount = Math.round(message.messageCount);
+        }
+        if (message.lastPublishedAt !== undefined) {
+            obj.lastPublishedAt = message.lastPublishedAt.toISOString();
+        }
+        return obj;
+    },
+    create(base) {
+        return exports.StreamTopicMetadata.fromPartial(base !== null && base !== void 0 ? base : {});
+    },
+    fromPartial(object) {
+        var _a, _b, _c, _d, _e, _f;
+        const message = createBaseStreamTopicMetadata();
+        message.namespace = (_a = object.namespace) !== null && _a !== void 0 ? _a : '';
+        message.topic = (_b = object.topic) !== null && _b !== void 0 ? _b : '';
+        message.tenantId = (_c = object.tenantId) !== null && _c !== void 0 ? _c : '';
+        message.latestCursor = (_d = object.latestCursor) !== null && _d !== void 0 ? _d : undefined;
+        message.messageCount = (_e = object.messageCount) !== null && _e !== void 0 ? _e : 0;
+        message.lastPublishedAt = (_f = object.lastPublishedAt) !== null && _f !== void 0 ? _f : undefined;
         return message;
     },
 };
@@ -448,6 +714,14 @@ exports.V1StreamsDefinition = {
             responseStream: true,
             options: {},
         },
+        getTopicMetadata: {
+            name: 'GetTopicMetadata',
+            requestType: exports.GetStreamTopicMetadataRequest,
+            requestStream: false,
+            responseType: exports.StreamTopicMetadata,
+            responseStream: false,
+            options: {},
+        },
     },
 };
 function bytesFromBase64(b64) {

package/protoc-es/v1/streams_pb.d.ts

@@ -29,6 +29,13 @@ export type PublishStreamMessageRequest = Message<"v1.PublishStreamMessageReques
      * @generated from field: int64 producer_seq = 5;
      */
     producerSeq: bigint;
+    /**
+     * instead of payload: a ref returned by the stream payload upload endpoint,
+     * for payloads too large for one gRPC message
+     *
+     * @generated from field: string payload_ref = 6;
+     */
+    payloadRef: string;
 };
 /**
  * Describes the message v1.PublishStreamMessageRequest.
@@ -104,12 +111,76 @@ export type StreamEntry = Message<"v1.StreamEntry"> & {
      * @generated from field: google.protobuf.Timestamp created_at = 3;
      */
     createdAt?: Timestamp | undefined;
+    /**
+     * instead of payload: fetch it from the stream payload endpoint by this ref
+     *
+     * @generated from field: string payload_ref = 4;
+     */
+    payloadRef: string;
 };
 /**
  * Describes the message v1.StreamEntry.
  * Use `create(StreamEntrySchema)` to create a new message.
  */
 export declare const StreamEntrySchema: GenMessage<StreamEntry>;
+/**
+ * @generated from message v1.GetStreamTopicMetadataRequest
+ */
+export type GetStreamTopicMetadataRequest = Message<"v1.GetStreamTopicMetadataRequest"> & {
+    /**
+     * @generated from field: string namespace = 1;
+     */
+    namespace: string;
+    /**
+     * @generated from field: string topic = 2;
+     */
+    topic: string;
+};
+/**
+ * Describes the message v1.GetStreamTopicMetadataRequest.
+ * Use `create(GetStreamTopicMetadataRequestSchema)` to create a new message.
+ */
+export declare const GetStreamTopicMetadataRequestSchema: GenMessage<GetStreamTopicMetadataRequest>;
+/**
+ * Counts and the latest message cover only what's within the tenant's retention.
+ *
+ * @generated from message v1.StreamTopicMetadata
+ */
+export type StreamTopicMetadata = Message<"v1.StreamTopicMetadata"> & {
+    /**
+     * @generated from field: string namespace = 1;
+     */
+    namespace: string;
+    /**
+     * @generated from field: string topic = 2;
+     */
+    topic: string;
+    /**
+     * @generated from field: string tenant_id = 3;
+     */
+    tenantId: string;
+    /**
+     * resumes a subscription after the newest retained message; unset when none is retained
+     *
+     * @generated from field: optional string latest_cursor = 4;
+     */
+    latestCursor?: string | undefined;
+    /**
+     * @generated from field: int64 message_count = 5;
+     */
+    messageCount: bigint;
+    /**
+     * when the newest retained message was stored; unset when none is retained
+     *
+     * @generated from field: google.protobuf.Timestamp last_published_at = 6;
+     */
+    lastPublishedAt?: Timestamp | undefined;
+};
+/**
+ * Describes the message v1.StreamTopicMetadata.
+ * Use `create(StreamTopicMetadataSchema)` to create a new message.
+ */
+export declare const StreamTopicMetadataSchema: GenMessage<StreamTopicMetadata>;
 /**
  * @generated from service v1.V1Streams
  */
@@ -130,4 +201,12 @@ export declare const V1Streams: GenService<{
         input: typeof SubscribeStreamRequestSchema;
         output: typeof StreamMessageSchema;
     };
+    /**
+     * @generated from rpc v1.V1Streams.GetTopicMetadata
+     */
+    getTopicMetadata: {
+        methodKind: "unary";
+        input: typeof GetStreamTopicMetadataRequestSchema;
+        output: typeof StreamTopicMetadataSchema;
+    };
 }>;

package/protoc-es/v1/streams_pb.js

@@ -3,13 +3,13 @@
 // @generated from file v1/streams.proto (package v1, syntax proto3)
 /* eslint-disable */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.V1Streams = exports.StreamEntrySchema = exports.StreamMessageSchema = exports.SubscribeStreamRequestSchema = exports.PublishStreamMessageResponseSchema = exports.PublishStreamMessageRequestSchema = exports.file_v1_streams = void 0;
+exports.V1Streams = exports.StreamTopicMetadataSchema = exports.GetStreamTopicMetadataRequestSchema = exports.StreamEntrySchema = exports.StreamMessageSchema = exports.SubscribeStreamRequestSchema = exports.PublishStreamMessageResponseSchema = exports.PublishStreamMessageRequestSchema = exports.file_v1_streams = void 0;
 const codegenv2_1 = require("@bufbuild/protobuf/codegenv2");
 const wkt_1 = require("@bufbuild/protobuf/wkt");
 /**
  * Describes the file v1/streams.proto.
  */
-exports.file_v1_streams = (0, codegenv2_1.fileDesc)("ChB2MS9zdHJlYW1zLnByb3RvEgJ2MSJ7ChtQdWJsaXNoU3RyZWFtTWVzc2FnZVJlcXVlc3QSEQoJbmFtZXNwYWNlGAEgASgJEg0KBXRvcGljGAIgASgJEg8KB3BheWxvYWQYAyABKAwSEwoLcHJvZHVjZXJfaWQYBCABKAkSFAoMcHJvZHVjZXJfc2VxGAUgASgDIh4KHFB1Ymxpc2hTdHJlYW1NZXNzYWdlUmVzcG9uc2UiWgoWU3Vic2NyaWJlU3RyZWFtUmVxdWVzdBIRCgluYW1lc3BhY2UYASABKAkSDQoFdG9waWMYAiABKAkSEwoGY3Vyc29yGAMgASgJSACIAQFCCQoHX2N1cnNvciJRCg1TdHJlYW1NZXNzYWdlEiAKB2VudHJpZXMYASADKAsyDy52MS5TdHJlYW1FbnRyeRIOCgZoYW5ndXAYAiABKAgSDgoGY3Vyc29yGAMgASgJIl4KC1N0cmVhbUVudHJ5Eg8KB3BheWxvYWQYASABKAwSDgoGY3Vyc29yGAIgASgJEi4KCmNyZWF0ZWRfYXQYAyABKAsyGi5nb29nbGUucHJvdG9idWYuVGltZXN0YW1wMpsBCglWMVN0cmVhbXMSTgoHUHVibGlzaBIfLnYxLlB1Ymxpc2hTdHJlYW1NZXNzYWdlUmVxdWVzdBogLnYxLlB1Ymxpc2hTdHJlYW1NZXNzYWdlUmVzcG9uc2UiABI+CglTdWJzY3JpYmUSGi52MS5TdWJzY3JpYmVTdHJlYW1SZXF1ZXN0GhEudjEuU3RyZWFtTWVzc2FnZSIAMAFCQlpAZ2l0aHViLmNvbS9oYXRjaGV0LWRldi9oYXRjaGV0L2ludGVybmFsL3NlcnZpY2VzL3NoYXJlZC9wcm90by92MWIGcHJvdG8z", [wkt_1.file_google_protobuf_timestamp]);
+exports.file_v1_streams = (0, codegenv2_1.fileDesc)("ChB2MS9zdHJlYW1zLnByb3RvEgJ2MSKQAQobUHVibGlzaFN0cmVhbU1lc3NhZ2VSZXF1ZXN0EhEKCW5hbWVzcGFjZRgBIAEoCRINCgV0b3BpYxgCIAEoCRIPCgdwYXlsb2FkGAMgASgMEhMKC3Byb2R1Y2VyX2lkGAQgASgJEhQKDHByb2R1Y2VyX3NlcRgFIAEoAxITCgtwYXlsb2FkX3JlZhgGIAEoCSIeChxQdWJsaXNoU3RyZWFtTWVzc2FnZVJlc3BvbnNlIloKFlN1YnNjcmliZVN0cmVhbVJlcXVlc3QSEQoJbmFtZXNwYWNlGAEgASgJEg0KBXRvcGljGAIgASgJEhMKBmN1cnNvchgDIAEoCUgAiAEBQgkKB19jdXJzb3IiUQoNU3RyZWFtTWVzc2FnZRIgCgdlbnRyaWVzGAEgAygLMg8udjEuU3RyZWFtRW50cnkSDgoGaGFuZ3VwGAIgASgIEg4KBmN1cnNvchgDIAEoCSJzCgtTdHJlYW1FbnRyeRIPCgdwYXlsb2FkGAEgASgMEg4KBmN1cnNvchgCIAEoCRIuCgpjcmVhdGVkX2F0GAMgASgLMhouZ29vZ2xlLnByb3RvYnVmLlRpbWVzdGFtcBITCgtwYXlsb2FkX3JlZhgEIAEoCSJBCh1HZXRTdHJlYW1Ub3BpY01ldGFkYXRhUmVxdWVzdBIRCgluYW1lc3BhY2UYASABKAkSDQoFdG9waWMYAiABKAkixgEKE1N0cmVhbVRvcGljTWV0YWRhdGESEQoJbmFtZXNwYWNlGAEgASgJEg0KBXRvcGljGAIgASgJEhEKCXRlbmFudF9pZBgDIAEoCRIaCg1sYXRlc3RfY3Vyc29yGAQgASgJSACIAQESFQoNbWVzc2FnZV9jb3VudBgFIAEoAxI1ChFsYXN0X3B1Ymxpc2hlZF9hdBgGIAEoCzIaLmdvb2dsZS5wcm90b2J1Zi5UaW1lc3RhbXBCEAoOX2xhdGVzdF9jdXJzb3Iy7QEKCVYxU3RyZWFtcxJOCgdQdWJsaXNoEh8udjEuUHVibGlzaFN0cmVhbU1lc3NhZ2VSZXF1ZXN0GiAudjEuUHVibGlzaFN0cmVhbU1lc3NhZ2VSZXNwb25zZSIAEj4KCVN1YnNjcmliZRIaLnYxLlN1YnNjcmliZVN0cmVhbVJlcXVlc3QaES52MS5TdHJlYW1NZXNzYWdlIgAwARJQChBHZXRUb3BpY01ldGFkYXRhEiEudjEuR2V0U3RyZWFtVG9waWNNZXRhZGF0YVJlcXVlc3QaFy52MS5TdHJlYW1Ub3BpY01ldGFkYXRhIgBCQlpAZ2l0aHViLmNvbS9oYXRjaGV0LWRldi9oYXRjaGV0L2ludGVybmFsL3NlcnZpY2VzL3NoYXJlZC9wcm90by92MWIGcHJvdG8z", [wkt_1.file_google_protobuf_timestamp]);
 /**
  * Describes the message v1.PublishStreamMessageRequest.
  * Use `create(PublishStreamMessageRequestSchema)` to create a new message.
@@ -35,6 +35,16 @@ exports.StreamMessageSchema = (0, codegenv2_1.messageDesc)(exports.file_v1_strea
  * Use `create(StreamEntrySchema)` to create a new message.
  */
 exports.StreamEntrySchema = (0, codegenv2_1.messageDesc)(exports.file_v1_streams, 4);
+/**
+ * Describes the message v1.GetStreamTopicMetadataRequest.
+ * Use `create(GetStreamTopicMetadataRequestSchema)` to create a new message.
+ */
+exports.GetStreamTopicMetadataRequestSchema = (0, codegenv2_1.messageDesc)(exports.file_v1_streams, 5);
+/**
+ * Describes the message v1.StreamTopicMetadata.
+ * Use `create(StreamTopicMetadataSchema)` to create a new message.
+ */
+exports.StreamTopicMetadataSchema = (0, codegenv2_1.messageDesc)(exports.file_v1_streams, 6);
 /**
  * @generated from service v1.V1Streams
  */

package/v1/client/features/streams.d.ts

@@ -7,6 +7,17 @@ export type StreamEvent = {
     /** when the message was stored */
     createdAt?: Date;
 };
+/** Covers only messages within the tenant's retention. */
+export type StreamTopicMetadata = {
+    namespace: string;
+    topic: string;
+    tenantId: string;
+    messageCount: number;
+    /** resumes events() after the newest retained message; undefined when none is retained */
+    latestCursor?: string;
+    /** when the newest retained message was stored; undefined when none is retained */
+    lastPublishedAt?: Date;
+};
 export type StreamCallOptions = {
     namespace?: string;
     cursor?: string;
@@ -20,11 +31,15 @@ export type StreamCallOptions = {
  */
 export declare class StreamsClient {
     private _config;
+    private _client;
     private _grpc;
     private producers;
     private publishChains;
     constructor(client: HatchetClient);
     private get grpc();
+    private static withRetries;
+    private uploadPayload;
+    private fetchPayload;
     private publishOrdered;
     /**
      * Resolves once the message is stored. A client's messages to a topic are
@@ -34,6 +49,13 @@ export declare class StreamsClient {
      * @param options - optional namespace override
      */
     publish(topic: string, message: Uint8Array | string, options?: StreamCallOptions): Promise<void>;
+    /**
+     * Describes topic's retained messages. Rejects with a NOT_FOUND gRPC error
+     * for a topic nothing was published to.
+     * @param topic - the topic to describe
+     * @param options - optional namespace override
+     */
+    topicMetadata(topic: string, options?: Pick<StreamCallOptions, 'namespace'>): Promise<StreamTopicMetadata>;
     /**
      * Yields topic's messages after options.cursor, or from the oldest retained.
      * If the connection drops, it resubscribes from the last message yielded, so

package/v1/client/features/streams.js

@@ -34,6 +34,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.StreamsClient = void 0;
 const crypto_1 = require("crypto");
+const axios_1 = require("axios");
 const nice_grpc_1 = require("nice-grpc");
 const grpc_helpers_1 = require("../../../util/grpc-helpers");
 const grpc_error_1 = require("../../../util/grpc-error");
@@ -58,7 +59,27 @@ const permanentSubscribeErrors = new Set([
 ]);
 const resubscribeMinDelayMs = 250;
 const resubscribeMaxDelayMs = 10000;
+// the server's limit for a payload sent over gRPC (MaxStreamMessagePayloadBytes)
+const maxInlinePayloadBytes = 4 * 1024 * 1024 - 5 * 1024;
+// the server's limit for an uploaded payload (MaxStreamUploadedPayloadBytes)
+const maxUploadedPayloadBytes = 64 * 1024 * 1024;
+const payloadRequestAttempts = 3;
+function httpStatus(err) {
+    var _a;
+    return (0, axios_1.isAxiosError)(err) ? (_a = err.response) === null || _a === void 0 ? void 0 : _a.status : undefined;
+}
+// a payload fetch that got no response, a 5xx or a 429 may succeed later
+function isTemporaryHttpError(err) {
+    if (!(0, axios_1.isAxiosError)(err)) {
+        return false;
+    }
+    const status = httpStatus(err);
+    return status === undefined || status >= 500 || status === 429;
+}
 function shouldResubscribe(err, attempt) {
+    if (isTemporaryHttpError(err)) {
+        return true;
+    }
     const code = (0, grpc_error_1.getGrpcErrorCode)(err);
     if (code === undefined || permanentSubscribeErrors.has(code)) {
         return false;
@@ -82,6 +103,7 @@ class StreamsClient {
         // stored promise always resolves, so one failure doesn't block later calls
         this.publishChains = new Map();
         this._config = client.config;
+        this._client = client;
     }
     get grpc() {
         if (!this._grpc) {
@@ -90,21 +112,46 @@ class StreamsClient {
         }
         return this._grpc;
     }
-    publishOrdered(key_1, namespace_1, topic_1, payload_1) {
-        return __awaiter(this, arguments, void 0, function* (key, namespace, topic, payload, retryGap = true) {
+    // retries a 5xx or network failure; a 4xx won't succeed on retry
+    static withRetries(request) {
+        return __awaiter(this, void 0, void 0, function* () {
+            for (let attempt = 1;; attempt += 1) {
+                try {
+                    return yield request();
+                }
+                catch (err) {
+                    const status = httpStatus(err);
+                    if ((status !== undefined && status < 500) || attempt >= payloadRequestAttempts) {
+                        throw err;
+                    }
+                }
+            }
+        });
+    }
+    uploadPayload(payload) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const res = yield StreamsClient.withRetries(() => this._client.api.v1StreamPayloadUpload(this._config.tenant_id, {
+                payload: new Blob([payload]),
+            }));
+            return res.data.ref;
+        });
+    }
+    fetchPayload(payloadRef, signal) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const res = yield StreamsClient.withRetries(() => this._client.api.v1StreamPayloadGet(this._config.tenant_id, { ref: payloadRef }, { format: 'arraybuffer', signal }));
+            return new Uint8Array(res.data);
+        });
+    }
+    publishOrdered(key_1, namespace_1, topic_1, body_1) {
+        return __awaiter(this, arguments, void 0, function* (key, namespace, topic, body, retryGap = true) {
             let producer = this.producers.get(key);
             if (!producer) {
                 producer = { producerId: (0, crypto_1.randomUUID)(), seq: 0 };
                 this.producers.set(key, producer);
             }
             try {
-                yield this.grpc.publish({
-                    namespace,
-                    topic,
-                    payload,
-                    producerId: producer.producerId,
-                    producerSeq: producer.seq,
-                });
+                yield this.grpc.publish(Object.assign(Object.assign({ namespace,
+                    topic }, body), { producerId: producer.producerId, producerSeq: producer.seq }));
             }
             catch (err) {
                 // it may still land, and a different payload under its seq would be dropped as a duplicate
@@ -113,7 +160,7 @@ class StreamsClient {
                 }
                 // a gap stored nothing (e.g. the watermark passed cursor retention), so resend as the new producer
                 if (retryGap && (0, grpc_error_1.getGrpcErrorCode)(err) === nice_grpc_1.Status.FAILED_PRECONDITION) {
-                    return this.publishOrdered(key, namespace, topic, payload, false);
+                    return this.publishOrdered(key, namespace, topic, body, false);
                 }
                 throw err;
             }
@@ -133,12 +180,41 @@ class StreamsClient {
             const payload = typeof message === 'string' ? new TextEncoder().encode(message) : message;
             const namespace = (_a = options === null || options === void 0 ? void 0 : options.namespace) !== null && _a !== void 0 ? _a : '';
             const key = `${namespace}\u0000${topic}`;
+            if (payload.byteLength > maxUploadedPayloadBytes) {
+                throw new Error(`payload exceeds maximum size of ${maxUploadedPayloadBytes} bytes`);
+            }
+            // uploaded before taking its place in the topic's order, so uploads overlap
+            const uploaded = payload.byteLength > maxInlinePayloadBytes ? this.uploadPayload(payload) : undefined;
+            // awaited in order below; this only keeps an early failure from reporting as unhandled
+            uploaded === null || uploaded === void 0 ? void 0 : uploaded.catch(() => { });
             const previous = (_b = this.publishChains.get(key)) !== null && _b !== void 0 ? _b : Promise.resolve();
-            const publishPromise = previous.then(() => this.publishOrdered(key, namespace, topic, payload));
+            const publishPromise = previous.then(() => __awaiter(this, void 0, void 0, function* () {
+                return this.publishOrdered(key, namespace, topic, uploaded ? { payloadRef: yield uploaded } : { payload });
+            }));
             this.publishChains.set(key, publishPromise.catch(() => { }));
             return publishPromise;
         });
     }
+    /**
+     * Describes topic's retained messages. Rejects with a NOT_FOUND gRPC error
+     * for a topic nothing was published to.
+     * @param topic - the topic to describe
+     * @param options - optional namespace override
+     */
+    topicMetadata(topic, options) {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            const res = yield this.grpc.getTopicMetadata({ namespace: (_a = options === null || options === void 0 ? void 0 : options.namespace) !== null && _a !== void 0 ? _a : '', topic });
+            return {
+                namespace: res.namespace,
+                topic: res.topic,
+                tenantId: res.tenantId,
+                messageCount: res.messageCount,
+                latestCursor: res.latestCursor,
+                lastPublishedAt: res.lastPublishedAt,
+            };
+        });
+    }
     /**
      * Yields topic's messages after options.cursor, or from the oldest retained.
      * If the connection drops, it resubscribes from the last message yielded, so
@@ -167,11 +243,15 @@ class StreamsClient {
                             if (msg.hangup) {
                                 return yield __await(void 0);
                             }
-                            failures = 0;
                             for (const entry of msg.entries) {
+                                const payload = entry.payloadRef
+                                    ? yield __await(this.fetchPayload(entry.payloadRef, signal))
+                                    : entry.payload;
                                 ({ cursor } = entry);
+                                // reset only on delivery, so a payload fetch failing on every reconnect still backs off
+                                failures = 0;
                                 yield yield __await({
-                                    payload: entry.payload,
+                                    payload,
                                     cursor: entry.cursor,
                                     createdAt: entry.createdAt,
                                 });

package/v1/client/worker/worker-internal.js

@@ -639,7 +639,7 @@ class InternalWorker {
     }
     shutdown() {
         return __awaiter(this, void 0, void 0, function* () {
-            var _a, _b;
+            var _a;
             this.killing = true;
             this.setStatus(health_server_1.workerStatus.UNHEALTHY);
             this.logger.info('Starting to exit...');
@@ -670,15 +670,17 @@ class InternalWorker {
             catch (e) {
                 this.logger.error(`Could not stop durable listener: ${e.message}`);
             }
-            (_a = this.listener) === null || _a === void 0 ? void 0 : _a.stopStream();
             this.logger.info('Gracefully exiting hatchet worker, running tasks will attempt to finish...');
-            // attempt to wait for futures to finish
-            yield Promise.all(Object.values(this.futures).map(({ promise }) => promise));
+            // Awaits the underlying work rather than the cancelable wrapper, which rejects as soon as a
+            // cancellation arrives while the task is still winding down.
+            yield Promise.allSettled(Object.values(this.futures).map(({ inner }) => inner));
             this.logger.info('Successfully finished pending tasks.');
-            // Unregistering stops the heartbeat, so it has to wait until the running tasks are done:
-            // otherwise the engine sees a dead worker and reassigns tasks that are about to complete.
+            // Unregistering closes the action stream and stops the heartbeat, so it has to wait until
+            // the running tasks are done. Stopping the heartbeat earlier lets the engine treat the worker
+            // as dead and reassign tasks that are about to complete, and closing the stream earlier makes
+            // the engine reject every heartbeat sent while those tasks finish.
             try {
-                yield ((_b = this.listener) === null || _b === void 0 ? void 0 : _b.unregister());
+                yield ((_a = this.listener) === null || _a === void 0 ? void 0 : _a.unregister());
             }
             catch (e) {
                 this.logger.error(`Could not unregister listener: ${e.message}`);

package/v1/examples/durable-streams/benchmark.d.ts

@@ -0,0 +1,34 @@
+import { HatchetClient } from '../..';
+import { StreamEvent } from '../../client/features/streams';
+export declare const MiB: number;
+export declare const maxInlinePayloadBytes: number;
+export declare const describeBenchmark: jest.Describe;
+export type LatencyStats = {
+    count: number;
+    avgMs: number;
+    p50Ms: number;
+    p99Ms: number;
+    maxMs: number;
+};
+export declare function latencyStats(durationsMs: number[]): LatencyStats;
+export declare function formatLatency(stats: LatencyStats, digits?: number): string;
+export declare function formatSize(bytes: number): string;
+export type PublishLatencyStats = LatencyStats & {
+    throughputPerSec: number;
+};
+/**
+ * Times `count` sequential publishes, after `warmupCount` unmeasured ones
+ * absorb connection setup and topic creation.
+ */
+export declare function measurePublishLatency(hatchet: HatchetClient, topic: string, payload: Uint8Array | string, count: number, warmupCount?: number): Promise<PublishLatencyStats>;
+export declare function formatPublishLatency(stats: PublishLatencyStats): string;
+export type LiveSubscription = {
+    /** resolves with the next message delivered after the call */
+    nextEvent: () => Promise<StreamEvent>;
+    close: () => Promise<void>;
+};
+/**
+ * Subscribes to topic and resolves once the subscription is live, so a
+ * benchmark's timings don't include subscribe setup.
+ */
+export declare function openLiveSubscription(hatchet: HatchetClient, topic: string): Promise<LiveSubscription>;

package/v1/examples/durable-streams/benchmark.js

@@ -0,0 +1,120 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __asyncValues = (this && this.__asyncValues) || function (o) {
+    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
+    var m = o[Symbol.asyncIterator], i;
+    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
+    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
+    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.describeBenchmark = exports.maxInlinePayloadBytes = exports.MiB = void 0;
+exports.latencyStats = latencyStats;
+exports.formatLatency = formatLatency;
+exports.formatSize = formatSize;
+exports.measurePublishLatency = measurePublishLatency;
+exports.formatPublishLatency = formatPublishLatency;
+exports.openLiveSubscription = openLiveSubscription;
+const perf_hooks_1 = require("perf_hooks");
+exports.MiB = 1024 * 1024;
+// the server's per-message gRPC limit (MaxStreamMessagePayloadBytes); larger payloads are uploaded
+exports.maxInlinePayloadBytes = 4 * exports.MiB - 5 * 1024;
+// benchmarks run only when asked for (test_durable_streams.sh --bench)
+exports.describeBenchmark = process.env.HATCHET_E2E_BENCHMARKS ? describe : describe.skip;
+function percentile(sorted, p) {
+    return sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * p))];
+}
+function latencyStats(durationsMs) {
+    const sorted = [...durationsMs].sort((a, b) => a - b);
+    return {
+        count: durationsMs.length,
+        avgMs: durationsMs.reduce((sum, d) => sum + d, 0) / durationsMs.length,
+        p50Ms: percentile(sorted, 0.5),
+        p99Ms: percentile(sorted, 0.99),
+        maxMs: sorted[sorted.length - 1],
+    };
+}
+function formatLatency(stats, digits = 2) {
+    const fmt = (ms) => ms.toFixed(digits);
+    return `avg=${fmt(stats.avgMs)}ms p50=${fmt(stats.p50Ms)}ms p99=${fmt(stats.p99Ms)}ms max=${fmt(stats.maxMs)}ms`;
+}
+function formatSize(bytes) {
+    if (bytes % exports.MiB === 0)
+        return `${bytes / exports.MiB}MiB`;
+    if (bytes % 1024 === 0)
+        return `${bytes / 1024}KiB`;
+    return `${bytes}B`;
+}
+/**
+ * Times `count` sequential publishes, after `warmupCount` unmeasured ones
+ * absorb connection setup and topic creation.
+ */
+function measurePublishLatency(hatchet_1, topic_1, payload_1, count_1) {
+    return __awaiter(this, arguments, void 0, function* (hatchet, topic, payload, count, warmupCount = 5) {
+        for (let i = 0; i < warmupCount; i += 1) {
+            yield hatchet.streams.publish(topic, payload);
+        }
+        const durationsMs = [];
+        const start = perf_hooks_1.performance.now();
+        for (let i = 0; i < count; i += 1) {
+            const before = perf_hooks_1.performance.now();
+            yield hatchet.streams.publish(topic, payload);
+            durationsMs.push(perf_hooks_1.performance.now() - before);
+        }
+        const totalMs = perf_hooks_1.performance.now() - start;
+        return Object.assign(Object.assign({}, latencyStats(durationsMs)), { throughputPerSec: (count / totalMs) * 1000 });
+    });
+}
+function formatPublishLatency(stats) {
+    return `${formatLatency(stats)} throughput=${stats.throughputPerSec.toFixed(2)}/s`;
+}
+/**
+ * Subscribes to topic and resolves once the subscription is live, so a
+ * benchmark's timings don't include subscribe setup.
+ */
+function openLiveSubscription(hatchet, topic) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const abort = new AbortController();
+        let onEvent;
+        const consumer = (() => __awaiter(this, void 0, void 0, function* () {
+            var _a, e_1, _b, _c;
+            try {
+                for (var _d = true, _e = __asyncValues(hatchet.streams.events(topic, { signal: abort.signal })), _f; _f = yield _e.next(), _a = _f.done, !_a; _d = true) {
+                    _c = _f.value;
+                    _d = false;
+                    const ev = _c;
+                    onEvent === null || onEvent === void 0 ? void 0 : onEvent(ev);
+                }
+            }
+            catch (e_1_1) { e_1 = { error: e_1_1 }; }
+            finally {
+                try {
+                    if (!_d && !_a && (_b = _e.return)) yield _b.call(_e);
+                }
+                finally { if (e_1) throw e_1.error; }
+            }
+        }))();
+        const nextEvent = () => new Promise((resolve) => {
+            onEvent = resolve;
+        });
+        // the subscription is live once a first message comes through
+        const ready = nextEvent();
+        yield hatchet.streams.publish(topic, 'ready');
+        yield ready;
+        return {
+            nextEvent,
+            close: () => __awaiter(this, void 0, void 0, function* () {
+                abort.abort();
+                yield consumer;
+            }),
+        };
+    });
+}

package/version.d.ts

@@ -1 +1 @@
-export declare const HATCHET_VERSION = "1.35.0";
+export declare const HATCHET_VERSION = "1.36.0";

package/version.js

@@ -1,4 +1,4 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.HATCHET_VERSION = void 0;
-exports.HATCHET_VERSION = '1.35.0';
+exports.HATCHET_VERSION = '1.36.0';

package/v1/examples/durable-streams/publish-latency.d.ts

@@ -1,15 +0,0 @@
-import { HatchetClient } from '../..';
-export type PublishLatencyStats = {
-    count: number;
-    avgMs: number;
-    p50Ms: number;
-    p99Ms: number;
-    maxMs: number;
-    throughputPerSec: number;
-};
-/**
- * Times `count` sequential publishes, after `warmupCount` unmeasured ones
- * absorb connection setup and topic creation.
- */
-export declare function measurePublishLatency(hatchet: HatchetClient, topic: string, payload: Uint8Array | string, count: number, warmupCount?: number): Promise<PublishLatencyStats>;
-export declare function formatPublishLatency(stats: PublishLatencyStats): string;

package/v1/examples/durable-streams/publish-latency.js

@@ -1,50 +0,0 @@
-"use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.measurePublishLatency = measurePublishLatency;
-exports.formatPublishLatency = formatPublishLatency;
-const perf_hooks_1 = require("perf_hooks");
-function percentile(sorted, p) {
-    return sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * p))];
-}
-/**
- * Times `count` sequential publishes, after `warmupCount` unmeasured ones
- * absorb connection setup and topic creation.
- */
-function measurePublishLatency(hatchet_1, topic_1, payload_1, count_1) {
-    return __awaiter(this, arguments, void 0, function* (hatchet, topic, payload, count, warmupCount = 5) {
-        for (let i = 0; i < warmupCount; i += 1) {
-            yield hatchet.streams.publish(topic, payload);
-        }
-        const durationsMs = [];
-        const start = perf_hooks_1.performance.now();
-        for (let i = 0; i < count; i += 1) {
-            const before = perf_hooks_1.performance.now();
-            yield hatchet.streams.publish(topic, payload);
-            durationsMs.push(perf_hooks_1.performance.now() - before);
-        }
-        const totalMs = perf_hooks_1.performance.now() - start;
-        const sorted = [...durationsMs].sort((a, b) => a - b);
-        return {
-            count,
-            avgMs: durationsMs.reduce((sum, d) => sum + d, 0) / count,
-            p50Ms: percentile(sorted, 0.5),
-            p99Ms: percentile(sorted, 0.99),
-            maxMs: sorted[sorted.length - 1],
-            throughputPerSec: (count / totalMs) * 1000,
-        };
-    });
-}
-function formatPublishLatency(stats) {
-    const fmt = (ms) => ms.toFixed(2);
-    return (`avg=${fmt(stats.avgMs)}ms p50=${fmt(stats.p50Ms)}ms p99=${fmt(stats.p99Ms)}ms ` +
-        `max=${fmt(stats.maxMs)}ms throughput=${fmt(stats.throughputPerSec)}/s`);
-}