apache-iggy 0.10.1-edge.1 → 0.10.1-edge.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (159) hide show
  1. package/README.md +310 -27
  2. package/dist/client/client.config.d.ts +2 -0
  3. package/dist/client/client.config.js +13 -2
  4. package/dist/client/client.config.test.js +15 -2
  5. package/dist/client/client.connection-string.js +13 -1
  6. package/dist/client/client.connection-string.test.js +43 -1
  7. package/dist/client/client.connection.d.ts +33 -135
  8. package/dist/client/client.connection.js +190 -453
  9. package/dist/client/client.connection.test.js +244 -669
  10. package/dist/client/client.d.ts +4 -35
  11. package/dist/client/client.js +8 -101
  12. package/dist/client/client.operation.d.ts +120 -0
  13. package/dist/client/client.operation.js +428 -0
  14. package/dist/client/client.operation.test.d.ts +2 -0
  15. package/dist/client/client.operation.test.js +285 -0
  16. package/dist/client/client.socket.d.ts +92 -209
  17. package/dist/client/client.socket.js +1205 -981
  18. package/dist/client/client.socket.test.js +4883 -667
  19. package/dist/client/client.type.d.ts +65 -22
  20. package/dist/client/index.d.ts +1 -0
  21. package/dist/client/index.js +1 -0
  22. package/dist/client.e2e.js +3 -5
  23. package/dist/e2e/metadata-changes.e2e.d.ts +2 -0
  24. package/dist/{wire/stream/purge-stream.command.js → e2e/metadata-changes.e2e.js} +10 -20
  25. package/dist/e2e/metadata-changes.suite.d.ts +17 -0
  26. package/dist/e2e/metadata-changes.suite.js +328 -0
  27. package/dist/e2e/resource-options.e2e.d.ts +2 -0
  28. package/dist/e2e/resource-options.e2e.js +32 -0
  29. package/dist/e2e/resource-options.suite.d.ts +15 -0
  30. package/dist/e2e/resource-options.suite.js +272 -0
  31. package/dist/e2e/tcp.cluster.e2e.js +1 -0
  32. package/dist/e2e/tcp.consumer-stream.e2e.js +4 -0
  33. package/dist/e2e/tcp.parallel.e2e.js +1 -5
  34. package/dist/e2e/tcp.stream.e2e.js +0 -3
  35. package/dist/e2e/tcp.system.e2e.js +6 -0
  36. package/dist/e2e/tcp.token.e2e.js +8 -2
  37. package/dist/e2e/tcp.topic.e2e.js +0 -3
  38. package/dist/e2e/tcp.user.e2e.js +46 -3
  39. package/dist/e2e/tls.system.e2e.js +24 -12
  40. package/dist/index.d.ts +2 -1
  41. package/dist/index.js +1 -1
  42. package/dist/stream/consumer-stream.js +97 -12
  43. package/dist/wire/cluster/cluster.type.d.ts +2 -2
  44. package/dist/wire/cluster/cluster.utils.d.ts +1 -0
  45. package/dist/wire/cluster/cluster.utils.js +4 -1
  46. package/dist/wire/command-set.d.ts +9 -7
  47. package/dist/wire/command-set.js +20 -7
  48. package/dist/wire/command.code.d.ts +4 -9
  49. package/dist/wire/command.code.js +10 -10
  50. package/dist/wire/command.utils.d.ts +3 -2
  51. package/dist/wire/command.utils.js +3 -2
  52. package/dist/wire/error.code.js +15 -1
  53. package/dist/wire/error.code.test.js +22 -1
  54. package/dist/wire/error.utils.d.ts +4 -2
  55. package/dist/wire/error.utils.js +6 -3
  56. package/dist/wire/identifier.utils.d.ts +2 -0
  57. package/dist/wire/identifier.utils.js +2 -0
  58. package/dist/wire/index.d.ts +1 -0
  59. package/dist/wire/index.js +1 -0
  60. package/dist/wire/message/header.type.d.ts +1 -1
  61. package/dist/wire/message/message-batch.test.js +11 -5
  62. package/dist/wire/message/poll-messages.command.d.ts +12 -1
  63. package/dist/wire/message/poll-messages.command.js +64 -16
  64. package/dist/wire/message/poll-messages.command.test.js +143 -5
  65. package/dist/wire/message/poll.utils.d.ts +4 -0
  66. package/dist/wire/message/poll.utils.js +6 -1
  67. package/dist/wire/message/send-messages.command.d.ts +3 -3
  68. package/dist/wire/message/send-messages.command.js +103 -3
  69. package/dist/wire/message/send-messages.command.test.js +113 -2
  70. package/dist/wire/offset/delete-offset.command.d.ts +11 -4
  71. package/dist/wire/offset/delete-offset.command.js +1 -1
  72. package/dist/wire/offset/get-offset.command.d.ts +1 -1
  73. package/dist/wire/offset/offset-write.command.test.d.ts +2 -0
  74. package/dist/wire/offset/offset-write.command.test.js +43 -0
  75. package/dist/wire/offset/offset.utils.js +10 -6
  76. package/dist/wire/offset/store-offset.command.d.ts +10 -0
  77. package/dist/wire/offset/store-offset.command.js +1 -1
  78. package/dist/wire/options.utils.d.ts +46 -0
  79. package/dist/wire/options.utils.js +57 -0
  80. package/dist/wire/partition/partition-route.d.ts +12 -0
  81. package/dist/wire/partition/partition-route.js +40 -0
  82. package/dist/wire/partition/partition-route.test.d.ts +2 -0
  83. package/dist/wire/partition/partition-route.test.js +42 -0
  84. package/dist/wire/session/login-with-token.command.d.ts +2 -2
  85. package/dist/wire/session/login-with-token.command.js +7 -2
  86. package/dist/wire/session/login.command.d.ts +2 -2
  87. package/dist/wire/session/login.command.js +7 -2
  88. package/dist/wire/stream/create-stream.command.d.ts +5 -2
  89. package/dist/wire/stream/create-stream.command.js +20 -2
  90. package/dist/wire/stream/index.d.ts +1 -1
  91. package/dist/wire/stream/index.js +1 -1
  92. package/dist/wire/stream/stream-options.d.ts +77 -0
  93. package/dist/wire/stream/stream-options.js +101 -0
  94. package/dist/wire/stream/stream-options.test.d.ts +2 -0
  95. package/dist/wire/stream/stream-options.test.js +103 -0
  96. package/dist/wire/stream/stream.utils.d.ts +9 -0
  97. package/dist/wire/stream/stream.utils.js +17 -3
  98. package/dist/wire/stream/update-stream.command.d.ts +23 -5
  99. package/dist/wire/stream/update-stream.command.js +10 -17
  100. package/dist/wire/system/describe-options.command.d.ts +23 -6
  101. package/dist/wire/system/describe-options.command.js +21 -2
  102. package/dist/wire/system/get-metadata-changes.command.d.ts +97 -0
  103. package/dist/wire/system/get-metadata-changes.command.js +170 -0
  104. package/dist/wire/system/get-metadata-changes.command.test.d.ts +2 -0
  105. package/dist/wire/system/get-metadata-changes.command.test.js +467 -0
  106. package/dist/wire/system/index.d.ts +4 -0
  107. package/dist/wire/system/index.js +4 -0
  108. package/dist/wire/system/metadata-changes.utils.d.ts +482 -0
  109. package/dist/wire/system/metadata-changes.utils.js +816 -0
  110. package/dist/wire/system/snapshot.command.d.ts +64 -0
  111. package/dist/wire/system/snapshot.command.js +76 -0
  112. package/dist/wire/system/snapshot.command.test.d.ts +2 -0
  113. package/dist/wire/system/snapshot.command.test.js +86 -0
  114. package/dist/wire/system/update-resource.command.d.ts +53 -0
  115. package/dist/wire/system/update-resource.command.js +130 -0
  116. package/dist/wire/system/update-resource.command.test.d.ts +2 -0
  117. package/dist/wire/system/update-resource.command.test.js +497 -0
  118. package/dist/wire/token/create-token.command.d.ts +1 -1
  119. package/dist/wire/token/create-token.command.js +1 -1
  120. package/dist/wire/token/create-token.command.test.js +6 -0
  121. package/dist/wire/topic/create-topic.command.d.ts +20 -12
  122. package/dist/wire/topic/create-topic.command.js +11 -4
  123. package/dist/wire/topic/index.d.ts +1 -2
  124. package/dist/wire/topic/index.js +1 -2
  125. package/dist/wire/topic/topic.utils.d.ts +14 -5
  126. package/dist/wire/topic/topic.utils.js +26 -11
  127. package/dist/wire/topic/topic.utils.test.js +13 -13
  128. package/dist/wire/topic/update-topic.command.d.ts +22 -15
  129. package/dist/wire/topic/update-topic.command.js +12 -34
  130. package/dist/wire/user/create-user.command.d.ts +12 -2
  131. package/dist/wire/user/create-user.command.js +7 -3
  132. package/dist/wire/user/index.d.ts +1 -0
  133. package/dist/wire/user/index.js +1 -0
  134. package/dist/wire/user/update-user.command.d.ts +25 -7
  135. package/dist/wire/user/update-user.command.js +18 -31
  136. package/dist/wire/user/user.utils.d.ts +2 -0
  137. package/dist/wire/user/user.utils.js +5 -3
  138. package/dist/wire/vsr/header.d.ts +23 -8
  139. package/dist/wire/vsr/header.js +32 -10
  140. package/dist/wire/vsr/header.test.js +5 -1
  141. package/dist/wire/vsr/index.d.ts +8 -2
  142. package/dist/wire/vsr/index.js +42 -8
  143. package/dist/wire/vsr/operation.d.ts +18 -6
  144. package/dist/wire/vsr/operation.js +46 -16
  145. package/dist/wire/vsr/operation.test.js +32 -8
  146. package/dist/wire/vsr/register.d.ts +10 -6
  147. package/dist/wire/vsr/register.js +24 -7
  148. package/dist/wire/vsr/register.test.js +28 -8
  149. package/dist/wire/vsr/reply.d.ts +4 -1
  150. package/dist/wire/vsr/reply.js +42 -7
  151. package/dist/wire/vsr/reply.test.js +11 -1
  152. package/dist/wire/vsr/session.d.ts +9 -10
  153. package/dist/wire/vsr/session.js +32 -23
  154. package/dist/wire/vsr/session.test.js +33 -7
  155. package/dist/wire/vsr/vsr.test.js +57 -6
  156. package/package.json +3 -4
  157. package/dist/wire/stream/purge-stream.command.d.ts +0 -22
  158. package/dist/wire/topic/purge-topic.command.d.ts +0 -24
  159. package/dist/wire/topic/purge-topic.command.js +0 -39
package/README.md CHANGED
@@ -22,7 +22,7 @@ note: previous works on node.js http client has been moved to [iggy-node-http-cl
22
22
  npm i --save apache-iggy@edge
23
23
  ```
24
24
 
25
- Use the `edge` package with server 0.9.0 or `edge`. Stable Node SDK 0.8.0 uses the older wire protocol. For a local server with the example credentials, follow the [example setup](../../examples/node/README.md#running-examples).
25
+ Use matching SDK and server builds from the coordinated VSR release. Older builds are incompatible. This release requires fresh server storage (`IGGY-METADATA-CHANGE-LOG-1`). Existing incompatible data is refused unchanged. For a local server with the example credentials, follow the [example setup](../../examples/node/README.md#running-examples).
26
26
 
27
27
  ## basic usage
28
28
 
@@ -51,29 +51,203 @@ Codes absent from the SDK command table use `Operation::NonReplicated` and
51
51
  carry the command code in the request header's reserved field. The server
52
52
  remains authoritative for classifying or rejecting extension commands.
53
53
 
54
- Sends encode `Partitioning.PartitionId`, `Partitioning.Balanced` or
55
- `Partitioning.MessageKey` in the payload. The server resolves the target
56
- partition at admission.
57
-
58
- VSR works over TCP and TLS. It restricts `Client` to one pooled connection because authentication, request sequencing, and consumer-group assignments belong to one consensus session. Configurations requesting more than one pooled connection fail before a socket is opened.
59
-
60
- VSR authentication translates the existing password and personal-access-token
61
- login APIs into the register handshake required by the consensus protocol. A
62
- disconnect or eviction invalidates the session, and later work must register a
63
- new session. Transient not-committed responses retry the exact encoded request
64
- within one bounded deadline. A disconnected mutation is never replayed under a
65
- new session.
54
+ Sends always carry an explicit partition id. The client resolves
55
+ `Partitioning.Balanced` round-robin and `Partitioning.MessageKey` as XXH32
56
+ (seed 0) of the key modulo the topic's partition count, as the Rust SDK does.
57
+ The count is cached per topic. The client reads it again after a refused send,
58
+ and after the client deletes a stream or a topic or creates or deletes
59
+ partitions. A raw `SendMessages` command without an explicit partition id
60
+ fails with error code 5 before the client sends it.
61
+
62
+ Partition commands carry a context containing the partition incarnation, owner
63
+ generation and metadata operation. Pass the context of an earlier poll reply to
64
+ keep an offset write or continuation tied to that history. Otherwise the client
65
+ captures the route context. Non-admission retries and uncertain-operation replay
66
+ preserve the first captured context, request identity and payload. Route refreshes
67
+ change the destination. Lifecycle error 88 retries with a fresh ID and the same
68
+ context. A send without caller-supplied context can refresh history once after
69
+ error 87; explicit contexts return that error. A new call can capture newer
70
+ metadata progress.
71
+
72
+ Route discovery uses command 105. Sends, polls and offset reads/writes use the
73
+ partition primary; other commands use the metadata primary. One logical client
74
+ owns a shared session, a monotonic request counter and one multiplexed link per
75
+ replica. Healthy links survive primary changes. Each link dials the address
76
+ that its node advertises in the cluster roster. With a one-node roster, the
77
+ link dials the configured endpoint instead. A single node behind a port
78
+ mapping or a port-forward thus stays reachable. The legacy `poolSize` option
79
+ accepts only one logical client; it does not limit replica links or concurrency.
80
+ The public `Client._pool` property has been removed. Applications should use the
81
+ client methods directly; the client owns its replica connections.
82
+
83
+ A backup replica can miss a stream, topic or partition that the metadata primary
84
+ already created. When such a replica replies "not found" (error 1009, 1010,
85
+ 2010, 2011 or 3007) to a route lookup or a partition command, the call looks up
86
+ the route again and retries. If the deadline ends the call first, it fails with
87
+ error 58. Only a "not found" reply from the metadata primary is final. A leader
88
+ change can make the client's view of the metadata primary out of date. If the
89
+ client believes that the replica that replied is the metadata primary, it reads
90
+ the cluster roster from that replica. If the client has no link to the metadata
91
+ primary, it also reads that roster. If that roster names this replica as the
92
+ metadata primary, the reply is final.
93
+
94
+ VSR authentication translates password and personal-access-token login into the
95
+ Register handshake. Replacement links bind the existing session. Confirmed
96
+ session expiry requires a new Register; an uncertain mutation is never replayed
97
+ under that new session. An eviction that ends the session does not prove the
98
+ outcome of a mutation that the client already wrote. That call ends with
99
+ `OperationUncertainError`. A stale client eviction drops only its link, because
100
+ the server keeps the session. The client binds the same session on a new link,
101
+ as after any lost link, so retained operations and group membership stay.
102
+ Explicit relogin logs out the previous session first.
103
+
104
+ The client signs in by itself as the last sign-in of the caller, or with the
105
+ configured `credentials` while the caller never signed in. After a logout, or
106
+ after the server refuses those credentials, it does not sign in by itself, and
107
+ a call fails with status 40 until the caller signs in again. A sign-in or
108
+ logout of the caller ends each earlier call that has to wait for a new session:
109
+ with status 40 if it sent nothing, or with `OperationUncertainError` if its
110
+ outcome is unknown. An auto-commit or Next poll, which has no operation ID, ends with status 57 instead. A typed send and a group poll can send more than one
111
+ command. If the caller signs in or logs out before the next command of such a
112
+ call, the call fails with status 40. Only a call that waits for the client's own sign-in goes on
113
+ under the new session. If the server refuses that sign-in, the call fails with
114
+ status 40, and its `cause` holds the refusal. A call whose deadline ends while
115
+ it waits for a session fails with status 58. So does a call that replicas refuse with 89 until its deadline.
116
+
117
+ A reply with status 30 (`StaleClient`) says that the server no longer attaches
118
+ the link to the session. This happens, for example, after the user was deleted
119
+ or deactivated. A 30 refusal before commit fails a first attempt with that
120
+ error. After an earlier written attempt, it does not prove the outcome, so that
121
+ call ends with `OperationUncertainError`. The client closes that link and binds
122
+ the session on a new link, as after any lost link. A replica that replaced its
123
+ client table refuses a live session the same way, so the bind decides. A live
124
+ session stays, with its retained operations and group membership. An ended
125
+ session refuses the bind, and the client signs in again.
126
+
127
+ A committed 30 comes from the replicated session table, so that session ended.
128
+ It is a proved outcome, like any other committed rejection, and the client
129
+ drops the session at once and signs in again.
130
+
131
+ ### Mutation recovery and limits
132
+
133
+ Receipt access follows the server's [session and revocation policy](../../core/server/README.md#sessions-and-retry-receipts).
134
+ An unresolved operation cannot recover its receipt after its original session
135
+ ends, even after a new login or account reactivation. A fresh send can duplicate
136
+ that operation.
137
+
138
+ Each node counts the messages in its stream, topic and system stats. After a
139
+ successful send, a node that is not the partition primary shows the new count
140
+ only after the next prepare or commit broadcast. When no other write follows,
141
+ this delay is at most `cluster.commit_broadcast_interval` (500 ms by default).
142
+ See [metadata freshness and counters](../../core/server/README.md#metadata-freshness-and-counters).
143
+
144
+ Mutations share one FIFO lane per partition history, with a separate metadata
145
+ lane. An uncertain mutation remains retained and blocks later mutations in its
146
+ lane. `OperationUncertainError.operationId` and `LaneBlockedError.operationId`
147
+ are exact `bigint` values. Use `resolveOperation(operationId, { deadline })` to
148
+ replay the retained frame, where `deadline` is an absolute `Date.now()` timestamp
149
+ in milliseconds. The default operation budget is 30 seconds, including admission,
150
+ routing, link waits and retries.
151
+
152
+ Resolution requires correlated committed-result proof. A proved business
153
+ rejection returns the original error and releases the lane. An unproved denial
154
+ keeps an uncertain operation retained. The client keeps the proved outcomes of
155
+ its last `maxPendingOperations` replays, also across a session reset. If a
156
+ resolver already proved the outcome, a later `resolveOperation` of the same ID
157
+ returns it without a request. A `resolveOperation` call that waits for another
158
+ replay of the same ID and reaches its deadline fails with
159
+ `OperationUncertainError`.
160
+
161
+ `abandonOperation(operationId)` releases the client lane without proving that
162
+ the mutation failed. A replay that is already running keeps the lane until it
163
+ ends, and the client keeps the outcome that it proves. Logout, session reset and
164
+ destruction discard retained operations with their outcomes still unknown.
165
+ `OperationNotRetainedError` does not establish the outcome: a reset or an
166
+ abandon came before a proved outcome, or the proved outcome aged out.
167
+
168
+ **Breaking change:** `Client.destroy()` aborts in-flight calls immediately and
169
+ discards retained operations. It does not wait for outstanding calls to finish.
170
+ Await those calls before destroying the client when their results are needed.
171
+
172
+ Auto-commit and Next polls hold their partition lane only during the active call.
173
+ Error 57, a lost reply or another unknown outcome ends the call immediately with status 57 and no
174
+ retained operation ID. These polls never replay after uncertainty. Known
175
+ non-admission errors 58/89 remain retryable; pure reads may retry lost replies.
176
+ Before uncertainty, lifecycle error 88 retries with a fresh ID within the same
177
+ deadline. After uncertainty, only committed-result proof can release the lane.
178
+
179
+ Configure `maxPendingOperations` (default 256) and `maxPendingOperationBytes`
180
+ (default 64 MiB) on the client. Admission waits before allocating an ID or freezing
181
+ a mutation frame. Both limits must be positive. The byte budget includes the
182
+ 256-byte header and body; smaller positive budgets reject mutations that cannot
183
+ fit while reads and login remain available. Oversized frames fail locally.
184
+ Completion, abandonment and reset release the operation slot. Writer aliases keep bytes charged until they release
185
+ the frame. These limits cover admitted active/retained frames, not caller buffers,
186
+ queued API input snapshots, or read/control bodies. `sendBinaryRequest` preserves
187
+ its immediate payload snapshot before awaiting a custom provider.
188
+
189
+ An explicit sign-in uses its caller's credentials and supersedes automatic or
190
+ queued sign-ins. A failed explicit attempt, including invalid input, timeout or
191
+ cancellation, withdraws credentials from automatic sign-in until another explicit
192
+ sign-in succeeds. This also discards a session whose Register succeeded when a
193
+ later step of the explicit sign-in fails. Earlier calls keep their original
194
+ lineage; only SDK recovery continues them under a replacement session.
195
+
196
+ An established session has its own lifetime. Password or PAT changes and PAT
197
+ expiry do not end it. Logout, session lease expiry, or user deactivation end the
198
+ session and prevent its bind proof from restoring it.
66
199
 
67
200
  The client pings every `heartbeatInterval` milliseconds, 5000 by default, which
68
201
  keeps an idle session alive when the server's `[heartbeat]` eviction is enabled.
69
202
  `heartbeatInterval` also accepts a duration expression such as `"10s"` or
70
203
  `"1h 30m"`, like the Rust SDK.
71
- With the default heartbeat settings, a group member becomes eligible for
72
- eviction after 36 s of silence (1.2 x the 30 s interval); the verifier checks
73
- once per interval. Raising the client interval past that window, or setting it
74
- to 0 to disable client heartbeats, exposes an idle consumer-group member to
75
- eviction; a connection holding no group membership is left alone. Any other
76
- unusable value is rejected instead of silently disabling the heartbeat.
204
+ With the default heartbeat settings, a client that sends nothing for 36 s
205
+ (1.2 x the 30 s interval) stops renewing its session, and the server ends the
206
+ session about 36 s later. The verifier checks once per interval and also evicts
207
+ the connection of a consumer-group member that is silent past 36 s. If the
208
+ client interval exceeds 36 s, the verifier evicts an idle group member. If it
209
+ exceeds about 72 s, or is 0 to disable client heartbeats, any idle session
210
+ expires. Any other unusable value is rejected instead of silently disabling the
211
+ heartbeat.
212
+
213
+ A replica link redials on its own after it drops, with jittered exponential
214
+ backoff. `reconnect.maxRetries` limits one burst of redials, not the life of
215
+ the client. While the client stays signed in, each heartbeat restarts a link
216
+ that used its budget, with a fresh budget. A request does not redial a link. It
217
+ retries within its deadline until the link returns. If the deadline ends before
218
+ the request was sent and no replica refused it, it fails with status 58.
219
+ `destroy()` and logout stop the restarts.
220
+
221
+ When every link used its budget, a request fails with `NotConnectedError` and
222
+ does not wait for its deadline. When no endpoint accepts a dial before its retry
223
+ budget or its deadline ends, a sign-in also fails with `NotConnectedError`, and
224
+ its `cause` holds the last dial error. In both cases a later call can succeed,
225
+ because the heartbeat restarts the links and each sign-in dials again. If
226
+ reconnection or heartbeats are disabled, nothing restarts a link that used its
227
+ budget, so the request fails with a plain `Error` instead.
228
+
229
+ When a poll fails with status 30, 40, 57, 58 or 89, `singleConsumerStream` and
230
+ `groupConsumerStream` poll again after `interval`. They also poll again after a
231
+ poll deadline, after a poll with unknown progress, after `NotConnectedError` and
232
+ after `ConnectionClosedError`. That error reports a lost connected socket. The
233
+ client binds the session again, signs in again,
234
+ reconnects or routes again after these on its own. Any other error ends the
235
+ stream. The server can also refuse a sign-in with status 40, for example for an
236
+ inactive user. So after a 40 the stream waits for the next sign-in, and a
237
+ refused sign-in ends the stream. If the reply of an auto-commit poll is lost,
238
+ the stream can skip that batch.
239
+
240
+ `singleConsumerStream` and `groupConsumerStream` sign in before they return the
241
+ stream, and `groupConsumerStream` also joins its group. If the sign-in or the
242
+ join fails, the call destroys its client and rejects with that error. If no
243
+ endpoint accepts a dial before the retry budget or the 30 second sign-in
244
+ deadline ends, the error is `NotConnectedError`. When a stream closes, it
245
+ destroys its client. A group stream that is still signed in first leaves its
246
+ group, so that the server gives its partitions to the other members at once. If
247
+ the leave fails or takes more than 5 seconds, the membership ends when the
248
+ session lease expires. If the client of a destroyed stream has no session, or
249
+ loses it, the stream destroys the client at once. It does not wait for a poll
250
+ or a sign-in that still runs.
77
251
 
78
252
  `sendBinaryRequest(code, payload)` sends an arbitrary command code. Known replicated commands use their registered operation, while unknown codes reach the server as non-replicated requests and are rejected by servers that do not register them.
79
253
 
@@ -89,7 +263,7 @@ try {
89
263
  }
90
264
  ```
91
265
 
92
- The client includes its npm package version and the binary protocol crate
266
+ The client includes its npm package version and the binary wire protocol
93
267
  version in VSR registration. An incompatible server rejects registration with
94
268
  a protocol-version error instead of accepting a mismatched wire contract.
95
269
 
@@ -122,7 +296,8 @@ const stats = await client.system.getStats();
122
296
  Supported schemes are `iggy://` (TCP, default) and `iggy+tcp://`. Credentials
123
297
  are `username:password` or a single personal access token. Options mirror the
124
298
  other SDKs: `tls`, `tls_domain`, `tls_ca_file`, `reconnection_retries`,
125
- `reconnection_interval`, `heartbeat_interval` and `nodelay`. `reestablish_after`
299
+ `reconnection_interval`, `heartbeat_interval`, `nodelay`, `max_pending_operations`
300
+ and `max_pending_operation_bytes`. `reestablish_after`
126
301
  is accepted for format compatibility but has no Node equivalent.
127
302
 
128
303
  note: `SimpleClient` does not accept a connection string: it wraps an existing
@@ -144,14 +319,120 @@ Durations accept the same expressions as the Rust SDK, for example `500ms`,
144
319
  `0`, `unlimited`, `disabled` and `none` map to zero. Unit-less numbers such as
145
320
  `5` are rejected.
146
321
 
322
+ ### resource options
323
+
324
+ A stream can carry its own data encryption key. Build it with `StreamEncryptionKey.from` from exactly 32 random bytes, or from their canonical padded base64, and pass it to `stream.create` as `encryptionKey`. Never derive the key from a password, because its fingerprint is readable by anyone who can read the stream. A key that is not 32 bytes or canonical base64 is refused with error 70 before anything is sent.
325
+
326
+ A `StreamEncryptionKey` keeps its own copy of the bytes and prints only its fingerprint through `util.inspect`, `JSON.stringify` and string conversion, so create arguments carrying it can be logged. Entries passed through `options` print as given, so an `encryption_key` entry there is refused with error 70 before anything is sent.
327
+
328
+ The key travels inside the create request over whatever transport the client uses, so connect over TLS when the key must stay confidential. The server stores the key wrapped and never returns it: reads report only `encryptionKeySource`, `encryptionKeyFingerprint` and `encryptionKekFingerprints`. Without a key, a server with encryption enabled generates one for the stream and reports the source `server`.
329
+
330
+ ```ts
331
+ import { randomBytes } from "node:crypto";
332
+ import { Client, StreamEncryptionKey } from "apache-iggy";
333
+
334
+ const client = new Client({
335
+ transport: "TLS",
336
+ options: { host: "localhost", port: 8090 },
337
+ credentials: { username: "iggy", password: "iggy" },
338
+ });
339
+
340
+ const key = StreamEncryptionKey.from(randomBytes(32));
341
+ const stream = await client.stream.create({ name: "orders", encryptionKey: key });
342
+ console.log(stream.encryptionKeySource); // "stream"
343
+ console.log(stream.encryptionKeyFingerprint === key.fingerprint()); // true
344
+ ```
345
+
346
+ `stream.update`, `topic.update` and `user.update` change only the keys they name and leave every other setting as it is, so an update without `name` keeps the name. `reset` returns a key to the server default. Use the `StreamOptionKey`, `TopicOptionKey` and `UserOptionKey` constants to name keys.
347
+
348
+ Only the topic keys `message_expiry` and `max_topic_size` can be reset. Settings fixed at create, such as the topic compression algorithm, the partition resize policy or the stream key, have no update field.
349
+
350
+ The SDK checks the typed fields and `reset` before sending, in the server's order and with its error codes (4041 for an unknown key, 4105 for a read-only key, 4103 for a create-only key, 4104 for a key without a default, then the name, username or status error). For a key this build has no field or constant for, pass `options` entries and `rawReset` names. The SDK sends those as they are and the server decides, so a refusal comes back from the server.
351
+
352
+ The server returns 4108 when a non-root caller tries to rename root or change root's status, or when root tries to deactivate itself.
353
+
354
+ ```ts
355
+ await client.topic.update({
356
+ streamId: "orders",
357
+ topicId: "events",
358
+ messageExpiry: 7_200_000_000n,
359
+ reset: ["max_topic_size"],
360
+ });
361
+ ```
362
+
363
+ Reads return every setting in `options` (keys the client set) and `derivedOptions` (keys the server filled). `u64` values stay `bigint`, so values above 2^53 keep their precision. `system.describeOptions({ scope: OptionsScope.Stream })` lists the keys a server accepts per scope, with `updatable`, `resettable`, `secret`, `readOnly` and `critical`.
364
+
365
+ ### metadata changes
366
+
367
+ `system.getMetadataChanges` reads the node's change log: one record per public result of a committed metadata operation, such as a stream created, a topic renamed, a user's permissions changed or a session closed. A mirror keeps a cursor, asks for the records after it, and stores the returned `next` cursor. The call needs root or the global `manage_servers` permission.
368
+
369
+ ```ts
370
+ import { setTimeout as sleep } from "node:timers/promises";
371
+ import {
372
+ Client, MetadataChangeStart, MetadataChangeStatus,
373
+ type MetadataChangeGroup, type MetadataCurrentPageToken,
374
+ } from "apache-iggy";
375
+
376
+ const follow = async (client: Client, groups: MetadataChangeGroup[]) => {
377
+ let start: MetadataChangeStart = MetadataChangeStart.Earliest;
378
+ for (;;) {
379
+ const page = await client.system.getMetadataChanges({ start, groups });
380
+ switch (page.status) {
381
+ case MetadataChangeStatus.Ok:
382
+ for (const record of page.records)
383
+ console.log(record.op, record.kind, record.actorUserId, record.body);
384
+ if (page.next) start = MetadataChangeStart.After(page.next);
385
+ if (!page.hasMore) await sleep(1000);
386
+ break;
387
+ case MetadataChangeStatus.Behind:
388
+ await sleep(100);
389
+ break;
390
+ default:
391
+ start = await rebuild(client, groups);
392
+ }
393
+ }
394
+ };
395
+
396
+ /** Pages the current state, then returns the position to follow from. */
397
+ const rebuild = async (client: Client, groups: MetadataChangeGroup[]) => {
398
+ let pageToken: MetadataCurrentPageToken | undefined;
399
+ for (;;) {
400
+ const page = await client.system.getMetadataChanges({
401
+ start: MetadataChangeStart.Current(pageToken), groups,
402
+ });
403
+ if (page.status === MetadataChangeStatus.Behind) {
404
+ await sleep(100);
405
+ continue;
406
+ }
407
+ if (page.status !== MetadataChangeStatus.Ok) {
408
+ pageToken = undefined;
409
+ continue;
410
+ }
411
+ for (const item of page.items)
412
+ console.log(item.group, item.body);
413
+ if (page.currentPageToken) pageToken = page.currentPageToken;
414
+ else if (page.next) return MetadataChangeStart.After(page.next);
415
+ else pageToken = undefined;
416
+ }
417
+ };
418
+ ```
419
+
420
+ `expired` means the log no longer holds the history after the cursor, and `reset` means the cursor belongs to another history, for example after the data directory was rebuilt. Either way the mirror rebuilds from current state pages and then follows the changes after the anchor they return.
421
+
422
+ Current pages need the parent groups of every selected group, for example `topics` with `streams`. Keep a rebuilt state private until a change page answers `ok` with `hasMore` false and a `headOp` at or above the last current page's.
423
+
424
+ Records narrow by `kind` and items by `group`, so `record.kind === "stream_created"` types `record.body` as a stream. A group or kind newer than this build arrives as `"unknown"` with the raw body and the wire codes in `groupCode` and `kindCode`, while a record of a newer kind keeps its known `group`, so a mirror that cannot apply it can stop without advancing its cursor. Ops, timestamps and 128-bit ids are `bigint`.
425
+
426
+ A query that breaks the request rules, such as a limit outside 1 to 1000 or a time bound with a latest or current start, is refused with error 3 before anything is sent. `MetadataChangeCursor` and `MetadataCurrentPageToken` print and parse as the `mc1_` and `cs1_` text forms the HTTP API and the CLI use, and `JSON.stringify` writes them as that text.
427
+
147
428
  ## use sources
148
429
 
149
- Cluster auto-commit polling over TCP/TLS keeps group membership on the coordinator
150
- and uses separate connections to partition primaries. It requires server support
151
- for binary commands 14, 103 and 104. Pause binary auto-commit consumers for the
152
- whole upgrade: upgrade every server first, then the SDKs, and restart consumers
153
- so they rejoin their groups. Older SDKs can lose membership when a backup refuses
154
- an offset commit; the new SDK does not fall back to legacy polling.
430
+ Cluster polling uses command 15 (`BindSession`), route discovery 105 and poll
431
+ 100. Commands 14, 103, 104 and 123 are retired. Pause consumers for the coordinated
432
+ server/SDK upgrade and restart them to rejoin their groups. There is no fallback
433
+ to retired polling commands.
434
+
435
+ Stream, topic and user updates use the generic update command 16. The former update commands 35, 204 and 304 are retired.
155
436
 
156
437
  ### Install
157
438
 
@@ -180,6 +461,8 @@ npm run test:unit
180
461
 
181
462
  e2e test expect an iggy-server at tcp://127.0.0.1:8090
182
463
 
464
+ The resource options tests also expect the server to run with encryption enabled and a config key (`IGGY_ENCRYPTION_ENABLED=true` and `IGGY_ENCRYPTION_KEY` set to the base64 of 32 bytes), so a stream created without a key reports a server-generated one. The keyed metadata changes case registered from the same file needs the config key too.
465
+
183
466
  ```bash
184
467
  npm run test:e2e
185
468
  ```
@@ -1,5 +1,7 @@
1
1
  import type { ClientConfig, ClientConfigOrString } from './client.type.js';
2
2
  export declare const DEFAULT_MAX_RESPONSE_FRAME_SIZE: number;
3
+ export declare const DEFAULT_MAX_PENDING_OPERATIONS = 256;
4
+ export declare const DEFAULT_MAX_PENDING_OPERATION_BYTES: number;
3
5
  /**
4
6
  * The server evicts a connection silent for 36 s (1.2 x its 30 s heartbeat
5
7
  * interval). 5 s matches the other SDKs and survives several skipped pings.
@@ -16,7 +16,10 @@
16
16
  // under the License.
17
17
  import { MAX_U32 } from '../constant.js';
18
18
  import { parseConnectionString } from './client.connection-string.js';
19
+ import { HEADER_SIZE } from '../wire/vsr/header.js';
19
20
  export const DEFAULT_MAX_RESPONSE_FRAME_SIZE = 64 * 1024 * 1024;
21
+ export const DEFAULT_MAX_PENDING_OPERATIONS = 256;
22
+ export const DEFAULT_MAX_PENDING_OPERATION_BYTES = 64 * 1024 * 1024;
20
23
  /**
21
24
  * The server evicts a connection silent for 36 s (1.2 x its 30 s heartbeat
22
25
  * interval). 5 s matches the other SDKs and survives several skipped pings.
@@ -29,10 +32,16 @@ export const normalizeClientConfig = (config) => {
29
32
  config = parseConnectionString(config);
30
33
  const maxResponseFrameSize = config.maxResponseFrameSize ?? DEFAULT_MAX_RESPONSE_FRAME_SIZE;
31
34
  if (!Number.isSafeInteger(maxResponseFrameSize) ||
32
- maxResponseFrameSize < 256)
35
+ maxResponseFrameSize < HEADER_SIZE)
33
36
  throw new TypeError('maxResponseFrameSize must be a safe integer of at least 256 bytes');
37
+ const maxPendingOperations = config.maxPendingOperations ?? DEFAULT_MAX_PENDING_OPERATIONS;
38
+ const maxPendingOperationBytes = config.maxPendingOperationBytes ?? DEFAULT_MAX_PENDING_OPERATION_BYTES;
39
+ if (!Number.isSafeInteger(maxPendingOperations) || maxPendingOperations <= 0)
40
+ throw new TypeError('maxPendingOperations must be a positive safe integer');
41
+ if (!Number.isSafeInteger(maxPendingOperationBytes) || maxPendingOperationBytes <= 0)
42
+ throw new TypeError('maxPendingOperationBytes must be a positive safe integer');
34
43
  if ((config.poolSize?.min ?? 1) > 1 || (config.poolSize?.max ?? 1) > 1)
35
- throw new TypeError('VSR clients currently support exactly one pooled connection');
44
+ throw new TypeError('poolSize supports exactly one logical client; replica links are managed independently');
36
45
  // Only 0 disables the heartbeat. Anything else unusable has to throw here:
37
46
  // the default below is nullish-only, so a value that is negative, fractional
38
47
  // or above MAX_HEARTBEAT_INTERVAL reaches setInterval, which clamps it to
@@ -65,6 +74,8 @@ export const normalizeClientConfig = (config) => {
65
74
  ...config,
66
75
  options: { ...config.options },
67
76
  maxResponseFrameSize,
77
+ maxPendingOperations,
78
+ maxPendingOperationBytes,
68
79
  heartbeatInterval,
69
80
  poolSize: { min: 1, max: 1 }
70
81
  };
@@ -73,13 +73,26 @@ describe('normalizeClientConfig', () => {
73
73
  reconnect: { enabled: true, interval: 1000, maxRetries }
74
74
  }), /reconnect\.maxRetries/);
75
75
  });
76
- it('restricts the client to one pooled connection', () => {
76
+ it('preserves the one-logical-client pool option', () => {
77
77
  const normalized = normalizeClientConfig(config());
78
78
  assert.deepEqual(normalized.poolSize, { min: 1, max: 1 });
79
79
  assert.throws(() => normalizeClientConfig({
80
80
  ...config(),
81
81
  poolSize: { max: 2 }
82
- }), /exactly one pooled connection/);
82
+ }), /exactly one logical client/);
83
+ });
84
+ it('applies and validates shared mutation admission limits', () => {
85
+ const defaults = normalizeClientConfig(config());
86
+ assert.equal(defaults.maxPendingOperations, 256);
87
+ assert.equal(defaults.maxPendingOperationBytes, 64 * 1024 * 1024);
88
+ const bounded = normalizeClientConfig({ ...config(), maxPendingOperations: 1, maxPendingOperationBytes: 256 });
89
+ assert.equal(bounded.maxPendingOperations, 1);
90
+ assert.equal(bounded.maxPendingOperationBytes, 256);
91
+ for (const maxPendingOperations of [0, -1, 1.5, Number.NaN, Number.MAX_SAFE_INTEGER + 1])
92
+ assert.throws(() => normalizeClientConfig({ ...config(), maxPendingOperations }), /maxPendingOperations/);
93
+ assert.equal(normalizeClientConfig({ ...config(), maxPendingOperationBytes: 1 }).maxPendingOperationBytes, 1);
94
+ for (const maxPendingOperationBytes of [0, -1, 256.5, Number.NaN, Number.MAX_SAFE_INTEGER + 1])
95
+ assert.throws(() => normalizeClientConfig({ ...config(), maxPendingOperationBytes }), /maxPendingOperationBytes/);
83
96
  });
84
97
  it('supports TLS transport', () => {
85
98
  const normalized = normalizeClientConfig({
@@ -95,7 +95,7 @@ export const parseConnectionString = (connectionString) => {
95
95
  tls: false,
96
96
  reconnect: { ...CONNECTION_STRING_RECONNECT }
97
97
  };
98
- const { tls, reconnect, heartbeatInterval, ...transportOptions } = options;
98
+ const { tls, reconnect, heartbeatInterval, maxPendingOperations, maxPendingOperationBytes, ...transportOptions } = options;
99
99
  const config = {
100
100
  transport: tls ? 'TLS' : 'TCP',
101
101
  options: {
@@ -112,6 +112,10 @@ export const parseConnectionString = (connectionString) => {
112
112
  };
113
113
  if (heartbeatInterval !== undefined)
114
114
  config.heartbeatInterval = heartbeatInterval;
115
+ if (maxPendingOperations !== undefined)
116
+ config.maxPendingOperations = maxPendingOperations;
117
+ if (maxPendingOperationBytes !== undefined)
118
+ config.maxPendingOperationBytes = maxPendingOperationBytes;
115
119
  return config;
116
120
  };
117
121
  const parseConnectionOptions = (optionsString) => {
@@ -169,6 +173,14 @@ const parseConnectionOptions = (optionsString) => {
169
173
  case 'heartbeat_interval':
170
174
  parsed.heartbeatInterval = parseDuration(value);
171
175
  break;
176
+ case 'max_pending_operations':
177
+ case 'max_pending_operation_bytes': {
178
+ const limit = parseNumber(name, value);
179
+ if (!Number.isSafeInteger(limit) || limit <= 0)
180
+ throw new TypeError(`option "${name}" must be a positive safe integer`);
181
+ parsed[name === 'max_pending_operations' ? 'maxPendingOperations' : 'maxPendingOperationBytes'] = limit;
182
+ break;
183
+ }
172
184
  default:
173
185
  throw new TypeError(`unknown option "${name}"`);
174
186
  }
@@ -17,8 +17,9 @@
17
17
  import assert from 'node:assert/strict';
18
18
  import { describe, it } from 'node:test';
19
19
  import { MAX_U32 } from '../constant.js';
20
+ import { HEADER_SIZE } from '../wire/vsr/header.js';
20
21
  import { parseConnectionString, parseDuration } from './client.connection-string.js';
21
- import { DEFAULT_HEARTBEAT_INTERVAL, normalizeClientConfig } from './client.config.js';
22
+ import { DEFAULT_HEARTBEAT_INTERVAL, DEFAULT_MAX_PENDING_OPERATIONS, DEFAULT_MAX_PENDING_OPERATION_BYTES, normalizeClientConfig } from './client.config.js';
22
23
  describe('parseConnectionString', () => {
23
24
  it('parses the default scheme with password credentials', () => {
24
25
  assert.deepEqual(parseConnectionString('iggy://iggy:secret@127.0.0.1:8090'), {
@@ -62,6 +63,36 @@ describe('parseConnectionString', () => {
62
63
  heartbeatInterval: 10000
63
64
  });
64
65
  });
66
+ for (const scheme of ['iggy', 'iggy+tcp']) {
67
+ it(`maps mutation limits to client options for ${scheme}`, () => {
68
+ const config = parseConnectionString(`${scheme}://iggy:secret@localhost:8090` +
69
+ '?max_pending_operations=7&max_pending_operation_bytes=1024&nodelay=true');
70
+ assert.equal(config.maxPendingOperations, 7);
71
+ assert.equal(config.maxPendingOperationBytes, 1024);
72
+ assert.deepEqual(config.options, { host: 'localhost', port: 8090, noDelay: true });
73
+ const normalized = normalizeClientConfig(config);
74
+ assert.equal(normalized.maxPendingOperations, config.maxPendingOperations);
75
+ assert.equal(normalized.maxPendingOperationBytes, config.maxPendingOperationBytes);
76
+ });
77
+ }
78
+ for (const [option, field, minimum] of [
79
+ ['max_pending_operations', 'maxPendingOperations', 1],
80
+ ['max_pending_operation_bytes', 'maxPendingOperationBytes', 1]
81
+ ]) {
82
+ it(`accepts the minimum and safe-integer maximum for ${option}`, () => {
83
+ for (const value of [minimum, Number.MAX_SAFE_INTEGER]) {
84
+ const config = parseConnectionString(`iggy://iggy:secret@localhost:8090?${option}=${value}`);
85
+ assert.equal(config[field], value);
86
+ assert.equal(normalizeClientConfig(config)[field], value);
87
+ }
88
+ });
89
+ it(`rejects invalid and overflowing ${option}`, () => {
90
+ for (const value of ['', '0', '-1', '1.5', 'NaN', 'Infinity', '1e3',
91
+ String(minimum - 1), String(Number.MAX_SAFE_INTEGER + 1), '999999999999999999999']) {
92
+ assert.throws(() => parseConnectionString(`iggy://iggy:secret@localhost:8090?${option}=${value}`), TypeError, `accepted ${option}=${value}`);
93
+ }
94
+ });
95
+ }
65
96
  it('applies unlimited/1s reconnection defaults to partial options', () => {
66
97
  // retries alone keep the 1s interval; interval alone keeps unlimited.
67
98
  assert.deepEqual(parseConnectionString('iggy://iggy:secret@localhost:8090?reconnection_retries=3').reconnect, { enabled: true, interval: 1000, maxRetries: 3 });
@@ -137,6 +168,7 @@ describe('parseConnectionString', () => {
137
168
  'iggy://iggy:hunter2@localhost:8090?unknown=value',
138
169
  'iggy://iggy:hunter2@localhost:8090?tls=maybe',
139
170
  'iggy://iggy:hunter2@localhost:8090?reconnection_retries=three',
171
+ 'iggy://iggy:hunter2@localhost:8090?max_pending_operations=0',
140
172
  'iggy://iggy:hunter2@localhost:70000'
141
173
  ]) {
142
174
  try {
@@ -199,8 +231,18 @@ describe('normalizeClientConfig with connection strings', () => {
199
231
  password: 'secret'
200
232
  });
201
233
  assert.equal(normalized.heartbeatInterval, DEFAULT_HEARTBEAT_INTERVAL);
234
+ assert.equal(normalized.maxPendingOperations, DEFAULT_MAX_PENDING_OPERATIONS);
235
+ assert.equal(normalized.maxPendingOperationBytes, DEFAULT_MAX_PENDING_OPERATION_BYTES);
202
236
  assert.deepEqual(normalized.poolSize, { min: 1, max: 1 });
203
237
  });
238
+ it('preserves the unspecified mutation limit when only one is configured', () => {
239
+ const count = normalizeClientConfig('iggy://iggy:secret@localhost:8090?max_pending_operations=7');
240
+ assert.equal(count.maxPendingOperations, 7);
241
+ assert.equal(count.maxPendingOperationBytes, DEFAULT_MAX_PENDING_OPERATION_BYTES);
242
+ const bytes = normalizeClientConfig(`iggy://iggy:secret@localhost:8090?max_pending_operation_bytes=${HEADER_SIZE}`);
243
+ assert.equal(bytes.maxPendingOperations, DEFAULT_MAX_PENDING_OPERATIONS);
244
+ assert.equal(bytes.maxPendingOperationBytes, HEADER_SIZE);
245
+ });
204
246
  it('rejects reconnect intervals beyond the node timer ceiling', () => {
205
247
  // Parses to 3_600_000_000 ms; setInterval would clamp it back to 1 ms.
206
248
  assert.throws(() => normalizeClientConfig('iggy://iggy:secret@localhost:8090?reconnection_interval=1000h'), /reconnect\.interval/);