@subsquid/portal-client 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (131) hide show
  1. package/lib/client.d.ts +40 -18
  2. package/lib/client.d.ts.map +1 -1
  3. package/lib/client.js +199 -222
  4. package/lib/client.js.map +1 -1
  5. package/lib/query/common/data.d.ts +21 -0
  6. package/lib/query/common/data.d.ts.map +1 -0
  7. package/lib/query/common/data.js +3 -0
  8. package/lib/query/common/data.js.map +1 -0
  9. package/lib/query/common/query.d.ts +7 -0
  10. package/lib/query/common/query.d.ts.map +1 -0
  11. package/lib/query/common/query.js +3 -0
  12. package/lib/query/common/query.js.map +1 -0
  13. package/lib/query/evm/data.d.ts +159 -0
  14. package/lib/query/evm/data.d.ts.map +1 -0
  15. package/lib/query/evm/data.js +3 -0
  16. package/lib/query/evm/data.js.map +1 -0
  17. package/lib/query/evm/fields.d.ts +50 -0
  18. package/lib/query/evm/fields.d.ts.map +1 -0
  19. package/lib/query/evm/fields.js +3 -0
  20. package/lib/query/evm/fields.js.map +1 -0
  21. package/lib/query/evm/index.d.ts +4 -0
  22. package/lib/query/evm/index.d.ts.map +1 -0
  23. package/lib/query/evm/index.js +7 -0
  24. package/lib/query/evm/index.js.map +1 -0
  25. package/lib/query/evm/query.d.ts +54 -0
  26. package/lib/query/evm/query.d.ts.map +1 -0
  27. package/lib/query/evm/query.js +3 -0
  28. package/lib/query/evm/query.js.map +1 -0
  29. package/lib/query/evm/schema.d.ts +322 -0
  30. package/lib/query/evm/schema.d.ts.map +1 -0
  31. package/lib/query/evm/schema.js +224 -0
  32. package/lib/query/evm/schema.js.map +1 -0
  33. package/lib/query/index.d.ts +576 -8
  34. package/lib/query/index.d.ts.map +1 -1
  35. package/lib/query/index.js +15 -24
  36. package/lib/query/index.js.map +1 -1
  37. package/lib/query/solana/data.d.ts +115 -0
  38. package/lib/query/solana/data.d.ts.map +1 -0
  39. package/lib/query/solana/data.js +3 -0
  40. package/lib/query/solana/data.js.map +1 -0
  41. package/lib/query/solana/fields.d.ts +31 -0
  42. package/lib/query/solana/fields.d.ts.map +1 -0
  43. package/lib/query/solana/fields.js +3 -0
  44. package/lib/query/solana/fields.js.map +1 -0
  45. package/lib/query/solana/index.d.ts +4 -0
  46. package/lib/query/solana/index.d.ts.map +1 -0
  47. package/lib/query/solana/index.js +7 -0
  48. package/lib/query/solana/index.js.map +1 -0
  49. package/lib/query/solana/query.d.ts +74 -0
  50. package/lib/query/solana/query.d.ts.map +1 -0
  51. package/lib/query/solana/query.js +3 -0
  52. package/lib/query/solana/query.js.map +1 -0
  53. package/lib/query/solana/schema.d.ts +160 -0
  54. package/lib/query/solana/schema.d.ts.map +1 -0
  55. package/lib/query/solana/schema.js +130 -0
  56. package/lib/query/solana/schema.js.map +1 -0
  57. package/lib/query/substrate/data.d.ts +98 -0
  58. package/lib/query/substrate/data.d.ts.map +1 -0
  59. package/lib/query/substrate/data.js +3 -0
  60. package/lib/query/substrate/data.js.map +1 -0
  61. package/lib/query/substrate/fields.d.ts +22 -0
  62. package/lib/query/substrate/fields.d.ts.map +1 -0
  63. package/lib/query/substrate/fields.js +3 -0
  64. package/lib/query/substrate/fields.js.map +1 -0
  65. package/lib/query/substrate/index.d.ts +4 -0
  66. package/lib/query/substrate/index.d.ts.map +1 -0
  67. package/lib/query/substrate/index.js +7 -0
  68. package/lib/query/substrate/index.js.map +1 -0
  69. package/lib/query/substrate/query.d.ts +50 -0
  70. package/lib/query/substrate/query.d.ts.map +1 -0
  71. package/lib/query/substrate/query.js +3 -0
  72. package/lib/query/substrate/query.js.map +1 -0
  73. package/lib/query/substrate/schema.d.ts +102 -0
  74. package/lib/query/substrate/schema.d.ts.map +1 -0
  75. package/lib/query/substrate/schema.js +88 -0
  76. package/lib/query/substrate/schema.js.map +1 -0
  77. package/lib/query/type-util.d.ts +13 -0
  78. package/lib/query/type-util.d.ts.map +1 -0
  79. package/lib/query/type-util.js +3 -0
  80. package/lib/query/type-util.js.map +1 -0
  81. package/lib/query/util.d.ts +5 -0
  82. package/lib/query/util.d.ts.map +1 -0
  83. package/lib/query/util.js +23 -0
  84. package/lib/query/util.js.map +1 -0
  85. package/lib/util.d.ts +19 -0
  86. package/lib/util.d.ts.map +1 -0
  87. package/lib/util.js +57 -0
  88. package/lib/util.js.map +1 -0
  89. package/package.json +3 -3
  90. package/src/client.ts +261 -278
  91. package/src/query/common/data.ts +24 -0
  92. package/src/query/common/query.ts +6 -0
  93. package/src/query/evm/data.ts +182 -0
  94. package/src/query/evm/fields.ts +105 -0
  95. package/src/query/evm/index.ts +3 -0
  96. package/src/query/evm/query.ts +59 -0
  97. package/src/query/evm/schema.ts +277 -0
  98. package/src/query/index.ts +19 -36
  99. package/src/query/solana/data.ts +132 -0
  100. package/src/query/solana/fields.ts +42 -0
  101. package/src/query/solana/index.ts +3 -0
  102. package/src/query/solana/query.ts +89 -0
  103. package/src/query/solana/schema.ts +164 -0
  104. package/src/query/substrate/data.ts +101 -0
  105. package/src/query/substrate/fields.ts +30 -0
  106. package/src/query/substrate/index.ts +3 -0
  107. package/src/query/substrate/query.ts +60 -0
  108. package/src/query/substrate/schema.ts +114 -0
  109. package/src/query/type-util.ts +25 -0
  110. package/src/query/util.ts +23 -0
  111. package/src/util.ts +56 -0
  112. package/lib/query/common.d.ts +0 -56
  113. package/lib/query/common.d.ts.map +0 -1
  114. package/lib/query/common.js +0 -16
  115. package/lib/query/common.js.map +0 -1
  116. package/lib/query/evm.d.ts +0 -267
  117. package/lib/query/evm.d.ts.map +0 -1
  118. package/lib/query/evm.js +0 -245
  119. package/lib/query/evm.js.map +0 -1
  120. package/lib/query/solana.d.ts +0 -224
  121. package/lib/query/solana.d.ts.map +0 -1
  122. package/lib/query/solana.js +0 -121
  123. package/lib/query/solana.js.map +0 -1
  124. package/lib/query/substrate.d.ts +0 -173
  125. package/lib/query/substrate.d.ts.map +0 -1
  126. package/lib/query/substrate.js +0 -71
  127. package/lib/query/substrate.js.map +0 -1
  128. package/src/query/common.ts +0 -83
  129. package/src/query/evm.ts +0 -677
  130. package/src/query/solana.ts +0 -438
  131. package/src/query/substrate.ts +0 -288
package/src/client.ts CHANGED
@@ -1,16 +1,8 @@
1
1
  import {HttpBody, HttpClient, HttpClientOptions, HttpError, HttpResponse, RequestOptions} from '@subsquid/http-client'
2
- import {
3
- addErrorContext,
4
- createFuture,
5
- Future,
6
- last,
7
- unexpectedCase,
8
- wait,
9
- withErrorContext,
10
- } from '@subsquid/util-internal'
11
- import {Readable} from 'stream'
12
- import {GetBlock, evm, getBlockSchema, PortalBlock, PortalQuery, Query, solana, substrate} from './query'
2
+ import {addErrorContext, last, unexpectedCase, wait, withErrorContext} from '@subsquid/util-internal'
13
3
  import {cast} from '@subsquid/util-internal-validation'
4
+ import {Semaphore, Timer} from './util'
5
+ import {getQuery, type AnyQuery, type GetQueryBlock} from './query'
14
6
 
15
7
  export * from './query'
16
8
 
@@ -29,19 +21,19 @@ export interface PortalClientOptions {
29
21
 
30
22
  /**
31
23
  * Minimum number of bytes to return.
32
- * @default 10_485_760 (10MB)
24
+ * @deprecated not used
33
25
  */
34
26
  minBytes?: number
35
27
 
36
28
  /**
37
29
  * Maximum number of bytes to return.
38
- * @default minBytes
30
+ * @default 52_428_800 (50MB)
39
31
  */
40
32
  maxBytes?: number
41
33
 
42
34
  /**
43
35
  * Maximum time between stream data in milliseconds for return.
44
- * @default 300
36
+ * @default 500
45
37
  */
46
38
  maxIdleTime?: number
47
39
 
@@ -70,7 +62,6 @@ export interface PortalRequestOptions {
70
62
  export interface PortalStreamOptions {
71
63
  request?: Omit<PortalRequestOptions, 'abort'>
72
64
 
73
- minBytes?: number
74
65
  maxBytes?: number
75
66
  maxIdleTime?: number
76
67
  maxWaitTime?: number
@@ -78,10 +69,34 @@ export interface PortalStreamOptions {
78
69
  headPollInterval?: number
79
70
  }
80
71
 
72
+ export interface PortalStreamBaseResponse {
73
+ headNumber?: number
74
+ finalizedHeadNumber?: number
75
+ finalizedHeadHash?: string
76
+ }
77
+
78
+ export interface PortalStreamDataResponse extends PortalStreamBaseResponse {
79
+ type: 'data'
80
+ data: AsyncIterable<Uint8Array> | null
81
+ }
82
+
83
+ export interface PortalStreamNoDataResponse extends PortalStreamBaseResponse {
84
+ type: 'no-data'
85
+ }
86
+
87
+ export interface PortalStreamForkResponse extends PortalStreamBaseResponse {
88
+ type: 'fork'
89
+ previousBlocks: BlockRef[]
90
+ }
91
+
92
+ export type PortalStreamResponse = PortalStreamDataResponse | PortalStreamNoDataResponse | PortalStreamForkResponse
93
+
81
94
  export type PortalStreamData<B> = {
82
95
  blocks: B[]
83
- finalizedHead?: BlockRef
84
96
  meta: {
97
+ finalizedHeadNumber?: number
98
+ finalizedHeadHash?: string
99
+ headNumber?: number
85
100
  bytes: number
86
101
  }
87
102
  }
@@ -97,7 +112,6 @@ export class PortalClient {
97
112
  private url: URL
98
113
  private client: HttpClient
99
114
  private headPollInterval: number
100
- private minBytes: number
101
115
  private maxBytes: number
102
116
  private maxIdleTime: number
103
117
  private maxWaitTime: number
@@ -106,9 +120,8 @@ export class PortalClient {
106
120
  this.url = new URL(options.url)
107
121
  this.client = options.http instanceof HttpClient ? options.http : new HttpClient(options.http)
108
122
  this.headPollInterval = options.headPollInterval ?? 0
109
- this.minBytes = options.minBytes ?? 10 * 1024 * 1024
110
- this.maxBytes = options.maxBytes ?? this.minBytes
111
- this.maxIdleTime = options.maxIdleTime ?? 300
123
+ this.maxBytes = options.maxBytes ?? 50 * 1024 * 1024
124
+ this.maxIdleTime = options.maxIdleTime ?? 500
112
125
  this.maxWaitTime = options.maxWaitTime ?? 5_000
113
126
  }
114
127
 
@@ -122,92 +135,56 @@ export class PortalClient {
122
135
  return u.toString()
123
136
  }
124
137
 
125
- async getHead(options?: PortalRequestOptions): Promise<BlockRef | undefined> {
126
- const res = await this.request('GET', this.getDatasetUrl('head'), options)
138
+ async getHead(options?: PortalRequestOptions, finalized?: boolean): Promise<BlockRef | undefined> {
139
+ const res = await this.request('GET', this.getDatasetUrl(!finalized ? 'head' : 'finalized-head'), options)
127
140
  return res.body ?? undefined
128
141
  }
129
142
 
130
- async getFinalizedHead(options?: PortalRequestOptions): Promise<BlockRef | undefined> {
131
- const res = await this.request('GET', this.getDatasetUrl('finalized-head'), options)
132
- return res.body ?? undefined
143
+ getFinalizedHead(options?: PortalRequestOptions): Promise<BlockRef | undefined> {
144
+ return this.getHead(options, true)
133
145
  }
134
146
 
135
147
  /**
136
- * @deprecated
148
+ * Waits until the finalized head reaches or exceeds the specified block number.
149
+ * @param blockNumber - The block number to wait for finalization
150
+ * @param options - Request options including abort signal and poll interval
151
+ * @returns The finalized head once it reaches the target block number
137
152
  */
138
- async getFinalizedHeight(options?: PortalRequestOptions): Promise<number> {
139
- let {body} = await this.request<string>('GET', this.getDatasetUrl('finalized-stream/height'), options)
140
- let height = parseInt(body)
141
- return height
142
- }
143
-
144
- getFinalizedQuery<Q extends Query>(query: Q, options?: PortalRequestOptions): Promise<GetBlock<Q>[]> {
145
- // FIXME: is it needed or it is better to always use stream?
146
- return this.request<Buffer>('POST', this.getDatasetUrl(`finalized-stream`), {
147
- ...options,
148
- json: query,
149
- })
150
- .catch(
151
- withErrorContext({
152
- archiveQuery: query,
153
- })
154
- )
155
- .then((res) => {
156
- let blocks = res.body
157
- .toString('utf8')
158
- .trimEnd()
159
- .split('\n')
160
- .map((line) => JSON.parse(line))
161
- return blocks
162
- })
163
- }
153
+ async waitForFinalization(
154
+ blockNumber: number,
155
+ options?: PortalRequestOptions & {pollInterval?: number}
156
+ ): Promise<BlockRef> {
157
+ let {pollInterval = this.headPollInterval || 1000, ...requestOptions} = options ?? {}
158
+
159
+ while (true) {
160
+ options?.abort?.throwIfAborted()
161
+
162
+ let head = await this.getFinalizedHead(requestOptions)
163
+ if (head && head.number >= blockNumber) {
164
+ return head
165
+ }
164
166
 
165
- getQuery<Q extends PortalQuery = PortalQuery, R extends PortalBlock = PortalBlock>(
166
- query: Q,
167
- options?: PortalRequestOptions
168
- ): Promise<R[]> {
169
- // FIXME: is it needed or it is better to always use stream?
170
- return this.request<Buffer>('POST', this.getDatasetUrl(`stream`), {
171
- ...options,
172
- json: query,
173
- })
174
- .catch(
175
- withErrorContext({
176
- archiveQuery: query,
177
- })
178
- )
179
- .then((res) => {
180
- let blocks = res.body
181
- .toString('utf8')
182
- .trimEnd()
183
- .split('\n')
184
- .map((line) => JSON.parse(line))
185
- return blocks
186
- })
167
+ await wait(pollInterval, options?.abort)
168
+ }
187
169
  }
188
170
 
189
- getFinalizedStream<Q extends evm.Query | solana.Query | substrate.Query>(
171
+ getStream<Q extends AnyQuery>(
190
172
  query: Q,
191
- options?: PortalStreamOptions
192
- ): PortalStream<GetBlock<Q>> {
173
+ options?: PortalStreamOptions,
174
+ finalized?: boolean
175
+ ): PortalStream<GetQueryBlock<Q>> {
193
176
  return createPortalStream(query, this.getStreamOptions(options), async (q, o) =>
194
- this.getStreamRequest('finalized-stream', q, o)
177
+ this.getStreamRequest(!finalized ? 'stream' : 'finalized-stream', q, o)
195
178
  )
196
179
  }
197
180
 
198
- getStream<Q extends evm.Query | solana.Query | substrate.Query>(
199
- query: Q,
200
- options?: PortalStreamOptions
201
- ): PortalStream<GetBlock<Q>> {
202
- return createPortalStream(query, this.getStreamOptions(options), async (q, o) =>
203
- this.getStreamRequest('stream', q, o)
204
- )
181
+ getFinalizedStream<Q extends AnyQuery>(query: Q, options?: PortalStreamOptions): PortalStream<GetQueryBlock<Q>> {
182
+ return this.getStream(query, options, true)
205
183
  }
206
184
 
207
185
  private getStreamOptions(options?: PortalStreamOptions) {
208
186
  let {
209
187
  headPollInterval = this.headPollInterval,
210
- minBytes = this.minBytes,
211
188
  maxBytes = this.maxBytes,
212
189
  maxIdleTime = this.maxIdleTime,
213
190
  maxWaitTime = this.maxWaitTime,
@@ -216,7 +193,6 @@ export class PortalClient {
216
193
 
217
194
  return {
218
195
  headPollInterval,
219
- minBytes,
220
196
  maxBytes,
221
197
  maxIdleTime,
222
198
  maxWaitTime,
@@ -224,9 +200,13 @@ export class PortalClient {
224
200
  }
225
201
  }
226
202
 
227
- private async getStreamRequest(path: string, query: PortalQuery, options?: PortalRequestOptions) {
203
+ private async getStreamRequest(
204
+ path: string,
205
+ query: AnyQuery,
206
+ options?: PortalRequestOptions
207
+ ): Promise<PortalStreamResponse> {
228
208
  try {
229
- let res = await this.request<Readable | undefined>('POST', this.getDatasetUrl(path), {
209
+ let res = await this.request<AsyncIterable<Uint8Array> | undefined>('POST', this.getDatasetUrl(path), {
230
210
  ...options,
231
211
  json: query,
232
212
  stream: true,
@@ -236,28 +216,29 @@ export class PortalClient {
236
216
  })
237
217
  )
238
218
 
219
+ let headers = getPortalStreamHeaders(res.headers)
220
+
239
221
  switch (res.status) {
240
222
  case 200:
241
- let finalizedHead = getFinalizedHeadHeader(res.headers)
242
- let stream = res.body ? splitLines(res.body) : undefined
243
-
244
223
  return {
245
- finalizedHead,
246
- stream,
224
+ ...headers,
225
+ type: 'data',
226
+ data: res.body ?? null,
247
227
  }
248
228
  case 204:
249
229
  return {
250
- finalizedHead: getFinalizedHeadHeader(res.headers),
230
+ ...headers,
231
+ type: 'no-data',
251
232
  }
252
233
  default:
253
234
  throw unexpectedCase(res.status)
254
235
  }
255
236
  } catch (e: any) {
256
- if (isForkHttpError(e) && query.fromBlock != null && query.parentBlockHash != null) {
257
- e = new ForkException(e.response.body.previousBlocks, {
258
- number: query.fromBlock - 1,
259
- hash: query.parentBlockHash,
260
- })
237
+ if (isForkHttpError(e)) {
238
+ return {
239
+ type: 'fork',
240
+ previousBlocks: e.response.body.previousBlocks,
241
+ }
261
242
  }
262
243
 
263
244
  throw addErrorContext(e, {
@@ -284,78 +265,79 @@ function isForkHttpError(err: unknown): err is HttpError<{previousBlocks: BlockR
284
265
  return true
285
266
  }
286
267
 
287
- function createPortalStream<Q extends Query>(
268
+ function createPortalStream<Q extends AnyQuery>(
288
269
  query: Q,
289
270
  options: Required<PortalStreamOptions>,
290
- requestStream: (
291
- query: Q,
292
- options?: PortalRequestOptions
293
- ) => Promise<{finalizedHead?: BlockRef; stream?: AsyncIterable<string[]> | null | undefined}>
294
- ): PortalStream<GetBlock<Q>> {
271
+ requestStream: (query: Q, options?: PortalRequestOptions) => Promise<PortalStreamResponse>
272
+ ): PortalStream<GetQueryBlock<Q>> {
295
273
  let {headPollInterval, request, ...bufferOptions} = options
296
274
 
297
- let buffer = new PortalStreamBuffer<GetBlock<Q>>(bufferOptions)
275
+ let buffer = new PortalStreamBuffer<GetQueryBlock<Q>>(bufferOptions)
298
276
 
299
- let schema = getBlockSchema(query)
300
- let {fromBlock = 0, toBlock, parentBlockHash} = query
277
+ let [normalizedQuery, schema] = getQuery(query)
278
+ let {fromBlock = 0, toBlock, parentBlockHash} = normalizedQuery
301
279
 
302
280
  const ingest = async () => {
303
- if (buffer.signal.aborted) return
304
- if (toBlock != null && fromBlock > toBlock) return
305
-
306
- let res = await requestStream(
307
- {
308
- ...query,
309
- fromBlock,
310
- parentBlockHash,
311
- },
312
- {
313
- ...request,
314
- abort: buffer.signal,
281
+ while (!buffer.signal.aborted) {
282
+ if (toBlock != null && fromBlock > toBlock) break
283
+
284
+ let res = await requestStream(
285
+ {
286
+ ...normalizedQuery,
287
+ fromBlock,
288
+ parentBlockHash,
289
+ },
290
+ {
291
+ ...request,
292
+ abort: buffer.signal,
293
+ }
294
+ )
295
+
296
+ if (res.type === 'fork') {
297
+ throw new ForkException(fromBlock, parentBlockHash!, res.previousBlocks)
315
298
  }
316
- )
317
299
 
318
- const finalizedHead = res.finalizedHead
300
+ buffer.updateHead(res)
319
301
 
320
- // we are on head
321
- if (!('stream' in res)) {
322
- await buffer.put({blocks: [], meta: {bytes: 0}, finalizedHead})
323
- buffer.flush()
324
- if (headPollInterval > 0) {
325
- await wait(headPollInterval, buffer.signal)
302
+ if (res.type === 'no-data') {
303
+ buffer.flush()
304
+ if (headPollInterval > 0) {
305
+ await wait(headPollInterval, buffer.signal)
306
+ }
307
+ continue
326
308
  }
327
- return ingest()
328
- }
329
309
 
330
- // no data left on this range
331
- if (res.stream == null) return
310
+ if (!res.data) break
332
311
 
333
- try {
334
- for await (let data of res.stream) {
335
- buffer.signal.throwIfAborted()
312
+ try {
313
+ for await (let data of splitLines(res.data)) {
314
+ buffer.signal.throwIfAborted()
336
315
 
337
- let blocks: GetBlock<Q>[] = []
338
- let bytes = 0
316
+ let blocks: GetQueryBlock<Q>[] = []
317
+ let bytes = 0
339
318
 
340
- for (let line of data) {
341
- let block = cast(schema, JSON.parse(line))
342
- blocks.push(block)
343
- bytes += line.length
319
+ for (let line of data) {
320
+ let block = cast(schema, JSON.parse(line)) as GetQueryBlock<Q>
321
+ blocks.push(block)
322
+ bytes += line.length
344
323
 
345
- fromBlock = block.header.number + 1
346
- parentBlockHash = block.header.hash
324
+ fromBlock = block.header.number + 1
325
+ parentBlockHash = block.header.hash
326
+ }
327
+
328
+ await buffer.put(blocks, bytes)
347
329
  }
348
330
 
349
- await buffer.put({blocks, finalizedHead, meta: {bytes}})
350
- }
351
- } catch (err) {
352
- if (buffer.signal.aborted) return
353
- if (!isStreamAbortedError(err)) {
354
- throw err
331
+ buffer.flush()
332
+ } catch (err) {
333
+ if (buffer.signal.aborted) return
334
+ if (!isStreamAbortedError(err)) {
335
+ throw err
336
+ }
355
337
  }
356
338
  }
357
339
 
358
- return ingest()
340
+ buffer.flush()
359
341
  }
360
342
 
361
343
  ingest().then(
@@ -363,174 +345,170 @@ function createPortalStream<Q extends Query>(
363
345
  (err) => buffer.fail(err)
364
346
  )
365
347
 
366
- return buffer.iterate()
348
+ return buffer
367
349
  }
368
350
 
369
351
  class PortalStreamBuffer<B> {
370
- private buffer: PortalStreamData<B> | undefined
371
- private state: 'pending' | 'ready' | 'failed' | 'closed' = 'pending'
372
- private error: unknown
352
+ private blocks: B[] = []
353
+ private bytes = 0
373
354
 
374
- private readyFuture: Future<void> = createFuture()
375
- private takeFuture: Future<void> = createFuture()
376
- private putFuture: Future<void> = createFuture()
355
+ private headNumber?: number
356
+ private finalizedHeadNumber?: number
357
+ private finalizedHeadHash?: string
358
+ private headUpdated = false
377
359
 
378
- private idleTimeout: ReturnType<typeof setTimeout> | undefined
379
- private waitTimeout: ReturnType<typeof setTimeout> | undefined
360
+ private state: 'open' | 'failed' | 'closed' = 'open'
361
+ private error: unknown
380
362
 
381
- private minBytes: number
382
- private maxBytes: number
383
- private maxIdleTime: number
384
- private maxWaitTime: number
363
+ private waitTimeouted = false
364
+ private putMutex = new Semaphore(false)
365
+ private takeMutex = new Semaphore(true)
385
366
 
367
+ private maxBytes: number
368
+ private idleTimer: Timer
369
+ private waitTimer: Timer
386
370
  private abortController = new AbortController()
387
371
 
388
372
  get signal() {
389
373
  return this.abortController.signal
390
374
  }
391
375
 
392
- constructor(options: {maxWaitTime: number; maxBytes: number; maxIdleTime: number; minBytes: number}) {
393
- this.maxWaitTime = options.maxWaitTime
394
- this.minBytes = options.minBytes
395
- this.maxBytes = Math.max(options.maxBytes, options.minBytes)
396
- this.maxIdleTime = options.maxIdleTime
376
+ constructor(options: {maxWaitTime: number; maxBytes: number; maxIdleTime: number}) {
377
+ this.maxBytes = options.maxBytes
378
+ this.idleTimer = new Timer(options.maxIdleTime, () => this.flush())
379
+ this.waitTimer = new Timer(options.maxWaitTime, () => (this.waitTimeouted = true))
397
380
  }
398
381
 
399
382
  async take(): Promise<PortalStreamData<B> | undefined> {
400
- if (this.state === 'pending') {
401
- this.waitTimeout = setTimeout(() => this._ready(), this.maxWaitTime)
383
+ if (this.isTerminated()) {
384
+ return this.collect()
402
385
  }
403
386
 
404
- await Promise.all([this.readyFuture.promise(), this.putFuture.promise()])
405
-
406
- if (this.state === 'failed') {
407
- throw this.error
408
- }
387
+ this.waitTimer.start()
388
+ await this.putMutex.wait()
389
+ this.waitTimer.stop()
409
390
 
410
- let result = this.buffer
411
- this.buffer = undefined
391
+ let result = this.collect()
412
392
 
413
- if (this.state === 'closed') {
414
- return result
415
- }
416
-
417
- if (result == null) {
418
- throw new Error('Buffer is empty')
419
- }
420
-
421
- this.takeFuture.resolve()
422
-
423
- this.readyFuture = createFuture()
424
- this.putFuture = createFuture()
425
- this.takeFuture = createFuture()
426
- this.state = 'pending'
393
+ this.putMutex.unready()
394
+ this.takeMutex.ready()
395
+ this.waitTimeouted = false
427
396
 
428
397
  return result
429
398
  }
430
399
 
431
- async put(data: PortalStreamData<B>) {
432
- if (this.state === 'closed' || this.state === 'failed') {
433
- throw new Error('Buffer is closed')
434
- }
435
-
436
- if (this.idleTimeout != null) {
437
- clearTimeout(this.idleTimeout)
438
- this.idleTimeout = undefined
439
- }
400
+ updateHead(head: {headNumber?: number; finalizedHeadNumber?: number; finalizedHeadHash?: string}) {
401
+ this.headNumber = head.headNumber
402
+ this.finalizedHeadNumber = head.finalizedHeadNumber
403
+ this.finalizedHeadHash = head.finalizedHeadHash
404
+ this.headUpdated = true
405
+ }
440
406
 
441
- if (this.buffer == null) {
442
- this.buffer = {blocks: [], meta: {bytes: 0}}
407
+ async put(blocks: B[], bytes: number) {
408
+ if (this.isTerminated()) {
409
+ throw new Error('Buffer is closed')
443
410
  }
444
411
 
445
- this.buffer.blocks.push(...data.blocks)
446
- this.buffer.finalizedHead = data.finalizedHead
447
- this.buffer.meta.bytes += data.meta.bytes
412
+ this.idleTimer.stop()
448
413
 
449
- this.putFuture.resolve()
414
+ this.blocks.push(...blocks)
415
+ this.bytes += bytes
450
416
 
451
- if (this.buffer.meta.bytes >= this.minBytes) {
452
- this.readyFuture.resolve()
417
+ if (this.waitTimeouted || this.bytes >= this.maxBytes) {
418
+ this.flush()
453
419
  }
454
420
 
455
- if (this.buffer.meta.bytes >= this.maxBytes) {
456
- await this.takeFuture.promise()
421
+ if (this.bytes >= this.maxBytes) {
422
+ this.takeMutex.unready()
423
+ await this.takeMutex.wait()
457
424
  }
458
425
 
459
- if (this.state === 'pending') {
460
- this.idleTimeout = setTimeout(() => this._ready(), this.maxIdleTime)
461
- }
426
+ this.idleTimer.start()
462
427
  }
463
428
 
464
429
  flush() {
465
- if (this.buffer == null) return
466
- this._ready()
430
+ if (!this.hasData()) return
431
+ this.stopTimers()
432
+ this.putMutex.ready()
467
433
  }
468
434
 
469
435
  close() {
470
- if (this.state === 'closed' || this.state === 'failed') return
436
+ if (this.isTerminated()) return
471
437
  this.state = 'closed'
472
- this._cleanup()
438
+ this.stopTimers()
439
+ this.putMutex.ready()
440
+ this.takeMutex.reject(new Error('Buffer closed'))
441
+ this.abortController.abort()
473
442
  }
474
443
 
475
444
  fail(err: any) {
476
- if (this.state === 'closed' || this.state === 'failed') return
445
+ if (this.isTerminated()) return
477
446
  this.state = 'failed'
478
447
  this.error = err
479
- this._cleanup()
448
+ this.stopTimers()
449
+ this.putMutex.ready()
450
+ this.takeMutex.reject(new Error('Buffer closed'))
451
+ this.abortController.abort()
480
452
  }
481
453
 
482
- iterate() {
454
+ [Symbol.asyncIterator](): AsyncIterator<PortalStreamData<B>> {
483
455
  return {
484
- [Symbol.asyncIterator]: (): AsyncIterator<PortalStreamData<B>> => {
485
- return {
486
- next: async (): Promise<IteratorResult<PortalStreamData<B>>> => {
487
- const value = await this.take()
488
- if (value == null) {
489
- return {done: true, value: undefined}
490
- }
491
- return {done: false, value}
492
- },
493
- return: async (): Promise<IteratorResult<PortalStreamData<B>>> => {
494
- this.close()
495
- return {done: true, value: undefined}
496
- },
497
- throw: async (error?: any): Promise<IteratorResult<PortalStreamData<B>>> => {
498
- this.fail(error)
499
- throw error
500
- },
456
+ next: async () => {
457
+ let value = await this.take()
458
+ if (value == null) {
459
+ return {done: true, value: undefined}
501
460
  }
461
+ return {done: false, value}
462
+ },
463
+ return: async () => {
464
+ this.close()
465
+ return {done: true, value: undefined}
466
+ },
467
+ throw: async (error?: unknown) => {
468
+ this.fail(error)
469
+ throw error
502
470
  },
503
471
  }
504
472
  }
505
473
 
506
- private _ready() {
507
- if (this.state === 'pending') {
508
- this.state = 'ready'
509
- this.readyFuture.resolve()
510
- }
511
- if (this.idleTimeout != null) {
512
- clearTimeout(this.idleTimeout)
513
- this.idleTimeout = undefined
514
- }
515
- if (this.waitTimeout != null) {
516
- clearTimeout(this.waitTimeout)
517
- this.waitTimeout = undefined
518
- }
474
+ private hasData(): boolean {
475
+ return this.blocks.length > 0 || this.headUpdated
519
476
  }
520
477
 
521
- private _cleanup() {
522
- if (this.idleTimeout != null) {
523
- clearTimeout(this.idleTimeout)
524
- this.idleTimeout = undefined
478
+ private isTerminated(): boolean {
479
+ return this.state !== 'open'
480
+ }
481
+
482
+ private isFailed(): boolean {
483
+ return this.state === 'failed'
484
+ }
485
+
486
+ private collect(): PortalStreamData<B> | undefined {
487
+ if (!this.hasData()) {
488
+ if (this.isFailed()) throw this.error
489
+ return undefined
525
490
  }
526
- if (this.waitTimeout != null) {
527
- clearTimeout(this.waitTimeout)
528
- this.waitTimeout = undefined
491
+
492
+ let result: PortalStreamData<B> = {
493
+ blocks: this.blocks,
494
+ meta: {
495
+ bytes: this.bytes,
496
+ headNumber: this.headNumber,
497
+ finalizedHeadNumber: this.finalizedHeadNumber,
498
+ finalizedHeadHash: this.finalizedHeadHash,
499
+ },
529
500
  }
530
- this.readyFuture.resolve()
531
- this.putFuture.resolve()
532
- this.takeFuture.resolve()
533
- this.abortController.abort()
501
+
502
+ this.blocks = []
503
+ this.bytes = 0
504
+ this.headUpdated = false
505
+
506
+ return result
507
+ }
508
+
509
+ private stopTimers() {
510
+ this.idleTimer.stop()
511
+ this.waitTimer.stop()
534
512
  }
535
513
  }
536
514
 
@@ -574,16 +552,21 @@ class LineSplitter {
574
552
  }
575
553
 
576
554
  export class ForkException extends Error {
577
- readonly name = 'ForkError'
578
-
579
- constructor(readonly previousBlocks: BlockRef[], readonly head: BlockRef) {
580
- let parent = last(previousBlocks)
555
+ constructor(public blockNumber: number, public parentBlockHash: string, public previousBlocks: BlockRef[]) {
556
+ let base = last(previousBlocks)
581
557
  super(
582
- `expected ${head.number + 1} to have parent ${parent.number}#${parent.hash}, but got ${head.number}#${
583
- head.hash
584
- }`
558
+ `expected block ${blockNumber} to have parent ${base.number}#${parentBlockHash}, ` +
559
+ `but got ${base.number}#${base.hash} as a parent instead`
585
560
  )
586
561
  }
562
+
563
+ get name(): string {
564
+ return 'ForkException'
565
+ }
566
+
567
+ get isSubsquidForkException(): true {
568
+ return true
569
+ }
587
570
  }
588
571
 
589
572
  export function isForkException(err: unknown): err is ForkException {
@@ -592,16 +575,16 @@ export function isForkException(err: unknown): err is ForkException {
592
575
  return false
593
576
  }
594
577
 
595
- function getFinalizedHeadHeader(headers: HttpResponse['headers']) {
596
- let finalizedHeadHash = headers.get('X-Sqd-Finalized-Head-Hash')
597
- let finalizedHeadNumber = headers.get('X-Sqd-Finalized-Head-Number')
578
+ function getPortalStreamHeaders(headers: HttpResponse['headers']) {
579
+ let finalizedHeadHash = headers.get('x-sqd-finalized-head-hash')
580
+ let finalizedHeadNumber = headers.get('x-sqd-finalized-head-number')
581
+ let headNumber = headers.get('x-sqd-head-number')
598
582
 
599
- return finalizedHeadHash != null && finalizedHeadNumber != null
600
- ? {
601
- hash: finalizedHeadHash,
602
- number: parseInt(finalizedHeadNumber),
603
- }
604
- : undefined
583
+ return {
584
+ headNumber: headNumber ? parseInt(headNumber) : undefined,
585
+ finalizedHeadNumber: finalizedHeadNumber ? parseInt(finalizedHeadNumber) : undefined,
586
+ finalizedHeadHash: finalizedHeadHash ?? undefined,
587
+ }
605
588
  }
606
589
 
607
590
  function isStreamAbortedError(err: unknown) {