@subsquid/portal-client 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/client.d.ts +40 -18
- package/lib/client.d.ts.map +1 -1
- package/lib/client.js +199 -222
- package/lib/client.js.map +1 -1
- package/lib/query/common/data.d.ts +21 -0
- package/lib/query/common/data.d.ts.map +1 -0
- package/lib/query/common/data.js +3 -0
- package/lib/query/common/data.js.map +1 -0
- package/lib/query/common/query.d.ts +7 -0
- package/lib/query/common/query.d.ts.map +1 -0
- package/lib/query/common/query.js +3 -0
- package/lib/query/common/query.js.map +1 -0
- package/lib/query/evm/data.d.ts +159 -0
- package/lib/query/evm/data.d.ts.map +1 -0
- package/lib/query/evm/data.js +3 -0
- package/lib/query/evm/data.js.map +1 -0
- package/lib/query/evm/fields.d.ts +50 -0
- package/lib/query/evm/fields.d.ts.map +1 -0
- package/lib/query/evm/fields.js +3 -0
- package/lib/query/evm/fields.js.map +1 -0
- package/lib/query/evm/index.d.ts +4 -0
- package/lib/query/evm/index.d.ts.map +1 -0
- package/lib/query/evm/index.js +7 -0
- package/lib/query/evm/index.js.map +1 -0
- package/lib/query/evm/query.d.ts +54 -0
- package/lib/query/evm/query.d.ts.map +1 -0
- package/lib/query/evm/query.js +3 -0
- package/lib/query/evm/query.js.map +1 -0
- package/lib/query/evm/schema.d.ts +322 -0
- package/lib/query/evm/schema.d.ts.map +1 -0
- package/lib/query/evm/schema.js +224 -0
- package/lib/query/evm/schema.js.map +1 -0
- package/lib/query/index.d.ts +576 -8
- package/lib/query/index.d.ts.map +1 -1
- package/lib/query/index.js +15 -24
- package/lib/query/index.js.map +1 -1
- package/lib/query/solana/data.d.ts +115 -0
- package/lib/query/solana/data.d.ts.map +1 -0
- package/lib/query/solana/data.js +3 -0
- package/lib/query/solana/data.js.map +1 -0
- package/lib/query/solana/fields.d.ts +31 -0
- package/lib/query/solana/fields.d.ts.map +1 -0
- package/lib/query/solana/fields.js +3 -0
- package/lib/query/solana/fields.js.map +1 -0
- package/lib/query/solana/index.d.ts +4 -0
- package/lib/query/solana/index.d.ts.map +1 -0
- package/lib/query/solana/index.js +7 -0
- package/lib/query/solana/index.js.map +1 -0
- package/lib/query/solana/query.d.ts +74 -0
- package/lib/query/solana/query.d.ts.map +1 -0
- package/lib/query/solana/query.js +3 -0
- package/lib/query/solana/query.js.map +1 -0
- package/lib/query/solana/schema.d.ts +160 -0
- package/lib/query/solana/schema.d.ts.map +1 -0
- package/lib/query/solana/schema.js +130 -0
- package/lib/query/solana/schema.js.map +1 -0
- package/lib/query/substrate/data.d.ts +98 -0
- package/lib/query/substrate/data.d.ts.map +1 -0
- package/lib/query/substrate/data.js +3 -0
- package/lib/query/substrate/data.js.map +1 -0
- package/lib/query/substrate/fields.d.ts +22 -0
- package/lib/query/substrate/fields.d.ts.map +1 -0
- package/lib/query/substrate/fields.js +3 -0
- package/lib/query/substrate/fields.js.map +1 -0
- package/lib/query/substrate/index.d.ts +4 -0
- package/lib/query/substrate/index.d.ts.map +1 -0
- package/lib/query/substrate/index.js +7 -0
- package/lib/query/substrate/index.js.map +1 -0
- package/lib/query/substrate/query.d.ts +50 -0
- package/lib/query/substrate/query.d.ts.map +1 -0
- package/lib/query/substrate/query.js +3 -0
- package/lib/query/substrate/query.js.map +1 -0
- package/lib/query/substrate/schema.d.ts +102 -0
- package/lib/query/substrate/schema.d.ts.map +1 -0
- package/lib/query/substrate/schema.js +88 -0
- package/lib/query/substrate/schema.js.map +1 -0
- package/lib/query/type-util.d.ts +13 -0
- package/lib/query/type-util.d.ts.map +1 -0
- package/lib/query/type-util.js +3 -0
- package/lib/query/type-util.js.map +1 -0
- package/lib/query/util.d.ts +5 -0
- package/lib/query/util.d.ts.map +1 -0
- package/lib/query/util.js +23 -0
- package/lib/query/util.js.map +1 -0
- package/lib/util.d.ts +19 -0
- package/lib/util.d.ts.map +1 -0
- package/lib/util.js +57 -0
- package/lib/util.js.map +1 -0
- package/package.json +3 -3
- package/src/client.ts +261 -278
- package/src/query/common/data.ts +24 -0
- package/src/query/common/query.ts +6 -0
- package/src/query/evm/data.ts +182 -0
- package/src/query/evm/fields.ts +105 -0
- package/src/query/evm/index.ts +3 -0
- package/src/query/evm/query.ts +59 -0
- package/src/query/evm/schema.ts +277 -0
- package/src/query/index.ts +19 -36
- package/src/query/solana/data.ts +132 -0
- package/src/query/solana/fields.ts +42 -0
- package/src/query/solana/index.ts +3 -0
- package/src/query/solana/query.ts +89 -0
- package/src/query/solana/schema.ts +164 -0
- package/src/query/substrate/data.ts +101 -0
- package/src/query/substrate/fields.ts +30 -0
- package/src/query/substrate/index.ts +3 -0
- package/src/query/substrate/query.ts +60 -0
- package/src/query/substrate/schema.ts +114 -0
- package/src/query/type-util.ts +25 -0
- package/src/query/util.ts +23 -0
- package/src/util.ts +56 -0
- package/lib/query/common.d.ts +0 -56
- package/lib/query/common.d.ts.map +0 -1
- package/lib/query/common.js +0 -16
- package/lib/query/common.js.map +0 -1
- package/lib/query/evm.d.ts +0 -267
- package/lib/query/evm.d.ts.map +0 -1
- package/lib/query/evm.js +0 -245
- package/lib/query/evm.js.map +0 -1
- package/lib/query/solana.d.ts +0 -224
- package/lib/query/solana.d.ts.map +0 -1
- package/lib/query/solana.js +0 -121
- package/lib/query/solana.js.map +0 -1
- package/lib/query/substrate.d.ts +0 -173
- package/lib/query/substrate.d.ts.map +0 -1
- package/lib/query/substrate.js +0 -71
- package/lib/query/substrate.js.map +0 -1
- package/src/query/common.ts +0 -83
- package/src/query/evm.ts +0 -677
- package/src/query/solana.ts +0 -438
- package/src/query/substrate.ts +0 -288
package/src/client.ts
CHANGED
|
@@ -1,16 +1,8 @@
|
|
|
1
1
|
import {HttpBody, HttpClient, HttpClientOptions, HttpError, HttpResponse, RequestOptions} from '@subsquid/http-client'
|
|
2
|
-
import {
|
|
3
|
-
addErrorContext,
|
|
4
|
-
createFuture,
|
|
5
|
-
Future,
|
|
6
|
-
last,
|
|
7
|
-
unexpectedCase,
|
|
8
|
-
wait,
|
|
9
|
-
withErrorContext,
|
|
10
|
-
} from '@subsquid/util-internal'
|
|
11
|
-
import {Readable} from 'stream'
|
|
12
|
-
import {GetBlock, evm, getBlockSchema, PortalBlock, PortalQuery, Query, solana, substrate} from './query'
|
|
2
|
+
import {addErrorContext, last, unexpectedCase, wait, withErrorContext} from '@subsquid/util-internal'
|
|
13
3
|
import {cast} from '@subsquid/util-internal-validation'
|
|
4
|
+
import {Semaphore, Timer} from './util'
|
|
5
|
+
import {getQuery, type AnyQuery, type GetQueryBlock} from './query'
|
|
14
6
|
|
|
15
7
|
export * from './query'
|
|
16
8
|
|
|
@@ -29,19 +21,19 @@ export interface PortalClientOptions {
|
|
|
29
21
|
|
|
30
22
|
/**
|
|
31
23
|
* Minimum number of bytes to return.
|
|
32
|
-
* @
|
|
24
|
+
* @deprecated not used
|
|
33
25
|
*/
|
|
34
26
|
minBytes?: number
|
|
35
27
|
|
|
36
28
|
/**
|
|
37
29
|
* Maximum number of bytes to return.
|
|
38
|
-
* @default
|
|
30
|
+
* @default 52_428_800 (50MB)
|
|
39
31
|
*/
|
|
40
32
|
maxBytes?: number
|
|
41
33
|
|
|
42
34
|
/**
|
|
43
35
|
* Maximum time between stream data in milliseconds for return.
|
|
44
|
-
* @default
|
|
36
|
+
* @default 500
|
|
45
37
|
*/
|
|
46
38
|
maxIdleTime?: number
|
|
47
39
|
|
|
@@ -70,7 +62,6 @@ export interface PortalRequestOptions {
|
|
|
70
62
|
export interface PortalStreamOptions {
|
|
71
63
|
request?: Omit<PortalRequestOptions, 'abort'>
|
|
72
64
|
|
|
73
|
-
minBytes?: number
|
|
74
65
|
maxBytes?: number
|
|
75
66
|
maxIdleTime?: number
|
|
76
67
|
maxWaitTime?: number
|
|
@@ -78,10 +69,34 @@ export interface PortalStreamOptions {
|
|
|
78
69
|
headPollInterval?: number
|
|
79
70
|
}
|
|
80
71
|
|
|
72
|
+
export interface PortalStreamBaseResponse {
|
|
73
|
+
headNumber?: number
|
|
74
|
+
finalizedHeadNumber?: number
|
|
75
|
+
finalizedHeadHash?: string
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
export interface PortalStreamDataResponse extends PortalStreamBaseResponse {
|
|
79
|
+
type: 'data'
|
|
80
|
+
data: AsyncIterable<Uint8Array> | null
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
export interface PortalStreamNoDataResponse extends PortalStreamBaseResponse {
|
|
84
|
+
type: 'no-data'
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
export interface PortalStreamForkResponse extends PortalStreamBaseResponse {
|
|
88
|
+
type: 'fork'
|
|
89
|
+
previousBlocks: BlockRef[]
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
export type PortalStreamResponse = PortalStreamDataResponse | PortalStreamNoDataResponse | PortalStreamForkResponse
|
|
93
|
+
|
|
81
94
|
export type PortalStreamData<B> = {
|
|
82
95
|
blocks: B[]
|
|
83
|
-
finalizedHead?: BlockRef
|
|
84
96
|
meta: {
|
|
97
|
+
finalizedHeadNumber?: number
|
|
98
|
+
finalizedHeadHash?: string
|
|
99
|
+
headNumber?: number
|
|
85
100
|
bytes: number
|
|
86
101
|
}
|
|
87
102
|
}
|
|
@@ -97,7 +112,6 @@ export class PortalClient {
|
|
|
97
112
|
private url: URL
|
|
98
113
|
private client: HttpClient
|
|
99
114
|
private headPollInterval: number
|
|
100
|
-
private minBytes: number
|
|
101
115
|
private maxBytes: number
|
|
102
116
|
private maxIdleTime: number
|
|
103
117
|
private maxWaitTime: number
|
|
@@ -106,9 +120,8 @@ export class PortalClient {
|
|
|
106
120
|
this.url = new URL(options.url)
|
|
107
121
|
this.client = options.http instanceof HttpClient ? options.http : new HttpClient(options.http)
|
|
108
122
|
this.headPollInterval = options.headPollInterval ?? 0
|
|
109
|
-
this.
|
|
110
|
-
this.
|
|
111
|
-
this.maxIdleTime = options.maxIdleTime ?? 300
|
|
123
|
+
this.maxBytes = options.maxBytes ?? 50 * 1024 * 1024
|
|
124
|
+
this.maxIdleTime = options.maxIdleTime ?? 500
|
|
112
125
|
this.maxWaitTime = options.maxWaitTime ?? 5_000
|
|
113
126
|
}
|
|
114
127
|
|
|
@@ -122,92 +135,56 @@ export class PortalClient {
|
|
|
122
135
|
return u.toString()
|
|
123
136
|
}
|
|
124
137
|
|
|
125
|
-
async getHead(options?: PortalRequestOptions): Promise<BlockRef | undefined> {
|
|
126
|
-
const res = await this.request('GET', this.getDatasetUrl('head'), options)
|
|
138
|
+
async getHead(options?: PortalRequestOptions, finalized?: boolean): Promise<BlockRef | undefined> {
|
|
139
|
+
const res = await this.request('GET', this.getDatasetUrl(!finalized ? 'head' : 'finalized-head'), options)
|
|
127
140
|
return res.body ?? undefined
|
|
128
141
|
}
|
|
129
142
|
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
return res.body ?? undefined
|
|
143
|
+
getFinalizedHead(options?: PortalRequestOptions): Promise<BlockRef | undefined> {
|
|
144
|
+
return this.getHead(options, true)
|
|
133
145
|
}
|
|
134
146
|
|
|
135
147
|
/**
|
|
136
|
-
*
|
|
148
|
+
* Waits until the finalized head reaches or exceeds the specified block number.
|
|
149
|
+
* @param blockNumber - The block number to wait for finalization
|
|
150
|
+
* @param options - Request options including abort signal and poll interval
|
|
151
|
+
* @returns The finalized head once it reaches the target block number
|
|
137
152
|
*/
|
|
138
|
-
async
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
withErrorContext({
|
|
152
|
-
archiveQuery: query,
|
|
153
|
-
})
|
|
154
|
-
)
|
|
155
|
-
.then((res) => {
|
|
156
|
-
let blocks = res.body
|
|
157
|
-
.toString('utf8')
|
|
158
|
-
.trimEnd()
|
|
159
|
-
.split('\n')
|
|
160
|
-
.map((line) => JSON.parse(line))
|
|
161
|
-
return blocks
|
|
162
|
-
})
|
|
163
|
-
}
|
|
153
|
+
async waitForFinalization(
|
|
154
|
+
blockNumber: number,
|
|
155
|
+
options?: PortalRequestOptions & {pollInterval?: number}
|
|
156
|
+
): Promise<BlockRef> {
|
|
157
|
+
let {pollInterval = this.headPollInterval || 1000, ...requestOptions} = options ?? {}
|
|
158
|
+
|
|
159
|
+
while (true) {
|
|
160
|
+
options?.abort?.throwIfAborted()
|
|
161
|
+
|
|
162
|
+
let head = await this.getFinalizedHead(requestOptions)
|
|
163
|
+
if (head && head.number >= blockNumber) {
|
|
164
|
+
return head
|
|
165
|
+
}
|
|
164
166
|
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
options?: PortalRequestOptions
|
|
168
|
-
): Promise<R[]> {
|
|
169
|
-
// FIXME: is it needed or it is better to always use stream?
|
|
170
|
-
return this.request<Buffer>('POST', this.getDatasetUrl(`stream`), {
|
|
171
|
-
...options,
|
|
172
|
-
json: query,
|
|
173
|
-
})
|
|
174
|
-
.catch(
|
|
175
|
-
withErrorContext({
|
|
176
|
-
archiveQuery: query,
|
|
177
|
-
})
|
|
178
|
-
)
|
|
179
|
-
.then((res) => {
|
|
180
|
-
let blocks = res.body
|
|
181
|
-
.toString('utf8')
|
|
182
|
-
.trimEnd()
|
|
183
|
-
.split('\n')
|
|
184
|
-
.map((line) => JSON.parse(line))
|
|
185
|
-
return blocks
|
|
186
|
-
})
|
|
167
|
+
await wait(pollInterval, options?.abort)
|
|
168
|
+
}
|
|
187
169
|
}
|
|
188
170
|
|
|
189
|
-
|
|
171
|
+
getStream<Q extends AnyQuery>(
|
|
190
172
|
query: Q,
|
|
191
|
-
options?: PortalStreamOptions
|
|
192
|
-
|
|
173
|
+
options?: PortalStreamOptions,
|
|
174
|
+
finalized?: boolean
|
|
175
|
+
): PortalStream<GetQueryBlock<Q>> {
|
|
193
176
|
return createPortalStream(query, this.getStreamOptions(options), async (q, o) =>
|
|
194
|
-
this.getStreamRequest('finalized-stream', q, o)
|
|
177
|
+
this.getStreamRequest(!finalized ? 'stream' : 'finalized-stream', q, o)
|
|
195
178
|
)
|
|
196
179
|
}
|
|
197
180
|
|
|
198
|
-
|
|
199
|
-
query
|
|
200
|
-
options?: PortalStreamOptions
|
|
201
|
-
): PortalStream<GetBlock<Q>> {
|
|
202
|
-
return createPortalStream(query, this.getStreamOptions(options), async (q, o) =>
|
|
203
|
-
this.getStreamRequest('stream', q, o)
|
|
204
|
-
)
|
|
181
|
+
getFinalizedStream<Q extends AnyQuery>(query: Q, options?: PortalStreamOptions): PortalStream<GetQueryBlock<Q>> {
|
|
182
|
+
return this.getStream(query, options, true)
|
|
205
183
|
}
|
|
206
184
|
|
|
207
185
|
private getStreamOptions(options?: PortalStreamOptions) {
|
|
208
186
|
let {
|
|
209
187
|
headPollInterval = this.headPollInterval,
|
|
210
|
-
minBytes = this.minBytes,
|
|
211
188
|
maxBytes = this.maxBytes,
|
|
212
189
|
maxIdleTime = this.maxIdleTime,
|
|
213
190
|
maxWaitTime = this.maxWaitTime,
|
|
@@ -216,7 +193,6 @@ export class PortalClient {
|
|
|
216
193
|
|
|
217
194
|
return {
|
|
218
195
|
headPollInterval,
|
|
219
|
-
minBytes,
|
|
220
196
|
maxBytes,
|
|
221
197
|
maxIdleTime,
|
|
222
198
|
maxWaitTime,
|
|
@@ -224,9 +200,13 @@ export class PortalClient {
|
|
|
224
200
|
}
|
|
225
201
|
}
|
|
226
202
|
|
|
227
|
-
private async getStreamRequest(
|
|
203
|
+
private async getStreamRequest(
|
|
204
|
+
path: string,
|
|
205
|
+
query: AnyQuery,
|
|
206
|
+
options?: PortalRequestOptions
|
|
207
|
+
): Promise<PortalStreamResponse> {
|
|
228
208
|
try {
|
|
229
|
-
let res = await this.request<
|
|
209
|
+
let res = await this.request<AsyncIterable<Uint8Array> | undefined>('POST', this.getDatasetUrl(path), {
|
|
230
210
|
...options,
|
|
231
211
|
json: query,
|
|
232
212
|
stream: true,
|
|
@@ -236,28 +216,29 @@ export class PortalClient {
|
|
|
236
216
|
})
|
|
237
217
|
)
|
|
238
218
|
|
|
219
|
+
let headers = getPortalStreamHeaders(res.headers)
|
|
220
|
+
|
|
239
221
|
switch (res.status) {
|
|
240
222
|
case 200:
|
|
241
|
-
let finalizedHead = getFinalizedHeadHeader(res.headers)
|
|
242
|
-
let stream = res.body ? splitLines(res.body) : undefined
|
|
243
|
-
|
|
244
223
|
return {
|
|
245
|
-
|
|
246
|
-
|
|
224
|
+
...headers,
|
|
225
|
+
type: 'data',
|
|
226
|
+
data: res.body ?? null,
|
|
247
227
|
}
|
|
248
228
|
case 204:
|
|
249
229
|
return {
|
|
250
|
-
|
|
230
|
+
...headers,
|
|
231
|
+
type: 'no-data',
|
|
251
232
|
}
|
|
252
233
|
default:
|
|
253
234
|
throw unexpectedCase(res.status)
|
|
254
235
|
}
|
|
255
236
|
} catch (e: any) {
|
|
256
|
-
if (isForkHttpError(e)
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
}
|
|
237
|
+
if (isForkHttpError(e)) {
|
|
238
|
+
return {
|
|
239
|
+
type: 'fork',
|
|
240
|
+
previousBlocks: e.response.body.previousBlocks,
|
|
241
|
+
}
|
|
261
242
|
}
|
|
262
243
|
|
|
263
244
|
throw addErrorContext(e, {
|
|
@@ -284,78 +265,79 @@ function isForkHttpError(err: unknown): err is HttpError<{previousBlocks: BlockR
|
|
|
284
265
|
return true
|
|
285
266
|
}
|
|
286
267
|
|
|
287
|
-
function createPortalStream<Q extends
|
|
268
|
+
function createPortalStream<Q extends AnyQuery>(
|
|
288
269
|
query: Q,
|
|
289
270
|
options: Required<PortalStreamOptions>,
|
|
290
|
-
requestStream: (
|
|
291
|
-
|
|
292
|
-
options?: PortalRequestOptions
|
|
293
|
-
) => Promise<{finalizedHead?: BlockRef; stream?: AsyncIterable<string[]> | null | undefined}>
|
|
294
|
-
): PortalStream<GetBlock<Q>> {
|
|
271
|
+
requestStream: (query: Q, options?: PortalRequestOptions) => Promise<PortalStreamResponse>
|
|
272
|
+
): PortalStream<GetQueryBlock<Q>> {
|
|
295
273
|
let {headPollInterval, request, ...bufferOptions} = options
|
|
296
274
|
|
|
297
|
-
let buffer = new PortalStreamBuffer<
|
|
275
|
+
let buffer = new PortalStreamBuffer<GetQueryBlock<Q>>(bufferOptions)
|
|
298
276
|
|
|
299
|
-
let schema =
|
|
300
|
-
let {fromBlock = 0, toBlock, parentBlockHash} =
|
|
277
|
+
let [normalizedQuery, schema] = getQuery(query)
|
|
278
|
+
let {fromBlock = 0, toBlock, parentBlockHash} = normalizedQuery
|
|
301
279
|
|
|
302
280
|
const ingest = async () => {
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
281
|
+
while (!buffer.signal.aborted) {
|
|
282
|
+
if (toBlock != null && fromBlock > toBlock) break
|
|
283
|
+
|
|
284
|
+
let res = await requestStream(
|
|
285
|
+
{
|
|
286
|
+
...normalizedQuery,
|
|
287
|
+
fromBlock,
|
|
288
|
+
parentBlockHash,
|
|
289
|
+
},
|
|
290
|
+
{
|
|
291
|
+
...request,
|
|
292
|
+
abort: buffer.signal,
|
|
293
|
+
}
|
|
294
|
+
)
|
|
295
|
+
|
|
296
|
+
if (res.type === 'fork') {
|
|
297
|
+
throw new ForkException(fromBlock, parentBlockHash!, res.previousBlocks)
|
|
315
298
|
}
|
|
316
|
-
)
|
|
317
299
|
|
|
318
|
-
|
|
300
|
+
buffer.updateHead(res)
|
|
319
301
|
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
302
|
+
if (res.type === 'no-data') {
|
|
303
|
+
buffer.flush()
|
|
304
|
+
if (headPollInterval > 0) {
|
|
305
|
+
await wait(headPollInterval, buffer.signal)
|
|
306
|
+
}
|
|
307
|
+
continue
|
|
326
308
|
}
|
|
327
|
-
return ingest()
|
|
328
|
-
}
|
|
329
309
|
|
|
330
|
-
|
|
331
|
-
if (res.stream == null) return
|
|
310
|
+
if (!res.data) break
|
|
332
311
|
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
312
|
+
try {
|
|
313
|
+
for await (let data of splitLines(res.data)) {
|
|
314
|
+
buffer.signal.throwIfAborted()
|
|
336
315
|
|
|
337
|
-
|
|
338
|
-
|
|
316
|
+
let blocks: GetQueryBlock<Q>[] = []
|
|
317
|
+
let bytes = 0
|
|
339
318
|
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
319
|
+
for (let line of data) {
|
|
320
|
+
let block = cast(schema, JSON.parse(line)) as GetQueryBlock<Q>
|
|
321
|
+
blocks.push(block)
|
|
322
|
+
bytes += line.length
|
|
344
323
|
|
|
345
|
-
|
|
346
|
-
|
|
324
|
+
fromBlock = block.header.number + 1
|
|
325
|
+
parentBlockHash = block.header.hash
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
await buffer.put(blocks, bytes)
|
|
347
329
|
}
|
|
348
330
|
|
|
349
|
-
|
|
350
|
-
}
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
331
|
+
buffer.flush()
|
|
332
|
+
} catch (err) {
|
|
333
|
+
if (buffer.signal.aborted) return
|
|
334
|
+
if (!isStreamAbortedError(err)) {
|
|
335
|
+
throw err
|
|
336
|
+
}
|
|
355
337
|
}
|
|
356
338
|
}
|
|
357
339
|
|
|
358
|
-
|
|
340
|
+
buffer.flush()
|
|
359
341
|
}
|
|
360
342
|
|
|
361
343
|
ingest().then(
|
|
@@ -363,174 +345,170 @@ function createPortalStream<Q extends Query>(
|
|
|
363
345
|
(err) => buffer.fail(err)
|
|
364
346
|
)
|
|
365
347
|
|
|
366
|
-
return buffer
|
|
348
|
+
return buffer
|
|
367
349
|
}
|
|
368
350
|
|
|
369
351
|
class PortalStreamBuffer<B> {
|
|
370
|
-
private
|
|
371
|
-
private
|
|
372
|
-
private error: unknown
|
|
352
|
+
private blocks: B[] = []
|
|
353
|
+
private bytes = 0
|
|
373
354
|
|
|
374
|
-
private
|
|
375
|
-
private
|
|
376
|
-
private
|
|
355
|
+
private headNumber?: number
|
|
356
|
+
private finalizedHeadNumber?: number
|
|
357
|
+
private finalizedHeadHash?: string
|
|
358
|
+
private headUpdated = false
|
|
377
359
|
|
|
378
|
-
private
|
|
379
|
-
private
|
|
360
|
+
private state: 'open' | 'failed' | 'closed' = 'open'
|
|
361
|
+
private error: unknown
|
|
380
362
|
|
|
381
|
-
private
|
|
382
|
-
private
|
|
383
|
-
private
|
|
384
|
-
private maxWaitTime: number
|
|
363
|
+
private waitTimeouted = false
|
|
364
|
+
private putMutex = new Semaphore(false)
|
|
365
|
+
private takeMutex = new Semaphore(true)
|
|
385
366
|
|
|
367
|
+
private maxBytes: number
|
|
368
|
+
private idleTimer: Timer
|
|
369
|
+
private waitTimer: Timer
|
|
386
370
|
private abortController = new AbortController()
|
|
387
371
|
|
|
388
372
|
get signal() {
|
|
389
373
|
return this.abortController.signal
|
|
390
374
|
}
|
|
391
375
|
|
|
392
|
-
constructor(options: {maxWaitTime: number; maxBytes: number; maxIdleTime: number
|
|
393
|
-
this.
|
|
394
|
-
this.
|
|
395
|
-
this.
|
|
396
|
-
this.maxIdleTime = options.maxIdleTime
|
|
376
|
+
constructor(options: {maxWaitTime: number; maxBytes: number; maxIdleTime: number}) {
|
|
377
|
+
this.maxBytes = options.maxBytes
|
|
378
|
+
this.idleTimer = new Timer(options.maxIdleTime, () => this.flush())
|
|
379
|
+
this.waitTimer = new Timer(options.maxWaitTime, () => (this.waitTimeouted = true))
|
|
397
380
|
}
|
|
398
381
|
|
|
399
382
|
async take(): Promise<PortalStreamData<B> | undefined> {
|
|
400
|
-
if (this.
|
|
401
|
-
|
|
383
|
+
if (this.isTerminated()) {
|
|
384
|
+
return this.collect()
|
|
402
385
|
}
|
|
403
386
|
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
throw this.error
|
|
408
|
-
}
|
|
387
|
+
this.waitTimer.start()
|
|
388
|
+
await this.putMutex.wait()
|
|
389
|
+
this.waitTimer.stop()
|
|
409
390
|
|
|
410
|
-
let result = this.
|
|
411
|
-
this.buffer = undefined
|
|
391
|
+
let result = this.collect()
|
|
412
392
|
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
if (result == null) {
|
|
418
|
-
throw new Error('Buffer is empty')
|
|
419
|
-
}
|
|
420
|
-
|
|
421
|
-
this.takeFuture.resolve()
|
|
422
|
-
|
|
423
|
-
this.readyFuture = createFuture()
|
|
424
|
-
this.putFuture = createFuture()
|
|
425
|
-
this.takeFuture = createFuture()
|
|
426
|
-
this.state = 'pending'
|
|
393
|
+
this.putMutex.unready()
|
|
394
|
+
this.takeMutex.ready()
|
|
395
|
+
this.waitTimeouted = false
|
|
427
396
|
|
|
428
397
|
return result
|
|
429
398
|
}
|
|
430
399
|
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
clearTimeout(this.idleTimeout)
|
|
438
|
-
this.idleTimeout = undefined
|
|
439
|
-
}
|
|
400
|
+
updateHead(head: {headNumber?: number; finalizedHeadNumber?: number; finalizedHeadHash?: string}) {
|
|
401
|
+
this.headNumber = head.headNumber
|
|
402
|
+
this.finalizedHeadNumber = head.finalizedHeadNumber
|
|
403
|
+
this.finalizedHeadHash = head.finalizedHeadHash
|
|
404
|
+
this.headUpdated = true
|
|
405
|
+
}
|
|
440
406
|
|
|
441
|
-
|
|
442
|
-
|
|
407
|
+
async put(blocks: B[], bytes: number) {
|
|
408
|
+
if (this.isTerminated()) {
|
|
409
|
+
throw new Error('Buffer is closed')
|
|
443
410
|
}
|
|
444
411
|
|
|
445
|
-
this.
|
|
446
|
-
this.buffer.finalizedHead = data.finalizedHead
|
|
447
|
-
this.buffer.meta.bytes += data.meta.bytes
|
|
412
|
+
this.idleTimer.stop()
|
|
448
413
|
|
|
449
|
-
this.
|
|
414
|
+
this.blocks.push(...blocks)
|
|
415
|
+
this.bytes += bytes
|
|
450
416
|
|
|
451
|
-
if (this.
|
|
452
|
-
this.
|
|
417
|
+
if (this.waitTimeouted || this.bytes >= this.maxBytes) {
|
|
418
|
+
this.flush()
|
|
453
419
|
}
|
|
454
420
|
|
|
455
|
-
if (this.
|
|
456
|
-
|
|
421
|
+
if (this.bytes >= this.maxBytes) {
|
|
422
|
+
this.takeMutex.unready()
|
|
423
|
+
await this.takeMutex.wait()
|
|
457
424
|
}
|
|
458
425
|
|
|
459
|
-
|
|
460
|
-
this.idleTimeout = setTimeout(() => this._ready(), this.maxIdleTime)
|
|
461
|
-
}
|
|
426
|
+
this.idleTimer.start()
|
|
462
427
|
}
|
|
463
428
|
|
|
464
429
|
flush() {
|
|
465
|
-
if (this.
|
|
466
|
-
this.
|
|
430
|
+
if (!this.hasData()) return
|
|
431
|
+
this.stopTimers()
|
|
432
|
+
this.putMutex.ready()
|
|
467
433
|
}
|
|
468
434
|
|
|
469
435
|
close() {
|
|
470
|
-
if (this.
|
|
436
|
+
if (this.isTerminated()) return
|
|
471
437
|
this.state = 'closed'
|
|
472
|
-
this.
|
|
438
|
+
this.stopTimers()
|
|
439
|
+
this.putMutex.ready()
|
|
440
|
+
this.takeMutex.reject(new Error('Buffer closed'))
|
|
441
|
+
this.abortController.abort()
|
|
473
442
|
}
|
|
474
443
|
|
|
475
444
|
fail(err: any) {
|
|
476
|
-
if (this.
|
|
445
|
+
if (this.isTerminated()) return
|
|
477
446
|
this.state = 'failed'
|
|
478
447
|
this.error = err
|
|
479
|
-
this.
|
|
448
|
+
this.stopTimers()
|
|
449
|
+
this.putMutex.ready()
|
|
450
|
+
this.takeMutex.reject(new Error('Buffer closed'))
|
|
451
|
+
this.abortController.abort()
|
|
480
452
|
}
|
|
481
453
|
|
|
482
|
-
|
|
454
|
+
[Symbol.asyncIterator](): AsyncIterator<PortalStreamData<B>> {
|
|
483
455
|
return {
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
if (value == null) {
|
|
489
|
-
return {done: true, value: undefined}
|
|
490
|
-
}
|
|
491
|
-
return {done: false, value}
|
|
492
|
-
},
|
|
493
|
-
return: async (): Promise<IteratorResult<PortalStreamData<B>>> => {
|
|
494
|
-
this.close()
|
|
495
|
-
return {done: true, value: undefined}
|
|
496
|
-
},
|
|
497
|
-
throw: async (error?: any): Promise<IteratorResult<PortalStreamData<B>>> => {
|
|
498
|
-
this.fail(error)
|
|
499
|
-
throw error
|
|
500
|
-
},
|
|
456
|
+
next: async () => {
|
|
457
|
+
let value = await this.take()
|
|
458
|
+
if (value == null) {
|
|
459
|
+
return {done: true, value: undefined}
|
|
501
460
|
}
|
|
461
|
+
return {done: false, value}
|
|
462
|
+
},
|
|
463
|
+
return: async () => {
|
|
464
|
+
this.close()
|
|
465
|
+
return {done: true, value: undefined}
|
|
466
|
+
},
|
|
467
|
+
throw: async (error?: unknown) => {
|
|
468
|
+
this.fail(error)
|
|
469
|
+
throw error
|
|
502
470
|
},
|
|
503
471
|
}
|
|
504
472
|
}
|
|
505
473
|
|
|
506
|
-
private
|
|
507
|
-
|
|
508
|
-
this.state = 'ready'
|
|
509
|
-
this.readyFuture.resolve()
|
|
510
|
-
}
|
|
511
|
-
if (this.idleTimeout != null) {
|
|
512
|
-
clearTimeout(this.idleTimeout)
|
|
513
|
-
this.idleTimeout = undefined
|
|
514
|
-
}
|
|
515
|
-
if (this.waitTimeout != null) {
|
|
516
|
-
clearTimeout(this.waitTimeout)
|
|
517
|
-
this.waitTimeout = undefined
|
|
518
|
-
}
|
|
474
|
+
private hasData(): boolean {
|
|
475
|
+
return this.blocks.length > 0 || this.headUpdated
|
|
519
476
|
}
|
|
520
477
|
|
|
521
|
-
private
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
478
|
+
private isTerminated(): boolean {
|
|
479
|
+
return this.state !== 'open'
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
private isFailed(): boolean {
|
|
483
|
+
return this.state === 'failed'
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
private collect(): PortalStreamData<B> | undefined {
|
|
487
|
+
if (!this.hasData()) {
|
|
488
|
+
if (this.isFailed()) throw this.error
|
|
489
|
+
return undefined
|
|
525
490
|
}
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
this.
|
|
491
|
+
|
|
492
|
+
let result: PortalStreamData<B> = {
|
|
493
|
+
blocks: this.blocks,
|
|
494
|
+
meta: {
|
|
495
|
+
bytes: this.bytes,
|
|
496
|
+
headNumber: this.headNumber,
|
|
497
|
+
finalizedHeadNumber: this.finalizedHeadNumber,
|
|
498
|
+
finalizedHeadHash: this.finalizedHeadHash,
|
|
499
|
+
},
|
|
529
500
|
}
|
|
530
|
-
|
|
531
|
-
this.
|
|
532
|
-
this.
|
|
533
|
-
this.
|
|
501
|
+
|
|
502
|
+
this.blocks = []
|
|
503
|
+
this.bytes = 0
|
|
504
|
+
this.headUpdated = false
|
|
505
|
+
|
|
506
|
+
return result
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
private stopTimers() {
|
|
510
|
+
this.idleTimer.stop()
|
|
511
|
+
this.waitTimer.stop()
|
|
534
512
|
}
|
|
535
513
|
}
|
|
536
514
|
|
|
@@ -574,16 +552,21 @@ class LineSplitter {
|
|
|
574
552
|
}
|
|
575
553
|
|
|
576
554
|
export class ForkException extends Error {
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
constructor(readonly previousBlocks: BlockRef[], readonly head: BlockRef) {
|
|
580
|
-
let parent = last(previousBlocks)
|
|
555
|
+
constructor(public blockNumber: number, public parentBlockHash: string, public previousBlocks: BlockRef[]) {
|
|
556
|
+
let base = last(previousBlocks)
|
|
581
557
|
super(
|
|
582
|
-
`expected ${
|
|
583
|
-
|
|
584
|
-
}`
|
|
558
|
+
`expected block ${blockNumber} to have parent ${base.number}#${parentBlockHash}, ` +
|
|
559
|
+
`but got ${base.number}#${base.hash} as a parent instead`
|
|
585
560
|
)
|
|
586
561
|
}
|
|
562
|
+
|
|
563
|
+
get name(): string {
|
|
564
|
+
return 'ForkException'
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
get isSubsquidForkException(): true {
|
|
568
|
+
return true
|
|
569
|
+
}
|
|
587
570
|
}
|
|
588
571
|
|
|
589
572
|
export function isForkException(err: unknown): err is ForkException {
|
|
@@ -592,16 +575,16 @@ export function isForkException(err: unknown): err is ForkException {
|
|
|
592
575
|
return false
|
|
593
576
|
}
|
|
594
577
|
|
|
595
|
-
function
|
|
596
|
-
let finalizedHeadHash = headers.get('
|
|
597
|
-
let finalizedHeadNumber = headers.get('
|
|
578
|
+
function getPortalStreamHeaders(headers: HttpResponse['headers']) {
|
|
579
|
+
let finalizedHeadHash = headers.get('x-sqd-finalized-head-hash')
|
|
580
|
+
let finalizedHeadNumber = headers.get('x-sqd-finalized-head-number')
|
|
581
|
+
let headNumber = headers.get('x-sqd-head-number')
|
|
598
582
|
|
|
599
|
-
return
|
|
600
|
-
?
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
: undefined
|
|
583
|
+
return {
|
|
584
|
+
headNumber: headNumber ? parseInt(headNumber) : undefined,
|
|
585
|
+
finalizedHeadNumber: finalizedHeadNumber ? parseInt(finalizedHeadNumber) : undefined,
|
|
586
|
+
finalizedHeadHash: finalizedHeadHash ?? undefined,
|
|
587
|
+
}
|
|
605
588
|
}
|
|
606
589
|
|
|
607
590
|
function isStreamAbortedError(err: unknown) {
|