@subsquid/batch-processor 0.1.0-portal-api.a8e10e → 0.1.0-portal-api.d0738f
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/run.js +14 -13
- package/lib/run.js.map +1 -1
- package/package.json +2 -2
- package/src/run.ts +15 -15
package/lib/run.js
CHANGED
|
@@ -87,16 +87,20 @@ class Processor {
|
|
|
87
87
|
}
|
|
88
88
|
await this.initMetrics(head?.number ?? -1, await chainHeight.get());
|
|
89
89
|
while (true) {
|
|
90
|
-
let getStream = this.db.supportsHotBlocks
|
|
90
|
+
let getStream = this.db.supportsHotBlocks
|
|
91
|
+
? this.src.getStream.bind(this.src)
|
|
92
|
+
: this.src.getFinalizedStream.bind(this.src);
|
|
91
93
|
head = getStateHead(state);
|
|
92
94
|
try {
|
|
93
95
|
for await (let data of getStream({ after: head })) {
|
|
94
96
|
state = await this.processBatch(state, data, await chainHeight.get());
|
|
95
97
|
}
|
|
98
|
+
break; // Stream completed successfully, exit loop
|
|
96
99
|
}
|
|
97
100
|
catch (e) {
|
|
98
101
|
if (!(0, util_internal_data_source_1.isForkException)(e) || !this.db.supportsHotBlocks)
|
|
99
102
|
throw e;
|
|
103
|
+
// Handle fork and continue loop to retry
|
|
100
104
|
let chain = state.finalizedHead
|
|
101
105
|
? [state.finalizedHead, ...state.unfinalizedHeads]
|
|
102
106
|
: state.unfinalizedHeads;
|
|
@@ -112,7 +116,6 @@ class Processor {
|
|
|
112
116
|
state.unfinalizedHeads = chain.slice(1, rollbackIndex + 1);
|
|
113
117
|
}
|
|
114
118
|
}
|
|
115
|
-
break;
|
|
116
119
|
}
|
|
117
120
|
this.reportFinalStatus();
|
|
118
121
|
}
|
|
@@ -129,10 +132,8 @@ class Processor {
|
|
|
129
132
|
updateProgressMetrics(chainHeight, indexerHeight, time) {
|
|
130
133
|
this.metrics.setChainHeight(chainHeight);
|
|
131
134
|
this.metrics.setLastProcessedBlock(indexerHeight);
|
|
132
|
-
let left;
|
|
133
|
-
let processed;
|
|
134
|
-
left = this.metrics.getChainHeight() - this.metrics.getLastProcessedBlock();
|
|
135
|
-
processed = this.metrics.getLastProcessedBlock();
|
|
135
|
+
let left = this.metrics.getChainHeight() - this.metrics.getLastProcessedBlock();
|
|
136
|
+
let processed = this.metrics.getLastProcessedBlock();
|
|
136
137
|
this.metrics.updateProgress(processed, left, time);
|
|
137
138
|
}
|
|
138
139
|
async processBatch(state, data, chainHeight) {
|
|
@@ -141,11 +142,11 @@ class Processor {
|
|
|
141
142
|
return state;
|
|
142
143
|
let prevHead = getStateHead(state);
|
|
143
144
|
// Validate data continuity
|
|
144
|
-
if (prevHead &&
|
|
145
|
+
if (prevHead && prevHead.number >= blocks[0].header.number) {
|
|
145
146
|
throw new Error('Data is not continuous');
|
|
146
147
|
}
|
|
147
148
|
let unfinalizedIndex = 0;
|
|
148
|
-
if (finalizedHeadData
|
|
149
|
+
if (finalizedHeadData != null) {
|
|
149
150
|
unfinalizedIndex = blocks.findIndex((b) => b.header.number > finalizedHeadData.number);
|
|
150
151
|
}
|
|
151
152
|
let nextHead = (0, util_internal_1.last)(blocks).header;
|
|
@@ -185,9 +186,9 @@ class Processor {
|
|
|
185
186
|
state.unfinalizedHeads.push({ number: blocks[i].header.number, hash: blocks[i].header.hash });
|
|
186
187
|
}
|
|
187
188
|
await this.db.transactHot2({
|
|
188
|
-
finalizedHead:
|
|
189
|
-
baseHead:
|
|
190
|
-
newBlocks:
|
|
189
|
+
finalizedHead: toHashAndHeight(finalizedHeadData ?? state.finalizedHead),
|
|
190
|
+
baseHead: toHashAndHeight(prevHead ?? state.finalizedHead),
|
|
191
|
+
newBlocks: blocks.map((b) => toHashAndHeight(b.header)),
|
|
191
192
|
}, (store, start, end) => {
|
|
192
193
|
return this.handler({
|
|
193
194
|
store,
|
|
@@ -235,11 +236,11 @@ function findRollbackIndex(currentChain, forkChain) {
|
|
|
235
236
|
const currentBlock = currentChain[currentIndex];
|
|
236
237
|
const forkBlock = forkChain[forkIndex];
|
|
237
238
|
if (currentBlock.number > forkBlock.number) {
|
|
238
|
-
|
|
239
|
+
forkIndex++;
|
|
239
240
|
continue;
|
|
240
241
|
}
|
|
241
242
|
if (currentBlock.number < forkBlock.number) {
|
|
242
|
-
|
|
243
|
+
currentIndex++;
|
|
243
244
|
continue;
|
|
244
245
|
}
|
|
245
246
|
if (currentBlock.hash !== forkBlock.hash) {
|
package/lib/run.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run.js","sourceRoot":"","sources":["../src/run.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkDA,kBAaC;AA/DD,6CAA6C;AAC7C,2DAA8E;AAC9E,+FAAgF;AAChF,kDAAmC;AAEnC,uCAAiC;AACjC,mFAA0G;AAC1G,iCAAgD;AAChD,oDAA2B;AAE3B,MAAM,GAAG,GAAG,IAAA,qBAAY,EAAC,qBAAqB,CAAC,CAAA;AA0B/C;;;;;;;;;;;;;GAaG;AACH,SAAgB,GAAG,CACf,GAAsB,EACtB,EAAmB,EACnB,WAAqE;IAErE,IAAA,0BAAU,EACN,GAAG,EAAE;QACD,OAAO,IAAI,SAAS,CAAC,GAAG,EAAE,EAAE,EAAE,WAAW,CAAC,CAAC,GAAG,EAAE,CAAA;IACpD,CAAC,EACD,CAAC,GAAG,EAAE,EAAE;QACJ,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;IAClB,CAAC,CACJ,CAAA;AACL,CAAC;AAED,MAAM,SAAS;IAKX,YACY,GAAkB,EAClB,EAAe,EACf,OAAyD;QAFzD,QAAG,GAAH,GAAG,CAAe;QAClB,OAAE,GAAF,EAAE,CAAa;QACf,YAAO,GAAP,OAAO,CAAkD;QAP7D,YAAO,GAAG,IAAI,iBAAO,EAAE,CAAA;QAEvB,kBAAa,GAAG,KAAK,CAAA;IAM1B,CAAC;IAEJ,KAAK,CAAC,GAAG;QACL,IAAI,OAAO,GAAG,IAAI,CAAC,EAAE,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACpH,IAAI,WAAW,GAAG,IAAI,yBAAS,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC,EAAE,KAAM,CAAC,CAAA;QAEtF,IAAI,KAAK,GAAmB;YACxB,aAAa,EAAE,SAAS;YACxB,gBAAgB,EAAE,EAAE;SACvB,CAAA;QACD,IAAI,IAAI,CAAC,EAAE,CAAC,iBAAiB,EAAE,CAAC;YAC5B,IAAI,OAAO,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAA;YACrC,KAAK,CAAC,aAAa,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,CAAA;YAC1E,KAAK,CAAC,gBAAgB,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,CAAA;QACxD,CAAC;aAAM,CAAC;YACJ,IAAI,OAAO,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAA;YACrC,KAAK,CAAC,aAAa,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,CAAA;QAC9E,CAAC;QAED,IAAI,IAAI,GAAG,YAAY,CAAC,KAAK,CAAC,CAAA;QAC9B,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC;YACf,GAAG,CAAC,IAAI,CAAC,4BAA4B,IAAI,CAAC,MAAM,EAAE,CAAC,CAAA;QACvD,CAAC;QACD,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,MAAM,IAAI,CAAC,CAAC,EAAE,MAAM,WAAW,CAAC,GAAG,EAAE,CAAC,CAAA;QAEnE,OAAO,IAAI,EAAE,CAAC;YACV,IAAI,SAAS,GAAG,IAAI,CAAC,EAAE,CAAC,iBAAiB,CAAC,CAAC,
|
|
1
|
+
{"version":3,"file":"run.js","sourceRoot":"","sources":["../src/run.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkDA,kBAaC;AA/DD,6CAA6C;AAC7C,2DAA8E;AAC9E,+FAAgF;AAChF,kDAAmC;AAEnC,uCAAiC;AACjC,mFAA0G;AAC1G,iCAAgD;AAChD,oDAA2B;AAE3B,MAAM,GAAG,GAAG,IAAA,qBAAY,EAAC,qBAAqB,CAAC,CAAA;AA0B/C;;;;;;;;;;;;;GAaG;AACH,SAAgB,GAAG,CACf,GAAsB,EACtB,EAAmB,EACnB,WAAqE;IAErE,IAAA,0BAAU,EACN,GAAG,EAAE;QACD,OAAO,IAAI,SAAS,CAAC,GAAG,EAAE,EAAE,EAAE,WAAW,CAAC,CAAC,GAAG,EAAE,CAAA;IACpD,CAAC,EACD,CAAC,GAAG,EAAE,EAAE;QACJ,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;IAClB,CAAC,CACJ,CAAA;AACL,CAAC;AAED,MAAM,SAAS;IAKX,YACY,GAAkB,EAClB,EAAe,EACf,OAAyD;QAFzD,QAAG,GAAH,GAAG,CAAe;QAClB,OAAE,GAAF,EAAE,CAAa;QACf,YAAO,GAAP,OAAO,CAAkD;QAP7D,YAAO,GAAG,IAAI,iBAAO,EAAE,CAAA;QAEvB,kBAAa,GAAG,KAAK,CAAA;IAM1B,CAAC;IAEJ,KAAK,CAAC,GAAG;QACL,IAAI,OAAO,GAAG,IAAI,CAAC,EAAE,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACpH,IAAI,WAAW,GAAG,IAAI,yBAAS,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC,EAAE,KAAM,CAAC,CAAA;QAEtF,IAAI,KAAK,GAAmB;YACxB,aAAa,EAAE,SAAS;YACxB,gBAAgB,EAAE,EAAE;SACvB,CAAA;QACD,IAAI,IAAI,CAAC,EAAE,CAAC,iBAAiB,EAAE,CAAC;YAC5B,IAAI,OAAO,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAA;YACrC,KAAK,CAAC,aAAa,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,CAAA;YAC1E,KAAK,CAAC,gBAAgB,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,CAAA;QACxD,CAAC;aAAM,CAAC;YACJ,IAAI,OAAO,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAA;YACrC,KAAK,CAAC,aAAa,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,CAAA;QAC9E,CAAC;QAED,IAAI,IAAI,GAAG,YAAY,CAAC,KAAK,CAAC,CAAA;QAC9B,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC;YACf,GAAG,CAAC,IAAI,CAAC,4BAA4B,IAAI,CAAC,MAAM,EAAE,CAAC,CAAA;QACvD,CAAC;QACD,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,MAAM,IAAI,CAAC,CAAC,EAAE,MAAM,WAAW,CAAC,GAAG,EAAE,CAAC,CAAA;QAEnE,OAAO,IAAI,EAAE,CAAC;YACV,IAAI,SAAS,GAAG,IAAI,CAAC,EAAE,CAAC,iBAAiB;gBACrC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;gBACnC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YAEhD,IAAI,GAAG,YAAY,CAAC,KAAK,CAAC,CAAA;YAC1B,IAAI,CAAC;gBACD,IAAI,KAAK,EAAE,IAAI,IAAI,IAAI,SAAS,CAAC,EAAC,KAAK,EAAE,IAAI,EAAC,CAAC,EAAE,CAAC;oBAC9C,KAAK,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC,GAAG,EAAE,CAAC,CAAA;gBACzE,CAAC;gBACD,MAAK,CAAC,2CAA2C;YACrD,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACT,IAAI,CAAC,IAAA,2CAAe,EAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,iBAAiB;oBAAE,MAAM,CAAC,CAAA;gBAE9D,yCAAyC;gBACzC,IAAI,KAAK,GAAG,KAAK,CAAC,aAAa;oBAC3B,CAAC,CAAC,CAAC,KAAK,CAAC,aAAa,EAAE,GAAG,KAAK,CAAC,gBAAgB,CAAC;oBAClD,CAAC,CAAC,KAAK,CAAC,gBAAgB,CAAA;gBAC5B,IAAI,aAAa,GAAG,iBAAiB,CAAC,KAAK,EAAE,CAAC,CAAC,cAAc,CAAC,CAAA;gBAC9D,IAAI,aAAa,KAAK,CAAC,CAAC,EAAE,CAAC;oBACvB,IAAI,KAAK,CAAC,aAAa,IAAI,IAAI;wBAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAA;oBAC1E,KAAK,CAAC,gBAAgB,GAAG,EAAE,CAAA;gBAC/B,CAAC;qBAAM,CAAC;oBACJ,MAAM,YAAY,GAAG,KAAK,CAAC,aAAa,CAAC,CAAA;oBACzC,GAAG,CAAC,IAAI,CAAC,sCAAsC,IAAA,iBAAU,EAAC,YAAY,CAAC,EAAE,CAAC,CAAA;oBAE1E,KAAK,CAAC,gBAAgB,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,aAAa,GAAG,CAAC,CAAC,CAAA;gBAC9D,CAAC;YACL,CAAC;QACL,CAAC;QAED,IAAI,CAAC,iBAAiB,EAAE,CAAA;IAC5B,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,KAAa,EAAE,WAAmB;QACxD,IAAI,CAAC,qBAAqB,CAAC,WAAW,EAAE,KAAK,CAAC,CAAA;QAC9C,IAAI,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,yBAAyB,IAAI,OAAO,CAAC,GAAG,CAAC,eAAe,CAAA;QAC/E,IAAI,IAAI,IAAI,IAAI;YAAE,OAAM;QACxB,IAAI,CAAC,qBAAqB,EAAE,CAAA;QAC5B,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAA;QACtB,IAAI,MAAM,GAAG,MAAM,IAAA,wDAAsB,EAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAA;QAC9D,GAAG,CAAC,IAAI,CAAC,yCAAyC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAA;IACpE,CAAC;IAEO,qBAAqB,CAAC,WAAmB,EAAE,aAAqB,EAAE,IAAa;QACnF,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,WAAW,CAAC,CAAA;QACxC,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC,aAAa,CAAC,CAAA;QACjD,IAAI,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,qBAAqB,EAAE,CAAA;QAC/E,IAAI,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,qBAAqB,EAAE,CAAA;QACpD,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,CAAC,CAAA;IACtD,CAAC;IAEO,KAAK,CAAC,YAAY,CACtB,KAAqB,EACrB,IAAmB,EACnB,WAAmB;QAEnB,IAAI,EAAC,MAAM,EAAE,aAAa,EAAE,iBAAiB,EAAC,GAAG,IAAI,CAAA;QAErD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,KAAK,CAAA;QAErC,IAAI,QAAQ,GAAG,YAAY,CAAC,KAAK,CAAC,CAAA;QAElC,2BAA2B;QAC3B,IAAI,QAAQ,IAAI,QAAQ,CAAC,MAAM,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACzD,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAA;QAC7C,CAAC;QAED,IAAI,gBAAgB,GAAG,CAAC,CAAA;QACxB,IAAI,iBAAiB,IAAI,IAAI,EAAE,CAAC;YAC5B,gBAAgB,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,iBAAkB,CAAC,MAAM,CAAC,CAAA;QAC3F,CAAC;QAED,IAAI,QAAQ,GAAG,IAAA,oBAAI,EAAC,MAAM,CAAC,CAAC,MAAM,CAAA;QAClC,IAAI,OAAO,GAAG,QAAQ,CAAC,MAAM,IAAI,WAAW,CAAA;QAE5C,IAAI,gBAAgB,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,CAAA;QAE9C,+BAA+B;QAC/B,IAAI,gBAAgB,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,CAAA;YACrD,KAAK,CAAC,aAAa,GAAG,EAAC,MAAM,EAAE,YAAY,CAAC,MAAM,EAAE,IAAI,EAAE,YAAY,CAAC,IAAI,EAAC,CAAA;YAC5E,KAAK,CAAC,gBAAgB,GAAG,EAAE,CAAA;YAE3B,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAClB;gBACI,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAC,MAAM,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAC;gBACzE,QAAQ,EAAE,eAAe,CAAC,QAAQ,CAAC;gBACnC,OAAO;aACV,EACD,CAAC,KAAK,EAAE,EAAE;gBACN,OAAO,IAAI,CAAC,OAAO,CAAC;oBAChB,KAAK;oBACL,MAAM;oBACN,MAAM,EAAE,OAAO;iBAClB,CAAC,CAAA;YACN,CAAC,CACJ,CAAA;QACL,CAAC;aAAM,CAAC;YACJ,IAAA,gBAAM,EAAC,IAAI,CAAC,EAAE,CAAC,iBAAiB,CAAC,CAAA;YAEjC,IAAI,YAAY,GAAyB,MAAM,CAAC,gBAAgB,GAAG,CAAC,CAAC,EAAE,MAAM,CAAA;YAC7E,IAAI,iBAAiB,EAAE,IAAI,IAAI,iBAAiB,EAAE,MAAM,IAAI,IAAI,EAAE,CAAC;gBAC/D,YAAY,GAAG,EAAC,IAAI,EAAE,iBAAiB,CAAC,IAAI,EAAE,MAAM,EAAE,iBAAiB,CAAC,MAAM,EAAC,CAAA;YACnF,CAAC;YACD,KAAK,CAAC,aAAa,GAAG,YAAY,IAAI,KAAK,CAAC,aAAa,CAAA;YAEzD,2DAA2D;YAC3D,IAAI,KAAK,CAAC,aAAa,EAAE,CAAC;gBACtB,IAAI,aAAa,GAAG,KAAK,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,aAAc,CAAC,MAAM,CAAC,CAAA;gBACnG,KAAK,CAAC,gBAAgB,GAAG,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,gBAAgB,CAAC,KAAK,CAAC,aAAa,CAAC,CAAA;YACjG,CAAC;YAED,6BAA6B;YAC7B,KAAK,IAAI,CAAC,GAAG,gBAAgB,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACpD,KAAK,CAAC,gBAAgB,CAAC,IAAI,CAAC,EAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,EAAC,CAAC,CAAA;YAC/F,CAAC;YAED,MAAM,IAAI,CAAC,EAAE,CAAC,YAAY,CACtB;gBACI,aAAa,EAAE,eAAe,CAAC,iBAAiB,IAAI,KAAK,CAAC,aAAa,CAAC;gBACxE,QAAQ,EAAE,eAAe,CAAC,QAAQ,IAAI,KAAK,CAAC,aAAa,CAAC;gBAC1D,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;aAC1D,EACD,CAAC,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;gBAClB,OAAO,IAAI,CAAC,OAAO,CAAC;oBAChB,KAAK;oBACL,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC;oBAChC,MAAM,EAAE,OAAO;iBAClB,CAAC,CAAA;YACN,CAAC,CACJ,CAAA;QACL,CAAC;QAED,IAAI,cAAc,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,CAAA;QAE5C,IAAI,CAAC,qBAAqB,CAAC,WAAW,EAAE,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC,CAAA;QACxE,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,MAAM,EAAE,IAAA,oBAAa,EAAC,MAAM,CAAC,EAAE,gBAAgB,EAAE,cAAc,CAAC,CAAA;QAElG,IAAI,CAAC,YAAY,EAAE,CAAA;QAEnB,OAAO,KAAK,CAAA;IAChB,CAAC;IAEO,YAAY;QAChB,IAAI,IAAI,CAAC,iBAAiB,IAAI,IAAI,EAAE,CAAC;YACjC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAA;YACtC,IAAI,CAAC,iBAAiB,GAAG,UAAU,CAAC,GAAG,EAAE;gBACrC,IAAI,CAAC,iBAAiB,GAAG,SAAS,CAAA;gBAClC,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;oBACrB,IAAI,CAAC,aAAa,GAAG,KAAK,CAAA;oBAC1B,IAAI,CAAC,YAAY,EAAE,CAAA;gBACvB,CAAC;YACL,CAAC,EAAE,IAAK,CAAC,CAAA;QACb,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,aAAa,GAAG,IAAI,CAAA;QAC7B,CAAC;IACL,CAAC;IAEO,iBAAiB;QACrB,IAAI,IAAI,CAAC,iBAAiB,IAAI,IAAI,EAAE,CAAC;YACjC,YAAY,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAA;QACxC,CAAC;QACD,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACrB,IAAI,CAAC,aAAa,GAAG,KAAK,CAAA;YAC1B,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAA;QAC1C,CAAC;IACL,CAAC;CACJ;AAED,SAAS,iBAAiB,CAAC,YAAwB,EAAE,SAAqB;IACtE,IAAI,YAAY,GAAG,CAAC,CAAA;IACpB,IAAI,SAAS,GAAG,CAAC,CAAA;IACjB,IAAI,eAAe,GAAG,CAAC,CAAC,CAAA;IAExB,OAAO,YAAY,GAAG,YAAY,CAAC,MAAM,IAAI,SAAS,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC;QACxE,MAAM,YAAY,GAAG,YAAY,CAAC,YAAY,CAAC,CAAA;QAC/C,MAAM,SAAS,GAAG,SAAS,CAAC,SAAS,CAAC,CAAA;QAEtC,IAAI,YAAY,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC;YACzC,SAAS,EAAE,CAAA;YACX,SAAQ;QACZ,CAAC;QAED,IAAI,YAAY,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC;YACzC,YAAY,EAAE,CAAA;YACd,SAAQ;QACZ,CAAC;QAED,IAAI,YAAY,CAAC,IAAI,KAAK,SAAS,CAAC,IAAI,EAAE,CAAC;YACvC,OAAO,eAAe,CAAA;QAC1B,CAAC;QAED,eAAe,GAAG,YAAY,CAAA;QAC9B,YAAY,EAAE,CAAA;QACd,SAAS,EAAE,CAAA;IACf,CAAC;IAED,OAAO,eAAe,CAAA;AAC1B,CAAC;AAED,SAAS,eAAe,CAAC,GAAa;IAClC,OAAO,EAAC,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAC,CAAA;AAC/C,CAAC;AAED,SAAS,UAAU,CAAC,aAA4B;IAC5C,OAAO,EAAC,MAAM,EAAE,aAAa,CAAC,MAAM,EAAE,IAAI,EAAE,aAAa,CAAC,IAAI,EAAC,CAAA;AACnE,CAAC;AAED,SAAS,YAAY,CAAC,KAAqB;IACvC,OAAO,IAAA,yBAAS,EAAC,KAAK,CAAC,gBAAgB,CAAC,IAAI,KAAK,CAAC,aAAa,CAAA;AACnE,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@subsquid/batch-processor",
|
|
3
|
-
"version": "0.1.0-portal-api.
|
|
3
|
+
"version": "0.1.0-portal-api.d0738f",
|
|
4
4
|
"description": "ETL processor",
|
|
5
5
|
"license": "GPL-3.0-or-later",
|
|
6
6
|
"repository": "git@github.com:subsquid/squid.git",
|
|
@@ -20,7 +20,7 @@
|
|
|
20
20
|
"@subsquid/util-internal-prometheus-server": "^1.3.0",
|
|
21
21
|
"@subsquid/util-internal-range": "^0.3.0",
|
|
22
22
|
"prom-client": "^14.2.0",
|
|
23
|
-
"@subsquid/util-internal-data-source": "0.0.1-portal-api.
|
|
23
|
+
"@subsquid/util-internal-data-source": "0.0.1-portal-api.d0738f"
|
|
24
24
|
},
|
|
25
25
|
"devDependencies": {
|
|
26
26
|
"@types/node": "^18.18.14",
|
package/src/run.ts
CHANGED
|
@@ -98,16 +98,20 @@ class Processor<B extends BlockBase, S> {
|
|
|
98
98
|
await this.initMetrics(head?.number ?? -1, await chainHeight.get())
|
|
99
99
|
|
|
100
100
|
while (true) {
|
|
101
|
-
let getStream = this.db.supportsHotBlocks
|
|
101
|
+
let getStream = this.db.supportsHotBlocks
|
|
102
|
+
? this.src.getStream.bind(this.src)
|
|
103
|
+
: this.src.getFinalizedStream.bind(this.src)
|
|
102
104
|
|
|
103
105
|
head = getStateHead(state)
|
|
104
106
|
try {
|
|
105
107
|
for await (let data of getStream({after: head})) {
|
|
106
108
|
state = await this.processBatch(state, data, await chainHeight.get())
|
|
107
109
|
}
|
|
110
|
+
break // Stream completed successfully, exit loop
|
|
108
111
|
} catch (e) {
|
|
109
112
|
if (!isForkException(e) || !this.db.supportsHotBlocks) throw e
|
|
110
113
|
|
|
114
|
+
// Handle fork and continue loop to retry
|
|
111
115
|
let chain = state.finalizedHead
|
|
112
116
|
? [state.finalizedHead, ...state.unfinalizedHeads]
|
|
113
117
|
: state.unfinalizedHeads
|
|
@@ -122,8 +126,6 @@ class Processor<B extends BlockBase, S> {
|
|
|
122
126
|
state.unfinalizedHeads = chain.slice(1, rollbackIndex + 1)
|
|
123
127
|
}
|
|
124
128
|
}
|
|
125
|
-
|
|
126
|
-
break
|
|
127
129
|
}
|
|
128
130
|
|
|
129
131
|
this.reportFinalStatus()
|
|
@@ -142,10 +144,8 @@ class Processor<B extends BlockBase, S> {
|
|
|
142
144
|
private updateProgressMetrics(chainHeight: number, indexerHeight: number, time?: bigint): void {
|
|
143
145
|
this.metrics.setChainHeight(chainHeight)
|
|
144
146
|
this.metrics.setLastProcessedBlock(indexerHeight)
|
|
145
|
-
let left
|
|
146
|
-
let processed
|
|
147
|
-
left = this.metrics.getChainHeight() - this.metrics.getLastProcessedBlock()
|
|
148
|
-
processed = this.metrics.getLastProcessedBlock()
|
|
147
|
+
let left = this.metrics.getChainHeight() - this.metrics.getLastProcessedBlock()
|
|
148
|
+
let processed = this.metrics.getLastProcessedBlock()
|
|
149
149
|
this.metrics.updateProgress(processed, left, time)
|
|
150
150
|
}
|
|
151
151
|
|
|
@@ -161,13 +161,13 @@ class Processor<B extends BlockBase, S> {
|
|
|
161
161
|
let prevHead = getStateHead(state)
|
|
162
162
|
|
|
163
163
|
// Validate data continuity
|
|
164
|
-
if (prevHead &&
|
|
164
|
+
if (prevHead && prevHead.number >= blocks[0].header.number) {
|
|
165
165
|
throw new Error('Data is not continuous')
|
|
166
166
|
}
|
|
167
167
|
|
|
168
168
|
let unfinalizedIndex = 0
|
|
169
|
-
if (finalizedHeadData
|
|
170
|
-
unfinalizedIndex = blocks.findIndex((b) => b.header.number > finalizedHeadData
|
|
169
|
+
if (finalizedHeadData != null) {
|
|
170
|
+
unfinalizedIndex = blocks.findIndex((b) => b.header.number > finalizedHeadData!.number)
|
|
171
171
|
}
|
|
172
172
|
|
|
173
173
|
let nextHead = last(blocks).header
|
|
@@ -217,9 +217,9 @@ class Processor<B extends BlockBase, S> {
|
|
|
217
217
|
|
|
218
218
|
await this.db.transactHot2(
|
|
219
219
|
{
|
|
220
|
-
finalizedHead:
|
|
221
|
-
baseHead:
|
|
222
|
-
newBlocks:
|
|
220
|
+
finalizedHead: toHashAndHeight(finalizedHeadData ?? state.finalizedHead),
|
|
221
|
+
baseHead: toHashAndHeight(prevHead ?? state.finalizedHead),
|
|
222
|
+
newBlocks: blocks.map((b) => toHashAndHeight(b.header)),
|
|
223
223
|
},
|
|
224
224
|
(store, start, end) => {
|
|
225
225
|
return this.handler({
|
|
@@ -277,12 +277,12 @@ function findRollbackIndex(currentChain: BlockRef[], forkChain: BlockRef[]): num
|
|
|
277
277
|
const forkBlock = forkChain[forkIndex]
|
|
278
278
|
|
|
279
279
|
if (currentBlock.number > forkBlock.number) {
|
|
280
|
-
|
|
280
|
+
forkIndex++
|
|
281
281
|
continue
|
|
282
282
|
}
|
|
283
283
|
|
|
284
284
|
if (currentBlock.number < forkBlock.number) {
|
|
285
|
-
|
|
285
|
+
currentIndex++
|
|
286
286
|
continue
|
|
287
287
|
}
|
|
288
288
|
|