envio 3.5.0-alpha.2 → 3.5.0-rc.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (129) hide show
  1. package/package.json +6 -6
  2. package/src/Batch.res +8 -8
  3. package/src/Batch.res.mjs +4 -3
  4. package/src/BatchProcessing.res +24 -0
  5. package/src/BatchProcessing.res.mjs +12 -0
  6. package/src/ChainFetching.res +20 -19
  7. package/src/ChainFetching.res.mjs +14 -14
  8. package/src/ChainId.res +58 -0
  9. package/src/ChainId.res.mjs +49 -0
  10. package/src/ChainId.resi +27 -0
  11. package/src/ChainMap.res +12 -26
  12. package/src/ChainMap.res.mjs +5 -41
  13. package/src/ChainMap.resi +5 -20
  14. package/src/ChainMetadata.res +1 -1
  15. package/src/ChainMetadata.res.mjs +3 -2
  16. package/src/ChainState.res +45 -24
  17. package/src/ChainState.res.mjs +40 -16
  18. package/src/ChainState.resi +3 -1
  19. package/src/Config.res +43 -26
  20. package/src/Config.res.mjs +19 -16
  21. package/src/ContractRegisterContext.res +1 -1
  22. package/src/ContractRegisterContext.res.mjs +1 -1
  23. package/src/Core.res +1 -0
  24. package/src/CrossChainState.res +90 -22
  25. package/src/CrossChainState.res.mjs +55 -16
  26. package/src/CrossChainState.resi +5 -1
  27. package/src/Env.res +0 -3
  28. package/src/Env.res.mjs +0 -3
  29. package/src/EventConfigBuilder.res +1 -1
  30. package/src/EventProcessing.res +4 -4
  31. package/src/EventProcessing.res.mjs +5 -5
  32. package/src/FetchState.res +489 -153
  33. package/src/FetchState.res.mjs +399 -129
  34. package/src/FinalizeBackfill.res +61 -0
  35. package/src/FinalizeBackfill.res.mjs +44 -0
  36. package/src/HandlerRegister.res +8 -8
  37. package/src/HandlerRegister.res.mjs +5 -4
  38. package/src/HandlerRegister.resi +1 -1
  39. package/src/InMemoryStore.res +4 -0
  40. package/src/InMemoryStore.res.mjs +1 -1
  41. package/src/InMemoryTable.res +15 -15
  42. package/src/InMemoryTable.res.mjs +14 -14
  43. package/src/IndexerLoop.res +9 -2
  44. package/src/IndexerLoop.res.mjs +8 -1
  45. package/src/IndexerState.res +52 -23
  46. package/src/IndexerState.res.mjs +52 -14
  47. package/src/IndexerState.resi +11 -6
  48. package/src/Internal.res +19 -17
  49. package/src/Internal.res.mjs +18 -9
  50. package/src/LoadLayer.res +10 -2
  51. package/src/LoadLayer.res.mjs +3 -1
  52. package/src/LogSelection.res +2 -2
  53. package/src/Main.res +7 -11
  54. package/src/Main.res.mjs +4 -6
  55. package/src/Metrics.res +6 -6
  56. package/src/Metrics.res.mjs +4 -3
  57. package/src/Persistence.res +23 -4
  58. package/src/PgStorage.res +447 -100
  59. package/src/PgStorage.res.mjs +285 -99
  60. package/src/RawEvent.res +1 -2
  61. package/src/RawEvent.res.mjs +1 -1
  62. package/src/Rollback.res +18 -20
  63. package/src/Rollback.res.mjs +9 -9
  64. package/src/RollbackCommit.res +2 -2
  65. package/src/RollbackCommit.res.mjs +3 -3
  66. package/src/SafeCheckpointTracking.res +2 -2
  67. package/src/SimulateDeadInputTracker.res +17 -21
  68. package/src/SimulateDeadInputTracker.res.mjs +9 -8
  69. package/src/SimulateItems.res +8 -10
  70. package/src/SimulateItems.res.mjs +9 -10
  71. package/src/Sink.res +2 -2
  72. package/src/Sink.res.mjs +3 -2
  73. package/src/TestIndexer.res +14 -14
  74. package/src/TestIndexer.res.mjs +16 -10
  75. package/src/UserContext.res +2 -2
  76. package/src/UserContext.res.mjs +2 -1
  77. package/src/Utils.res +0 -4
  78. package/src/Utils.res.mjs +1 -6
  79. package/src/bindings/ClickHouse.res +20 -2
  80. package/src/bindings/ClickHouse.res.mjs +25 -14
  81. package/src/db/EntityFilter.res +0 -4
  82. package/src/db/EntityFilter.res.mjs +1 -8
  83. package/src/db/EntityHistory.res +1 -1
  84. package/src/db/IndexCatalog.res +248 -0
  85. package/src/db/IndexCatalog.res.mjs +222 -0
  86. package/src/db/IndexDefinition.res +131 -0
  87. package/src/db/IndexDefinition.res.mjs +138 -0
  88. package/src/db/IndexManager.res +145 -0
  89. package/src/db/IndexManager.res.mjs +130 -0
  90. package/src/db/InternalTable.res +54 -23
  91. package/src/db/InternalTable.res.mjs +38 -23
  92. package/src/db/Table.res +24 -14
  93. package/src/db/Table.res.mjs +21 -16
  94. package/src/sources/AddressSet.res +5 -4
  95. package/src/sources/AddressStore.res +25 -14
  96. package/src/sources/AddressStore.res.mjs +10 -12
  97. package/src/sources/Evm.res +2 -2
  98. package/src/sources/Evm.res.mjs +1 -1
  99. package/src/sources/EvmChain.res +3 -3
  100. package/src/sources/EvmChain.res.mjs +3 -3
  101. package/src/sources/EvmHyperSyncSource.res +7 -7
  102. package/src/sources/EvmHyperSyncSource.res.mjs +6 -6
  103. package/src/sources/Fuel.res +2 -2
  104. package/src/sources/Fuel.res.mjs +1 -1
  105. package/src/sources/FuelHyperSyncClient.res +3 -0
  106. package/src/sources/FuelHyperSyncClient.res.mjs +1 -0
  107. package/src/sources/FuelHyperSyncSource.res +5 -7
  108. package/src/sources/FuelHyperSyncSource.res.mjs +5 -5
  109. package/src/sources/HyperSync.resi +2 -2
  110. package/src/sources/HyperSyncClient.res +5 -0
  111. package/src/sources/HyperSyncClient.res.mjs +1 -0
  112. package/src/sources/RpcSource.res +13 -14
  113. package/src/sources/RpcSource.res.mjs +13 -12
  114. package/src/sources/SimulateSource.res +42 -15
  115. package/src/sources/SimulateSource.res.mjs +27 -9
  116. package/src/sources/Source.res +1 -5
  117. package/src/sources/SourceManager.res +12 -12
  118. package/src/sources/SourceManager.res.mjs +11 -10
  119. package/src/sources/SourceManager.resi +2 -2
  120. package/src/sources/Svm.res +4 -5
  121. package/src/sources/Svm.res.mjs +5 -4
  122. package/src/sources/SvmHyperSyncClient.res +3 -0
  123. package/src/sources/SvmHyperSyncClient.res.mjs +1 -0
  124. package/src/sources/SvmHyperSyncSource.res +4 -4
  125. package/src/sources/SvmHyperSyncSource.res.mjs +3 -3
  126. package/src/tui/Tui.res +1 -1
  127. package/src/tui/Tui.res.mjs +2 -1
  128. package/src/tui/components/CustomHooks.res +5 -4
  129. package/src/tui/components/CustomHooks.res.mjs +4 -2
@@ -3,6 +3,7 @@
3
3
  import * as Env from "./Env.res.mjs";
4
4
  import * as Utils from "./Utils.res.mjs";
5
5
  import * as Config from "./Config.res.mjs";
6
+ import * as ChainId from "./ChainId.res.mjs";
6
7
  import * as Logging from "./Logging.res.mjs";
7
8
  import * as ChainMap from "./ChainMap.res.mjs";
8
9
  import * as EntityId from "./EntityId.res.mjs";
@@ -38,7 +39,7 @@ function getIndexingAddressesByChain(state) {
38
39
  let dcDict = state.entities[Config.EnvioAddresses.name];
39
40
  if (dcDict !== undefined) {
40
41
  Object.values(dcDict).forEach(entity => {
41
- let chainIdStr = entity.chain_id.toString();
42
+ let chainIdStr = ChainId.toString(entity.chain_id);
42
43
  let arr = byChain[chainIdStr];
43
44
  let contracts;
44
45
  if (arr !== undefined) {
@@ -116,7 +117,7 @@ function handleWriteBatch(state, updatedEntities, checkpointIds, checkpointChain
116
117
  for (let i = 0, i_finish = checkpointIds.length; i < i_finish; ++i) {
117
118
  let checkpointId = checkpointIds[i];
118
119
  let change = {};
119
- state.progressBlockByChain[checkpointChainIds[i].toString()] = checkpointBlockNumbers[i];
120
+ state.progressBlockByChain[ChainId.toString(checkpointChainIds[i])] = checkpointBlockNumbers[i];
120
121
  change["block"] = checkpointBlockNumbers[i];
121
122
  change["chainId"] = checkpointChainIds[i];
122
123
  change["eventsProcessed"] = checkpointEventsProcessed[i];
@@ -157,15 +158,14 @@ function handleWriteBatch(state, updatedEntities, checkpointIds, checkpointChain
157
158
  function makeInitialState(config, processConfigChains, indexingAddressesByChain) {
158
159
  let chainKeys = Object.keys(processConfigChains);
159
160
  let chains = chainKeys.map(chainIdStr => {
160
- let chainId = Stdlib_Option.getOr(Stdlib_Int.fromString(chainIdStr, undefined), 0);
161
- let chain = ChainMap.Chain.makeUnsafe(chainId);
161
+ let chain = ChainId.normalizeOrThrow(chainIdStr);
162
162
  if (!ChainMap.has(config.chainMap, chain)) {
163
163
  Stdlib_JsError.throwWithMessage(`Chain ` + chainIdStr + ` is not configured in config.yaml`);
164
164
  }
165
165
  let processChainConfig = processConfigChains[chainIdStr];
166
166
  let indexingAddresses = Stdlib_Option.getOr(indexingAddressesByChain[chainIdStr], []);
167
167
  return {
168
- id: chainId,
168
+ id: chain,
169
169
  startBlock: processChainConfig.startBlock,
170
170
  endBlock: processChainConfig.endBlock,
171
171
  maxReorgDepth: 0,
@@ -222,9 +222,12 @@ function getSimulateEndBlock(simulateItems, config, startBlock) {
222
222
  }
223
223
 
224
224
  function parseBlockRange(chainIdStr, config, rawChainConfig, progressBlock) {
225
- let id = Stdlib_Int.fromString(chainIdStr, undefined);
226
- let chainId = id !== undefined ? id : Stdlib_JsError.throwWithMessage(`Invalid chain ID "` + chainIdStr + `": expected a numeric chain ID`);
227
- let chain = ChainMap.Chain.makeUnsafe(chainId);
225
+ let chain;
226
+ try {
227
+ chain = ChainId.normalizeOrThrow(chainIdStr);
228
+ } catch (exn) {
229
+ chain = Stdlib_JsError.throwWithMessage(`Invalid chain ID "` + chainIdStr + `": expected a numeric chain ID`);
230
+ }
228
231
  if (!ChainMap.has(config.chainMap, chain)) {
229
232
  Stdlib_JsError.throwWithMessage(`Chain ` + chainIdStr + ` is not configured in config.yaml`);
230
233
  }
@@ -315,6 +318,9 @@ function makeInMemoryStorage(state) {
315
318
  initialize: async (param, param$1, param$2, param$3) => Stdlib_JsError.throwWithMessage("TestIndexer: initialize should not be called; the initial state is derived from config."),
316
319
  resumeInitialState: async () => Stdlib_JsError.throwWithMessage("TestIndexer: resumeInitialState should not be called; the initial state is derived from config."),
317
320
  loadOrThrow: async (filter, table) => handleLoad(state, table.tableName, filter),
321
+ ensureQueryIndexes: async (param, param$1) => {},
322
+ ensureSchemaIndexes: async param => {},
323
+ finalizeBackfill: async (param, param$1, param$2) => {},
318
324
  dumpEffectCache: async () => {},
319
325
  reset: async () => {},
320
326
  setChainMeta: async param => {},
@@ -410,7 +416,7 @@ function createTestIndexer() {
410
416
  let chainIds = [];
411
417
  let chains = Object.create(null);
412
418
  ChainMap.values(config.chainMap).forEach(chainConfig => {
413
- let chainIdStr = chainConfig.id.toString();
419
+ let chainIdStr = ChainId.toString(chainConfig.id);
414
420
  chainIds.push(chainConfig.id);
415
421
  let chainObj = Object.create(null);
416
422
  Object.defineProperty(Object.defineProperty(Object.defineProperty(Object.defineProperty(Object.defineProperty(chainObj, "id", {
@@ -443,7 +449,7 @@ function createTestIndexer() {
443
449
  if (state.processInProgress) {
444
450
  Stdlib_JsError.throwWithMessage(`Cannot access ` + contract.name + `.addresses while indexer.process() is running. ` + "Wait for process() to complete before reading contract addresses.");
445
451
  }
446
- return Stdlib_Array.filterMap(Stdlib_Option.getOr(getIndexingAddressesByChain(state)[chainConfig.id.toString()], []), ia => {
452
+ return Stdlib_Array.filterMap(Stdlib_Option.getOr(getIndexingAddressesByChain(state)[ChainId.toString(chainConfig.id)], []), ia => {
447
453
  if (ia.contractName === contract.name) {
448
454
  return Primitive_option.some(ia.address);
449
455
  }
@@ -52,7 +52,7 @@ EffectContext.prototype = effectContextPrototype;
52
52
  @new
53
53
  external makeEffectContext: (
54
54
  contextParams,
55
- ~chainId: option<int>,
55
+ ~chainId: option<ChainId.t>,
56
56
  ~effectName: string,
57
57
  ~defaultShouldCache: bool,
58
58
  ~callEffect: (Internal.effect, Internal.effectInput) => promise<Internal.effectOutput>,
@@ -318,7 +318,7 @@ let handlerTraps: Utils.Proxy.traps<contextParams> = {
318
318
  | "chain" =>
319
319
  let chainId = params.item->Internal.getItemChainId
320
320
  params.chains
321
- ->Utils.Dict.dangerouslyGetByIntNonOption(chainId)
321
+ ->ChainId.Dict.dangerouslyGetNonOption(chainId)
322
322
  ->(Utils.magic: option<Internal.chainInfo> => unknown)
323
323
  | _ =>
324
324
  switch params.config.userEntitiesByName->Utils.Dict.dangerouslyGetNonOption(prop) {
@@ -12,6 +12,7 @@ import * as InMemoryStore from "./InMemoryStore.res.mjs";
12
12
  import * as InMemoryTable from "./InMemoryTable.res.mjs";
13
13
  import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
14
14
  import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
15
+ import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
15
16
  import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
16
17
 
17
18
  let paramsByThis = new WeakMap();
@@ -54,7 +55,7 @@ function initEffect(params) {
54
55
  Stdlib_JsError.throwWithMessage(`The cross-chain effect "` + caller.name + `" cannot call the chain-scoped effect "` + effect.name + `", because a cross-chain effect isn't tied to a single chain. Make "` + effect.name + `" cross-chain (\`crossChain: true\`), or make "` + caller.name + `" chain-scoped (\`crossChain: false\`).`);
55
56
  }
56
57
  let tmp;
57
- tmp = scope === "crossChain" ? undefined : scope;
58
+ tmp = scope === "crossChain" ? undefined : Primitive_option.some(scope);
58
59
  let effectContext = new EffectContext(params, tmp, effect.name, effect.defaultShouldCache, makeCaller(effect));
59
60
  let effectArgs_cacheKey = Utils.Hash.makeOrThrow(S$RescriptSchema.reverseConvertOrThrow(input, effect.input));
60
61
  let effectArgs_checkpointId = params.checkpointId;
package/src/Utils.res CHANGED
@@ -218,10 +218,6 @@ module Dict = {
218
218
 
219
219
  @set_index
220
220
  external setByInt: (dict<'a>, int, 'a) => unit = ""
221
-
222
- let incrementByInt: (dict<int>, int) => unit = %raw(`(dict, key) => {
223
- dict[key]++
224
- }`)
225
221
  }
226
222
 
227
223
  module Math = {
package/src/Utils.res.mjs CHANGED
@@ -161,10 +161,6 @@ let updateImmutable = ((dict, key, value) => ({...dict, [key]: value}));
161
161
 
162
162
  let shallowCopy = ((dict) => ({...dict}));
163
163
 
164
- let incrementByInt = ((dict, key) => {
165
- dict[key]++
166
- });
167
-
168
164
  let Dict = {
169
165
  $$delete: Stdlib_Dict.$$delete,
170
166
  getOrInsertEmptyDict: getOrInsertEmptyDict,
@@ -181,8 +177,7 @@ let Dict = {
181
177
  deleteInPlace: deleteInPlace,
182
178
  unsafeDeleteUndefinedFieldsInPlace: unsafeDeleteUndefinedFieldsInPlace,
183
179
  updateImmutable: updateImmutable,
184
- shallowCopy: shallowCopy,
185
- incrementByInt: incrementByInt
180
+ shallowCopy: shallowCopy
186
181
  };
187
182
 
188
183
  function minOptInt(a, b) {
@@ -44,9 +44,15 @@ let getClickHouseFieldType = (
44
44
  ~fieldType: Table.fieldType,
45
45
  ~isNullable: bool,
46
46
  ~isArray: bool,
47
+ ~chainIdMode: ChainId.mode=Int32,
47
48
  ): string => {
48
49
  let baseType = switch fieldType {
49
50
  | Int32 => "Int32"
51
+ | ChainId =>
52
+ switch chainIdMode {
53
+ | Int32 => "Int32"
54
+ | Int64 => "UInt64"
55
+ }
50
56
  | Uint32 => "UInt32"
51
57
  | UInt52 => "UInt64"
52
58
  | UInt64 => "UInt64"
@@ -123,6 +129,7 @@ let makeClickHouseEntitySchema = (table: Table.table): S.t<Internal.entity> => {
123
129
  dateSchema
124
130
  }
125
131
  }
132
+ | ChainId => ChainId.schema->S.toUnknown
126
133
  // ClickHouse returns UInt64 values as strings, need to parse to float
127
134
  | UInt52 => {
128
135
  let uint52Schema =
@@ -337,6 +344,7 @@ let makeCreateHistoryTableQuery = (
337
344
  ~database: string,
338
345
  ~replicated: bool=false,
339
346
  ~onCluster: bool=false,
347
+ ~chainIdMode: ChainId.mode=Int32,
340
348
  ) => {
341
349
  let tableEngine = replicated ? "ReplicatedMergeTree" : "MergeTree()"
342
350
  let fieldDefinitions = entityConfig.table.fields->Array.filterMap(field => {
@@ -348,6 +356,7 @@ let makeCreateHistoryTableQuery = (
348
356
  ~fieldType=field.fieldType,
349
357
  ~isNullable=field.isNullable,
350
358
  ~isArray=field.isArray,
359
+ ~chainIdMode,
351
360
  )
352
361
  `\`${fieldName}\` ${clickHouseType}`
353
362
  })
@@ -446,6 +455,7 @@ let makeCreateCheckpointsTableQuery = (
446
455
  ~database: string,
447
456
  ~replicated: bool=false,
448
457
  ~onCluster: bool=false,
458
+ ~chainIdMode: ChainId.mode=Int32,
449
459
  ) => {
450
460
  let tableEngine = replicated ? "ReplicatedMergeTree" : "MergeTree()"
451
461
  let idField = (#id: InternalTable.Checkpoints.field :> string)
@@ -459,9 +469,10 @@ let makeCreateCheckpointsTableQuery = (
459
469
  )} (
460
470
  \`${idField}\` ${getClickHouseFieldType(~fieldType=UInt64, ~isNullable=false, ~isArray=false)},
461
471
  \`${chainIdField}\` ${getClickHouseFieldType(
462
- ~fieldType=Int32,
472
+ ~fieldType=ChainId,
463
473
  ~isNullable=false,
464
474
  ~isArray=false,
475
+ ~chainIdMode,
465
476
  )},
466
477
  \`${blockNumberField}\` ${getClickHouseFieldType(
467
478
  ~fieldType=Int32,
@@ -528,6 +539,7 @@ let initialize = async (
528
539
  ~database: string,
529
540
  ~entities: array<Internal.entityConfig>,
530
541
  ~enums as _: array<Table.enumConfig<Table.enum>>,
542
+ ~chainIdMode: ChainId.mode=Int32,
531
543
  ) => {
532
544
  try {
533
545
  let databaseEngine = Env.ClickHouse.databaseEngine()
@@ -600,12 +612,18 @@ let initialize = async (
600
612
  ~database,
601
613
  ~replicated,
602
614
  ~onCluster=ddlOnCluster,
615
+ ~chainIdMode,
603
616
  ),
604
617
  })
605
618
  ),
606
619
  )->Utils.Promise.ignoreValue
607
620
  await client->exec({
608
- query: makeCreateCheckpointsTableQuery(~database, ~replicated, ~onCluster=ddlOnCluster),
621
+ query: makeCreateCheckpointsTableQuery(
622
+ ~database,
623
+ ~replicated,
624
+ ~onCluster=ddlOnCluster,
625
+ ~chainIdMode,
626
+ ),
609
627
  })
610
628
 
611
629
  // The client pools HTTP connections, so consecutive statements may reach
@@ -3,6 +3,7 @@
3
3
  import * as Env from "../Env.res.mjs";
4
4
  import * as Table from "../db/Table.res.mjs";
5
5
  import * as Utils from "../Utils.res.mjs";
6
+ import * as ChainId from "../ChainId.res.mjs";
6
7
  import * as Logging from "../Logging.res.mjs";
7
8
  import * as Persistence from "../Persistence.res.mjs";
8
9
  import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
@@ -13,7 +14,8 @@ import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js
13
14
  import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
14
15
  import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
15
16
 
16
- function getClickHouseFieldType(fieldType, isNullable, isArray) {
17
+ function getClickHouseFieldType(fieldType, isNullable, isArray, chainIdModeOpt) {
18
+ let chainIdMode = chainIdModeOpt !== undefined ? chainIdModeOpt : "int32";
17
19
  let baseType;
18
20
  if (typeof fieldType !== "object") {
19
21
  switch (fieldType) {
@@ -27,6 +29,9 @@ function getClickHouseFieldType(fieldType, isNullable, isArray) {
27
29
  case "UInt64" :
28
30
  baseType = "UInt64";
29
31
  break;
32
+ case "ChainId" :
33
+ baseType = chainIdMode === "int32" ? "Int32" : "UInt64";
34
+ break;
30
35
  case "Number" :
31
36
  baseType = "Float64";
32
37
  break;
@@ -99,6 +104,9 @@ function makeClickHouseEntitySchema(table) {
99
104
  f.isArray ? S$RescriptSchema.array(uint52Schema) : uint52Schema
100
105
  );
101
106
  break;
107
+ case "ChainId" :
108
+ fieldSchema = ChainId.schema;
109
+ break;
102
110
  case "Date" :
103
111
  let dateSchema = Utils.Schema.clickHouseDate;
104
112
  fieldSchema = f.isNullable ? S$RescriptSchema.$$null(dateSchema) : (
@@ -244,9 +252,10 @@ function databaseEngineName(engineSpec) {
244
252
  return engineSpec.trim().split("(")[0].split(" ")[0].trim();
245
253
  }
246
254
 
247
- function makeCreateHistoryTableQuery(entityConfig, database, replicatedOpt, onClusterOpt) {
255
+ function makeCreateHistoryTableQuery(entityConfig, database, replicatedOpt, onClusterOpt, chainIdModeOpt) {
248
256
  let replicated = replicatedOpt !== undefined ? replicatedOpt : false;
249
257
  let onCluster = onClusterOpt !== undefined ? onClusterOpt : false;
258
+ let chainIdMode = chainIdModeOpt !== undefined ? chainIdModeOpt : "int32";
250
259
  let tableEngine = replicated ? "ReplicatedMergeTree" : "MergeTree()";
251
260
  let fieldDefinitions = Stdlib_Array.filterMap(entityConfig.table.fields, field => {
252
261
  if (field.TAG !== "Field") {
@@ -254,7 +263,7 @@ function makeCreateHistoryTableQuery(entityConfig, database, replicatedOpt, onCl
254
263
  }
255
264
  let field$1 = field._0;
256
265
  let fieldName = Table.getClickHouseDbFieldName(field$1);
257
- let clickHouseType = getClickHouseFieldType(field$1.fieldType, field$1.isNullable, field$1.isArray);
266
+ let clickHouseType = getClickHouseFieldType(field$1.fieldType, field$1.isNullable, field$1.isArray, chainIdMode);
258
267
  return `\`` + fieldName + `\` ` + clickHouseType;
259
268
  });
260
269
  let options = entityConfig.storage.clickhouseOptions;
@@ -306,28 +315,29 @@ function makeCreateHistoryTableQuery(entityConfig, database, replicatedOpt, onCl
306
315
  onCluster ? ` ON CLUSTER '{cluster}'` : ""
307
316
  ) + ` (
308
317
  ` + fieldDefinitions.join(",\n ") + `,
309
- \`` + EntityHistory.checkpointIdFieldName + `\` ` + getClickHouseFieldType("UInt64", false, false) + `,
318
+ \`` + EntityHistory.checkpointIdFieldName + `\` ` + getClickHouseFieldType("UInt64", false, false, undefined) + `,
310
319
  \`` + EntityHistory.changeFieldName + `\` ` + getClickHouseFieldType({
311
320
  type: "Enum",
312
321
  config: EntityHistory.RowAction.config
313
- }, false, false) + `
322
+ }, false, false, undefined) + `
314
323
  )
315
324
  ENGINE = ` + tableEngine + partitionByClause + `
316
325
  ORDER BY (` + orderByColumns + `)` + ttlClause;
317
326
  }
318
327
 
319
- function makeCreateCheckpointsTableQuery(database, replicatedOpt, onClusterOpt) {
328
+ function makeCreateCheckpointsTableQuery(database, replicatedOpt, onClusterOpt, chainIdModeOpt) {
320
329
  let replicated = replicatedOpt !== undefined ? replicatedOpt : false;
321
330
  let onCluster = onClusterOpt !== undefined ? onClusterOpt : false;
331
+ let chainIdMode = chainIdModeOpt !== undefined ? chainIdModeOpt : "int32";
322
332
  let tableEngine = replicated ? "ReplicatedMergeTree" : "MergeTree()";
323
333
  return `CREATE TABLE IF NOT EXISTS ` + database + `.\`` + InternalTable.Checkpoints.table.tableName + `\`` + (
324
334
  onCluster ? ` ON CLUSTER '{cluster}'` : ""
325
335
  ) + ` (
326
- \`` + "id" + `\` ` + getClickHouseFieldType("UInt64", false, false) + `,
327
- \`` + "chain_id" + `\` ` + getClickHouseFieldType("Int32", false, false) + `,
328
- \`` + "block_number" + `\` ` + getClickHouseFieldType("Int32", false, false) + `,
329
- \`` + "block_hash" + `\` ` + getClickHouseFieldType("String", true, false) + `,
330
- \`` + "events_processed" + `\` ` + getClickHouseFieldType("UInt64", false, false) + `
336
+ \`` + "id" + `\` ` + getClickHouseFieldType("UInt64", false, false, undefined) + `,
337
+ \`` + "chain_id" + `\` ` + getClickHouseFieldType("ChainId", false, false, chainIdMode) + `,
338
+ \`` + "block_number" + `\` ` + getClickHouseFieldType("Int32", false, false, undefined) + `,
339
+ \`` + "block_hash" + `\` ` + getClickHouseFieldType("String", true, false, undefined) + `,
340
+ \`` + "events_processed" + `\` ` + getClickHouseFieldType("UInt64", false, false, undefined) + `
331
341
  )
332
342
  ENGINE = ` + tableEngine + `
333
343
  ORDER BY (` + "id" + `)`;
@@ -358,7 +368,8 @@ FROM (
358
368
  WHERE \`` + EntityHistory.changeFieldName + `\` = '` + "SET" + `'`;
359
369
  }
360
370
 
361
- async function initialize(client, database, entities, param) {
371
+ async function initialize(client, database, entities, param, chainIdModeOpt) {
372
+ let chainIdMode = chainIdModeOpt !== undefined ? chainIdModeOpt : "int32";
362
373
  try {
363
374
  let databaseEngine = Env.ClickHouse.databaseEngine();
364
375
  let databaseEngineClause = databaseEngine !== undefined ? ` ENGINE = ` + databaseEngine : "";
@@ -399,10 +410,10 @@ async function initialize(client, database, entities, param) {
399
410
  query: `CREATE DATABASE IF NOT EXISTS ` + database + databaseOnClusterClause + databaseEngineClause
400
411
  });
401
412
  await Promise.all(entities.map(entityConfig => client.exec({
402
- query: makeCreateHistoryTableQuery(entityConfig, database, replicated, ddlOnCluster)
413
+ query: makeCreateHistoryTableQuery(entityConfig, database, replicated, ddlOnCluster, chainIdMode)
403
414
  })));
404
415
  await client.exec({
405
- query: makeCreateCheckpointsTableQuery(database, replicated, ddlOnCluster)
416
+ query: makeCreateCheckpointsTableQuery(database, replicated, ddlOnCluster, chainIdMode)
406
417
  });
407
418
  if (hasReplicatedDatabaseEngine) {
408
419
  await client.exec({
@@ -176,10 +176,6 @@ let parseGetWhereOrThrow = (filter: dict<dict<unknown>>, ~entityName, ~table: Ta
176
176
  JsError.throwWithMessage(
177
177
  `The field "${apiFieldName}" on entity "${entityName}" is a derived field and cannot be used in getWhere(). Use the source entity's indexed field instead.`,
178
178
  )
179
- | Some(Field({isPrimaryKey: false, isIndex: false, linkedEntity: None})) =>
180
- JsError.throwWithMessage(
181
- `The field "${apiFieldName}" on entity "${entityName}" does not have an index. To use it in getWhere(), add the @index directive in your schema.graphql:\n\n ${apiFieldName}: ... @index\n\nThen run 'pnpm envio codegen' to regenerate.`,
182
- )
183
179
  | Some(Field(_)) => ()
184
180
  }
185
181
 
@@ -142,14 +142,7 @@ function parseGetWhereOrThrow(filter, entityName, table) {
142
142
  });
143
143
  let match = Table.getFieldByApiName(table, apiFieldName);
144
144
  if (match !== undefined) {
145
- if (match.TAG === "Field") {
146
- let match$1 = match._0;
147
- if (match$1.isPrimaryKey || match$1.isIndex || match$1.linkedEntity !== undefined) {
148
-
149
- } else {
150
- Stdlib_JsError.throwWithMessage(`The field "` + apiFieldName + `" on entity "` + entityName + `" does not have an index. To use it in getWhere(), add the @index directive in your schema.graphql:\n\n ` + apiFieldName + `: ... @index\n\nThen run 'pnpm envio codegen' to regenerate.`);
151
- }
152
- } else {
145
+ if (match.TAG !== "Field") {
153
146
  Stdlib_JsError.throwWithMessage(`The field "` + apiFieldName + `" on entity "` + entityName + `" is a derived field and cannot be used in getWhere(). Use the source entity's indexed field instead.`);
154
147
  }
155
148
  } else {
@@ -64,7 +64,7 @@ let historyTableName = (~entityName, ~entityIndex) => {
64
64
  }
65
65
 
66
66
  type safeReorgBlocks = {
67
- chainIds: array<int>,
67
+ chainIds: array<ChainId.t>,
68
68
  blockNumbers: array<int>,
69
69
  }
70
70
 
@@ -0,0 +1,248 @@
1
+ // What PostgreSQL actually holds, read straight from pg_catalog and classified
2
+ // against what the indexer wanted. Postgres stays authoritative: nothing is
3
+ // counted as present until it has been read back from here.
4
+
5
+ type row = {
6
+ tableName: string,
7
+ indexName: string,
8
+ method: string,
9
+ // 1/0 rather than bools: postgres.js hands booleans back inconsistently
10
+ // depending on the driver's type resolution, so the query casts them.
11
+ isValid: int,
12
+ isUnique: int,
13
+ // An index with a WHERE clause only covers rows inside its predicate, so it
14
+ // can't stand in for the unrestricted index a filter needs.
15
+ isPartial: int,
16
+ isExpression: int,
17
+ // The predicate itself, for the error a partial index under our name earns.
18
+ predicate: option<string>,
19
+ columns: array<string>,
20
+ // "ASC"/"DESC" per column, in the same order as `columns`.
21
+ directions: array<string>,
22
+ }
23
+
24
+ let rowsSchema = S.array(
25
+ S.object((s): row => {
26
+ tableName: s.field("tableName", S.string),
27
+ indexName: s.field("indexName", S.string),
28
+ method: s.field("method", S.string),
29
+ isValid: s.field("isValid", S.int),
30
+ isUnique: s.field("isUnique", S.int),
31
+ isPartial: s.field("isPartial", S.int),
32
+ isExpression: s.field("isExpression", S.int),
33
+ predicate: s.field("predicate", S.null(S.string)),
34
+ columns: s.field("columns", S.array(S.string)),
35
+ directions: s.field("directions", S.array(S.string)),
36
+ }),
37
+ )
38
+
39
+ // One row per index, with its key columns aggregated in ordinal order.
40
+ // INCLUDE columns are excluded (they don't affect what the index can serve),
41
+ // and expression columns come back as their printed definition so an
42
+ // expression index can never be mistaken for a plain-column one.
43
+ //
44
+ // Both interpolated values are identifiers, not user input: `pgSchema` comes
45
+ // from the operator's own environment, and `indexName` is always a generated
46
+ // name — a codegen-validated `[A-Za-z_][A-Za-z0-9_]*` prefix plus a base36
47
+ // hash, so neither can carry a quote.
48
+ let makeQuery = (~pgSchema, ~indexName=?) =>
49
+ `SELECT
50
+ t.relname AS "tableName",
51
+ i.relname AS "indexName",
52
+ am.amname AS "method",
53
+ CASE WHEN ix.indisvalid AND ix.indisready THEN 1 ELSE 0 END AS "isValid",
54
+ CASE WHEN ix.indisunique THEN 1 ELSE 0 END AS "isUnique",
55
+ CASE WHEN ix.indpred IS NOT NULL THEN 1 ELSE 0 END AS "isPartial",
56
+ CASE WHEN ix.indexprs IS NOT NULL THEN 1 ELSE 0 END AS "isExpression",
57
+ pg_get_expr(ix.indpred, ix.indrelid, true) AS "predicate",
58
+ array_agg(
59
+ CASE WHEN k.attnum = 0
60
+ THEN pg_get_indexdef(ix.indexrelid, k.ord::int, true)
61
+ ELSE a.attname
62
+ END ORDER BY k.ord
63
+ ) AS "columns",
64
+ array_agg(
65
+ CASE WHEN (ix.indoption[k.ord - 1] & 1) = 1 THEN 'DESC' ELSE 'ASC' END ORDER BY k.ord
66
+ ) AS "directions"
67
+ FROM pg_index ix
68
+ JOIN pg_class i ON i.oid = ix.indexrelid
69
+ JOIN pg_class t ON t.oid = ix.indrelid
70
+ JOIN pg_namespace n ON n.oid = t.relnamespace
71
+ JOIN pg_am am ON am.oid = i.relam
72
+ JOIN LATERAL unnest(ix.indkey) WITH ORDINALITY AS k(attnum, ord) ON k.ord <= ix.indnkeyatts
73
+ LEFT JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = k.attnum
74
+ WHERE n.nspname = '${pgSchema}'${switch indexName {
75
+ | Some(indexName) => ` AND i.relname = '${indexName}'`
76
+ | None => ""
77
+ }}
78
+ GROUP BY t.relname, i.relname, am.amname, ix.indisvalid, ix.indisready,
79
+ ix.indisunique, ix.indpred, ix.indrelid, (ix.indexprs IS NOT NULL);`
80
+
81
+ type entry = {
82
+ name: string,
83
+ tableName: string,
84
+ method: string,
85
+ columns: array<IndexDefinition.column>,
86
+ isValid: bool,
87
+ isUnique: bool,
88
+ isPartial: bool,
89
+ isExpression: bool,
90
+ predicate: option<string>,
91
+ }
92
+
93
+ let fromRow = (row: row): entry => {
94
+ name: row.indexName,
95
+ tableName: row.tableName,
96
+ method: row.method,
97
+ columns: row.columns->Array.mapWithIndex((name, idx) => {
98
+ IndexDefinition.name,
99
+ direction: switch row.directions->Array.get(idx) {
100
+ | Some("DESC") => Table.Desc
101
+ | _ => Asc
102
+ },
103
+ }),
104
+ isValid: row.isValid === 1,
105
+ isUnique: row.isUnique === 1,
106
+ isPartial: row.isPartial === 1,
107
+ isExpression: row.isExpression === 1,
108
+ predicate: row.predicate,
109
+ }
110
+
111
+ // A btree's key columns are ordered, so an index leading with the requested
112
+ // columns can serve them. The reverse isn't true, which is why order matters.
113
+ let leadsWith = (entry, definition: IndexDefinition.t) =>
114
+ definition.columns->Array.length <= entry.columns->Array.length &&
115
+ definition.columns->Array.everyWithIndex((column, idx) =>
116
+ switch entry.columns->Array.get(idx) {
117
+ | Some(actual) => actual.name === column.name && actual.direction === column.direction
118
+ | None => false
119
+ }
120
+ )
121
+
122
+ // How much of an existing index has to line up before it counts as coverage.
123
+ //
124
+ // A schema-declared `@index` is a promise about the physical schema, so it is
125
+ // matched `Exact`: whether some composite index happens to exist must not
126
+ // change what a fresh database ends up with, or two indexers on the same
127
+ // schema would hold different tables.
128
+ //
129
+ // An automatic getWhere index is an optimization nobody declared, so
130
+ // `LeadingColumns` is enough — an existing composite that already starts with
131
+ // the column serves the query, and building a second one would only cost write
132
+ // amplification.
133
+ type coverage = Exact | LeadingColumns
134
+
135
+ let coverageKey = coverage =>
136
+ switch coverage {
137
+ | Exact => "="
138
+ | LeadingColumns => "^"
139
+ }
140
+
141
+ // Why an index can't serve a request, in the order the checks are worth
142
+ // reporting. `None` means it can.
143
+ let rejectReason = (entry, definition: IndexDefinition.t, ~coverage) =>
144
+ if entry.tableName !== definition.tableName {
145
+ Some(`it is on table "${entry.tableName}"`)
146
+ } else if !entry.isValid {
147
+ Some("PostgreSQL reports it as invalid or not ready")
148
+ } else if entry.isPartial {
149
+ Some(
150
+ `it is partial (WHERE ${entry.predicate->Option.getOr(
151
+ "...",
152
+ )}), so it only covers part of the table`,
153
+ )
154
+ } else if entry.isExpression {
155
+ Some("it indexes an expression rather than plain columns")
156
+ } else if entry.method !== definition.method {
157
+ Some(`it uses the ${entry.method} access method, not ${definition.method}`)
158
+ } else if !(entry->leadsWith(definition)) {
159
+ Some(
160
+ `it covers (${entry.columns
161
+ ->Array.map(IndexDefinition.columnKey)
162
+ ->Array.joinUnsafe(", ")}) instead of leading with the requested columns`,
163
+ )
164
+ } else if coverage === Exact && entry.columns->Array.length !== definition.columns->Array.length {
165
+ Some(
166
+ `it covers (${entry.columns
167
+ ->Array.map(IndexDefinition.columnKey)
168
+ ->Array.joinUnsafe(", ")}) rather than exactly the declared columns`,
169
+ )
170
+ } else {
171
+ None
172
+ }
173
+
174
+ let satisfies = (entry, definition, ~coverage) =>
175
+ entry->rejectReason(definition, ~coverage)->Option.isNone
176
+
177
+ // The index is byte-for-byte what `definition` asks for, valid or not. Used to
178
+ // decide ownership, where even an exact-coverage match would be too loose: a
179
+ // unique index covering the same columns still isn't one the indexer created.
180
+ let isExactly = (entry, definition: IndexDefinition.t) =>
181
+ entry.tableName === definition.tableName &&
182
+ entry.method === definition.method &&
183
+ !entry.isUnique &&
184
+ !entry.isPartial &&
185
+ !entry.isExpression &&
186
+ entry.columns->Array.length === definition.columns->Array.length &&
187
+ entry->leadsWith(definition)
188
+
189
+ type t = {
190
+ byName: dict<entry>,
191
+ // `find` answers "is this index already covered?", which runs per filtered
192
+ // column on every batched getWhere load. Scanning every index in the schema
193
+ // there would make a hot path grow with the size of the schema, so the
194
+ // answer is memoized by identity and thrown away whenever the catalog moves
195
+ // — which only happens when an index is built, resynced or reloaded.
196
+ mutable covering: dict<string>,
197
+ }
198
+
199
+ let make = () => {byName: Dict.make(), covering: Dict.make()}
200
+
201
+ let set = (catalog, entry) => {
202
+ catalog.byName->Dict.set(entry.name, entry)
203
+ catalog.covering = Dict.make()
204
+ }
205
+
206
+ let remove = (catalog, name) => {
207
+ catalog.byName->Utils.Dict.deleteInPlace(name)
208
+ catalog.covering = Dict.make()
209
+ }
210
+
211
+ let fromRows = (~rows) => {
212
+ let catalog = make()
213
+ rows->Array.forEach(row => catalog->set(row->fromRow))
214
+ catalog
215
+ }
216
+
217
+ let getByName = (catalog, name) => catalog.byName->Utils.Dict.dangerouslyGetNonOption(name)
218
+
219
+ let entries = catalog => catalog.byName->Dict.valuesToArray
220
+
221
+ let size = catalog => catalog.byName->Dict.keysToArray->Array.length
222
+
223
+ // Postgres has no zero-length identifier, so this can't collide with a name.
224
+ let nothingCovers = ""
225
+
226
+ let find = (catalog, definition, ~coverage) => {
227
+ let cacheKey = `${coverage->coverageKey}${definition->IndexDefinition.key}`
228
+ switch catalog.covering->Utils.Dict.dangerouslyGetNonOption(cacheKey) {
229
+ | Some(name) if name === nothingCovers => None
230
+ | Some(name) => catalog.byName->Utils.Dict.dangerouslyGetNonOption(name)
231
+ | None =>
232
+ let found = catalog->entries->Array.find(entry => entry->satisfies(definition, ~coverage))
233
+ catalog.covering->Dict.set(
234
+ cacheKey,
235
+ switch found {
236
+ | Some(entry) => entry.name
237
+ | None => nothingCovers
238
+ },
239
+ )
240
+ found
241
+ }
242
+ }
243
+
244
+ let invalidNames = catalog =>
245
+ catalog
246
+ ->entries
247
+ ->Array.filterMap(entry => entry.isValid ? None : Some(entry.name))
248
+ ->Array.toSorted(String.compare)