envio 3.15.0-subgraph → 3.16.0-subgraph

package/src/subgraph/graph-ts.ts

@@ -1406,11 +1406,16 @@ function toRow(entityType: string, entity: Entity, id?: string): Record<string,
     // others (`check_invalid_fields` refuses them from 0.0.8 on).
     if (fields !== undefined && !fields.has(entry.key)) continue;
     const column = refFields.has(entry.key) ? `${entry.key}_id` : entry.key;
-    // An id is a string column whatever the subgraph declared it as, and so is
-    // a relation, which carries one. Every other `Bytes` field is a bytes
-    // column, which is what `bytes_type: uint8array` makes of it.
-    const isId = entry.key === "id" || refFields.has(entry.key) || listFields.has(entry.key);
     const declared = schema.entityFieldTypes[entityType]?.[entry.key];
+    // An id, a relation and a stored list of entities hold ids: bytes when the
+    // id they name is `Bytes`, text otherwise. Every other `Bytes` field is a
+    // bytes column, which is what `bytes_type: uint8array` makes of it.
+    const idColumn =
+      entry.key === "id" || refFields.has(entry.key) || listFields.has(entry.key)
+        ? declaredKind(schema, declared) === "Bytes"
+          ? "bytes"
+          : "text"
+        : undefined;
     if (
       declared?.nullableElements &&
       entry.value.kind === ValueKind.ARRAY &&
@@ -1426,17 +1431,17 @@ function toRow(entityType: string, entity: Entity, id?: string): Record<string,
     row[column] =
       timestampFields.has(entry.key) && entry.value.kind !== ValueKind.NULL
         ? new Date(Number(entry.value.toTimestamp() / 1000n))
-      : fromValue(entry.value, isId);
+      : fromValue(entry.value, idColumn);
   }
   return row;
 }
 
-function fromValue(value: Value, isId = false): unknown {
+function fromValue(value: Value, idColumn?: "bytes" | "text"): unknown {
   switch (value.kind) {
     case ValueKind.NULL:
       return null;
     case ValueKind.BYTES:
-      return isId ? value.toBytes().toHexString() : Uint8Array.from(value.toBytes());
+      return idColumn === "text" ? value.toBytes().toHexString() : Uint8Array.from(value.toBytes());
     case ValueKind.BIGINT:
       return value.toBigInt().valueOf();
     case ValueKind.INT8:
@@ -1445,9 +1450,9 @@ function fromValue(value: Value, isId = false): unknown {
     case ValueKind.BIGDECIMAL:
       return value.toBigDecimal().value;
     case ValueKind.ARRAY:
-      return value.toArray().map((item) => fromValue(item, isId));
+      return value.toArray().map((item) => fromValue(item, idColumn));
     case ValueKind.STRING:
-      return withoutNul(value.data);
+      return idColumn === "bytes" ? graphIdToStoredId("Bytes", withoutNul(value.data)) : withoutNul(value.data);
     default:
       return value.data;
   }
@@ -1508,8 +1513,7 @@ function toValue(value: unknown, kind?: string): Value {
   if (Array.isArray(value)) return Value.fromArray(value.map((item) => toValue(item, kind)));
   switch (kind) {
     case "Bytes":
-      // A bytes column reads back as the bytes themselves; an id column
-      // declared `Bytes` is stored as text, and reads back hex.
+      // A bytes column reads back as the bytes themselves.
       if (value instanceof Uint8Array) return Value.fromBytes(Bytes.fromUint8Array(value));
       return typeof value === "string" ? Value.fromBytes(Bytes.fromHexString(value)) : Value.fromNull();
     case "BigInt":
@@ -1685,15 +1689,54 @@ export function resetRegisterStore(): void {
  * needs it spelled out — a list column refuses a missing value.
  */
 function withUnsetAsNull(scope: Scope, entityType: string, row: Record<string, unknown>) {
-  const refFields = new Set(scope.schema.entityRefFields[entityType] ?? []);
-  for (const [field, declared] of Object.entries(scope.schema.entityFieldTypes[entityType] ?? {})) {
-    if (declared.derivedFrom) continue;
-    const column = refFields.has(field) ? `${field}_id` : field;
+  for (const column of storedColumnsOfEntity(scope, entityType)) {
     if (!(column in row)) row[column] = null;
   }
   return row;
 }
 
+const columnsBySchema = new WeakMap<object, Map<string, string[]>>();
+
+/** An entity's stored columns: a reference field's is `<field>_id`, a derived one has none. */
+function storedColumnsOfEntity(scope: Scope, entityType: string): string[] {
+  let byType = columnsBySchema.get(scope.schema);
+  if (!byType) {
+    byType = new Map();
+    columnsBySchema.set(scope.schema, byType);
+  }
+  let columns = byType.get(entityType);
+  if (!columns) {
+    const refFields = new Set(scope.schema.entityRefFields[entityType] ?? []);
+    columns = Object.entries(scope.schema.entityFieldTypes[entityType] ?? {})
+      .filter(([, declared]) => !declared.derivedFrom)
+      .map(([field]) => (refFields.has(field) ? `${field}_id` : field));
+    byType.set(entityType, columns);
+  }
+  return columns;
+}
+
+/**
+ * graph-ts names an entity by a string, a `Bytes` id by its hex; graph-node
+ * stores that id as the bytes, so `0xAB` and `0xab` are one entity. `kind` is
+ * the column's declared kind, a relation's being its target's id kind.
+ */
+function graphIdToStoredId(kind: string | undefined, id: string): unknown {
+  if (kind !== "Bytes") return id;
+  const digits = id.startsWith("0x") || id.startsWith("0X") ? id.slice(2) : id;
+  if (digits.length % 2 !== 0 || !HEX_DIGITS.test(digits)) {
+    throw new Error(`Envio Subgraph: \`${id}\` names an entity whose id is Bytes, and isn't hex.`);
+  }
+  return new Uint8Array(Buffer.from(digits, "hex"));
+}
+
+function storedIdToGraphId(id: unknown): string {
+  return id instanceof Uint8Array ? toHex(id) : String(id);
+}
+
+function entityIdToStoredId(scope: Scope, entityType: string, id: string): unknown {
+  return graphIdToStoredId(scope.schema.entityFieldTypes[entityType]?.id?.kind, withoutNul(id));
+}
+
 /**
  * graph-node's EntityCache::set validates a save: an update that sets every
  * non-nullable field passes as it is (a null counts as set), otherwise it is
@@ -1752,26 +1795,29 @@ function fileAccess(scope: Scope, entityType: string, id: string): boolean | nul
 
 const storeImpl = {
   get(entityType: string, id: string): Entity | null {
-    id = withoutNul(id);
     const scope = currentScope();
+    const storedId = entityIdToStoredId(scope, entityType, id);
+    id = storedIdToGraphId(storedId);
     if (fileAccess(scope, entityType, id) === false) return null;
     if (scope.mode === "register") return registerRead(scope, entityType, id, false);
-    return toEntity(entityType, entityContext(entityType).getSync(id));
+    return toEntity(entityType, entityContext(entityType).getSync(storedId));
   },
   get_in_block(entityType: string, id: string): Entity | null {
-    id = withoutNul(id);
     const scope = currentScope();
+    const storedId = entityIdToStoredId(scope, entityType, id);
+    id = storedIdToGraphId(storedId);
     if (fileAccess(scope, entityType, id) === false) return null;
     if (scope.mode === "register") return registerRead(scope, entityType, id, true);
-    return toEntity(entityType, entityContext(entityType).getInBlockSync(id));
+    return toEntity(entityType, entityContext(entityType).getInBlockSync(storedId));
   },
   set(entityType: string, id: string, data: Entity): void {
-    id = withoutNul(id);
     const scope = currentScope();
+    const storedId = entityIdToStoredId(scope, entityType, id);
+    id = storedIdToGraphId(storedId);
     const own = fileAccess(scope, entityType, id);
     if (scope.file) scope.file.written.add(`${entityType}\u0000${id}`);
     const row = toRow(entityType, data, id);
-    row.id = id;
+    row.id = storedId;
     // graph-node merges a save onto the stored entity (EntityCache::set is an
     // Update, merged by merge_remove_null_fields): a field this instance
     // doesn't set keeps its stored value, an explicit null clears it. ENS's
@@ -1792,20 +1838,27 @@ const storeImpl = {
     }
     // A file data source's first write of an id is a new entity of its own,
     // whatever a chain handler stored under the same id.
-    const stored = own === false ? undefined : table.getSync(id);
+    // A save that sets every column leaves the merge nothing to keep, so the
+    // store isn't read: for an insert-only entity that read was a database
+    // lookup and a second preload round per event.
+    const replacesWhole =
+      !scope.schema.skipDuplicateEntities.includes(entityType) &&
+      storedColumnsOfEntity(scope, entityType).every((column) => column in row);
+    const stored = own === false || replacesWhole ? undefined : table.getSync(storedId);
     assertRequiredFields(scope, entityType, id, data, stored);
     // graph-node inserts a skipDuplicates type with ON CONFLICT DO NOTHING, so
     // an id an earlier block stored keeps that block's write; saves within
     // this block merge in its entity cache first.
-    if (stored && scope.schema.skipDuplicateEntities.includes(entityType) && !table.getInBlockSync(id)) return;
+    if (stored && scope.schema.skipDuplicateEntities.includes(entityType) && !table.getInBlockSync(storedId)) return;
     table.set(stored ? { ...stored, ...row } : withUnsetAsNull(scope, entityType, row));
   },
   remove(entityType: string, id: string): void {
-    id = withoutNul(id);
     const scope = currentScope();
+    const storedId = entityIdToStoredId(scope, entityType, id);
+    id = storedIdToGraphId(storedId);
     if (fileAccess(scope, entityType, id) === false) return;
     if (scope.mode === "register") return registerWrite(scope, entityType, id, null);
-    entityContext(entityType).deleteUnsafe(id);
+    entityContext(entityType).deleteUnsafe(storedId);
   },
   /**
    * `graph codegen` names the entity the derived field is declared on and the
@@ -1833,10 +1886,15 @@ const storeImpl = {
     const back = schema.entityRefFields[target]?.includes(declared.derivedFrom)
       ? `${declared.derivedFrom}_id`
       : declared.derivedFrom;
-    const rows = entityContext(target).getWhereSync({ [back]: { _eq: id } });
+    const backKind = declaredKind(schema, schema.entityFieldTypes[target]?.[declared.derivedFrom]);
+    const rows = entityContext(target).getWhereSync({
+      [back]: { _eq: graphIdToStoredId(backKind, withoutNul(id)) },
+    });
     // A file data source sees the related entities it wrote, and no others.
     const visible = file
-      ? rows.filter((row: any) => file.entities.has(target) && file.written.has(`${target}\u0000${row.id}`))
+      ? rows.filter(
+          (row: any) => file.entities.has(target) && file.written.has(`${target}\u0000${storedIdToGraphId(row.id)}`),
+        )
       : rows;
     // graph-node answers in key order (`EntityCache::load_related` walks a
     // BTreeSet of entity keys): by id, as the id's type orders.
@@ -1845,8 +1903,9 @@ const storeImpl = {
   },
 };
 
+// A Bytes id's lowercase hex orders as its bytes do.
 function byId(a: { id: unknown }, b: { id: unknown }): number {
-  return codePointOrder(String(a.id), String(b.id));
+  return codePointOrder(storedIdToGraphId(a.id), storedIdToGraphId(b.id));
 }
 
 function byNumericId(a: { id: unknown }, b: { id: unknown }): number {

package/src/subgraph/runtime.ts

@@ -1437,7 +1437,6 @@ export async function registerSubgraph(config: SubgraphConfig, logger: unknown):
 
   const schema: SubgraphSchema = {
     timestampFields: config.timestampFields ?? {},
-    bytesIdEntities: config.bytesIdEntities ?? [],
     entityListFields: config.entityListFields ?? {},
     entityFields: config.entityFields ?? {},
     entityRefFields: config.entityRefFields ?? {},

package/src/subgraph/scope.ts

@@ -6,7 +6,6 @@ export type Mode = "handler" | "register";
 
 export type SubgraphSchema = {
   timestampFields: Record<string, string[]>;
-  bytesIdEntities: string[];
   entityListFields: Record<string, string[]>;
   entityFields: Record<string, string[]>;
   entityRefFields: Record<string, string[]>;

package/svm.schema.json

@@ -220,7 +220,7 @@
           "minimum": 0
         },
         "block_throttle": {
-          "description": "The minimum number of new blocks the chain head must advance by before the indexer fetches again once it has caught up. Useful on chains with very fast blocks, where fetching every block is wasteful. Unset, 0 and 1 all fetch on every new block.",
+          "description": "Once caught up, wait for this many new slots before fetching them. Delays new instructions by up to as many slots.",
           "type": [
             "integer",
             "null"
@@ -303,7 +303,7 @@
       "type": "object",
       "properties": {
         "name": {
-          "description": "A unique project-wide name for this program (used in generated code).",
+          "description": "A unique project-wide name for this program (used in the generated types).",
           "type": "string"
         },
         "program_id": {
@@ -357,7 +357,7 @@
       "type": "object",
       "properties": {
         "name": {
-          "description": "Name of the instruction in the HyperIndex generated code. Should be unique per program.",
+          "description": "Name of the instruction in the generated types. Should be unique per program.",
           "type": "string"
         },
         "discriminator": {

package/src/PgWriting.res

@@ -1,36 +0,0 @@
-let columns = (fields: array<Table.field>, ~kinds: array<Staging.kind>): array<Staging.column> =>
-  fields->Array.mapWithIndex((field, index) => {
-    Staging.name: field->Table.getPgDbFieldName,
-    kind: kinds->Array.getUnsafe(index),
-    isNullable: field.isNullable,
-  })
-
-// Lays a batch into the arena, column by column. The values are the ones the
-// table's own schema produced — a bigint as its digits, a date as its text, a
-// boolean as the number the statement's cast expects — so nothing here has to
-// know what a column means, only where its values go.
-//
-// The rows of a column are written in order because a variable-width slot's row
-// starts where the row before it ended.
-let stage = (
-  arena,
-  ~table,
-  ~columns: array<Staging.column>,
-  ~values: array<array<unknown>>,
-  ~rows,
-) => {
-  let stage = arena->Staging.begin(~table, ~rows, ~columns)
-  try {
-    for column in 0 to columns->Array.length - 1 {
-      let values = values->Array.getUnsafe(column)
-      for row in 0 to rows - 1 {
-        stage->Staging.writeValue(~column, ~row, values->Array.getUnsafe(row))
-      }
-    }
-    stage->Staging.commit
-  } catch {
-  | exn =>
-    stage->Staging.abort
-    throw(exn)
-  }
-}

package/src/PgWriting.res.mjs

@@ -1,34 +0,0 @@
-// Generated by ReScript, PLEASE EDIT WITH CARE
-
-import * as Table from "./db/Table.res.mjs";
-import * as Staging from "./Staging.res.mjs";
-
-function columns(fields, kinds) {
-  return fields.map((field, index) => ({
-    name: Table.getPgDbFieldName(field),
-    kind: kinds[index],
-    isNullable: field.isNullable
-  }));
-}
-
-function stage(arena, table, columns, values, rows) {
-  let stage$1 = Staging.begin(arena, table, rows, columns);
-  try {
-    for (let column = 0, column_finish = columns.length; column < column_finish; ++column) {
-      let values$1 = values[column];
-      for (let row = 0; row < rows; ++row) {
-        Staging.writeValue(stage$1, column, row, values$1[row]);
-      }
-    }
-    return Staging.commit(stage$1);
-  } catch (exn) {
-    Staging.abort(stage$1);
-    throw exn;
-  }
-}
-
-export {
-  columns,
-  stage,
-}
-/* Table Not a pure module */

package/src/db/IndexDefinition.res

@@ -1,47 +0,0 @@
-// What the indexer wants an index to be: a table, its ordered key columns with
-// their directions, and an access method. The addon names it, matches it
-// against what PostgreSQL holds and builds it.
-
-type column = {
-  name: string,
-  direction: Table.indexFieldDirection,
-}
-
-type t = {
-  tableName: string,
-  columns: array<column>,
-  method: string,
-}
-
-let make = (~tableName, ~columns, ~method="btree") => {tableName, columns, method}
-
-let single = (~tableName, ~column) =>
-  make(~tableName, ~columns=[{name: column, direction: Table.Asc}])
-
-let fromIndexFields = (~tableName, ~indexFields: array<Table.compositeIndexField>) =>
-  make(
-    ~tableName,
-    ~columns=indexFields->Array.map(({fieldName, direction}) => {name: fieldName, direction}),
-  )
-
-let toInput = ({tableName, columns, method}: t): Core.pgIndexInput => {
-  tableName,
-  columns: columns->Array.map(({name, direction}): Core.pgIndexColumnInput => {
-    name,
-    direction: switch direction {
-    | Table.Asc => "Asc"
-    | Desc => "Desc"
-    },
-  }),
-  method,
-}
-
-let describe = (definition: t) =>
-  `${definition.tableName}(${definition.columns
-    ->Array.map(({name, direction}) =>
-      switch direction {
-      | Asc => name
-      | Desc => `${name} DESC`
-      }
-    )
-    ->Array.joinUnsafe(", ")}) using ${definition.method}`

package/src/db/IndexDefinition.res.mjs

@@ -1,60 +0,0 @@
-// Generated by ReScript, PLEASE EDIT WITH CARE
-
-
-function make(tableName, columns, methodOpt) {
-  let method = methodOpt !== undefined ? methodOpt : "btree";
-  return {
-    tableName: tableName,
-    columns: columns,
-    method: method
-  };
-}
-
-function single(tableName, column) {
-  return make(tableName, [{
-      name: column,
-      direction: "Asc"
-    }], undefined);
-}
-
-function fromIndexFields(tableName, indexFields) {
-  return make(tableName, indexFields.map(param => ({
-    name: param.fieldName,
-    direction: param.direction
-  })), undefined);
-}
-
-function toInput(param) {
-  return {
-    tableName: param.tableName,
-    columns: param.columns.map(param => {
-      let tmp;
-      tmp = param.direction === "Asc" ? "Asc" : "Desc";
-      return {
-        name: param.name,
-        direction: tmp
-      };
-    }),
-    method: param.method
-  };
-}
-
-function describe(definition) {
-  return definition.tableName + `(` + definition.columns.map(param => {
-    let name = param.name;
-    if (param.direction === "Asc") {
-      return name;
-    } else {
-      return name + ` DESC`;
-    }
-  }).join(", ") + `) using ` + definition.method;
-}
-
-export {
-  make,
-  single,
-  fromIndexFields,
-  toInput,
-  describe,
-}
-/* No side effect */