@o1-labs/mina-archive-node-graphql 0.0.6

package/src/services/blocks-service/blocks-service.ts

@@ -0,0 +1,407 @@
+import type postgres from 'postgres';
+import {
+  Blocks,
+  UserCommand,
+  ZkAppCommand,
+  FeeTransfer,
+} from '../../blockchain/types.js';
+import type {
+  BlockQueryInput,
+  BlockSortByInput,
+} from '../../resolvers-types.js';
+import { IBlocksService } from './blocks-service.interface.js';
+import {
+  TracingState,
+  extractTraceStateFromOptions,
+} from '../../tracing/tracer.js';
+import {
+  BLOCK_RANGE_SIZE,
+  ENABLE_BLOCK_TRANSACTION_DETAILS,
+} from '../../server/server.js';
+
+export { BlocksService };
+export type { BlockRow, UserCommandRow, ZkAppCommandRow, FeeTransferRow };
+
+// Interface representing the expected row structure from the database
+interface BlockRow {
+  id: number;
+  height: number;
+  creator: string;
+  state_hash: string;
+  parent_hash: string;
+  timestamp: string;
+  internal_command_type?: string;
+  coinbase_amount?: string;
+}
+
+interface UserCommandRow {
+  block_id: number;
+  hash: string;
+  kind: string;
+  from: string;
+  to: string;
+  amount: string;
+  fee: string;
+  memo: string;
+  nonce: number;
+  status: string;
+  failure_reason: string | null;
+}
+
+interface ZkAppCommandRow {
+  block_id: number;
+  hash: string;
+  fee_payer: string;
+  fee: string;
+  memo: string;
+  status: string;
+  failure_reasons_ids: number[] | string | null;
+}
+
+interface FeeTransferRow {
+  block_id: number;
+  recipient: string;
+  fee: string;
+  type: string;
+}
+
+class BlocksService implements IBlocksService {
+  private readonly client: postgres.Sql;
+
+  constructor(client: postgres.Sql) {
+    this.client = client;
+  }
+
+  async getBlocks(
+    query: BlockQueryInput | null | undefined,
+    limit: number | null | undefined,
+    sortBy: BlockSortByInput | null | undefined,
+    options: unknown
+  ): Promise<Blocks> {
+    const tracingState = extractTraceStateFromOptions(options);
+    return (await this.getBlockData(query, limit, sortBy, { tracingState })) ?? [];
+  }
+
+  async getBlockData(
+    query: BlockQueryInput | null | undefined,
+    limit: number | null | undefined,
+    sortBy: BlockSortByInput | null | undefined,
+    { tracingState }: { tracingState: TracingState }
+  ): Promise<Blocks> {
+    const sqlSpan = tracingState.startSpan('blocks.SQL');
+    const rows = await this.executeBlocksQuery(query, limit, sortBy);
+    sqlSpan.end();
+
+    const blockRows = rows as unknown as BlockRow[];
+
+    let userCommandsByBlock = new Map<number, UserCommand[]>();
+    let zkappCommandsByBlock = new Map<number, ZkAppCommand[]>();
+    let feeTransfersByBlock = new Map<number, FeeTransfer[]>();
+
+    if (ENABLE_BLOCK_TRANSACTION_DETAILS && blockRows.length > 0) {
+      const txSpan = tracingState.startSpan('blocks.transactionDetails.SQL');
+      const blockIds = blockRows.map((r) => r.id);
+      [userCommandsByBlock, zkappCommandsByBlock, feeTransfersByBlock] =
+        await this.fetchTransactionDetails(blockIds);
+      txSpan.end();
+    }
+
+    const processingSpan = tracingState.startSpan('blocks.processing');
+    const blocks = this.rowsToBlocks(
+      blockRows,
+      userCommandsByBlock,
+      zkappCommandsByBlock,
+      feeTransfersByBlock
+    );
+    processingSpan.end();
+    return blocks;
+  }
+
+  private async executeBlocksQuery(
+    query: BlockQueryInput | null | undefined,
+    limit: number | null | undefined,
+    sortBy: BlockSortByInput | null | undefined
+  ) {
+    const blockHeightGte = query?.blockHeight_gte;
+    const blockHeightLt = query?.blockHeight_lt;
+    const dateTimeGte = query?.dateTime_gte;
+    const dateTimeLt = query?.dateTime_lt;
+    const canonical = query?.canonical;
+    const inBestChain = query?.inBestChain;
+    const orderBy = sortBy === 'BLOCKHEIGHT_DESC' ? 'DESC' : 'ASC';
+    const limitValue = Math.min(limit ?? 200, BLOCK_RANGE_SIZE);
+
+    // Build the SQL query for blocks with transactions
+    // Archive Node uses a junction table blocks_internal_commands
+    let sql = `
+      SELECT
+        b.id,
+        b.state_hash,
+        b.parent_hash,
+        b.height,
+        b.timestamp,
+        pk.value as creator,
+        COALESCE(ic.fee, '0') as coinbase_amount
+      FROM blocks b
+      INNER JOIN public_keys pk ON b.creator_id = pk.id
+      LEFT JOIN (
+        SELECT
+          bic.block_id,
+          ic.fee
+        FROM internal_commands ic
+        INNER JOIN blocks_internal_commands bic ON ic.id = bic.internal_command_id
+        WHERE ic.command_type = 'coinbase'
+      ) AS ic ON b.id = ic.block_id
+      WHERE TRUE
+    `;
+
+    const params: (string | number)[] = [];
+    let paramIndex = 1;
+
+    if (canonical === true) {
+      sql += ` AND b.chain_status = 'canonical'`;
+    } else if (canonical === false) {
+      sql += ` AND b.chain_status <> 'canonical'`;
+    }
+
+    if (blockHeightGte) {
+      sql += ` AND b.height >= $${paramIndex}`;
+      params.push(blockHeightGte);
+      paramIndex++;
+    }
+
+    if (blockHeightLt) {
+      sql += ` AND b.height < $${paramIndex}`;
+      params.push(blockHeightLt);
+      paramIndex++;
+    }
+
+    if (dateTimeGte) {
+      // Convert ISO timestamp to Unix milliseconds
+      const timestampMs = new Date(dateTimeGte).getTime().toString();
+      sql += ` AND b.timestamp >= $${paramIndex}`;
+      params.push(timestampMs);
+      paramIndex++;
+    }
+
+    if (dateTimeLt) {
+      // Convert ISO timestamp to Unix milliseconds
+      const timestampMs = new Date(dateTimeLt).getTime().toString();
+      sql += ` AND b.timestamp < $${paramIndex}`;
+      params.push(timestampMs);
+      paramIndex++;
+    }
+
+    const best_chain_til_canonical_cte =
+      `
+        WITH RECURSIVE best_chain_til_canonical AS (
+            SELECT
+                id,
+                parent_id,
+                height
+            FROM blocks
+            WHERE height = (SELECT MAX(height) FROM blocks)
+            AND chain_status = 'pending'
+
+            UNION
+
+            SELECT
+               potential_parent.id,
+               potential_parent.parent_id,
+               potential_parent.height
+            FROM blocks potential_parent
+            JOIN best_chain_til_canonical ON potential_parent.id = best_chain_til_canonical.parent_id
+            WHERE potential_parent.chain_status <> 'canonical'
+        )
+        `
+    if (inBestChain === true) {
+      sql =
+        best_chain_til_canonical_cte
+        + sql
+        + ` AND (b.id IN (SELECT id FROM best_chain_til_canonical) OR b.chain_status = 'canonical')`;
+    } else if (inBestChain === false) {
+      sql =
+        best_chain_til_canonical_cte
+        + sql
+        + ` AND (b.id NOT IN (SELECT id FROM best_chain_til_canonical)
+                 OR b.chain_status = 'orphaned')`;
+
+    }
+
+    sql += ` ORDER BY b.height ${orderBy}`;
+    sql += ` LIMIT $${paramIndex}`;
+    params.push(limitValue);
+
+    return this.client.unsafe(sql, params);
+  }
+
+  private async fetchTransactionDetails(
+    blockIds: number[]
+  ): Promise<
+    [
+      Map<number, UserCommand[]>,
+      Map<number, ZkAppCommand[]>,
+      Map<number, FeeTransfer[]>,
+    ]
+  > {
+    const [ucRows, zkRows, ftRows] = await Promise.all([
+      this.fetchUserCommands(blockIds),
+      this.fetchZkAppCommands(blockIds),
+      this.fetchFeeTransfers(blockIds),
+    ]);
+
+    return [
+      this.groupBy(
+        ucRows as unknown as UserCommandRow[],
+        (r) => r.block_id,
+        this.mapUserCommandRow
+      ),
+      this.groupBy(
+        zkRows as unknown as ZkAppCommandRow[],
+        (r) => r.block_id,
+        this.mapZkAppCommandRow
+      ),
+      this.groupBy(
+        ftRows as unknown as FeeTransferRow[],
+        (r) => r.block_id,
+        this.mapFeeTransferRow
+      ),
+    ];
+  }
+
+  private async fetchUserCommands(blockIds: number[]) {
+    const sql = `
+      SELECT
+        buc.block_id,
+        uc.hash,
+        uc.command_type AS kind,
+        pk_from.value AS "from",
+        pk_to.value AS "to",
+        COALESCE(uc.amount, '0') AS amount,
+        uc.fee,
+        uc.memo,
+        uc.nonce,
+        buc.status,
+        buc.failure_reason
+      FROM blocks_user_commands buc
+      INNER JOIN user_commands uc ON buc.user_command_id = uc.id
+      INNER JOIN public_keys pk_from ON uc.source_id = pk_from.id
+      INNER JOIN public_keys pk_to ON uc.receiver_id = pk_to.id
+      WHERE buc.block_id = ANY($1)
+      ORDER BY buc.block_id, buc.sequence_no
+    `;
+    return this.client.unsafe(sql, [blockIds]);
+  }
+
+  private async fetchZkAppCommands(blockIds: number[]) {
+    const sql = `
+      SELECT
+        bzkc.block_id,
+        zkc.hash,
+        pk_fee.value AS fee_payer,
+        fpb.fee,
+        zkc.memo,
+        bzkc.status,
+        bzkc.failure_reasons_ids
+      FROM blocks_zkapp_commands bzkc
+      INNER JOIN zkapp_commands zkc ON bzkc.zkapp_command_id = zkc.id
+      INNER JOIN zkapp_fee_payer_body fpb ON zkc.zkapp_fee_payer_body_id = fpb.id
+      INNER JOIN public_keys pk_fee ON fpb.public_key_id = pk_fee.id
+      WHERE bzkc.block_id = ANY($1)
+      ORDER BY bzkc.block_id, bzkc.sequence_no
+    `;
+    return this.client.unsafe(sql, [blockIds]);
+  }
+
+  private async fetchFeeTransfers(blockIds: number[]) {
+    const sql = `
+      SELECT
+        bic.block_id,
+        pk.value AS recipient,
+        ic.fee,
+        ic.command_type AS type
+      FROM blocks_internal_commands bic
+      INNER JOIN internal_commands ic ON bic.internal_command_id = ic.id
+      INNER JOIN public_keys pk ON ic.receiver_id = pk.id
+      WHERE bic.block_id = ANY($1)
+        AND ic.command_type IN ('fee_transfer', 'fee_transfer_via_coinbase')
+      ORDER BY bic.block_id, bic.sequence_no
+    `;
+    return this.client.unsafe(sql, [blockIds]);
+  }
+
+  groupBy<TRow, TResult>(
+    rows: TRow[],
+    keyFn: (row: TRow) => number,
+    mapFn: (row: TRow) => TResult
+  ): Map<number, TResult[]> {
+    const map = new Map<number, TResult[]>();
+    for (const row of rows) {
+      const key = keyFn(row);
+      let arr = map.get(key);
+      if (!arr) {
+        arr = [];
+        map.set(key, arr);
+      }
+      arr.push(mapFn(row));
+    }
+    return map;
+  }
+
+  mapUserCommandRow(row: UserCommandRow): UserCommand {
+    return {
+      hash: row.hash,
+      kind: row.kind,
+      from: row.from,
+      to: row.to,
+      amount: row.amount,
+      fee: row.fee,
+      memo: row.memo,
+      nonce: row.nonce,
+      status: row.status,
+      failureReason: row.failure_reason,
+    };
+  }
+
+  mapZkAppCommandRow(row: ZkAppCommandRow): ZkAppCommand {
+    return {
+      hash: row.hash,
+      feePayer: row.fee_payer,
+      fee: row.fee,
+      memo: row.memo,
+      status: row.status,
+      failureReason: row.failure_reasons_ids
+        ? String(row.failure_reasons_ids)
+        : null,
+    };
+  }
+
+  mapFeeTransferRow(row: FeeTransferRow): FeeTransfer {
+    return {
+      recipient: row.recipient,
+      fee: row.fee,
+      type: row.type,
+    };
+  }
+
+  rowsToBlocks(
+    rows: BlockRow[],
+    userCommandsByBlock: Map<number, UserCommand[]> = new Map(),
+    zkappCommandsByBlock: Map<number, ZkAppCommand[]> = new Map(),
+    feeTransfersByBlock: Map<number, FeeTransfer[]> = new Map()
+  ): Blocks {
+    return rows.map((row) => ({
+      blockHeight: row.height,
+      creator: row.creator,
+      stateHash: row.state_hash,
+      parentHash: ENABLE_BLOCK_TRANSACTION_DETAILS ? row.parent_hash : '',
+      dateTime: new Date(parseInt(row.timestamp)).toISOString(),
+      transactions: {
+        coinbase: row.coinbase_amount || '0',
+        userCommands: userCommandsByBlock.get(row.id) ?? [],
+        zkappCommands: zkappCommandsByBlock.get(row.id) ?? [],
+        feeTransfer: feeTransfersByBlock.get(row.id) ?? [],
+      },
+    }));
+  }
+}

package/src/services/data-adapters/database-row-adapters.ts

@@ -0,0 +1,261 @@
+import type {
+  Action,
+  Event,
+  BlocksWithTransactionsMap,
+  FieldElementIdWithValueMap,
+  BlockInfo,
+} from '../../blockchain/types.js';
+import type { ArchiveNodeDatabaseRow } from '../../db/sql/events-actions/types.js';
+import {
+  createTransactionInfo,
+  createEvent,
+  createAction,
+} from '../../blockchain/utils.js';
+import { filterBestTip } from '../../consensus/mina-consensus.js';
+
+export {
+  partitionBlocks,
+  getElementIdFieldValues,
+  mapActionOrEvent,
+  removeRedundantEmittedFields,
+  sortAndFilterBlocks,
+};
+
+/**
+ * Partitions the given rows into a map where the keys are block hashes
+ * and the values are maps of transaction hashes to array of rows.
+ *
+ * For each row, if a block with the same block hash exists in the map,
+ * it adds the row to the corresponding transaction in the map.
+ * If the block does not exist, it creates a new map entry with the block hash.
+ *
+ * If the transaction does not exist within a block, it creates a new transaction entry with the transaction hash.
+ *
+ * @param rows The array of rows to be partitioned.
+ * @returns A map where the keys are block hashes and the values are maps of transaction hashes to array of rows.
+ */
+function partitionBlocks(rows: ArchiveNodeDatabaseRow[]) {
+  const blocks: BlocksWithTransactionsMap = new Map();
+  if (rows.length === 0) return blocks;
+
+  for (let i = 0; i < rows.length; i++) {
+    const { state_hash: blockHash, hash: transactionHash } = rows[i];
+    const blockData = blocks.get(blockHash);
+
+    if (blockData === undefined) {
+      const firstEntry = new Map();
+      firstEntry.set(transactionHash, [rows[i]]);
+      blocks.set(blockHash, firstEntry);
+    } else {
+      const blockDataRows = blockData.get(transactionHash);
+      if (blockDataRows) {
+        blockDataRows.push(rows[i]);
+      } else {
+        blockData.set(transactionHash, [rows[i]]);
+      }
+    }
+  }
+  return blocks;
+}
+
+/**
+ * Extracts 'id' and 'field' properties from each row in the provided array and maps them together.
+ *
+ * This function iterates over each row in the given array, and for each row, it retrieves
+ * the 'id' and 'field' properties. It then stores these values in a Map where the key is
+ * the stringified 'id' and the value is the 'field'.
+ *
+ * @param rows An array of ArchiveNodeDatabaseRow objects to process.
+ * @returns A Map where the keys are the stringified 'id' properties from the input rows and
+ *          the values are the corresponding 'field' properties.
+ */
+function getElementIdFieldValues(rows: ArchiveNodeDatabaseRow[]) {
+  const elementIdValues: FieldElementIdWithValueMap = new Map();
+  for (let i = 0; i < rows.length; i++) {
+    const { field_id, field_value } = rows[i];
+    elementIdValues.set(field_id.toString(), field_value);
+  }
+  return elementIdValues;
+}
+
+function createUniqueEventId(
+  zkapp_account_update_id: number,
+  zkapp_event_array_id: number
+) {
+  return [zkapp_account_update_id, zkapp_event_array_id].join(',');
+}
+
+/**
+ * Removes redundant fields from an array of rows based on unique event and account update identifiers.
+ *
+ * There are redundant fields in the database output because a single action/event can be made up of multiple fields, which means
+ * we get rows with the same account update id and event array id but different field values.
+ *
+ * We remove rows that refer to the same event/action that is emitted, but just keep one row since
+ * one row can be used to derive all event/action field values (by it's element_ids).
+ *
+ * @example
+ * The following rows are returned from the database, which represents one event/action:
+ * [
+ *    { id: 1, field: "1", element_ids: [3, 2, 1], zkapp_event_array_id: 1, zkapp_account_update_id: 1, zkapp_account_updates_ids: [1, 2, 3] },
+ *    { id: 2, field: "2", element_ids: [3, 2, 1], zkapp_event_array_id: 1, zkapp_account_update_id: 1, zkapp_account_updates_ids: [1, 2, 3] },
+ *    { id: 3, field: "3", element_ids: [3, 2, 1], zkapp_event_array_id: 1, zkapp_account_update_id: 1, zkapp_account_updates_ids: [1, 2, 3] },
+ * ]
+ *
+ * `id` corresponds to the field id, the `field` corresponds to the field value, and the `element_ids` correspond to the field ids. Since these rows
+ * represent a single event/action, they are all redundant because they have the same event array id and account update id.
+ *
+ * The output of this function will be:
+ * [
+ *   { id: 1, field: "1", element_ids: [3, 2, 1], zkapp_event_array_id: 1, zkapp_account_update_id: 1, zkapp_account_updates_ids: [1, 2, 3] },
+ * ]
+ *
+ *
+ * @param archiveNodeRow An array of ArchiveNodeDatabaseRow objects to process.
+ * @returns A new array of ArchiveNodeDatabaseRow objects where redundant fields have been removed.
+ * @throws {Error} If no matching account update is found for a given account update id and event array id.
+ */
+function removeRedundantEmittedFields(
+  archiveNodeRow: ArchiveNodeDatabaseRow[]
+) {
+  const inOrderTransactionRows: ArchiveNodeDatabaseRow[][] = [];
+  const seenEventOrActionIds = new Set<string>();
+  for (let i = 0; i < archiveNodeRow.length; i++) {
+    const currentRow = archiveNodeRow[i];
+    const {
+      event_field_elements_id, // The unique id for the field array in the current row
+      event_element_ids, // The list of element ids in the event (list of event_field_elements_id)
+      zkapp_account_update_id, // The unique id for the account update that emitted the event/action
+      zkapp_account_updates_ids, // List of all account update ids inside the transaction
+    } = currentRow;
+
+    // Create a unique Id consisting of the account update id and event/action id emitted
+    // This is used to check if we have already seen this event/action before.
+    const uniqueEventId = createUniqueEventId(
+      zkapp_account_update_id,
+      event_field_elements_id
+    );
+
+    if (!seenEventOrActionIds.has(uniqueEventId)) {
+      // Since multiple events/actions can be emitted in a single account update, we want to put back the event/action
+      // in the correct place. To do this, we need to know the index of the event array id in the list of event array ids (these stored in order by the Archive Node)
+      const emittedEventOrActionIndexes = findAllIndexes(
+        event_element_ids,
+        event_field_elements_id
+      );
+
+      // Since multiple account updates can be emitted in a single transaction, we need to know the index of the account update id in the list of account update ids
+      const accountUpdateIndexes = findAllIndexes(
+        zkapp_account_updates_ids,
+        zkapp_account_update_id
+      );
+
+      if (accountUpdateIndexes.length === 0) {
+        throw new Error(
+          `No matching account update found for the given account update ID (${zkapp_account_update_id}) and event array ID (${event_field_elements_id}).`
+        );
+      }
+
+      const accountUpdateIdIndex = accountUpdateIndexes[0];
+
+      // Put the event/action back in the correct place. The specific event/action should go into the index of the account update that emitted it.
+      // When we put this all together, the transaction will be in the correct order.
+      emittedEventOrActionIndexes.forEach((eventOrActionIndex) => {
+        if (inOrderTransactionRows[accountUpdateIdIndex] === undefined) {
+          inOrderTransactionRows[accountUpdateIdIndex] = [];
+        }
+        inOrderTransactionRows[accountUpdateIdIndex][eventOrActionIndex] =
+          currentRow;
+      });
+      seenEventOrActionIds.add(uniqueEventId);
+    }
+  }
+  return inOrderTransactionRows.flat();
+}
+/**
+ * Maps an array of database rows into an array of Action or Event instances.
+ *
+ * This function iterates over the given array of rows and for each row, it retrieves the element
+ * ids. It then looks up each element id in the provided map of field values and appends the corresponding
+ * value to an array, which is used to create a new Action or Event instance, depending on the
+ * provided kind.
+ *
+ * If the kind is 'event', a TransactionInfo object is created from the current row, and this information
+ * along with the array of values is used to create a new Event instance. If the kind is 'action',
+ * in addition to the TransactionInfo, the account update id from the current row is also used to
+ * create a new Action instance. The created instances are appended to the output array.
+ *
+ * If the element id does not exist in the provided map, the function simply continues with the next id.
+ *
+ * @param kind The type of instances to create. Can be either 'action' or 'event'.
+ * @param rows An array of ArchiveNodeDatabaseRow objects to map into Action or Event instances.
+ * @param elementIdFieldValues A map of element ids to their corresponding field values.
+ * @returns An array of Action or Event instances.
+ */
+function mapActionOrEvent(
+  kind: 'action' | 'event',
+  rows: ArchiveNodeDatabaseRow[],
+  elementIdFieldValues: FieldElementIdWithValueMap
+) {
+  const data: (Event | Action)[] = [];
+  for (let i = 0; i < rows.length; i++) {
+    const { zkapp_account_update_id, event_field_element_ids } = rows[i];
+    const transactionInfo = createTransactionInfo(rows[i]);
+    const elementIdToFieldValues = getFieldValuesFromElementIds(
+      event_field_element_ids,
+      elementIdFieldValues
+    );
+
+    if (kind === 'event') {
+      const event = createEvent(
+        zkapp_account_update_id.toString(),
+        elementIdToFieldValues,
+        transactionInfo
+      );
+      data.push(event);
+    } else {
+      const { zkapp_account_update_id } = rows[i];
+      const action = createAction(
+        zkapp_account_update_id.toString(),
+        elementIdToFieldValues,
+        transactionInfo
+      );
+      data.push(action);
+    }
+  }
+  return data;
+}
+
+function getFieldValuesFromElementIds(
+  element_ids: number[],
+  elementIdFieldValues: FieldElementIdWithValueMap
+) {
+  const elementIdToFieldValues = [];
+  for (const elementId of element_ids) {
+    const elementIdFieldValue = elementIdFieldValues.get(elementId.toString());
+    if (elementIdFieldValue === undefined) continue;
+    elementIdToFieldValues.push(elementIdFieldValue);
+  }
+  return elementIdToFieldValues;
+}
+
+function sortAndFilterBlocks<T extends { blockInfo: BlockInfo }>(data: T[]) {
+  data.sort((a, b) => {
+    if (a.blockInfo.height < b.blockInfo.height) return -1;
+    if (a.blockInfo.height > b.blockInfo.height) return 1;
+    if (a.blockInfo.timestamp < b.blockInfo.timestamp) return -1;
+    if (a.blockInfo.timestamp > b.blockInfo.timestamp) return 1;
+    return 0;
+  });
+  filterBestTip(data);
+}
+
+function findAllIndexes<T>(arr: T[], target: T): number[] {
+  const indexes: number[] = [];
+  arr.forEach((element, index) => {
+    if (element === target) {
+      indexes.push(index);
+    }
+  });
+  return indexes;
+}

package/src/services/events-service/events-service.interface.ts

@@ -0,0 +1,6 @@
+import { Events } from '../../blockchain/types.js';
+import { EventFilterOptionsInput } from '../../resolvers-types.js';
+
+export interface IEventsService {
+  getEvents(input: EventFilterOptionsInput, options: unknown): Promise<Events>;
+}