@subsquid/solana-ingest 0.0.1

package/bin/run.js

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+
+require('../lib/main')

package/lib/ingest.d.ts

@@ -0,0 +1,12 @@
+import { Command, Ingest, IngestOptions, Range } from '@subsquid/util-internal-ingest-cli';
+interface Options extends IngestOptions {
+    votes: boolean;
+}
+export declare class SolanaIngest extends Ingest<Options> {
+    protected getLoggingNamespace(): string;
+    protected hasRpc(): 'required' | boolean;
+    protected setUpProgram(program: Command): void;
+    protected getBlocks(range: Range): AsyncIterable<object[]>;
+}
+export {};
+//# sourceMappingURL=ingest.d.ts.map

package/lib/ingest.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ingest.d.ts","sourceRoot":"","sources":["../src/ingest.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,OAAO,EAAE,MAAM,EAAE,aAAa,EAAE,KAAK,EAAC,MAAM,oCAAoC,CAAA;AAKxF,UAAU,OAAQ,SAAQ,aAAa;IACnC,KAAK,EAAE,OAAO,CAAA;CACjB;AAGD,qBAAa,YAAa,SAAQ,MAAM,CAAC,OAAO,CAAC;IAC7C,SAAS,CAAC,mBAAmB,IAAI,MAAM;IAIvC,SAAS,CAAC,MAAM,IAAI,UAAU,GAAG,OAAO;IAIxC,SAAS,CAAC,YAAY,CAAC,OAAO,EAAE,OAAO;cAUtB,SAAS,CAAC,KAAK,EAAE,KAAK,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC;CAiBpE"}

package/lib/ingest.js

@@ -0,0 +1,44 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SolanaIngest = void 0;
+const util_internal_1 = require("@subsquid/util-internal");
+const util_internal_ingest_cli_1 = require("@subsquid/util-internal-ingest-cli");
+const util_internal_json_1 = require("@subsquid/util-internal-json");
+const mapping_1 = require("./mapping");
+class SolanaIngest extends util_internal_ingest_cli_1.Ingest {
+    getLoggingNamespace() {
+        return 'sqd:solana-ingest';
+    }
+    hasRpc() {
+        return false;
+    }
+    setUpProgram(program) {
+        program.description('Data ingestion tool for Solana');
+        program.options.forEach(option => {
+            if (option.attributeName() == 'rawArchive') {
+                option.required = true;
+            }
+        });
+        program.option('--no-votes', 'Exclude vote transactions');
+    }
+    async *getBlocks(range) {
+        let votes = this.options().votes;
+        for await (let blocks of this.archive().getRawBlocks(range)) {
+            yield blocks.map(raw => {
+                try {
+                    let block = (0, mapping_1.mapRawBlock)(raw, !votes);
+                    return (0, util_internal_json_1.toJSON)(block);
+                }
+                catch (err) {
+                    throw (0, util_internal_1.addErrorContext)(err, {
+                        blockHash: raw.hash,
+                        blockHeight: raw.height,
+                        blockSlot: raw.slot
+                    });
+                }
+            });
+        }
+    }
+}
+exports.SolanaIngest = SolanaIngest;
+//# sourceMappingURL=ingest.js.map

package/lib/ingest.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ingest.js","sourceRoot":"","sources":["../src/ingest.ts"],"names":[],"mappings":";;;AAAA,2DAAuD;AACvD,iFAAwF;AACxF,qEAAmD;AACnD,uCAA+C;AAQ/C,MAAa,YAAa,SAAQ,iCAAe;IACnC,mBAAmB;QACzB,OAAO,mBAAmB,CAAA;IAC9B,CAAC;IAES,MAAM;QACZ,OAAO,KAAK,CAAA;IAChB,CAAC;IAES,YAAY,CAAC,OAAgB;QACnC,OAAO,CAAC,WAAW,CAAC,gCAAgC,CAAC,CAAA;QACrD,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE;YAC7B,IAAI,MAAM,CAAC,aAAa,EAAE,IAAI,YAAY,EAAE,CAAC;gBACzC,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAA;YAC1B,CAAC;QACL,CAAC,CAAC,CAAA;QACF,OAAO,CAAC,MAAM,CAAC,YAAY,EAAE,2BAA2B,CAAC,CAAA;IAC7D,CAAC;IAES,KAAK,CAAC,CAAC,SAAS,CAAC,KAAY;QACnC,IAAI,KAAK,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,KAAK,CAAA;QAChC,IAAI,KAAK,EAAE,IAAI,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC,YAAY,CAAW,KAAK,CAAC,EAAE,CAAC;YACpE,MAAM,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;gBACnB,IAAI,CAAC;oBACD,IAAI,KAAK,GAAG,IAAA,qBAAW,EAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAA;oBACpC,OAAO,IAAA,2BAAM,EAAC,KAAK,CAAC,CAAA;gBACxB,CAAC;gBAAC,OAAM,GAAQ,EAAE,CAAC;oBACf,MAAM,IAAA,+BAAe,EAAC,GAAG,EAAE;wBACvB,SAAS,EAAE,GAAG,CAAC,IAAI;wBACnB,WAAW,EAAE,GAAG,CAAC,MAAM;wBACvB,SAAS,EAAE,GAAG,CAAC,IAAI;qBACtB,CAAC,CAAA;gBACN,CAAC;YACL,CAAC,CAAC,CAAA;QACN,CAAC;IACL,CAAC;CACJ;AApCD,oCAoCC"}

package/lib/main.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=main.d.ts.map

package/lib/main.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"main.d.ts","sourceRoot":"","sources":["../src/main.ts"],"names":[],"mappings":""}

package/lib/main.js

@@ -0,0 +1,5 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const ingest_1 = require("./ingest");
+new ingest_1.SolanaIngest().run();
+//# sourceMappingURL=main.js.map

package/lib/main.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"main.js","sourceRoot":"","sources":["../src/main.ts"],"names":[],"mappings":";;AAAA,qCAAqC;AAErC,IAAI,qBAAY,EAAE,CAAC,GAAG,EAAE,CAAA"}

package/lib/mapping.d.ts

@@ -0,0 +1,199 @@
+import { Block } from '@subsquid/solana-normalization';
+import { GetSrcType } from '@subsquid/util-internal-validation';
+declare const RawBlock: import("@subsquid/util-internal-validation").Validator<{
+    hash: string;
+    height: number;
+    slot: number;
+    block: {
+        blockHeight: number;
+        blockTime: number;
+        blockhash: string;
+        parentSlot: number;
+        previousBlockhash: string;
+        transactions: {
+            meta: {
+                fee: bigint;
+                preBalances: bigint[];
+                postBalances: bigint[];
+                logMessages: string[] | null;
+                computeUnitsConsumed?: bigint | undefined;
+                err?: any;
+                preTokenBalances?: {
+                    accountIndex: number;
+                    mint: string;
+                    uiTokenAmount: {
+                        amount: bigint;
+                        decimals: number;
+                    };
+                    owner?: string | undefined;
+                    programId?: string | undefined;
+                }[] | undefined;
+                postTokenBalances?: {
+                    accountIndex: number;
+                    mint: string;
+                    uiTokenAmount: {
+                        amount: bigint;
+                        decimals: number;
+                    };
+                    owner?: string | undefined;
+                    programId?: string | undefined;
+                }[] | undefined;
+                innerInstructions?: {
+                    index: number;
+                    instructions: {
+                        accounts: number[];
+                        data: string;
+                        programIdIndex: number;
+                        stackHeight: number | null;
+                    }[];
+                }[] | undefined;
+                loadedAddresses?: {
+                    readonly: string[];
+                    writable: string[];
+                } | undefined;
+                rewards?: {
+                    pubkey: string;
+                    lamports: number;
+                    postBalance: bigint;
+                    rewardType?: string | undefined;
+                    commission?: number | undefined;
+                }[] | undefined;
+                returnData?: {
+                    programId: string;
+                    data: [string, "base64"];
+                } | undefined;
+            };
+            transaction: {
+                message: {
+                    accountKeys: string[];
+                    header: {
+                        numReadonlySignedAccounts: number;
+                        numReadonlyUnsignedAccounts: number;
+                        numRequiredSignatures: number;
+                    };
+                    instructions: {
+                        accounts: number[];
+                        data: string;
+                        programIdIndex: number;
+                        stackHeight: number | null;
+                    }[];
+                    recentBlockhash: string;
+                    addressTableLookups?: {
+                        accountKey: string;
+                        readonlyIndexes: number[];
+                        writableIndexes: number[];
+                    }[] | undefined;
+                };
+                signatures: string[];
+            };
+            version: number | "legacy";
+        }[];
+        rewards: {
+            pubkey: string;
+            lamports: number;
+            postBalance: bigint;
+            rewardType?: string | undefined;
+            commission?: number | undefined;
+        }[];
+    };
+}, {
+    hash: string;
+    height: number;
+    slot: number;
+    block: {
+        blockHeight: number;
+        blockTime: number;
+        blockhash: string;
+        parentSlot: number;
+        previousBlockhash: string;
+        transactions: {
+            meta: {
+                fee: string | number;
+                preBalances: (string | number)[];
+                postBalances: (string | number)[];
+                logMessages: string[] | null;
+                computeUnitsConsumed?: string | number | null | undefined;
+                err?: any;
+                preTokenBalances?: {
+                    accountIndex: number;
+                    mint: string;
+                    uiTokenAmount: {
+                        amount: string;
+                        decimals: number;
+                    };
+                    owner?: string | null | undefined;
+                    programId?: string | null | undefined;
+                }[] | null | undefined;
+                postTokenBalances?: {
+                    accountIndex: number;
+                    mint: string;
+                    uiTokenAmount: {
+                        amount: string;
+                        decimals: number;
+                    };
+                    owner?: string | null | undefined;
+                    programId?: string | null | undefined;
+                }[] | null | undefined;
+                innerInstructions?: {
+                    index: number;
+                    instructions: {
+                        accounts: number[];
+                        data: string;
+                        programIdIndex: number;
+                        stackHeight: number | null;
+                    }[];
+                }[] | null | undefined;
+                loadedAddresses?: {
+                    readonly: string[];
+                    writable: string[];
+                } | null | undefined;
+                rewards?: {
+                    pubkey: string;
+                    lamports: number;
+                    postBalance: string | number;
+                    rewardType?: string | null | undefined;
+                    commission?: number | null | undefined;
+                }[] | null | undefined;
+                returnData?: {
+                    programId: string;
+                    data: [string, "base64"];
+                } | null | undefined;
+            };
+            transaction: {
+                message: {
+                    accountKeys: string[];
+                    header: {
+                        numReadonlySignedAccounts: number;
+                        numReadonlyUnsignedAccounts: number;
+                        numRequiredSignatures: number;
+                    };
+                    instructions: {
+                        accounts: number[];
+                        data: string;
+                        programIdIndex: number;
+                        stackHeight: number | null;
+                    }[];
+                    recentBlockhash: string;
+                    addressTableLookups?: {
+                        accountKey: string;
+                        readonlyIndexes: number[];
+                        writableIndexes: number[];
+                    }[] | null | undefined;
+                };
+                signatures: string[];
+            };
+            version: number | "legacy";
+        }[];
+        rewards: {
+            pubkey: string;
+            lamports: number;
+            postBalance: string | number;
+            rewardType?: string | null | undefined;
+            commission?: number | null | undefined;
+        }[];
+    };
+}>;
+export type RawBlock = GetSrcType<typeof RawBlock>;
+export declare function mapRawBlock(raw: unknown, noVotes: boolean): Block;
+export {};
+//# sourceMappingURL=mapping.d.ts.map

package/lib/mapping.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"mapping.d.ts","sourceRoot":"","sources":["../src/mapping.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,KAAK,EAAc,MAAM,gCAAgC,CAAA;AAEjE,OAAO,EAA6B,UAAU,EAAc,MAAM,oCAAoC,CAAA;AActG,QAAA,MAAM,QAAQ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAKZ,CAAA;AAGF,MAAM,MAAM,QAAQ,GAAG,UAAU,CAAC,OAAO,QAAQ,CAAC,CAAA;AAGlD,wBAAgB,WAAW,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,GAAG,KAAK,CAOjE"}

package/lib/mapping.js

@@ -0,0 +1,47 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.mapRawBlock = void 0;
+const solana_normalization_1 = require("@subsquid/solana-normalization");
+const solana_rpc_data_1 = require("@subsquid/solana-rpc-data");
+const util_internal_validation_1 = require("@subsquid/util-internal-validation");
+const GetBlock = (0, util_internal_validation_1.object)({
+    blockHeight: util_internal_validation_1.NAT,
+    blockTime: util_internal_validation_1.NAT,
+    blockhash: util_internal_validation_1.B58,
+    parentSlot: util_internal_validation_1.NAT,
+    previousBlockhash: util_internal_validation_1.B58,
+    transactions: (0, util_internal_validation_1.array)(solana_rpc_data_1.Transaction),
+    rewards: (0, util_internal_validation_1.array)(solana_rpc_data_1.Reward)
+});
+const RawBlock = (0, util_internal_validation_1.object)({
+    hash: util_internal_validation_1.B58,
+    height: util_internal_validation_1.NAT,
+    slot: util_internal_validation_1.NAT,
+    block: GetBlock
+});
+function mapRawBlock(raw, noVotes) {
+    (0, util_internal_validation_1.assertValidity)(RawBlock, raw);
+    let block = (0, solana_normalization_1.mapRpcBlock)(raw);
+    if (noVotes) {
+        removeVotes(block);
+    }
+    return block;
+}
+exports.mapRawBlock = mapRawBlock;
+function removeVotes(block) {
+    let removed = new Set();
+    for (let i of block.instructions) {
+        if (i.programId == 'Vote111111111111111111111111111111111111111') {
+            removed.add(i.transactionIndex);
+        }
+    }
+    function kept(item) {
+        return !removed.has(item.transactionIndex);
+    }
+    block.transactions = block.transactions.filter(kept);
+    block.instructions = block.instructions.filter(kept);
+    block.logs = block.logs.filter(kept);
+    block.balances = block.balances.filter(kept);
+    block.tokenBalances = block.tokenBalances.filter(kept);
+}
+//# sourceMappingURL=mapping.js.map

package/lib/mapping.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"mapping.js","sourceRoot":"","sources":["../src/mapping.ts"],"names":[],"mappings":";;;AAAA,yEAAiE;AACjE,+DAA6D;AAC7D,iFAAsG;AAGtG,MAAM,QAAQ,GAAG,IAAA,iCAAM,EAAC;IACpB,WAAW,EAAE,8BAAG;IAChB,SAAS,EAAE,8BAAG;IACd,SAAS,EAAE,8BAAG;IACd,UAAU,EAAE,8BAAG;IACf,iBAAiB,EAAE,8BAAG;IACtB,YAAY,EAAE,IAAA,gCAAK,EAAC,6BAAW,CAAC;IAChC,OAAO,EAAE,IAAA,gCAAK,EAAC,wBAAM,CAAC;CACzB,CAAC,CAAA;AAGF,MAAM,QAAQ,GAAG,IAAA,iCAAM,EAAC;IACpB,IAAI,EAAE,8BAAG;IACT,MAAM,EAAE,8BAAG;IACX,IAAI,EAAE,8BAAG;IACT,KAAK,EAAE,QAAQ;CAClB,CAAC,CAAA;AAMF,SAAgB,WAAW,CAAC,GAAY,EAAE,OAAgB;IACtD,IAAA,yCAAc,EAAC,QAAQ,EAAE,GAAG,CAAC,CAAA;IAC7B,IAAI,KAAK,GAAG,IAAA,kCAAW,EAAC,GAAG,CAAC,CAAA;IAC5B,IAAI,OAAO,EAAE,CAAC;QACV,WAAW,CAAC,KAAK,CAAC,CAAA;IACtB,CAAC;IACD,OAAO,KAAK,CAAA;AAChB,CAAC;AAPD,kCAOC;AAGD,SAAS,WAAW,CAAC,KAAY;IAC7B,IAAI,OAAO,GAAG,IAAI,GAAG,EAAU,CAAA;IAE/B,KAAK,IAAI,CAAC,IAAI,KAAK,CAAC,YAAY,EAAE,CAAC;QAC/B,IAAI,CAAC,CAAC,SAAS,IAAI,6CAA6C,EAAE,CAAC;YAC/D,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAA;QACnC,CAAC;IACL,CAAC;IAED,SAAS,IAAI,CAAC,IAAgC;QAC1C,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;IAC9C,CAAC;IAED,KAAK,CAAC,YAAY,GAAG,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;IACpD,KAAK,CAAC,YAAY,GAAG,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;IACpD,KAAK,CAAC,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;IACpC,KAAK,CAAC,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;IAC5C,KAAK,CAAC,aAAa,GAAG,KAAK,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;AAC1D,CAAC"}

package/package.json

@@ -0,0 +1,33 @@
+{
+  "name": "@subsquid/solana-ingest",
+  "version": "0.0.1",
+  "description": "Data fetcher and normalizer for Solana",
+  "license": "GPL-3.0-or-later",
+  "repository": "git@github.com:subsquid/squid.git",
+  "publishConfig": {
+    "access": "public"
+  },
+  "bin": {
+    "solana-ingest": "./bin/run.js"
+  },
+  "files": [
+    "bin",
+    "lib",
+    "src"
+  ],
+  "dependencies": {
+    "@subsquid/solana-normalization": "^0.0.0",
+    "@subsquid/solana-rpc-data": "^0.0.0",
+    "@subsquid/util-internal": "^3.2.0",
+    "@subsquid/util-internal-ingest-cli": "^0.0.2",
+    "@subsquid/util-internal-json": "^1.2.3",
+    "@subsquid/util-internal-validation": "^0.4.0"
+  },
+  "devDependencies": {
+    "@types/node": "^18.18.14",
+    "typescript": "~5.3.2"
+  },
+  "scripts": {
+    "build": "rm -rf lib && tsc"
+  }
+}

package/src/ingest.ts

@@ -0,0 +1,48 @@
+import {addErrorContext} from '@subsquid/util-internal'
+import {Command, Ingest, IngestOptions, Range} from '@subsquid/util-internal-ingest-cli'
+import {toJSON} from '@subsquid/util-internal-json'
+import {mapRawBlock, RawBlock} from './mapping'
+
+
+interface Options extends IngestOptions {
+    votes: boolean
+}
+
+
+export class SolanaIngest extends Ingest<Options> {
+    protected getLoggingNamespace(): string {
+        return 'sqd:solana-ingest'
+    }
+
+    protected hasRpc(): 'required' | boolean {
+        return false
+    }
+
+    protected setUpProgram(program: Command) {
+        program.description('Data ingestion tool for Solana')
+        program.options.forEach(option => {
+            if (option.attributeName() == 'rawArchive') {
+                option.required = true
+            }
+        })
+        program.option('--no-votes', 'Exclude vote transactions')
+    }
+
+    protected async *getBlocks(range: Range): AsyncIterable<object[]> {
+        let votes = this.options().votes
+        for await (let blocks of this.archive().getRawBlocks<RawBlock>(range)) {
+            yield blocks.map(raw => {
+                try {
+                    let block = mapRawBlock(raw, !votes)
+                    return toJSON(block)
+                } catch(err: any) {
+                    throw addErrorContext(err, {
+                        blockHash: raw.hash,
+                        blockHeight: raw.height,
+                        blockSlot: raw.slot
+                    })
+                }
+            })
+        }
+    }
+}

package/src/main.ts

@@ -0,0 +1,3 @@
+import {SolanaIngest} from './ingest'
+
+new SolanaIngest().run()

package/src/mapping.ts

@@ -0,0 +1,56 @@
+import {Block, mapRpcBlock} from '@subsquid/solana-normalization'
+import {Reward, Transaction} from '@subsquid/solana-rpc-data'
+import {array, assertValidity, B58, GetSrcType, NAT, object} from '@subsquid/util-internal-validation'
+
+
+const GetBlock = object({
+    blockHeight: NAT,
+    blockTime: NAT,
+    blockhash: B58,
+    parentSlot: NAT,
+    previousBlockhash: B58,
+    transactions: array(Transaction),
+    rewards: array(Reward)
+})
+
+
+const RawBlock = object({
+    hash: B58,
+    height: NAT,
+    slot: NAT,
+    block: GetBlock
+})
+
+
+export type RawBlock = GetSrcType<typeof RawBlock>
+
+
+export function mapRawBlock(raw: unknown, noVotes: boolean): Block {
+    assertValidity(RawBlock, raw)
+    let block = mapRpcBlock(raw)
+    if (noVotes) {
+        removeVotes(block)
+    }
+    return block
+}
+
+
+function removeVotes(block: Block): void {
+    let removed = new Set<number>()
+
+    for (let i of block.instructions) {
+        if (i.programId == 'Vote111111111111111111111111111111111111111') {
+            removed.add(i.transactionIndex)
+        }
+    }
+
+    function kept(item: {transactionIndex: number}): boolean {
+        return !removed.has(item.transactionIndex)
+    }
+
+    block.transactions = block.transactions.filter(kept)
+    block.instructions = block.instructions.filter(kept)
+    block.logs = block.logs.filter(kept)
+    block.balances = block.balances.filter(kept)
+    block.tokenBalances = block.tokenBalances.filter(kept)
+}