scale.rb 0.2.17 → 0.3.1

data/lib/scale/version.rb

@@ -1,3 +1,3 @@
 module Scale
-  VERSION = "0.2.17".freeze
+  VERSION = "0.3.1".freeze
 end

data/lib/scale_bytes.rb

@@ -0,0 +1,63 @@
+module Scale
+  class Bytes
+    attr_reader :data, :bytes
+    attr_reader :offset
+
+    def initialize(data)
+      if (data.class == Array) && data.is_byte_array?
+        @bytes = data
+      elsif (data.class == String) && data.start_with?("0x") && (data.length % 2 == 0)
+        arr = data[2..].scan(/../).map(&:hex)
+        @bytes = arr
+      else
+        raise "Provided data is not valid"
+      end
+
+      @data = data
+      @offset = 0
+    end
+
+    def reset_offset
+      @offset = 0
+    end
+
+    def get_next_bytes(length)
+      result = @bytes[@offset...@offset + length]
+      if result.length < length
+        str = @data[(2 + @offset * 2)..]
+        str = str.length > 40 ? (str[0...40]).to_s + "..." : str
+        raise "No enough data: #{str}, expect length: #{length}, but #{result.length}" 
+      end
+      @offset += length
+      result
+    rescue RangeError => ex
+      puts "length: #{length}"
+      puts ex.message
+      puts ex.backtrace
+    end
+
+    def get_remaining_bytes
+      @bytes[offset..]
+    end
+
+    def to_hex_string
+      @bytes.bytes_to_hex
+    end
+
+    def to_bin_string
+      @bytes.bytes_to_bin
+    end
+
+    def to_ascii 
+      @bytes[0...offset].pack("C*") + "<================================>" + @bytes[offset..].pack("C*")
+    end
+
+    def ==(other)
+      bytes == other.bytes && offset == other.offset
+    end
+
+    def to_s
+      green(@bytes[0...offset].bytes_to_hex) + yellow(@bytes[offset..].bytes_to_hex[2..])
+    end
+  end
+end

data/lib/substrate_client.rb

@@ -39,6 +39,7 @@ class SubstrateClient
     @url = url
     @request_id = 1
     @metadata_cache = {}
+    init_types_and_metadata
   end
 
   def request(method, params)
@@ -121,8 +122,8 @@ class SubstrateClient
     block = self.chain_getBlock(block_hash)
     SubstrateClient::Helper.decode_block(block)
   rescue => ex
-    puts ex.message
-    puts ex.backtrace.join("\n\t")
+    Scale::Types.logger.error ex
+    Scale::Types.logger.error ex.backtrace.join("\n\t")
   end
 
   def get_block_events(block_hash=nil)
@@ -136,16 +137,18 @@ class SubstrateClient
     [events_data, decoded]
   end
 
-  # Plain: client.get_storage("Sudo", "Key")
-  # Plain: client.get_storage("Balances", "TotalIssuance")
-  # Map: client.get_storage("System", "Account", ["0xd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d"])
-  # DoubleMap: client.get_storage("ImOnline", "AuthoredBlocks", [2818, "0x749ddc93a65dfec3af27cc7478212cb7d4b0c0357fef35a0163966ab5333b757"])
   def get_storage(module_name, storage_name, params = nil, block_hash = nil)
     self.init_types_and_metadata(block_hash)
 
-    storage_key, return_type = SubstrateClient::Helper.generate_storage_key_from_metadata(@metadata, module_name, storage_name, params)
+    storage_key, return_type, storage_item = SubstrateClient::Helper.generate_storage_key_from_metadata(@metadata, module_name, storage_name, params)
+
     data = self.state_getStorage(storage_key, block_hash)
-    return unless data
+
+    if data.nil?
+      return if storage_item[:modifier] == "Optional"
+
+      data = storage_item[:fallback]
+    end
 
     bytes = Scale::Bytes.new(data)
     type = Scale::Types.get(return_type)

data/lib/type_builder.rb

@@ -0,0 +1,280 @@
+module Scale
+  module Types
+    class << self
+
+      # type_info: type_string or type_def
+      #   type_string: hard coded type name, Compact, H128, Vec<Compact>, (U32, U128), ...
+      #   type_def   : struct, enum, set
+      #
+      # if type_string start_with Scale::Types::, it is treat as a hard coded type
+      def get(type_info)
+        if type_info.class == ::String 
+          if type_info.start_with?('Scale::Types::')
+            return get_hard_coded_type(type_info)
+          end
+
+          # find the final type from registry
+          type_info = fix_name(type_info)
+          type_info = get_final_type_from_registry(type_info)
+        end
+
+        build_type(type_info)
+      end
+
+      private
+        def build_type(type_info)
+          # 1. hard coded types, 2. Vec<...>, 3. Option<...>, 4. [xx; x], 5. (x, y)
+          if type_info.class == ::String 
+            type_string = fix_name(type_info)
+            if type_string =~ /\AVec<.+>\z/ 
+              build_vec(type_string)
+            elsif type_info =~ /\AOption<.+>\z/
+              build_option(type_string)
+            elsif type_info =~ /\A\[.+;\s*\d+\]\z/
+              build_array(type_string)
+            elsif type_info =~ /\A\(.+\)\z/
+              build_tuple(type_string)
+            else
+              get_hard_coded_type(type_string)
+            end
+
+          # 5. Struct, 6. Enum, 7. Set
+          else 
+
+            type_info.transform_keys!(&:to_sym)
+            if type_info[:type] == "struct"
+              build_struct(type_info)
+            elsif type_info[:type] == "enum"
+              build_enum(type_info)
+            elsif type_info[:type] == "set"
+              build_set(type_info)
+            else
+              raise Scale::TypeBuildError.new("Failed to build a type from #{type_info}")
+            end
+
+          end
+        end
+
+        def get_final_type_from_registry(type_info)
+          type_registry = TypeRegistry.instance
+          if type_registry.types.nil?
+            raise TypeRegistryNotLoadYet
+          end
+          TypeRegistry.instance.get(type_info)
+        end
+
+        def get_hard_coded_type(type_name)
+          # type_name = rename(type_name)
+          type_name = (type_name.start_with?("Scale::Types::") ? type_name : "Scale::Types::#{type_name}")
+          type_name.constantize2
+        rescue => e
+          raise Scale::TypeBuildError.new("Failed to get the hard coded type named `#{type_name}`")
+        end
+
+        def build_vec(type_string)
+          inner_type_str = type_string.scan(/\AVec<(.+)>\z/).first.first
+          inner_type = get(inner_type_str)
+
+          type_name = "Vec_#{inner_type.name.gsub('Scale::Types::', '')}_"
+
+          if !Scale::Types.const_defined?(type_name)
+            klass = Class.new do
+              include Scale::Types::Vec
+              inner_type inner_type
+            end
+            Scale::Types.const_set type_name, klass
+          else
+            Scale::Types.const_get type_name
+          end
+        end
+
+        def build_option(type_string)
+          inner_type_str = type_string.scan(/\AOption<(.+)>\z/).first.first
+
+          # an exception
+          # https://substrate.dev/docs/en/knowledgebase/advanced/codec#options
+          return get("Scale::Types::OptionBool") if inner_type_str.camelize2 == "Bool"
+
+          inner_type = get(inner_type_str)
+
+          type_name = "Option_#{inner_type.name.gsub('Scale::Types::', '')}_"
+
+          if !Scale::Types.const_defined?(type_name)
+            klass = Class.new do
+              include Scale::Types::Option
+              inner_type inner_type
+            end
+            Scale::Types.const_set type_name, klass
+          else
+            Scale::Types.const_get type_name
+          end
+        end
+
+        def build_array(type_string)
+          scan_result = type_string.scan /\[(.+);\s*(\d+)\]/
+
+          #
+          inner_type_str = scan_result[0][0]
+          inner_type = get(inner_type_str)
+
+          #
+          len = scan_result[0][1].to_i
+
+          type_name = "Array_#{inner_type.name.gsub('Scale::Types::', '')}_#{len}_"
+
+          if !Scale::Types.const_defined?(type_name)
+            klass = Class.new do
+              include Scale::Types::Array
+              inner_type inner_type
+              length len
+            end
+            Scale::Types.const_set type_name, klass
+          else
+            Scale::Types.const_get type_name
+          end
+        end
+
+        def build_tuple(type_string)
+          scan_result = type_string.scan /\A\((.+)\)\z/
+          inner_types_str = scan_result[0][0]
+          inner_type_strs = inner_types_str.split(",").map do |inner_type_str| 
+            inner_type_str.strip
+          end
+
+          inner_types = inner_type_strs.map do |inner_type_str|
+            get(inner_type_str)
+          end
+
+          type_name = "Tuple_#{inner_types.map {|inner_type| inner_type.name.gsub('Scale::Types::', '')}.join("_")}_"
+          if !Scale::Types.const_defined?(type_name)
+            klass = Class.new do
+              include Scale::Types::Tuple
+              inner_types(*inner_types)
+            end
+            Scale::Types.const_set type_name, klass
+          else
+            Scale::Types.const_get type_name
+          end
+        end
+
+        def build_struct(type_info)
+          # items: {"a" => Type}
+          items = type_info[:type_mapping].map do |item|
+            item_name = item[0]
+            item_type = get(item[1])
+            [item_name, item_type]
+          end.to_h
+
+          partials = []
+          items.each_pair do |item_name, item_type|
+            partials << item_name.camelize2 + 'In' + item_type.name.gsub('Scale::Types::', '') 
+          end
+          type_name = "Struct_#{partials.join('_')}_"
+
+          if !Scale::Types.const_defined?(type_name)
+            klass = Class.new do
+              include Scale::Types::Struct
+              items(**items)
+            end
+            Scale::Types.const_set type_name, klass
+          else
+            Scale::Types.const_get type_name
+          end
+        end
+
+        # not implemented: ["Compact", "Hex"]
+        def build_enum(type_info)
+          # type_mapping: [["Item1", "Compact"], [["Item2", "Hex"]]
+          if type_info.has_key?(:type_mapping)
+            # items: {a: Type}
+            items = type_info[:type_mapping].map do |item|
+              item_name = item[0]
+              item_type = get(item[1])
+              [item_name.to_sym, item_type]
+            end.to_h
+
+            partials = []
+            items.each_pair do |item_name, item_type|
+              partials << item_name.to_s.camelize2 + 'In' + item_type.name.gsub('Scale::Types::', '') 
+            end
+            type_name = "Enum_#{partials.join('_')}_"
+
+            if !Scale::Types.const_defined?(type_name)
+              klass = Class.new do
+                include Scale::Types::Enum
+                items(**items)
+              end
+
+              return Scale::Types.const_set type_name, klass
+            else
+              return Scale::Types.const_get type_name
+            end
+          end
+
+          # value_list: [1, "hello"]
+          if type_info.has_key?(:value_list)
+            type_name = "Enum#{type_info[:value_list].map {|value| value.to_s.camelize2}.join}"
+
+            if !Scale::Types.const_defined?(type_name)
+              klass = Class.new do
+                include Scale::Types::Enum
+                values *type_info[:value_list]
+              end
+              return Scale::Types.const_set type_name, klass
+            else
+              return Scale::Types.const_get type_name
+            end
+          end
+        end
+
+
+        # {
+          # value_type: u32,
+          # value_list: {
+          #   "TransactionPayment" => 0b00000001,
+          #   "Transfer" => 0b00000010,
+          #   "Reserve" => 0b00000100,
+          #   ...
+          # }
+        # }
+        def build_set(type_info)
+          type_name = "Set#{type_info[:value_list].keys.map(&:camelize2).join("")}"
+          if !Scale::Types.const_defined?(type_name)
+            bytes_length = type_info[:value_type][1..].to_i / 8
+            klass = Class.new do
+              include Scale::Types::Set
+              items type_info[:value_list], bytes_length
+            end
+            return Scale::Types.const_set type_name, klass
+          else
+            return Scale::Types.const_get type_name
+          end
+          Scale::Types.const_set fix(name), klass
+        end
+
+        def fix_name(type)
+          type = type.gsub("T::", "")
+            .gsub("<T>", "")
+            .gsub("<T as Trait>::", "")
+            .delete("\n")
+            .gsub(/(u)(\d+)/, 'U\2')
+          return "Bool" if type == "bool"
+          return "Null" if type == "()"
+          return "String" if type == "Vec<u8>"
+          return "Compact" if type == "Compact<u32>" || type == "Compact<U32>"
+          return "Address" if type == "<Lookup as StaticLookup>::Source"
+          return "Compact" if type == "<Balance as HasCompact>::Type"
+          return "Compact" if type == "<BlockNumber as HasCompact>::Type"
+          return "Compact" if type =~ /\ACompact<[a-zA-Z0-9\s]*>\z/
+          return "CompactMoment" if type == "<Moment as HasCompact>::Type"
+          return "CompactMoment" if type == "Compact<Moment>"
+          return "InherentOfflineReport" if type == "<InherentOfflineReport as InherentOfflineReport>::Inherent"
+          return "AccountData" if type == "AccountData<Balance>"
+          return "EventRecord" if type == "EventRecord<Event, Hash>"
+
+          type
+        end
+
+    end
+  end
+end

data/lib/type_registry.rb

@@ -0,0 +1,91 @@
+module Scale
+
+  class TypeRegistry
+    include Singleton
+
+    # init by load, and will not change
+    attr_reader :spec_name, :types
+    attr_reader :versioning, :custom_types # optional
+
+    # will change by different spec version
+    attr_accessor :spec_version # optional
+    attr_accessor :metadata
+
+    def load(spec_name: nil, custom_types: nil)
+      @spec_name = nil
+      @types = nil
+      @versioning = nil
+      @custom_types = nil
+
+      default_types, _, _ = load_chain_spec_types("default")
+
+      if spec_name
+        begin
+          @spec_name = spec_name
+          spec_types, @versioning, @spec_version = load_chain_spec_types(spec_name)
+          @types = default_types.merge(spec_types)
+        rescue => ex
+          # TODO: check different errors
+          Scale::Types.logger.error "There is no types json file named #{spec_name}"
+          @types = default_types
+        end
+      else
+        @spec_name = "default"
+        @types = default_types
+      end
+
+      self.custom_types = custom_types
+      true
+    end
+
+    def get(type_name)
+      all_types = self.all_types
+      type_traverse(type_name, all_types)
+    end
+
+    def custom_types=(custom_types)
+      @custom_types = custom_types.stringify_keys if (not custom_types.nil?) && custom_types.class.name == "Hash"
+    end
+
+    def all_types
+      all_types = {}.merge(@types)
+
+      if @spec_version && @versioning
+        @versioning.each do |item|
+          if @spec_version >= item["runtime_range"][0] && 
+              ( item["runtime_range"][1].nil? || @spec_version <= item["runtime_range"][1] )
+            all_types.merge!(item["types"])
+          end
+        end
+      end
+
+      all_types.merge!(@custom_types) if @custom_types
+      all_types
+    end
+
+    private
+
+      def load_chain_spec_types(spec_name)
+        file = File.join File.expand_path("../..", __FILE__), "lib", "type_registry", "#{spec_name}.json"
+        json_string = File.open(file).read
+        json = JSON.parse(json_string)
+
+        runtime_id = json["runtime_id"]
+
+        [json["types"], json["versioning"], runtime_id]
+      end
+
+      def type_traverse(type, types)
+        if type.class == ::String 
+          real_type = types[type]
+
+          return type if real_type.nil? || real_type == type
+
+          type_traverse(real_type, types)
+        else
+          type
+        end
+      end
+  end
+
+end