bitcoin-ruby 0.0.1 → 0.0.2

data/lib/bitcoin/storage/storage.rb

@@ -1,3 +1,5 @@
+# encoding: ascii-8bit
+
 # The storage implementation supports different backends, which inherit from
 # Storage::StoreBase and implement the same interface.
 # Each backend returns Storage::Models objects to easily access helper methods and metadata.
@@ -11,7 +13,7 @@ module Bitcoin::Storage
   @log = Bitcoin::Logger.create(:storage)
   def self.log; @log; end
 
-  BACKENDS = [:dummy, :sequel]
+  BACKENDS = [:dummy, :sequel, :utxo]
   BACKENDS.each do |name|
     module_eval <<-EOS
       def self.#{name} config, *args
@@ -22,7 +24,7 @@ module Bitcoin::Storage
 
   module Backends
 
-    BACKENDS.each {|b| autoload("#{b.to_s.capitalize}Store", "bitcoin/storage/#{b}") }
+    BACKENDS.each {|b| autoload("#{b.to_s.capitalize}Store", "bitcoin/storage/#{b}/#{b}_store.rb") }
 
     # Base class for storage backends.
     # Every backend must overwrite the "Not implemented" methods
@@ -40,12 +42,102 @@ module Bitcoin::Storage
       # orphan branch (not connected to main branch / genesis block)
       ORPHAN = 2
 
-      attr_reader :log
+      # possible script types
+      SCRIPT_TYPES = [:unknown, :pubkey, :hash160, :multisig, :p2sh]
+      if Bitcoin.namecoin?
+        [:name_new, :name_firstupdate, :name_update].each {|n| SCRIPT_TYPES << n }
+      end
+
+      DEFAULT_CONFIG = {
+        sqlite_pragmas: {
+          # journal_mode pragma
+          journal_mode: false,
+          # synchronous pragma
+          synchronous: false,
+          # cache_size pragma
+          # positive specifies number of cache pages to use,
+          # negative specifies cache size in kilobytes.
+          cache_size: -200_000,
+        }
+      }
+
+      SEQUEL_ADAPTERS = { :sqlite => "sqlite3", :postgres => "pg", :mysql => "mysql" }
+
+      attr_reader :log, :config
+
+      attr_accessor :config
 
       def initialize(config = {}, getblocks_callback = nil)
-        @config = config
-        @getblocks_callback = getblocks_callback
+        base = self.class.ancestors.select {|a| a.name =~ /StoreBase$/ }[0]::DEFAULT_CONFIG
+        @config = base.merge(self.class::DEFAULT_CONFIG).merge(config)
         @log    = config[:log] || Bitcoin::Storage.log
+        @log.level = @config[:log_level]  if @config[:log_level]
+        init_sequel_store
+        @getblocks_callback = getblocks_callback
+        @checkpoints = Bitcoin.network[:checkpoints] || {}
+        @watched_addrs = []
+      end
+
+      def init_sequel_store
+        return  unless (self.is_a?(SequelStore) || self.is_a?(UtxoStore)) && @config[:db]
+        @config[:db].sub!("~", ENV["HOME"])
+        @config[:db].sub!("<network>", Bitcoin.network_name.to_s)
+        adapter = SEQUEL_ADAPTERS[@config[:db].split(":").first] rescue nil
+        Bitcoin.require_dependency(adapter, gem: adapter)  if adapter
+        connect
+      end
+
+      # connect to database
+      def connect
+        Sequel.extension(:core_extensions, :sequel_3_dataset_methods)
+        @db = Sequel.connect(@config[:db].sub("~", ENV["HOME"]))
+        @db.extend_datasets(Sequel::Sequel3DatasetMethods)
+        sqlite_pragmas; migrate; check_metadata
+        log.info { "opened #{backend_name} store #{@db.uri}" }
+      end
+
+      # check if schema is up to date and migrate to current version if necessary
+      def migrate
+        migrations_path = File.join(File.dirname(__FILE__), "#{backend_name}/migrations")
+        Sequel.extension :migration
+        unless Sequel::Migrator.is_current?(@db, migrations_path)
+          Sequel::Migrator.run(@db, migrations_path)
+          unless (v = @db[:schema_info].first) && v[:magic] && v[:backend]
+            @db[:schema_info].update(
+              magic: Bitcoin.network[:magic_head].hth, backend: backend_name)
+          end
+        end
+      end
+
+      # check that database network magic and backend match the ones we are using
+      def check_metadata
+        version = @db[:schema_info].first
+        unless version[:magic] == Bitcoin.network[:magic_head].hth
+          name = Bitcoin::NETWORKS.find{|n,d| d[:magic_head].hth == version[:magic]}[0]
+          raise "Error: DB #{@db.url} was created for '#{name}' network!"
+        end
+        unless version[:backend] == backend_name
+          if version[:backend] == "sequel" && backend_name == "utxo"
+            log.warn { "Note: The 'utxo' store is now the default backend.
+            To keep using the full storage, change the configuration to use storage: 'sequel::#{@db.url}'.
+            To use the new storage backend, delete or move #{@db.url}, or specify a different database path in the config." }
+          end
+          raise "Error: DB #{@db.url} was created for '#{version[:backend]}' backend!"
+        end
+      end
+
+      # set pragma options for sqlite (if it is sqlite)
+      def sqlite_pragmas
+        return  unless (@db.is_a?(Sequel::SQLite::Database) rescue false)
+        @config[:sqlite_pragmas].each do |name, value|
+          @db.pragma_set name, value
+          log.debug { "set sqlite pragma #{name} to #{value}" }
+        end
+      end
+
+      # name of the storage backend currently in use ("sequel" or "utxo")
+      def backend_name
+        self.class.name.split("::")[-1].split("Store")[0].downcase
       end
 
       # reset the store; delete all data
@@ -53,6 +145,16 @@ module Bitcoin::Storage
         raise "Not implemented"
       end
 
+      # handle a new block incoming from the network
+      def new_block blk
+        time = Time.now
+        res = store_block(blk)
+        log.info { "block #{blk.hash} " +
+          "[#{res[0]}, #{['main', 'side', 'orphan'][res[1]]}] " +
+          "(#{"%.4fs, %3dtx, %.3fkb" % [(Time.now - time), blk.tx.size, blk.payload.bytesize.to_f/1000]})" }  if res && res[1]
+        res
+      end
+
       # store given block +blk+.
       # determine branch/chain and dept of block. trigger reorg if side branch becomes longer
       # than current main chain and connect orpans.
@@ -60,9 +162,17 @@ module Bitcoin::Storage
         log.debug { "new block #{blk.hash}" }
 
         existing = get_block(blk.hash)
-        return [existing.depth, existing.chain]  if existing && existing.chain == MAIN
+        if existing && existing.chain == MAIN
+          log.debug { "=> exists (#{existing.depth}, #{existing.chain})" }
+          return [existing.depth]
+        end
+
+        prev_block = get_block(blk.prev_block.reverse_hth)
+        unless @config[:skip_validation]
+          validator = blk.validator(self, prev_block)
+          validator.validate(rules: [:syntax], raise_errors: true)
+        end
 
-        prev_block = get_block(hth(blk.prev_block.reverse))
         if !prev_block || prev_block.chain == ORPHAN
           if blk.hash == Bitcoin.network[:genesis_hash]
             log.debug { "=> genesis (0)" }
@@ -70,24 +180,38 @@ module Bitcoin::Storage
           else
             depth = prev_block ? prev_block.depth + 1 : 0
             log.debug { "=> orphan (#{depth})" }
+            return [0, 2]  unless in_sync?
             return persist_block(blk, ORPHAN, depth)
           end
         end
         depth = prev_block.depth + 1
+
+        checkpoint = @checkpoints[depth]
+        if checkpoint && blk.hash != checkpoint
+          log.warn "Block #{depth} doesn't match checkpoint #{checkpoint}"
+          exit  if depth > get_depth # TODO: handle checkpoint mismatch properly
+        end
         if prev_block.chain == MAIN
-          next_block = prev_block.get_next_block
-          if next_block && next_block.chain == MAIN
-            log.debug { "=> side (#{depth})" }
-            return persist_block(blk, SIDE, depth)
-          else
+          if prev_block == get_head
             log.debug { "=> main (#{depth})" }
-            return persist_block(blk, MAIN, depth)
+            if !@config[:skip_validation] && ( !@checkpoints.any? || depth > @checkpoints.keys.last )
+              if self.class.name =~ /UtxoStore/
+                @config[:utxo_cache] = 0
+                @config[:block_cache] = 120
+              end
+              validator.validate(rules: [:context], raise_errors: true)
+            end
+            return persist_block(blk, MAIN, depth, prev_block.work)
+          else
+            log.debug { "=> side (#{depth})" }
+            return persist_block(blk, SIDE, depth, prev_block.work)
           end
         else
           head = get_head
-          if prev_block.depth + 1 <= head.depth
+          if prev_block.work + blk.block_work  <= head.work
             log.debug { "=> side (#{depth})" }
-            return persist_block(blk, SIDE, depth)
+            validator.validate(rules: [:context], raise_errors: true)  unless @config[:skip_validation]
+            return persist_block(blk, SIDE, depth, prev_block.work)
           else
             log.debug { "=> reorg" }
             new_main, new_side = [], []
@@ -102,8 +226,8 @@ module Bitcoin::Storage
             end
             log.debug { "new main: #{new_main.inspect}" }
             log.debug { "new side: #{new_side.inspect}" }
-            update_blocks([[new_main, {:chain => MAIN}], [new_side, {:chain => SIDE}]])
-            return persist_block(blk, MAIN, depth)
+            reorg(new_side.reverse, new_main.reverse)
+            return persist_block(blk, MAIN, depth, prev_block.work)
           end
         end
       end
@@ -114,13 +238,17 @@ module Bitcoin::Storage
       end
 
       # update +attrs+ for block with given +hash+.
-      # typically used to update chain.
+      # typically used to update the chain value during reorg.
       def update_block(hash, attrs)
         raise "Not implemented"
       end
 
+      def new_tx(tx)
+        store_tx(tx)
+      end
+
       # store given +tx+
-      def store_tx(tx)
+      def store_tx(tx, validate = true)
         raise "Not implemented"
       end
 
@@ -146,7 +274,14 @@ module Bitcoin::Storage
 
       # compute blockchain locator
       def get_locator pointer = get_head
-        return [Bitcoin::hth("\x00"*32)]  if get_depth == -1
+        if @locator
+          locator, head = @locator
+          if head == get_head
+            return locator
+          end
+        end
+
+        return [("\x00"*32).hth]  if get_depth == -1
         locator = []
         step = 1
         while pointer && pointer.hash != Bitcoin::network[:genesis_hash]
@@ -159,6 +294,7 @@ module Bitcoin::Storage
           step *= 2  if locator.size > 10
         end
         locator << Bitcoin::network[:genesis_hash]
+        @locator = [locator, get_head]
         locator
       end
 
@@ -203,6 +339,11 @@ module Bitcoin::Storage
         raise "Not implemented"
       end
 
+      # Grab the position of a tx in a given block
+      def get_idx_from_tx_hash(tx_hash)
+        raise "Not implemented"
+      end
+
       # collect all txouts containing the
       # given +script+
       def get_txouts_for_pk_script(script)
@@ -225,19 +366,83 @@ module Bitcoin::Storage
         nil
       end
 
+
+      # store address +hash160+
+      def store_addr(txout_id, hash160)
+        addr = @db[:addr][:hash160 => hash160]
+        addr_id = addr[:id]  if addr
+        addr_id ||= @db[:addr].insert({:hash160 => hash160})
+        @db[:addr_txout].insert({:addr_id => addr_id, :txout_id => txout_id})
+      end
+
+      # parse script and collect address/txout mappings to index
+      def parse_script txout, i
+        addrs, names = [], []
+        # skip huge script in testnet3 block 54507 (998000 bytes)
+        return [SCRIPT_TYPES.index(:unknown), [], []]  if txout.pk_script.bytesize > 10_000
+        script = Bitcoin::Script.new(txout.pk_script) rescue nil
+        if script
+          if script.is_hash160? || script.is_pubkey?
+            addrs << [i, script.get_hash160]
+          elsif script.is_multisig?
+            script.get_multisig_pubkeys.map do |pubkey|
+              addrs << [i, Bitcoin.hash160(pubkey.unpack("H*")[0])]
+            end
+          elsif Bitcoin.namecoin? && script.is_namecoin?
+            addrs << [i, script.get_hash160]
+            names << [i, script]
+          else
+            log.debug { "Unknown script type"}# #{tx.hash}:#{txout_idx}" }
+          end
+          script_type = SCRIPT_TYPES.index(script.type)
+        else
+          log.error { "Error parsing script"}# #{tx.hash}:#{txout_idx}" }
+          script_type = SCRIPT_TYPES.index(:unknown)
+        end
+        [script_type, addrs, names]
+      end
+
+      def add_watched_address address
+        hash160 = Bitcoin.hash160_from_address(address)
+        @db[:addr].insert(hash160: hash160)  unless @db[:addr][hash160: hash160]
+        @watched_addrs << hash160  unless @watched_addrs.include?(hash160)
+      end
+
+      def rescan
+        raise "Not implemented"
+      end
+
       # import satoshi bitcoind blk0001.dat blockchain file
       def import filename, max_depth = nil
-        File.open(filename) do |file|
-          until file.eof?
-            magic = file.read(4)
-            raise "invalid network magic"  unless Bitcoin.network[:magic_head] == magic
-            size = file.read(4).unpack("L")[0]
-            blk = Bitcoin::P::Block.new(file.read(size))
-            depth, chain = store_block(blk)
-            break  if max_depth && depth >= max_depth
+        if File.file?(filename)
+          log.info { "Importing #{filename}" }
+          File.open(filename) do |file|
+            until file.eof?
+              magic = file.read(4)
+              raise "invalid network magic"  unless Bitcoin.network[:magic_head] == magic
+              size = file.read(4).unpack("L")[0]
+              blk = Bitcoin::P::Block.new(file.read(size))
+              depth, chain = new_block(blk)
+              break  if max_depth && depth >= max_depth
+            end
           end
+        elsif File.directory?(filename)
+          Dir.entries(filename).sort.each do |file|
+            next  unless file =~ /^blk.*?\.dat$/
+            import(File.join(filename, file), max_depth)
+          end
+        else
+          raise "Import dir/file #{filename} not found"
         end
       end
+
+      def in_sync?
+        (get_head && (Time.now - get_head.time).to_i < 3600) ? true : false
+      end
+
     end
   end
 end
+
+# TODO: someday sequel will support #blob directly and #to_sequel_blob will be gone
+class String; def blob; to_sequel_blob; end; end

data/lib/bitcoin/storage/utxo/migrations/001_base_schema.rb

@@ -0,0 +1,52 @@
+Sequel.migration do
+
+
+  up do
+
+    $stdout.puts "Running migration #{__FILE__}"
+
+    binary = adapter_scheme == :postgres ? :bytea : :varchar
+
+    alter_table :schema_info do
+      add_column :magic, :varchar # network magic-head
+      add_column :backend, :varchar # storage backend
+    end
+
+    next  if tables.include?(:blk)
+
+    create_table :blk do
+      primary_key :id
+      column :hash, binary, :null => false, :unique => true, :index => true
+      column :depth, :int, :null => false, :index => true
+      column :version, :bigint, :null => false
+      column :prev_hash, binary, :null => false, :index => true
+      column :mrkl_root, binary, :null => false
+      column :time, :bigint, :null => false
+      column :bits, :bigint, :null => false
+      column :nonce, :bigint, :null => false
+      column :blk_size, :int, :null => false
+      column :chain, :int, :null => false
+      column :work, binary, :index => true
+      column :aux_pow, binary
+    end
+
+    create_table :addr do
+      primary_key :id
+      column :hash160, String, :null => false, :index => true
+    end
+
+    create_table :addr_txout do
+      column :addr_id, :int, :null => false, :index => true
+      column :txout_id, :int, :null => false, :index => true
+    end
+
+    create_table :names do
+      column :txout_id, :int, :null => false, :index => true
+      column :hash, binary, :index => true
+      column :name, binary, :index => true
+      column :value, binary
+    end
+
+  end
+
+end

data/lib/bitcoin/storage/utxo/migrations/002_utxo.rb

@@ -0,0 +1,18 @@
+Sequel.migration do
+
+  up do
+
+    $stdout.puts "Running migration #{__FILE__}"
+
+    create_table :utxo do
+      primary_key :id
+      column :tx_hash, String, null: false, index: true
+      column :tx_idx, :int, null: false, index: true
+      column :blk_id, :int, null: false, index: true
+        column :pk_script, (@db.adapter_scheme == :postgres ? :bytea : :blob), null: false
+      column :value, :bigint, null: false, index: true
+    end
+
+  end
+
+end

data/lib/bitcoin/storage/utxo/utxo_store.rb

@@ -0,0 +1,361 @@
+Bitcoin.require_dependency :sequel, message:
+  "Note: You will also need an adapter for your database like sqlite3, mysql2, postgresql"
+
+module Bitcoin::Storage::Backends
+
+  # Storage backend using Sequel to connect to arbitrary SQL databases.
+  # Inherits from StoreBase and implements its interface.
+  class UtxoStore < StoreBase
+
+
+    # possible script types
+    SCRIPT_TYPES = [:unknown, :pubkey, :hash160, :multisig, :p2sh]
+    if Bitcoin.namecoin?
+      [:name_new, :name_firstupdate, :name_update].each {|n| SCRIPT_TYPES << n }
+    end
+
+    # sequel database connection
+    attr_accessor :db
+
+    DEFAULT_CONFIG = {
+      # cache head block; it is only updated when new block comes in,
+      # so this should only be used by the store receiving new blocks.
+      cache_head: false,
+      # cache this many utxo records before syncing to disk.
+      # this should only be enabled during initial sync, because
+      # with it the store cannot reorg properly.
+      utxo_cache: 250,
+      # cache this many blocks.
+      # NOTE: this is also the maximum number of blocks the store can reorg.
+      block_cache: 120,
+      # keep an index of utxos for all addresses, not just the ones
+      # we are explicitly told about.
+      index_all_addrs: false
+    }
+
+    # create sequel store with given +config+
+    def initialize config, *args
+      super config, *args
+      @spent_outs, @new_outs, @watched_addrs = [], [], []
+      @deleted_utxos, @tx_cache, @block_cache = {}, {}, {}
+    end
+
+    # connect to database
+    def connect
+      super
+      load_watched_addrs
+#      rescan
+
+    end
+
+    # reset database; delete all data
+    def reset
+      [:blk, :utxo, :addr, :addr_txout].each {|table| @db[table].delete }
+      @head = nil
+    end
+
+    # persist given block +blk+ to storage.
+    def persist_block blk, chain, depth, prev_work = 0
+      load_watched_addrs
+      @db.transaction do
+        attrs = {
+          :hash => blk.hash.htb.blob,
+          :depth => depth,
+          :chain => chain,
+          :version => blk.ver,
+          :prev_hash => blk.prev_block.reverse.blob,
+          :mrkl_root => blk.mrkl_root.reverse.blob,
+          :time => blk.time,
+          :bits => blk.bits,
+          :nonce => blk.nonce,
+          :blk_size => blk.payload.bytesize,
+          :work => (prev_work + blk.block_work).to_s
+        }
+        existing = @db[:blk].filter(:hash => blk.hash.htb.blob)
+        if existing.any?
+          existing.update attrs
+          block_id = existing.first[:id]
+        else
+          block_id = @db[:blk].insert(attrs)
+        end
+
+        if @config[:block_cache] > 0
+          @block_cache.shift  if @block_cache.size > @config[:block_cache]
+          @deleted_utxos.shift  if @deleted_utxos.size > @config[:block_cache]
+          @block_cache[blk.hash] = blk
+        end
+
+        if chain == MAIN
+          persist_transactions(blk.tx, block_id, depth)
+          @tx_cache = {}
+          @head = wrap_block(attrs.merge(id: block_id))  if chain == MAIN
+        end
+        return depth, chain
+      end
+    end
+
+    def persist_transactions txs, block_id, depth
+      txs.each.with_index do |tx, tx_blk_idx|
+        tx.in.each.with_index do |txin, txin_tx_idx|
+          next  if txin.coinbase?
+          size = @new_outs.size
+          @new_outs.delete_if {|o| o[0][:tx_hash] == txin.prev_out.reverse.hth &&
+            o[0][:tx_idx] == txin.prev_out_index }
+          @spent_outs << {
+            tx_hash: txin.prev_out.reverse.hth.to_sequel_blob,
+            tx_idx: txin.prev_out_index  }  if @new_outs.size == size
+        end
+        tx.out.each.with_index do |txout, txout_tx_idx|
+          _, a, n = *parse_script(txout, txout_tx_idx)
+          @new_outs << [{
+              :tx_hash => tx.hash.blob,
+              :tx_idx => txout_tx_idx,
+              :blk_id => block_id,
+              :pk_script => txout.pk_script.blob,
+              :value => txout.value },
+            @config[:index_all_addrs] ? a : a.select {|a| @watched_addrs.include?(a[1]) },
+            Bitcoin.namecoin? ? n : [] ]
+        end
+      end
+      flush_spent_outs(depth)  if @spent_outs.size > @config[:utxo_cache]
+      flush_new_outs(depth)  if @new_outs.size > @config[:utxo_cache]
+    end
+
+    def reorg new_side, new_main
+      new_side.each do |block_hash|
+        raise "trying to remove non-head block!"  unless get_head.hash == block_hash
+        depth = get_depth
+        blk = @db[:blk][hash: block_hash.htb.blob]
+        delete_utxos = @db[:utxo].where(blk_id: blk[:id])
+        @db[:addr_txout].where("txout_id IN ?", delete_utxos.map{|o|o[:id]}).delete
+
+        delete_utxos.delete
+        (@deleted_utxos[depth] || []).each do |utxo|
+          utxo[:pk_script] = utxo[:pk_script].to_sequel_blob
+          utxo_id = @db[:utxo].insert(utxo)
+          addrs = Bitcoin::Script.new(utxo[:pk_script]).get_addresses
+          addrs.each do |addr|
+            hash160 = Bitcoin.hash160_from_address(addr)
+            store_addr(utxo_id, hash160)
+          end
+        end
+
+        @db[:blk].where(id: blk[:id]).update(chain: SIDE)
+      end
+
+      new_main.each do |block_hash|
+        block = @db[:blk][hash: block_hash.htb.blob]
+        blk = @block_cache[block_hash]
+        persist_transactions(blk.tx, block[:id], block[:depth])
+        @db[:blk].where(id: block[:id]).update(chain: MAIN)
+      end
+    end
+
+    def flush_spent_outs depth
+      log.time "flushed #{@spent_outs.size} spent txouts in %.4fs" do
+        if @spent_outs.any?
+          @spent_outs.each_slice(250) do |slice|
+            if @db.adapter_scheme == :postgres
+              condition = slice.map {|o| "(tx_hash = '#{o[:tx_hash]}' AND tx_idx = #{o[:tx_idx]})" }.join(" OR ")
+            else
+              condition = slice.map {|o| "(tx_hash = X'#{o[:tx_hash].hth}' AND tx_idx = #{o[:tx_idx]})" }.join(" OR ")
+            end
+            @db["DELETE FROM addr_txout WHERE EXISTS
+                   (SELECT 1 FROM utxo WHERE
+                     utxo.id = addr_txout.txout_id AND (#{condition}));"].all
+            @db["DELETE FROM utxo WHERE #{condition};"].first
+
+          end
+        end
+        @spent_outs = []
+      end
+    end
+
+    def flush_new_outs depth
+      log.time "flushed #{@new_outs.size} new txouts in %.4fs" do
+        new_utxo_ids = @db[:utxo].insert_multiple(@new_outs.map{|o|o[0]})
+        @new_outs.each.with_index do |d, idx|
+          d[1].each do |i, hash160|
+            next  unless i && hash160
+            store_addr(new_utxo_ids[idx], hash160)
+          end
+        end
+
+        @new_outs.each.with_index do |d, idx|
+          d[2].each do |i, script|
+            next  unless i && script
+            store_name(script, new_utxo_ids[idx])
+          end
+        end
+        @new_outs = []
+      end
+    end
+
+    def add_watched_address address
+      hash160 = Bitcoin.hash160_from_address(address)
+      @db[:addr].insert(hash160: hash160)  unless @db[:addr][hash160: hash160]
+      @watched_addrs << hash160  unless @watched_addrs.include?(hash160)
+    end
+
+    def load_watched_addrs
+      @watched_addrs = @db[:addr].all.map{|a| a[:hash160] }  unless @config[:index_all_addrs]
+    end
+
+    def rescan
+      load_watched_addrs
+      @rescan_lock ||= Monitor.new
+      @rescan_lock.synchronize do
+        log.info { "Rescanning #{@db[:utxo].count} utxos for #{@watched_addrs.size} addrs" }
+        count = @db[:utxo].count; n = 100_000
+        @db[:utxo].order(:id).each_slice(n).with_index do |slice, index|
+          log.debug { "rescan progress: %.2f%" % (100.0 / count * (index*n)) }
+          slice.each do |utxo|
+            next  if utxo[:pk_script].bytesize >= 10_000
+            hash160 = Bitcoin::Script.new(utxo[:pk_script]).get_hash160
+            if @config[:index_all_addrs] || @watched_addrs.include?(hash160)
+              log.info { "Found utxo for address #{Bitcoin.hash160_to_address(hash160)}: " +
+                "#{utxo[:tx_hash][0..8]}:#{utxo[:tx_idx]} (#{utxo[:value]})" }
+              addr = @db[:addr][hash160: hash160]
+              addr_utxo = {addr_id: addr[:id], txout_id: utxo[:id]}
+              @db[:addr_txout].insert(addr_utxo)  unless @db[:addr_txout][addr_utxo]
+            end
+          end
+        end
+      end
+    end
+
+    # check if block +blk_hash+ exists
+    def has_block(blk_hash)
+      !!@db[:blk].where(:hash => blk_hash.htb.blob).get(1)
+    end
+
+    # check if transaction +tx_hash+ exists
+    def has_tx(tx_hash)
+      !!@db[:utxo].where(:tx_hash => tx_hash.blob).get(1)
+    end
+
+    # get head block (highest block from the MAIN chain)
+    def get_head
+      (@config[:cache_head] && @head) ? @head :
+        @head = wrap_block(@db[:blk].filter(:chain => MAIN).order(:depth).last)
+    end
+
+    # get depth of MAIN chain
+    def get_depth
+      return -1  unless get_head
+      get_head.depth
+    end
+
+    # get block for given +blk_hash+
+    def get_block(blk_hash)
+      wrap_block(@db[:blk][:hash => blk_hash.htb.blob])
+    end
+
+    # get block by given +depth+
+    def get_block_by_depth(depth)
+      wrap_block(@db[:blk][:depth => depth, :chain => MAIN])
+    end
+
+    # get block by given +prev_hash+
+    def get_block_by_prev_hash(prev_hash)
+      wrap_block(@db[:blk][:prev_hash => prev_hash.htb.blob, :chain => MAIN])
+    end
+
+    # get block by given +tx_hash+
+    def get_block_by_tx(tx_hash)
+      block_id = @db[:utxo][tx_hash: tx_hash.blob][:blk_id]
+      get_block_by_id(block_id)
+    end
+
+    # get block by given +id+
+    def get_block_by_id(block_id)
+      wrap_block(@db[:blk][:id => block_id])
+    end
+
+    # get transaction for given +tx_hash+
+    def get_tx(tx_hash)
+      @tx_cache[tx_hash] ||= wrap_tx(tx_hash)
+    end
+
+    # get transaction by given +tx_id+
+    def get_tx_by_id(tx_id)
+      get_tx(tx_id)
+    end
+
+    def get_txout_by_id(id)
+      wrap_txout(@db[:utxo][id: id])
+    end
+
+    # get corresponding Models::TxOut for +txin+
+    def get_txout_for_txin(txin)
+      wrap_txout(@db[:utxo][tx_hash: txin.prev_out.reverse.hth.blob, tx_idx: txin.prev_out_index])
+    end
+
+    # get all Models::TxOut matching given +script+
+    def get_txouts_for_pk_script(script)
+      utxos = @db[:utxo].filter(pk_script: script.blob).order(:blk_id)
+      utxos.map {|utxo| wrap_txout(utxo) }
+    end
+
+    # get all Models::TxOut matching given +hash160+
+    def get_txouts_for_hash160(hash160, unconfirmed = false)
+      addr = @db[:addr][hash160: hash160]
+      return []  unless addr
+      @db[:addr_txout].where(addr_id: addr[:id]).map {|ao| wrap_txout(@db[:utxo][id: ao[:txout_id]]) }.compact
+    end
+
+    def get_balance hash160
+      get_txouts_for_hash160(hash160).map(&:value).inject(:+) || 0
+    end
+
+    # wrap given +block+ into Models::Block
+    def wrap_block(block)
+      return nil  unless block
+
+      data = {:id => block[:id], :depth => block[:depth], :chain => block[:chain],
+        :work => block[:work].to_i, :hash => block[:hash].hth}
+      blk = Bitcoin::Storage::Models::Block.new(self, data)
+
+      blk.ver = block[:version]
+      blk.prev_block = block[:prev_hash].reverse
+      blk.mrkl_root = block[:mrkl_root].reverse
+      blk.time = block[:time].to_i
+      blk.bits = block[:bits]
+      blk.nonce = block[:nonce]
+
+      if cached = @block_cache[block[:hash].hth]
+        blk.tx = cached.tx
+      end
+
+      blk.recalc_block_hash
+      blk
+    end
+
+    # wrap given +transaction+ into Models::Transaction
+    def wrap_tx(tx_hash)
+      utxos = @db[:utxo].where(tx_hash: tx_hash.blob)
+      return nil  unless utxos.any?
+      data = { blk_id: utxos.first[:blk_id] }
+      tx = Bitcoin::Storage::Models::Tx.new(self, data)
+      tx.hash = tx_hash # utxos.first[:tx_hash].hth
+      utxos.each {|u| tx.out[u[:tx_idx]] = wrap_txout(u) }
+      return tx
+    end
+
+    # wrap given +output+ into Models::TxOut
+    def wrap_txout(utxo)
+      return nil  unless utxo
+      data = {id: utxo[:id], tx_id: utxo[:tx_hash], tx_idx: utxo[:tx_idx]}
+      txout = Bitcoin::Storage::Models::TxOut.new(self, data)
+      txout.value = utxo[:value]
+      txout.pk_script = utxo[:pk_script]
+      txout
+    end
+
+    def check_consistency(*args)
+      log.warn { "Utxo store doesn't support consistency check" }
+    end
+
+  end
+
+end