txcatcher 0.1.0

data/lib/txcatcher/initializer.rb

@@ -0,0 +1,148 @@
+module TxCatcher
+
+  VERSION = File.read(File.expand_path('../../', File.dirname(__FILE__)) + '/VERSION')
+
+  class << self
+    attr_accessor :db_connection, :rpc_node, :current_block_height, :config_dir
+  end
+
+  module Initializer
+
+    GEM_ROOT = File.expand_path('../..', File.dirname(__FILE__))
+    MIGRATIONS_ROOT = GEM_ROOT + '/db/migrations/'
+
+    module ConfigFile
+
+      class << self
+
+        # Determine config dir or set default. Useful when we want to
+        # have different settings for production or staging or development environments.
+        def set!(path=nil)
+          @@config_file = path and return if path
+          @@config_file = ENV['HOME'] + '/.txcatcher/config.yml'
+          ARGV.each_with_index do |a,i|
+            if a =~ /\A--config-file=.+/
+              @@config_file = File.expand_path(a.sub('--config-file=', ''))
+              ARGV.delete_at(i)
+              break
+            elsif a =~ /\A-c\Z/
+              @@config_file = File.expand_path(ARGV[1])
+              ARGV.delete_at(i) and ARGV.delete_at(i+1)
+              break
+            end
+          end
+          TxCatcher::Config.config_dir = File.dirname(@@config_file)
+          puts "Using config file: #{@@config_file}"
+        end
+
+        def path
+          @@config_file
+        end
+
+      end
+
+    end
+
+    def prepare
+      ConfigFile.set!
+      create_config_files
+      read_config_file
+      connect_to_db
+      connect_to_rpc_node
+      run_migrations if migrations_pending?
+      set_server_port
+    end
+
+    def add_route(path, &block)
+      @routes[path] = block
+    end
+
+    def create_config_files
+      FileUtils.mkdir_p(ConfigFile.path) unless File.exist?(ConfigFile.path)
+
+      unless File.exist?(ConfigFile.path)
+        puts "\e[1;33mWARNING!\e[0m \e[33mNo file #{ConfigFile.path} was found. Created a sample one for you.\e[0m"
+        puts "You should edit it and try starting the server again.\n"
+
+        FileUtils.cp(GEM_ROOT + '/templates/config.yml', ConfigFile.path)
+        config_contents = File.read(ConfigFile.path)
+        config_contents.sub!("$home", File.expand_path('~'))
+        File.open(ConfigFile.path, "w") { |f| f.write(config_contents) }
+        puts "Shutting down now.\n\n"
+        exit
+      end
+
+    end
+
+    def read_config_file
+      YAML.load_file(ConfigFile.path).each do |k,v|
+        TxCatcher::Config.send(k + '=', v)
+      end
+    end
+
+    def set_server_port
+      # Setting default port to 9498
+      unless ARGV.include?("-p")
+        ARGV.push "-p"
+        if Config.server_port
+          ARGV.push Config.server_port.to_s
+        else
+          ARGV.push "9498"
+        end
+      end
+    end
+
+    def connect_to_db
+
+      # symbolize keys for convenience
+      db_config = TxCatcher::Config.db.keys_to_sym
+
+      db_name = if db_config[:adapter] == 'sqlite'
+        if db_config[:db_path].start_with?("/")
+          db_config[:db_path]
+        else
+          "#{File.dirname(ConfigFile.path)}/#{db_config[:db_path]}"
+        end
+      else
+        db_config[:db_name]
+      end
+
+      TxCatcher.db_connection = Sequel.connect(
+        "#{db_config[:adapter]}://"                                                   +
+        "#{db_config[:user]}#{(":" if db_config[:user])}"                             +
+        "#{db_config[:password]}#{("@" if db_config[:user] || db_config[:password])}" +
+        "#{db_config[:host]}#{(":" if db_config[:port])}"                             +
+        "#{db_config[:port]}#{("/" if db_config[:host] || db_config[:port])}"         +
+        "#{db_name}"
+      )
+    end
+
+    # Connects to all bitcoin daemons listed in config
+    def connect_to_rpc_node
+      n = TxCatcher::Config.rpcnode
+      print "Checking #{n["name"]} RPC connection... "
+      begin
+        TxCatcher.rpc_node = BitcoinRPC.new("http://#{n["user"]}:#{n["password"]}@#{n["host"]}:#{n["port"]}")
+        TxCatcher.current_block_height = TxCatcher.rpc_node.getblockcount
+        print "balance: #{TxCatcher.rpc_node.getbalance}\n"
+        print "current block height: #{TxCatcher.current_block_height}\n"
+      rescue Errno::ECONNREFUSED
+        print "ERROR, cannot connect using connection data #{n["name"]}\n"
+        exit
+      end
+    end
+
+    def run_migrations
+      Sequel.extension :migration
+      print "\nPending migrations for the database detected. Migrating..."
+      Sequel::Migrator.run(TxCatcher.db_connection, MIGRATIONS_ROOT)
+      print "done\n\n"
+    end
+
+    def migrations_pending?
+      !Sequel::Migrator.is_current?(TxCatcher.db_connection, MIGRATIONS_ROOT)
+    end
+
+  end
+
+end

data/lib/txcatcher/models/address.rb

@@ -0,0 +1,13 @@
+module TxCatcher
+
+  class Address < Sequel::Model
+    one_to_many :deposits
+
+    def received_in_btc
+      Satoshi.new(self.received, from_unit: :satoshi).to_btc
+    end
+
+  end
+
+
+end

data/lib/txcatcher/models/deposit.rb

@@ -0,0 +1,23 @@
+module TxCatcher
+
+  class Deposit < Sequel::Model
+    many_to_one :transaction
+    many_to_one :address
+
+    attr_accessor :address_string
+
+    def before_save
+      if @address_string
+        self.address = Address.find_or_create(address: @address_string)
+      end
+
+      self.address.update(received: self.address.received + self.amount)
+    end
+
+    def amount_in_btc
+      Satoshi.new(self.amount, from_unit: :satoshi).to_btc
+    end
+
+  end
+
+end

data/lib/txcatcher/models/transaction.rb

@@ -0,0 +1,55 @@
+module TxCatcher
+
+  class Transaction < Sequel::Model
+    one_to_many :deposits
+
+    def before_validation
+      return if !self.new? || !self.deposits.empty?
+      parse_transaction
+      assign_transaction_attrs
+      @tx_hash["vout"].uniq { |out| out["n"] }.each do |out|
+        amount = Satoshi.new(out["value"], from_unit: :btc).to_i if out["value"]
+        address = out["scriptPubKey"]["addresses"]&.first
+        # Do not create a new deposit unless it actually makes sense to create one
+        if address && amount && amount > 0
+          self.deposits << Deposit.new(amount: amount, address_string: address)
+        end
+      end
+    end
+
+    def after_save
+      self.deposits.each do |d|
+        d.transaction = self
+        d.save
+      end
+    end
+
+    def tx_hash
+      @tx_hash || parse_transaction
+    end
+
+    def confirmations
+      if self.block_height
+        TxCatcher.current_block_height - self.block_height + 1
+      else
+        0
+      end
+    end
+
+    private
+
+      def parse_transaction
+        @tx_hash = TxCatcher.rpc_node.decoderawtransaction(self.hex)
+      end
+
+      def assign_transaction_attrs
+        self.txid = @tx_hash["txid"]
+      end
+
+      def validate
+        errors.add(:base, "No outputs for this transactions") if self.deposits.empty?
+      end
+
+  end
+
+end

data/lib/txcatcher/server.rb

@@ -0,0 +1,74 @@
+module TxCatcher
+
+  class Server < Goliath::API
+
+    use Goliath::Rack::Params
+
+    def response(env)
+      uri = env["REQUEST_URI"]
+      return route_for(uri)
+    end
+
+    def route_for(path)
+      puts "[REQUEST: #{path}]"
+      if path =~ /\A\/addr\/[0-9a-zA-Z]+\/utxo/
+        utxo(path)
+      elsif path.start_with? "/addr/"
+        address(path)
+      elsif path.start_with? "/tx/send"
+        broadcast_tx(params["rawtx"])
+      else
+        [404, {}, { error: "404, not found" }.to_json]
+      end
+    end
+
+    def address(path)
+      path = path.split("/").delete_if { |i| i.empty? }
+      addr = path.last
+
+      model = Address.where(address: addr).eager(deposits: :transactions).first
+      if model
+        transactions_ids = model.deposits.map { |d| d.transaction.txid }
+        deposits = model.deposits.map do |d|
+          t = d.transaction
+          {
+            txid:          t.txid,
+            amount:        d.amount_in_btc,
+            satoshis:      d.amount,
+            confirmations: 0
+          }
+        end
+        [200, {}, { address: model.address, received: model.received, deposits: deposits }.to_json]
+      else
+        [200, {}, { address: addr, received: 0, deposits: [] }.to_json]
+      end
+    end
+
+    def utxo(path)
+      path = path.split("/").delete_if { |i| i.empty? }
+      path.pop
+      addr = path.last
+
+      model        = Address.where(address: addr).eager(deposits: :transactions).first
+      transactions = model.deposits.map { |d| d.transaction }
+      utxos = transactions.map do |t|
+        outs = t.tx_hash["vout"].select { |out| out["scriptPubKey"]["addresses"] == [addr] }
+        outs.map! do |out|
+          out["confirmations"] = t.confirmations || 0
+          out["txid"]          = t.txid
+          out
+        end
+        outs
+      end.flatten
+      [200, {}, utxos.to_json]
+    end
+
+    def broadcast_tx(txhex)
+      TxCatcher.rpc_node.sendrawtransaction(txhex)
+      tx = TxCatcher.rpc_node.decoderawtransaction(txhex)
+      [200, {}, tx.to_json]
+    end
+
+  end
+
+end

data/lib/txcatcher/utils/hash_string_to_sym_keys.rb

@@ -0,0 +1,24 @@
+class Hash
+
+  # Replace String keys in the current hash with symbol keys
+  def keys_to_sym!
+    new_hash = keys_to_sym
+    self.clear
+    new_hash.each do |k,v|
+      self[k] = v
+    end
+  end
+
+  def keys_to_sym
+    symbolized_hash = {} 
+    self.each do |k,v|
+      if k =~ /\A[a-zA-Z0-9!?_]+\Z/
+        symbolized_hash[k.to_sym] = v
+      else
+        symbolized_hash[k] = v
+      end
+    end
+    symbolized_hash
+  end
+
+end

data/lib/txcatcher.rb

@@ -0,0 +1,21 @@
+require 'rubygems'
+require 'yaml'
+require 'json'
+require 'thread'
+require 'satoshi-unit'
+require 'sequel'
+Sequel.extension :migration
+
+require_relative 'txcatcher/utils/hash_string_to_sym_keys'
+require_relative 'txcatcher/bitcoin_rpc'
+require_relative 'txcatcher/config'
+require_relative 'txcatcher/initializer'
+require_relative 'txcatcher/catcher'
+require_relative 'txcatcher/cleaner'
+
+include TxCatcher::Initializer
+prepare
+
+require_relative 'txcatcher/models/transaction'
+require_relative 'txcatcher/models/address'
+require_relative 'txcatcher/models/deposit'

data/spec/catcher_spec.rb

@@ -0,0 +1,96 @@
+require 'rubygems'
+require 'ffi-rzmq'
+
+require_relative 'spec_helper'
+require_relative '../lib/txcatcher/models/address'
+require_relative '../lib/txcatcher/models/transaction'
+require_relative '../lib/txcatcher/catcher'
+
+RSpec.describe TxCatcher::Catcher do
+
+  before(:all) do
+    @tx_sock     = ZMQ::Context.create(1).socket(ZMQ::PUB)
+    @block_sock  = ZMQ::Context.create(2).socket(ZMQ::PUB)
+    @tx_sock.bind("ipc:///tmp/bitcoind_test.rawtx")
+    @block_sock.bind("ipc:///tmp/bitcoind_test.hashblock")
+    @hextx = File.read(File.dirname(__FILE__) + "/fixtures/transaction.txt").strip
+    @rawtx = unhexlify(File.read(File.dirname(__FILE__) + "/fixtures/transaction.txt"))
+    @catcher = TxCatcher::Catcher.new(name: "bitcoind_test")
+    sleep 1
+  end
+
+  after(:all) do
+    @tx_sock.unbind('ipc:///tmp/bitcoind_test.rawtx')
+    @block_sock.unbind('ipc:///tmp/bitcoind_test.hashblock')
+  end
+
+  it "creates a new transaction in the DB" do
+    @tx_sock.send_string('rawtx', ZMQ::SNDMORE)
+    @tx_sock.send_string(@rawtx)
+
+    i = 0
+    until (@catcher.sockets.values&.first && @catcher.sockets.values.first[:last_message]) || i > 10
+      sleep 1 and i += 1
+    end
+
+    i = 0
+    until tx = TxCatcher::Transaction.last
+      sleep 1 and i += 1
+    end
+    expect(tx.hex).to eq(@hextx)
+
+  end
+
+  it "upon receiving a new block updates transactions block height" do
+    transaction = TxCatcher::Transaction.create(hex: @hextx)
+    expect(TxCatcher.rpc_node).to receive(:getblock).and_return({ "height" => TxCatcher.current_block_height + 1, "tx" => [transaction.txid]})
+    @block_sock.send_string('hashblock', ZMQ::SNDMORE)
+    @block_sock.send_string("hello")
+
+    i = 0
+    until transaction.reload.confirmations == 1
+      sleep 1 and i += 1
+    end
+    expect(transaction.block_height).to eq(TxCatcher.current_block_height)
+
+  end
+
+  describe "error logging" do
+
+    before(:each) do
+      @error_log = File.dirname(__FILE__) + "/config/error.log"
+      TxCatcher::Config.config_dir = File.dirname(@error_log)
+    end
+
+    after(:each) do
+      File.delete(@error_log) if File.exists?(@error_log)
+    end
+
+    it "ignores validation errors" do
+      tx = eval File.read(File.dirname(__FILE__) + "/fixtures/transaction_decoded_no_outputs.txt")
+      allow(TxCatcher.rpc_node).to receive(:decoderawtransaction).and_return(tx)
+      @tx_sock.send_string('rawtx', ZMQ::SNDMORE)
+      @tx_sock.send_string(@rawtx)
+
+      i = 0
+      until (@catcher.sockets.values&.first && @catcher.sockets.values.first[:last_message]) || i > 10
+        sleep 1 and i += 1
+      end
+      expect(File.exists?(@error_log)).to be_falsey
+    end
+
+    it "logs all other errors" do
+      allow(TxCatcher.rpc_node).to receive(:decoderawtransaction).and_raise(StandardError)
+      @tx_sock.send_string('rawtx', ZMQ::SNDMORE)
+      @tx_sock.send_string(@rawtx)
+
+      i = 0
+      until File.exists?(@error_log)
+        sleep 1 and i += 1
+      end
+      expect(File.read(@error_log)).not_to be_empty
+    end
+
+  end
+
+end

data/spec/cleaner_spec.rb

@@ -0,0 +1,60 @@
+require 'rubygems'
+require 'ffi-rzmq'
+
+require_relative 'spec_helper'
+require_relative '../lib/txcatcher/models/address'
+require_relative '../lib/txcatcher/models/transaction'
+require_relative '../lib/txcatcher/cleaner'
+
+RSpec.describe TxCatcher::Cleaner do
+
+  before(:each) do
+    allow(TxCatcher.rpc_node).to receive(:decoderawtransaction).and_return({ "vout" => []})
+  end
+
+  it "doesn't clean anything if transaction count is below threshold" do
+    create_transactions(9)
+    clean_transactions
+    expect(TxCatcher::Transaction.count).to eq(9)
+    expect(TxCatcher::Deposit.count).to     eq(9)
+    expect(TxCatcher::Address.count).to     eq(9)
+  end
+
+  it "cleans excessive transactions from the DB" do
+    create_transactions(15)
+    clean_transactions
+    expect(TxCatcher::Transaction.count).to eq(9)
+    expect(TxCatcher::Deposit.count).to     eq(9)
+    expect(TxCatcher::Address.count).to     eq(9)
+  end
+
+  it "doesn't delete the address upon cleanup if it has another deposit associated with it" do
+    create_transactions(15)
+
+    d = TxCatcher::Deposit.create(address_string: "addr1", amount: 0)
+    tx = TxCatcher::Transaction.last
+    tx.deposits << d
+    tx.save
+
+    clean_transactions
+    expect(TxCatcher::Transaction.count).to eq(9)
+    expect(TxCatcher::Deposit.count).to     eq(10)
+    expect(TxCatcher::Address.count).to     eq(10)
+  end
+
+
+  def create_transactions(n)
+    (1..n).to_a.each do |i|
+      d  = TxCatcher::Deposit.new(address_string: "addr#{i}", amount: 0)
+      tx = TxCatcher::Transaction.new
+      tx.deposits << d
+      tx.save
+    end
+  end
+
+  def clean_transactions
+    TxCatcher::Cleaner.start(run_once: true)
+    sleep 1 until TxCatcher::Cleaner.stopped?
+  end
+
+end

data/spec/config/config.yml

@@ -0,0 +1,22 @@
+db:
+  adapter: sqlite
+  db_path: txcatcher_test.db
+
+  # No need to set these options for sqlite,
+  # but other DBs may require them.
+  #
+  #user: username
+  #password: password
+  #host: hostname
+  #port: 1234
+
+rpcnode:
+  name: bitcoind
+  user: roman
+  password: 12990duxcibciwebf
+  host: 127.0.0.1
+  port: 8332
+
+# zeromq: bitcoind
+max_db_transactions_stored: 10
+db_clean_period_seconds: 300

data/spec/fixtures/transaction.txt

@@ -0,0 +1 @@
+0100000001b0130468da00458400a40db0509a2d41fb6b3cf8cc54bd494a5ae4439979e9af01000000fdfe000048304502210085ba337244b670a3241af0a47cc05cdbf633dfda94c26fe8c9cfa77ab4e14bb002203d046ccc53d715d0632420c964eca9d8d6bca922fc95dd09344016e53793e59e01483045022100b111fe6e28d07aff12494976e4fd168f0265f21cc3c5abf5e83f4b872124e1930220669cd375691ef92f905efaaece66f672902294d283529c2b9ffcd488c861e766014c6952210204568228ddc9bd466bd346a9789eeec116ccdd1fda7823c264740b1261b5c4c221025df4d301519e9fb509361c025c5286d23ecf89be4060418ce2cec1e4a58c48d421031dc9a14a2daf2f20581e885c466fa6f6ace0e00a155fce9df476aadc390b2d3553aeffffffff02e046d11b0000000017a91403d7a39c85363805fb19e972c1e5c5956a75b016871e8e704f0000000017a9143921d410a0360f72d07b607173923b1e8ebab7578700000000

data/spec/fixtures/transaction_decoded_no_outputs.txt

@@ -0,0 +1 @@
+{"txid"=>"faf8368fec418a971e8fa94943e951490759498e0b2d3c4d5b9b12212c4f2e5a", "hash"=>"faf8368fec418a971e8fa94943e951490759498e0b2d3c4d5b9b12212c4f2e5a", "size"=>371, "vsize"=>371, "version"=>1, "locktime"=>0, "vin"=>[{"txid"=>"afe9799943e45a4a49bd54ccf83c6bfb412d9a50b00da400844500da680413b0", "vout"=>1, "scriptSig"=>{"asm"=>"0 304502210085ba337244b670a3241af0a47cc05cdbf633dfda94c26fe8c9cfa77ab4e14bb002203d046ccc53d715d0632420c964eca9d8d6bca922fc95dd09344016e53793e59e[ALL] 3045022100b111fe6e28d07aff12494976e4fd168f0265f21cc3c5abf5e83f4b872124e1930220669cd375691ef92f905efaaece66f672902294d283529c2b9ffcd488c861e766[ALL] 52210204568228ddc9bd466bd346a9789eeec116ccdd1fda7823c264740b1261b5c4c221025df4d301519e9fb509361c025c5286d23ecf89be4060418ce2cec1e4a58c48d421031dc9a14a2daf2f20581e885c466fa6f6ace0e00a155fce9df476aadc390b2d3553ae", "hex"=>"0048304502210085ba337244b670a3241af0a47cc05cdbf633dfda94c26fe8c9cfa77ab4e14bb002203d046ccc53d715d0632420c964eca9d8d6bca922fc95dd09344016e53793e59e01483045022100b111fe6e28d07aff12494976e4fd168f0265f21cc3c5abf5e83f4b872124e1930220669cd375691ef92f905efaaece66f672902294d283529c2b9ffcd488c861e766014c6952210204568228ddc9bd466bd346a9789eeec116ccdd1fda7823c264740b1261b5c4c221025df4d301519e9fb509361c025c5286d23ecf89be4060418ce2cec1e4a58c48d421031dc9a14a2daf2f20581e885c466fa6f6ace0e00a155fce9df476aadc390b2d3553ae"}, "sequence"=>4294967295}], "vout"=>[]}

data/spec/models/address_spec.rb

@@ -0,0 +1,5 @@
+require_relative '../spec_helper'
+require_relative '../../lib/txcatcher/models/address'
+
+RSpec.describe TxCatcher::Address do
+end

data/spec/models/transaction_spec.rb

@@ -0,0 +1,26 @@
+require_relative '../spec_helper'
+
+RSpec.describe TxCatcher::Transaction do
+
+  before(:each) do
+    @hextx = File.read(File.dirname(__FILE__) + "/../fixtures/transaction.txt").strip
+    @transaction = TxCatcher::Transaction.create(hex: @hextx)
+  end
+
+  it "parses rawtx using bitcoind" do
+    expect(@transaction.txid).to eq("faf8368fec418a971e8fa94943e951490759498e0b2d3c4d5b9b12212c4f2e5a")
+  end
+
+  it "creates addresses and deposits" do
+    expect(TxCatcher::Address.where(address: "323LGzCm43NgbtoYJhT6oKSwmKFTQ7AHzH").first.received).to eq(466700000)
+    expect(TxCatcher::Address.where(address: "36u6xv2TvZqPPYdogzfLZpMAXrYdW4Vwjp").first.received).to eq(1332776478)
+  end
+
+  it "doesn't create duplicate deposits if valid? called manually before save" do
+    transaction = TxCatcher::Transaction.new(hex: @hextx)
+    transaction.valid?
+    transaction.save
+    expect(transaction.deposits.size).to eq(2)
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,46 @@
+require 'rubygems'
+require 'yaml'
+require 'json'
+require 'satoshi-unit'
+require 'sequel'
+Sequel.extension :migration
+
+db_file = File.dirname(__FILE__) + "/config/txcatcher_test.db"
+File.delete(db_file) if File.exists?(db_file)
+
+require_relative '../lib/txcatcher/utils/hash_string_to_sym_keys'
+require_relative '../lib/txcatcher/config'
+require_relative '../lib/txcatcher/initializer'
+require_relative '../lib/txcatcher/bitcoin_rpc'
+
+include TxCatcher::Initializer
+ConfigFile.set!(File.dirname(__FILE__) + "/config/config.yml")
+
+read_config_file
+connect_to_db
+run_migrations if migrations_pending?
+connect_to_rpc_node
+
+require_relative '../lib/txcatcher/models/transaction'
+require_relative '../lib/txcatcher/models/address'
+require_relative '../lib/txcatcher/models/deposit'
+
+def unhexlify(msg)
+  msg.scan(/../).collect { |c| c.to_i(16).chr }.join
+end 
+
+RSpec.configure do |config|
+
+  config.before(:all) do
+  end
+
+  config.after(:all) do
+  end
+
+  config.before(:each) do
+    TxCatcher::Transaction.select_all.delete
+    TxCatcher::Deposit.select_all.delete
+    TxCatcher::Address.select_all.delete
+  end
+
+end

data/templates/config.yml

@@ -0,0 +1,23 @@
+db:
+  adapter: sqlite
+  db_path: txcatcher_test.db
+
+  # No need to set these options for sqlite,
+  # but other DBs may require them.
+  #
+  #user: username
+  #password: password
+  #host: hostname
+  #port: 1234
+
+rpcnode:
+  name: bitcoind
+  user: roman
+  password: 12990duxcibciwebf
+  host: 127.0.0.1
+  port: 8332
+
+zeromq: bitcoind
+max_db_transactions_stored: 100000
+db_clean_period_seconds: 300
+server_port: 9498