txcatcher 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 19e98ad3469e868a632b24ac5b511e557be28c9c
+  data.tar.gz: 83b80df4f5cd3b2367adfbc26e9782184c8901f7
+SHA512:
+  metadata.gz: 6e5e8b687a4feb36016a0ebe7f8e07cb0c604dc9add1d31b6ce0ca412fa871f7222f45f102cef733017276277b301a52dd9f3de56a06deaa7581ef2eb1819cb5
+  data.tar.gz: 8436da5751fdf20ce4ef8d45d0d58842d3f807e73ae8fe0b83412c919b41e1bccb47d2c780e07c5cc9d6ae3f083953e195ca73c8bc1dd8bbf2b370126a1f3efc

data/.document

@@ -0,0 +1,5 @@
+lib/**/*.rb
+bin/*
+- 
+features/**/*.feature
+LICENSE.txt

data/.rspec

@@ -0,0 +1 @@
+--color

data/Gemfile

@@ -0,0 +1,16 @@
+source "https://rubygems.org"
+
+gem "goliath"
+gem "sequel"
+gem "ffi-rzmq"
+gem "satoshi-unit"
+
+group :development do
+  gem "bundler"
+  gem "jeweler"
+end
+
+group :test do
+  gem "rspec"
+  gem "sqlite3"
+end

data/Gemfile.lock

@@ -0,0 +1,116 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.4.0)
+    async-rack (0.5.1)
+      rack (~> 1.1)
+    builder (3.2.3)
+    descendants_tracker (0.0.4)
+      thread_safe (~> 0.3, >= 0.3.1)
+    diff-lcs (1.3)
+    einhorn (0.7.4)
+    em-synchrony (1.0.6)
+      eventmachine (>= 1.0.0.beta.1)
+    em-websocket (0.3.8)
+      addressable (>= 2.1.1)
+      eventmachine (>= 0.12.9)
+    eventmachine (1.2.5)
+    faraday (0.9.2)
+      multipart-post (>= 1.2, < 3)
+    ffi (1.9.18)
+    ffi-rzmq (2.0.5)
+      ffi-rzmq-core (>= 1.0.6)
+    ffi-rzmq-core (1.0.6)
+      ffi
+    git (1.3.0)
+    github_api (0.16.0)
+      addressable (~> 2.4.0)
+      descendants_tracker (~> 0.0.4)
+      faraday (~> 0.8, < 0.10)
+      hashie (>= 3.4)
+      mime-types (>= 1.16, < 3.0)
+      oauth2 (~> 1.0)
+    goliath (1.0.5)
+      async-rack
+      einhorn
+      em-synchrony (>= 1.0.0)
+      em-websocket (= 0.3.8)
+      eventmachine (>= 1.0.0.beta.4)
+      http_parser.rb (>= 0.6.0)
+      log4r
+      multi_json
+      rack (>= 1.2.2)
+      rack-contrib
+      rack-respond_to
+    hashie (3.5.6)
+    highline (1.7.8)
+    http_parser.rb (0.6.0)
+    jeweler (2.3.7)
+      builder
+      bundler (>= 1)
+      git (>= 1.2.5)
+      github_api (~> 0.16.0)
+      highline (>= 1.6.15)
+      nokogiri (>= 1.5.10)
+      psych (~> 2.2)
+      rake
+      rdoc
+      semver2
+    jwt (1.5.6)
+    log4r (1.1.10)
+    mime-types (2.99.3)
+    mini_portile2 (2.2.0)
+    multi_json (1.12.1)
+    multi_xml (0.6.0)
+    multipart-post (2.0.0)
+    nokogiri (1.8.0)
+      mini_portile2 (~> 2.2.0)
+    oauth2 (1.4.0)
+      faraday (>= 0.8, < 0.13)
+      jwt (~> 1.0)
+      multi_json (~> 1.3)
+      multi_xml (~> 0.5)
+      rack (>= 1.2, < 3)
+    psych (2.2.4)
+    rack (1.6.8)
+    rack-accept-media-types (0.9)
+    rack-contrib (1.5.0)
+      rack (~> 1.4)
+    rack-respond_to (0.9.8)
+      rack-accept-media-types (>= 0.6)
+    rake (12.0.0)
+    rdoc (5.1.0)
+    rspec (3.6.0)
+      rspec-core (~> 3.6.0)
+      rspec-expectations (~> 3.6.0)
+      rspec-mocks (~> 3.6.0)
+    rspec-core (3.6.0)
+      rspec-support (~> 3.6.0)
+    rspec-expectations (3.6.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.6.0)
+    rspec-mocks (3.6.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.6.0)
+    rspec-support (3.6.0)
+    satoshi-unit (0.2.2)
+    semver2 (3.4.2)
+    sequel (4.49.0)
+    sqlite3 (1.3.13)
+    thread_safe (0.3.6)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler
+  ffi-rzmq
+  goliath
+  jeweler
+  rspec
+  satoshi-unit
+  sequel
+  sqlite3
+
+BUNDLED WITH
+   1.15.4

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2017 Roman Snitko
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,11 @@
+= txcatcher
+
+Dependencies
+------------
+
+In order for bitcoin-core and litecoin-core to stream new transactions
+and blocks, we need ZeroMQ. Ubuntu/Debian installation instructions are here:
+
+    http://zeromq.org/distro:debian
+
+you can safely use a package weirdly located at download.opensuse.org.

data/Rakefile

@@ -0,0 +1,28 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://guides.rubygems.org/specification-reference/ for more options
+  gem.name = "txcatcher"
+  gem.homepage = "http://github.com/snitko/txcatcher"
+  gem.license = "MIT"
+  gem.summary = %Q{An lightweight version of Bitpay's Insight, allows to check Bitcoin/Litecoin addresses}
+  gem.description = %Q{Ccurrently, the only job of this gem is to collect all new Bitcoin/Litecoin transactions, store them in a DB, index addresses.}
+  gem.email = "roman.snitko@gmail.com"
+  gem.authors = ["Roman Snitko"]
+  # dependencies defined in Gemfile
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+Dir.glob('lib/tasks/*.rake').each { |r| load r }

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/bin/txcatcher

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+
+require 'goliath'
+require_relative '../lib/txcatcher'
+require_relative '../lib/txcatcher/server'
+require 'ffi-rzmq'
+
+TxCatcher::Catcher.new(name: TxCatcher::Config.zeromq)
+TxCatcher::Cleaner.start

data/db/migrations/001_create_transactions.rb

@@ -0,0 +1,11 @@
+Sequel.migration do
+  change do
+    create_table(:transactions) do
+      primary_key :id
+      String :txid, index: true
+      String :hex,  text: true
+      Integer :amount
+      Integer :block_height
+    end
+  end
+end

data/db/migrations/002_create_addresses.rb

@@ -0,0 +1,9 @@
+Sequel.migration do
+  change do
+    create_table(:addresses) do
+      primary_key :id
+      String  :address, index: true
+      Integer :received, default: 0
+    end
+  end
+end

data/db/migrations/003_create_deposits.rb

@@ -0,0 +1,10 @@
+Sequel.migration do
+  change do
+    create_table(:deposits) do
+      primary_key :id
+      Integer     :amount, null: false
+      foreign_key :transaction_id, index: true
+      foreign_key :address_id, index: true
+    end
+  end
+end

data/db/schema.rb

@@ -0,0 +1,35 @@
+Sequel.migration do
+  change do
+    create_table(:addresses, :ignore_index_errors=>true) do
+      primary_key :id
+      String :address, :size=>255
+      Integer :received, :default=>0
+      
+      index [:address]
+    end
+    
+    create_table(:deposits, :ignore_index_errors=>true) do
+      primary_key :id
+      Integer :amount, :null=>false
+      Integer :transaction_id
+      Integer :address_id
+      
+      index [:address_id]
+      index [:transaction_id]
+    end
+    
+    create_table(:schema_info) do
+      Integer :version, :default=>0, :null=>false
+    end
+    
+    create_table(:transactions, :ignore_index_errors=>true) do
+      primary_key :id
+      String :txid, :size=>255
+      String :hex, :text=>true
+      Integer :amount
+      Integer :block_height
+      
+      index [:txid]
+    end
+  end
+end

data/lib/tasks/db.rake

@@ -0,0 +1,52 @@
+require 'rubygems'
+require 'yaml'
+require 'json'
+require 'sequel'
+require_relative '../txcatcher/utils/hash_string_to_sym_keys'
+require_relative '../txcatcher/config'
+require_relative '../txcatcher/initializer'
+
+Sequel.extension :migration
+
+namespace :db do
+
+  task :environment do
+    include TxCatcher::Initializer
+    ConfigFile.set!
+    create_config_files
+    read_config_file
+    connect_to_db
+  end
+
+  desc "Migrates the database"
+  task :migrate, [:step] => :environment do |t, args|
+    target = args[:step] && (step = args[:step].to_i) > 0 ?
+               current_migration_version + step : nil
+
+    Sequel::Migrator.run(TxCatcher.db_connection, MIGRATIONS_ROOT, target: target)
+    puts "Migrated DB to version #{current_migration_version}"
+    dump_schema
+  end
+
+  desc "Rollbacks database migrations"
+  task :rollback, [:step] => :environment do |t, args|
+    target = args[:step] && (step = args[:step].to_i) > 0 ?
+      current_migration_version - step : current_migration_version - 1
+
+    Sequel::Migrator.run(TxCatcher.db_connection, MIGRATIONS_ROOT, target: target)
+    puts "Rolled back DB to version #{current_migration_version}"
+    dump_schema
+  end
+
+  def current_migration_version
+    db = TxCatcher.db_connection
+    Sequel::Migrator.migrator_class(MIGRATIONS_ROOT).new(db, MIGRATIONS_ROOT, {}).current
+  end
+
+  def dump_schema
+    TxCatcher.db_connection.extension :schema_dumper
+    open('db/schema.rb', 'w') do |f|
+      f.puts TxCatcher.db_connection.dump_schema_migration(same_db: false)
+    end
+  end
+end

data/lib/txcatcher/bitcoin_rpc.rb

@@ -0,0 +1,30 @@
+# Credits for this file go to
+# Mikica Ivosevic https://github.com/mikicaivosevic/bitcoin-rpc-ruby/
+
+require 'net/http'
+require 'uri'
+require 'json'
+
+class BitcoinRPC
+  def initialize(service_url)
+    @uri = URI.parse(service_url)
+  end
+
+  def method_missing(name, *args)
+    post_body = { 'method' => name, 'params' => args, 'id' => 'jsonrpc' }.to_json
+    resp = JSON.parse( http_post_request(post_body) )
+    raise JSONRPCError, resp['error'].to_s + " (method: '#{name}')" if resp['error']
+    resp['result']
+  end
+
+  def http_post_request(post_body)
+    http    = Net::HTTP.new(@uri.host, @uri.port)
+    request = Net::HTTP::Post.new(@uri.request_uri)
+    request.basic_auth @uri.user, @uri.password
+    request.content_type = 'application/json'
+    request.body = post_body
+    http.request(request).body
+  end
+
+  class JSONRPCError < RuntimeError; end
+end

data/lib/txcatcher/catcher.rb

@@ -0,0 +1,103 @@
+module TxCatcher
+
+  class Catcher
+
+    attr_accessor :name, :sockets
+
+    def initialize(name:, socket: "ipc:///tmp/")
+     @queue = {}
+     @sockets  = {}
+
+      {'rawtx' => "#{socket}#{name}.rawtx", 'hashblock' => "#{socket}#{name}.hashblock"}.each do |channel, address|
+        puts "Start listening on #{name} #{channel}... (#{address})"
+        listen_to_zeromq_message(channel: channel, address: address)
+      end
+    end
+
+    private
+
+      def hexlify(s)
+        a = []
+        s.each_byte do |b|
+          a << sprintf('%02X', b)
+        end
+        a.join
+      end
+
+      def listen_to_zeromq_message(channel:, address:)
+        @queue[channel] = Queue.new
+
+        # This thread is responsible for actions after the messages from ZeroMQ is parsed,
+        # typically it's writing data to DB through the models. We start it
+        # before we start listening to any messages from ZeroMQ.
+        queue_thread = Thread.new do
+          loop do
+            puts "in #{channel} queue: #{@queue[channel].size}"
+            if @queue[channel].empty?
+              sleep 1
+            else
+              begin
+                @queue[channel].pop.call
+              rescue Sequel::ValidationFailed => e
+                $stdout.puts "[WARNING #{Time.now.to_s}] #{e.class} #{e.to_s}\n"
+              rescue StandardError => e
+                File.open(TxCatcher::Config.config_dir + "/error.log", "a") do |f|
+                  f.puts "[ERROR #{Time.now.to_s}] #{e.class} #{e.to_s}\n  #{e.backtrace.join("\n  ")}\n\n"
+                end
+              end
+            end
+          end
+        end
+
+        # Now we can start receiving messages from ZeroMQ.
+        # On every received message we call a handler method, which parses it
+        # appropriately (each ZeroMQ channel has its own handler method) and then
+        # adds additional tasks, such as writing to the DB, in the queue.
+        # They queue itself is handled in the thread created above.
+        key = "#{channel}#{address}"
+        handler_thread = Thread.new do
+          context = ZMQ::Context.new
+          socket  = context.socket(ZMQ::SUB)
+          socket.setsockopt(ZMQ::SUBSCRIBE, channel)
+          socket.connect(address)
+          @sockets[key] = { object: socket }
+          loop do
+            topic   = []
+            message = []
+            socket.recv_multipart(topic, message)
+            message
+            if message[1]
+              message_hex                  = hexlify(message[1].copy_out_string).downcase
+              @sockets[key][:last_message] = message_hex
+              send("handle_#{channel}", message_hex)
+            end
+          end
+        end
+
+      end # listen_to_zeromq_message
+
+      def handle_rawtx(txhex)
+        $stdout.print "received tx hex: #{txhex[0..50]}...\n"
+
+        @queue["rawtx"] << ( Proc.new {
+          tx = TxCatcher::Transaction.new(hex: txhex)
+          tx.save
+          $stdout.puts "tx #{tx.txid} saved (id: #{tx.id}), deposits (outputs):"
+          tx.deposits.each do |d|
+            $stdout.puts "  id: #{d.id}, addr: #{d.address.address}, amount: #{Satoshi.new(d.amount, from_unit: :satoshi).to_btc}"
+          end
+        })
+      end
+
+      def handle_hashblock(block_hex)
+        block_hash   = TxCatcher.rpc_node.getblock(block_hex)
+        transactions = block_hash["tx"]
+        height       = TxCatcher.current_block_height = block_hash["height"].to_i
+        $stdout.puts "Block #{height} mined, transactions received:\n  #{transactions.join("  \n")}"
+        @queue["hashblock"] << ( Proc.new {
+          Transaction.where(txid: transactions).update(block_height: height)
+        })
+      end
+
+  end # class Catcher
+end

data/lib/txcatcher/cleaner.rb

@@ -0,0 +1,65 @@
+require 'thread'
+
+module TxCatcher
+
+  # Cleans DB so that its size doesn't go above Config.max_db_transactions_stored
+  class Cleaner
+
+    class << self
+
+      def stopped?
+        @@stopped
+      end
+    
+      def start(run_once: false)
+        @@stopped = false
+        @@stop    = false
+        Thread.new do
+          loop do
+            @@cleaning_counter = { transactions: 0, deposits: 0, addresses: 0 }
+            db_tx_count = TxCatcher::Transaction.last.id - TxCatcher::Transaction.first.id + 1
+            $stdout.print "Cleaning transactions in DB..."
+            $stdout.print "#{db_tx_count} now, needs to be below #{Config.max_db_transactions_stored}\n"
+            if db_tx_count > Config.max_db_transactions_stored
+              number_to_delete = (db_tx_count - Config.max_db_transactions_stored) + (Config.max_db_transactions_stored*0.1).round
+              clean_transactions(number_to_delete)
+              $stdout.puts "DB cleaned: #{@@cleaning_counter.to_s}"
+            else
+              $stdout.puts "Nothing to be cleaned from DB, size is below threshold."
+            end
+            if @stop || run_once
+              @@stopped = true
+              break
+            end
+            sleep Config.db_clean_period_seconds
+          end # loop
+        end # Thread
+        @@stopped
+      end
+
+      def stop
+        @@stop = true
+      end
+
+      def clean_transactions(n)
+        transactions = Transaction.order("created_at ASC").limit(n).each do |t|
+          clean_deposits(t)
+          t.delete
+          @@cleaning_counter[:transactions] += 1
+        end
+      end
+
+      def clean_deposits(transaction)
+        transaction.deposits.each do |d|
+          if d.address && d.address.deposits.count == 1
+            d.address.delete
+            @@cleaning_counter[:addresses] += 1
+          end
+          d.delete
+          @@cleaning_counter[:deposits] += 1
+        end
+      end
+
+    end # class << self
+  end # Cleaner
+end # TxCatcher

data/lib/txcatcher/config.rb

@@ -0,0 +1,20 @@
+require 'ostruct'
+
+module TxCatcher
+
+  class << (Config = OpenStruct.new)
+    def [](key_chain)
+      key_chain = key_chain.to_s.split('.')
+      config    = self.public_send(key_chain.shift)
+      key_chain.each do |key|
+        if config.kind_of?(Hash)
+          config = config[key] || config[key.to_sym]
+        else
+          return
+        end
+      end
+      config
+    end
+  end
+
+end