txcatcher 0.2.4 → 0.2.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b03f8177e761f95c4e004bdee74aa320bb19495799a2752a0fc0cf476840025a
-  data.tar.gz: 2604d5b41d3d7f8c750c91d06abfaa453bc7d15326ad96484cada3f3bd543d4c
+  metadata.gz: f29602463f10ef91c91d32eef97b9a587f9005b7e63c1d4d5fb13585299f3c82
+  data.tar.gz: 0b698f2d3528a4ba5395828079904fdf67574d0bdea8660c7583092cf1f903e1
 SHA512:
-  metadata.gz: 4cde58e8673c4b62e4acde2cb541fb1dd5e1a4d1fda9b07d10b259ebed15c3191508098561e3cdead7e49858ff3473458d25c01ed95791581e5f8012106babda
-  data.tar.gz: 12aabc4580072213a7010cb9c7d271aaf77b6829a03a2157420850f7ec1ecb1a7e627f37cea1f74eb64084a984c61c5586ba19d3ae97384ede4c36ef4ce0ca52
+  metadata.gz: 2e1a21f9973c1cf4703228958522ffe1d4dd7e7a641aa3cdcbda0df8d825552833bfe7e3d926672d3de8565ab5b85d049fe61fcb70b5bf748e41124a9e1d74ec
+  data.tar.gz: 5b9613759bd60fc5a36c2b7a8d7647cc81400928224eaf099abaae66ea4f2b72489d267f1e01df22eee5d49efb7830a7b3609fc07a02b5c0d6c4c618d70b6bfd

data/VERSION

@@ -1 +1 @@
-0.2.4
+0.2.6

data/lib/txcatcher.rb

@@ -1,4 +1,5 @@
 require 'rubygems'
+require 'digest'
 require 'yaml'
 require 'json'
 require 'thread'

data/lib/txcatcher/catcher.rb

@@ -84,8 +84,16 @@ module TxCatcher
         LOGGER.report "received tx hex: #{txhex[0..50]}..."
         @queue["rawtx"] << ( Proc.new {
           tx = TxCatcher::Transaction.new(hex: txhex)
-          tx.save
-          LOGGER.report "tx #{tx.txid} saved (id: #{tx.id}), deposits (outputs):"
+          begin
+            LOGGER.report "tx #{tx.txid} caught (id: #{tx.id}), deposits (outputs):"
+            tx.save
+          rescue Sequel::ValidationFailed => e
+            if tx.errors[:txid].include?("is already taken")
+              LOGGER.report "  it's already in DB, no need to save it!"
+            else
+              raise e
+            end
+          end
           tx.deposits.each do |d|
             LOGGER.report "  id: #{d.id}, addr: #{d.address.address}, amount: #{CryptoUnit.new(Config["currency"], d.amount, from_unit: :primary).to_standart}"
           end
@@ -98,8 +106,15 @@ module TxCatcher
         height       = TxCatcher.current_block_height = block_hash["height"].to_i
         LOGGER.report "*** Block #{height} mined, transactions received:\n  #{transactions.join("  \n")}"
         @queue["hashblock"] << ( Proc.new {
+          existing_transactions   = Transaction.where(txid: transactions).map(&:txid)
+          LOGGER.report "*** Block #{height} mined, transactions received:\n  #{transactions.join("  \n")}"
           Transaction.where(txid: transactions).update(block_height: height)
         })
+        # Update RBF transactions and deposits if a transaction with lower fee (no associated deposit) got
+        # accidentally confirmed.
+        TxCatcher::Transaction.where(block_height: height).exclude(rbf_next_transaction_id: nil).each do |t|
+          t.force_deposit_association_on_rbf!
+        end
       end
 
   end # class Catcher

data/lib/txcatcher/models/address.rb

@@ -7,6 +7,22 @@ module TxCatcher
       CryptoUnit.new(Config["currency"], self.received, from_unit: :primary).to_standart
     end
 
+    def self.find_or_catch_and_create(a)
+      # Even if there are not transactions to this address yet, we still create it,
+      # because calling this method means someone is interested in this address and we need
+      # to track it and maybe force catching it not just through mempool and ZeroMQ (see catcher.rb),
+      # but also directly through querying RPC (slow, but at least we won't miss it).
+      Address.find_or_create(address: a)
+
+      if addr&.deposits.empty? && addr.created_at < (Time.now - 3600)
+        # The address is in the DB, which means someone has been checking it,
+        # but no deposits were associated with it and it's been more than 1 hour
+        # since someone first got interested. Let's query the RPC directly, see if there were any transactions to this
+        # address.
+
+      end
+    end
+
   end
 
 

data/lib/txcatcher/models/deposit.rb

@@ -6,6 +6,12 @@ module TxCatcher
 
     attr_accessor :address_string
 
+    plugin :serialization, :json, :rbf_transaction_ids
+
+    def before_save
+      self.rbf_transaction_ids
+    end
+
     def before_save
       if @address_string
         self.address = Address.find_or_create(address: @address_string)

data/lib/txcatcher/models/transaction.rb

@@ -5,6 +5,23 @@ module TxCatcher
     plugin :validation_helpers
     one_to_many :deposits
 
+    def self.find_or_catch(txid)
+      if tx = self.where(txid: txid).first
+        tx
+      else
+        self.catch(txid)
+      end
+    end
+
+    def self.catch(txid)
+      if txhex = TxCatcher.rpc_node.getrawtransaction(txid)
+        LOGGER.report "received tx hex: #{txhex[0..50]}... (fetched via manual RPC request)"
+        tx = self.new(hex: txhex)
+        tx.save
+        tx
+      end
+    end
+
     # Updates only those transactions that have changed
     def self.update_all(transactions)
       transactions_to_update = transactions.select { |t| !t.column_changes.empty? }
@@ -13,14 +30,15 @@ module TxCatcher
     end
 
     def before_validation
-      return if !self.new? || !self.deposits.empty?
-      parse_transaction
+      return if !self.new? || (!self.deposits.empty? && !self.rbf?)
       assign_transaction_attrs
-      @tx_hash["vout"].uniq { |out| out["n"] }.each do |out|
+      self.tx_hash["vout"].uniq { |out| out["n"] }.each do |out|
         amount = CryptoUnit.new(Config["currency"], out["value"], from_unit: :standart).to_i if out["value"]
         address = out["scriptPubKey"]["addresses"]&.first
         # Do not create a new deposit unless it actually makes sense to create one
-        if address && amount && amount > 0
+        if rbf?
+          self.rbf_previous_transaction.deposits.each { |d| self.deposits << d }
+        elsif address && amount && amount > 0
           self.deposits << Deposit.new(amount: amount, address_string: address)
         end
       end
@@ -32,13 +50,19 @@ module TxCatcher
 
     def after_create
       self.deposits.each do |d|
-        d.transaction = self
+        d.transaction_id = self.id
+        if self.rbf?
+          d.rbf_transaction_ids ||= []
+          d.rbf_transaction_ids.push(self.rbf_previous_transaction.id)
+          d.rbf_transaction_ids = d.rbf_transaction_ids.uniq
+        end
         d.save
+        self.rbf_previous_transaction&.update(rbf_next_transaction_id: self.id)
       end
     end
 
     def tx_hash
-      @tx_hash || parse_transaction
+      @tx_hash ||= parse_transaction
     end
 
     def confirmations
@@ -80,20 +104,85 @@ module TxCatcher
       blocks_to_check
     end
 
+    def rbf?
+      return true if self.rbf_previous_transaction_id
+      # 1. Find transactions that are like this one (inputs, outputs).
+      previous_unmarked_transactions = Transaction.where(inputs_outputs_hash: self.inputs_outputs_hash, block_height: nil, rbf_next_transaction_id: nil)
+                                        .exclude(id: self.id)
+                                        .order(Sequel.desc(:created_at)).eager(:deposits).to_a.select { |t| !t.deposits.empty? }
+      unless previous_unmarked_transactions.empty?
+        @rbf_previous_transaction = previous_unmarked_transactions.first
+        self.rbf_previous_transaction_id = @rbf_previous_transaction.id
+        true
+      else
+        false
+      end
+    end
+
+    def rbf_previous_transaction
+      @rbf_previous_transaction ||= Transaction.where(id: self.rbf_previous_transaction_id).first
+    end
+
+    def rbf_next_transaction
+      @rbf_next_transaction ||= Transaction.where(id: self.rbf_next_transaction_id).first
+    end
+
+    def input_hexes
+      @input_hexes ||= self.tx_hash["vin"].map { |input| input["scriptSig"]["hex"] }.compact.sort
+    end
+
+    def output_addresses
+      @output_addresses ||= self.tx_hash["vout"].map { |output| output["scriptPubKey"]["addresses"]&.join(",") }.compact.sort
+    end
+
+    # Sometimes, even though an RBF transaction with higher fee was broadcasted,
+    # miners accept one the lower-fee transactions instead. However, in txcatcher database, the
+    # deposits are already associated with the latest transaction. In this case,
+    # we need to find the deposits in the DB set their transaction_id field to current transaction id.
+    def force_deposit_association_on_rbf!
+      tx = self
+      while tx && tx.deposits.empty? do
+        tx = tx.rbf_next_transaction
+      end
+      tx.deposits.each do |d|
+        d.update(transaction_id: self.id)
+      end
+    end
+
+    def to_json
+      #self.tx_hash.to_json
+      self.tx_hash.merge(confirmations: self.confirmations, block_height: self.block_height).to_json
+    end
+
     private
 
       def parse_transaction
-        @tx_hash = TxCatcher.rpc_node.decoderawtransaction(self.hex)
+        TxCatcher.rpc_node.decoderawtransaction(self.hex)
       end
 
       def assign_transaction_attrs
-        self.txid = @tx_hash["txid"]
+        self.txid = self.tx_hash["txid"] unless self.txid
+        # In order to be able to identify RBF - those are normally transactions with
+        # identical inputs and outputs - we hash inputs and outputs that hash serves
+        # as an identifier that we store in our DB and thus can search all
+        # previous transactions which the current transaction might be an RBF transaction to.
+        #
+        # A few comments:
+        #
+        # 1. Although an RBF transaction may techinically have different outputs as per
+        #    protocol specification, it is true in most cases that outputs will also be
+        #    the same (that's how most wallets implement RBF). Thus,
+        #    we're also incorporating outputs into the hashed value.
+        #
+        # 2. For inputs, we're using input hexes, because pruned bitcoin-core
+        #    doesn't provide addresses.
+        self.inputs_outputs_hash ||= Digest::SHA256.hexdigest((self.input_hexes + self.output_addresses).join(""))
       end
 
       def validate
         super
         validates_unique :txid
-        errors.add(:base, "No outputs for this transactions") if self.deposits.empty?
+        errors.add(:base, "No outputs for this transaction") if !self.rbf? && self.deposits.empty?
       end
 
   end

data/lib/txcatcher/server.rb

@@ -23,6 +23,8 @@ module TxCatcher
         utxo(path)
       elsif path.start_with? "/addr/"
         address(path)
+      elsif path.start_with? "/tx/"
+        tx(path)
       elsif path.start_with? "/tx/send"
         broadcast_tx(params["rawtx"])
       elsif path.start_with? "/feerate"
@@ -39,8 +41,8 @@ module TxCatcher
       path = path.sub(/\?.*/, '').split("/").delete_if { |i| i.empty? }
       addr = path.last
 
-      address  = Address.where(address: addr).first
-      if address
+      address  = Address.find_or_create(address: addr)
+      unless address.deposits.empty?
         deposits       = Deposit.where(address_id: address.id)
         deposits_count = deposits.count
         deposits       = deposits.eager(:transaction).limit(params["limit"] || 100)
@@ -49,14 +51,15 @@ module TxCatcher
           t = d.transaction
           t.update(protected: true) unless t.protected
           t.check_block_height!(dont_save: true)
-
-          {
+          result = {
             txid:          t.txid,
             amount:        d.amount_in_btc,
             satoshis:      d.amount,
             confirmations: t.confirmations,
-            block_height:  t.block_height
+            block_height:  t.block_height,
           }
+          result.merge!({ rbf: "yes", rbf_previous_txid: t.rbf_previous_transaction.txid }) if t.rbf?
+          result
         end
         return [200, {}, { address: address.address, received: address.received, deposits_count: deposits_count, deposits_shown: deposits.size, deposits: deposits }.to_json]
       else
@@ -69,8 +72,8 @@ module TxCatcher
       path.pop
       addr = path.last
 
-      address  = Address.where(address: addr).first
-      return [200, {}, "{}"] unless address
+      address  = Address.find_or_create(address: addr)
+      return [200, {}, "{}"] unless address.deposits.empty?
       deposits = Deposit.where(address_id: address.id).limit(params["limit"] || 100).eager(:transaction)
 
       transactions = deposits.map { |d| d.transaction }
@@ -91,6 +94,10 @@ module TxCatcher
         outs.map! do |out|
           out["confirmations"]   = t.confirmations || 0
           out["txid"]            = t.txid
+          if t.rbf?
+            out["rbf"]               = "yes"
+            out["rbf_previous_txid"] = t.rbf_previous_transaction.txid
+          end
           out
         end
         outs
@@ -99,6 +106,19 @@ module TxCatcher
       return [200, {}, utxos.to_json]
     end
 
+    def tx(path)
+      path = path.sub(/\?.*/, '').split("/").delete_if { |i| i.empty? }
+      txid = path.last
+      tx = Transaction.find_or_catch(txid)
+      tx = Transaction.where(txid: txid).first
+
+      if tx && !tx.deposits.empty?
+        return [200, {}, tx.to_json]
+      else
+        return [404, {}, "Transaction not found"]
+      end
+    end
+
     def broadcast_tx(txhex)
       TxCatcher.rpc_node.sendrawtransaction(txhex)
       tx = TxCatcher.rpc_node.decoderawtransaction(txhex)

data/spec/cleaner_spec.rb

@@ -9,7 +9,7 @@ require_relative '../lib/txcatcher/cleaner'
 RSpec.describe TxCatcher::Cleaner do
 
   before(:each) do
-    allow(TxCatcher.rpc_node).to receive(:decoderawtransaction).and_return({ "vout" => []})
+    # allow(TxCatcher.rpc_node).to receive(:decoderawtransaction).and_return({ "vout" => []})
   end
 
   it "doesn't clean anything if transaction count is below threshold" do
@@ -43,7 +43,7 @@ RSpec.describe TxCatcher::Cleaner do
   end
 
   it "protects checked transactions" do
-    protected_txs = create_transactions(3, { protected: true })
+    protected_txs = create_transactions(3, { protected: true, prefix: "protected" })
     regular_txs   = create_transactions(15)
     clean_transactions
     expect(TxCatcher::Transaction.count).to eq(12)
@@ -53,9 +53,11 @@ RSpec.describe TxCatcher::Cleaner do
 
 
   def create_transactions(n, attrs={})
+    prefix = attrs.delete(:prefix)
     (1..n).to_a.map do |i|
       d  = TxCatcher::Deposit.new(address_string: "addr#{i}", amount: 0)
-      tx = TxCatcher::Transaction.new(attrs)
+      tx = TxCatcher::Transaction.new(attrs.merge(hex: File.read(File.dirname(__FILE__) + "/fixtures/transaction.txt").strip))
+      tx.txid = "#{prefix}_tx#{i}"
       tx.deposits << d
       tx.save
       tx

data/spec/logger_spec.rb

@@ -49,7 +49,7 @@ RSpec.describe TxCatcher::LOGGER do
   end
 
   it "converts Exception into a text for logging" do
-    expect($stdout).to receive(:print).with("StandardError\n[no backtrace]\n")
+    expect($stdout).to receive(:print).with("StandardError - StandardError\n[no backtrace]\n")
     expect($stdout).to receive(:print).with("\n\n")
     TxCatcher::LOGGER.report StandardError.new, :error
   end

data/spec/models/transaction_spec.rb

@@ -3,8 +3,8 @@ require_relative '../spec_helper'
 RSpec.describe TxCatcher::Transaction do
 
   class TxCatcher::Transaction
-    def assign_transaction_attrs
-      self.txid = @tx_hash["txid"] unless self.txid
+    def assign_tx_hash(h)
+      @tx_hash = h
     end
   end
 
@@ -50,4 +50,52 @@ RSpec.describe TxCatcher::Transaction do
     ).to eq([transaction2.id, transaction3.id])
   end
 
+  it "handles an RBF transaction" do
+    deposit_ids = @transaction.deposits.map(&:id)
+    rbf_tx = TxCatcher::Transaction.new(hex: @hextx)
+    rbf_tx_hash = @transaction.tx_hash
+    rbf_tx_hash["txid"] = "rbftxid1"
+    rbf_tx_hash["locktime"] = "1"
+    rbf_tx.assign_tx_hash(rbf_tx_hash)
+    rbf_tx.save
+    expect(@transaction.reload.rbf_next_transaction_id).to eq(rbf_tx.id)
+    expect(rbf_tx.rbf_previous_transaction_id).to eq(@transaction.id)
+    expect(@transaction.deposits).to be_empty
+    expect(rbf_tx.reload.deposits.map(&:id)).to eq(deposit_ids)
+    rbf_tx.deposits.each do |d|
+      expect(d.rbf_transaction_ids).to eq([@transaction.id])
+    end
+
+    rbf_tx2 = TxCatcher::Transaction.new(hex: @hextx)
+    rbf_tx_hash["txid"] = "rbftxid2"
+    rbf_tx_hash["locktime"] = "2"
+    rbf_tx2.assign_tx_hash(rbf_tx_hash)
+    rbf_tx2.save
+    expect(rbf_tx.reload.rbf_next_transaction_id).to eq(rbf_tx2.id)
+    expect(rbf_tx2.rbf_previous_transaction_id).to eq(rbf_tx.id)
+    expect(rbf_tx.deposits).to be_empty
+    expect(rbf_tx2.reload.deposits.map(&:id)).to eq(deposit_ids)
+    rbf_tx2.deposits.each do |d|
+      expect(d.rbf_transaction_ids).to eq([@transaction.id, rbf_tx.id])
+    end
+  end
+
+  it "forces deposit association with itself when confirmed, but another RBF transaction is associated with the deposit" do
+    deposit_ids = @transaction.deposits.map(&:id)
+    rbf_tx = TxCatcher::Transaction.new(hex: @hextx)
+    rbf_tx_hash = @transaction.tx_hash
+    rbf_tx_hash["txid"] = "rbftxid1"
+    rbf_tx_hash["locktime"] = "1"
+    rbf_tx.assign_tx_hash(rbf_tx_hash)
+    rbf_tx.save
+
+    @transaction.reload.force_deposit_association_on_rbf!
+    expect(rbf_tx.reload.deposits).to be_empty
+    expect(@transaction.reload.deposits.map(&:id)).to eq(deposit_ids)
+    @transaction.deposits.each do |d|
+      expect(d.rbf_transaction_ids).to eq([@transaction.id])
+    end
+
+  end
+
 end

data/spec/spec_helper.rb

@@ -1,4 +1,5 @@
 require 'rubygems'
+require 'digest'
 require 'yaml'
 require 'json'
 require 'crypto-unit'

data/txcatcher.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = "txcatcher".freeze
-  s.version = "0.2.4"
+  s.version = "0.2.6"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: txcatcher
 version: !ruby/object:Gem::Version
-  version: 0.2.4
+  version: 0.2.6
 platform: ruby
 authors:
 - Roman Snitko