txcatcher 0.1.85 → 0.1.86

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b38221c572549590a66aeee17bd40d9faf33cd23
-  data.tar.gz: 2f23c67720cc64544ed1feda49c656f5fca4e538
+  metadata.gz: 965c94026bab04e89b4371d58f6a4025ed22d2e1
+  data.tar.gz: d01ca825c2f1e0670a131c5d971d2452ce3b3f55
 SHA512:
-  metadata.gz: c895162372eacf4102e6cf051e897a0af141fe4779e5865635121ebb57036954f6fd48f3e370cfa323a9795788cd0b5815ffec8a34a4c29962094bb82d924ead
-  data.tar.gz: 73141d20ac2af88b1cbafbff568becbf8349ecb87df09ecebe58431ccb459c5325be8446076fd6277e5cee9a8eeb0c0911fd8a20cad06a36db1c98342cbd16ec
+  metadata.gz: dc4da965fd01031c17163e97518bfffcd1edb1be06a68a7e7b8cfa3c9f7e65d5394434c2d5cd346127a596e843487107d1076e5b443ea8d66dcf27f8108b9b18
+  data.tar.gz: c77d3e23861d9d619748842b10e3c809c8e4f8343051826f4803e54023cc09c8f152ebb191c476de509cd5c4276fe29e96b356f8fd92a0ae07b4863b4215b414

data/Gemfile

@@ -5,6 +5,8 @@ gem "sequel"
 gem "ffi-rzmq"
 gem "crypto-unit"
 gem "sentry-raven"
+gem "aws-sdk-ses"
+gem "faraday"
 
 group :development do
   gem "bundler"

data/Gemfile.lock

@@ -4,6 +4,15 @@ GEM
     addressable (2.4.0)
     async-rack (0.5.1)
       rack (~> 1.1)
+    aws-partitions (1.68.0)
+    aws-sdk-core (3.17.0)
+      aws-partitions (~> 1.0)
+      aws-sigv4 (~> 1.0)
+      jmespath (~> 1.0)
+    aws-sdk-ses (1.6.0)
+      aws-sdk-core (~> 3)
+      aws-sigv4 (~> 1.0)
+    aws-sigv4 (1.0.2)
     builder (3.2.3)
     crypto-unit (0.3.3)
     descendants_tracker (0.0.4)
@@ -57,6 +66,7 @@ GEM
       rake
       rdoc
       semver2
+    jmespath (1.3.1)
     jwt (1.5.6)
     log4r (1.1.10)
     mime-types (2.99.3)
@@ -105,8 +115,10 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
+  aws-sdk-ses
   bundler
   crypto-unit
+  faraday
   ffi-rzmq
   goliath
   jeweler

data/VERSION

@@ -1 +1 @@
-0.1.85
+0.1.86

data/bin/txcatcher-monitor

@@ -0,0 +1,148 @@
+#!/usr/bin/env ruby
+
+# This script is designed to be launched with cron every n minutes to knock on
+# txcatcher's API and check whether new transactions are being picked up by it.
+#
+# New transactions are being fetched from a third-party source (currently: blockcypher.com)
+# and config is read from the same config file as the one txcatcher uses.
+#
+# If an incosistency is found, reports are sent to email and to the logfile
+# placed in your config_dir/txcatcher_monitor_errors.log
+# Successful checks are logged into config_dir/txcatcher_monitor.log
+#
+# To run this script, specify config dir like this:
+#
+#   txcatcher-monitor -c ~/.txcatcher/
+#
+# or
+#
+#   txcatcher-monitor --config-dir ~/.txcatcher/
+#
+# When you generated an example config file with txcatcher on the first run,
+# you should look at "monitor" section for settings related to this script.
+#
+# Just in case, here they are:
+#
+# monitor:
+#   txcatcher_url: "https://txcatcher-btc-mainnet.mydomain.com"
+#   blockchain_source_url: "https://api.blockcypher.com/v1/btc/main/txs"
+#   aws_ses:
+#     region: 'us-west-2'
+#     access_key: YOUR_AWS_KEY
+#     secret_access_key: YOUR_AWS_SECRET_ACCESS_KEY
+#   alert_mail:
+#     from: "alert@email.com"
+#     to:
+#       - "your@email.com"
+#     subject: "TxCatcher Alert"
+#
+#  However remember, you'll also need a 'logger' section in this file.
+
+require "rubygems"
+require "faraday"
+require "aws-sdk-ses"
+require "yaml"
+require 'time'
+require_relative "../lib/txcatcher/config"
+require_relative "../lib/txcatcher/logger"
+require_relative "../lib/txcatcher/initializer"
+
+def send_alert(subject, text)
+
+  return unless TxCatcher::Config["monitor"]["alert_mail"]
+
+  # Do not send emails more often than once in 1 hour
+  email_time_file = TxCatcher::Config.config_dir + "/txcatcher_monitor_last_email_time.txt"
+  t = Time.parse(File.read(email_time_file)) if File.exists?(email_time_file)
+  return if t && t > Time.now - 3600
+
+  ses_config = TxCatcher::Config["monitor"]["aws_ses"]
+  ses = Aws::SES::Client.new(
+    region: ses_config["region"],
+    access_key_id: ses_config["access_key"],
+    secret_access_key: ses_config["secret_access_key"]
+  )
+
+  resp = ses.send_email({
+    destination: { to_addresses: TxCatcher::Config["monitor"]["alert_mail"]["to"] },
+    source: TxCatcher::Config["monitor"]["alert_mail"]["from"],
+    message: {
+      body: { text: { charset: "utf-8", data: text }},
+      subject: { charset: "utf-8", data: subject   }
+    }
+  })
+
+  File.open(email_time_file, 'w') { |f| f.write Time.now.to_s }
+
+end
+
+def latest_output_addr
+
+  response = Faraday.get do |req|
+    req.url TxCatcher::Config["monitor"]["blockchain_source_url"]
+    req.options.timeout = 5
+    req.options.open_timeout = 5
+  end
+  txs = JSON.parse(response.body)
+
+  txs.each do |tx|
+    if tx["outputs"] && tx["outputs"].first["addresses"]
+      tx["outputs"].first["addresses"].each do |addr|
+        # Blockcypher returns old Litecoin P2SH addresses which start with 3, which
+        # txcatcher doesn't support. Let's just ignore them.
+        if addr[0] != "3" || !TxCatcher::Config["monitor"]["ignore_3_litecoin_addr"]
+          return addr
+        end
+      end
+    end
+  end
+
+end
+
+def try_txcatcher(addr, sleep_time: 3)
+
+  raise StandardError
+
+  url = TxCatcher::Config["monitor"]["txcatcher_url"] + "/addr/#{addr}/utxo"
+
+  response = {}
+  3.times do
+    sleep sleep_time # let's wait, perhaps txcatcher hasn't caught up
+    break unless (response = fetch_txcatcher_for_addr(addr)).empty?
+  end
+
+  if response.empty?
+    send_alert(
+      "TxCatcher response empty",
+      "TxCatcher returned an empty {} for #{url}"
+    )
+    TxCatcher::Logger.report("Checked #{url}, got empty response", :error)
+  else
+    TxCatcher::Logger.report("Checked #{url}, got #{response.size} utxos", :info)
+  end
+
+end
+
+def fetch_txcatcher_for_addr(addr)
+  response = Faraday.get do |req|
+    req.url TxCatcher::Config["monitor"]["txcatcher_url"] + "/addr/#{addr}1/utxo"
+    req.options.timeout = 5
+    req.options.open_timeout = 5
+  end
+  JSON.parse(response.body)
+end
+
+begin
+  include TxCatcher::Initializer
+  ConfigFile.set!
+  TxCatcher::Logger.set_log_file_names!(log: "txcatcher_monitor.log", error: "txcatcher_monitor_errors.log")
+  read_config_file
+  try_txcatcher(latest_output_addr)
+rescue StandardError => e
+  TxCatcher::Logger.report(e, :error)
+  send_alert(
+    "TxCatcher monitor error",
+    "While checking #{TxCatcher::Config["monitor"]["txcatcher_url"]} response, got the following error\n\n" +
+    "#{e.to_s}\n\n#{e.backtrace.join("\n")}"
+  )
+end

data/lib/txcatcher/logger.rb

@@ -10,8 +10,16 @@ module TxCatcher
       unknown: 5
     }
 
+    @@log_file_name       = "txcatcher.log"
+    @@error_log_file_name = "error.log"
+
     class << self
 
+      def set_log_file_names!(log: nil, error: nil)
+        @@log_file_name       = log   if log
+        @@error_log_file_name = error if error
+      end
+
       def report(message, log_level=:info, data: nil, timestamp: false)
         [:logfile, :stdout, :sentry].each do |out|
           if LOG_LEVELS[log_level] >= LOG_LEVELS[Config["logger"]["#{out}_level"].to_sym]
@@ -29,7 +37,7 @@ module TxCatcher
         end
 
         def report_to_logfile(message, log_level, data: nil, timestamp: true) # always gonna be forcing timestamp to be true here
-          fn = LOG_LEVELS[log_level] >= LOG_LEVELS[:error] ? "error.log" : "txcatcher.log"
+          fn = LOG_LEVELS[log_level] >= LOG_LEVELS[:error] ? @@error_log_file_name : @@log_file_name
           fn = TxCatcher::Config.config_dir + "/#{fn}"
 
           File.open(fn, "a") do |f|
@@ -40,6 +48,7 @@ module TxCatcher
         end
 
         def report_to_sentry(e, log_level, data: nil, timestamp: timestamp)
+          return unless TxCatcher::Config["logger"]["sentry_dsn"]
           data ||= {}
           data.merge!(environment: Config["environment"], host: Config["host"], currency: Config["currency"])
           Raven.tags_context data

data/templates/config.yml

@@ -24,6 +24,24 @@ logger:
   sentry_level: error
   sentry_dsn: null
 
+monitor:
+  # REPLACE with your txcatcher url
+  txcatcher_url: "https://txcatcher-btc-mainnet.mydomain.com"
+
+  # LEAVE as is if on bitcoin mainnet, replace with
+  #   https://api.blockcypher.com/v1/ltc/main/txs for Litecoin
+  #   https://api.blockcypher.com/v1/btc/test3/txs for Bitcoin TESTNET
+  blockchain_source_url: "https://api.blockcypher.com/v1/btc/main/txs"
+  aws_ses:
+    region: 'us-west-2'
+    access_key: YOUR_AWS_KEY
+    secret_access_key: YOUR_AWS_SECRET_ACCESS_KEY
+  alert_mail:
+    from: "alert@email.com"
+    to:
+      - "your@email.com"
+    subject: "TxCatcher Alert"
+
 zeromq: bitcoind
 max_db_transactions_stored: 100000
 db_clean_period_seconds: 300

data/txcatcher.gemspec

@@ -2,19 +2,19 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: txcatcher 0.1.85 ruby lib
+# stub: txcatcher 0.1.86 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "txcatcher"
-  s.version = "0.1.85"
+  s.version = "0.1.86"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Roman Snitko"]
-  s.date = "2017-09-25"
-  s.description = "Currently, the only job of this gem is to collect all new Bitcoin/Litecoin transactions, store them in a DB, index addresses."
+  s.date = "2018-03-05"
+  s.description = "Ccurrently, the only job of this gem is to collect all new Bitcoin/Litecoin transactions, store them in a DB, index addresses."
   s.email = "roman.snitko@gmail.com"
-  s.executables = ["goliath.log", "goliath_stdout.log", "txcatcher"]
+  s.executables = ["txcatcher", "txcatcher-monitor"]
   s.extra_rdoc_files = [
     "LICENSE.txt",
     "README.md"
@@ -28,12 +28,13 @@ Gem::Specification.new do |s|
     "README.md",
     "Rakefile",
     "VERSION",
-    "bin/goliath.log",
-    "bin/goliath_stdout.log",
     "bin/txcatcher",
+    "bin/txcatcher-monitor",
     "db/migrations/001_create_transactions.rb",
     "db/migrations/002_create_addresses.rb",
     "db/migrations/003_create_deposits.rb",
+    "db/migrations/004_add_timestamps_to_transactions.rb",
+    "db/migrations/005_add_protected_flag_to_transactions.rb",
     "db/schema.rb",
     "lib/tasks/db.rake",
     "lib/txcatcher.rb",
@@ -42,20 +43,22 @@ Gem::Specification.new do |s|
     "lib/txcatcher/cleaner.rb",
     "lib/txcatcher/config.rb",
     "lib/txcatcher/initializer.rb",
+    "lib/txcatcher/logger.rb",
     "lib/txcatcher/models/address.rb",
     "lib/txcatcher/models/deposit.rb",
     "lib/txcatcher/models/transaction.rb",
     "lib/txcatcher/server.rb",
+    "lib/txcatcher/utils/crypto_unit.rb",
     "lib/txcatcher/utils/hash_string_to_sym_keys.rb",
     "spec/catcher_spec.rb",
     "spec/cleaner_spec.rb",
     "spec/config/config.yml.sample",
-    "spec/config/txcatcher_test.db",
     "spec/fixtures/transaction.txt",
     "spec/fixtures/transaction_decoded_no_outputs.txt",
-    "spec/models/address_spec.rb",
+    "spec/logger_spec.rb",
     "spec/models/transaction_spec.rb",
     "spec/spec_helper.rb",
+    "spec/utils/crypto_unit.rb",
     "templates/config.yml",
     "txcatcher.gemspec"
   ]
@@ -71,14 +74,20 @@ Gem::Specification.new do |s|
       s.add_runtime_dependency(%q<goliath>, [">= 0"])
       s.add_runtime_dependency(%q<sequel>, [">= 0"])
       s.add_runtime_dependency(%q<ffi-rzmq>, [">= 0"])
-      s.add_runtime_dependency(%q<satoshi-unit>, [">= 0"])
+      s.add_runtime_dependency(%q<crypto-unit>, [">= 0"])
+      s.add_runtime_dependency(%q<sentry-raven>, [">= 0"])
+      s.add_runtime_dependency(%q<aws-sdk-ses>, [">= 0"])
+      s.add_runtime_dependency(%q<faraday>, [">= 0"])
       s.add_development_dependency(%q<bundler>, [">= 0"])
       s.add_development_dependency(%q<jeweler>, [">= 0"])
     else
       s.add_dependency(%q<goliath>, [">= 0"])
       s.add_dependency(%q<sequel>, [">= 0"])
       s.add_dependency(%q<ffi-rzmq>, [">= 0"])
-      s.add_dependency(%q<satoshi-unit>, [">= 0"])
+      s.add_dependency(%q<crypto-unit>, [">= 0"])
+      s.add_dependency(%q<sentry-raven>, [">= 0"])
+      s.add_dependency(%q<aws-sdk-ses>, [">= 0"])
+      s.add_dependency(%q<faraday>, [">= 0"])
       s.add_dependency(%q<bundler>, [">= 0"])
       s.add_dependency(%q<jeweler>, [">= 0"])
     end
@@ -86,7 +95,10 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<goliath>, [">= 0"])
     s.add_dependency(%q<sequel>, [">= 0"])
     s.add_dependency(%q<ffi-rzmq>, [">= 0"])
-    s.add_dependency(%q<satoshi-unit>, [">= 0"])
+    s.add_dependency(%q<crypto-unit>, [">= 0"])
+    s.add_dependency(%q<sentry-raven>, [">= 0"])
+    s.add_dependency(%q<aws-sdk-ses>, [">= 0"])
+    s.add_dependency(%q<faraday>, [">= 0"])
     s.add_dependency(%q<bundler>, [">= 0"])
     s.add_dependency(%q<jeweler>, [">= 0"])
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: txcatcher
 version: !ruby/object:Gem::Version
-  version: 0.1.85
+  version: 0.1.86
 platform: ruby
 authors:
 - Roman Snitko
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-03-03 00:00:00.000000000 Z
+date: 2018-03-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: goliath
@@ -80,6 +80,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-ses
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -112,9 +140,8 @@ description: Ccurrently, the only job of this gem is to collect all new Bitcoin/
   transactions, store them in a DB, index addresses.
 email: roman.snitko@gmail.com
 executables:
-- goliath.log
-- goliath_stdout.log
 - txcatcher
+- txcatcher-monitor
 extensions: []
 extra_rdoc_files:
 - LICENSE.txt
@@ -128,9 +155,8 @@ files:
 - README.md
 - Rakefile
 - VERSION
-- bin/goliath.log
-- bin/goliath_stdout.log
 - bin/txcatcher
+- bin/txcatcher-monitor
 - db/migrations/001_create_transactions.rb
 - db/migrations/002_create_addresses.rb
 - db/migrations/003_create_deposits.rb

data/bin/goliath.log

@@ -1,2 +0,0 @@
-[16007:INFO] 2017-09-23 23:23:59 :: Starting server on http://0.0.0.0:9498 in development mode. Watch out for stones.
-[16007:INFO] 2017-09-23 23:24:49 :: Stopping server...