fluent-plugin-claymore 1.0.0

data/example/fluentd.conf

@@ -0,0 +1,13 @@
+<source>
+  @type tail
+  path *.txt
+  read_from_head true
+  tag claymore.data
+  <parse>
+    @type claymore
+  </parse>
+</source>
+
+<match **>
+  @type stdout
+</match>

data/fluent-plugin-claymore.gemspec

@@ -0,0 +1,29 @@
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name    = 'fluent-plugin-claymore'
+  spec.version = '1.0.0'
+  spec.authors = ['Timothy Stott']
+  spec.email   = ['stott.timothy@gmail.com']
+
+  spec.summary       = 'Fluentd parser plugin for Claymore Dual Miner logs'
+  spec.description   = 'Extract time series metrics from Claymore Dual Miner logs'
+  spec.homepage      = 'https://github.com/timstott/fluent-plugin-claymore'
+  spec.license       = 'Apache-2.0'
+
+  test_files, files  = `git ls-files -z`.split("\x0").partition do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.files         = files
+  spec.executables   = files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = test_files
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler', '~> 1.14'
+  spec.add_development_dependency 'rake', '~> 12.0'
+  spec.add_development_dependency 'rubocop', '~> 0.50.0'
+  spec.add_development_dependency 'test-unit', '~> 3.0'
+  spec.add_development_dependency 'timecop', '~> 0.9'
+  spec.add_runtime_dependency 'fluentd', ['>= 0.14.10', '< 2']
+end

data/lib/claymore/asset_symbol.rb

@@ -0,0 +1,11 @@
+module Claymore
+  # Provides method to extract asset symbole
+  module AssetSymbol
+    ASSET_REGEXP = /(?<asset>DRC|ETH|LBC|PASC|SC)(:| -)/
+
+    def asset_symbol(line)
+      match = line.match(ASSET_REGEXP)
+      match && match[:asset]
+    end
+  end
+end

data/lib/claymore/gpu_hash_rate.rb

@@ -0,0 +1,55 @@
+require 'claymore/asset_symbol'
+
+module Claymore
+  # Extracts asset, gpu index and gpu hash rate
+  # Sets hash rate to -1 when gpu is off
+  #
+  # Example input:
+  # 05:45:16:028 2100 ETH: GPU0 29.586 Mh/s, GPU1 off
+  #
+  # Example output:
+  # [
+  #   { 'asset' => 'ETH', 'gpu' => 0, 'hash_rate' => 29.586, 'type' => 'GPU_HASH_RATE' },
+  #   { 'asset' => 'ETH', 'gpu' => 1, 'hash_rate' => -1.0, 'type' => 'GPU_HASH_RATE' }
+  # ]
+  class GPUHashRate
+    include AssetSymbol
+
+    RATES_REGEXP = %r{GPU(?<index>\d+) (?<rate>\d+(?:\.\d+)? Mh\/s|off)}
+    LINE_REGEXP = Regexp.new("#{ASSET_REGEXP.source} #{RATES_REGEXP.source}")
+
+    def self.call(line)
+      new(line).call
+    end
+
+    attr_reader :line
+
+    def initialize(line)
+      @line = line
+    end
+
+    # rubocop:disable Metrics/MethodLength
+    def call
+      (match = LINE_REGEXP.match(line)) || return
+
+      raw_rates.each_with_object([]) do |(raw_index, raw_rate), acc|
+        hash_rate = raw_rate == 'off' ? -1.0 : raw_rate.to_f.round(3)
+        index = raw_index.to_i
+
+        acc << {
+          'type' => 'GPU_HASH_RATE',
+          'asset' => match[:asset],
+          'gpu' => index,
+          'hash_rate' => hash_rate
+        }
+      end
+    end
+    # rubocop:enable Metrics/MethodLength
+
+    private
+
+    def raw_rates
+      line.scan(RATES_REGEXP)
+    end
+  end
+end

data/lib/claymore/total_hash_rate.rb

@@ -0,0 +1,37 @@
+require 'claymore/asset_symbol'
+
+module Claymore
+  # Extracts total hash rate with asset symbol
+  #
+  # Example input:
+  # 05:45:16:028 2100 ETH - Total Speed: 90.118 Mh/s, Total Shares: 237, Rejected: 0, Time: 06:50
+  #
+  # Example output:
+  # { 'asset' => 'ETH', 'hash_rate' => 90.118, 'type' => 'TOTAL_HASH_RATE' }
+  class TotalHashRate
+    include AssetSymbol
+
+    TOTAL_RATE_REGEXP = %r{Total Speed: (?<rate>\d+\.\d+ Mh\/s)}
+    LINE_REGEXP = Regexp.new("#{ASSET_REGEXP.source} #{TOTAL_RATE_REGEXP.source}")
+
+    def self.call(line)
+      new(line).call
+    end
+
+    attr_reader :line
+
+    def initialize(line)
+      @line = line
+    end
+
+    def call
+      (match = LINE_REGEXP.match(line)) || return
+
+      {
+        'asset' => match[:asset],
+        'hash_rate' => match[:rate].to_f.round(3),
+        'type' => 'TOTAL_HASH_RATE'
+      }
+    end
+  end
+end

data/lib/fluent/plugin/parser_claymore.rb

@@ -0,0 +1,70 @@
+require 'fluent/plugin/parser'
+require 'claymore/gpu_hash_rate'
+require 'claymore/total_hash_rate'
+
+module Fluent
+  module Plugin
+    class ClaymoreParser < Fluent::Plugin::Parser
+      Fluent::Plugin.register_parser('claymore', self)
+
+      # Extract gpu index, temperature, old and new fan speed
+      #
+      # Example:
+      # 09:27:02:820 1834 GPU 4 temp = 45, old fan speed = 0, new fan speed = 75
+      # { 'gpu' => 4, 'old_fan' => 0, 'new_fan' => 75, 'temperature' => 45 }
+      GPU_TEMP = lambda do |text|
+        match = text.match(/GPU (?<gpu>\d) temp = (?<temperature>\d+), old.+= (?<old_fan>\d+), new.+= (?<new_fan>\d+)/)
+        match.names.zip(match.captures).map { |(k, v)| [k, v.to_i] }.push(%w[type GPU_TEMP]).to_h if match
+      end
+
+      # Extract gpu share found
+      #
+      # Example:
+      # 11:04:02:920 234c ETH: 12/17/17-11:04:02 - SHARE FOUND - (GPU 5)
+      # { 'asset' => 'ETH', 'gpu' => 5, 'share_found' => 1 }
+      GPU_SHARE_FOUND = lambda do |text|
+        match = text.match(/(?<asset>[A-Z]{2,}):.+SHARE FOUND.+\(GPU (?<gpu>\d+)/)
+        { 'type' => 'GPU_SHARE_FOUND', 'asset' => match[:asset], 'gpu' => match[:gpu].to_i, 'count' => 1 } if match
+      end
+
+      # Extract connection lost
+      #
+      # Example:
+      # 20:15:08:451 2338 ETH: Connection lost, retry in 20 sec..
+      # { 'asset' => 'ETH', 'connection_lost' => 1 }
+      CONNECTION_LOST = lambda do |text|
+        match = text.match(/(?<asset>[A-Z]{2,}):.+Connection lost/)
+        { 'type' => 'CONNECTION_LOST', 'asset' => match[:asset], 'count' => 1 } if match
+      end
+
+      INCORRECT_SHARE = lambda do |text|
+        match = text.match(/GPU #(?<gpu>\d+) got incorrect share/)
+        { 'type' => 'INCORRECT_SHARE', 'gpu' => match[:gpu].to_i, 'count' => 1 } if match
+      end
+
+      EXTRACTORS = [
+        CONNECTION_LOST,
+        Claymore::GPUHashRate,
+        Claymore::TotalHashRate,
+        GPU_SHARE_FOUND,
+        GPU_TEMP,
+        INCORRECT_SHARE
+      ].freeze
+
+      def parse(text)
+        EXTRACTORS
+          .map { |extractor| extractor.call(text) }
+          .reject { |result| result.nil? || result.empty? }
+          .compact
+          .flatten
+          .each { |result| yield time, result }
+      end
+
+      # Claymore filename include the date however,
+      # Claymore log line include time without date
+      def time
+        parse_time(nil)
+      end
+    end
+  end
+end

data/test/claymore/test_gpu_hash_rate.rb

@@ -0,0 +1,43 @@
+require 'helper'
+require 'claymore/gpu_hash_rate.rb'
+
+class GPUHashRateTest < Test::Unit::TestCase
+  test 'return nil when no match' do
+    assert_nil service('ETH: GPU0')
+    assert_nil service('GPU0 24.1 Mh/s')
+    assert_nil service('SC - Total Speed: 292.684 Mh/s')
+  end
+
+  test 'extracts individual GPU hash rates when available' do
+    text = 'ETH: GPU0 24.314 Mh/s, GPU1 24.01 Mh/s, GPU20 24.1 Mh/s'
+    assert_equal service(text), [
+      { 'asset' => 'ETH', 'gpu' => 0, 'hash_rate' => 24.314, 'type' => 'GPU_HASH_RATE' },
+      { 'asset' => 'ETH', 'gpu' => 1, 'hash_rate' => 24.01, 'type' => 'GPU_HASH_RATE' },
+      { 'asset' => 'ETH', 'gpu' => 20, 'hash_rate' => 24.1, 'type' => 'GPU_HASH_RATE' }
+    ]
+  end
+
+  test 'extracts asset name' do
+    text = 'SC: GPU0 292.862 Mh/s'
+    assert_equal service(text), [
+      { 'asset' => 'SC', 'gpu' => 0, 'hash_rate' => 292.862, 'type' => 'GPU_HASH_RATE' }
+    ]
+  end
+
+  test 'assigns -1 hash rate value when GPU is off' do
+    text = 'ETH: GPU0 29 Mh/s, GPU1 off'
+    assert_equal service(text), [
+      { 'asset' => 'ETH', 'gpu' => 0, 'hash_rate' => 29, 'type' => 'GPU_HASH_RATE' },
+      { 'asset' => 'ETH', 'gpu' => 1, 'hash_rate' => -1.0, 'type' => 'GPU_HASH_RATE' }
+    ]
+
+    text = 'LBC: GPU0 off'
+    assert_equal service(text), [
+      { 'asset' => 'LBC', 'gpu' => 0, 'hash_rate' => -1.0, 'type' => 'GPU_HASH_RATE' }
+    ]
+  end
+
+  def service(text)
+    Claymore::GPUHashRate.call(text)
+  end
+end

data/test/claymore/test_total_hash_rate.rb

@@ -0,0 +1,23 @@
+require 'helper'
+require 'claymore/total_hash_rate.rb'
+
+class TotalHashRateTest < Test::Unit::TestCase
+  test 'return nil when no match' do
+    assert_nil service('ETH: job is the same')
+    assert_nil service('ETH: checking pool connection...')
+    assert_nil service('ETH: GPU0 29 Mh/s, GPU1 off')
+  end
+
+  test 'extracts asset name and total hash rate' do
+    line = 'ETH - Total Speed: 90.118 Mh/s, Total Shares: 237, Rejected: 0, Time: 06:50'
+    assert_equal ({
+      'asset' => 'ETH',
+      'hash_rate' => 90.118,
+      'type' => 'TOTAL_HASH_RATE'
+    }), service(line)
+  end
+
+  def service(line)
+    Claymore::TotalHashRate.call(line)
+  end
+end

data/test/helper.rb

@@ -0,0 +1,9 @@
+$LOAD_PATH.unshift(File.expand_path('../../', __FILE__))
+require 'test-unit'
+require 'timecop'
+require 'fluent/test'
+require 'fluent/test/driver/parser'
+require 'fluent/test/helpers'
+
+Test::Unit::TestCase.include(Fluent::Test::Helpers)
+Test::Unit::TestCase.extend(Fluent::Test::Helpers)

data/test/plugin/test_parser_claymore.rb

@@ -0,0 +1,82 @@
+require 'helper'
+require 'fluent/plugin/parser_claymore.rb'
+
+class ClaymoreParserTest < Test::Unit::TestCase
+  setup do
+    Fluent::Test.setup
+  end
+
+  test 'uses current Time instead of file/line time' do
+    t1 = '2017-12-20T07:30:05.123+00:00'
+    Timecop.freeze(t1) do
+      parse('') do |time, _record|
+        assert_equal time, Time.parse(t1)
+      end
+    end
+  end
+
+  test 'returns nothing when no match' do
+    parse('blahblah') do |_time, record|
+      assert_nil record
+    end
+  end
+
+  test 'extracts gpu hash rates' do
+    line = '17:22:59:067 25e8 ETH: GPU0 24.314 Mh/s, GPU1 24.01 Mh/s, GPU2 24.1 Mh/s'
+    records = []
+    parse(line) { |_time, record| records.push record }
+    assert_equal records, [
+      { 'asset' => 'ETH', 'gpu' => 0, 'hash_rate' => 24.314, 'type' => 'GPU_HASH_RATE' },
+      { 'asset' => 'ETH', 'gpu' => 1, 'hash_rate' => 24.01, 'type' => 'GPU_HASH_RATE' },
+      { 'asset' => 'ETH', 'gpu' => 2, 'hash_rate' => 24.1, 'type' => 'GPU_HASH_RATE' }
+    ]
+  end
+
+  test 'extracts gpu hash rates with off gpus' do
+    line = '1559:05:45:16:028 2100 ETH: GPU0 29.586 Mh/s, GPU1 off'
+    records = []
+    parse(line) { |_time, record| records.push record }
+    assert_equal records, [
+      { 'asset' => 'ETH', 'gpu' => 0, 'hash_rate' => 29.586, 'type' => 'GPU_HASH_RATE' },
+      { 'asset' => 'ETH', 'gpu' => 1, 'hash_rate' => -1, 'type' => 'GPU_HASH_RATE' }
+    ]
+  end
+
+  test 'extracts gpu temperature, old and new fan speed' do
+    line = '09:27:02:820 1834 GPU 4 temp = 45, old fan speed = 0, new fan speed = 75'
+    parse(line) do |_time, record|
+      assert_equal record, 'gpu' => 4, 'old_fan' => 0, 'new_fan' => 75, 'temperature' => 45, 'type' => 'GPU_TEMP'
+    end
+  end
+
+  test 'extracts gpu share found' do
+    line = '11:04:02:920 234c ETH: 12/17/17-11:04:02 - SHARE FOUND - (GPU 5)'
+    parse(line) do |_time, record|
+      assert_equal record, 'asset' => 'ETH', 'gpu' => 5, 'count' => 1, 'type' => 'GPU_SHARE_FOUND'
+    end
+  end
+
+  test 'extracts connection lost' do
+    line = '20:15:08:451 2338 ETH: Connection lost, retry in 20 sec...'
+    parse(line) do |_time, record|
+      assert_equal record, 'asset' => 'ETH', 'count' => 1, 'type' => 'CONNECTION_LOST'
+    end
+  end
+
+  test 'extracts incorrect share' do
+    line = '05:07:23:959 acc GPU #3 got incorrect share. If you see this warning often, make sure you did not overclock it too much!'
+    parse(line) do |_time, record|
+      assert_equal record, 'gpu' => 3, 'count' => 1, 'type' => 'INCORRECT_SHARE'
+    end
+  end
+
+  private
+
+  def parse(txt, &block)
+    create_driver({}).instance.parse(txt, &block)
+  end
+
+  def create_driver(conf)
+    Fluent::Test::Driver::Parser.new(Fluent::Plugin::ClaymoreParser).configure(conf)
+  end
+end

metadata

@@ -0,0 +1,157 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-claymore
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Timothy Stott
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-01-25 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.50.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.50.0
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.10
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.10
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+description: Extract time series metrics from Claymore Dual Miner logs
+email:
+- stott.timothy@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".rubocop.yml"
+- ".rubocop_todo.yml"
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- Rakefile
+- circle.yml
+- example/1516312455_log.txt
+- example/fluentd.conf
+- fluent-plugin-claymore.gemspec
+- lib/claymore/asset_symbol.rb
+- lib/claymore/gpu_hash_rate.rb
+- lib/claymore/total_hash_rate.rb
+- lib/fluent/plugin/parser_claymore.rb
+- test/claymore/test_gpu_hash_rate.rb
+- test/claymore/test_total_hash_rate.rb
+- test/helper.rb
+- test/plugin/test_parser_claymore.rb
+homepage: https://github.com/timstott/fluent-plugin-claymore
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.14
+signing_key: 
+specification_version: 4
+summary: Fluentd parser plugin for Claymore Dual Miner logs
+test_files:
+- test/claymore/test_gpu_hash_rate.rb
+- test/claymore/test_total_hash_rate.rb
+- test/helper.rb
+- test/plugin/test_parser_claymore.rb