zold 0.16.4 → 0.16.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6878147c3eac159d72548009bed74eda692af6b465f82ae47c2fd071de818512
-  data.tar.gz: 42401944fc5709a6fb6ea30de6f4af5ebc789538ca87ed204c0386792fbd3011
+  metadata.gz: c73944cf9e705895a711529a97a3ab56cd4561724c1b2009416db8355ae9a0e9
+  data.tar.gz: ece33aa66164e63afebc6f2383d4ad18997fa616abf0311b0956d62ec38f10f6
 SHA512:
-  metadata.gz: 97e5df269652087a17d29b3b603cc6b26992e189ce7005cb944ede9559884fdd8d0c1c60bb0463dd5a4dc267f1d6f57c9ad867864b98b679b8fce3431da4a6df
-  data.tar.gz: 52ac1e64157997658b5b4be0230960ca61099b4aabdbab7ba43faefd69d4b948a5a4b4a4e0d123c8a0e92915a362aeb63ee333a37a98e4f559f8916231aafacb
+  metadata.gz: 911f89497f640f092f3f7f2acfc192cb7154be4be191f5a75e4101045bd71432cb7235f31595a1c0290de9ce5848cc60e23e803b447f4a2c1e4e02cd449ae012
+  data.tar.gz: f98b5bb83459de3cd8c1576e9736a3f873a5a44255e66a0417307804dccb15702f72a9323b6defb43816e121c1388421d25106376cbff54e8fad563ef6dd06ed

data/lib/zold/cache.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+# Copyright (c) 2018 Yegor Bugayenko
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the 'Software'), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+# Cache.
+# Author:: Yegor Bugayenko (yegor256@gmail.com)
+# Copyright:: Copyright (c) 2018 Yegor Bugayenko
+# License:: MIT
+module Zold
+  # Cache
+  class Cache
+    def initialize
+      @hash = {}
+      @mutex = Mutex.new
+    end
+
+    def get(key, lifetime: 60 * 60)
+      @mutex.synchronize do
+        rec = @hash[key]
+        rec = nil if !rec.nil? && rec[:start] < Time.now - rec[:lifetime]
+        if rec.nil?
+          @hash[key] = {
+            value: yield,
+            start: Time.now,
+            lifetime: lifetime
+          }
+        end
+        @hash[key][:value]
+      end
+    end
+  end
+end

data/lib/zold/commands/node.rb

@@ -23,6 +23,7 @@
 require 'open3'
 require 'slop'
 require 'backtrace'
+require 'concurrent'
 require_relative '../version'
 require_relative '../age'
 require_relative '../score'
@@ -78,8 +79,8 @@ module Zold
           "The strength of the score (default: #{Score::STRENGTH})",
           default: Score::STRENGTH
         o.integer '--threads',
-          'How many threads to use for scores finding (default: 2)',
-          default: 2
+          "How many threads to use for scores finding (default: #{Concurrent.processor_count})",
+          default: Concurrent.processor_count
         o.bool '--dump-errors',
           'Make HTTP front-end errors visible in the log (false by default)',
           default: false
@@ -324,8 +325,12 @@ module Zold
         @log.info('Metronome hasn\'t been started because of --no-metronome')
         return metronome
       end
-      require_relative 'routines/spread'
-      metronome.add(Routines::Spread.new(opts, @wallets, @remotes, log: @log))
+      # This was a good idea to spread wallets among other nodes,
+      # but with the growing number of wallets it's obvious that this
+      # doesn't make a lot of sense. We better focus of HungryWallets
+      # implementation, where wallets are being pulled if they are missed.
+      # require_relative 'routines/spread'
+      # metronome.add(Routines::Spread.new(opts, @wallets, @remotes, log: @log))
       unless opts['standalone']
         require_relative 'routines/reconnect'
         metronome.add(Routines::Reconnect.new(opts, @remotes, farm, network: opts['network'], log: @log))

data/lib/zold/node/async_entrance.rb

@@ -74,7 +74,7 @@ module Zold
             loop do
               VerboseThread.new(@log).run(true) { take }
               break if @pool.shuttingdown?
-              sleep Random.rand(100) / 100
+              sleep(1 + Random.rand(100) / 100)
             end
           end
         end
@@ -119,6 +119,7 @@ in #{Age.new(start, limit: 0.05)}")
       opts = queue
       return if opts.empty?
       id = opts[0]
+      Thread.current.thread_variable_set(:wallet, id.to_s)
       body = Futex.new(file(id), log: @log).open do |f|
         b = File.exist?(f) ? IO.read(f) : ''
         FileUtils.rm_f(f)

data/lib/zold/node/farm.rb

@@ -24,6 +24,7 @@ require 'time'
 require 'open3'
 require 'backtrace'
 require 'futex'
+require 'json'
 require_relative '../log'
 require_relative '../score'
 require_relative '../age'
@@ -59,16 +60,24 @@ module Zold
     end
 
     def to_text
-      @threads.map do |t|
-        trace = t.backtrace || []
-        "#{t.name}: status=#{t.status}; alive=#{t.alive?};\n  #{trace.join("\n  ")}"
-      end.join("\n")
+      [
+        "Current time: #{Time.now.utc.iso8601}",
+        JSON.pretty_generate(to_json),
+        @threads.map do |t|
+          trace = t.backtrace || []
+          [
+            "#{t.name}: status=#{t.status}; alive=#{t.alive?}",
+            'Vars: ' + t.thread_variables.map { |v| "#{v}=\"#{t.thread_variable_get(v)}\"" }.join('; '),
+            "  #{trace.join("\n  ")}"
+          ].join("\n")
+        end
+      ].flatten.join("\n\n")
     end
 
     def to_json
       {
         threads: @threads.map do |t|
-          "#{t.name}/#{t.status}/#{t.alive? ? 'A' : 'D'}"
+          "#{t.name}/#{t.status}/#{t.alive? ? 'alive' : 'dead'}"
         end.join(', '),
         cleanup: @cleanup.status,
         pipeline: @pipeline.size,
@@ -177,6 +186,7 @@ module Zold
       return unless s.port == port
       return unless s.strength >= strength
       Thread.current.name = s.to_mnemo
+      Thread.current.thread_variable_set(:start, Time.now.utc.iso8601)
       score = @farmer.up(s)
       @log.debug("New score discovered: #{score}")
       save(threads, [score])

data/lib/zold/node/farmers.rb

@@ -51,6 +51,7 @@ module Zold
         bin = File.expand_path(File.join(File.dirname(__FILE__), '../../../bin/zold'))
         raise "Zold binary not found at #{bin}" unless File.exist?(bin)
         Open3.popen2e("ruby #{bin} --skip-upgrades --low-priority next \"#{score}\"") do |stdin, stdout, thr|
+          Thread.current.thread_variable_set(:pid, thr.pid.to_s)
           @log.debug("Scoring started in proc ##{thr.pid} \
 for #{score.value}/#{score.strength} at #{score.host}:#{score.port}")
           begin
@@ -73,6 +74,7 @@ for #{score.value}/#{score.strength} at #{score.host}:#{score.port}")
                 break
               end
               sleep 0.25
+              Thread.current.thread_variable_set(:buffer, buffer.length.to_s)
             end
             after = Score.parse(buffer.strip)
             @log.debug("Next score #{after.value}/#{after.strength} found in proc ##{thr.pid} \

data/lib/zold/node/front.rb

@@ -26,8 +26,6 @@ require 'eventmachine'
 require 'thin'
 require 'json'
 require 'sinatra/base'
-require 'cachy'
-require 'moneta'
 require 'get_process_mem'
 require 'diffy'
 require 'usagewatch_ext'
@@ -41,6 +39,7 @@ require_relative '../copies'
 require_relative '../log'
 require_relative '../id'
 require_relative '../http'
+require_relative '../cache'
 
 # The web front of the node.
 # Author:: Yegor Bugayenko (yegor256@gmail.com)
@@ -51,7 +50,6 @@ module Zold
   class Front < Sinatra::Base
     configure do
       Thread.current.name = 'sinatra'
-      Cachy.cache_store = Moneta.new(:Memory)
       set :bind, '0.0.0.0'
       set :suppress_messages, true
       set :start, Time.now
@@ -79,10 +77,12 @@ module Zold
       set :remotes, nil? # to be injected at node.rb
       set :copies, nil? # to be injected at node.rb
       set :node_alias, nil? # to be injected at node.rb
+      set :cache, Cache.new
     end
     use Rack::Deflater
 
     before do
+      Thread.current.thread_variable_set(:uri, request.url)
       @start = Time.now
       if !settings.halt.empty? && params[:halt] && params[:halt] == settings.halt
         settings.log.error('Halt signal received, shutting the front end down...')
@@ -189,10 +189,10 @@ in #{Age.new(@start, limit: 1)}")
         protocol: settings.protocol,
         score: score.to_h,
         pid: Process.pid,
-        cpus: Cachy.cache(:a_cpus) { Concurrent.processor_count },
+        cpus: settings.cache.get(:cpus) { Concurrent.processor_count },
         memory: GetProcessMem.new.bytes.to_i,
         platform: RUBY_PLATFORM,
-        load: Cachy.cache(:a_load, expires_in: 5 * 60) { Usagewatch.uw_load.to_f },
+        load: settings.cache.get(:load, lifetime: 5 * 60) { Usagewatch.uw_load.to_f },
         threads: "#{Thread.list.select { |t| t.status == 'run' }.count}/#{Thread.list.count}",
         wallets: total_wallets,
         remotes: settings.remotes.all.count,
@@ -390,6 +390,7 @@ in #{Age.new(@start, limit: 1)}")
         Thread.list.map do |t|
           [
             "#{t.name}: status=#{t.status}; alive=#{t.alive?}",
+            'Vars: ' + t.thread_variables.map { |v| "#{v}=\"#{t.thread_variable_get(v)}\"" }.join('; '),
             t.backtrace.nil? ? 'NO BACKTRACE' : "  #{t.backtrace.join("\n  ")}"
           ].join("\n")
         end
@@ -427,14 +428,20 @@ in #{Age.new(@start, limit: 1)}")
       yield header
     end
 
+    # @todo #513:30min This method is temporarily disabled since it
+    #  takes a lot of time (when the amount of wallets is big, like 40K). However,
+    #  we must find a way to count them somehow faster.
     def total_wallets
-      Cachy.cache(:a_wallets, expires_in: 5 * 60) { settings.wallets.all.count }
+      return 256 if settings.network == Wallet::MAIN_NETWORK
+      settings.wallets.all.count
     end
 
     def score
-      best = Cachy.cache(:a_score, expires_in: 60) { settings.farm.best }
-      raise 'Score is empty, there is something wrong with the Farm!' if best.empty?
-      best[0]
+      settings.cache.get(:score, lifetime: settings.network == Wallet::MAIN_NETWORK ? 60 : 1) do
+        b = settings.farm.best
+        raise 'Score is empty, there is something wrong with the Farm!' if b.empty?
+        b[0]
+      end
     end
 
     def copy_of(id)

data/lib/zold/node/spread_entrance.rb

@@ -69,6 +69,7 @@ module Zold
               if @remotes.all.empty?
                 @log.info("There are no remotes, won\'t spread #{id}")
               else
+                Thread.current.thread_variable_set(:wallet, id.to_s)
                 Push.new(wallets: @wallets, remotes: @remotes, log: @log).run(
                   ['push', "--ignore-node=#{@address}", id.to_s] +
                   (@ignore_score_weakeness ? ['--ignore-score-weakness'] : [])

data/lib/zold/version.rb

@@ -25,6 +25,6 @@
 # Copyright:: Copyright (c) 2018 Yegor Bugayenko
 # License:: MIT
 module Zold
-  VERSION = '0.16.4'
+  VERSION = '0.16.5'
   PROTOCOL = 2
 end

data/test/node/test_front.rb

@@ -109,6 +109,19 @@ class FrontTest < Minitest::Test
     end
   end
 
+  def test_increments_score
+    FakeNode.new(log: test_log).run(['--threads=1', '--strength=1', '--no-metronome']) do |port|
+      3.times do |i|
+        assert_equal_wait(true) do
+          response = Zold::Http.new(uri: "http://localhost:#{port}/", score: nil).get
+          assert_equal('200', response.code, response.body)
+          score = Zold::Score.parse_json(Zold::JsonPage.new(response.body).to_hash['score'])
+          score.value >= i
+        end
+      end
+    end
+  end
+
   def test_renders_wallet_pages
     FakeHome.new(log: test_log).run do |home|
       FakeNode.new(log: test_log).run(['--ignore-score-weakness', '--standalone']) do |port|

data/test/test__helper.rb

@@ -24,7 +24,6 @@ gem 'openssl'
 require 'openssl'
 require 'minitest/autorun'
 require 'concurrent'
-require 'moneta'
 
 STDOUT.sync = true
 

data/test/test_cache.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+# Copyright (c) 2018 Yegor Bugayenko
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the 'Software'), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+require 'minitest/autorun'
+require 'threads'
+require_relative '../lib/zold/cache'
+
+# Cache test.
+# Author:: Yegor Bugayenko (yegor256@gmail.com)
+# Copyright:: Copyright (c) 2018 Yegor Bugayenko
+# License:: MIT
+class TestCache < Minitest::Test
+  def test_caches
+    cache = Zold::Cache.new
+    first = cache.get(:hey, lifetime: 5) { Random.rand }
+    second = cache.get(:hey) { Random.rand }
+    assert(first == second)
+  end
+
+  def test_caches_and_expires
+    cache = Zold::Cache.new
+    first = cache.get(:hey, lifetime: 0.01) { Random.rand }
+    sleep 0.1
+    second = cache.get(:hey) { Random.rand }
+    assert(first != second)
+  end
+
+  def test_caches_in_threads
+    cache = Zold::Cache.new
+    Threads.new(10).assert(100) do
+      cache.get(:hey, lifetime: 0.0001) { Random.rand }
+    end
+  end
+end

data/zold.gemspec

@@ -55,7 +55,6 @@ and suggests a different architecture for digital wallet maintenance.'
   s.rdoc_options = ['--charset=UTF-8']
   s.extra_rdoc_files = ['README.md', 'LICENSE.txt']
   s.add_runtime_dependency 'backtrace', '~>0'
-  s.add_runtime_dependency 'cachy', '~>0'
   s.add_runtime_dependency 'concurrent-ruby', '~>1.0'
   s.add_runtime_dependency 'cucumber', '~>3' # has to stay here for Heroku
   s.add_runtime_dependency 'diffy', '~>3'
@@ -64,7 +63,6 @@ and suggests a different architecture for digital wallet maintenance.'
   s.add_runtime_dependency 'futex', '~>0'
   s.add_runtime_dependency 'get_process_mem', '~>0'
   s.add_runtime_dependency 'json', '~>2'
-  s.add_runtime_dependency 'moneta', '~>1'
   s.add_runtime_dependency 'openssl', '~>2'
   s.add_runtime_dependency 'posix-spawn', '~>0.3'
   s.add_runtime_dependency 'rainbow', '~>3'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: zold
 version: !ruby/object:Gem::Version
-  version: 0.16.4
+  version: 0.16.5
 platform: ruby
 authors:
 - Yegor Bugayenko
@@ -24,20 +24,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: cachy
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
   requirement: !ruby/object:Gem::Requirement
@@ -150,20 +136,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2'
-- !ruby/object:Gem::Dependency
-  name: moneta
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1'
 - !ruby/object:Gem::Dependency
   name: openssl
   requirement: !ruby/object:Gem::Requirement
@@ -544,6 +516,7 @@ files:
 - lib/zold.rb
 - lib/zold/age.rb
 - lib/zold/amount.rb
+- lib/zold/cache.rb
 - lib/zold/cached_wallets.rb
 - lib/zold/commands/alias.rb
 - lib/zold/commands/args.rb
@@ -646,6 +619,7 @@ files:
 - test/test__helper.rb
 - test/test_age.rb
 - test/test_amount.rb
+- test/test_cache.rb
 - test/test_cached_wallets.rb
 - test/test_copies.rb
 - test/test_dir_items.rb
@@ -746,6 +720,7 @@ test_files:
 - test/test__helper.rb
 - test/test_age.rb
 - test/test_amount.rb
+- test/test_cache.rb
 - test/test_cached_wallets.rb
 - test/test_copies.rb
 - test/test_dir_items.rb