zold 0.18.8 → 0.18.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 253fd71080a2529f04a374fa90311a1035ed907dbe6084a61f6d3c0f6f834d77
-  data.tar.gz: '0857238f9834a99cc2ef85f5e6f1d547cb8013f4c5e5697ddd9ae7d4366fea34'
+  metadata.gz: 6ad955cd4ed1c34ff735ca6684945186c29c07bf970f4bdc130ffdc0b3164922
+  data.tar.gz: 9344af265b58567d3aa2afda28815c6efaaddf0235a379d85f1ae69433112cd0
 SHA512:
-  metadata.gz: 1976dcc0a25d82d2722495ce617670fe6bae75e19890a087a8b1fbff209d64e312da3ddbf6b3c20e5d8148b4055e80c4ba705d9bc546fde5ee535e88b4437043
-  data.tar.gz: 516f380178059ba3bc7d09319b7bf6451113008c09218d99fb7bf1129084507cc0a8bdba6d9368c50ced632fff4df716d4ae441fbdd08c9704ad219cf0b20d21
+  metadata.gz: fba208cc63dc399eb24dd6c015f6eada0e22c9a3ca2b22d6a5612576b6ea264f2e3bc6cb57c0a69618694544928faa5f919d779c6e29d7da5137d40325e9b62e
+  data.tar.gz: bbfd1eec62d4f29272c21273aef71d9d110bbe491237b264c18f571e687d724b606861141e2efacadc3551fafe7917b2d896ed88eccc39b80f86066e226d81ee

data/bin/zold

@@ -36,6 +36,7 @@ require_relative '../lib/zold/wallets'
 require_relative '../lib/zold/tree_wallets'
 require_relative '../lib/zold/sync_wallets'
 require_relative '../lib/zold/cached_wallets'
+require_relative '../lib/zold/hungry_wallets'
 require_relative '../lib/zold/log'
 require_relative '../lib/zold/key'
 require_relative '../lib/zold/age'

data/fixtures/scripts/distribute-wallet.sh

@@ -4,7 +4,7 @@ function start_node {
   port=$(reserve_port)
   mkdir ${port}
   cd ${port}
-  zold node --trace --invoice=DISTRWALLET@ffffffffffffffff \
+  zold node --trace --invoice=DISTRWALLET@ffffffffffffffff --tolerate-edges \
     --host=127.0.0.1 --port=${port} --bind-port=${port} \
     --threads=0 --routine-immediately --never-reboot > log.txt 2>&1 &
   pid=$!
@@ -36,7 +36,7 @@ zold --public-key=id_rsa.pub create 0000000000000000
 zold pay --private-key=id_rsa 0000000000000000 NOPREFIX@aaaabbbbccccdddd 4.95 'For the book'
 zold remote clean
 zold remote add 127.0.0.1 ${first}
-zold push 0000000000000000
+zold push 0000000000000000 --tolerate-edges
 zold remote clean
 zold remote add 127.0.0.1 ${second}
 
@@ -45,7 +45,7 @@ zold remote add 127.0.0.1 ${second}
 # delay between them, in order to give the first node a chance to distribute
 # the wallet.
 i=0
-until zold fetch 0000000000000000 --ignore-score-weakness; do
+until zold fetch 0000000000000000 --ignore-score-weakness --tolerate-edges; do
   echo 'Failed to fetch, let us try again'
   ((i++)) || sleep 0
   if ((i==5)); then
@@ -73,7 +73,7 @@ fi
 # have the wallet very soon.
 rm -f ${second}/**/*.z
 i=0
-until zold fetch 0000000000000000 --ignore-score-weakness; do
+until zold fetch 0000000000000000 --ignore-score-weakness --tolerate-edges; do
   echo 'Failed to fetch, let us try again'
   ((i++)) || sleep 0
   if ((i==5)); then

data/fixtures/scripts/pull-on-start.sh

@@ -3,7 +3,8 @@
 port=$(reserve_port)
 mkdir server
 cd server
-zold node --trace --invoice=PULLONSTART@ffffffffffffffff --no-metronome \
+zold remote clean
+zold node --trace --invoice=PULLONSTART@ffffffffffffffff --no-metronome --tolerate-edges \
   --host=127.0.0.1 --port=${port} --bind-port=${port} \
   --threads=0 --standalone --pretty=full 2>&1 &
 cd ..
@@ -14,16 +15,16 @@ zold remote clean
 zold remote add 127.0.0.1 ${port}
 
 zold --public-key=id_rsa.pub create abcdabcdabcdabcd
-zold push abcdabcdabcdabcd
+zold push abcdabcdabcdabcd --tolerate-edges
 zold remove abcdabcdabcdabcd
-zold invoice abcdabcdabcdabcd
+zold invoice abcdabcdabcdabcd --tolerate-edges
 
 second_port=$(reserve_port)
 mkdir second
 cd second
 zold remote clean
 zold remote add 127.0.0.1 ${port}
-zold node --trace --invoice=abcdabcdabcdabcd --no-metronome \
+zold node --trace --invoice=abcdabcdabcdabcd --no-metronome --tolerate-edges \
   --host=127.0.0.1 --port=${second_port} --bind-port=${second_port} \
   --threads=0 &
 

data/fixtures/scripts/push-and-pull.sh

@@ -4,7 +4,7 @@ port=$(reserve_port)
 
 mkdir server
 cd server
-zold node --trace --invoice=PUSHNPULL@ffffffffffffffff \
+zold node --trace --invoice=PUSHNPULL@ffffffffffffffff --tolerate-edges \
   --host=127.0.0.1 --port=${port} --bind-port=${port} \
   --threads=0 --standalone 2>&1 &
 pid=$!
@@ -29,13 +29,13 @@ zold show 0000000000000000
 zold taxes debt 0000000000000000
 
 zold remote show
-zold push
-zold push 0000000000000000
-until zold fetch 0000000000000000 --ignore-score-weakness; do
+zold push --tolerate-edges
+zold push 0000000000000000 --tolerate-edges
+until zold fetch 0000000000000000 --ignore-score-weakness --tolerate-edges; do
   echo 'Failed to fetch, let us try again'
   sleep 1
 done
-zold fetch
+zold fetch --tolerate-edges
 zold diff 0000000000000000
 zold merge
 zold merge 0000000000000000

data/fixtures/scripts/redeploy-on-upgrade.sh

@@ -5,7 +5,7 @@ function start_node {
   cd $1
   zold remote clean
   zold node $3 --nohup --nohup-command='touch restarted' --nohup-log=log --nohup-max-cycles=0 --nohup-log-truncate=10240 \
-    --expose-version=$2 --save-pid=pid --routine-immediately \
+    --expose-version=$2 --save-pid=pid --routine-immediately --tolerate-edges \
     --verbose --trace --invoice=REDEPLOY@ffffffffffffffff \
     --host=127.0.0.1 --port=$1 --bind-port=$1 --threads=1 --strength=20 > /dev/null 2>&1
   wait_for_port $1

data/fixtures/scripts/spread-wallets.sh

@@ -4,7 +4,8 @@ function start_node {
   port=$(reserve_port)
   mkdir ${port}
   cd ${port}
-  zold node --trace --invoice=SPREADWALLETS@ffffffffffffffff \
+  zold remote clean
+  zold node --trace --invoice=SPREADWALLETS@ffffffffffffffff --tolerate-edges \
     --host=127.0.0.1 --port=${port} --bind-port=${port} \
     --threads=0 > log.txt 2>&1 &
   pid=$!
@@ -18,20 +19,19 @@ first=$(start_node)
 second=$(start_node)
 trap "halt_nodes ${first} ${second}" EXIT
 
-zold --home=${first} remote clean
 zold --home=${first} remote add 127.0.0.1 ${second}
-zold --home=${second} remote clean
 zold --home=${second} remote add 127.0.0.1 ${first}
 
 zold --public-key=id_rsa.pub create 0000000000000000
 zold pay --private-key=id_rsa 0000000000000000 NOPREFIX@aaaabbbbccccdddd 4.95 'To help you, dude!'
+zold remote clean
 zold remote add 127.0.0.1 ${first}
-zold push 0000000000000000
+zold push 0000000000000000 --tolerate-edges
 zold remote clean
 zold remote add 127.0.0.1 ${second}
 
 i=0
-until zold fetch 0000000000000000 --ignore-score-weakness; do
+until zold fetch 0000000000000000 --ignore-score-weakness --tolerate-edges; do
   echo 'Failed to fetch, let us try again'
   ((i++)) || sleep 0
   if ((i==5)); then

data/lib/zold/cached_wallets.rb

@@ -21,6 +21,7 @@
 # SOFTWARE.
 
 require 'zache'
+require 'delegate'
 require_relative 'endless'
 require_relative 'thread_pool'
 
@@ -30,7 +31,7 @@ require_relative 'thread_pool'
 # License:: MIT
 module Zold
   # Collection of local wallets
-  class CachedWallets
+  class CachedWallets < SimpleDelegator
     def initialize(wallets)
       @wallets = wallets
       @zache = Zache.new
@@ -41,22 +42,7 @@ module Zold
           @zache.clean
         end
       end
-    end
-
-    def to_s
-      @wallets.to_s
-    end
-
-    def path
-      @wallets.path
-    end
-
-    def all
-      @wallets.all
-    end
-
-    def count
-      @wallets.count
+      super(wallets)
     end
 
     def acq(id, exclusive: false)

data/lib/zold/commands/fetch.rb

@@ -66,6 +66,9 @@ Available options:"
         o.array '--ignore-node',
           'Ignore this node and don\'t fetch from it',
           default: []
+        o.bool '--tolerate-edges',
+          'Don\'t fail if only "edge" (not default ones) nodes accepted the wallet',
+          default: false
         o.bool '--quiet-if-absent',
           'Don\'t fail if the wallet is absent in all remote nodes',
           default: false
@@ -91,15 +94,22 @@ Available options:"
       total = Concurrent::AtomicFixnum.new
       nodes = Concurrent::AtomicFixnum.new
       done = Concurrent::AtomicFixnum.new
+      defaults = Concurrent::AtomicFixnum.new
       @remotes.iterate(@log) do |r|
         nodes.increment
         total.increment(fetch_one(id, r, cps, opts))
+        defaults.increment if r.default?
         done.increment
       end
       raise "There are no remote nodes, run 'zold remote reset'" if nodes.value.zero?
-      raise "No nodes out of #{nodes.value} have the wallet #{id}" if done.value.zero? && !opts['quiet-if-absent']
+      unless opts['quiet-if-absent']
+        raise "No nodes out of #{nodes.value} have the wallet #{id}" if done.value.zero?
+        if defaults.value.zero? && !opts['tolerate-edges']
+          raise "There are only edge nodes, run 'zold remote reset' or use --tolerate-edges"
+        end
+      end
       @log.info("#{done.value} copies of #{id} fetched in #{Age.new(start)} with the total score of \
-#{total.value} from #{nodes.value} nodes")
+#{total.value} from #{nodes.value} nodes (#{defaults.value} defaults)")
       list = cps.all.map do |c|
         "  ##{c[:name]}: #{c[:score]} #{Wallet.new(c[:path]).mnemo} \
 #{Size.new(File.size(c[:path]))}/#{Age.new(File.mtime(c[:path]))}"

data/lib/zold/commands/invoice.rb

@@ -51,6 +51,9 @@ Available options:"
         o.integer '--length',
           'The length of the invoice prefix (default: 8)',
           default: 8
+        o.bool '--tolerate-edges',
+          'Don\'t fail if only "edge" (not default ones) nodes have the wallet',
+          default: false
         o.string '--network',
           'The name of the network we work in',
           default: 'test'
@@ -67,7 +70,8 @@ Available options:"
       unless @wallets.acq(id, &:exists?)
         require_relative 'pull'
         Pull.new(wallets: @wallets, remotes: @remotes, copies: @copies, log: @log).run(
-          ['pull', id.to_s, "--network=#{opts['network']}"]
+          ['pull', id.to_s, "--network=#{opts['network']}"] +
+          (opts['tolerate-edges'] ? ['--tolerate-edges'] : [])
         )
       end
       inv = @wallets.acq(id) do |wallet|

data/lib/zold/commands/merge.rb

@@ -78,6 +78,12 @@ Available options:"
       cps = cps.all.sort_by { |c| c[:score] }.reverse
       patch = Patch.new(@wallets, log: @log)
       score = 0
+      cps.each_with_index do |c, idx|
+        wallet = Wallet.new(c[:path])
+        name = "#{c[:name]}/#{idx}/#{c[:score]}"
+        merge_one(opts, patch, wallet, name)
+        score += c[:score]
+      end
       @wallets.acq(id) do |w|
         if w.exists?
           merge_one(opts, patch, w, 'localhost')
@@ -86,12 +92,6 @@ Available options:"
           @log.debug("Local copy of #{id} is absent, nothing to merge")
         end
       end
-      cps.each_with_index do |c, idx|
-        wallet = Wallet.new(c[:path])
-        name = "#{c[:name]}/#{idx}/#{c[:score]}"
-        merge_one(opts, patch, wallet, name)
-        score += c[:score]
-      end
       modified = @wallets.acq(id, exclusive: true) { |w| patch.save(w.path, overwrite: true) }
       if modified
         @log.info("#{cps.count} copies with the total score of #{score} successfully merged \

data/lib/zold/commands/node.rb

@@ -30,8 +30,10 @@ require_relative 'thread_badge'
 require_relative '../version'
 require_relative '../age'
 require_relative '../metronome'
+require_relative '../thread_pool'
 require_relative '../wallet'
 require_relative '../wallets'
+require_relative '../hungry_wallets'
 require_relative '../remotes'
 require_relative '../verbose_thread'
 require_relative '../node/farmers'
@@ -95,6 +97,9 @@ module Zold
         o.bool '--ignore-score-weakness',
           'Ignore score weakness of incoming requests and register those nodes anyway',
           default: false
+        o.bool '--tolerate-edges',
+          'Don\'t fail if only "edge" (not default ones) nodes accepted/have the wallet',
+          default: false
         o.boolean '--nohup',
           'Run it in background, rebooting when a higher version is available in the network',
           default: false
@@ -134,8 +139,8 @@ module Zold
           'Maximum amount of memory we can consume, quit if we take more than that, in Mb (default: 512)',
           default: 512
         o.integer '--queue-limit',
-          'The maximum number of wallets to be accepted via PUSH and stored in the queue (default: 4096)',
-          default: 4096
+          'The maximum number of wallets to be accepted via PUSH and stored in the queue (default: 256)',
+          default: 256
         o.integer '--gc-age',
           'Maximum time in seconds to keep an empty and unused wallet on the disk',
           default: 60 * 60 * 24 * 10
@@ -213,8 +218,10 @@ module Zold
         Zold::Remote.new(remotes: @remotes).run(['remote', 'remove', host, port.to_s])
         @log.info("Removed current node (#{address}) from list of remotes")
       end
+      hungry = Zold::ThreadPool.new('hungry', log: @log)
+      wts = Zold::HungryWallets.new(@wallets, @remotes, @copies, hungry, log: @log, network: opts['network'])
       Front.set(:zache, Zache.new)
-      Front.set(:wallets, @wallets)
+      Front.set(:wallets, wts)
       Front.set(:remotes, @remotes)
       Front.set(:copies, @copies)
       Front.set(:address, address)
@@ -222,18 +229,7 @@ module Zold
       Front.set(:opts, opts)
       Front.set(:dump_errors, opts['dump-errors'])
       Front.set(:port, opts['bind-port'])
-      node_alias = opts[:alias] || address
-      unless node_alias.eql?(address) || node_alias =~ /^[A-Za-z0-9]{4,16}$/
-        raise "Alias should be a 4 to 16 char long alphanumeric string: #{node_alias}"
-      end
-      Front.set(:node_alias, node_alias)
-      invoice = opts[:invoice]
-      unless invoice.include?('@')
-        require_relative 'invoice'
-        invoice = Invoice.new(
-          wallets: @wallets, remotes: @remotes, copies: @copies, log: @log
-        ).run(['invoice', invoice, "--network=#{opts['network']}"])
-      end
+      Front.set(:node_alias, node_alias(opts, address))
       entrance = SafeEntrance.new(
         NoSpamEntrance.new(
           NoDupEntrance.new(
@@ -241,22 +237,22 @@ module Zold
               SpreadEntrance.new(
                 SyncEntrance.new(
                   Entrance.new(
-                    @wallets,
-                    @remotes, @copies, address,
+                    wts, @remotes, @copies, address,
                     log: @log, network: opts['network']
                   ),
                   File.join(home, '.zoldata/sync-entrance'),
                   log: @log
                 ),
-                @wallets, @remotes, address,
+                wts, @remotes, address,
                 log: @log,
-                ignore_score_weakeness: opts['ignore-score-weakness']
+                ignore_score_weakeness: opts['ignore-score-weakness'],
+                tolerate_edges: opts['tolerate-edges']
               ),
               File.join(home, '.zoldata/async-entrance'),
               log: @log,
               queue_limit: opts['queue-limit']
             ),
-            @wallets,
+            wts,
             log: @log
           ),
           period: opts['allow-spam'] ? 0 : 60 * 60,
@@ -266,7 +262,10 @@ module Zold
       )
       entrance.start do |ent|
         Front.set(:entrance, ent)
-        farm = Farm.new(invoice, File.join(home, 'farm'), log: @log, farmer: farmer(opts), strength: opts[:strength])
+        farm = Farm.new(
+          invoice(opts), File.join(home, 'farm'),
+          log: @log, farmer: farmer(opts), strength: opts[:strength]
+        )
         farm.start(host, opts[:port], threads: opts[:threads]) do |f|
           Front.set(:farm, f)
           metronome(f, opts).start do |metronome|
@@ -277,11 +276,32 @@ module Zold
           end
         end
       end
+      hungry.kill
       @log.info('Thanks for helping Zold network!')
     end
 
     private
 
+    def invoice(opts)
+      invoice = opts['invoice']
+      unless invoice.include?('@')
+        require_relative 'invoice'
+        invoice = Invoice.new(wallets: @wallets, remotes: @remotes, copies: @copies, log: @log).run(
+          ['invoice', invoice, "--network=#{opts['network']}"] +
+          (opts['tolerate-edges'] ? ['--tolerate-edges'] : [])
+        )
+      end
+      invoice
+    end
+
+    def node_alias(opts, address)
+      a = opts[:alias] || address
+      unless a.eql?(address) || a =~ /^[A-Za-z0-9]{4,16}$/
+        raise "Alias should be a 4 to 16 char long alphanumeric string: #{a}"
+      end
+      a
+    end
+
     def farmer(opts)
       case opts['farmer'].downcase.strip
       when 'plain'

data/lib/zold/commands/push.rb

@@ -58,6 +58,12 @@ Available options:"
         o.bool '--ignore-score-weakness',
           'Don\'t complain when their score is too weak',
           default: false
+        o.bool '--tolerate-edges',
+          'Don\'t fail if only "edge" (not default ones) nodes accepted the wallet',
+          default: false
+        o.bool '--quiet-if-missed',
+          'Don\'t fail if the wallet wasn\'t delivered to any remotes',
+          default: false
         o.array '--ignore-node',
           'Ignore this node and don\'t push to it',
           default: []
@@ -76,19 +82,26 @@ Available options:"
     private
 
     def push(id, opts)
-      total = 0
-      nodes = 0
-      done = 0
+      total = Concurrent::AtomicFixnum.new
+      nodes = Concurrent::AtomicFixnum.new
+      done = Concurrent::AtomicFixnum.new
+      defaults = Concurrent::AtomicFixnum.new
       start = Time.now
       @remotes.iterate(@log) do |r|
-        nodes += 1
-        total += push_one(id, r, opts)
-        done += 1
+        nodes.increment
+        total.increment(push_one(id, r, opts))
+        defaults.increment if r.default?
+        done.increment
+      end
+      raise "There are no remote nodes, run 'zold remote reset'" if nodes.value.zero?
+      unless opts['quiet-if-missed']
+        raise "No nodes out of #{nodes} accepted the wallet #{id}" if done.value.zero?
+        if defaults.value.zero? && !opts['tolerate-edges']
+          raise "There are only edge nodes, run 'zold remote reset' or use --tolerate-edges"
+        end
       end
-      raise "There are no remote nodes, run 'zold remote reset'" if nodes.zero?
-      raise "No nodes out of #{nodes} accepted the wallet #{id}" if done.zero?
-      @log.info("Push finished to #{done} nodes out of #{nodes} in #{Age.new(start)}, \
-total score for #{id} is #{total}")
+      @log.info("Push finished to #{done.value} nodes (#{defaults.value} defaults) \
+out of #{nodes.value} in #{Age.new(start)}, total score for #{id} is #{total.value}")
     end
 
     def push_one(id, r, opts)

data/lib/zold/hungry_wallets.rb

@@ -21,14 +21,43 @@
 # SOFTWARE.
 
 require 'delegate'
+require_relative 'log'
+require_relative 'thread_pool'
+require_relative 'commands/pull'
 
+# Wallets that PULL what's missing, in the background.
+#
+# Author:: Yegor Bugayenko (yegor256@gmail.com)
+# Copyright:: Copyright (c) 2018 Yegor Bugayenko
+# License:: MIT
 module Zold
   # Wallets decorator that adds missing wallets to the queue to be pulled later.
   class HungryWallets < SimpleDelegator
-    # @todo #280:30min Add to the queue. Once in there, try
-    #  to pull it as soon as possible as is described in #280.
+    def initialize(wallets, remotes, copies, pool,
+      log: Log::NULL, network: 'test')
+      @wallets = wallets
+      @remotes = remotes
+      @copies = copies
+      @log = log
+      @network = network
+      @pool = pool
+      @queue = Queue.new
+      @pool.add do
+        Endless.new('hungry', log: log).run do
+          id = @queue.pop
+          Pull.new(wallets: @wallets, remotes: @remotes, copies: @copies, log: @log).run(
+            ['pull', id.to_s, "--network=#{@network}"]
+          )
+        end
+      end
+      super(wallets)
+    end
+
     def acq(id, exclusive: false)
-      yield super(id, exclusive: exclusive)
+      @wallets.acq(id, exclusive: exclusive) do |wallet|
+        @queue << id unless wallet.exists?
+        yield wallet
+      end
     end
   end
 end

data/lib/zold/node/entrance.rb

@@ -79,7 +79,7 @@ module Zold
       end
       modified = Merge.new(
         wallets: @wallets, copies: copies.root, log: @log
-      ).run(['merge', id.to_s])
+      ).run(['merge', id.to_s, '--no-baseline'])
       Clean.new(wallets: @wallets, copies: copies.root, log: @log).run(['clean', id.to_s])
       copies.remove(localhost, Remotes::PORT)
       if modified.empty?

data/lib/zold/node/spread_entrance.rb

@@ -41,13 +41,15 @@ require_relative '../commands/clean'
 module Zold
   # The entrance
   class SpreadEntrance
-    def initialize(entrance, wallets, remotes, address, log: Log::NULL, ignore_score_weakeness: false)
+    def initialize(entrance, wallets, remotes, address, log: Log::NULL,
+      ignore_score_weakeness: false, tolerate_edges: false)
       @entrance = entrance
       @wallets = wallets
       @remotes = remotes
       @address = address
       @log = log
       @ignore_score_weakeness = ignore_score_weakeness
+      @tolerate_edges = tolerate_edges
       @mutex = Mutex.new
       @push = ThreadPool.new('spread-entrance')
     end
@@ -75,7 +77,8 @@ module Zold
               Thread.current.thread_variable_set(:wallet, id.to_s)
               Push.new(wallets: @wallets, remotes: @remotes, log: @log).run(
                 ['push', "--ignore-node=#{@address}", id.to_s] +
-                (@ignore_score_weakeness ? ['--ignore-score-weakness'] : [])
+                (@ignore_score_weakeness ? ['--ignore-score-weakness'] : []) +
+                (@tolerate_edges ? ['--tolerate-edges'] : [])
               )
             end
             @mutex.synchronize { @seen.delete(id) }

data/lib/zold/remotes.rb

@@ -88,6 +88,10 @@ module Zold
         Http.new(uri: "http://#{@host}:#{@port}#{path}", score: @score, network: @network)
       end
 
+      def default?
+        !DEFS.find { |r| r[0] == @host && r[1].to_i == @port }.nil?
+      end
+
       def to_s
         "#{@host}:#{@port}/#{@idx}"
       end

data/lib/zold/sync_wallets.rb

@@ -21,6 +21,7 @@
 # SOFTWARE.
 
 require 'futex'
+require 'delegate'
 require_relative 'log'
 
 # Synchronized collection of wallets.
@@ -32,27 +33,12 @@ require_relative 'log'
 # License:: MIT
 module Zold
   # Synchronized collection of wallets
-  class SyncWallets
+  class SyncWallets < SimpleDelegator
     def initialize(wallets, log: Log::NULL, dir: wallets.path)
       @wallets = wallets
       @log = log
       @dir = dir
-    end
-
-    def to_s
-      @wallets.to_s
-    end
-
-    def path
-      @wallets.path
-    end
-
-    def all
-      @wallets.all
-    end
-
-    def count
-      @wallets.count
+      super(wallets)
     end
 
     def acq(id, exclusive: false)

data/lib/zold/thread_pool.rb

@@ -44,17 +44,22 @@ module Zold
       idx = Concurrent::AtomicFixnum.new
       mutex = Mutex.new
       list = set.dup
-      [threads, set.count].min.times do
+      total = [threads, set.count].min
+      latch = Concurrent::CountDownLatch.new(total)
+      total.times do |i|
         add do
+          Thread.current.name = "#{@title}-#{i}"
           loop do
             r = mutex.synchronize { list.pop }
             break if r.nil?
             yield(r, idx.increment - 1)
           end
+        ensure
+          latch.count_down
         end
       end
-      @threads.each(&:join)
-      @threads.clear
+      latch.wait
+      kill
     end
 
     # Add a new thread
@@ -62,6 +67,7 @@ module Zold
       raise 'Block must be given to start()' unless block_given?
       latch = Concurrent::CountDownLatch.new(1)
       thread = Thread.start do
+        Thread.current.name = @title
         VerboseThread.new(@log).run do
           latch.count_down
           yield
@@ -75,22 +81,29 @@ module Zold
       @threads << thread
     end
 
+    def join(sec)
+      @threads.each { |t| t.join(sec) }
+    end
+
     # Kill them all immediately and close the pool
     def kill
       if @threads.empty?
         @log.debug("Thread pool \"#{@title}\" terminated with no threads")
         return
       end
-      @log.info("Stopping \"#{@title}\" thread pool with #{@threads.count} threads: \
+      @log.debug("Stopping \"#{@title}\" thread pool with #{@threads.count} threads: \
 #{@threads.map { |t| "#{t.name}/#{t.status}" }.join(', ')}...")
       start = Time.new
       @threads.each do |t|
         (t.thread_variable_get(:kids) || []).each(&:kill)
         t.kill
         raise "Failed to join the thread \"#{t.name}\" in \"#{@title}\" pool" unless t.join(0.1)
-        (Thread.current.thread_variable_get(:kids) || []).delete(t)
+        Thread.current.thread_variable_set(
+          :kids,
+          (Thread.current.thread_variable_get(:kids) || []) - [t]
+        )
       end
-      @log.info("Thread pool \"#{@title}\" terminated all threads in #{Age.new(start)}, \
+      @log.debug("Thread pool \"#{@title}\" terminated all threads in #{Age.new(start)}, \
 it was alive for #{Age.new(@start)}: #{@threads.map { |t| "#{t.name}/#{t.status}" }.join(', ')}")
       @threads.clear
     end

data/lib/zold/verbose_thread.rb

@@ -35,6 +35,7 @@ module Zold
     end
 
     def run(safe = false)
+      Thread.current.report_on_exception = false
       yield
     rescue StandardError => e
       @log.error(Backtrace.new(e).to_s)

data/lib/zold/version.rb

@@ -25,7 +25,7 @@
 # Copyright:: Copyright (c) 2018 Yegor Bugayenko
 # License:: MIT
 module Zold
-  VERSION = '0.18.8'
+  VERSION = '0.18.9'
   PROTOCOL = 2
   REPO = 'zold-io/zold'
 end

data/test/commands/test_node.rb

@@ -39,11 +39,7 @@ require_relative '../node/fake_node'
 # Copyright:: Copyright (c) 2018 Yegor Bugayenko
 # License:: MIT
 class TestNode < Zold::Test
-  # @todo #306:30min This test is failing from time to time
-  #  We should find a way to check that tests involved in thread concurrency
-  #  are always working
   def test_push_and_fetch
-    skip
     FakeHome.new(log: test_log).run do |home|
       FakeNode.new(log: test_log).run do |port|
         wallets = home.wallets
@@ -51,7 +47,7 @@ class TestNode < Zold::Test
         remotes = home.remotes
         remotes.add('localhost', port)
         Zold::Push.new(wallets: wallets, remotes: remotes, log: test_log).run(
-          ['push', '--ignore-score-weakness']
+          ['push', '--ignore-score-weakness', '--tolerate-edges']
         )
         copies = home.copies(wallet)
         begin
@@ -59,7 +55,7 @@ class TestNode < Zold::Test
           Zold::Fetch.new(
             wallets: wallets, copies: copies.root,
             remotes: remotes, log: test_log
-          ).run(['fetch', '--ignore-score-weakness'])
+          ).run(['fetch', '--ignore-score-weakness', '--tolerate-edges'])
         rescue StandardError => _
           sleep 1
           retry if (retries += 1) < 3

data/test/commands/test_pull.rb

@@ -42,7 +42,7 @@ class TestPull < Zold::Test
       stub_request(:get, "http://localhost:4096/wallet/#{id}/size").to_return(status: 200, body: '10000')
       stub_request(:get, "http://localhost:4096/wallet/#{id}").to_return(status: 200, body: json)
       Zold::Pull.new(wallets: home.wallets, remotes: remotes, copies: home.copies.root.to_s, log: test_log).run(
-        ['--ignore-this-stupid-option', 'pull', id.to_s]
+        ['--ignore-this-stupid-option', 'pull', id.to_s, '--tolerate-edges']
       )
       home.wallets.acq(Zold::Id.new(id)) do |wallet|
         assert(wallet.exists?)

data/test/commands/test_push.rb

@@ -43,7 +43,7 @@ class TestPush < Zold::Test
       remotes.add('localhost', 80)
       stub_request(:put, "http://localhost:80/wallet/#{wallet.id}").to_return(status: 304)
       Zold::Push.new(wallets: home.wallets, remotes: remotes, log: test_log).run(
-        ['--ignore-this-stupid-option', 'push', wallet.id.to_s]
+        ['--ignore-this-stupid-option', 'push', wallet.id.to_s, '--tolerate-edges']
       )
     end
   end
@@ -58,7 +58,7 @@ class TestPush < Zold::Test
       stub_request(:put, "http://localhost:80/wallet/#{wallet_a.id}").to_return(status: 304)
       stub_request(:put, "http://localhost:80/wallet/#{wallet_b.id}").to_return(status: 304)
       Zold::Push.new(wallets: home.wallets, remotes: remotes, log: log).run(
-        ['--ignore-this-stupid-option --threads 2', 'push', wallet_a.id.to_s, wallet_b.id.to_s]
+        ['--tolerate-edges', '--threads=2', 'push', wallet_a.id.to_s, wallet_b.id.to_s]
       )
     end
   end

data/test/node/fake_node.rb

@@ -44,8 +44,8 @@ class FakeNode
       RandomPort::Pool::SINGLETON.acquire do |port|
         node = Thread.new do
           Thread.current.name = 'fake_node'
+          Thread.current.abort_on_exception = true
           Zold::VerboseThread.new(@log).run do
-            Thread.current.abort_on_exception = true
             require_relative '../../lib/zold/commands/node'
             Zold::Node.new(wallets: home.wallets, remotes: home.remotes, copies: home.copies.root, log: @log).run(
               [

data/test/test__helper.rb

@@ -63,10 +63,14 @@ module Zold
     def assert_equal_wait(expected, max: 30)
       start = Time.now
       loop do
-        actual = yield
-        if expected == actual
-          assert_equal(expected, actual)
-          break
+        begin
+          actual = yield
+          if expected == actual
+            assert_equal(expected, actual)
+            break
+          end
+        rescue StandardError => e
+          test_log.debug(e.message)
         end
         sleep 1
         sec = Time.now - start

data/test/test_hungry_wallets.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+# Copyright (c) 2018 Yegor Bugayenko
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the 'Software'), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+require 'minitest/autorun'
+require 'webmock/minitest'
+require_relative 'test__helper'
+require_relative 'fake_home'
+require_relative '../lib/zold/key'
+require_relative '../lib/zold/thread_pool'
+require_relative '../lib/zold/wallets'
+require_relative '../lib/zold/hungry_wallets'
+
+# HungryWallets test.
+# Author:: Yegor Bugayenko (yegor256@gmail.com)
+# Copyright:: Copyright (c) 2018 Yegor Bugayenko
+# License:: MIT
+class TestHungryWallets < Zold::Test
+  def test_pulls_wallet
+    FakeHome.new(log: test_log).run do |home|
+      id = Zold::Id.new
+      get = stub_request(:get, "http://localhost:4096/wallet/#{id}/size").to_return(status: 404)
+      remotes = home.remotes
+      remotes.add('localhost', 4096)
+      pool = Zold::ThreadPool.new('test', log: test_log)
+      wallets = Zold::HungryWallets.new(
+        home.wallets, remotes, File.join(home.dir, 'copies'),
+        pool, log: test_log
+      )
+      wallets.acq(id) { |w| assert(!w.exists?) }
+      pool.join(1)
+      assert_requested(get)
+    end
+  end
+end

data/test/test_thread_pool.rb

@@ -69,6 +69,14 @@ class TestThreadPool < Zold::Test
     assert_equal('0 1 2', indexes.to_a.sort.join(' '))
   end
 
+  def test_runs_with_exceptions
+    assert_raises do
+      Zold::ThreadPool.new('test', log: test_log).run(1) do
+        raise 'intended'
+      end
+    end
+  end
+
   def test_adds_and_stops
     pool = Zold::ThreadPool.new('test', log: test_log)
     pool.add do

data/test/test_zold.rb

@@ -34,9 +34,10 @@ require_relative '../lib/zold/age'
 # License:: MIT
 class TestZold < Zold::Test
   Dir.new('fixtures/scripts').select { |f| f =~ /\.sh$/ && !f.start_with?('_') }.each do |f|
-    define_method("test_#{f.gsub(/\.sh$/, '').gsub(/[^a-z]/, '_')}") do
+    method = "test_#{f.gsub(/\.sh$/, '').gsub(/[^a-z]/, '_')}"
+    define_method(method) do
       start = Time.now
-      test_log.info("\n\n#{f} running...")
+      test_log.info("\n\n#{method} running (script at #{f})...")
       Dir.mktmpdir do |dir|
         FileUtils.cp('fixtures/id_rsa.pub', dir)
         FileUtils.cp('fixtures/id_rsa', dir)

data/zold.gemspec

@@ -61,12 +61,12 @@ and suggests a different architecture for digital wallet maintenance.'
   s.test_files = s.files.grep(%r{^(test|features)/})
   s.rdoc_options = ['--charset=UTF-8']
   s.extra_rdoc_files = ['README.md', 'LICENSE.txt']
-  s.add_runtime_dependency 'backtrace', '0.3.0'
+  s.add_runtime_dependency 'backtrace', '~>0'
   s.add_runtime_dependency 'concurrent-ruby', '1.1.3'
   s.add_runtime_dependency 'cucumber', '3.1.2' # has to stay here for Heroku
   s.add_runtime_dependency 'diffy', '3.2.1'
-  s.add_runtime_dependency 'futex', '0.7.0'
-  s.add_runtime_dependency 'get_process_mem', '0.2.3'
+  s.add_runtime_dependency 'futex', '~>0'
+  s.add_runtime_dependency 'get_process_mem', '~>0.2'
   s.add_runtime_dependency 'json', '2.1.0'
   s.add_runtime_dependency 'memory_profiler', '0.9.12'
   s.add_runtime_dependency 'openssl', '2.1.2'
@@ -79,12 +79,12 @@ and suggests a different architecture for digital wallet maintenance.'
   s.add_runtime_dependency 'slop', '4.6.2'
   s.add_runtime_dependency 'sys-proctable', '1.2.1'
   s.add_runtime_dependency 'thin', '1.7.2'
-  s.add_runtime_dependency 'threads', '0.3.0'
+  s.add_runtime_dependency 'threads', '~>0'
   s.add_runtime_dependency 'typhoeus', '1.3.1'
   s.add_runtime_dependency 'usagewatch_ext', '0.2.1'
-  s.add_runtime_dependency 'xcop', '0.6'
-  s.add_runtime_dependency 'zache', '0.4.0'
-  s.add_runtime_dependency 'zold-score', '0.4.3'
+  s.add_runtime_dependency 'xcop', '~>0'
+  s.add_runtime_dependency 'zache', '~>0'
+  s.add_runtime_dependency 'zold-score', '0.4.4'
   s.add_development_dependency 'codecov', '0.1.13'
   s.add_development_dependency 'minitest', '5.11.3'
   s.add_development_dependency 'minitest-fail-fast', '0.1.0'

metadata

@@ -1,29 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: zold
 version: !ruby/object:Gem::Version
-  version: 0.18.8
+  version: 0.18.9
 platform: ruby
 authors:
 - Yegor Bugayenko
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-12-14 00:00:00.000000000 Z
+date: 2018-12-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: backtrace
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.0
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.0
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
   requirement: !ruby/object:Gem::Requirement
@@ -70,30 +70,30 @@ dependencies:
   name: futex
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.7.0
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.7.0
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: get_process_mem
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.2.3
+        version: '0.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.2.3
+        version: '0.2'
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
@@ -266,16 +266,16 @@ dependencies:
   name: threads
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.0
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.0
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: typhoeus
   requirement: !ruby/object:Gem::Requirement
@@ -308,44 +308,44 @@ dependencies:
   name: xcop
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.6'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.6'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: zache
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.4.0
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.4.0
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: zold-score
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.4.3
+        version: 0.4.4
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.4.3
+        version: 0.4.4
 - !ruby/object:Gem::Dependency
   name: codecov
   requirement: !ruby/object:Gem::Requirement
@@ -656,6 +656,7 @@ files:
 - test/test_gem.rb
 - test/test_hexnum.rb
 - test/test_http.rb
+- test/test_hungry_wallets.rb
 - test/test_id.rb
 - test/test_json_page.rb
 - test/test_key.rb
@@ -689,7 +690,7 @@ licenses:
 - MIT
 metadata: {}
 post_install_message: |-
-  Thanks for installing Zold 0.18.8!
+  Thanks for installing Zold 0.18.9!
     Study our White Paper: https://papers.zold.io/wp.pdf
     Read our blog posts: https://blog.zold.io
     Try online wallet at: https://wts.zold.io
@@ -765,6 +766,7 @@ test_files:
 - test/test_gem.rb
 - test/test_hexnum.rb
 - test/test_http.rb
+- test/test_hungry_wallets.rb
 - test/test_id.rb
 - test/test_json_page.rb
 - test/test_key.rb