zold 0.20.1 → 0.20.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dec6ef3362deffe905c48db392a63691b2af28e0e1d7935705a7ccee03f72630
-  data.tar.gz: d8bc9621312050b27e2d25897585f6f8a9b5d2b0f9d4681c9a08e52ad04990d3
+  metadata.gz: d066e22fce978f4fdee5ebf56c935c3f05a09a1c7ad57c6da8ed6388db63eb84
+  data.tar.gz: e505ab03aafc00413ba0f71513fb4bd0f5e06794da6704a2ea335275285222cb
 SHA512:
-  metadata.gz: 50bf436dc8d8323b83c7fae037e9de22818921e67ede7d1a2fce49c29dcffe515e1f6695199bbd36ec64668aba138782cdb82725f07ae13c2db74f78f1210b2f
-  data.tar.gz: 603a47739743f9c77c3a4f6208802534d1ad7c59b9a15d5759324c78ddba22939ae7b4b1181181e26b0de2445ac747eef31aab14e45ffbc640d9791ec8b4e7f1
+  metadata.gz: 4313b16d11fb3226c35a0db9b3e7367b9550b008c2be560c3281ae05c8bae9f5249f53b5a7fbd8cd429f3f8207db098e6dada0ed3af68cb1bff927a1f45ce804
+  data.tar.gz: e2a483f2eff040a1429e96c0a515a00c8a7566ab200ed23ab18b6d6d5dc89c6fecda4d107ef0c21f3481df4759cf0ae6f70ca0c94b5f9270f798ee211854f737

data/bin/zold

@@ -29,9 +29,11 @@ require 'slop'
 require 'rainbow'
 require 'backtrace'
 require 'memory_profiler'
+require 'get_process_mem'
 require_relative '../lib/zold'
 require_relative '../lib/zold/version'
 require_relative '../lib/zold/wallet'
+require_relative '../lib/zold/dir_items'
 require_relative '../lib/zold/wallets'
 require_relative '../lib/zold/tree_wallets'
 require_relative '../lib/zold/sync_wallets'
@@ -40,6 +42,7 @@ require_relative '../lib/zold/hungry_wallets'
 require_relative '../lib/zold/log'
 require_relative '../lib/zold/key'
 require_relative '../lib/zold/age'
+require_relative '../lib/zold/size'
 require_relative '../lib/zold/amount'
 require_relative '../lib/zold/copies'
 require_relative '../lib/zold/remotes'
@@ -159,20 +162,27 @@ FileUtils.mkdir_p(home)
 Dir.chdir(home)
 log.debug("Home directory: #{home}")
 
-zoldata = File.join(home, '.zoldata')
+zdata = File.join(home, '.zoldata')
 
 unless opts['skip-upgrades']
-  Zold::Upgrades.new(Zold::VersionFile.new(File.join(zoldata, 'version')), 'upgrades', { command: command, network: opts['network']}).run
+  Zold::Upgrades.new(Zold::VersionFile.new(File.join(zdata, 'version')), 'upgrades', { command: command, network: opts['network']}).run
 end
 
+locks = File.join(zdata, 'locks')
+Zold::DirItems.new(locks).fetch.each do |f|
+  file = File.join(locks, f)
+  if File.mtime(file) < Time.now - 60
+    File.delete(file)
+  end
+end
 wallets = Zold::SyncWallets.new(
   Zold::CachedWallets.new(
     command == 'node' ? Zold::TreeWallets.new(home) : Zold::Wallets.new(home)
   ),
   log: log,
-  dir: File.join(zoldata, 'locks')
+  dir: locks
 )
-fremotes = File.join(zoldata, 'remotes')
+fremotes = File.join(zdata, 'remotes')
 remotes = Zold::Remotes.new(file: fremotes, network: opts['network'])
 if File.exist?(fremotes)
   log.debug("Remote nodes: #{remotes.all.count} total")
@@ -180,9 +190,10 @@ else
   remotes.masters
   log.debug("Default remotes have been set: #{remotes.all.count} total")
 end
-copies = File.join(zoldata, 'copies')
+copies = File.join(zdata, 'copies')
 
 log.debug("Network: #{opts['network']} (#{opts['network'] == Zold::Wallet::MAINET ? 'main' : 'test'} net)")
+log.debug("Memory footprint at start is #{Zold::Size.new(GetProcessMem.new.bytes.to_i)}")
 
 cmd = lambda do
   begin
@@ -262,6 +273,11 @@ else
   code = cmd.call
 end
 
-exit(code) unless code.zero?
+log.debug("Memory footprint at the end is #{Zold::Size.new(GetProcessMem.new.bytes.to_i)}")
+if code.zero?
+  log.debug("Failed in in #{Zold::Age.new(start)}")
+  exit(code)
+else
+  log.debug("Successfully finished in #{Zold::Age.new(start)}")
+end
 
-log.debug("Successfully finished in #{Zold::Age.new(start)}")

data/fixtures/merge/into-no-wallet/copies/0123456789abcdef/scores.zc

@@ -1 +1 @@
-1,0.0.0.0,4096,50,NOW
+1,0.0.0.0,4096,50,NOW,M

data/fixtures/merge/random-expenses/copies/0123456789abcdef/scores.zc

@@ -1,5 +1,5 @@
-1,0.0.0.0,4096,10,NOW
-2,0.0.0.0,4096,20,NOW
-3,0.0.0.0,4096,30,NOW
-4,0.0.0.0,4096,40,NOW
-5,0.0.0.0,4096,50,NOW
+1,0.0.0.0,4096,10,NOW,M
+2,0.0.0.0,4096,20,NOW,E
+3,0.0.0.0,4096,30,NOW,E
+4,0.0.0.0,4096,40,NOW,E
+5,0.0.0.0,4096,50,NOW,E

data/fixtures/merge/simple-case/copies/0123456789abcdef/scores.zc

@@ -1 +1 @@
-1,0.0.0.0,4096,50,NOW
+1,0.0.0.0,4096,50,NOW,M

data/lib/zold/age.rb

@@ -22,6 +22,7 @@
 
 require 'time'
 require 'rainbow'
+require_relative 'txn'
 
 # Age in seconds.
 # Author:: Yegor Bugayenko (yegor256@gmail.com)
@@ -31,7 +32,7 @@ module Zold
   # Age
   class Age
     def initialize(time, limit: nil)
-      @time = time.nil? || time.is_a?(Time) ? time : Time.parse(time)
+      @time = time.nil? || time.is_a?(Time) ? time : Txn.parse_time(time)
       @limit = limit
     end
 

data/lib/zold/amount.rb

@@ -50,12 +50,15 @@ module Zold
       raise "The amount is too small: #{@zents}" if @zents < -MAX
     end
 
+    # Just zero, for convenience.
     ZERO = Amount.new(zents: 0)
 
+    # Convert it to zents and return as an integer.
     def to_i
       @zents
     end
 
+    # Convert to ZLD and return as a string. If you need float, you should use <tt>to_f()</tt> later.
     def to_zld(digits = 2)
       format("%0.#{digits}f", @zents.to_f / 2**FRACTION)
     end

data/lib/zold/cached_wallets.rb

@@ -38,7 +38,7 @@ module Zold
       @clean = ThreadPool.new('cached-wallets')
       @clean.add do
         Endless.new('cached_wallets').run do
-          sleep 60
+          sleep 5
           @zache.clean
         end
       end
@@ -47,7 +47,7 @@ module Zold
 
     def acq(id, exclusive: false)
       @wallets.acq(id, exclusive: exclusive) do |wallet|
-        c = @zache.get(id.to_s, lifetime: 5 * 60) { wallet }
+        c = @zache.get(id.to_s, lifetime: 15) { wallet }
         res = yield c
         c.flush if exclusive
         res

data/lib/zold/commands/calculate.rb

@@ -79,7 +79,7 @@ Available options:"
       strength = opts[:strength]
       raise "Invalid strength: #{strength}" if strength <= 0 || strength > 8
       score = Zold::Score.new(
-        time: Time.parse(opts[:time]), host: opts[:host], port: opts[:port].to_i,
+        time: Txn.parse_time(opts[:time]), host: opts[:host], port: opts[:port].to_i,
         invoice: opts[:invoice], strength: strength
       )
       loop do

data/lib/zold/commands/clean.rb

@@ -33,6 +33,7 @@ require_relative '../size'
 require_relative '../log'
 require_relative '../http'
 require_relative '../copies'
+require_relative '../thread_pool'
 
 # CLEAN command.
 # Author:: Yegor Bugayenko (yegor256@gmail.com)
@@ -51,10 +52,14 @@ module Zold
       opts = Slop.parse(args, help: true, suppress_errors: true) do |o|
         o.banner = "Usage: zold clean [ID...] [options]
 Available options:"
+        o.integer '--threads',
+          "How many threads to use for cleaning copies (default: #{[Concurrent.processor_count / 2, 2].max})",
+          default: [Concurrent.processor_count / 2, 2].max
         o.bool '--help', 'Print instructions'
       end
       mine = Args.new(opts, @log).take || return
-      (mine.empty? ? @wallets.all : mine.map { |i| Id.new(i) }).each do |id|
+      list = mine.empty? ? @wallets.all : mine.map { |i| Id.new(i) }
+      ThreadPool.new('clean', log: @log).run(opts['threads'], list.uniq) do |id|
         clean(Copies.new(File.join(@copies, id), log: @log), opts)
       end
     end

data/lib/zold/commands/fetch.rb

@@ -91,7 +91,7 @@ Available options:"
       end
       mine = Args.new(opts, @log).take || return
       list = mine.empty? ? @wallets.all : mine.map { |i| Id.new(i) }
-      ThreadPool.new('fetch', log: @log).run(opts['threads'], list) do |id|
+      ThreadPool.new('fetch', log: @log).run(opts['threads'], list.uniq) do |id|
         fetch(id, Copies.new(File.join(@copies, id)), opts)
       end
     end
@@ -112,7 +112,9 @@ Available options:"
         done.increment
       end
       unless opts['quiet-if-absent']
-        raise "No nodes out of #{nodes.value} have the wallet #{id}" if done.value.zero?
+        if done.value.zero?
+          raise "No nodes out of #{nodes.value} have the wallet #{id}; run 'zold remote update' and try again"
+        end
         if masters.value.zero? && !opts['tolerate-edges']
           raise EdgesOnly, "There are only edge nodes, run 'zold remote update' or use --tolerate-edges"
         end
@@ -140,7 +142,6 @@ run 'zold remote update' or use --tolerate-quorum=1"
       size = r.http(uri + '/size').get
       r.assert_code(200, size)
       res = r.http(uri).get(timeout: 2 + size.body.to_i * 0.01 / 1024)
-      raise "Wallet #{id} not found" if res.status == '404'
       r.assert_code(200, res)
       json = JsonPage.new(res.body, uri).to_hash
       score = Score.parse_json(json['score'])
@@ -161,7 +162,7 @@ run 'zold remote update' or use --tolerate-quorum=1"
         if wallet.balance.negative? && !wallet.root?
           raise "The balance of #{id} is #{wallet.balance} and it's not a root wallet"
         end
-        copy = cps.add(IO.read(f), score.host, score.port, score.value)
+        copy = cps.add(IO.read(f), score.host, score.port, score.value, master: r.master?)
         @log.info("#{r} returned #{wallet.mnemo} #{Age.new(json['mtime'])}/#{json['copies']}c \
 as copy ##{copy}/#{cps.all.count} in #{Age.new(start, limit: 4)}: \
 #{Rainbow(score.value).green} (#{json['version']})")

data/lib/zold/commands/merge.rb

@@ -72,7 +72,8 @@ Available options:"
       end
       mine = Args.new(opts, @log).take || return
       modified = []
-      (mine.empty? ? @wallets.all : mine.map { |i| Id.new(i) }).each do |id|
+      list = mine.empty? ? @wallets.all : mine.map { |i| Id.new(i) }
+      list.uniq.each do |id|
         next unless merge(id, Copies.new(File.join(@copies, id)), opts)
         modified << id
         next if opts['skip-propagate']

data/lib/zold/commands/node.rb

@@ -23,6 +23,7 @@
 require 'open3'
 require 'slop'
 require 'backtrace'
+require 'fileutils'
 require 'zache'
 require 'concurrent'
 require 'zold/score'
@@ -132,6 +133,15 @@ module Zold
         o.bool '--no-cache',
           'Skip caching of front JSON pages (will seriously slow down, mostly useful for testing)',
           default: false
+        o.boolean '--skip-audit',
+          'Don\'t report audit information to the console every minute',
+          default: false
+        o.boolean '--skip-reconnect',
+          'Don\'t reconnect to the network every minute (for testing)',
+          default: false
+        o.boolean '--not-hungry',
+          'Don\'t do hugry pulling of missed nodes (mostly for testing)',
+          default: false
         o.bool '--allow-spam',
           'Don\'t filter the incoming spam via PUT requests (duplicate wallets)',
           default: false
@@ -139,11 +149,14 @@ module Zold
           'Skip Out Of Memory check and never exit, no matter how much RAM is consumed',
           default: false
         o.integer '--oom-limit',
-          'Maximum amount of memory we can consume, quit if we take more than that, in Mb (default: 256)',
-          default: 256
+          'Maximum amount of memory we can consume, quit if we take more than that, in Mb (default: 512)',
+          default: 512
         o.integer '--queue-limit',
           'The maximum number of wallets to be accepted via PUSH and stored in the queue (default: 256)',
           default: 256
+        o.bool '--skip-gc',
+          'Don\'t run garbage collector and never remove any wallets from the disk',
+          default: false
         o.integer '--gc-age',
           'Maximum time in seconds to keep an empty and unused wallet on the disk',
           default: 60 * 60 * 24 * 10
@@ -221,8 +234,17 @@ module Zold
         Zold::Remote.new(remotes: @remotes).run(['remote', 'remove', host, port.to_s])
         @log.info("Removed current node (#{address}) from list of remotes")
       end
-      hungry = Zold::ThreadPool.new('hungry', log: @log)
-      wts = Zold::HungryWallets.new(@wallets, @remotes, @copies, hungry, log: @log, network: opts['network'])
+      if File.exist?(@copies)
+        FileUtils.rm_rf(@copies)
+        @log.info("Directory #{@copies} deleted")
+      end
+      wts = @wallets
+      if opts['not-hungry']
+        @log.info('Hungry pulling disabled because of --not-hungry')
+      else
+        hungry = Zold::ThreadPool.new('hungry', log: @log)
+        wts = Zold::HungryWallets.new(@wallets, @remotes, @copies, hungry, log: @log, network: opts['network'])
+      end
       Front.set(:zache, Zache.new(dirty: true))
       Front.set(:wallets, wts)
       Front.set(:remotes, @remotes)
@@ -279,7 +301,7 @@ module Zold
           end
         end
       end
-      hungry.kill
+      hungry.kill unless opts['not-hungry']
       @log.info('Thanks for helping Zold network!')
     end
 
@@ -383,14 +405,28 @@ module Zold
     def metronome(farm, opts)
       metronome = Metronome.new(@log)
       if opts['no-metronome']
-        @log.info('Metronome hasn\'t been started because of --no-metronome')
+        @log.info("Metronome hasn't been started because of --no-metronome")
         return metronome
       end
-      require_relative 'routines/gc'
-      metronome.add(Routines::Gc.new(opts, @wallets, log: @log))
+      if opts['skip-gc']
+        @log.info('Garbage collection is disabled because of --skip-gc')
+      else
+        require_relative 'routines/gc'
+        metronome.add(Routines::Gc.new(opts, @wallets, log: @log))
+      end
+      if opts['skip-audit']
+        @log.info('Audit is disabled because of --skip-audit')
+      else
+        require_relative 'routines/audit'
+        metronome.add(Routines::Audit.new(opts, @wallets, log: @log))
+      end
       unless opts['standalone']
-        require_relative 'routines/reconnect'
-        metronome.add(Routines::Reconnect.new(opts, @remotes, farm, network: opts['network'], log: @log))
+        if opts['skip-reconnect']
+          @log.info('Reconnect is disabled because of --skip-reconnect')
+        else
+          require_relative 'routines/reconnect'
+          metronome.add(Routines::Reconnect.new(opts, @remotes, farm, network: opts['network'], log: @log))
+        end
       end
       @log.info('Metronome started (use --no-metronome to disable it)')
       metronome

data/lib/zold/commands/push.rb

@@ -83,7 +83,7 @@ Available options:"
       end
       mine = Args.new(opts, @log).take || return
       list = mine.empty? ? @wallets.all : mine.map { |i| Id.new(i) }
-      ThreadPool.new('push', log: @log).run(opts['threads'], list) do |id|
+      ThreadPool.new('push', log: @log).run(opts['threads'], list.uniq) do |id|
         push(id, opts)
       end
     end
@@ -104,7 +104,9 @@ Available options:"
         done.increment
       end
       unless opts['quiet-if-missed']
-        raise "No nodes out of #{nodes} accepted the wallet #{id}" if done.value.zero?
+        if done.value.zero?
+          raise "No nodes out of #{nodes} accepted the wallet #{id}; run 'zold remote update' and try again"
+        end
         if masters.value.zero? && !opts['tolerate-edges']
           raise EdgesOnly, "There are only edge nodes, run 'zold remote update' or use --tolerate-edges"
         end

data/lib/zold/commands/routines/audit.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+# Copyright (c) 2018 Yegor Bugayenko
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the 'Software'), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+require 'get_process_mem'
+require_relative '../../size'
+
+# Audit and report as much as we can to the command line.
+# Author:: Yegor Bugayenko (yegor256@gmail.com)
+# Copyright:: Copyright (c) 2018 Yegor Bugayenko
+# License:: MIT
+module Zold
+  # Routines module
+  module Routines
+    # Audit the system
+    class Audit
+      def initialize(opts, wallets, log: Log::NULL)
+        @opts = opts
+        @wallets = wallets
+        @log = log
+      end
+
+      def exec(_ = 0)
+        sleep(60) unless @opts['routine-immediately']
+        @log.info(
+          'Audit: ' + [
+            "memory used: #{Size.new(GetProcessMem.new.bytes.to_i)}",
+            "threads total: #{Thread.list.count}",
+            "wallets: #{@wallets.count}"
+          ].join('; ')
+        )
+      end
+    end
+  end
+end

data/lib/zold/commands/routines/gc.rb

@@ -47,7 +47,7 @@ module Zold
         removed = 0
         @wallets.all.each do |id|
           seen += 1
-          next unless @wallets.acq(id) { |w| w.exists? && w.txns.empty? && w.mtime < Time.now - @opts['gc-age'] }
+          next unless @wallets.acq(id) { |w| w.exists? && w.mtime < Time.now - @opts['gc-age'] && w.txns.empty? }
           cmd.run(args + [id.to_s])
           removed += 1
         end

data/lib/zold/copies.rb

@@ -89,7 +89,7 @@ module Zold
     end
 
     # Returns the name of the copy
-    def add(content, host, port, score, time = Time.now)
+    def add(content, host, port, score, time: Time.now, master: false)
       raise "Content can't be empty" if content.empty?
       raise 'TCP port must be of type Integer' unless port.is_a?(Integer)
       raise "TCP port can't be negative: #{port}" if port.negative?
@@ -121,7 +121,8 @@ module Zold
           host: host,
           port: port,
           score: score,
-          time: time
+          time: time,
+          master: master
         }
         save(list)
         name
@@ -134,24 +135,27 @@ module Zold
           {
             name: name,
             path: File.join(@dir, "#{name}#{Copies::EXT}"),
+            total: scores.count,
+            master: scores.any? { |s| s[:master] },
             score: scores.select { |s| s[:time] > Time.now - 24 * 60 * 60 }
               .map { |s| s[:score] }
               .inject(&:+) || 0
           }
-        end.select { |c| File.exist?(c[:path]) }.sort_by { |c| c[:score] }.reverse
+        end.select { |c| File.exist?(c[:path]) }.sort_by { |c| [c[:master] ? 1 : 0, c[:score].to_s] }.reverse
       end
     end
 
     def load
       FileUtils.mkdir_p(File.dirname(file))
       FileUtils.touch(file)
-      CSV.read(file).map do |s|
+      CSV.read(file).select { |s| s.count == 6 }.map do |s|
         {
           name: s[0],
           host: s[1],
           port: s[2].to_i,
           score: s[3].to_i,
-          time: Time.parse(s[4])
+          time: Txn.parse_time(s[4]),
+          master: s[5] == 'M'
         }
       end
     end
@@ -163,9 +167,12 @@ module Zold
         file,
         list.map do |r|
           [
-            r[:name], r[:host],
-            r[:port], r[:score],
-            r[:time].utc.iso8601
+            r[:name],
+            r[:host],
+            r[:port],
+            r[:score],
+            r[:time].utc.iso8601,
+            r[:master] ? 'M' : 'E'
           ].join(',')
         end.join("\n")
       )