zold 0.14.36 → 0.14.37

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e0ac9a7fc3634c563abae9dce0f68ca709d46490
-  data.tar.gz: 59bb7653c11021d3d46c6cab7124480c9db2a103
+  metadata.gz: e60384f0c0c624aa8351fbb37ba157ab55ecec6c
+  data.tar.gz: 45c724b470053a6d46787589ff8a8815f8bd2dc2
 SHA512:
-  metadata.gz: cc4d99b314585fc077c82779463f09505d71f787a560087a2d55f3ab17184580df1aa0f12de8e0afb49914a0f2bff5d5064f343d579a9fe0d289562cacf94efe
-  data.tar.gz: 1995bae0601aa6cd84b5de1f3eeb58b27914f2d4d2935a503fa5226c88d12eaa659a6a76bcac58dae8b7103dac813ba90be1e66597490af4bf899e8f77ead092
+  metadata.gz: '08171d0370b3d30ae360159cc144fe7fb687110c09bad086b3c7ce37b984f49e76419dd864d55a22696e0a582a6b569b4ffb43ec04ec01c9dfdae30b144708b5'
+  data.tar.gz: cdd6f7908645858c273be17c94b3741aab8a5dd15fdfff65450a9d01ad79f34d97d290942c1529342eed4c31785fed78a650ad556b2dfe2036c25d8ba20e3514

data/bin/zold

@@ -31,10 +31,13 @@ end
 
 require 'slop'
 require 'rainbow'
+require 'cachy'
+require 'moneta'
 require_relative '../lib/zold'
 require_relative '../lib/zold/version'
 require_relative '../lib/zold/wallet'
 require_relative '../lib/zold/wallets'
+require_relative '../lib/zold/tree_wallets'
 require_relative '../lib/zold/sync_wallets'
 require_relative '../lib/zold/log'
 require_relative '../lib/zold/key'
@@ -46,6 +49,8 @@ require_relative '../lib/zold/version_file'
 
 Thread.current.name = 'main'
 
+Cachy.cache_store = Moneta.new(:Memory)
+
 Encoding.default_external = Encoding::UTF_8
 Encoding.default_internal = Encoding::UTF_8
 
@@ -158,9 +163,16 @@ Available options:"
     Zold::ProtocolUp.new(Dir.pwd, log).exec
     require_relative '../upgrades/rename_foreign_wallets'
     Zold::RenameForeignWallets.new(Dir.pwd, opts['network'], log).exec
+    if command == 'node'
+      require_relative '../upgrades/move_wallets_into_tree'
+      Zold::MoveWalletsIntoTree.new(Dir.pwd, log).exec
+    end
   end
 
-  wallets = Zold::SyncWallets.new(Zold::Wallets.new('.'), File.join(zoldata, 'locks'), log: log)
+  wallets = Zold::SyncWallets.new(
+    command == 'node' ? Zold::TreeWallets.new('.') : Zold::Wallets.new('.'),
+    File.join(zoldata, 'locks'), log: log
+  )
   remotes = Zold::Remotes.new(file: File.join(zoldata, 'remotes'), network: opts['network'])
   copies = File.join(zoldata, 'copies')
 
@@ -183,6 +195,9 @@ Available options:"
   when 'show'
     require_relative '../lib/zold/commands/show'
     Zold::Show.new(wallets: wallets, log: log).run(args)
+  when 'list'
+    require_relative '../lib/zold/commands/list'
+    Zold::List.new(wallets: wallets, log: log).run(args)
   when 'fetch'
     require_relative '../lib/zold/commands/fetch'
     Zold::Fetch.new(wallets: wallets, remotes: remotes, copies: copies, log: log).run(args)

data/fixtures/scripts/distribute-wallet.sh

@@ -67,15 +67,11 @@ if [ $(echo ${json} | jq -r '.entrance.history_size') == "0" ]; then
   echo "The history doesn't have a wallet, it's a bug"
   exit 6
 fi
-if [ ! $(echo ${json} | jq -r '.wallets') == "1" ]; then
-  echo "The wallet is not there for some reason, it's a bug"
-  exit 7
-fi
 
 # Now, we remove the wallet from the second node and expect the first
 # one to "spread" it again, almost immediately. The second node should
 # have the wallet very soon.
-rm -f ${second}/0000000000000000.z
+rm -f ${second}/**/*.z
 i=0
 until zold fetch 0000000000000000 --ignore-score-weakness; do
   echo 'Failed to fetch, let us try again'

data/fixtures/scripts/spread-wallets.sh

@@ -51,7 +51,3 @@ if [ ! $(echo ${json} | jq -r '.entrance.history_size') == "1" ]; then
   echo "The history doesn't have a wallet, it's a bug"
   exit -1
 fi
-if [ ! $(echo ${json} | jq -r '.wallets') == "1" ]; then
-  echo "The wallet is not there for some reason, it's a bug"
-  exit -1
-fi

data/lib/zold/atomic_file.rb

@@ -41,6 +41,7 @@ module Zold
 
     def write(content)
       raise 'Content can\'t be nil' if content.nil?
+      FileUtils.mkdir_p(File.dirname(@file))
       @mutex.synchronize do
         File.open(@file, 'wb') do |f|
           f.write(content)

data/lib/zold/node/async_entrance.rb

@@ -35,6 +35,9 @@ module Zold
     # How many threads to use for processing
     THREADS = Concurrent.processor_count * 8
 
+    # Queue length
+    MAX_QUEUE = Concurrent.processor_count * 64
+
     # Max items in the queue. If there will be more, push() requests
     # will be rejected.
     MAX_QUEUE = 128
@@ -65,7 +68,7 @@ module Zold
       @entrance.start do
         FileUtils.mkdir_p(@dir)
         @pool = Concurrent::FixedThreadPool.new(
-          AsyncEntrance::THREADS, max_queue: AsyncEntrance::THREADS, fallback_policy: :abort
+          AsyncEntrance::THREADS, max_queue: AsyncEntrance::MAX_QUEUE, fallback_policy: :abort
         )
         AsyncEntrance::THREADS.times do |t|
           @pool.post do

data/lib/zold/node/farm.rb

@@ -57,7 +57,8 @@ module Zold
 
     def to_text
       @threads.map do |t|
-        "#{t.name}: status=#{t.status}; alive=#{t.alive?};\n  #{t.backtrace.join("\n  ")}"
+        trace = t.backtrace || []
+        "#{t.name}: status=#{t.status}; alive=#{t.alive?};\n  #{trace.join("\n  ")}"
       end.join("\n")
     end
 

data/lib/zold/node/front.rb

@@ -25,6 +25,7 @@ STDOUT.sync = true
 require 'json'
 require 'sinatra/base'
 require 'webrick'
+require 'cachy'
 require 'get_process_mem'
 require 'diffy'
 require 'usagewatch_ext'
@@ -166,9 +167,8 @@ while #{settings.address} is in '#{settings.network}'"
         memory: GetProcessMem.new.bytes.to_i,
         platform: RUBY_PLATFORM,
         load: Usagewatch.uw_load.to_f,
-        uptime: `uptime`.strip,
         threads: "#{Thread.list.select { |t| t.status == 'run' }.count}/#{Thread.list.count}",
-        wallets: settings.wallets.all.count,
+        wallets: Cachy.cache(:a_key, expires_in: 5 * 60) { settings.wallets.all.count },
         remotes: settings.remotes.all.count,
         nscore: settings.remotes.all.map { |r| r[:score] }.inject(&:+) || 0,
         farm: settings.farm.to_json,

data/lib/zold/sync_wallets.rb

@@ -59,10 +59,10 @@ module Zold
             break if lock.flock(File::LOCK_EX | File::LOCK_NB)
             sleep 0.1
             cycles += 1
-            delay = Time.now - start
+            delay = (Time.now - start).round(2)
             if delay > @timeout
               raise "##{Process.pid}/#{Thread.current.name} can't get exclusive access to the wallet #{id} \
-because of the lock at #{lock.path}: #{File.read(lock)}"
+because of the lock at #{lock.path}, after #{delay}s of waiting: #{File.read(lock)}"
             end
             if (cycles % 20).zero? && delay > 10
               @log.info("##{Process.pid}/#{Thread.current.name} still waiting for \

data/lib/zold/tree_wallets.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+# Copyright (c) 2018 Yegor Bugayenko
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the 'Software'), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+require 'pathname'
+require_relative 'id'
+require_relative 'wallet'
+
+# The local collection of wallets.
+# Author:: Yegor Bugayenko (yegor256@gmail.com)
+# Copyright:: Copyright (c) 2018 Yegor Bugayenko
+# License:: MIT
+module Zold
+  # Collection of local wallets, in a tree of directories
+  class TreeWallets
+    def initialize(dir)
+      @dir = dir
+    end
+
+    def to_s
+      path
+    end
+
+    def path
+      FileUtils.mkdir_p(@dir)
+      File.expand_path(@dir)
+    end
+
+    # Returns the list of their IDs (as plain text)
+    def all
+      Dir.glob("#{path}/**/*#{Wallet::EXTENSION}").select do |f|
+        basename = File.basename(f, Wallet::EXTENSION)
+        File.file?(f) &&
+          !File.directory?(f) &&
+          basename =~ /^[0-9a-fA-F]{16}$/ &&
+          Id.new(basename).to_s == basename
+      end.map { |w| File.basename(w, Wallet::EXTENSION) }
+    end
+
+    def find(id)
+      raise 'Id can\'t be nil' if id.nil?
+      raise 'Id must be of type Id' unless id.is_a?(Id)
+      yield Zold::Wallet.new(File.join(path, (id.to_s.split('', 5).take(4) + [id.to_s]).join('/')))
+    end
+  end
+end

data/lib/zold/version.rb

@@ -25,6 +25,6 @@
 # Copyright:: Copyright (c) 2018 Yegor Bugayenko
 # License:: MIT
 module Zold
-  VERSION = '0.14.36'
+  VERSION = '0.14.37'
   PROTOCOL = 2
 end

data/test/test__helper.rb

@@ -24,6 +24,8 @@ gem 'openssl'
 require 'openssl'
 require 'minitest/autorun'
 require 'concurrent'
+require 'cachy'
+require 'moneta'
 
 STDOUT.sync = true
 
@@ -36,6 +38,8 @@ if ENV['CI'] == 'true'
   SimpleCov.formatter = SimpleCov::Formatter::Codecov
 end
 
+Cachy.cache_store = Moneta.new(:Memory)
+
 module Minitest
   class Test
     def assert_wait(max: 30)

data/test/test_tree_wallets.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+# Copyright (c) 2018 Yegor Bugayenko
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the 'Software'), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+require 'minitest/autorun'
+require 'tmpdir'
+require_relative '../lib/zold/key'
+require_relative '../lib/zold/id'
+require_relative '../lib/zold/tree_wallets'
+
+# TreeWallets test.
+# Author:: Yegor Bugayenko (yegor256@gmail.com)
+# Copyright:: Copyright (c) 2018 Yegor Bugayenko
+# License:: MIT
+class TestTreeWallets < Minitest::Test
+  def test_adds_wallet
+    Dir.mktmpdir do |dir|
+      wallets = Zold::TreeWallets.new(dir)
+      id = Zold::Id.new('abcd0123abcd0123')
+      wallets.find(id) do |wallet|
+        wallet.init(id, Zold::Key.new(file: 'fixtures/id_rsa.pub'))
+        assert_equal(1, wallets.all.count)
+        assert_equal(id.to_s, wallets.all[0])
+        assert(wallet.path.end_with?('/a/b/c/d/abcd0123abcd0123.z'), wallet.path)
+      end
+    end
+  end
+end

data/upgrades/move_wallets_into_tree.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+# Copyright (c) 2018 Yegor Bugayenko
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the 'Software'), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+require 'fileutils'
+require_relative '../lib/zold/version'
+require_relative '../lib/zold/wallet'
+
+module Zold
+  # Move wallets into tree
+  class MoveWalletsIntoTree
+    def initialize(home, log)
+      @home = home
+      @log = log
+    end
+
+    def exec
+      Dir.new(@home).each do |path|
+        next unless path =~ /^[a-f0-9]{16}#{Wallet::EXTENSION}$/
+        f = File.join(@home, path)
+        target = File.join(@home, (path.split('', 5).take(4) + [path]).join('/'))
+        FileUtils.mkdir_p(File.dirname(target))
+        FileUtils.mv(f, target)
+        @log.info("Wallet #{path} moved to #{target}")
+      end
+    end
+  end
+end

data/zold.gemspec

@@ -54,6 +54,7 @@ and suggests a different architecture for digital wallet maintenance.'
   s.test_files = s.files.grep(%r{^(test|features)/})
   s.rdoc_options = ['--charset=UTF-8']
   s.extra_rdoc_files = ['README.md', 'LICENSE.txt']
+  s.add_runtime_dependency 'cachy', '0.4.2'
   s.add_runtime_dependency 'concurrent-ruby', '~>1.0'
   s.add_runtime_dependency 'cucumber', '~>3.1' # has to stay here for Heroku
   s.add_runtime_dependency 'diffy', '~>3.2'
@@ -61,6 +62,7 @@ and suggests a different architecture for digital wallet maintenance.'
   s.add_runtime_dependency 'dry-types', '~>0.13.2'
   s.add_runtime_dependency 'get_process_mem', '0.2.2'
   s.add_runtime_dependency 'json', '~>1.8'
+  s.add_runtime_dependency 'moneta', '1.0.0'
   s.add_runtime_dependency 'openssl', '~>2.1'
   s.add_runtime_dependency 'rainbow', '~>3.0'
   s.add_runtime_dependency 'rake', '~>12.3' # has to stay here for Heroku

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: zold
 version: !ruby/object:Gem::Version
-  version: 0.14.36
+  version: 0.14.37
 platform: ruby
 authors:
 - Yegor Bugayenko
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-08-16 00:00:00.000000000 Z
+date: 2018-08-17 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: cachy
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.4.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.4.2
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
   requirement: !ruby/object:Gem::Requirement
@@ -108,6 +122,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.8'
+- !ruby/object:Gem::Dependency
+  name: moneta
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
 - !ruby/object:Gem::Dependency
   name: openssl
   requirement: !ruby/object:Gem::Requirement
@@ -481,6 +509,7 @@ files:
 - lib/zold/signature.rb
 - lib/zold/sync_wallets.rb
 - lib/zold/tax.rb
+- lib/zold/tree_wallets.rb
 - lib/zold/txn.rb
 - lib/zold/type.rb
 - lib/zold/upgrades.rb
@@ -539,6 +568,7 @@ files:
 - test/test_signature.rb
 - test/test_sync_wallets.rb
 - test/test_tax.rb
+- test/test_tree_wallets.rb
 - test/test_txn.rb
 - test/test_upgrades.rb
 - test/test_verbose_thread.rb
@@ -548,6 +578,7 @@ files:
 - test/test_zold.rb
 - test/upgrades/test_protocol_up.rb
 - upgrades/2.rb
+- upgrades/move_wallets_into_tree.rb
 - upgrades/protocol_up.rb
 - upgrades/rename_foreign_wallets.rb
 - zold.gemspec
@@ -630,6 +661,7 @@ test_files:
 - test/test_signature.rb
 - test/test_sync_wallets.rb
 - test/test_tax.rb
+- test/test_tree_wallets.rb
 - test/test_txn.rb
 - test/test_upgrades.rb
 - test/test_verbose_thread.rb