zold 0.17.4 → 0.17.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 622a8738f318299d195dd0efbe8cdb8bc41aac3432a964b775a31a1b78cc592f
-  data.tar.gz: 06ed79ff0550b5e4f66eb28af66051b14b3d840caf4ea936cfd40b36e6693124
+  metadata.gz: c06af13af0d59a21c9855dc36c17c13cfe838286f7aadcb1ac3b130ed934d65d
+  data.tar.gz: 4349ebaee96c42a6224b92a4fc9a398f28d0f7c07352a34fd965d63bbf00b706
 SHA512:
-  metadata.gz: d0e4e8282e87e826b31e3662f0139aa4e0634c4d5e7d2cdb13b48a8958bb48557c007605a4f07d79f50b678acf7beab1860ddb1283b2819b00b66f208ab3b850
-  data.tar.gz: 9a4fb162a36f9db69b3c5916ffca1658c75ea3450b975e05d29054a6b6da991e46e4bdb6c201394a724dcf1461284e1d1a101a000a09bdfebec2420a5c9957f5
+  metadata.gz: 69b146cdf8bcc399e5213e1c8af4faa2aba937fd93a96ee845a0c9552ffbfe24fcb34d37c724298319aeddfde91ab2b30546f9afcb517232959000893ef40b4b
+  data.tar.gz: 82943553b346434e975041142b6ac21ffe301bae24695dc509613ef9b8fd8c0a625003f1b060360550b3eab1076f3cd42ad02a7f5de7b7d405f09534f36a3c10

data/lib/zold/commands/node.rb

@@ -41,6 +41,7 @@ require_relative '../node/spread_entrance'
 require_relative '../node/async_entrance'
 require_relative '../node/sync_entrance'
 require_relative '../node/nodup_entrance'
+require_relative '../node/nospam_entrance'
 require_relative '../node/front'
 require_relative '../node/trace'
 require_relative '../node/farm'
@@ -123,6 +124,9 @@ module Zold
         o.bool '--no-cache',
           'Skip caching of front JSON pages (will seriously slow down, mostly useful for testing)',
           default: false
+        o.bool '--allow-spam',
+          'Don\'t filter the incoming spam via PUT requests (duplicate wallets)',
+          default: false
         o.bool '--skip-oom',
           'Skip Out Of Memory check and never exit, no matter how much RAM is consumed',
           default: false
@@ -225,27 +229,32 @@ module Zold
         ).run(['invoice', invoice, "--network=#{opts['network']}"])
       end
       SafeEntrance.new(
-        NoDupEntrance.new(
-          AsyncEntrance.new(
-            SpreadEntrance.new(
-              SyncEntrance.new(
-                Entrance.new(
-                  @wallets,
-                  @remotes, @copies, address,
-                  log: @log, network: opts['network']
+        NoSpamEntrance.new(
+          NoDupEntrance.new(
+            AsyncEntrance.new(
+              SpreadEntrance.new(
+                SyncEntrance.new(
+                  Entrance.new(
+                    @wallets,
+                    @remotes, @copies, address,
+                    log: @log, network: opts['network']
+                  ),
+                  File.join(home, '.zoldata/sync-entrance'),
+                  log: @log
                 ),
-                File.join(home, '.zoldata/sync-entrance'),
-                log: @log
+                @wallets, @remotes, address,
+                log: @log,
+                ignore_score_weakeness: opts['ignore-score-weakness']
               ),
-              @wallets, @remotes, address,
+              File.join(home, '.zoldata/async-entrance'),
               log: @log,
-              ignore_score_weakeness: opts['ignore-score-weakness']
+              queue_limit: opts['queue-limit']
             ),
-            File.join(home, '.zoldata/async-entrance'),
-            log: @log,
-            queue_limit: opts['queue-limit']
+            @wallets,
+            log: @log
           ),
-          @wallets
+          period: opts['allow-spam'] ? 0 : 60 * 60,
+          log: @log
         ),
         network: opts['network']
       ).start do |entrance|

data/lib/zold/dir_items.rb

@@ -21,7 +21,6 @@
 # SOFTWARE.
 
 require 'shellwords'
-require 'posix/spawn'
 
 # Items in a directory.
 #
@@ -40,12 +39,7 @@ module Zold
     end
 
     def fetch(recursive: true)
-      spawn = POSIX::Spawn::Child.new(
-        'find',
-        *([@dir, '-type', 'f', '-print'] + (recursive ? [] : ['-maxdepth', '1']))
-      )
-      raise spawn.err unless spawn.status.success?
-      spawn.out
+      `find #{([@dir, '-type', 'f', '-print'] + (recursive ? [] : ['-maxdepth', '1'])).join(' ')}`
         .strip
         .split(' ')
         .select { |f| f.start_with?(@dir) && f.length > @dir.length }

data/lib/zold/node/farmers.rb

@@ -24,7 +24,6 @@ require 'open3'
 require 'backtrace'
 require 'zold/score'
 require 'shellwords'
-require 'posix/spawn'
 require_relative '../log'
 require_relative '../age'
 
@@ -54,67 +53,6 @@ module Zold
       end
     end
 
-    # In a child process
-    class Spawn
-      def initialize(log: Log::NULL)
-        @log = log
-      end
-
-      def up(score)
-        raise "We are farming the score already: #{score}" if
-          POSIX::Spawn::Child.new('ps', 'ax').out.include?(score.to_s.split(' ').take(4).join(' '))
-        start = Time.now
-        bin = File.expand_path(File.join(File.dirname(__FILE__), '../../../bin/zold'))
-        raise "Zold binary not found at #{bin}" unless File.exist?(bin)
-        cmd = [
-          'ruby',
-          Shellwords.escape(bin),
-          '--skip-upgrades',
-          "--info-tid=#{Thread.current.thread_variable_get(:tid)}",
-          "--info-thread=#{Shellwords.escape(Thread.current.name)}",
-          "--info-start=#{Time.now.utc.iso8601}",
-          '--low-priority',
-          'next',
-          Shellwords.escape(score)
-        ].join(' ')
-        Open3.popen2e(cmd) do |stdin, stdout, thr|
-          Thread.current.thread_variable_set(:pid, thr.pid.to_s)
-          at_exit { Farmers.kill(@log, thr.pid, start) }
-          @log.debug("Scoring started in proc ##{thr.pid} \
-for #{score.value}/#{score.strength} at #{score.host}:#{score.port}")
-          begin
-            stdin.close
-            buffer = +''
-            loop do
-              begin
-                buffer << stdout.read_nonblock(16 * 1024)
-                # rubocop:disable Lint/HandleExceptions
-              rescue IO::WaitReadable => _
-                # rubocop:enable Lint/HandleExceptions
-                # nothing to do here
-              rescue StandardError => e
-                @log.error(buffer)
-                raise e
-              end
-              break if buffer.end_with?("\n") && thr.value.to_i.zero?
-              if stdout.closed?
-                raise "Failed to calculate the score (##{thr.value}): #{buffer}" unless thr.value.to_i.zero?
-                break
-              end
-              sleep(1)
-              Thread.current.thread_variable_set(:buffer, buffer.length.to_s)
-            end
-            after = Score.parse(buffer.strip)
-            @log.debug("Next score #{after.value}/#{after.strength} found in proc ##{thr.pid} \
-for #{after.host}:#{after.port} in #{Age.new(start)}: #{after.suffixes}")
-            after
-          ensure
-            Farmers.kill(@log, thr.pid, start)
-          end
-        end
-      end
-    end
-
     # In a child process using fork
     class Fork
       def initialize(log: Log::NULL)

data/lib/zold/node/front.rb

@@ -29,7 +29,6 @@ require 'sinatra/base'
 require 'concurrent'
 require 'backtrace'
 require 'zache'
-require 'posix/spawn'
 require_relative '../version'
 require_relative '../size'
 require_relative '../wallet'
@@ -313,7 +312,7 @@ in #{Age.new(@start, limit: 1)}")
           "Balance: #{wallet.balance.to_zld(8)} ZLD (#{wallet.balance.to_i} zents)",
           "Transactions: #{wallet.txns.count}",
           "Taxes: #{Tax.new(wallet).paid} paid, the debt is #{Tax.new(wallet).debt}",
-          "File size: #{wallet.size} bytes (#{Copies.new(File.join(settings.copies, wallet.id)).all.count} copies)",
+          "File size: #{Size.new(wallet.size)}, #{Copies.new(File.join(settings.copies, wallet.id)).all.count} copies",
           "Modified: #{wallet.mtime.utc.iso8601} (#{Age.new(wallet.mtime.utc.iso8601)} ago)",
           "Digest: #{wallet.digest}"
         ].join("\n")
@@ -458,10 +457,7 @@ in #{Age.new(@start, limit: 1)}")
     end
 
     def processes
-      []
-      # This is temporarily disabled. We suspect that this line causes
-      # memory leakage.
-      # POSIX::Spawn::Child.new('ps', 'ax').out.split("\n").select { |t| t.include?('zold') }
+      `ps ax`.split("\n").select { |t| t.include?('zold') }
     end
 
     def pretty(json)

data/lib/zold/node/nodup_entrance.rb

@@ -31,7 +31,7 @@ require_relative '../wallet'
 # Copyright:: Copyright (c) 2018 Yegor Bugayenko
 # License:: MIT
 module Zold
-  # The safe entrance
+  # The entrance that ignores dups
   class NoDupEntrance
     def initialize(entrance, wallets, log: Log::NULL)
       @entrance = entrance
@@ -53,10 +53,10 @@ module Zold
       before = @wallets.acq(id) { |w| w.exists? ? w.digest : '' }
       after = OpenSSL::Digest::SHA256.new(body).hexdigest
       if before == after
-        @log.debug("Duplicate of #{id} ignored (#{Size.new(body.length)} bytes)")
+        @log.debug("Duplicate of #{id} ignored #{Size.new(body.length)}")
         return []
       end
-      @log.debug("New content for #{id} arrived (#{Size.new(body.length)} bytes)")
+      @log.debug("New content for #{id} arrived #{Size.new(body.length)}")
       @entrance.push(id, body)
     end
   end

data/lib/zold/node/nospam_entrance.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+# Copyright (c) 2018 Yegor Bugayenko
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the 'Software'), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+require 'tempfile'
+require 'openssl'
+require 'zache'
+require_relative '../log'
+require_relative '../size'
+
+# The entrance that ignores something we've seen already.
+# Author:: Yegor Bugayenko (yegor256@gmail.com)
+# Copyright:: Copyright (c) 2018 Yegor Bugayenko
+# License:: MIT
+module Zold
+  # The no-spam entrance
+  class NoSpamEntrance
+    def initialize(entrance, period: 60 * 60, log: Log::NULL)
+      @entrance = entrance
+      @log = log
+      @period = period
+      @zache = Zache.new
+    end
+
+    def start
+      raise 'Block must be given to start()' unless block_given?
+      @entrance.start { yield(self) }
+    end
+
+    def to_json
+      @entrance.to_json
+    end
+
+    # Returns a list of modifed wallets (as Zold::Id)
+    def push(id, body)
+      before = @zache.get(id.to_s, lifetime: @period) { '' }
+      after = hash(id, body)
+      if before == after
+        @log.debug("Spam of #{id} ignored #{Size.new(body.length)}")
+        return []
+      end
+      @zache.put(id.to_s, after)
+      @entrance.push(id, body)
+    end
+
+    private
+
+    def hash(id, body)
+      OpenSSL::Digest::SHA256.new(id.to_s + ' ' + body).hexdigest
+    end
+  end
+end

data/lib/zold/version.rb

@@ -25,6 +25,6 @@
 # Copyright:: Copyright (c) 2018 Yegor Bugayenko
 # License:: MIT
 module Zold
-  VERSION = '0.17.4'
+  VERSION = '0.17.5'
   PROTOCOL = 2
 end

data/test/node/fake_node.rb

@@ -79,6 +79,7 @@ class FakeNode
         ensure
           Zold::Front.stop!
           node.join
+          sleep(0.25)
         end
       end
     end

data/test/node/test_farmers.rb

@@ -30,7 +30,7 @@ require_relative '../../lib/zold/verbose_thread'
 class FarmersTest < Zold::Test
   def test_calculates_next_score
     before = Zold::Score.new(host: 'some-host', port: 9999, invoice: 'NOPREFIX4@ffffffffffffffff', strength: 3)
-    [Zold::Farmers::Plain, Zold::Farmers::Spawn, Zold::Farmers::Fork].each do |farmer_class|
+    [Zold::Farmers::Plain, Zold::Farmers::Fork].each do |farmer_class|
       farmer = farmer_class.new(log: test_log)
       after = farmer.up(before)
       assert_equal(1, after.value)
@@ -41,21 +41,20 @@ class FarmersTest < Zold::Test
   end
 
   def test_calculates_large_score
-    log = TestLogger.new(test_log)
-    thread = Thread.start do
-      [Zold::Farmers::Spawn, Zold::Farmers::Fork].each do |farmer_class|
-        farmer = farmer_class.new(log: log)
+    [Zold::Farmers::Plain, Zold::Farmers::Fork].each do |type|
+      log = TestLogger.new(test_log)
+      thread = Thread.start do
+        farmer = type.new(log: log)
         farmer.up(Zold::Score.new(host: 'a', port: 1, invoice: 'NOPREFIX4@ffffffffffffffff', strength: 20))
       end
+      sleep(0.1)
+      thread.kill
+      thread.join
     end
-    assert_wait { !log.msgs.find { |m| m.include?('Scoring started') }.nil? }
-    thread.kill
-    thread.join
-    assert(log.msgs.find { |m| m.include?('killed') })
   end
 
   def test_kills_farmer
-    [Zold::Farmers::Plain, Zold::Farmers::Spawn, Zold::Farmers::Fork].each do |type|
+    [Zold::Farmers::Plain, Zold::Farmers::Fork].each do |type|
       farmer = type.new(log: test_log)
       thread = Thread.start do
         Zold::VerboseThread.new(test_log).run do
@@ -67,25 +66,4 @@ class FarmersTest < Zold::Test
       thread.join
     end
   end
-
-  def test_avoid_duplicate_processes
-    log = TestLogger.new(test_log)
-    time = Time.now
-    thread = Thread.start do
-      [Zold::Farmers::Spawn, Zold::Farmers::Fork].each do |farmer_class|
-        farmer = farmer_class.new(log: log)
-        farmer.up(Zold::Score.new(time: time, host: 'a', port: 1, invoice: 'prefixone@ffffffffffffffff', strength: 20))
-      end
-    end
-    assert_wait { !log.msgs.find { |m| m.include?('Scoring started') }.nil? }
-    assert_raises do
-      [Zold::Farmers::Spawn, Zold::Farmers::Fork].each do |farmer_class|
-        farmer_class.new(log: log).up(
-          Zold::Score.new(time: time, host: 'a', port: 1, invoice: 'prefixtwo@ffffffffffffffff', strength: 20)
-        )
-      end
-    end
-    thread.kill
-    thread.join
-  end
 end

data/test/node/test_nodup_entrance.rb

@@ -31,7 +31,7 @@ require_relative 'fake_entrance'
 # Author:: Yegor Bugayenko (yegor256@gmail.com)
 # Copyright:: Copyright (c) 2018 Yegor Bugayenko
 # License:: MIT
-class TestAsyncEntrance < Zold::Test
+class TestNoDupEntrance < Zold::Test
   def test_ignores_dup
     FakeHome.new(log: test_log).run do |home|
       wallet = home.create_wallet

data/test/node/test_nospam_entrance.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+# Copyright (c) 2018 Yegor Bugayenko
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the 'Software'), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+require 'minitest/autorun'
+require_relative '../test__helper'
+require_relative '../../lib/zold/id'
+require_relative '../../lib/zold/node/nospam_entrance'
+require_relative 'fake_entrance'
+
+# NoSpamEntrance test.
+# Author:: Yegor Bugayenko (yegor256@gmail.com)
+# Copyright:: Copyright (c) 2018 Yegor Bugayenko
+# License:: MIT
+class TestNoSpamEntrance < Zold::Test
+  def test_ignores_spam
+    Zold::NoSpamEntrance.new(RealEntrance.new, log: test_log).start do |e|
+      id = Zold::Id.new
+      content = 'hello'
+      assert(!e.push(id, content).empty?)
+      assert(e.push(id, content).empty?)
+      assert(e.push(id, content).empty?)
+    end
+  end
+
+  class RealEntrance < FakeEntrance
+    def push(id, _)
+      [id]
+    end
+  end
+end

data/zold.gemspec

@@ -71,7 +71,6 @@ and suggests a different architecture for digital wallet maintenance.'
   s.add_runtime_dependency 'memory_profiler', '0.9.12'
   s.add_runtime_dependency 'openssl', '2.1.2'
   s.add_runtime_dependency 'parallel', '1.12.1'
-  s.add_runtime_dependency 'posix-spawn', '0.3.13'
   s.add_runtime_dependency 'rainbow', '3.0.0'
   s.add_runtime_dependency 'rake', '12.3.1' # has to stay here for Heroku
   s.add_runtime_dependency 'rubocop', '0.60.0' # has to stay here for Heroku
@@ -85,7 +84,7 @@ and suggests a different architecture for digital wallet maintenance.'
   s.add_runtime_dependency 'typhoeus', '1.3.1'
   s.add_runtime_dependency 'usagewatch_ext', '0.2.1'
   s.add_runtime_dependency 'xcop', '0.6'
-  s.add_runtime_dependency 'zache', '0.3.1'
+  s.add_runtime_dependency 'zache', '0.4.0'
   s.add_runtime_dependency 'zold-score', '0.3.4'
   s.add_development_dependency 'codecov', '0.1.13'
   s.add_development_dependency 'minitest', '5.11.3'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: zold
 version: !ruby/object:Gem::Version
-  version: 0.17.4
+  version: 0.17.5
 platform: ruby
 authors:
 - Yegor Bugayenko
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-11-30 00:00:00.000000000 Z
+date: 2018-12-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: backtrace
@@ -150,20 +150,6 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 1.12.1
-- !ruby/object:Gem::Dependency
-  name: posix-spawn
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 0.3.13
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 0.3.13
 - !ruby/object:Gem::Dependency
   name: rainbow
   requirement: !ruby/object:Gem::Requirement
@@ -352,14 +338,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.3.1
+        version: 0.4.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.3.1
+        version: 0.4.0
 - !ruby/object:Gem::Dependency
   name: zold-score
   requirement: !ruby/object:Gem::Requirement
@@ -615,6 +601,7 @@ files:
 - lib/zold/node/farmers.rb
 - lib/zold/node/front.rb
 - lib/zold/node/nodup_entrance.rb
+- lib/zold/node/nospam_entrance.rb
 - lib/zold/node/safe_entrance.rb
 - lib/zold/node/spread_entrance.rb
 - lib/zold/node/sync_entrance.rb
@@ -667,6 +654,7 @@ files:
 - test/node/test_farmers.rb
 - test/node/test_front.rb
 - test/node/test_nodup_entrance.rb
+- test/node/test_nospam_entrance.rb
 - test/node/test_safe_entrance.rb
 - test/node/test_spread_entrance.rb
 - test/node/test_sync_entrance.rb
@@ -711,7 +699,7 @@ licenses:
 - MIT
 metadata: {}
 post_install_message: |-
-  Thanks for installing Zold 0.17.4!
+  Thanks for installing Zold 0.17.5!
     Study our White Paper: https://papers.zold.io/wp.pdf
     Read our blog posts: https://blog.zold.io
     Try online wallet at: https://wts.zold.io
@@ -773,6 +761,7 @@ test_files:
 - test/node/test_farmers.rb
 - test/node/test_front.rb
 - test/node/test_nodup_entrance.rb
+- test/node/test_nospam_entrance.rb
 - test/node/test_safe_entrance.rb
 - test/node/test_spread_entrance.rb
 - test/node/test_sync_entrance.rb