miner_mover 0.0.0.3

data/demo/ractor.rb

@@ -0,0 +1,117 @@
+require 'miner_mover/run'
+require 'thread'
+
+include MinerMover
+
+run = Run.new.cfg_banner!(duration: 1)
+run.timer.timestamp!
+run.log "Starting"
+
+stop_mining = false
+Signal.trap("INT") {
+  run.timer.timestamp!
+  run.log " *** SIGINT ***  Stop Mining"
+  stop_mining = true
+}
+
+# the moving operation executes in its own Ractor
+mover = Ractor.new(run) { |r|
+  r.log "MOVE Moving operation started"
+
+  # use queue to distribute incoming ore to mover threads
+  queue = Thread::Queue.new
+
+  # store the mover threads in an array
+  movers = Array.new(r.num_movers) { |i|
+    Thread.new {
+      m = r.new_mover
+      m.log "MOVE Mover #{i} started"
+
+      loop {
+        # a mover picks up ore from the queue
+        r.debug && m.log("POP ")
+        ore = queue.pop
+        r.debug && m.log("POPD #{ore}")
+
+        break if ore == :quit
+
+        # load (and possibly move) the ore
+        m.load_ore ore
+      }
+
+      # move any remaining ore and quit
+      m.move_batch while m.batch > 0
+      m.log "QUIT #{m.status}"
+      m
+    }
+  }
+
+  # Miners feed this Ractor with ore
+  # Pass the ore into a queue for the movers
+  # When the miners say to quit, tell the movers to quit
+  r.log "WAIT Waiting for ore ..."
+  loop {
+    # when the Ractor gets ore, push it into the queue
+    ore = Ractor.recv
+    r.debug && r.log("RECV #{ore}")
+
+    break if ore == :quit
+
+    r.debug && r.log("PUSH #{ore}")
+    queue.push ore
+    r.debug && r.log("PSHD #{ore}")
+  }
+
+  # tell all the movers to quit and gather their results
+  r.num_movers.times { queue.push :quit }
+  movers.map { |thr| thr.value.ore_moved }.sum
+}
+
+# our mining operation executes in the main Ractor, here
+run.log "MINE Mining operation started  [ctrl-c] to stop"
+
+# store the miner threads in an array
+miners = Array.new(run.num_miners) { |i|
+  Thread.new {
+    m = run.new_miner
+    m.log "MINE Miner #{i} started"
+    ore_mined = 0
+
+    # miners wait for the SIGINT signal to quit
+    while !stop_mining
+      ore = m.mine_ore
+
+      # send any ore mined to the mover Ractor
+      if ore > 0
+        run.debug && m.log("SEND #{ore}")
+        mover.send ore
+        run.debug && m.log("SENT #{ore}")
+      end
+
+      ore_mined += ore
+
+      # stop mining after a while
+      if run.time_limit? or run.ore_limit?(ore_mined)
+        run.timer.timestamp!
+        m.log format("Mining limit reached: %s", Ore.display(ore_mined))
+        stop_mining = true
+      end
+    end
+
+    m.log format("MINE Miner %i finished after mining %s",
+                 i, Ore.display(ore_mined))
+    ore_mined
+  }
+}
+
+# wait on all mining threads to stop
+ore_mined = miners.map { |thr| thr.value }.sum
+run.log format("MINE %s mined (%i)", Ore.display(ore_mined), ore_mined)
+
+# tell mover to quit
+mover.send :quit
+
+# wait for results
+ore_moved = mover.take
+run.log format("MOVE %s moved (%i)", Ore.display(ore_moved), ore_moved)
+run.timer.timestamp!

data/demo/serial.rb

@@ -0,0 +1,50 @@
+require 'miner_mover/run'
+
+include MinerMover
+
+run = Run.new.cfg_banner!(duration: 1)
+run.timer.timestamp!
+run.log "Starting"
+
+stop_mining = false
+Signal.trap("INT") {
+  run.timer.timestamp!
+  run.log " *** SIGINT ***  Stop Mining"
+  stop_mining = true
+}
+
+# system 'cpulimit', "--pid=#{Process.pid}", '--limit=1', '--background'
+
+miner = run.new_miner
+run.log "MINE Mining operation initialized  [ctrl-c] to stop"
+
+mover = run.new_mover
+run.log "MOVE Moving operation initialized"
+
+ore_mined = 0
+
+# miner waits for the SIGINT signal to quit
+while !stop_mining
+  # mine the ore
+  ore = miner.mine_ore
+  ore_mined += ore
+
+  # load (and possibly move) the ore
+  mover.load_ore ore if ore > 0
+
+  # stop mining after a while
+  if run.time_limit? or run.ore_limit?(ore_mined)
+    run.timer.timestamp!
+    miner.log format("Mining limit reached: %s", Ore.display(ore_mined))
+    stop_mining = true
+  end
+end
+
+# miner has quit; move any remaining ore and quit
+mover.move_batch while mover.batch > 0
+run.log "QUIT #{mover.status}"
+
+ore_moved = mover.ore_moved
+run.log format("MINE %s mined (%i)", Ore.display(ore_mined), ore_mined)
+run.log format("MOVE %s moved (%i)", Ore.display(ore_moved), ore_moved)
+run.timer.timestamp!

data/demo/thread.rb

@@ -0,0 +1,91 @@
+require 'miner_mover/run'
+require 'thread'
+
+include MinerMover
+
+run = Run.new.cfg_banner!(duration: 1)
+run.timer.timestamp!
+run.log "Starting"
+
+stop_mining = false
+Signal.trap("INT") {
+  run.timer.timestamp!
+  run.log " *** SIGINT ***  Stop Mining"
+  stop_mining = true
+}
+
+run.log "MOVE Moving operation started"
+queue = Thread::Queue.new
+run.log "WAIT Waiting for ore ..."
+
+# store mover threads in an array
+movers = Array.new(run.num_movers) { |i|
+  Thread.new {
+    m = run.new_mover
+    run.log "MOVE Mover #{i} started"
+
+    loop {
+      # a mover picks up mined ore from the queue
+      run.debug && m.log("POP ")
+      ore = queue.pop
+      run.debug && m.log("POPD #{ore}")
+
+      break if ore == :quit
+
+      # load (and possibly move) the ore
+      m.load_ore ore
+    }
+
+    # move any remaining ore and quit
+    m.move_batch while m.batch > 0
+    m.log "QUIT #{m.status}"
+    m
+  }
+}
+
+
+run.log "MINE Mining operation started  [ctrl-c] to stop"
+# store the miner threads in an array
+miners = Array.new(run.num_miners) { |i|
+  Thread.new {
+    m = run.new_miner
+    m.log "MINE Miner #{i} started"
+    ore_mined = 0
+
+    # miners wait for the SIGINT signal to quit
+    while !stop_mining
+      ore = m.mine_ore
+
+      # send any ore mined to the movers
+      if ore > 0
+        run.debug && m.log("PUSH #{ore}")
+        queue.push ore
+        run.debug && m.log("PSHD #{ore}")
+      end
+
+      ore_mined += ore
+
+      # stop mining after a while
+      if run.time_limit? or run.ore_limit?(ore_mined)
+        run.timer.timestamp!
+        m.log format("Mining limit reached: %s", Ore.display(ore_mined))
+        stop_mining = true
+      end
+    end
+
+    m.log format("MINE Miner %i finished after mining %s",
+                 i, Ore.display(ore_mined))
+    ore_mined
+  }
+}
+
+# wait on all mining threads to stop
+ore_mined = miners.map { |thr| thr.value }.sum
+run.log format("MINE %s mined (%i)", Ore.display(ore_mined), ore_mined)
+
+# tell all the movers to quit; gather their results
+run.num_movers.times { queue.push :quit }
+
+ore_moved = movers.map { |thr| thr.value.ore_moved }.sum
+run.log format("MOVE %s moved (%i)", Ore.display(ore_moved), ore_moved)
+run.timer.timestamp!

data/lib/miner_mover/config.rb

@@ -0,0 +1,78 @@
+require 'dotcfg'
+
+module MinerMover
+  module Config
+    class Error < RuntimeError; end
+
+    GLOB = '*/*.cfg'.freeze
+
+    # reasonable defaults for all known keys
+    DEFAULT = {
+      main: {
+        num_miners: 3,
+        num_movers: 3,
+        time_limit: 5,
+        ore_limit: 100,
+        logging: true,
+      }.freeze,
+      miner: {
+        depth: 30,
+        partial_reward: false,
+        variance: 0,
+        logging: true,
+      }.freeze,
+      mover: {
+        batch_size: 10,
+        rate: 2,
+        work_type: :wait,
+        variance: 0,
+        logging: true,
+      }.freeze,
+    }.freeze
+
+    # return an array of strings representing file paths
+    def self.gather(*globs)
+      (globs.unshift GLOB).inject([]) { |memo, glob| memo + Dir[glob] }
+    end
+
+    # return a file path as a string, or nil
+    def self.recent(*globs)
+      mtime, newest = Time.at(0), nil
+      self.gather(*globs).each { |file|
+        mt = File.mtime(file)
+        mtime, newest = mt, file if mt > mtime
+      }
+      newest
+    end
+
+    # return a hash with :miner, :mover, :main keys
+    def self.process(file = nil, cfg: nil)
+      cfg ||= DotCfg.new(file || self.recent)
+
+      if cfg['miner'] or cfg['mover'] or cfg['main']
+        # convert string keys to symbols
+        miner = (cfg['miner'] || {}).transform_keys { |k| k.to_sym }
+        mover = (cfg['mover'] || {}).transform_keys { |k| k.to_sym }
+        main  = (cfg['main']  || {}).transform_keys { |k| k.to_sym }
+      elsif cfg[:miner] or cfg[:mover] or cfg[:main]
+        # assume all keys are symbols
+        miner = cfg[:miner] || {}
+        mover = cfg[:mover] || {}
+        main  = cfg[:main]  || {}
+      else
+        raise(Error, "couldn't find miner, mover, or main in #{file}")
+      end
+      { miner: DEFAULT[:miner].merge(miner),
+        mover: DEFAULT[:mover].merge(mover),
+        main:  DEFAULT[:main].merge(main) }
+    end
+
+    # rewrites the dotcfg file, filling in any defaults, using symbols for keys
+    def self.rewrite(file)
+      cfg = DotCfg.new(file)
+      hsh = self.process(cfg: cfg)
+      hsh.each { |k, v| cfg[k] = v }
+      cfg.save
+    end
+  end
+end

data/lib/miner_mover/run.rb

@@ -0,0 +1,67 @@
+require 'miner_mover/worker'
+require 'miner_mover/config'
+
+module MinerMover
+  class Run
+    def self.cfg_file(filename = nil)
+      f = filename || ARGV.shift || Config.recent
+      if f.nil?
+        raise(Config::Error, "no config file")
+      elsif !File.exist? f
+        raise(Config::Error, "can't find file #{f.inspect}")
+      elsif !File.readable? f
+        raise(Config::Error, "can't read file #{f.inspect}")
+      end
+      f
+    end
+
+    attr_accessor :debug, :logging
+    attr_accessor :num_miners, :num_movers
+    attr_accessor :cfg_file, :cfg, :miner, :mover, :timer
+    attr_accessor :time_limit, :ore_limit
+
+    def initialize(cfg_file: nil, timer: nil, debug: false)
+      @cfg_file = self.class.cfg_file(cfg_file)
+      @cfg = Config.process @cfg_file
+      main  = @cfg.fetch :main
+      @miner = @cfg.fetch :miner
+      @mover = @cfg.fetch :mover
+
+      @time_limit = main.fetch :time_limit
+      @ore_limit  = main.fetch :ore_limit
+      @logging    = main.fetch :logging
+      @num_miners = main.fetch :num_miners
+      @num_movers = main.fetch :num_movers
+
+      @timer = timer || CompSci::Timer.new
+      @debug = debug
+    end
+
+    def cfg_banner!(duration: 0)
+      log "USING: #{@cfg_file}"
+      pp @cfg
+      sleep duration if duration > 0
+      self
+    end
+
+    def new_miner
+      Miner.new(**@miner)
+    end
+
+    def new_mover
+      Mover.new(**@mover)
+    end
+
+    def ore_limit?(ore_mined)
+      Ore.block(ore_mined) > @ore_limit
+    end
+
+    def time_limit?
+      @timer.elapsed > @time_limit
+    end
+
+    def log msg
+      @logging and puts(MinerMover.log_fmt(@timer, ' (main) ', msg))
+    end
+  end
+end

data/lib/miner_mover/worker.rb

@@ -0,0 +1,164 @@
+require 'miner_mover'
+require 'compsci/timer'
+require 'compsci/fibonacci'
+
+module MinerMover
+  def self.work(duration, type = :wait, fib = 30)
+    case type
+    when :wait
+      sleep duration
+      duration
+    when :cpu
+      t = CompSci::Timer.new
+      CompSci::Fibonacci.classic(fib) while t.elapsed < duration
+      t.elapsed
+    when :instant
+      0
+    else
+      raise "unknown work type: #{type.inspect}"
+    end
+  end
+
+  class Worker
+    attr_accessor :variance, :logging
+    attr_reader :timer
+
+    def initialize(variance: 0, logging: false, timer: nil)
+      @variance = variance
+      @logging = logging
+      @timer = timer || CompSci::Timer.new
+    end
+
+    def id
+      self.object_id.to_s.rjust(8, '0')
+    end
+
+    def state
+      { id: self.id,
+        logging: @logging,
+        timer: @timer.elapsed_ms.round,
+        variance: @variance }
+    end
+
+    def to_s
+      self.state.to_s
+    end
+
+    def log msg
+      @logging && puts(MinerMover.log_fmt(@timer, self.id, msg))
+    end
+
+    # 4 levels:
+    # 0 - no variance
+    # 1 - 12.5% variance (squeeze = 2)
+    # 2 - 25%   variance (squeeze = 1)
+    # 3 - 50%   variance (squeeze = 0)
+    def varied n
+      case @variance
+      when 0
+        n
+      when 1..3
+        MinerMover.randomize(n, 3 - @variance)
+      else
+        raise "unexpected variance: #{@variance.inspect}"
+      end
+    end
+  end
+
+  class Miner < Worker
+    attr_accessor :depth, :partial_reward
+
+    def initialize(depth: 10,
+                   partial_reward: true,
+                   variance: 0,
+                   logging: false,
+                   timer: nil)
+      @partial_reward = partial_reward
+      @depth = depth
+      super(variance: variance, logging: logging, timer: timer)
+    end
+
+    def state
+      super.merge(depth: @depth, partial_reward: @partial_reward)
+    end
+
+    def mine_ore(depth = @depth)
+      log format("MINE Depth %i", depth)
+      ores, elapsed = CompSci::Timer.elapsed {
+        # every new depth is a new mining operation
+        Array.new(depth) { |d|
+          # mine ore by calculating fibonacci for that depth
+          mined = CompSci::Fibonacci.classic(self.varied(d).round)
+          @partial_reward ? rand(1 + mined) : mined
+        }
+      }
+      total = ores.sum
+      log format("MIND %s %s (%.2f s)",
+                 Ore.display(total), ores.inspect, elapsed)
+      total
+    end
+  end
+
+  class Mover < Worker
+    attr_reader :rate, :work_type, :batch, :batch_size, :batches, :ore_moved
+
+    def initialize(batch_size: 10,
+                   rate: 2,  # 2M ore per sec
+                   work_type: :cpu,
+                   variance: 0,
+                   logging: false,
+                   timer: nil)
+      @batch_size = batch_size * Ore::BLOCK
+      @rate = rate.to_f * Ore::BLOCK
+      @work_type = work_type
+      @batch, @batches, @ore_moved = 0, 0, 0
+      super(variance: variance, logging: logging, timer: timer)
+    end
+
+    def state
+      super.merge(work_type: @work_type,
+                  batch_size: @batch_size,
+                  batch: @batch,
+                  batches: @batches,
+                  ore_moved: @ore_moved)
+    end
+
+    def status
+      [format("Batch %s / %s %i%%",
+              Ore.units(@batch),
+              Ore.units(@batch_size),
+              @batch.to_f * 100 / @batch_size),
+       format("Moved %ix (%s)", @batches, Ore.units(@ore_moved)),
+      ].join(' | ')
+    end
+
+    def load_ore(amt)
+      @batch += amt
+      move_batch if @batch >= @batch_size
+      log format("LOAD %s", self.status)
+      @batch
+    end
+
+    def move(duration)
+      MinerMover.work(duration, @work_type)
+    end
+
+    def move_batch
+      raise "unexpected batch: #{@batch}" if @batch <= 0
+      amt = [@batch, @batch_size].min
+      duration = self.varied(amt / @rate)
+
+      log format("MOVE %s (%.2f s)", Ore.display(amt), duration)
+      _, elapsed = CompSci::Timer.elapsed { self.move(duration) }
+      log format("MOVD %s (%.2f s)", Ore.display(amt), elapsed)
+
+      # accounting
+      @ore_moved += amt
+      @batch -= amt
+      @batches += 1
+
+      # what moved
+      amt
+    end
+  end
+end

data/lib/miner_mover.rb

@@ -0,0 +1,46 @@
+module MinerMover
+  # called by Worker instances, available for general use
+  def self.log_fmt(timer, id, msg)
+    format("%s %s %s", timer.elapsed_display, id, msg)
+  end
+
+  # i +- 50% at squeeze 0
+  # i +- 25% at squeeze 1, 12.5% at squeeze 2, etc.
+  def self.randomize(i, squeeze = 0)
+    r, base = rand, 0.5
+    # every squeeze, increase the base closer to 1 and cut the rand in half
+    squeeze.times { |s|
+      r *= 0.5
+      base += 0.5 ** (s+2)
+    }
+    i * (base + r)
+  end
+
+  # ore is handled in blocks of 1M
+  module Ore
+    BLOCK = 1_000_000
+
+    # raw ore in, blocks out
+    def self.block(ore, size = BLOCK)
+      ore.to_f / size
+    end
+
+    # mostly used for display purposes
+    def self.units(ore)
+      if ore % BLOCK == 0 or ore > BLOCK * 100
+        format("%iM", self.block(ore).round)
+      elsif ore > BLOCK
+        format("%.2fM", self.block(ore))
+      elsif ore > 10_000
+        format("%iK", self.block(ore, 1_000).round)
+      else
+        format("%i", ore)
+      end
+    end
+
+    # entirely used for display purposes
+    def self.display(ore)
+      format("%s ore", self.units(ore))
+    end
+  end
+end

data/miner_mover.gemspec

@@ -0,0 +1,30 @@
+Gem::Specification.new do |s|
+  s.name = 'miner_mover'
+  s.summary = "This project provides a basic concurrency problem useful for" <<
+              " exploring different multitasking paradigms available in Ruby"
+  s.description = <<EOF
+Fundamentally, we have a set of miners and a set of movers. A miner takes some amount of time to mine ore, which is given to a mover. When a mover has enough ore for a full batch, the delivery takes some amount of time before more ore can be loaded.
+EOF
+    s.authors = ["Rick Hull"]
+  s.homepage = "https://github.com/rickhull/miner_mover"
+  s.license = "LGPL-3.0"
+
+  s.required_ruby_version = "~> 2"
+
+  s.version = File.read(File.join(__dir__, 'VERSION')).chomp
+
+  s.files = %w[miner_mover.gemspec VERSION README.md Rakefile]
+  s.files += Dir['lib/**/*.rb']
+  s.files += Dir['test/**/*.rb']
+  s.files += Dir['demo/**/*.rb']
+
+  s.add_dependency "compsci", "~> 0.3"
+  s.add_dependency "dotcfg", "~> 1.0"
+  s.add_dependency "fiber_scehduler", "~> 0.13"
+
+  s.add_development_dependency "buildar", "~> 3.0"
+  s.add_development_dependency "minitest", "~> 5.0"
+  s.add_development_dependency "rake", "~> 13.0" # CVE-2020-8130
+  s.add_development_dependency "flog", "~> 0"
+  s.add_development_dependency "flay", "~> 0"
+end

data/test/miner_mover.rb

@@ -0,0 +1,33 @@
+require 'minitest/autorun'
+require 'miner_mover/worker'
+
+describe MinerMover do
+  describe "MinerMover.work" do
+    it "rejects invalid work types" do
+      expect { MinerMover.work(2, :invalid) }.must_raise
+    end
+
+    it "sleeps for a duration to simulate waiting on an IO response" do
+      n = 0.1
+      expect(MinerMover.work(n, :wait)).must_equal n
+    end
+
+    it "performs fibonacci to simulate CPU work" do
+      expect(MinerMover.work(0.1, :cpu)).must_be(:<, 0.5)
+    end
+
+    it "returns instantly" do
+      expect(MinerMover.work(5, :instant)).must_equal 0
+    end
+  end
+
+  describe "MinerMover.mine_ore" do
+    before do
+      @miner = MinerMover::Miner.new(variance: 0, partial_reward: false)
+    end
+
+    it "mines to a depth, unsigned int" do
+      expect(@miner.mine_ore(1)).must_equal 0
+    end
+  end
+end