nera 0.0.1

data/lib/nera/nera_database.rb

@@ -0,0 +1,281 @@
+require 'fileutils'
+require 'yaml'
+#require 'filelock'
+require 'pstore'
+require 'pp'
+
+#* list of public methods
+#- initialize( path_to_filename )
+#
+#- self.create_table( path_to_basename )
+#--- method to create new table
+#--- ex. NERA::Database.create_table( "./db")
+#
+#- add( new_rec )
+#--- add a new record to the table.
+#--- new_rec must be specified by a Hash.
+#--- ex. db.add( { :key1 => 'test', :key2 => DateTime.now, ... } )
+#--- You don't need to specify the ':id' key. If it is specified, it is omitted.
+#--- During the transaction, files are locked with exclusive locks.
+#--- return true if the addition succeeded.
+#--- raise RuntimeError if the specified hash is not valid.
+#
+#- find_by_id( id )
+#--- returns records in Hash.
+#--- id can be Integer, Array of Integers, or Range of Integers.
+#--- During reading file, the files are locked with shared locks.
+#--- If there are several records, it returns Array of Hash.
+#
+#- find_all { |rec| (condition) }
+#--- returns the array of the matched records.
+#--- return nil if no record matched.
+#
+#- update( rec )
+#--- update a record
+#--- ex. db.update( {:id=>3, :key1=>'updated', :key2=>DateTime.now} )
+#--- returns nil if the record of the specified 'id' does not exist.
+#
+#- destroy( id )
+#--- destroy records
+#--- id may be an Integer, Array of Integers, Range
+#--- if the specified record is not found in the table, return nil.
+#
+#- transaction {  ..(transaction).. }
+#--- all the database files are exclusively locked during block is processed.
+#--- database files are unlocked at the end of the block. terminates.
+#--- sample -----------
+#
+#  db.transaction do
+#    arr = db.find_all do |record|
+#      record[:id] > 5
+#    end
+#  
+#    arr.each do |rec| rec[:state] = finished end
+#    db.update( arr)
+#  end
+#
+#--- end of sample ---
+#
+#
+#
+module NERA
+  class Database
+    # name of the file
+    @filename
+    
+    # true if files are exclusively locked
+    @in_transaction
+
+    # PStore object
+    @pstore
+    
+    # search a data file in this order
+    # classes of values
+    VALUE_CLASSES = [ String, Symbol, Integer, Float, Date, DateTime, TrueClass, FalseClass, NilClass ]
+
+    public
+    # ----------------------------------
+    def initialize( path_to_file )
+      @filename = path_to_file
+      unless File.exist?(@filename)
+        raise "No such database : #{@filename}"
+      end
+    end
+    
+    # ------------------------------------
+    def transaction
+      if @in_transaction     # nothing happens when the process is already in transaction
+        yield
+        return nil
+      end
+      @pstore = PStore.new(@filename)
+      @pstore.transaction( false) do
+        begin
+          @in_transaction = true
+          yield
+        ensure
+          @in_transaction = false
+          nil
+        end
+      end
+    end
+
+    private
+    # ------------------------------------
+    def readonly_transaction
+      if @in_transaction
+        yield
+        return nil
+      end
+      @pstore = PStore.new(@filename)
+      @pstore.transaction(true) do
+        yield
+      end
+    end
+
+    public
+    # ------------------------------------
+    def self.create_table( filename)
+      if File.exists?( filename)
+        $stderr.puts "Database #{filename} already exists."
+        return nil
+      end
+      db = PStore.new( filename)
+      db.transaction(false) do
+        db[:max_id] = 0
+      end
+      return true
+    end
+
+    public
+    # ------------------------------------
+    def add( new_rec )
+      unless valid_as_a_record?( new_rec)
+        raise ArgumentError, "This record is not valid."
+      end
+      new_id = nil
+      transaction do
+        new_id = @pstore[:max_id] + 1
+        new_rec[:id] = new_id
+        @pstore[:max_id] = new_id
+        @pstore[new_id] = new_rec
+      end
+      return new_id
+    end
+
+    private
+    # ------------------------------------------
+    def valid_as_a_record?( rec)
+      # check validity
+      raise "rec must be a Hash" unless rec.is_a?(Hash)
+      rec.each do |key,val|
+        unless VALUE_CLASSES.find do |type| val.is_a?(type) end
+          $stderr.puts "Types of this record is not valid : #{key}, #{val}, #{val.class.to_s}"
+          return false
+        end
+      end
+      return true
+    end
+
+    public
+    # --- find methods --------------------------
+    def find_by_id( ids)
+      if ids.is_a?(Integer)
+        readonly_transaction do
+          return @pstore[ids]
+        end
+      elsif ids.is_a?(Array)
+        found = []
+        readonly_transaction do
+          # find all ----
+          ids.each do |id|
+            found << @pstore[id]
+          end
+          return found
+        end
+      elsif ids.is_a?(Range)
+        found = []
+        readonly_transaction do
+          matched_ids = @pstore.roots.find_all do |id|
+            ids.include?(id)
+          end
+          return nil unless matched_ids
+          matched_ids.sort! do |id1,id2|
+            id1 <=> id2
+          end
+          matched_ids.each do |id|
+            found << @pstore[id]
+          end
+          found = nil if found.size == 0
+          return found
+        end
+      else 
+        raise ArgumentError, "Argument of the find method must be Integer, Array or Range. : #{ids.class}"
+      end
+    end
+
+    public
+    # --- find all ---
+    # usage: find_all { ..(condition).. }
+    def find_all
+      matched = []
+      readonly_transaction do
+        all_ids = @pstore.roots
+        all_ids.delete( :max_id)
+        all_ids.each do |id|
+          if yield @pstore[id].dup
+            matched << @pstore[id]
+          end
+        end
+      end
+      matched.sort! do |rec1,rec2|
+        rec1[:id] <=> rec2[:id]
+      end
+      matched = nil if matched.size == 0
+      return matched
+    end
+
+    public
+    # --- update method ---
+    #- update( rec )
+    #--- update a record
+    #--- ex. db.update( {:id=>3, :key1=>'updated', :key2=>DateTime.now} )
+    #--- returns nil if the record of the specified 'id' does not exist.
+    def update( rec)
+      unless valid_as_a_record?( rec)
+        $stderr.puts "This record is not valid."
+        return nil
+      end
+      id_to_update = rec[:id]
+      flag = nil
+      transaction do
+        unless @pstore.root?( id_to_update)
+          return nil
+        end
+        @pstore[id_to_update] = rec
+        flag = true
+      end
+      return flag
+    end
+    
+    public
+    #- destroy( id )
+    #--- destroy records
+    #--- id may be an Integer, Array of Integers, Range
+    #--- if the specified record is not found in the table, return nil.
+    def destroy( ids)
+      if ids.is_a?( Array)
+        status = []
+        transaction do
+          ids.each do |id|
+            status << destroy( id)
+          end
+        end
+        return status
+      elsif ids.is_a?( Range)
+        flag = nil
+        transaction do
+          id_array = @pstore.roots.find_all do |id|
+            ids.include?(id)
+          end
+          flag = destroy( id_array)
+        end
+        flag = flag.find do |f| f end
+        return flag
+      elsif ids.is_a?( Integer)
+        stat = nil
+        transaction do
+          stat = @pstore.delete(ids)
+        end
+        return stat
+      else
+        raise ArgumentError, "argument must be specified by Integer, Array of Integer, or Range. : #{ids.class}"
+      end
+      return true
+    end
+    
+  end
+  # ------ end of Database class --------
+  
+end
+# ------ end of NERA module --------

data/lib/nera/nera_db_folders.rb

@@ -0,0 +1,226 @@
+require 'fileutils'
+require 'nera_simulator_records'
+require 'nera_job_records'
+
+
+module NERA
+  
+  # This class contains the information of the folder structure
+  class DbFolders
+
+    #--------------------------------------
+    #-class methods -----------------------
+    #--------------------------------------
+    
+    def initialize( db_folder)
+      if db_folder =~ /\/$/
+        @db_folder = db_folder
+      else
+        @db_folder = db_folder + '/'
+      end
+      unless valid_db?( @db_folder)
+        raise RuntimeError, "Database #{@db_folder} is not a valid database folder."
+      end
+
+      # load simulator classes
+      Dir.glob( "#{@db_folder}/Simulator_classes/*.rb").each do |lib|
+        require lib
+      end
+    end
+
+    def valid_db?( fold)
+      return false unless FileTest.directory?( fold)
+      return false unless FileTest.directory?( fold + "Jobs/")
+      return false unless FileTest.directory?( fold + "Jobs/Include/")
+      return false unless FileTest.directory?( fold + "Tables/")
+      return false unless FileTest.directory?( fold + "Simulator_classes/")
+      return false unless FileTest.exist?( fold + "Tables/simulators.pstore")
+      return false unless FileTest.exist?( fold + "Tables/jobs.pstore")
+      return true
+    end
+    private :valid_db?
+
+    def self.create_db( db_folder)
+      if FileTest.directory?( db_folder)
+        raise RuntimeError, "Folder #{db_folder} already exists."
+      end
+      
+      if db_folder =~ /\/$/
+        db = db_folder
+      else
+        db = db_folder + '/'
+      end
+      FileUtils.mkdir( db, :verbose => true)
+      FileUtils.mkdir( db + "Jobs/", :verbose => true)
+      FileUtils.mkdir( db + "Jobs/Include/", :verbose => true)
+      FileUtils.mkdir( db + "Tables/", :verbose => true)
+      FileUtils.mkdir( db + "Simulator_classes/", :verbose => true)
+      f = NERA::SimulatorRecords.create_table( db + "Tables/simulators.pstore")
+      raise "must not happen" unless f
+      f = NERA::JobRecords.create_table( db + "Tables/jobs.pstore")
+      raise "must not happen" unless f
+      File.open( db + 'simulators.yml', 'w') do |io|
+        YAML.dump( [], io )
+        io.flush
+      end
+      File.open( db + 'Jobs/jobs.yml', 'w') do |io|
+        YAML.dump( [], io )
+        io.flush
+      end
+
+      return true
+    end
+
+    #--------------------------------------
+    #-public instance methods--------------
+    #--------------------------------------
+    public
+    def path_to_simulator_layer
+      return @db_folder
+    end
+
+    def path_to_simulators_table
+      return "#{@db_folder}Tables/simulators.pstore"
+    end
+
+    def path_to_simulators_yaml
+      return "#{@db_folder}simulators.yml"
+    end
+
+    def create_new_simulator_folder( sim_class)
+      unless sim_class.superclass == NERA::Simulator
+        raise ArgumentError, "must be a subclass of NERA::Simulator"
+      end
+      name = sim_class.to_s
+      name += '/'
+      
+      if FileTest.directory?( @db_folder + name) or FileTest.directory?( @db_folder + "Tables/" + name)
+        return nil
+      end
+      FileUtils.mkdir( @db_folder + name, :verbose => true)
+      FileUtils.mkdir( @db_folder + "Tables/" + name, :verbose => true)
+
+      File.open( path_to_parameters_yaml( sim_class) , 'w') do |io|
+        YAML.dump( [], io )
+        io.flush
+      end
+
+      return true
+    end
+    
+    def path_to_parameter_layer( sim_class)
+      unless sim_class.superclass == NERA::Simulator
+        raise ArgumentError, "must be a subclass of NERA::Simulator"
+      end
+      name = sim_class.to_s
+      name += '/'
+      
+      path = @db_folder + name
+      if FileTest.directory?(path)
+        return path
+      else
+        return nil
+      end
+    end
+    
+    def path_to_parameters_table( sim_class)
+      unless sim_class.superclass == NERA::Simulator
+        raise ArgumentError, "must be a subclass of NERA::Simulator"
+      end
+      name = sim_class.to_s
+      name += '/'
+      
+      path = @db_folder + 'Tables/' + name + 'parameters.pstore'
+      unless FileTest.directory?( @db_folder + 'Tables/' + name)
+        return nil
+      end
+      return path
+    end
+
+    def path_to_parameters_yaml( sim_class)
+      return path_to_parameter_layer( sim_class) + 'parameters.yml'
+    end
+    
+    def create_new_parameter_folder( sim_class, param_id)
+      p = path_to_parameter_layer( sim_class)
+      return nil unless p
+
+      if param_id.to_i > 99999 or param_id.to_i <= 0
+        raise RuntimeError, "The specified parameter id #{param_id} is out of range."
+      end
+      fold_name = sprintf("%05d/", param_id.to_i)
+      if FileTest.directory?( p + fold_name)
+        return nil
+      end
+      FileUtils.mkdir( p + fold_name)
+      p = path_to_parameters_table( sim_class)
+      p.sub!(/parameters.pstore$/, fold_name)
+      FileUtils.mkdir( p)
+
+      File.open( path_to_runs_yaml( sim_class, param_id) , 'w') do |io|
+        YAML.dump( [], io )
+        io.flush
+      end
+
+      return true
+    end
+    
+    def path_to_run_layer( sim_class, param_id)
+      p = path_to_parameter_layer( sim_class)
+      return nil unless p
+      if param_id.to_i > 99999 or param_id.to_i <= 0
+        return nil
+      end
+      fold_name = sprintf("%05d/", param_id.to_i)
+      path = p + fold_name
+      return path
+    end
+    
+    def path_to_runs_table( sim_class, param_id)
+      ppt = path_to_parameters_table( sim_class)
+      return nil unless ppt
+      if param_id.to_i > 99999 or param_id.to_i <= 0
+        return nil
+      end
+      fold_name = sprintf("%05d/", param_id.to_i)
+      path = ppt.sub(/parameters.pstore$/, fold_name)
+      raise "must not happen" unless path
+      return path + 'runs.pstore'
+    end
+
+    def path_to_runs_yaml( sim_class, param_id)
+      return path_to_run_layer( sim_class, param_id) + 'runs.yml'
+    end
+
+    def path_to_job_layer
+      return @db_folder + "Jobs/"
+    end
+
+    def path_to_jobs_table
+      return @db_folder + "Tables/jobs.pstore"
+    end
+
+    def path_to_jobs_yaml
+      return path_to_job_layer + "jobs.yml"
+    end
+
+    def path_to_job_script(job_id)
+      filename = sprintf("j%06d.sh", job_id)
+      return path_to_job_layer + filename
+    end
+
+    def path_to_include_layer
+      return @db_folder + "Jobs/Include/"
+    end
+
+    def search_include_files
+      return Dir.glob( path_to_include_layer + "j[0-9][0-9][0-9][0-9][0-9][0-9]*.tar.bz2")
+    end
+
+    def path_to_finished_jobs_file
+      return @db_folder + "Tables/finished_jobs.yml"
+    end
+    
+  end
+end
+    

data/lib/nera/nera_dialog.rb

@@ -0,0 +1,205 @@
+# -*- coding: utf-8 -*-
+require "readline"
+
+# module for CUI interactive interface
+#
+module Dialog
+
+  PROMPT = "> "
+
+  # ----------------
+  # Select one from the list specified by the array.
+  def select_one( arr, message = "Input the number", default = 0)
+    unless arr.is_a?(Array) and arr.size >= 1 and default.to_i >= 0 and default.to_i < arr.size
+      raise ArgumentError, "Arguments are not valid."
+    end
+
+    while true
+      $stdout.puts( "#{message} : [#{default.to_s}]")
+      arr.size.times do |i|
+        puts( "[#{i}]:\t#{arr[i]}")
+      end
+      buf = Readline.readline(PROMPT).chomp.strip
+      buf = default.to_s if buf==''
+
+      unless buf.match(/^[0-9]+$/)
+        $stdout.puts "The input #{buf} is not valid. Try again"
+        redo
+      end
+      num = Integer(buf)
+      if num >= 0 and num < arr.size
+        valid_input = true
+        return num
+      end
+      $stdout.puts "Your input is not valid. Try again."
+    end
+  end
+
+  # ---------------
+  def select_one_or_return_string( arr, message = "Input the number", default = 0)
+    unless arr.is_a?(Array) and arr.size >= 1
+      raise ArgumentError, "Arguments are not valid."
+    end
+
+    while true
+      $stdout.puts( "#{message} : [#{default.to_s}]")
+      arr.size.times do |i|
+        puts( "[#{i}]:\t#{arr[i]}")
+      end
+      buf = Readline.readline(PROMPT).chomp.strip
+      buf = default.to_s if buf==''
+
+      if buf.match(/^[0-9]+$/)
+        num = Integer(buf)
+        if num >= 0 and num < arr.size
+          valid_input = true
+          return num
+        end
+      else
+        return buf
+      end
+      $stdout.puts "Your input is not valid. Try again."
+    end
+  end
+
+  # ---------------
+  def select_many( arr, message = "Select multiple numbers (ex. 1,3,5-8)", default = 0)
+    unless arr.is_a?(Array) and arr.size >= 1
+      raise ArgumentError, "Arguments are not valid."
+    end
+
+    result = []
+    valid_input = nil
+    until valid_input
+      $stdout.puts( "#{message} : [#{default}]")
+      arr.size.times do |i|
+        puts( "[#{i}]:\t#{arr[i]}")
+      end
+
+      buf = Readline.readline(PROMPT).chomp
+      buf = default.to_s if buf==''
+      
+      valid_input = true
+      buf = buf.split(',').map do |e| e.strip end
+      buf.each do |e|
+        if e.match(/^[0-9]+$/)
+          num = Integer(e)
+          if num < 0 or num >= arr.size
+            valid_input = nil
+            break
+          else
+            result << num
+          end
+        elsif e.match(/^[0-9]+-[0-9]+$/)
+          f = e.split('-')
+          from = Integer(f[0])
+          to = Integer(f[1])
+          if from >= to
+            valid_input = nil
+            break
+          end
+          for i in from..to
+            result << i if i > 0 and i <= arr.size
+          end
+        else
+          valid_input = nil
+          break
+        end
+      end
+      $stdout.puts "The input #{buf} is not valid. Try again." unless valid_input
+    end
+    return result
+  end
+
+  # ---------------
+  def message( str)
+    $stdout.puts str
+  end
+
+  # ---------------
+  def set_multiple_values( arr, message = "Input the values")
+    unless arr.is_a?(Array)
+      raise ArgumentError, "Arguments are not valid."
+    end
+    
+    result = Hash.new
+    arr.each do |a|
+      unless a.is_a?(Array) and a.size == 3
+        raise ArgumentError, "Arguments are not valid."
+      end
+      key = a[0]
+      type = a[1]
+      val = a[2]
+      unless ( type == Integer or type == Float or type == String ) and val.is_a?(type)
+        raise ArgumentError, "Arguments are not valid."
+      end
+      result[key] = val
+    end
+    unless arr.size == result.size
+      raise ArgumentError, "There is a nonunique key."
+    end
+    
+    ok = nil
+    until ok
+      # show the list
+      arr.each_with_index do |elem,i|
+        $stdout.puts "[#{i}] :\t#{elem[0]}, #{elem[1]}, #{result[elem[0]]}"
+      end
+      $stdout.puts "[#{arr.size}] : \tOK"
+      # input
+      buf = Readline.readline(PROMPT).chomp.strip
+      unless buf.match(/^[0-9]+$/)
+        redo
+      end
+      input = buf.to_i
+      if input >= 0 and input < arr.size
+        $stdout.puts "Input #{arr[input][0]}  (#{arr[input][1]})"
+        buf = Readline.readline(PROMPT).chomp.strip
+        if arr[input][1] == Integer
+          begin
+            result[ arr[input][0] ] = Integer(buf)
+          rescue
+            $stdout.puts "The specified value is not convertible to Integer"
+          end
+        elsif arr[input][1] == Float
+          begin
+            result[ arr[input][0] ] = Float(buf)
+          rescue
+            $stdout.puts "The specified value is not convertible to Float"
+          end
+        elsif arr[input][1] == String
+          result[ arr[input][0] ] = buf.to_s
+        end
+      elsif input == arr.size
+        ok = true
+      end
+    end
+    return result
+  end
+  
+  # ---------------
+  def get_integer( message = "Input a integer", default = 0)
+    unless default.is_a?(Integer)
+      raise ArgumentError, "Arguments are not valid."
+    end
+
+    while true
+      $stdout.puts( "#{message} : [#{default}]")
+
+      buf = Readline.readline(PROMPT).chomp.strip
+      buf = default.to_s if buf==''
+      unless buf.match(/^[0-9]+$/)
+        $stdout.puts "The input #{buf} is not valid. Try again"
+        redo
+      end
+      num = Integer(buf)
+      if yield num
+        return num
+      else
+        $stdout.puts "The input #{buf} does not match the condition. Try again"
+      end
+    end
+  end
+  
+  module_function :select_one, :select_one_or_return_string, :select_many, :message, :set_multiple_values, :get_integer
+end