nera 0.0.1

data/lib/nera/nera_parameter_records.rb

@@ -0,0 +1,186 @@
+require 'nera_database'
+require 'nera_simulator'
+
+module NERA
+
+  # The parameter tables are accessed from this class
+  class ParameterRecords
+
+    # keys of the parameter table
+    ATTRIBUTES_PART = [ [:id, Integer],
+                        [:created_at, DateTime],
+                        [:updated_at, DateTime],
+                        [:in_trashbox?, Symbol] ]   # :active or :trashbox
+
+    # NERA::Database object
+    @db
+
+    # attributes (keys)
+    @keys
+    attr_reader :keys
+
+    # simulator_specific keys
+    @sim_class
+    
+    # argument is the path to database file
+    def initialize( db_file, sim_class)
+      unless sim_class.superclass == NERA::Simulator
+        raise ArgumentError, "Given simulator class is not a subclass of NERA::Simulator : #{sim_class}"
+      end
+      @sim_class = sim_class
+      @keys = ATTRIBUTES_PART.map do |k|
+        k[0]
+      end
+      sim_keys = @sim_class::Parameters.map do |x|
+        x[0]
+      end
+      found = @keys.find do |a|
+        sim_keys.include?(a)
+      end
+      if found
+        raise ArgumentError, "Invalid class. This class has a parameter : #{found}"
+      end
+      @keys += sim_keys
+      @db = NERA::Database.new( db_file)
+    end
+
+    # if file already exists, return false
+    def self.create_table( db_file, sim_class)
+      unless sim_class.superclass == NERA::Simulator
+        raise ArgumentError, "Given simulator class is not a subclass of NERA::Simulator : #{sim_class}"
+      end
+      ks = ATTRIBUTES_PART.map do |k|
+        k[0]
+      end
+      params = sim_class::Parameters.map do |x|
+        x[0]
+      end
+      found = ks.find do |a|
+        params.include?(a)
+      end
+      if found
+        raise ArgumentError, "Invalid class. This class has a parameter : #{found}"
+      end
+      NERA::Database.create_table( db_file)
+    end
+
+    #
+    def add( param_hash)
+      # check the argument
+      unless param_hash.is_a?(Hash)
+        raise ArgumentError, "Argument must be a Hash."
+      end
+
+      # check the argument
+      sim_keys = @sim_class::Parameters.map do |x|
+        x[0]
+      end
+      param_hash.each_pair do |param,value|
+        unless sim_keys.include?(param)
+          raise ArgumentError, "The given parameter set has unknown key."
+        end
+        a = @sim_class::Parameters.find do |p|
+          p[0] == param
+        end
+        unless value.is_a?( a[1] )
+          raise ArgumentError, "The given parameter set has invalid type."
+        end
+      end
+
+      # check duplication
+      found = @db.find_all do |rec|
+        f = true
+        @sim_class::Parameters.each do |p|
+          # set default values
+          rec[ p[0] ] = p[2] if rec[ p[0] ] == nil
+          param_hash[ p[0] ] = p[2] if param_hash[ p[0]] == nil
+          unless rec[p[0]] == param_hash[p[0]]
+            f = false
+            break
+          end
+        end
+        f
+      end
+      if found    # the given parameter set already exists.
+        return nil
+      end
+
+      # add a parameter
+      d = DateTime.now
+      h = {:created_at => d, :updated_at => d, :in_trashbox? => :active}
+      ps = param_hash.dup
+      @sim_class::Parameters.each do |p|
+        ps[ p[0]] = p[3] if ps[p[0]] == nil
+      end
+      h.merge!( ps)
+      @db.add(h)
+    end
+
+    def list_all
+      list = @db.find_all do |r|
+        true
+      end
+      return list.to_a
+    end
+
+    def list_active
+      list = @db.find_all do |r|
+        r[:in_trashbox?] == :active
+      end
+      return list.to_a
+    end
+
+    def list_trashbox
+      list = @db.find_all do |r|
+        r[:in_trashbox?] == :trashbox
+      end
+      return list.to_a
+    end
+
+    def find_by_id( id)
+      @db.find_by_id( id)
+    end
+
+    def update_to_state_active( id)
+      raise ArgumentError unless id.is_a?(Integer)
+      found = @db.find_by_id( id)
+      return nil unless found
+      return nil if found[:in_trashbox?] == :active
+
+      found[:in_trashbox?] = :active
+      @db.update( found)
+    end
+
+    def update_to_state_trashbox( id)
+      raise ArgumentError unless id.is_a?(Integer)
+      found = @db.find_by_id( id)
+      return nil unless found
+      return nil if found[:in_trashbox?] == :trashbox
+
+      found[:in_trashbox?] = :trashbox
+      @db.update( found)
+    end
+
+    def touch( id)
+      raise ArgumentError unless id.is_a?(Integer)
+      found = @db.find_by_id( id)
+      return nil unless found
+      found[:updated_at] = DateTime.now
+      @db.update( found)
+    end
+
+    def destroy( id)
+      raise ArgumentError unless id.is_a?(Integer)
+      found = @db.find_by_id( id)
+      return nil if found == nil or found[:in_trashbox?] == :active
+      @db.destroy(id)
+    end
+
+    def transaction
+      @db.transaction{
+        yield
+      }
+    end
+    
+  end
+end

data/lib/nera/nera_run_layer_controller.rb

@@ -0,0 +1,192 @@
+require 'date'
+require 'nera_db_folders'
+require 'nera_simulator_records'
+require 'nera_parameter_records'
+require 'nera_run_records'
+require 'nera_job_records'
+require 'nera_job_script'
+require 'fileutils'
+
+module NERA
+
+  class RunLayerController
+
+    # instance of NERA::DbFolders
+    @db_folders
+    # instance of NERA::Simulator class
+    @sim_instance
+    # id of the parameter set
+    @param_id
+    # instance of NERA::RunRecords
+    @run_records
+
+    def initialize( path_db_folder, sim_class, param_id)
+      raise ArgumentError unless path_db_folder.is_a?(String)
+      unless sim_class.is_a?(Class) and sim_class.superclass == NERA::Simulator and param_id.is_a?(Integer)
+        raise ArgumentError, "#{sim_class}, #{param_id} are not appropriate instances"
+      end
+      
+      @db_folders = NERA::DbFolders.new( path_db_folder)
+      @sim_instance = sim_class.new
+      @param_id = param_id
+      param_records = NERA::ParameterRecords.new( @db_folders.path_to_parameters_table( sim_class), sim_class)
+      found = param_records.find_by_id( @param_id)
+      @sim_instance.class::Parameters.each do |pa|
+        key = pa[0]
+        if found[key]
+          @sim_instance.param[key] = found[key]
+        else
+          @sim_instance.param[key] = pa[2]
+        end
+      end
+      
+      @run_records = NERA::RunRecords.new( @db_folders.path_to_runs_table( sim_class, @param_id) )
+    end
+    
+    def path_to_run_layer
+      return @db_folders.path_to_run_layer( @sim_instance.class, @param_id)
+    end
+    
+    def all_runs_list_in_csv
+      list = @run_records.list_all
+      header = @run_records.keys.dup
+      csv_list = []
+      list.each do |r_hash|
+        strings = []
+        header.each do |key|
+          if r_hash[key].is_a?(DateTime)
+            strings << r_hash[key].to_s.split('T')[0]
+          else
+            strings << r_hash[key].to_s
+          end
+        end
+        csv_list << strings.join(", ")
+      end
+      return header.join(", "), csv_list
+    end
+
+    def create_jobs( num_jobs, num_runs_per_job = 1)
+      unless num_jobs.is_a?(Integer) and num_jobs >= 0
+        raise ArgumentError, "num_jobs must be equal to or larger than zero."
+      end
+
+      job_ids = []
+      job_recs = NERA::JobRecords.new( @db_folders.path_to_jobs_table)
+      js = NERA::JobScript.new( @db_folders)
+      job_recs.transaction {
+        @run_records.transaction {
+          num_jobs.times do |j_idx|
+            run_stat = @run_records.add( num_runs_per_job)
+            rid = run_stat[0][:id]
+            jid = job_recs.add( @sim_instance.class, @param_id, rid, num_runs_per_job)
+            @run_records.set_job_id( rid, num_runs_per_job, jid)
+            sim_records = NERA::SimulatorRecords.new( @db_folders.path_to_simulators_table)
+            found = sim_records.list.find do |rec| rec[:name] == @sim_instance.class.to_s end
+            path_script = js.create_script( jid, found[:id], @param_id, @sim_instance, run_stat)
+            job_ids << jid
+          end
+        }
+      }
+      dump_in_yaml( job_recs)
+      return job_ids
+    end
+
+    def not_finished_jobs_list_in_csv
+      list = @run_records.list_all_not_finished
+      header = "job_id, created_at, run_id, num_runs, omp_threads, mpi_processes"
+
+      jobids = list.map do |rec|
+        rec[:job_id]
+      end
+      jobids = jobids.sort.uniq
+
+      csv_list = []
+      jobids.each do |jid|
+        strings = [ jid]
+        found = list.find_all do |r| r[:job_id] == jid end
+        found_first = found.first
+        strings << found_first[:created_at].to_s.split('T')[0]
+        strings << found_first[:id]
+        strings << found.size
+        strings << found_first[:omp_threads]
+        strings << found_first[:mpi_processes]
+        csv_list << strings.join(", ")
+      end
+      return header, csv_list
+    end
+
+    def cancel_jobs( job_ids)
+      unless job_ids.is_a?(Array)
+        raise ArgumentError, "job_ids must be an Array."
+      end
+      job_ids.each do |x|
+        raise ArgumentError, "each element of job_ids must be an Integer" unless x.is_a?(Integer)
+      end
+      
+      job_recs = NERA::JobRecords.new( @db_folders.path_to_jobs_table)
+      job_recs.transaction {
+        @run_records.transaction {
+          job_ids.each do |jid|
+            d = job_recs.destroy(jid)
+            next unless d
+            a = @run_records.destroy_job_id( jid)
+            raise "must not happen" unless a
+            FileUtils.rm( @db_folders.path_to_job_script(jid) )
+          end
+        }
+      }
+      dump_in_yaml( job_recs)
+      return true
+    end
+
+    def analysis_methods
+      found = @sim_instance.public_methods.find_all do |meth|
+        meth =~ /^analyze_/
+      end
+      return found
+    end
+    
+    def analyze( method_str)
+      unless method_str.is_a?(String) or method_str.is_a?(Symbol)
+        raise ArgumentError, "method_str must be a String or Symbol"
+      end
+
+      return nil unless method_str.to_s =~ /^analyze_/
+      
+      found = @sim_instance.public_methods.find do |m|
+        m == method_str.to_s
+      end
+      return nil unless found
+
+      FileUtils.cd( @db_folders.path_to_run_layer(@sim_instance.class, @param_id) ) {
+        @sim_instance.__send__( method_str)
+      }
+    end
+
+    def analyze_all
+      found = @sim_instance.public_methods.find_all do |m|
+        m =~ /^analyze_/
+      end
+      return nil unless found
+
+      FileUtils.cd( @db_folders.path_to_run_layer(@sim_instance.class, @param_id) ) {
+        found.each do |meth|
+          @sim_instance.__send__( meth)
+        end
+      }
+    end
+
+    private
+    def dump_in_yaml( job_recs)
+      File.open( @db_folders.path_to_runs_yaml( @sim_instance.class, @param_id), 'w') do |io|
+        YAML.dump( @run_records.list_all, io)
+        io.flush
+      end
+      File.open( @db_folders.path_to_jobs_yaml, 'w') do |io|
+        YAML.dump( job_recs.list_all, io)
+        io.flush
+      end
+    end
+
+  end
+end

data/lib/nera/nera_run_records.rb

@@ -0,0 +1,184 @@
+require 'date'
+require 'nera_database'
+
+module NERA
+  class RunRecords
+
+    # keys of run records
+    ATTRIBUTES = [ [:id, Integer], [:state, Symbol], [:created_at, DateTime],
+                   [:job_id, Integer], [:seed, Integer], 
+                   [:start_at, DateTime], [:finish_at, DateTime], 
+                   [:included_at, DateTime], [:real_time, Float], 
+                   [:user_time, Float], [:host_name, String], 
+                   [:omp_threads, Integer], [:mpi_processes, Integer]
+                 ]
+
+    # Maximum of the seed
+    SEED_MAX = 2**31
+
+    # NERA::Database object (table object)
+    @db
+    
+    # record
+    @keys
+    attr_accessor :keys
+    
+    # run record
+    def initialize( table_file)
+      @keys = ATTRIBUTES.map do |k| k[0] end
+      @db = NERA::Database.new( table_file)
+    end
+
+    def self.create_table(table_file)
+      NERA::Database.create_table( table_file)
+    end
+
+    public
+    def add( num_runs, omp_threads = 1, mpi_procs = 1)
+      # check arguments
+      unless num_runs.is_a?(Integer) and num_runs > 0
+        raise ArgumentError, "Number of runs must be a postive integer."
+      end
+      unless (omp_threads.is_a?(Integer) and omp_threads > 0) or omp_threads == nil
+        raise ArgumentError, "omp_threads must be a postive integer or nil"
+      end
+      unless (mpi_procs.is_a?(Integer) and mpi_procs > 0) or mpi_procs == nil
+        raise ArgumentError, "mpi_procs must be a postive integer or nil"
+      end
+
+      stats = []
+      @db.transaction {
+        seeds = find_unique_seed( num_runs)
+        d = DateTime.now
+        num_runs.times do |i|
+          h = {:state => :not_finished, :created_at => d, :job_id => nil, :seed => seeds[i], :omp_threads => omp_threads, :mpi_processes => mpi_procs}
+          new_id = @db.add( h)
+          found = @db.find_by_id( new_id)
+          stat = Hash.new
+          stat[:id] = found[:id]
+          stat[:seed] = found[:seed]
+          stat[:omp_threads] = found[:omp_threads]
+          stat[:mpi_processes] = found[:mpi_processes]
+          stats << stat
+        end
+      }
+      return stats
+    end
+
+    private
+    def find_unique_seed( num_runs)
+      seeds = []
+      until seeds.size == num_runs
+        sed = rand( SEED_MAX)
+        found = @db.find_all do |rec|
+          rec[:seed] == sed
+        end
+        seeds << sed unless found
+      end
+      return seeds
+    end
+    
+    public
+    def set_job_id( id, num_runs, j_id)
+      # check arguments
+      unless id.is_a?(Integer) and num_runs.is_a?(Integer) and j_id.is_a?(Integer)
+        raise ArgumentError, "Arguments must be Integers."
+      end
+
+      @db.transaction {
+        for i in id...(id+num_runs) do
+          r = @db.find_by_id(i)
+          raise RuntimeError, "The record #{id} is not found." unless r
+          raise RuntimeError, "The record #{id} already has job_id." if r[:job_id]
+          r[:job_id] = j_id
+          @db.update(r)
+        end
+        flag = true
+      }
+      return j_id
+    end
+    
+    def list_all
+      matched = @db.find_all do |rec|
+        true
+      end
+      return matched.to_a
+    end
+
+    def list_all_finished
+      matched = @db.find_all do |rec|
+        rec[:state] == :finished
+      end
+      return matched.to_a
+    end
+
+    def list_all_not_finished
+      matched = @db.find_all do |rec|
+        rec[:state] == :not_finished
+      end
+      return matched.to_a
+    end
+
+    def update_state_to_finished( id, start_at, finish_at, included_at, real_time, user_time, host_name)
+      unless id.is_a?(Integer) and start_at.is_a?(DateTime) and finish_at.is_a?(DateTime)
+        p id, start_at, finish_at
+        raise ArgumentError
+      end
+      unless included_at.is_a?(DateTime) and real_time.is_a?(Float) and user_time.is_a?(Float) and host_name.is_a?(String)
+        p included_at, real_time, user_time, host_name
+        raise ArgumentError
+      end
+
+      flag = nil
+      @db.transaction {
+        found = @db.find_by_id( id)
+        return nil unless found
+        return nil if found[:state] == :finished
+        found[:start_at] = start_at
+        found[:finish_at] = finish_at
+        found[:included_at] = included_at
+        found[:real_time] = real_time
+        found[:user_time] = user_time
+        found[:host_name] = host_name
+        found[:state] = :finished
+        flag = @db.update( found)
+      }
+      return flag
+    end
+    
+    def destroy( id)
+      raise ArgumentError unless id.is_a?(Integer)
+
+      flag = nil
+      @db.transaction {
+        found = @db.find_by_id(id)
+        return nil unless found
+        return nil if found[:state] == :finished
+        flag = @db.destroy(id)
+      }
+      return flag
+    end
+
+    def destroy_job_id( job_id)
+      raise ArgumentError unless job_id.is_a?(Integer)
+
+      @db.transaction {
+        found = list_all_not_finished.find_all do |rec|
+          rec[:job_id] == job_id
+        end
+        return nil if found.size == 0
+        found.each do |rec|
+          f = @db.destroy( rec[:id])
+          raise "must not happen" unless f
+        end
+      }
+      return true
+    end
+    
+    def transaction
+      @db.transaction{
+        yield
+      }
+    end
+  end
+end

data/lib/nera/nera_simulator.rb

@@ -0,0 +1,26 @@
+module NERA
+  
+  class Simulator
+
+    # instance variables
+    @param
+    attr_accessor :param
+
+    def initialize
+      @param = {}
+    end
+
+    # class methods, variables
+    @@inherited = []
+    
+    def self.inherited(subclass)
+      @@inherited << subclass
+    end
+
+    public
+    def self.inherited_simulators
+      return @@inherited
+    end
+  end
+  
+end

data/lib/nera/nera_simulator_layer_controller.rb

@@ -0,0 +1,66 @@
+require 'nera_db_folders'
+require 'nera_simulator_records'
+require 'nera_parameter_records'
+
+module NERA
+  
+  class SimulatorLayerController
+
+    # instance of NERA::DbFolders
+    @db_folders
+    # instance of NERA::SimulatorRecords
+    @sim_records
+    
+    def initialize( path_db_folds)
+      raise ArgumentError unless path_db_folds.is_a?(String)
+      
+      @db_folders = NERA::DbFolders.new( path_db_folds)
+      table_path = @db_folders.path_to_simulators_table
+      @sim_records = NERA::SimulatorRecords.new(table_path)
+
+      NERA::Simulator.inherited_simulators.each do |sim|
+        next if list.find do |rec| rec[:name] == sim.to_s end
+        found = sim::Parameters.find do |p|
+            p[0] == :in_trashbox? or p[0] == :created_at or p[0] == :updated_at
+        end
+        next if found
+        register_a_simulator( sim)
+      end
+    end
+
+    def list
+      @sim_records.list
+    end
+
+    def path_to_sim_layer
+      @db_folders.path_to_simulator_layer
+    end
+    
+    private
+    def register_a_simulator( sim_klass)
+      a = nil
+      @sim_records.transaction {
+        a = @sim_records.add( sim_klass)
+        return nil unless a
+        b = @db_folders.create_new_simulator_folder( sim_klass)
+        raise "The folder #{sim_klass.to_s}/ already exists." unless b
+        path = @db_folders.path_to_parameters_table( sim_klass)
+        c = NERA::ParameterRecords.create_table( path, sim_klass)
+        raise "The file #{path} already exists." unless c
+      }
+      dump_in_yaml
+      return a
+    end
+
+    private
+    def dump_in_yaml
+      ymlpath = @db_folders.path_to_simulators_yaml
+      File.open( ymlpath, 'w') do |io|
+        YAML::dump( @sim_records.list, io)
+        io.flush
+      end
+    end
+
+  end
+
+end