ctioga2 0.0

data/lib/ctioga2/data/backends/backends.rb

@@ -0,0 +1,39 @@
+# backends.rb: all the backends
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301
+# USA
+
+
+require 'ctioga2/data/backends/backend'
+
+# We try to look for all files under a ctioga2/metabuilder/types
+# directory anywhere on the $: path
+
+files = []
+for dir in $:
+  Dir[dir + '/ctioga2/data/backends/backends/**/*'].each do |f|
+    f =~ /ctioga2\/data\/backends\/backends\/(.*)\.[^.]+$/
+    files << $1
+  end
+end
+
+for file in files.uniq
+  begin
+    require "ctioga2/data/backends/backends/#{file}"
+  rescue Exception => e
+    warn "There was a problem trying to load 'ctioga2/data/backends/backends/#{file}': " 
+    warn "#{e.inspect}"
+  end
+end

data/lib/ctioga2/data/backends/backends/gnuplot.rb

@@ -0,0 +1,140 @@
+# gnuplot.rb: a backend to extract plots from gnuplot's files
+# Copyright (C) 2007,2009 Vincent Fourmond
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+
+
+
+require 'Dobjects/Dvector'
+require 'Dobjects/Function'
+
+# To feed data to fancyread
+require 'stringio'
+
+module CTioga2
+
+  Version::register_svn_info('$Revision: 61 $', '$Date: 2009-05-29 01:00:56 +0200 (Fri, 29 May 2009) $')
+
+
+  module Data
+
+    module Backends
+
+      class GnuplotBackend < Backend
+
+        include Dobjects
+        include Log
+        
+        describe 'gnuplot', 'Gnuplot files', <<EOD
+This backend hijacks gnuplot files to make extract data which they plot.
+No information is taken about the style !
+EOD
+
+        param_accessor :variables_overrides, 'vars', 
+        "Variable overrides", 'text', <<EOD
+A colon-separated override of local variables, such as a=1;b=3;c=5
+EOD
+
+        param_accessor :range, 'range', 
+        "Plotting X range", 'float-range', 
+        "The plotting X range, such as 0:2"
+        
+        # This is called by the architecture to get the data. It splits
+        # the set name into filename@cols, reads the file if necessary and
+        # calls get_data
+        def query_dataset(set)
+          set =~ /^(.*?)(?:@(\d+))?(?::(.*))?$/
+          filename = $1
+          if $2
+            number = $2.to_i - 1
+          else
+            number = 0
+          end
+          if $3
+            overrides = "#{@variable_overrides};#{$3}"
+          else
+            overrides = @variable_overrides
+          end
+          plots = run_gnuplot(filename, overrides)
+          return Dataset.new(set,plots[number])
+        end
+
+        # Runs gnuplot on the file, and returns all datasets found
+        # inside it. 
+        def run_gnuplot(filename, overrides = @variables_overrides)
+          date = File::mtime(filename)
+          # Get it from the cache !
+          debug "Running gnuplot on file #{filename}"
+          f = File.open(filename)
+          # We open a bidirectionnal connection to gnuplot:
+          gnuplot = IO.popen("gnuplot", "r+")
+          output = ""
+          gnuplot.puts "set term table"
+          for line in f
+            next if line =~ /set\s+term/
+            if overrides and line =~ /plot\s+/
+              debug "Found a plot, inserting variable overrides: #{overrides}"
+              line.gsub!(/plot\s+/, "#{overrides};plot ")
+            end
+            if @range and line =~ /plot\s+/
+              debug "Found a plot, inserting range: #{@range}"
+              line.gsub!(/plot\s+(\[[^\]]+\])?/, 
+                         "plot [#{@range}]")
+            end
+            gnuplot.print line 
+            gnuplot.flush
+            output += slurp(gnuplot)
+          end
+          
+          # Output a "\n" in the end.
+          
+          gnuplot.puts ""
+          gnuplot.flush
+          gnuplot.close_write
+          # Then we get all that is remaining:
+          output += gnuplot.read
+          gnuplot.close
+          
+          # Now, interaction with gnuplot is finished, and we want to
+          # parse that:
+          outputs = output.split("\n\n")
+          plots = []
+          for data in outputs
+            plots << Dvector.fancy_read(StringIO.new(data), [0,1])
+          end
+          # This block evaluates to plots:
+          return plots
+        end
+
+        # Gets all data from the given file until it blocks, and returns it.
+        def slurp(f, size = 10240)
+          str = ""
+          begin
+            while IO::select([f],[],[],0)
+              ret = f.readpartial(size)
+              if ret.empty?
+                return str
+              end
+              str << ret 
+            end
+          end
+          return str
+        end
+
+      end
+
+    end
+  end
+end

data/lib/ctioga2/data/backends/backends/math.rb

@@ -0,0 +1,121 @@
+# math.rb :the Math backend: data based on mathematical formulas.
+# Copyright (C) 2006 - 2009 Vincent Fourmond
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+
+
+require 'Dobjects/Dvector'
+require 'Dobjects/Function'
+
+module CTioga2
+
+  Version::register_svn_info('$Revision: 17 $', '$Date: 2009-04-28 22:22:22 +0200 (Tue, 28 Apr 2009) $')
+
+  module Data
+
+    module Backends
+
+      class MathBackend < Backend
+        
+        include Dobjects
+        include Math
+      
+        describe 'math', 'Mathematical functions', <<EOD
+This backend returns computations of mathematical formulas.
+EOD
+
+        # TODO: make provisions for 3-D datasets. Ideas: x(t):y(t):z(t)
+        # for parametric plots ? (possibly x(t):y1(t):y2(t):...:yn(t)) ?
+
+        param_accessor :samples, 'samples', "Samples", 'integer', 
+        "The number of points"
+        param_accessor :x_range, 'xrange',  "X Range", 'float-range', 
+        "X range (a:b)"
+        param_accessor :t_range, 'trange',  "T Range", 'float-range', 
+        "T range (a:b) (parametric plot)"
+
+        param_accessor :log, 'log',  "Logarithmic scale", 'boolean', 
+        "Space samples logarithmically"
+        
+        def initialize
+          super()
+          @samples = 100
+          @x_range = -10.0..10.0
+          @t_range = -10.0..10.0
+          @log = false
+        end
+
+        # This is called by the architecture to get the data. It first
+        # splits the set name into func@range.
+        def query_dataset(set)
+          if set =~ /(.*)@(.*)/
+            set = $1
+            range = $2
+          end          
+          name = "math: #{set}"
+          if set =~ /:/         # parametric
+            if range
+              set_param_from_string(:t_range, range)
+            end
+            varname = "t"
+            values = make_dvector(@t_range, @samples, @log)
+          else
+            if range
+              set_param_from_string(:x_range, range)
+            end
+            varname = "x"
+            values = make_dvector(@x_range, @samples, @log)
+            set = "x:#{set}"
+          end
+          return Dataset.dataset_from_spec(name, set) do |b|
+            get_data_column(b, varname, values)
+          end
+        end
+
+
+        protected
+        
+        # Turns a Range and a number of points into a Dvector
+        def make_dvector(range, nb_points, log = @log)
+          n = nb_points -1
+          a = Dvector.new(nb_points) { |i| 
+            i.to_f/(n.to_f)
+          }
+          # a is in [0:1] inclusive...
+          if log
+            delta = range.last/range.first
+            # delta is positive necessarily
+            a *= delta.log
+            a.exp!
+            a *= range.first
+          else
+            delta = range.last - range.first
+            a *= delta
+            a += range.first
+          end
+          return a
+        end
+
+        # Uses compute_formula to get data from 
+        def get_data_column(column, variable, values)
+          column.gsub!(/\b#{variable}\b/, "(column[0])")
+          Dvector.compute_formula(column, [values])
+        end
+
+
+      end
+    end
+  end
+end

data/lib/ctioga2/data/backends/backends/text.rb

@@ -0,0 +1,335 @@
+# text.rb : A simple backend to deal with basic text files.
+# Copyright (C) 2006 Vincent Fourmond
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+
+
+
+require 'Dobjects/Dvector'
+require 'Dobjects/Function'
+
+# For separated sets
+require 'stringio'
+
+module CTioga2
+
+  Version::register_svn_info('$Revision: 17 $', '$Date: 2009-04-28 22:22:22 +0200 (Tue, 28 Apr 2009) $')
+
+
+  module Data
+
+    # A module for easy use of NaN in operations
+    module NaN
+      NaN = 0.0/0.0
+      def nan
+        return NaN
+      end
+    end
+
+    module Backends
+
+      class TextBackend < Backend
+
+        # A constant holding a relation extension -> command to
+        # decompress (to be fed to sprintf with the filename as argument)
+        UNCOMPRESSORS = {
+          ".gz" => "gunzip -c %s",
+          ".bz2" => "bunzip2 -c %s",
+          ".lzma" => "unlzma -c %s",
+          ".lz" => "unlzma -c %s",
+        }
+
+        include Dobjects
+
+        describe 'text', 'Text format', <<EOD
+This backend can read text files in a format close to the one understood
+by gnuplot and the like.
+EOD
+
+        # Inherit the baseline handling, can be useful !
+        # inherit_parameters :base_line
+        
+        param_accessor :skip, 'skip', "Skip lines", 'integer', 
+        "Number of lines to be skipped at the beginning of the file"
+
+        param_accessor :default_column_spec, 'col', 
+        "Default column specification", 'text', 
+        "Which columns to use when the @1:2 syntax is not used"
+
+        param_accessor :split, 'split', "Split into subsets", 'boolean', 
+        "If true, splits files into subsets on blank/non number lines"
+
+
+        param_accessor :separator, 'separator', "Data columns separator", 
+        'regexp', 
+        "The columns separator. Defaults to /\s+/"
+
+
+        #     param_accessor :select, 'select', "Select lines", {:type => :string},
+        #     "Skips line where the code returns false"
+        
+        def initialize
+          @dummy = nil
+          @current = nil   
+          # Current is the name of the last file used. Necessary for '' specs.
+          @current_data = nil       # The data of the last file used.
+          @skip = 0
+          @included_modules = [NaN]    # to make sure we give them to
+          # Dvector.compute_formula
+          @default_column_spec = "1:2"
+
+          @separator = /\s+/
+
+          # We don't split data by default.
+          @split = false
+
+          super()
+
+          # Override Backend's cache - for now.
+          @cache = {}               # A cache file_name -> data
+
+        end
+
+        def extend(mod)
+          super
+          @included_modules << mod
+        end
+
+        # Expands specifications into few sets. This function will separate the
+        # set into a file spec and a col spec. Within the col spec, the 2##6
+        # keyword is used to expand to 2,3,4,5,6. 2## followed by a non-digit
+        # expands to 2,...,last column in the file. For now, the expansions
+        # stops on the first occurence found, and the second form doesn't
+        # work yet. But soon...
+        def expand_sets(spec)
+          if m = /(\d+)##(\D|$)/.match(spec)
+            a = m[1].to_i 
+            trail = m[2]
+            b = read_file(spec)
+            b = (b.length - 1) 
+            ret = []
+            a.upto(b) do |i|
+              ret << m.pre_match + i.to_s + trail + m.post_match
+            end
+            return ret
+          else
+            return super
+          end
+        end
+
+
+        protected
+
+
+        # Returns a IO object suitable to acquire data from it for
+        # the given _file_, which can be one of the following:
+        # * a real file name
+        # * a compressed file name
+        # * a pipe command.
+        def get_io_object(file)
+          if file == "-"
+            return $stdin
+          elsif file =~ /(.*?)\|\s*$/ # A pipe
+            return IO.popen($1)
+          elsif not File.readable?(file)
+            # Try to find a compressed version
+            for ext,method in UNCOMPRESSORS
+              if File.readable? "#{file}#{ext}"
+                info "Using compressed file #{file}#{ext} in stead of #{file}"
+                return IO.popen(method % "#{file}#{ext}")
+              end
+            end
+          else 
+            for ext, method in UNCOMPRESSORS
+              if file =~ /#{ext}$/ 
+                info "Taking file #{file} as a compressed file"
+                return IO.popen(method % file)
+              end
+            end
+            return File::open(file)
+          end
+          error "Could not open #{file}"
+          return nil
+        end
+
+        # A line is invalid if it is blank or starts
+        # neither with a digit nor +, - or .
+        #
+        # Maybe to be improved later.
+        InvalidLineRE = /^\s*$|^\s*[^\d+.\s-]+/
+
+        # Returns a string corresponding to the given _set_ of the
+        # given _io_ object.
+        #
+        # Sets are 1-based.
+        def get_set_string(io, set)
+          cur_set = 1
+          last_line_is_invalid = true
+          str = ""
+          line_number = 0
+          while line = io.gets
+            line_number += 1
+            if line =~ InvalidLineRE
+              debug "Found invalid line at #{line_number}"
+              if ! last_line_is_invalid
+                # We begin a new set.
+                cur_set += 1
+                debug "Found set #{cur_set} at line #{line_number}"
+                if(cur_set > set)
+                  return str
+                end
+              end
+              last_line_is_invalid = true
+            else
+              last_line_is_invalid = false
+              if cur_set == set
+                str += line
+              end
+            end
+          end
+          return str
+        end
+
+        # Returns an IO object corresponding to the given file.
+        def get_io_set(file)
+          if not @split
+            return get_io_object(file)
+          else
+            file =~ /(.*?)(?:#(\d+))?$/; # ; to make ruby-mode indent correctly.
+            filename = $1
+            if $2
+              set = $2.to_i
+            else
+              set = 1
+            end
+            debug "Trying to get set #{set} from file '#{filename}'"
+            str = get_set_string(get_io_object(filename), set)
+            return StringIO.new(str)
+          end
+        end
+
+        # Reads data from a file. If needed, extract the file from the
+        # columns specification.
+        #
+        # TODO: the cache really should include things such as time of
+        # last modification and various parameters that influence the
+        # reading of the file.
+        def read_file(file)
+          if file =~ /(.*)@.*/
+            file = $1
+          end
+          name = file               # As file will be modified.
+          if ! @cache.key?(file)    # Read the file if it is not cached.
+            fancy_read_options = {'index_col' => true,
+              'skip_first' => @skip,
+              'sep' => @separator
+            }
+            io_set = get_io_set(file)
+            debug "Fancy read '#{file}', options #{fancy_read_options.inspect}"
+            @cache[name] = Dvector.fancy_read(io_set, nil, fancy_read_options)
+          end
+          return @cache[name]
+        end
+
+
+        # This is called by the architecture to get the data. It
+        # splits the set name into filename@cols, reads the file if
+        # necessary and calls get_data
+        def query_dataset(set)
+          if set =~ /(.*)@(.*)/
+            col_spec = $2
+            file = $1
+          else
+            col_spec = @default_column_spec
+            file = set
+          end
+          if file.length > 0
+            @current_data = read_file(file)
+            @current = file
+          end
+
+          # Wether we need or not to compute formulas:
+          if col_spec =~ /\$/
+            compute_formulas = true
+          else
+            compute_formulas = false
+          end
+          
+          return Dataset.dataset_from_spec(set, col_spec) do |col|
+            get_data_column(col, compute_formulas)
+          end
+        end
+
+        # Gets the data corresponding to the given column. If
+        # _compute_formulas_ is true, the column specification is
+        # taken to be a formula (in the spirit of gnuplot's)
+        def get_data_column(column, compute_formulas = false)
+          if compute_formulas
+            formula = column.gsub(/\$(\d+)/, 'column[\1]')
+            debug "Using formula #{formula} for column spec: #{column}"
+            return Dvector.compute_formula(formula, 
+                                           @current_data,
+                                           @included_modules)
+          else
+            return @current_data[column.to_i].dup
+          end
+        end
+
+#         # Turns a target => values specification into something usable as
+#         # error bars, that is :xmin, :xmax and the like hashes. The rules
+#         # are the following:
+#         # * ?min/?max are passed on directly;
+#         # * ?e(abs) are transformed into ?min = ? - ?eabs, ?max = ? + ?eabs
+#         # * ?eu(p/?ed(own) are transformed respectively into ? +/- ?...
+#         # * ?er(el) become ?min = ?*(1 - ?erel, ?max = ?(1 + ?erel)
+#         # * ?erup/?erdown follow the same pattern...
+#         def compute_error_bars(values)
+#           target = {}
+#           for key in values.keys
+#             case key.to_s
+#             when /^[xy](min|max)?$/
+#               target[key] = values[key].dup # Just to make sure.
+#             when /^(.)e(a(bs?)?)?$/
+#               target["#{$1}min".to_sym] = values[$1.to_sym] - values[key]
+#               target["#{$1}max".to_sym] = values[$1.to_sym] + values[key]
+#             when /^(.)eu(p)?$/
+#               target["#{$1}max".to_sym] = values[$1.to_sym] + values[key]
+#             when /^(.)ed(o(wn?)?)?$/
+#               target["#{$1}min".to_sym] = values[$1.to_sym] - values[key]
+#             when /^(.)er(el?)?$/
+#               target["#{$1}min".to_sym] = values[$1.to_sym] * 
+#                 (values[key].neg + 1)
+#               target["#{$1}max".to_sym] = values[$1.to_sym] * 
+#                 (values[key] + 1)
+#             when /^(.)erd(o(wn?)?)?$/
+#               target["#{$1}min".to_sym] = values[$1.to_sym] * 
+#                 (values[key].neg + 1)
+#             when /^(.)erup?$/
+#               target["#{$1}max".to_sym] = values[$1.to_sym] * 
+#                 (values[key] + 1)
+#             else
+#               warn "Somehow, the target specification #{key} " +
+#                 "didn't make it through"
+#             end
+#           end
+#           return target
+#         end
+
+      end
+      
+    end
+
+  end
+end