ctioga2 0.0 → 0.1

data/lib/ctioga2/commands/parsers/file.rb

@@ -19,7 +19,7 @@ require 'ctioga2/commands/strings'
 
 module CTioga2
 
-  Version::register_svn_info('$Revision: 2 $', '$Date: 2009-04-25 14:03:30 +0200 (Sat, 25 Apr 2009) $')
+  Version::register_svn_info('$Revision: 151 $', '$Date: 2010-06-19 23:45:20 +0200 (Sat, 19 Jun 2010) $')
 
   module Commands
 
@@ -96,11 +96,13 @@ module CTioga2
                 # Problem: the space on the right of the = sign is
                 # *significant*. 
                 for o in args
-                  if o =~ /^\s*([\w-]+)\s*=(.*)/
+                  if o =~ /^\s*\/?([\w-]+)\s*=(.*)/
                     if cmd.has_option? $1
                       options[$1] = $2
                     else
-                      error "Command #{cmd.name} does not take option #{$1}"
+                      error { 
+                        "Command #{cmd.name} does not take option #{$1}" 
+                      }
                     end
                   end
                 end

data/lib/ctioga2/commands/type.rb

@@ -16,11 +16,16 @@ require 'ctioga2/metabuilder/types'
 
 module CTioga2
 
-  Version::register_svn_info('$Revision: 18 $', '$Date: 2009-04-28 23:43:54 +0200 (Tue, 28 Apr 2009) $')
+  Version::register_svn_info('$Revision: 173 $', '$Date: 2010-10-22 21:41:39 +0200 (Fri, 22 Oct 2010) $')
 
   module Commands
 
     # A named type, based on CTioga2::MetaBuilder::Type
+    #
+    # @todo *Structural* in real, I don't think it is necessary
+    # anymore to rely on MetaBuilder, as most types in CTioga2 already
+    # provide a from_text class function that does a nice job. I
+    # should convert as many things as possible to using that.
     class CommandType
 
       # The underlying  CTioga2::MetaBuilder::Type object.
@@ -43,7 +48,9 @@ module CTioga2
           return obj
         else 
           if obj.is_a? Symbol
-            warn "Converting type specifcation #{obj.inspect} to string at #{caller[1]}"
+            warn {
+              "Converting type specification #{obj.inspect} to string at #{caller[1]}"
+            }
             obj = obj.to_s
           end
           type = Interpreter::type(obj)
@@ -87,7 +94,7 @@ module CTioga2
 
       # Returns the long option for the option parser.
       #
-      # TODO: maybe this should be rethought a bit ?
+      # \todo maybe this should be rethought a bit ?
       def option_parser_long_option(name, param = nil)
         return @type.option_parser_long_option(name, param)
       end      

data/lib/ctioga2/commands/variables.rb

@@ -16,7 +16,7 @@ require 'ctioga2/commands/strings'
 
 module CTioga2
 
-  Version::register_svn_info('$Revision: 2 $', '$Date: 2009-04-25 14:03:30 +0200 (Sat, 25 Apr 2009) $')
+  Version::register_svn_info('$Revision: 155 $', '$Date: 2010-06-21 21:41:32 +0200 (Mon, 21 Jun 2010) $')
 
   module Commands
 
@@ -43,7 +43,7 @@ module CTioga2
     #    variable = value
     #   They are stored in the form on an InterpreterString
     # 
-    # TODO: The variables system should automatically transform
+    # \todo The variables system should automatically transform
     # recursive variables into immediate ones when there is no
     # variables replacement text.
     class Variables

data/lib/ctioga2/data/backends/backend.rb

@@ -21,7 +21,7 @@ require 'ctioga2/data/backends/description'
 
 module CTioga2
 
-  Version::register_svn_info('$Revision: 59 $', '$Date: 2009-05-28 23:15:50 +0200 (Thu, 28 May 2009) $')
+  Version::register_svn_info('$Revision: 155 $', '$Date: 2010-06-21 21:41:32 +0200 (Mon, 21 Jun 2010) $')
   
 
   module Data
@@ -36,7 +36,7 @@ module CTioga2
       # to retrive data from some source with a consistent way to do so,
       # independent the kind of source accessed.
       #
-      # TODO: update documentation.
+      # \todo update documentation.
       #
       # Subclasses should:
       # * provide a consistent method for creating themselves,
@@ -55,11 +55,11 @@ module CTioga2
       # * wether the actual reading of the data is done at initialization time
       #   or at query time is left to the implementor ;-) !
       #
-      # TODO: adapt to the new structure.
+      # \todo adapt to the new structure.
       #
-      # TODO: add back filters (with time)
+      # \todo add back filters (with time)
       #
-      # TODO: add a Cache ?
+      # \todo add a Cache ?
       class Backend
 
         # Include logging facilities...
@@ -106,7 +106,7 @@ This is the base class for backends. It should never be used directly.
 EOD
 
 
-        # TODO: the baseline should not be implemented.
+        # \todo the baseline should not be implemented.
         # It is much more efficient to ;
         # * implement a dataset subtraction command;
         # * use the add-dataset hook to automatically subtract a given
@@ -159,7 +159,7 @@ EOD
         # code while keeping old stuff working.
         def expand_sets(spec)
           if m = /(\d+)##(\d+)/.match(spec)
-            debug "Using expansion rule #1"
+            debug { "Using expansion rule #1" }
             a = m[1].to_i
             b = m[2].to_i
             ret = []
@@ -168,10 +168,10 @@ EOD
             end
             return ret
           elsif m = /\#<(\d+)<(.*?)>(\d+)>/.match(spec)
-            debug "Using expansion rule #2"
+            debug { "Using expansion rule #2" }
             from = m[1].to_i
             to = m[3].to_i
-            debug "Ruby code used for expansion: {|i| #{m[2]} }"
+            debug { "Ruby code used for expansion: {|i| #{m[2]} }" }
             code = eval "proc {|i| #{m[2]} }"
             ret = []
             from.upto(to) do |i|
@@ -179,13 +179,13 @@ EOD
             end
             return ret
           elsif m = /\#<\s*(\w+)\s*=\s*(\d+)\s*<(.*?)>\s*(\d+)\s*>/.match(spec)
-            debug "Using expansion rule #3"
+            debug { "Using expansion rule #3" }
             var = m[1]
             from = m[2].to_i
             to = m[4].to_i
             # Then we replace all occurences of the variable
             literal = '"' + m[3].gsub(/\b#{var}\b/, '#{' + var + '}') + '"'
-            debug "Ruby code used for expansion: {|#{var}| #{literal} }"
+            debug { "Ruby code used for expansion: {|#{var}| #{literal} }" }
             code = eval "proc {|#{var}| #{literal} }"
             ret = []
             from.upto(to) do |i|
@@ -197,10 +197,12 @@ EOD
           return [spec]
         rescue  Exception => ex
           # In case something went wrong in the eval.
-          warn "An error occured during expansion of '#{spec}': #{ex.message}"
-          debug "Error backtrace: #{ex.backtrace.join "\n"}"
-          warn "Ignoring, but you're nearly garanteed something will "+
+          warn { "An error occured during expansion of '#{spec}': #{ex.message}" }
+          debug { "Error backtrace: #{ex.backtrace.join "\n"}" }
+          warn {
+            "Ignoring, but you're nearly garanteed something will "+
             "fail later on"
+          }
           return [spec]
         end
 
@@ -242,7 +244,7 @@ EOD
         # * keys inside _exclude_ are removed.
         # * _supp_info_ is added
         #
-        # TODO: get the implementation back again.
+        # \todo get the implementation back again.
         def get_cached_entry(name, exclude = [], supp_info = {}, &code)
           raise YetUnimplemented
           state = save_state

data/lib/ctioga2/data/backends/backends.rb

@@ -33,7 +33,7 @@ for file in files.uniq
   begin
     require "ctioga2/data/backends/backends/#{file}"
   rescue Exception => e
-    warn "There was a problem trying to load 'ctioga2/data/backends/backends/#{file}': " 
-    warn "#{e.inspect}"
+    warn { "There was a problem trying to load 'ctioga2/data/backends/backends/#{file}': "  }
+    warn { "#{e.inspect}" }
   end
 end

data/lib/ctioga2/data/backends/backends/gnuplot.rb

@@ -25,7 +25,7 @@ require 'stringio'
 
 module CTioga2
 
-  Version::register_svn_info('$Revision: 61 $', '$Date: 2009-05-29 01:00:56 +0200 (Fri, 29 May 2009) $')
+  Version::register_svn_info('$Revision: 151 $', '$Date: 2010-06-19 23:45:20 +0200 (Sat, 19 Jun 2010) $')
 
 
   module Data
@@ -50,6 +50,10 @@ EOD
         param_accessor :range, 'range', 
         "Plotting X range", 'float-range', 
         "The plotting X range, such as 0:2"
+
+        param_accessor :samples, 'samples', 
+        "The number of samples", 'text', 
+        "The number of samples"
         
         # This is called by the architecture to get the data. It splits
         # the set name into filename@cols, reads the file if necessary and
@@ -76,20 +80,31 @@ EOD
         def run_gnuplot(filename, overrides = @variables_overrides)
           date = File::mtime(filename)
           # Get it from the cache !
-          debug "Running gnuplot on file #{filename}"
+          debug { "Running gnuplot on file #{filename}" }
           f = File.open(filename)
           # We open a bidirectionnal connection to gnuplot:
           gnuplot = IO.popen("gnuplot", "r+")
           output = ""
-          gnuplot.puts "set term table"
+          ## \todo determine gnuplot version for choosing which one we
+          ## want to use.
+          # gnuplot.puts "set term table"
+          gnuplot.puts "set table"
+          if @samples
+            overrides ||= ""
+            overrides += ";set samples #{@samples}"
+          end
           for line in f
             next if line =~ /set\s+term/
             if overrides and line =~ /plot\s+/
-              debug "Found a plot, inserting variable overrides: #{overrides}"
+              debug { 
+                "Found a plot, inserting variable overrides: #{overrides}" 
+              }
               line.gsub!(/plot\s+/, "#{overrides};plot ")
             end
             if @range and line =~ /plot\s+/
-              debug "Found a plot, inserting range: #{@range}"
+              debug { 
+                "Found a plot, inserting range: #{@range}" 
+              }
               line.gsub!(/plot\s+(\[[^\]]+\])?/, 
                          "plot [#{@range}]")
             end

data/lib/ctioga2/data/backends/backends/math.rb

@@ -21,7 +21,7 @@ require 'Dobjects/Function'
 
 module CTioga2
 
-  Version::register_svn_info('$Revision: 17 $', '$Date: 2009-04-28 22:22:22 +0200 (Tue, 28 Apr 2009) $')
+  Version::register_svn_info('$Revision: 155 $', '$Date: 2010-06-21 21:41:32 +0200 (Mon, 21 Jun 2010) $')
 
   module Data
 
@@ -36,7 +36,7 @@ module CTioga2
 This backend returns computations of mathematical formulas.
 EOD
 
-        # TODO: make provisions for 3-D datasets. Ideas: x(t):y(t):z(t)
+        # \todo make provisions for 3-D datasets. Ideas: x(t):y(t):z(t)
         # for parametric plots ? (possibly x(t):y1(t):y2(t):...:yn(t)) ?
 
         param_accessor :samples, 'samples', "Samples", 'integer', 

data/lib/ctioga2/data/backends/backends/text.rb

@@ -25,7 +25,7 @@ require 'stringio'
 
 module CTioga2
 
-  Version::register_svn_info('$Revision: 17 $', '$Date: 2009-04-28 22:22:22 +0200 (Tue, 28 Apr 2009) $')
+  Version::register_svn_info('$Revision: 191 $', '$Date: 2010-11-07 15:53:08 +0100 (Sun, 07 Nov 2010) $')
 
 
   module Data
@@ -49,6 +49,7 @@ module CTioga2
           ".bz2" => "bunzip2 -c %s",
           ".lzma" => "unlzma -c %s",
           ".lz" => "unlzma -c %s",
+          ".xz" => "unxz -c %s",
         }
 
         include Dobjects
@@ -76,9 +77,14 @@ EOD
         'regexp', 
         "The columns separator. Defaults to /\s+/"
 
+        param_accessor :param_regex, 'parameters', "Parameters parsing", 
+        'regexp', 
+        "Regular expression for extracting parameters from a file. Defaults to nil (ie nothing)"
 
-        #     param_accessor :select, 'select', "Select lines", {:type => :string},
-        #     "Skips line where the code returns false"
+        param_accessor :header_line_regex, 'header-line', 
+        'Header line regular expression', 
+        'regexp', 
+        "Regular expression indicating the header line (containing column names) (default /^##/"
         
         def initialize
           @dummy = nil
@@ -95,11 +101,20 @@ EOD
           # We don't split data by default.
           @split = false
 
+          @param_regex = nil
+
+          @header_line_regex = /^\#\#\s*/
+
           super()
 
           # Override Backend's cache - for now.
           @cache = {}               # A cache file_name -> data
 
+          @param_cache = {}     # Same thing as cache, but for parameters
+
+          @headers_cache = {}   # Same thing as cache, but for header
+                                # lines.
+
         end
 
         def extend(mod)
@@ -147,20 +162,20 @@ EOD
             # Try to find a compressed version
             for ext,method in UNCOMPRESSORS
               if File.readable? "#{file}#{ext}"
-                info "Using compressed file #{file}#{ext} in stead of #{file}"
+                info { "Using compressed file #{file}#{ext} in stead of #{file}" }
                 return IO.popen(method % "#{file}#{ext}")
               end
             end
           else 
             for ext, method in UNCOMPRESSORS
               if file =~ /#{ext}$/ 
-                info "Taking file #{file} as a compressed file"
+                info { "Taking file #{file} as a compressed file" }
                 return IO.popen(method % file)
               end
             end
             return File::open(file)
           end
-          error "Could not open #{file}"
+          error { "Could not open #{file}" }
           return nil
         end
 
@@ -182,11 +197,11 @@ EOD
           while line = io.gets
             line_number += 1
             if line =~ InvalidLineRE
-              debug "Found invalid line at #{line_number}"
+              debug { "Found invalid line at #{line_number}" }
               if ! last_line_is_invalid
                 # We begin a new set.
                 cur_set += 1
-                debug "Found set #{cur_set} at line #{line_number}"
+                debug { "Found set #{cur_set} at line #{line_number}" }
                 if(cur_set > set)
                   return str
                 end
@@ -214,32 +229,99 @@ EOD
             else
               set = 1
             end
-            debug "Trying to get set #{set} from file '#{filename}'"
+            debug { "Trying to get set #{set} from file '#{filename}'" }
             str = get_set_string(get_io_object(filename), set)
             return StringIO.new(str)
           end
         end
 
+        undef :param_regex=
+        # A proper writer for @param_regex
+        def param_regex=(val)
+          if val.is_a? Regexp
+            @param_regex = val
+          elsif val =~ /([^\\]|^)\(/     # Has capturing groups
+            @param_regex = /#{val}/
+          else                  # Treat as separator
+            @param_regex = /(\S+)\s*#{val}\s*(\S+)/
+          end
+        end
+
+        # Turns an array of comments into a hash[param] -> value
+        def parse_parameters(comments)
+          ret = {}
+          for line in comments
+            if line =~ @param_regex
+              ret[$1] = $2.to_f
+            end
+          end
+          return ret
+        end
+
+        # Turns an array of comments into a hash column name -> column
+        # number (1-based)
+        def parse_header_line(comments)
+          for line in comments
+            if line =~ @header_line_regex
+              colnames = line.gsub(@header_line_regex,'').split(@separator)
+              i = 1
+              ret = {}
+              for n in colnames
+                ret[n] = i
+                i += 1
+              end
+              return ret
+            end
+          end
+          return {}
+        end
+
         # Reads data from a file. If needed, extract the file from the
         # columns specification.
         #
-        # TODO: the cache really should include things such as time of
+        # \todo the cache really should include things such as time of
         # last modification and various parameters that influence the
-        # reading of the file.
+        # reading of the file, and the parameters read from the file
+        # using #parse_parameters
+        #
+        # \todo There should be a real global handling of meta-data
+        # extracted from files, so that they could be included for
+        # instance in the automatic labels ? (and we could have fun
+        # improving this one ?)
+        #
+        # \warning This needs Tioga r561
         def read_file(file)
           if file =~ /(.*)@.*/
             file = $1
           end
           name = file               # As file will be modified.
           if ! @cache.key?(file)    # Read the file if it is not cached.
+            comments = []
             fancy_read_options = {'index_col' => true,
               'skip_first' => @skip,
-              'sep' => @separator
+              'sep' => @separator,
+              'comment_out' => comments
             }
             io_set = get_io_set(file)
-            debug "Fancy read '#{file}', options #{fancy_read_options.inspect}"
+            debug { "Fancy read '#{file}', options #{fancy_read_options.inspect}" }
             @cache[name] = Dvector.fancy_read(io_set, nil, fancy_read_options)
+            if @param_regex
+              # Now parsing params
+              @param_cache[name] = parse_parameters(comments)
+              info { "Read #{@param_cache[name].size} parameters from #{name}" }
+              debug { "Parameters read: #{@param_cache[name].inspect}" }
+            end
+            if @header_line_regex
+              @headers_cache[name] = parse_header_line(comments)
+              info { "Read #{@headers_cache[name].size} column names from #{name}" }
+              debug { "Got: #{@headers_cache[name].inspect}" }
+            end
           end
+          ## @todo These are not very satisfying; ideally, the data
+          ## information should be embedded into @cache[name] rather
+          ## than as external variables. Well...
+          @current_parameters = @param_cache[name]
+          @current_header = @headers_cache[name]
           return @cache[name]
         end
 
@@ -268,17 +350,30 @@ EOD
           end
           
           return Dataset.dataset_from_spec(set, col_spec) do |col|
-            get_data_column(col, compute_formulas)
+            get_data_column(col, compute_formulas, 
+                            @current_parameters, @current_header)
           end
         end
 
         # Gets the data corresponding to the given column. If
         # _compute_formulas_ is true, the column specification is
         # taken to be a formula (in the spirit of gnuplot's)
-        def get_data_column(column, compute_formulas = false)
+        def get_data_column(column, compute_formulas = false, 
+                            parameters = nil, header = nil)
           if compute_formulas
-            formula = column.gsub(/\$(\d+)/, 'column[\1]')
-            debug "Using formula #{formula} for column spec: #{column}"
+            formula = column
+            if parameters
+              for k,v in parameters
+                formula.gsub!(/\b#{k}\b/, v.to_s)
+              end
+            end
+            formula.gsub!(/\$(\d+)/, 'column[\1]')
+            if header
+              for k,v in header
+                formula.gsub!("$#{k}$", "column[#{v}]")
+              end
+            end
+            debug { "Using formula #{formula} for column spec: #{column}" }
             return Dvector.compute_formula(formula, 
                                            @current_data,
                                            @included_modules)