cdo 1.2.7 → 1.3.0

data/gemspec

@@ -3,9 +3,9 @@ $:.unshift File.join(File.dirname(__FILE__),"..","lib")
 
 spec = Gem::Specification.new do |s|
   s.name              = "cdo"
-  s.version           = '1.2.7'
+  s.version           = '1.3.0'
   s.platform          = Gem::Platform::RUBY
-  s.files             = ["lib/cdo.rb","lib/cdo_oo.rb"] + ["gemspec","LICENSE"]
+  s.files             = ["lib/cdo.rb","lib/cdo_lib.rb"] + ["gemspec","LICENSE"]
   s.test_file         = "test/test_cdo.rb"
   s.description       = "Easy access to the Climate Data operators"
   s.summary           = "Easy access to the Climate Data operators"

data/lib/cdo.rb

@@ -3,116 +3,97 @@ require 'open3'
 require 'logger'
 require 'stringio'
 
-# Copyright (C) 2011-2013 Ralf Mueller, ralf.mueller@zmaw.de
-# See COPYING file for copying and redistribution conditions.
-# 
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; version 2 of the License.
-# 
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-
-# ==============================================================================
-# CDO calling mechnism
-module Cdo
-
-  VERSION = "1.2.5"
-  @@file  = StringIO.new
-
-  State = {
-    :debug       => false,
-    :returnCdf   => false,
-    :operators   => [],
-    :forceOutput => true,
-    :env         => {},
-    :log         => false,
-    :logger      => Logger.new(@@file),
-  }
-  State[:debug] = ENV.has_key?('DEBUG')
-  State[:logger].formatter = proc do |serverity, time, progname, msg|
-    msg
+class Cdo
+  OutputOperatorsPattern = /(diff|info|output|griddes|zaxisdes|show|ncode|ndate|nlevel|nmon|nvar|nyear|ntime|npar|gradsdes|pardes)/
+
+  attr_accessor :cdo, :returnCdf, :forceOutput, :env, :debug, :logging, :logFile
+  attr_reader   :operators, :filetypes
+
+  def initialize(cdo: 'cdo',
+                 returnCdf: false,
+                 returnFalseOnError: false,
+                 forceOutput: true,
+                 env: {},
+                 logging: false,
+                 logFile: StringIO.new,
+                 debug: false,
+                 returnNilOnError: false)
+
+    # setup path to cdo executable
+    @cdo = ENV.has_key?('CDO') ? ENV['CDO'] : cdo
+
+    @operators              = getOperators(binary=@cdo)
+    @returnCdf              = returnCdf
+    @forceOutput            = forceOutput
+    @env                    = env
+    @debug                  = ENV.has_key?('DEBUG') ? true : debug
+    @returnNilOnError       = returnNilOnError
+
+    @filetypes              = getFiletypes
+    @returnFalseOnError     = returnFalseOnError
+
+    @logging                = logging
+    @logFile                = logFile
+    @logger                 = Logger.new(@logFile,'a')
+    @logger.level           = Logger::INFO
   end
 
-  @@CDO = ENV['CDO'].nil? ? 'cdo' : ENV['CDO']
-
-  # Since cdo-1.5.4 undocumented operators are given with the -h option. For
-  # earlier version, they have to be provided manually
-  @@undocumentedOperators = %w[anomaly beta boxavg change_e5lsm change_e5mask
-    change_e5slm chisquare chvar cloudlayer cmd com command complextorect
-    covar0 covar0r daycount daylogs del29feb delday delete deltap deltap_fl
-    delvar diffv divcoslat dumplogo dumplogs duplicate eca_r1mm enlargegrid
-    ensrkhistspace ensrkhisttime eof3d eof3dspatial eof3dtime export_e5ml
-    export_e5res fc2gp fc2sp fillmiss fisher fldcovar fldrms fourier fpressure
-    gather gengrid geopotheight ggstat ggstats globavg gp2fc gradsdes
-    gridverify harmonic hourcount hpressure import_e5ml import_e5res
-    import_obs imtocomplex infos infov interpolate intgrid intgridbil
-    intgridtraj intpoint isosurface lmavg lmean lmmean lmstd log lsmean
-    meandiff2test mergegrid mod moncount monlogs mrotuv mrotuvb mulcoslat ncode
-    ncopy nmltest normal nvar outputbounds outputboundscpt outputcenter
-    outputcenter2 outputcentercpt outputkey outputtri outputvector outputvrml
-    pardup parmul pinfo pinfov pressure_fl pressure_hl read_e5ml remapcon1
-    remapdis1 retocomplex scalllogo scatter seascount select selgridname
-    seloperator selvar selzaxisname setrcaname setvar showvar sinfov smemlogo
-    snamelogo sort sortcode sortlevel sortname sorttaxis sorttimestamp sortvar
-    sp2fc specinfo spectrum sperclogo splitvar stimelogo studentt template1
-    template2 test test2 testdata thinout timcount timcovar tinfo transxy trms
-    tstepcount vardes vardup varmul varquot2test varrms vertwind write_e5ml
-    writegrid writerandom yearcount]
-
-  @@outputOperatorsPattern = /(diff|info|output|griddes|zaxisdes|show|ncode|ndate|nlevel|nmon|nvar|nyear|ntime|npar|gradsdes|pardes)/
-
-  private
-  def Cdo.getOperators(force=false)
-    # Do NOT compute anything, if it is not required
-    return State[:operators] unless (State[:operators].empty? or force)
-    cmd       = @@CDO + ' 2>&1'
-    help      = IO.popen(cmd).readlines.map {|l| l.chomp.lstrip}
-    if 5 >= help.size
-      warn "Operators could not get listed by running the CDO binary (#{@@CDO})"
-      pp help if Cdo.debug
-      exit
-    end
-    # in version 1.5.6 the output of '-h' has changed
-    State[:operators] = case 
-                        when Cdo.version < "1.5.6"
-                          (help[help.index("Operators:")+1].split + @@undocumentedOperators).uniq
-                        else
-                          help[(help.index("Operators:")+1)..help.index(help.find {|v| v =~ /CDO version/}) - 2].join(' ').split
-                        end
+  private # {{{
+
+  # split arguments into hash-like args and the rest
+  def Cdo.parseArgs(args)
+    operatorArgs = args.reject {|a| a.class == Hash}
+    opts = operatorArgs.empty? ? '' : ',' + operatorArgs.join(',')
+    io   = args.find {|a| a.class == Hash}
+    io   = {} if io.nil?
+    args.delete_if   {|a| a.class == Hash}
+    # join input streams together if possible
+    io[:input] = io[:input].join(' ') if io[:input].respond_to?(:join)
+
+    return [io,opts]
   end
 
-  def Cdo.hasError(cmd,retvals)
-    if (State[:debug])
-      puts("RETURNCODE: #{retvals[:returncode]}")
-    end
-    if ( 0 != retvals[:returncode] )
-      puts("Error in calling:")
-      puts(">>> "+cmd+"<<<")
-      puts(retvals[:stderr])
-      return true
+  # collect the complete list of possible operators
+  def getOperators(path2cdo)
+    if version <= '1.7.0' then
+      cmd       = path2cdo + ' 2>&1'
+      help      = IO.popen(cmd).readlines.map {|l| l.chomp.lstrip}
+      if 5 >= help.size
+        warn "Operators could not get listed by running the CDO binary (#{path2cdo})"
+        pp help if @debug
+        exit
+      end
+
+      @operators = help[(help.index("Operators:")+1)..help.index(help.find {|v| v =~ /CDO version/}) - 2].join(' ').split
     else
-      return false
+      cmd = "#{path2cdo} --operators"
+
+      @operators = IO.popen(cmd).readlines.map {|l| l.split(' ').first }
     end
   end
 
-  def Cdo.env=(envHash)
-    State[:env] = envHash
+  # get supported IO filetypes form the binary
+  def getFiletypes
+    _, _, stderr, _ = Open3.popen3(@cdo + " -V")
+    supported       = stderr.readlines.map(&:chomp)
+
+    supported.grep(/(Filetypes)/)[0].split(':')[1].split.map(&:downcase)
   end
-  def Cdo.env; State[:env]; end
 
-  def Cdo.call(cmd)
-    if (State[:debug])
+
+  # Execute the given cdo call and return all outputs
+  def _call(cmd)
+    if (@debug)
       puts '# DEBUG ====================================================================='
-      pp Cdo.env unless Cdo.env.empty?
+      pp @env unless @env.empty?
       puts 'CMD: '
       puts cmd
       puts '# DEBUG ====================================================================='
     end
-    stdin, stdout, stderr, wait_thr = Open3.popen3(Cdo.env,cmd)
 
+    @logger.info(cmd+"\n") if @logging
+
+    stdin, stdout, stderr, wait_thr = Open3.popen3(@env,cmd)
     {
       :stdout     => stdout.read,
       :stderr     => stderr.read,
@@ -120,269 +101,211 @@ module Cdo
     }
   end
 
-  def Cdo.run(cmd,ofile='',options='',returnCdf=false,force=nil,returnArray=nil,returnMaArray=nil)
-    cmd = "#{@@CDO} -O #{options} #{cmd} "
+  # Error handling for the given command
+  def _hasError(cmd,retvals)
+    if @debug
+      puts("RETURNCODE: #{retvals[:returncode]}")
+    end
+    if ( 0 != retvals[:returncode] )
+      puts("Error in calling:")
+      puts(">>> "+cmd+"<<<")
+      puts(retvals[:stderr])
+      @logger.error("FAILIURE in execution of:"+cmd+"| msg:"+retvals[:stderr]) if @logging
+      return true
+    else
+      return false
+    end
+  end
+
+  # command execution wrapper, which handles the possible return types
+  def _run(cmd,ofile='',options=nil,returnCdf=false,force=nil,returnArray=nil,returnMaArray=nil)
+    options = options.to_s
+
+    options << '-f nc' if options.empty? and ( \
+                                              (     returnCdf ) or \
+                                              ( not returnArray.nil? ) or \
+                                              ( not returnMaArray.nil?) \
+                                             )
+    cmd = "#{@cdo} -O #{options} #{cmd} "
+
     case ofile
     when $stdout
-      retvals = Cdo.call(cmd)
-      State[:logger].info(cmd+"\n") if State[:log]
-      unless hasError(cmd,retvals)
+      retvals = _call(cmd)
+      unless _hasError(cmd,retvals)
         return retvals[:stdout].split($/).map {|l| l.chomp.strip}
       else
-        raise ArgumentError,"CDO did NOT run successfully!"
+        if @returnNilOnError then
+          return nil
+        else
+          raise ArgumentError,"CDO did NOT run successfully!"
+        end
       end
     else
-      force = State[:forceOutput] if force.nil?
+      force = @forceOutput if force.nil?
       if force or not File.exists?(ofile.to_s)
         ofile = MyTempfile.path if ofile.nil?
         cmd << "#{ofile}"
-        retvals = call(cmd)
-        State[:logger].info(cmd+"\n") if State[:log]
-        if hasError(cmd,retvals)
-          raise ArgumentError,"CDO did NOT run successfully!"
+        retvals = _call(cmd)
+        if _hasError(cmd,retvals)
+          if @returnNilOnError then
+            return nil
+          else
+            raise ArgumentError,"CDO did NOT run successfully!"
+          end
         end
       else
-        warn "Use existing file '#{ofile}'" if Cdo.debug
+        warn "Use existing file '#{ofile}'" if @debug
       end
     end
+
     if not returnArray.nil?
-      Cdo.readArray(ofile,returnArray)
+      readArray(ofile,returnArray)
     elsif not returnMaArray.nil?
-      Cdo.readMaArray(ofile,returnMaArray)
-    elsif returnCdf or State[:returnCdf]
-      Cdo.readCdf(ofile)
+      readMaArray(ofile,returnMaArray)
+    elsif returnCdf or @returnCdf
+      readCdf(ofile)
     else
-      return ofile
+      ofile
     end
   end
 
-  def Cdo.parseArgs(args)
-    # splitinto hash-like args and the rest
-    operatorArgs = args.reject {|a| a.class == Hash}
-    opts = operatorArgs.empty? ? '' : ',' + operatorArgs.join(',')
-    io   = args.find {|a| a.class == Hash}
-    io   = {} if io.nil?
-    args.delete_if   {|a| a.class == Hash}
-    # join input streams together if possible
-    io[:input] = io[:input].join(' ') if io[:input].respond_to?(:join)
-    return [io,opts]
-  end
+  # Implementation of operator calls using ruby's meta programming skills
+  #
+  # args is expected to look like
+  #   [opt1,...,optN,:input => iStream,:output => oStream, :options => ' ']
+  #   where iStream could be another CDO call (timmax(selname(Temp,U,V,ifile.nc))
+  def method_missing(sym, *args, &block)
+    puts "Operator #{sym.to_s} is called" if @debug
 
-  def Cdo.method_missing(sym, *args, &block)
-    ## args is expected to look like [opt1,...,optN,:input => iStream,:output => oStream] where
-    # iStream could be another CDO call (timmax(selname(Temp,U,V,ifile.nc))
-    puts "Operator #{sym.to_s} is called" if State[:debug]
-    if getOperators.include?(sym.to_s)
+    if @operators.include?(sym.to_s)
       io, opts = Cdo.parseArgs(args)
-      if @@outputOperatorsPattern.match(sym)
-        run(" -#{sym.to_s}#{opts} #{io[:input]} ",$stdout)
+      if OutputOperatorsPattern.match(sym.to_s)
+        _run(" -#{sym.to_s}#{opts} #{io[:input]} ",$stdout)
       else
-        run(" -#{sym.to_s}#{opts} #{io[:input]} ",io[:output],io[:options],io[:returnCdf],io[:force],io[:returnArray],io[:returnMaArray])
+        _run(" -#{sym.to_s}#{opts} #{io[:input]} ",io[:output],io[:options],io[:returnCdf],io[:force],io[:returnArray],io[:returnMaArray])
       end
     else
+      return false if @returnFalseOnError
       raise ArgumentError,"Operator #{sym.to_s} not found"
     end
   end
 
-  def Cdo.loadCdf
+  # load the netcdf bindings
+  def loadCdf
     begin
       require "numru/netcdf_miss"
-      include NumRu
     rescue LoadError
       warn "Could not load ruby's netcdf bindings. Please install it."
       raise
     end
   end
 
-  def Cdo.getSupportedLibs(force=false)
-    return unless (State[:libs].nil? or force)
-    _, _, stderr, _ = Open3.popen3(@@CDO + " -V")
-    supported       = stderr.readlines.map(&:chomp)
-    with            = supported.grep(/(with|Features)/)[0].split(':')[1].split.map(&:downcase)
-    libs            = supported.grep(/library version/).map {|l| 
-      l.strip.split(':').map {|l| 
-        l.split.first.downcase
-      }[0,2]
-    }
-    State[:libs] = {}
-    with.flatten.each {|k| State[:libs][k]=true}
-    libs.each {|lib,version| State[:libs][lib] = version}
-  end
-
-  public
-  def Cdo.debug=(value)
-    State[:debug] = value
-  end
-  def Cdo.debug
-    State[:debug]
-  end
-  def Cdo.forceOutput=(value)
-    State[:forceOutput] = value
-  end
-  def Cdo.forceOutput
-    State[:forceOutput]
-  end
-  def Cdo.log=(value)
-    State[:log] = value
-  end
+  # }}}
 
-  def Cdo.log
-    State[:log]
-  end
+  public  # {{{
 
-  def Cdo.version
-    cmd     = @@CDO + ' 2>&1'
-    help    = IO.popen(cmd).readlines.map {|l| l.chomp.lstrip}
-    regexp  = %r{CDO version (\d.*), Copyright}
-    line    = help.find {|v| v =~ regexp}
-    version = regexp.match(line)[1]
+  # show Cdo's built-in help for given operator
+  def help(operator=nil)
+    if operator.nil?
+      puts _call([@cdo,'-h'].join(' '))[:stderr]
+    else
+      operator = operator.to_s
+      puts _call([@cdo,'-h',operator].join(' ')).values_at(:stdout,:stderr)
+    end
   end
 
-  def Cdo.setReturnCdf(value=true)
-    if value
-      Cdo.loadCdf
+  # collect logging messages
+  def collectLogs
+    if @logger.instance_variable_get(:'@logdev').filename.nil?
+      @logFile.rewind
+      return @logFile.read
+    else
+      return File.open(@logFile).readlines
     end
-    State[:returnCdf] = value
   end
-
-  def Cdo.unsetReturnCdf
-    setReturnCdf(false)
+  # print the loggin messaged
+  def showLog
+    puts collectLogs
   end
 
-  def Cdo.returnCdf
-    State[:returnCdf]
-  end
+  # check if cdo backend is working
+  def check
+    return false unless system("#@cdo -h 1>/dev/null 2>&1")
 
-  def Cdo.hasCdo?(bin=@@CDO)
-    return true if File.exists?(@@CDO) and File.executable?(@@CDO)
-    ENV['PATH'].split(File::PATH_SEPARATOR).each {|path| 
-      return true if File.exists?([path,bin].join(File::SEPARATOR))
-    }
-  end
+    retval = _call("#{@cdo} -V")
+    pp retval if @debug
 
-  # test if @@CDO can be used
-  def Cdo.checkCdo
-    unless hasCdo?(@@CDO)
-      warn "Testing application #@@CDO is not available!"
-      exit 1
-    else
-      puts "Using CDO: #@@CDO"
-      puts IO.popen(@@CDO + " -V").readlines
-    end
     return true
   end
 
-  def Cdo.setCdo(cdo)
-    puts "Will use #{cdo} instead of #@@CDO" if Cdo.debug
-    @@CDO = cdo
-    Cdo.getOperators(true)
-    Cdo.getSupportedLibs(true)
-  end
-
-  def Cdo.getCdo
-    @@CDO
+  def version
+    IO.popen("#{@cdo} -V 2>&1").readlines.first.split(/version/i).last.strip.split(' ').first
   end
 
-  def Cdo.operators
-    Cdo.getOperators if State[:operators].empty?
-    State[:operators]
+  # return cdf handle to given file readonly
+  def readCdf(iFile)
+    loadCdf
+    NumRu::NetCDF.open(iFile)
   end
 
-  def Cdo.libs
-    getSupportedLibs
-    State[:libs]
+  # return cdf handle opened in append more
+  def openCdf(iFile)
+    loadCdf
+    NumRu::NetCDF.open(iFile,'r+')
   end
 
-  def Cdo.hasLib?(lib)
-    return Cdo.libs.has_key?(lib)
-    return false
+  # return narray for given variable name
+  def readArray(iFile,varname)
+    filehandle = readCdf(iFile)
+    if filehandle.var_names.include?(varname)
+      # return the data array
+      filehandle.var(varname).get
+    else
+      raise ArgumentError, "Cannot find variable '#{varname}'"
+    end
   end
 
-  def Cdo.libsVersion(lib)
-    unless Cdo.hasLib?(lib)
-      raise ArgumentError, "Cdo does NOT have support for '#{lib}'"
+  # return a masked array for given variable name
+  def readMaArray(iFile,varname)
+    filehandle = readCdf(iFile)
+    if filehandle.var_names.include?(varname)
+      # return the data array
+      filehandle.var(varname).get_with_miss
     else
-      if State[:libs][lib].kind_of? String
-        return State[:libs][lib]
-      else
-        warn "No version information available about '#{lib}'"
-        return false
-      end
+      raise ArgumentError,"Cannot find variable '#{varname}'"
     end
   end
 
-  def Cdo.showlog
-    @@file.rewind
-    puts @@file.read
-  end
+  # }}}
+
+  # Addional operators: {{{
 
-  #==================================================================
-  # Addional operotors:
-  #------------------------------------------------------------------
-  def Cdo.boundaryLevels(args)
-    ilevels         = Cdo.showlevel(:input => args[:input])[0].split.map(&:to_f)
+  # compute vertical boundary levels from full levels
+  def boundaryLevels(args)
+    ilevels         = self.showlevel(:input => args[:input])[0].split.map(&:to_f)
     bound_levels    = Array.new(ilevels.size+1)
     bound_levels[0] = 0
-    (1..ilevels.size).each {|i| 
+    (1..ilevels.size).each {|i|
       bound_levels[i] =bound_levels[i-1] + 2*(ilevels[i-1]-bound_levels[i-1])
     }
     bound_levels
   end
 
-  def Cdo.thicknessOfLevels(args)
-    bound_levels = Cdo.boundaryLevels(args)
+  # compute level thicknesses from given full levels
+  def thicknessOfLevels(args)
+    bound_levels = self.boundaryLevels(args)
     delta_levels    = []
-    bound_levels.each_with_index {|v,i| 
+    bound_levels.each_with_index {|v,i|
       next if 0 == i
       delta_levels << v - bound_levels[i-1]
     }
     delta_levels
   end
 
-  def Cdo.readCdf(iFile)
-    Cdo.loadCdf unless State[:returnCdf] 
-    NetCDF.open(iFile)
-  end
-
-  def Cdo.openCdf(iFile)
-    Cdo.loadCdf unless State[:returnCdf] 
-    NetCDF.open(iFile,'r+')
-  end
-
-  def Cdo.readArray(iFile,varname)
-    filehandle = Cdo.readCdf(iFile)
-    if filehandle.var_names.include?(varname)
-      # return the data array
-      filehandle.var(varname).get
-    else
-      raise ArgumentError, "Cannot find variable '#{varname}'"
-    end
-  end
+  # }}}
 
-  def Cdo.readMaArray(iFile,varname)
-    filehandle = Cdo.readCdf(iFile)
-    if filehandle.var_names.include?(varname)
-      # return the data array
-      filehandle.var(varname).get_with_miss
-    else
-      raise ArgumentError,"Cannot find variable '#{varname}'"
-    end
-  end
-
-  def Cdo.help(operator=nil)
-    if operator.nil?
-      puts Cdo.call([@@CDO,'-h'].join(' '))[:stderr]
-    else
-      operator = operator.to_s unless String == operator.class
-      if Cdo.operators.include?(operator)
-        puts Cdo.call([@@CDO,'-h',operator].join(' '))[:stdout]
-      else
-        puts "Unknown operator #{operator}"
-      end
-    end
-  end
 end
-
+#
 # Helper module for easy temp file handling
 module MyTempfile
   require 'tempfile'
@@ -411,3 +334,5 @@ module MyTempfile
     @@_tempfiles.each {|f| print(f+" ") if f.kind_of? String}
   end
 end
+
+#vim:fdm=marker