caruby-core 1.4.1

data/lib/caruby/cli/application.rb

@@ -0,0 +1,36 @@
+require 'logger'
+require 'caruby/util/log'
+
+module CaRuby
+  module CLI
+    # Extends the standard Logger::Application to use the {CaRuby::Log} and add start
+    # functionality.
+    class Application < Logger::Application
+      # @param [String] appname the application name
+      def initialize(appname=nil)
+        super(appname)
+        @log = Log::instance.logger
+        @log.progname = @appname
+        @level = @log.level
+      end
+      
+      # Overrides Logger::Application start with the following enhancements:
+      # * pass arguments and a block to the application run method
+      # * improve the output messages
+      # * print an exception to stderr as well as the log
+      def start(*args, &block)
+        # Adapted from Logger.
+        status = 1
+        begin
+          log(INFO, "Starting #{@appname}...")
+          status = run(*args, &block)
+        rescue
+          log(FATAL, "#{@appname} detected an exception: #{$!}\n#{$@.qp}")
+          $stderr.puts "#{@appname} was unsuccessful: #{$!}.\nSee the log #{Log.instance.file} for more information."
+        ensure
+          log(INFO, "#{@appname} completed with status #{status}.")
+        end
+      end
+    end
+  end
+end

data/lib/caruby/cli/command.rb

@@ -0,0 +1,169 @@
+require 'optparse'
+require 'caruby/cli/application'
+
+module CaRuby
+  module CLI
+    # Command-line parsing errors.
+    class CommandError < StandardError; end
+      
+    # Command-line parser and executor.
+    class Command < Application
+      # Command line application wrapper.
+      # The specs parameter is an array of command line option and argument
+      # specifications as follows:
+      #
+      # Each option specification is an array in the form:
+      #   [:option, short, long, class, description]
+      # where:
+      # * :option is the option symbol, e.g. +:output+
+      # * short is the short option form, e.g. "-o"
+      # * long is the long option form, e.g. "--output FILE"
+      # * class is the option value class, e.g. Integer
+      # * description is the option usage, e.g. "Output file"
+      # The :option, long and description items are required; the short and class items can
+      # be omitted.
+      #
+      # Each command line argument specification is an array in the form:
+      #   [:arg, text]
+      # where:
+      # * :arg is the argument symbol, e.g. +:input+
+      # * text is the usage message text, e.g. 'input', '[input]' or 'input ...' 
+      # Both items are required.
+      #
+      # Built-in options include the following:
+      # * --help : print the help message and exit
+      # * --version : print the version and exit
+      # * --log FILE : log file
+      # * --debug : print debug messages to the log
+      # * --file FILE: configuration file containing other options
+      # This class processes these built-in options, with the exception of +--version+,
+      # which is a subclass responsibility. Subclasses are responsible for
+      # processing any remaining options.
+      #
+      # @param [(<Symbol>, <String, Class>)] specs the arguments and options
+      #   described above
+      # @yield [hash] the command executor
+      # @yieldparam [{Symbol => Object}] the argument and option symbol => value hash
+      def initialize(specs=[], &executor)
+        unless block_given? then
+          raise ArgumentError.new("Command #{self.class} is missing the required execution block" )
+        end
+        @executor = executor
+        @opt_specs, @arg_specs = specs.partition { |spec| spec[1][0, 1] == '-' }
+        @opt_specs.concat(DEF_OPTS)
+        super($0)
+      end
+  
+      # Runs this command by calling the +execute+ method on the parsed command line
+      # option or argument symbol => value hash.
+      def run
+        # the option => value hash
+        hash = get_opts
+        # this base class's options
+        handle_caruby_options(hash)
+        # add the argument => value hash
+        hash.merge!(get_args)
+        # call the block
+        @executor.call(hash)
+      end
+      
+      # Collects the command line options.
+      #
+      # @return [{Symbol => Object}] the option => value hash 
+      def get_opts
+        opts = {}
+        # the option parser
+        OptionParser.new do |parser|
+          arg_s = @arg_specs.map { |spec| spec[1] }.join(' ')
+          parser.banner = "Usage: #{parser.program_name} [options] #{arg_s}"
+          parser.separator ""
+          parser.separator "Options:"
+          opts = parse(parser)
+          @usage = parser.help
+        end
+        opts
+      end
+  
+      # Collects the non-option command line arguments.
+      #
+      # @return [{Symbol => Object}] the argument => value hash 
+      def get_args
+        return Hash::EMPTY_HASH if ARGV.empty?
+        args = {}
+        n = [ARGV.size, @arg_specs.size - 1].min
+        n.times { |i| args[@arg_specs[i].first] = ARGV[i] }
+        if n < ARGV.size then
+          arg, form = @arg_specs.last
+          if form.index('...') then
+            args[arg] = ARGV[n..-1]
+          elsif @arg_specs.size == ARGV.size then
+            args[arg] = ARGV[n]
+          else
+            halt("Too many arguments", 1)
+          end
+        end
+        args
+      end
+  
+      private
+      
+      DEF_OPTS = [
+        [:help, "--help", "Displays this help message"],
+        [:file, "--file FILE", "Configuration file containing other options"],
+        [:log, "--log FILE", "Log file"],
+        [:debug, "--debug", "Displays debug log messages"],
+      ]
+      
+      # @param [OptionParser] parser the option parser
+      # @return [{Symbol => Object}] the option => value hash
+      def parse(parser)
+        opts = {}
+        @opt_specs.each do |opt, *spec|
+          parser.on_tail(*spec) { |v| opts[opt] = v }
+        end
+        # build the option => value hash 
+        parser.parse!
+        opts
+      end
+      
+      # Processes the built-in options.
+      #
+      # @param [{Symbol => Object}] the option => value hash
+      def handle_caruby_options(opts)
+        # if help, then print usage and exit
+        if opts[:help] then halt end
+        
+        # open the log file
+        log = opts[:log]
+        debug = opts[:debug]
+        if log then
+          CaRuby::Log.instance.open(log, :debug => debug)
+        elsif debug then
+          CaRuby::logger.level = Logger::DEBUG
+        end
+        
+        # if there is a file option, then load additional options from the file
+        file = opts.delete(:file)
+        if file then
+          props = CaRuby::Properties.new(file)
+          props.each { |opt, arg| ARGV << "--#{opt}" << arg }
+          OptionParser.new do |p|
+            opts.merge!(parse(p)) { |ov, nv| ov ? ov : nv }
+          end
+        end
+      end
+      
+      # Prints the given error message and the program usage, then exits with status 1.
+      def fail(message=nil)
+        halt(message, 1)
+      end
+  
+      # Prints the given message and program usage, then exits with the given status.
+      def halt(message=nil, status=0)
+        print(message) if message
+        print(@usage)
+        exit(status)
+      end
+    end
+  end
+end

data/lib/caruby/csv/csv_mapper.rb

@@ -0,0 +1,157 @@
+require 'caruby/csv/csvio'
+require 'caruby/util/properties'
+
+module CaRuby
+  # Maps a CSV extract to a caBIG application.
+  class CsvMapper
+    attr_reader :csvio, :classes
+
+    # Creates a new CsvMapper from the following parameters:
+    # * the required mapping configuration file config
+    # * the required target class
+    # * the required CSV file name
+    # * additional CsvIO options as desired
+    #
+    # If the converter block is given to this method, then that block is called to convert
+    # source CSV field values as described in the FasterCSV.
+    def initialize(config, target, csv, options={}, &converter) # :yields: value, info
+      @target = target
+      # load the config
+      fld_path_hash = load_config(config)
+      # the default input fields are obtained by CsvIO from the first line of the input;
+      # the default output fields are the field mapping config keys in order
+      options[:headers] ||= config_headers(config) if options[:mode] =~ /^w/
+      # the CSV wrapper; do this before making the header map since the CsvIO-generated headers
+      # are used to build the header map
+      @csvio = CsvIO.new(csv, options) do |value, info|
+        # nonstring headers are determined later in this initializer
+        if value and @string_headers.include?(info.header) then
+          value
+        elsif block_given? then
+          # call custom converter first, if any
+          yield(value, info)
+        end
+      end
+      # the class => paths hash; populated in map_headers
+      @cls_paths_hash = LazyHash.new { Set.new }
+      # the path => header hash; do this after making the CsvIO
+      @cls_paths_hash, @hdr_map = map_headers(fld_path_hash)
+      # the top-level classes
+      klasses = @cls_paths_hash.keys
+      # include the target class
+      @cls_paths_hash[@target] ||= Set.new
+      # add superclass paths into subclass paths
+      @cls_paths_hash.each do |klass, paths|
+        @cls_paths_hash.each { |other, other_paths| paths.merge!(other_paths) if klass < other }
+      end
+      # include only concrete classes
+      @classes = @cls_paths_hash.keys
+      @cls_paths_hash.delete_if do |klass, paths|
+        klass.abstract? or klasses.any? { |other| other < klass }
+      end
+      # collect the non-string input fields for the custom CSVLoader converter
+      @string_headers = Set.new
+      @hdr_map.each do |path, cls_hdr_hash|
+        last = path.last
+        @string_headers.merge!(cls_hdr_hash.values) if AttributeMetadata === last and last.type == String
+      end
+    end
+
+    # Returns the given klass's mapped AttributeMetadata paths.
+    # The default klass is the target class.
+    def paths(klass=nil)
+      klass ||= @target
+      @cls_paths_hash[klass]
+    end
+
+    # Returns the header mapped by the given AttributeMetadata path and starting klass.
+    # The default klass is the target class.
+    def header(path, klass=nil)
+      klass ||= @target
+      @hdr_map[path][klass]
+    end
+
+    private
+
+    # Returns the field => path list hash from the field mapping configuration file.
+    def load_config(file)
+      begin
+        config = YAML::load_file(file)
+      rescue
+        raise ConfigurationError.new("Could not read field mapping configuration file #{file}: " + $!)
+      end
+    end
+
+    def config_headers(config)
+      File.open(config) do |file|
+        file.map { |line| line[/(^.+):/, 1] }.compact
+      end
+    end
+
+    # @param [{Symbol => <AttributeMetadata>}] config the field => path list configuration
+    # @return [({Symbol => <AttributeMetadata>}, {Class => {<AttributeMetadata> => Symbol>}})]
+    #   the class => paths hash and the path => class => header hash
+    def map_headers(config)
+      # the class => paths hash; populated in map_headers
+      cls_paths_hash = LazyHash.new { Set.new }
+      hdr_map = LazyHash.new { Hash.new }
+      config.each do |field, attr_list|
+        next if attr_list.blank?
+        # the header accessor method for the field
+        header = @csvio.accessor(field)
+        raise ConfigurationError.new("Field defined in field mapping configuration not found: #{field}") if header.nil?
+        attr_list.split(/,\s*/).each do |path_s|
+          klass, path = create_attribute_path(path_s)
+          hdr_map[path][klass] = header
+          # associate the class with the path
+          cls_paths_hash[klass] << path
+        end
+      end
+      [cls_paths_hash, hdr_map]
+    end
+
+    # Returns an array of AttributeMetadata or symbol objects for the period-delimited path string path_s in the
+    # pattern (_class_|_attribute_)(+.+_attribute_)*, e.g.:
+    #   ClinicalStudy.status
+    #   study.status
+    # The default starting class is this CvsMapper's target class.
+    # Raises ConfigurationError if the path string is malformed or an attribute is not found.
+    def create_attribute_path(path_s)
+      names = path_s.split('.')
+      # if the path starts with a capitalized class name, then resolve the class.
+      # otherwise, the target class is the start of the path.
+      klass = names.first =~ /^[A-Z]/ ? @target.domain_module.const_get(names.shift) : @target
+      # there must be at least one attribute
+      if names.empty? then
+        raise ConfigurationError.new("Attribute entry in CSV field mapping is not in <class>.<attribute> format: #{value}")
+      end
+      # build the AttributeMetadata path by traversing the names path
+      # if the name corresponds to a parent attribute, then add the attribute metadata.
+      # otherwise, if the name is a method, then add the method.
+      path = []
+      names.inject(klass) do |parent, name|
+        attr_md = parent.class.attribute_metadata(name) rescue nil
+        if attr_md then
+          # name is an attribute: add the attribute metadata and navigate to the attribute domain type
+          path << attr_md
+          attr_md.type
+        elsif parent.method_defined?(name) then
+          # name is not a pre-defined attribute but is a method: add the method symbol to the path and halt traversal
+          path << name.to_sym
+          break
+        else
+          # method not defined
+          raise ConfigurationError.new("CSV field mapping attribute not found: #{parent.qp}.#{name}")
+        end
+      end
+      # add remaining non-attribute symbols
+      tail = names[path.size..-1].map { |name| name.to_sym }
+      path.concat(tail)
+      # return the starting class and path
+      # Note that the starting class is not necessarily the first path AttributeMetadata declarer, since the
+      # starting class could be a concrete subclass of an abstract declarer. this is important, since the class
+      # must be instantiated.
+      [klass, path]
+    end
+  end
+end

data/lib/caruby/csv/csvio.rb

@@ -0,0 +1,185 @@
+require 'rubygems'
+gem 'fastercsv'
+
+require 'fileutils'
+require 'faster_csv'
+require 'caruby/util/options'
+require 'caruby/util/collection'
+
+# CsvIO reads or writes CSV records.
+# This class wraps a FasterCSV with the following modifications:
+# * relax the date parser to allow dd/mm/yyyy dates
+# * don't convert integer text with a leading zero to an octal number
+# * allow one custom converter with different semantics: if the converter block
+#   call returns nil, then continue conversion, otherwise return the converter
+#   result. This differs from FasterCSV converter semantics which calls converters
+#   as long the result ==  the input field value. The CsvIO converter semantics
+#   supports converters that intend a String result to be the converted result.
+#
+# CsvIO is Enumerable, but does not implement the complete Ruby IO interface.
+class CsvIO
+  include Enumerable
+
+  # Returns the CSV field access header symbols.
+  attr_reader :headers
+
+  # Opens the CSV file and calls the given block with this CsvIO as the argument.
+  #
+  # @see #initialize the supported options
+  def self.open(file, options=nil) # :yields: csvio
+    csvio = self.new(file, options)
+    if block_given? then
+      yield csvio
+      csvio.close
+    end
+  end
+
+  # #open the given CSV file and options, and call {#each} with the given block.
+  def self.foreach(file, options=nil, &block) # :yields: record
+    self.open(file, options=nil) { |csvio| csvio.each(&block) }
+  end
+
+  # Creates a new CsvIO for the specified source file.
+  # If a converter block is given, then it is added to the CSV converters list.
+  def initialize(file, options=nil, &converter)
+    # the CSV file open mode
+    mode = Options.get(:mode, options, "r")
+    # the CSV headers option; can be boolean or array
+    hdr_opt = Options.get(:headers, options)
+    # there is a header record by default for an input CSV file
+    hdr_opt ||= true if mode =~ /^r/
+    # make parent directories if necessary for an output CSV file
+    File.makedirs(File.dirname(file)) if mode =~ /^w/
+    # if headers aren't given, then convert the input CSV header record names to underscore symbols
+    hdr_cvtr = :symbol unless Enumerable === hdr_opt
+     # make a custom converter
+    custom = Proc.new { |f, info| convert(f, info, &converter) }
+    # open the CSV file
+    @csv = FasterCSV.open(file, mode, :headers => hdr_opt, :header_converters => hdr_cvtr, :return_headers => true, :write_headers => true, :converters => custom)
+    # the header => field name hash:
+    # if the header option is set to true, then read the input header line.
+    # otherwise, parse an empty string which mimics an input header line.
+    hdr_row = case hdr_opt
+    when true then
+      @csv.shift
+    when Enumerable then
+      ''.parse_csv(:headers => hdr_opt, :header_converters => :symbol, :return_headers => true)
+    else
+      raise ArgumentError.new("CSV headers option value not supported: #{hdr_opt}")
+    end
+    # the header row headers
+    @headers = hdr_row.headers
+    # the header name => symbol map
+    @hdr_sym_hash = hdr_row.to_hash.invert
+  end
+
+  # Closes the CSV file and trash file if necessary.
+  def close
+    @csv.close
+    @trash.close if @trash
+  end
+
+  # Returns the header accessor method for the given input header name.
+  def accessor(header)
+    @hdr_sym_hash[header]
+  end
+
+  # Sets the trash output file. This creates a separate CSV output file distinct from the input CSV file.
+  # This is useful for writing rejected rows from the input. The output file has a header row.
+  def trash=(file)
+    @trash = FasterCSV.open(file, 'w', :headers => true, :header_converters => :symbol, :write_headers => true)
+  end
+
+  # Writes the row to the trash file if the trash file is set.
+  #
+  #@param [{Symbol => Object}] row the rejected input row
+  def reject(row)
+    @trash << row if @trash
+  end
+
+  # Iterates over each CSV row, yielding a row for each iteration.
+  # This method closes the CSV file after the iteration completes.
+  def each
+    begin
+      # parse each line
+      @csv.each { |row| yield row }
+    ensure
+      close
+    end
+  end
+
+  # @return the next CSV row
+  # @see #each
+  def read
+    @csv.shift
+  end
+
+  alias :shift :read
+
+  # Writes the given row to the CSV file.
+  #
+  #@param [{Symbol => Object}] row the input row
+  def write(row)
+    @csv << row
+  end
+
+  alias :<< :write
+
+  private
+
+  # 3-letter months => month sequence hash.
+  MMM_MM_MAP = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec'].to_compact_hash_with_index do |mmm, index|
+    index < 9 ? ('0' + index.succ.to_s) : index.succ.to_s
+  end
+
+  # DateMatcher relaxes the FasterCSV DateMatcher to allow dd/mm/yyyy dates.
+  DateMatcher = / \A(?: (\w+,?\s+)?\w+\s+\d{1,2},?\s+\d{2,4} | \d{1,2}-\w{3}-\d{2,4} | \d{4}[-\/]\d{1,2}[-\/]\d{1,2} | \d{1,2}[-\/]\d{1,2}[-\/]\d{2,4} )\z /x
+
+  # @param f the input field value to convert
+  # @param info the CSV field info
+  # @return the converted value
+  def convert(f, info)
+    return if f.nil?
+    # the block has precedence
+    value = yield(f, info) if block_given?
+    # integer conversion
+    value ||= Integer(f) if f =~ /^[1-9]\d*$/
+    # date conversion
+    value ||= convert_date(f) if f =~ CsvIO::DateMatcher
+    # float conversion
+    value ||= (Float(f) rescue f) if f =~ /^\d+\.\d*$/ or f =~ /^\d*\.\d+$/
+    # return converted value or the input field if there was no conversion
+    value || f
+  end
+
+  # @param [String] the input field value
+  # @return [Date] the converted date
+  def convert_date(f)
+    # If input value is in dd-mmm-yy format, then reformat.
+    # Otherwise, parse as a Date if possible.
+    if f =~ /^\d{1,2}-\w{3}-\d{2,4}$/ then
+      ddmmyy = reformat_dd_mmm_yy_date(f) || return
+      convert_date(ddmmyy)
+#    elsif f =~ /^\w{3} \d{1,2}, \d{4}$/ then
+#      ddmmyy = reformat_mmm_dd_yyyy_date(f) || return
+#      convert_date(ddmmyy)
+    else
+      Date.parse(f, true) rescue nil
+    end
+  end
+
+  # @param [String] the input field value in dd-mmm-yy format
+  # @return [String] the reformatted date String in mm/dd/yy format
+  def reformat_dd_mmm_yy_date(f)
+    all, dd, mmm, yy = /^(\d{1,2})-([[:alpha:]]{3})-(\d{2,4})$/.match(f).to_a
+    mm = MMM_MM_MAP[mmm.downcase] || return
+    "#{mm}/#{dd}/#{yy}"
+  end
+#  # @param [String] the input field value in 'mmmd d, yyyy' format
+#  # @return [String] the reformatted date String in mm/dd/yyyy format
+#  def reformat_mmm_dd_yyyy_date(f)
+#    all, mmm, dd, yyyy = /^(\w{3}) (\d{1,2}), (\d{4})$/.match(f).to_a
+#    mm = MMM_MM_MAP[mmm.downcase] || return
+#    "#{mm}/#{dd}/#{yyyy}"
+#  end
+end