lwac 0.2.0

data/lib/lwac/export/format.rb

@@ -0,0 +1,310 @@
+
+
+module LWAC
+
+  # ------------------------------------------------------------
+  # Defines how output is written from the exporter tool
+  class Formatter
+    def initialize(config={})
+      @config = config
+    end
+
+    ## --------------------------------------------
+    # Open all output for writing
+    def open_output()
+    end
+
+    # Close output after all items have been written
+    def close_output()
+    end
+
+    # Write one line
+    def <<(data)
+      open_point()
+      add_data(data)
+      close_point()
+    end
+
+    ## --------------------------------------------
+    # Write keys if appropriate
+    def write_header
+    end
+
+    ## --------------------------------------------
+    # Open a single datapoint for writing, i.e. 
+    # one line of a CSV or a new file for XML output
+    def open_point()
+      $log.debug "Opening new point"
+      @data = nil
+    end
+
+    # Add a key-value set for a given line
+    def add_data(data)
+      $log.debug "Adding data: #{data}"
+      @data = data
+    end
+
+    # Close the current point
+    def close_point()
+      $log.debug "Closing point."
+    end
+  end
+
+
+  # ------------------------------------------------------------
+  # Formatters that support key-value pairs as selected
+  # by a 'fields' config item
+  class KeyValueFormatter < Formatter
+    require 'lwac/export/key_value_format'
+   
+    def initialize(config = {})
+      super(config)
+      raise "No fields in field listing" if (not (@config[:fields] and @config[:fields].length > 0) )
+      KeyValueFormat::compile_format_procedures( @config[:fields] )
+    end
+
+
+    def open_point()
+      $log.debug "KV: Opening new point"
+      @data = nil
+      @line = {}
+    end
+
+    def add_data(data)
+      $log.debug "KV: Adding data: #{data}"
+      @data = data
+      @line.merge! KeyValueFormat::produce_output_line( data, @config[:fields] )
+    end
+  end
+
+
+
+
+  # ------------------------------------------------------------
+  # Output to a single JSON file
+  class JSONFormatter < KeyValueFormatter 
+    require 'json'
+    # TODO: - sync after every write
+    #       - use formatter system]
+    #
+    #
+    def open_output()
+      $log.info "Opening #{@config[:filename]} for writing..."
+      @f = File.open( @config[:filename], 'w' )
+    end
+
+    def close_output()
+      $log.info "Closing output CSV..."
+      @f.close
+    end
+
+    def write_header
+      $log.info "Writing header"
+      @f.write( @config[:fields].keys )
+      @f.flush
+    end
+
+    def close_point()
+      super
+      @f.write(JSON.generate(@line.values))
+      @f.write("\n")
+      @f.flush
+    end
+  end
+
+
+
+  # ------------------------------------------------------------
+  # Output to a single CSV file
+  class CSVFormatter < KeyValueFormatter 
+    require 'csv'
+
+    def open_output()
+      $log.info "Opening #{@config[:filename]} for writing..."
+      $log.debug "Options for CSV: #{@config[:csv_opts]}"
+      @csv = CSV.open(@config[:filename], 'w', @config[:csv_opts] || {})
+    end
+
+    def close_output()
+      $log.info "Closing output CSV..."
+      @csv.close
+    end
+
+    def write_header
+      $log.info "Writing header"
+      @csv << @config[:fields].keys
+    end
+
+    def close_point()
+      super
+      @csv << (@line.values)
+    end
+  end
+
+
+
+
+
+  # ------------------------------------------------------------
+  # Output to individual CSVs
+  class MultiCSVFormatter < KeyValueFormatter 
+    require 'csv'
+    require 'fileutils'
+
+    def write_header
+      @config[:headers] = true
+    end
+
+    def close_point()
+      filename = get_filename( @data )
+      $log.debug "Writing point to file #{filename}..."
+      file_exists = File.exist?( filename )
+
+      # FIXME: don't keep opening/closing file
+      CSV.open( filename, "a" ){|cout|
+        cout << @line.keys if not file_exists and @config[:headers] 
+        cout << @line.values
+      }
+    end
+
+    private
+    def get_filename(data)
+      filename = eval( "\"#{@config[:filename]}\"" ).to_s
+      FileUtils.mkdir_p( File.dirname( filename ) ) if not File.exist?( File.dirname( filename ) )
+      return filename
+    rescue Exception => e
+      $log.error "Failed to generate filename."
+      $log.error "This data point will be skipped."
+      $log.debug "#{e.backtrace.join("\n")}"
+      return nil
+    end
+  end
+
+
+
+
+
+
+  # ------------------------------------------------------------
+  # Output to an erb template
+  class MultiTemplateFormatter < Formatter  
+    require 'erb'
+
+    def initialize( config )
+      super(config)
+      
+      raise "Template not found" if not File.exist?(@config[:template])
+      @template = File.read(@config[:template])
+    end
+
+    def close_point()
+      filename = get_filename( @data )
+      $log.debug "Writing point to file #{filename}..."
+      $log.warn "Overwriting (#{filename}) (you might have selected a non-unique key field)" if File.exist?(filename)
+
+      File.open(filename, 'w'){ |f|
+        f.write(apply_template(filename, @data))
+      }
+    end
+
+    private
+    def apply_template(filename, data)
+      return ERB.new(@template).result(binding)
+    rescue StandardError => e
+      $log.warn "Error running template #{@config[:template]}: #{e}"
+      $log.debug "#{e.backtrace.join("\n")}"
+    end
+
+
+    def get_filename(data)
+      filename = eval( "\"#{@config[:filename]}\"" ).to_s
+      FileUtils.mkdir_p( File.dirname(filename) ) if not File.exist?( File.dirname(filename) )
+      return filename
+    rescue Exception => e
+      $log.error "Failed to generate filename."
+      $log.error "This data point will be skipped."
+      $log.debug "#{e.backtrace.join("\n")}"
+      return nil
+    end
+  end
+
+
+
+
+  # ------------------------------------------------------------
+  # Output to an erb template
+  class MultiXMLFormatter < Formatter  
+    require 'rexml/document'
+    require 'rexml/formatters/transitive'
+
+    def close_point()
+      filename = get_filename( @data )
+      $log.debug "Writing point to file #{filename}..."
+      $log.warn "Overwriting (#{filename}) (you might have selected a non-unique key field)" if File.exist?(filename)
+
+      File.open(filename, 'w'){ |f|
+        xml = build_xml_doc( @data, nil, "data")
+
+        # Select a formatter
+        formatter = case(@config[:xml_format])
+                      when :whitespace
+                        REXML::Formatters::Transitive.new(@config[:xml_indent] || 2)
+                      when :pretty
+                        REXML::Formatters::Pretty.new(@config[:xml_indent] || 2)
+                      else
+                        REXML::Formatters::Default.new()
+                      end
+
+        # Compact if pretty
+        formatter.compact = true if @config[:xml_format] == :pretty
+
+        # Do output
+        formatter.write( xml, f )
+      }
+    end
+
+    private
+
+    # Recursively construct an XML doc from a resource 
+    def build_xml_doc(data, root=nil, name=nil)
+      name = data.__name if (not name) and (data.is_a? Resource)
+      node = REXML::Element.new( name, root )
+
+      if data.is_a? Resource then
+        data.__params.each{|p|
+          val = eval("data.#{p}")
+
+          if val.is_a? Resource
+            build_xml_doc( val, node, p.to_s )
+          else
+            build_xml_doc( val, node, p.to_s )
+          end
+        }
+      else
+        node.add_attributes({'type' => data.class.to_s})
+        if data.is_a? Array
+          data.each{|val|
+            build_xml_doc( val, node, 'value' )
+          }
+        else 
+          node.add_text( data.to_s )
+        end
+      end
+
+      return node
+    end
+
+    def get_filename(data)
+      filename = eval( "\"#{@config[:filename]}\"" ).to_s
+      FileUtils.mkdir_p( File.dirname(filename) ) if not File.exist?( File.dirname(filename) )
+      return filename
+    rescue Exception => e
+      $log.error "Failed to generate filename."
+      $log.error "This data point will be skipped."
+      $log.debug "#{e.backtrace.join("\n")}"
+      return nil
+    end
+  end
+
+
+end

data/lib/lwac/export/key_value_format.rb

@@ -0,0 +1,132 @@
+module LWAC
+
+  module KeyValueFormat 
+    # The output formatting system for the export tool uses these procedures.
+    #
+    # They are responsible for:
+    #  * Constructing lambda-function filters for data selection
+    #  * Constructing lambda-function output formatter scripts
+    #  * Running filters on data
+    #  * Producing output strings from formatters and data
+    #
+
+
+
+    # -----------------------------------------------------------------------------
+    # Compile formatting procedures
+    #
+    # Output format procedures are designed to handle output of missing values,
+    # formatting such as lower-case or normalised output.
+    #
+    # The format is described in a hash, as in the config file:
+    #  { :key_name => "variable.name", -and/or-
+    #    :key_name => {:var => 'variable.name', :condition => 'expression', :missing => 'NA'},
+    #    :key_name => {:expr => 'expression returning value'},
+    #    ...
+    # }
+    # 
+    # Where 'key_name' is used to form the name of a column in the CSV, and the value can be
+    # either a hash or a string.  Where a string is given, it is presumed to be the name
+    # of a resource value, i.e. sample.id, or sample.datapoint.id.  Where a hash is given,
+    # it can contain either
+    #  1) :var, :condition and :missing fields to describe how to get and format data simply
+    #  2) :expr, an expression that returns a value and may do more complex formatting
+    #
+    def self.compile_format_procedures( format )
+      $log.info "Compiling formatting procedures..."
+
+      format.each{|f, v|
+        $log.info "  Preparing field #{f}..."
+        # Make sure it's a hash
+        v = {:val => nil, :var => v, :expr => nil, :condition => nil, :missing => nil} if(not v.is_a? Hash)
+
+        # Don't allow people to define both a static value and a variable
+        criteria = 0
+        %w{val var expr}.each{|method| criteria += 1 if v[method.to_sym] != nil}
+        raise "No extraction method given for field '#{f}'."          if(criteria == 0)
+        raise "Multiple extraction methods given for field '#{f}' (#{v.keys.join(", ")})." if(criteria > 1)
+
+        # Construct lambdas for active fields
+        if v[:var] or v[:expr] then
+          $log.debug "Building expression for data extraction (#{f})..."
+          begin
+            if v[:expr] then
+              v[:lambda] = eval("lambda{|data|" + v[:expr] + "}")
+            elsif v[:var] then
+              v[:lambda] = eval("lambda{|data| return data." + v[:var] + "}")
+            end
+          rescue StandardError => e
+            $log.fatal "Error building expression for field: #{f}."
+            $log.fatal "Please review your configuration."
+            $log.fatal "The exact error was: \n#{e}"
+            $log.fatal "Backtrace: \n#{e.backtrace.join("\n")}"
+            exit(1)
+          end
+          $log.debug "Success so far..."
+        end
+
+        format[f] = v
+      }
+
+      $log.info "Done."
+    end
+
+
+
+
+    # -----------------------------------------------------------------------------
+    # Format data from the 'data' resource according to a set of rules
+    # given in the format hash.
+    #
+    # The hash is, roughly, organised thus:
+    #  output_field_name: data.path.to.var
+    # - OR -
+    #  output_field_name: {:val => static value, (optional) one of these must exist
+    #                      :var => path.to.var, (optional)
+    #                      :condition => "expression which must be true to be
+    #                      non-missing, default is simply true",
+    #                      :missing => "value for when it's missing, default is """ }
+    def self.produce_output_line( data, format )
+      line = {}
+     
+      current = nil
+      format.each{|f, v|
+        current = f
+        $log.debug "Processing field #{f}..."
+
+        # Look up info
+        if v[:lambda] then
+          val = v[:lambda].call(data)
+        elsif v[:val] then
+          val = v[:val]
+        else
+          $log.fatal "No way of finding var for #{f}!"
+          $log.fatal "Please check your config!"
+          exit(1)
+        end
+        
+        # Handle the condition of missingness
+        if(v[:condition])
+          x   = val
+          val = v[:missing] if not eval("#{v[:condition]}")
+        end
+
+        # add to line
+        line[f] = val
+      }
+      current = nil
+
+      return line 
+
+    rescue StandardError => e
+      $log.error "Error producing output: #{e}"
+      $log.error "This is probably a bug in your formatting expressions."
+      $log.error "Currently formatting '#{current}'." if current
+      $log.error "Backtrace: \n#{e.backtrace.join("\n")}"
+      $log.error "I'm going to continue because the alternative is giving up entirely"
+      return 'ERROR'
+      # exit(1)
+    end
+
+  end
+end

data/lib/lwac/export/resources.rb

@@ -0,0 +1,82 @@
+
+# -----------------------------------------------------------------------------
+# Provide a nice truncated output for summaries
+class String
+  def truncate(lim, ellipsis='...', pad=' ')
+    ellipsis = '' if self.length <= lim
+    return ellipsis[ellipsis.length - lim..-1] if lim <= ellipsis.length
+    return self[0..(lim - ellipsis.length)-1] + ellipsis + (pad * [lim - self.length, 0].max)
+  end
+end
+
+module LWAC
+
+  # -----------------------------------------------------------------------------
+  # This is similar to ruby's Struct system, in that it creates an object based on
+  # the input parameters, with the exception that it can be efficiently and
+  # recursively described.
+  class Resource
+
+    # Construct a resource from a hash of parameters and a name.
+    def initialize(name, params = {})
+      @params = []
+      params.each{ |p, v|
+        if(p) then
+          # Parse param
+          param   = sanitise_paramname(p)
+          raise "Duplicate parameters for resource #{name}: #{param}." if @params.include? param
+          val     = (v.is_a? Hash) ? Resource.new(param, v) : v
+
+          eval("@#{param} = val")
+          self.class.__send__(:attr_accessor, param)
+          @params << param
+        end
+      }
+      @name     = name
+    end
+
+    # Describe this resource in a nice terminal-friendly way
+    #  * trunc --- how to truncate the keys[0] and values[1]
+    #  * indent --- base indent
+    #  * notitle --- Don't output a header
+    def describe(trunc = [17, 50], indent=0, notitle=false)
+      str = "{\n"
+      str = "#{" "*indent}#{@name}#{str}" if not notitle
+      @params.each{|p|
+        # Load the value
+        val = eval("@#{p}")
+
+        # Output the string
+        str += "#{" "*indent}  #{p.truncate(trunc[0])}: "
+        if val.is_a? Resource
+          str += "#{val.describe(trunc, indent + 2, true)}" 
+        else
+          str += "#{val.to_s.truncate(trunc[1]).gsub("\n", '\n').gsub("\r", '\r').gsub("\t", '\t')}"
+        end
+        str += "\n"
+      }
+      str += "#{" "*indent}}"
+      return str
+    end
+
+    # Expose parameters to people who may wish to iterate over them
+    def __params
+      @params
+    end
+
+    # Expose name to people who may wish to iterate over them
+    def __name
+      @name
+    end
+
+  private
+
+    # Store a clean internal parameter name
+    def sanitise_paramname(p)
+      p.to_s.gsub(/[\s]/, "_").gsub(/[^a-zA-Z0-9_]/, "_")
+    end
+  end
+
+
+
+end