data_miner-ruby19 0.5.4

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Brighter Planet
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,271 @@
+=data_miner
+
+Programmatically import useful data into your ActiveRecord models.
+
+(see http://wiki.github.com/seamusabshere/data_miner for more examples)
+
+==Quick start
+
+You define <tt>data_miner</tt> blocks in your ActiveRecord models. For example, in <tt>app/models/country.rb</tt>:
+
+  class Country < ActiveRecord::Base
+    set_primary_key :iso_3166_code
+  
+    data_miner do
+      schema do
+        string   'iso_3166_code'
+        string   'name'
+      end
+    
+      import 'the official ISO country list',
+             :url => 'http://www.iso.org/iso/list-en1-semic-3.txt',
+             :skip => 2,
+             :headers => false,
+             :delimiter => ';',
+             :encoding => 'ISO-8859-1' do
+        key   'iso_3166_code', :field_number => 1
+        store 'name', :field_number => 0
+      end
+    end
+  end
+
+Now you can run:
+
+  irb(main):001:0> Country.run_data_miner!
+  => nil
+
+==Advanced usage
+
+This is how we linked together (http://data.brighterplanet.com/aircraft) the FAA's list of aircraft with the US Department of Transportations list of aircraft:
+
+  class Aircraft < ActiveRecord::Base
+    # Tell ActiveRecord that we want to use a string primary key.
+    # This makes it easier to repeatedly truncate and re-import this
+    # table without breaking associations.
+    set_primary_key :icao_code
+  
+    # A dictionary between BTS aircraft type codes and ICAO aircraft
+    # codes that uses string similarity instead of exact matching.
+    # This is preferable to typing everything out.
+    def self.bts_name_dictionary
+      # Sorry for documenting the LooseTightDictionary gem here, but it's useful
+      @_bts_dictionary ||= LooseTightDictionary.new(
+                                                    # The first argument is the source... the possible matches. Most Enumerables will do.
+                                                    RemoteTable.new(:url => 'http://www.transtats.bts.gov/Download_Lookup.asp?Lookup=L_AIRCRAFT_TYPE', :select => lambda { |record| record['Code'].to_i.between?(1, 998) }),
+                                                    # Tightenings optionally pull out what is important on both sides of a potential match
+                                                    :tightenings  => RemoteTable.new(:url => 'http://spreadsheets.google.com/pub?key=tiS_6CCDDM_drNphpYwE_iw&single=true&gid=0&output=csv', :headers => false),
+                                                    # Identities optionally require a particular capture from both sides of a match to be equal
+                                                    :identities   => RemoteTable.new(:url => 'http://spreadsheets.google.com/pub?key=tiS_6CCDDM_drNphpYwE_iw&single=true&gid=3&output=csv', :headers => false),
+                                                    # Blockings restrict comparisons to a subset where everything matches the blocking
+                                                    :blockings    => RemoteTable.new(:url => 'http://spreadsheets.google.com/pub?key=tiS_6CCDDM_drNphpYwE_iw&single=true&gid=4&output=csv', :headers => false),
+                                                    # This means that lookups that don't match a blocking won't be compared to possible matches that **do** match a blocking.
+                                                    # This is useful because we say /boeing/ and only boeings are matched against other boeings.
+                                                    :blocking_only => true,
+                                                    # Tell the dictionary how read things from the source.
+                                                    :right_reader => lambda { |record| record['Description'] }
+                                                   )
+    end
+  
+    # A dictionary between what appear to be ICAO aircraft names and
+    # objects of this class itself.
+    # Warning: self-referential (it calls Aircraft.all) so it should be run after the first DataMiner step.
+    def self.icao_name_dictionary
+      @_icao_dictionary ||= LooseTightDictionary.new Aircraft.all,
+                                                     :tightenings  => RemoteTable.new(:url => 'http://spreadsheets.google.com/pub?key=tiS_6CCDDM_drNphpYwE_iw&single=true&gid=0&output=csv', :headers => false),
+                                                     :identities   => RemoteTable.new(:url => 'http://spreadsheets.google.com/pub?key=tiS_6CCDDM_drNphpYwE_iw&single=true&gid=3&output=csv', :headers => false),
+                                                     :blockings    => RemoteTable.new(:url => 'http://spreadsheets.google.com/pub?key=tiS_6CCDDM_drNphpYwE_iw&single=true&gid=4&output=csv', :headers => false),
+                                                     :right_reader => lambda { |record| record.manufacturer_name.to_s + ' ' + record.name.to_s }
+    end
+
+    # This responds to the "Matcher" interface as defined by DataMiner.
+    # In other words, it takes Matcher#match(*args) and returns something.
+    class BtsMatcher
+      attr_reader :wants
+      def initialize(wants)
+        @wants = wants
+      end
+      def match(raw_faa_icao_record)
+        @_match ||= Hash.new
+        return @_match[raw_faa_icao_record] if @_match.has_key?(raw_faa_icao_record)
+        faa_icao_record = [ raw_faa_icao_record['Manufacturer'] + ' ' + raw_faa_icao_record['Model'] ]
+        bts_record = Aircraft.bts_name_dictionary.left_to_right faa_icao_record
+        retval = case wants
+        when :bts_aircraft_type_code
+          bts_record['Code']
+        when :bts_name
+          bts_record['Description']
+        end if bts_record
+        @_match[raw_faa_icao_record] = retval
+      end
+    end
+
+    # Another class that implements the "Matcher" interface as expected by DataMiner.
+    class FuelUseMatcher
+      def match(raw_fuel_use_record)
+        @_match ||= Hash.new
+        return @_match[raw_fuel_use_record] if @_match.has_key?(raw_fuel_use_record)
+        # First try assuming we have an ICAO code
+        aircraft_record = if raw_fuel_use_record['ICAO'] =~ /\A[0-9A-Z]+\z/
+          Aircraft.find_by_icao_code raw_fuel_use_record['ICAO']
+        end
+        # No luck? then try a fuzzy match
+        aircraft_record ||= if raw_fuel_use_record['Aircraft Name'].present?
+          Aircraft.icao_name_dictionary.left_to_right [ raw_fuel_use_record['Aircraft Name'] ]
+        end
+        if aircraft_record
+          @_match[raw_fuel_use_record] = aircraft_record.icao_code
+        else
+          # While we're developing the dictionary, we want it to blow up until we have 100% matchability
+          raise "Didn't find a match for #{raw_fuel_use_record['Aircraft Name']} (#{raw_fuel_use_record['ICAO']}), which we found in the fuel use spreadsheet"
+        end
+      end
+    end
+  
+    # This responds to the "Responder" interface as expected by Errata.
+    # Basically it lets you say "Is a DC plane" in the errata file and
+    # have it map to a Ruby method.
+    class Guru
+      def is_a_dc_plane?(row)
+        row['Designator'] =~ /^DC\d/i
+      end
+      def is_a_g159?(row)
+        row['Designator'] =~ /^G159$/
+      end
+      def is_a_galx?(row)
+        row['Designator'] =~ /^GALX$/
+      end
+      def method_missing(method_id, *args, &block)
+        if method_id.to_s =~ /\Ais_n?o?t?_?attributed_to_([^\?]+)/
+          manufacturer_name = $1
+          manufacturer_regexp = Regexp.new(manufacturer_name.gsub('_', ' ?'), Regexp::IGNORECASE)
+          matches = manufacturer_regexp.match(args.first['Manufacturer']) # row['Manufacturer'] =~ /mcdonnell douglas/i
+          method_id.to_s.include?('not_attributed') ? matches.nil? : !matches.nil?
+        else
+          super
+        end
+      end
+    end
+  
+    data_miner do
+      # In our app, we defined DataMiner::Run.allowed? to return false if a run
+      # has taken place in the last hour (among other things).
+      # By raising DataMiner::Skip, we skip this run but call it a success.
+      process "Don't re-import too often" do
+        raise DataMiner::Skip unless DataMiner::Run.allowed? Aircraft
+      end
+    
+      # Define the database schema in-line.
+      # It will destructively and automatically add/remove columns.
+      # This is "OK" because you can always just re-run the import script to get the data back.
+      # PS. if we were using DataMapper, we wouldn't need this.
+      schema :options => 'ENGINE=InnoDB default charset=utf8' do
+        string   'icao_code'
+        string   'manufacturer_name'
+        string   'name'
+        string   'bts_name'
+        string   'bts_aircraft_type_code'
+        string   'brighter_planet_aircraft_class_code'
+        string   'fuel_use_aircraft_name'
+        float    'm3'
+        string   'm3_units'
+        float    'm2'
+        string   'm2_units'
+        float    'm1'
+        string   'm1_units'
+        float    'endpoint_fuel'
+        string   'endpoint_fuel_units'
+        float    'seats'
+        float    'distance'
+        string   'distance_units'
+        float    'load_factor'
+        float    'freight_share'
+        float    'payload'
+        float    'weighting'
+        index    'bts_aircraft_type_code'
+      end
+
+      # The FAA publishes a document to help people identify aircraft by different names.
+      ('A'..'Z').each do |letter|
+        import( "ICAO aircraft codes starting with the letter #{letter} used by the FAA",
+                # The master URL of the source file (one for every letter)
+                :url => "http://www.faa.gov/air_traffic/publications/atpubs/CNT/5-2-#{letter}.htm",
+                # The RFC-style errata... note that it will use the Guru class we defined above. See the Errata gem for more details.
+                :errata => Errata.new(:url => 'http://spreadsheets.google.com/pub?key=tObVAGyqOkCBtGid0tJUZrw', :responder => Aircraft::Guru.new),
+                # If it's not UTF-8, you should say what it is so that we can iconv it!
+                :encoding => 'windows-1252',
+                # Nokogiri is being used to grab each row starting from the second
+                :row_xpath => '//table/tr[2]/td/table/tr',
+                # ditto... XPath for Nokogiri
+                :column_xpath => 'td' ) do
+          # The code that they use is in fact the ICAO code!
+          key 'icao_code', :field_name => 'Designator'
+          # We get this for free
+          store 'manufacturer_name', :field_name => 'Manufacturer'
+          # ditto
+          store 'name', :field_name => 'Model'
+          # Use the loose-tight dictionary.
+          # It gets the entire input row to play with before deciding on an output.
+          store 'bts_aircraft_type_code', :matcher => Aircraft::BtsMatcher.new(:bts_aircraft_type_code)
+          store 'bts_name', :matcher => Aircraft::BtsMatcher.new(:bts_name)
+        end
+      end
+
+      # Pull in some data that might only be important to Brighter Planet
+      import "Brighter Planet's aircraft class codes",
+             :url => 'http://static.brighterplanet.com/science/data/transport/air/bts_aircraft_type/bts_aircraft_types-brighter_planet_aircraft_classes.csv' do
+        key   'bts_aircraft_type_code', :field_name => 'bts_aircraft_type'
+        store 'brighter_planet_aircraft_class_code'
+      end
+
+      # Pull in fuel use equation (y = m3*x^3 + m2*x^2 + m1*x + endpoint_fuel).
+      # This data comes from the EEA.
+      import "pre-calculated fuel use equation coefficients",
+             :url => 'http://static.brighterplanet.com/science/data/transport/air/fuel_use/aircraft_fuel_use_formulae.ods',
+             :select => lambda { |row| row['ICAO'].present? or row['Aircraft Name'].present? } do
+        # We want to key on ICAO code, but since it's sometimes missing, use the loose-tight dictionary we defined above.
+        key   'icao_code', :matcher => Aircraft::FuelUseMatcher.new
+        # Keep the name for sanity checking. Yes, we have 3 different "name" fields... they should all refer to the same aircraft.
+        store 'fuel_use_aircraft_name', :field_name => 'Aircraft Name'
+        store 'm3'
+        store 'm2'
+        store 'm1'
+        store 'endpoint_fuel', :field_name => 'b'
+      end
+    
+      # Use arel and the weighted_average gem to do some crazy averaging.
+      # This assumes that you're dealing with the BTS T-100 flight segment data.
+      # See http://data.brighterplanet.com/flight_segments for a pre-sanitized version.
+      process "Derive some average flight characteristics from flight segments" do
+        FlightSegment.run_data_miner!
+        aircraft = Aircraft.arel_table
+        segments = FlightSegment.arel_table
+
+        conditional_relation = aircraft[:bts_aircraft_type_code].eq(segments[:bts_aircraft_type_code])
+        update_all "seats         = (#{FlightSegment.weighted_average_relation(:seats,         :weighted_by => :passengers                                           ).where(conditional_relation).to_sql})"
+        update_all "distance      = (#{FlightSegment.weighted_average_relation(:distance,      :weighted_by => :passengers                                           ).where(conditional_relation).to_sql})"
+        update_all "load_factor   = (#{FlightSegment.weighted_average_relation(:load_factor,   :weighted_by => :passengers                                           ).where(conditional_relation).to_sql})"
+        update_all "freight_share = (#{FlightSegment.weighted_average_relation(:freight_share, :weighted_by => :passengers                                           ).where(conditional_relation).to_sql})"
+        update_all "payload       = (#{FlightSegment.weighted_average_relation(:payload,       :weighted_by => :passengers, :disaggregate_by => :departures_performed).where(conditional_relation).to_sql})"
+      
+        update_all "weighting = (#{segments.project(segments[:passengers].sum).where(aircraft[:bts_aircraft_type_code].eq(segments[:bts_aircraft_type_code])).to_sql})"
+      end
+    
+      # And finally re-run the import of resources that depend on this resource.
+      # Don't worry about calling Aircraft.run_data_miner! at the top of AircraftManufacturer's data_miner block;
+      # that's the right way to do dependencies. It won't get called twice in the same run.
+      [ AircraftManufacturer ].each do |synthetic_resource|
+        process "Synthesize #{synthetic_resource}" do
+          synthetic_resource.run_data_miner!
+        end
+      end
+    end
+  end
+
+==Authors
+
+* Seamus Abshere <seamus@abshere.net>
+* Andy Rossmeissl <andy@rossmeissl.net>
+
+==Copyright
+
+Copyright (c) 2010 Brighter Planet. See LICENSE for details.

data/lib/data_miner.rb

@@ -0,0 +1,136 @@
+require 'active_support'
+require 'active_support/version'
+%w{
+  active_support/core_ext/array/conversions
+  active_support/core_ext/string/access
+  active_support/core_ext/string/multibyte
+}.each do |active_support_3_requirement|
+  require active_support_3_requirement
+end if ActiveSupport::VERSION::MAJOR == 3
+
+require 'active_record'
+require 'blockenspiel'
+require 'conversions'
+require 'errata'
+require 'remote_table'
+require 'escape'
+require 'andand'
+require 'log4r'
+require 'fileutils'
+require 'tmpdir'
+require 'zlib'
+
+require 'data_miner/attribute'
+require 'data_miner/base'
+require 'data_miner/dictionary'
+require 'data_miner/import'
+require 'data_miner/tap'
+require 'data_miner/process'
+require 'data_miner/run'
+require 'data_miner/schema'
+
+module DataMiner
+  class MissingHashColumn < StandardError; end
+  class Finish < StandardError; end
+  class Skip < StandardError; end
+  
+  mattr_accessor :logger
+  
+  def self.start_logging
+    return if logger
+
+    if defined? Rails
+      self.logger = Rails.logger
+    else
+      class_eval { include Log4r }
+      info_outputter = FileOutputter.new 'f1', :filename => 'data_miner.log'
+      error_outputter = Outputter.stderr
+      info_outputter.only_at DEBUG, INFO
+      error_outputter.only_at WARN, ERROR, FATAL
+      
+      self.logger = Logger.new 'data_miner'
+      logger.add info_outputter, error_outputter
+      ActiveRecord::Base.logger = logger
+    end
+  end
+  
+  def self.log_or_raise(message)
+    message = "[data_miner gem] #{message}"
+    if ENV['RAILS_ENV'] == 'production' or ENV['DONT_RAISE'] == 'true'
+      logger.error message
+    else
+      raise message
+    end
+  end
+  
+  def self.log_info(message)
+    logger.info "[data_miner gem] #{message}"
+  end
+  
+  def self.log_debug(message)
+    logger.debug "[data_miner gem] #{message}"
+  end
+  
+  def self.run(options = {})
+    DataMiner::Base.run options.merge(:preserve_call_stack_between_runs => true)
+    DataMiner::Base.call_stack.clear
+  end
+  
+  def self.resource_names
+    DataMiner::Base.resource_names
+  end
+  
+    # TODO this should probably live somewhere else
+  def self.backtick_with_reporting(cmd)
+    cmd = cmd.gsub /[ ]*\n[ ]*/m, ' '
+    output = `#{cmd}`
+    if not $?.success?
+      raise %{
+From the data_miner gem...
+
+Command failed:
+#{cmd}
+
+Output:
+#{output}
+}
+    end
+  end
+
+end
+
+ActiveRecord::Base.class_eval do
+  def self.x_data_miner(&block)
+    DataMiner.start_logging
+    
+    DataMiner.log_debug "Skipping data_miner block in #{self.name} because called as x_data_miner"
+  end
+  
+  def self.data_miner(&block)
+    DataMiner.start_logging
+    
+    DataMiner.log_debug "Database table `#{table_name}` doesn't exist. It might be created in the data_miner block, but if it's not, DataMiner probably won't work properly until you run a migration or otherwise fix the schema." unless table_exists?
+    
+    DataMiner.resource_names.push self.name unless DataMiner.resource_names.include? self.name
+
+    # this is class_eval'ed here so that each ActiveRecord descendant has its own copy, or none at all
+    class_eval do
+      cattr_accessor :data_miner_base
+      def self.data_miner_runs
+        DataMiner::Run.scoped :conditions => { :resource_name => name }
+      end
+      def self.run_data_miner!(options = {})
+        data_miner_base.run options
+      end
+      def self.execute_schema
+        schema = data_miner_base.steps.find { |s| s.instance_of?(DataMiner::Schema) }
+        schema.run(nil) if schema
+      end
+    end
+    self.data_miner_base = DataMiner::Base.new self
+
+    Blockenspiel.invoke block, data_miner_base
+    
+    data_miner_base.after_invoke
+  end
+end

data/lib/data_miner/attribute.rb

@@ -0,0 +1,233 @@
+module DataMiner
+  class Attribute
+    attr_accessor :step
+    attr_accessor :name
+    attr_accessor :options
+
+    delegate :resource, :to => :step
+
+    VALID_OPTIONS = [
+      :from_units,
+      :to_units,
+      :static,
+      :dictionary,
+      :matcher,
+      :field_name,
+      :delimiter,
+      :split,
+      :units,
+      :sprintf,
+      :nullify,
+      :overwrite,
+      :upcase,
+      :units_field_name,
+      :units_field_number,
+      :field_number,
+      :chars,
+      :synthesize
+    ]
+
+    def initialize(step, name, options = {})
+      options.symbolize_keys!
+
+      @step = step
+      @name = name
+      
+      invalid_option_keys = options.keys.select { |k| not VALID_OPTIONS.include? k }
+      DataMiner.log_or_raise "Invalid options: #{invalid_option_keys.map(&:inspect).to_sentence} (#{inspect})" if invalid_option_keys.any?
+      @options = options
+    end
+        
+    def inspect
+      "Attribute(#{resource}##{name})"
+    end
+
+    def value_in_dictionary(str)
+      dictionary.lookup str
+    end
+    
+    def value_in_source(row)
+      if wants_static?
+        value = static
+      elsif field_number
+        if field_number.is_a?(Range)
+          value = field_number.map { |n| row[n] }.join(delimiter)
+        else
+          value = row[field_number]
+        end
+      else
+        value = row[field_name]
+      end
+      return nil if value.nil?
+      return value if value.is_a?(ActiveRecord::Base) # escape valve for parsers that look up associations directly
+      value = value.to_s
+      value = value[chars] if wants_chars?
+      value = do_split(value) if wants_split?
+      # taken from old errata... maybe we want to do this here
+      value.gsub! /[ ]+/, ' '
+      # text.gsub!('- ', '-')
+      value.gsub! /([^\\])~/, '\1 '
+      value.strip!
+      value.upcase! if wants_upcase?
+      value = do_convert row, value if wants_conversion?
+      value = do_sprintf value if wants_sprintf?
+      value
+    end
+    
+    def match_row(row)
+      matcher.match row
+    end
+    
+    def value_from_row(row)
+      return match_row row if wants_matcher?
+      value = value_in_source row
+      return value if value.is_a? ActiveRecord::Base # carry through trapdoor
+      value = value_in_dictionary value if wants_dictionary?
+      value = synthesize.call(row) if wants_synthesize?
+      value
+    end
+        
+    # this will overwrite nils, even if wants_overwriting? is false
+    # returns true if an attr was changed, otherwise false
+    def set_record_from_row(record, row)
+      return false if !wants_overwriting? and !record.send(name).nil?
+      what_it_was = record.send name
+      what_it_should_be = value_from_row row
+
+      record.send "#{name}=", what_it_should_be
+      record.send "#{name}_units=", (to_units || unit_from_source(row)).to_s if wants_units?
+      
+      what_it_is = record.send name
+      if what_it_is.nil? and !what_it_should_be.nil?
+        DataMiner.log_debug "ActiveRecord didn't like trying to set #{resource}.#{name} = #{what_it_should_be} (it came out as nil)"
+        nil
+      elsif what_it_is == what_it_was
+        false
+      else
+        true
+      end
+    end
+
+    def unit_from_source(row)
+      row[units_field_name || units_field_number].to_s.strip.underscore.to_sym
+    end
+    
+    def do_convert(row, value)
+      DataMiner.log_or_raise "If you use :from_units, you need to set :to_units (#{inspect})" unless wants_units?
+      value.to_f.convert((from_units || unit_from_source(row)), (to_units || unit_from_source(row)))
+    end
+    
+    def do_sprintf(value)
+      if /\%[0-9\.]*f/.match sprintf
+        value = value.to_f
+      elsif /\%[0-9\.]*d/.match sprintf
+        value = value.to_i
+      end
+      sprintf % value
+    end
+    
+    def do_split(value)
+      pattern = split_options[:pattern] || /\s+/ # default is split on whitespace
+      keep = split_options[:keep] || 0           # default is keep first element
+      value.to_s.split(pattern)[keep].to_s
+    end
+  
+    def column_type
+      resource.columns_hash[name.to_s].type
+    end
+    
+    # Our wants and needs :)
+    def wants_split?
+      split_options.present?
+    end
+    def wants_sprintf?
+      sprintf.present?
+    end
+    def wants_upcase?
+      upcase.present?
+    end
+    def wants_static?
+      options.has_key? :static
+    end
+    def wants_nullification?
+      nullify != false
+    end
+    def wants_chars?
+      chars.present?
+    end
+    def wants_synthesize?
+      synthesize.is_a?(Proc)
+    end
+    def wants_overwriting?
+      overwrite != false
+    end
+    def wants_conversion?
+      from_units.present? or units_field_name.present? or units_field_number.present?
+    end
+    def wants_units?
+      to_units.present? or units_field_name.present? or units_field_number.present?
+    end
+    def wants_dictionary?
+      options[:dictionary].present?
+    end
+    def wants_matcher?
+      options[:matcher].present?
+    end
+
+    # Options that always have values
+    def field_name
+      (options[:field_name] || name).to_s
+    end
+    def delimiter
+      (options[:delimiter] || ', ')
+    end
+    
+    # Options that can't be referred to by their names
+    def split_options
+      options[:split]
+    end
+    
+    def from_units
+      options[:from_units]
+    end
+    def to_units
+      options[:to_units] || options[:units]
+    end
+    def sprintf
+      options[:sprintf]
+    end
+    def nullify
+      options[:nullify]
+    end
+    def overwrite
+      options[:overwrite]
+    end
+    def upcase
+      options[:upcase]
+    end
+    def units_field_name
+      options[:units_field_name]
+    end
+    def units_field_number
+      options[:units_field_number]
+    end
+    def field_number
+      options[:field_number]
+    end
+    def chars
+      options[:chars]
+    end
+    def synthesize
+      options[:synthesize]
+    end
+    def static
+      options[:static]
+    end
+    def dictionary
+      @_dictionary ||= (options[:dictionary].is_a?(Dictionary) ? options[:dictionary] : Dictionary.new(options[:dictionary]))
+    end
+    def matcher
+      @_matcher ||= (options[:matcher].is_a?(String) ? options[:matcher].constantize.new : options[:matcher])
+    end
+  end
+end