stockboy 0.5.0

data/lib/stockboy/railtie.rb

@@ -0,0 +1,28 @@
+require 'stockboy'
+
+# When loaded in a Rails project, Stockboy will assume the following defaults:
+#
+# == Configuration file
+#
+# If a file under +config/stockboy.rb+ exists, it will be loaded for setting up
+# the app-specific configuration options, like paths or registering custom
+# readers, filters, or providers.
+#
+# == Default template load paths
+#
+# +config/stockboy_jobs+ Will be set up as the default template load path.
+# This can be changed in the config file.
+#
+class Railtie < Rails::Railtie
+
+  initializer "stockboy.configure_rails_initialization" do
+    Stockboy.configure do |config|
+      config.template_load_paths = [Rails.root.join('config/stockboy_jobs')]
+    end
+
+    if File.exists?(config_file = Rails.root.join("config", "stockboy.rb"))
+      require config_file
+    end
+  end
+
+end

data/lib/stockboy/reader.rb

@@ -0,0 +1,59 @@
+require 'stockboy/dsl'
+
+module Stockboy
+
+  # Abstract class for defining data readers
+  #
+  # == Interface
+  #
+  # A reader must implement a +parse+ method for extracting an array of records
+  # from raw data. At this stage no data transformation is performed, only
+  # extracting field tokens for each record, based on the specific data
+  # serialization.
+  #
+  # String keys should be preferred, since these may be specified by the user;
+  # external inputs should not be symbolized (because symbols are never GC'd).
+  # Frozen strings for keys are a good idea, of course.
+  #
+  # @example
+  #   reader.parse("name,email\nArthur Dent,arthur@example.com")
+  #   # => [{"name" => "Arthur Dent", "email" => "arthur@example.com"}]
+  #
+  # @abstract
+  #
+  class Reader
+    extend Stockboy::DSL
+
+    # Initialize a new reader
+    #
+    # @param [Hash] opts
+    #
+    def initialize(opts={})
+      @encoding = opts.delete(:encoding)
+    end
+
+    # Take raw input (String) and extract an array of records
+    #
+    # @return [Array<Hash>]
+    #
+    def parse(data)
+      raise NoMethodError, "#{self.class}#parse needs implementation"
+    end
+
+  end
+
+
+  # @!macro [new] reader.skip_row_options
+  # [skip_header_rows]
+  #   If the file has a preamble before actual data to be ignored
+  #     skip_header_rows 4
+  # [skip_header_rows]
+  #   If the file has a summary or footer to be ignored
+  #     skip_footer_rows 4
+
+  # @!macro [new] reader.encoding_options
+  # [encoding]
+  #   String encoding format of the source data. All readers output UTF-8.
+  #     encoding 'Windows-1252'
+
+end

data/lib/stockboy/readers.rb

@@ -0,0 +1,11 @@
+require 'stockboy/registry'
+
+module Stockboy
+
+  # Registry of available readers
+  #
+  module Readers
+    extend Stockboy::Registry
+  end
+
+end

data/lib/stockboy/readers/csv.rb

@@ -0,0 +1,115 @@
+require 'stockboy/configuration'
+require 'stockboy/reader'
+require 'csv'
+
+module Stockboy::Readers
+
+  # Parse data from CSV into hashes
+  #
+  # All standard ::CSV options are respected and passed through
+  #
+  # @see
+  #   http://www.ruby-doc.org/stdlib-2.0.0/libdoc/csv/rdoc/CSV.html#DEFAULT_OPTIONS
+  #
+  class CSV < Stockboy::Reader
+
+    # @!group Options
+
+    # Override source file encoding
+    #
+    # @!attribute [rw] encoding
+    # @return [String]
+    #
+    dsl_attr :encoding
+
+    # Skip number of rows at start of file before data starts
+    #
+    # @!attribute [rw] skip_header_rows
+    # @return [Fixnum]
+    #
+    dsl_attr :skip_header_rows
+
+    # Skip number of rows at end of file after data ends
+    #
+    # @!attribute [rw] skip_footer_rows
+    # @return [Fixnum]
+    #
+    dsl_attr :skip_footer_rows
+
+    # @!attribute [rw] col_sep
+    #   @macro dsl_attr
+    #   @return [String]
+    #
+    # @!attribute [rw] row_sep
+    #   @macro dsl_attr
+    #   @return [String]
+    #
+    # @!attribute [rw] quote_char
+    #   @macro dsl_attr
+    #   @return [String]
+    #
+    # @!attribute [rw] headers
+    #   @macro dsl_attr
+    #   @return [Array, String]
+    #
+    ::CSV::DEFAULT_OPTIONS.keys.each do |opt|
+      dsl_attr opt, attr_accessor: false
+      define_method(opt)        { @csv_options[opt] }
+      define_method(:"#{opt}=") { |value| @csv_options[opt] = value }
+    end
+
+    # @!endgroup
+
+    # Initialize a new CSV reader
+    #
+    # All stdlib ::CSV options are respected.
+    # @see http://ruby-doc.org/stdlib-2.0.0/libdoc/csv/rdoc/CSV.html#method-c-new
+    #
+    # @param [Hash] opts
+    #
+    def initialize(opts={}, &block)
+      super
+      @csv_options = opts.reject {|k,v| !::CSV::DEFAULT_OPTIONS.keys.include?(k) }
+      @csv_options[:headers] = @csv_options.fetch(:headers, true)
+      @skip_header_rows = opts.fetch(:skip_header_rows, 0)
+      @skip_footer_rows = opts.fetch(:skip_footer_rows, 0)
+      DSL.new(self).instance_eval(&block) if block_given?
+    end
+
+    def parse(data)
+      chain = options[:header_converters] || []
+      chain << proc{ |h| h.freeze }
+      opts = options.merge(header_converters: chain)
+      ::CSV.parse(sanitize(data), opts).map &:to_hash
+    end
+
+    # Hash of all CSV-specific options
+    #
+    # @!attribute [r] options
+    #   @return [Hash]
+    #
+    def options
+      @csv_options
+    end
+
+    private
+
+    def sanitize(data)
+      data.force_encoding(encoding) if encoding
+      data = data.encode(universal_newline: true)
+                 .delete(0.chr)
+                 .chomp
+      from = row_start_index(data, skip_header_rows)
+      to = row_end_index(data, skip_footer_rows)
+      data[from..to]
+    end
+
+    def row_start_index(data, skip_rows)
+      Array.new(skip_rows).inject(0) { |i| data.index(/$/, i) + 1 }
+    end
+
+    def row_end_index(data, skip_rows)
+      Array.new(skip_rows).inject(-1) { |i| data.rindex(/$/, i) - 1 }
+    end
+  end
+end

data/lib/stockboy/readers/fixed_width.rb

@@ -0,0 +1,121 @@
+require 'stockboy/reader'
+require 'stockboy/configuration'
+
+module Stockboy::Readers
+
+  # For reading fixed-width data split by column widths
+  #
+  class FixedWidth < Stockboy::Reader
+
+    # @!group Options
+
+    # Widths of data columns with optional names
+    #
+    # Array format will use numeric indexes for field keys. Hash will use the
+    # keys for naming the fields.
+    #
+    # @return [Array<Fixnum>, Hash{Object=>Fixnum}]
+    # @example
+    #   reader.headers = [10, 5, 10, 42]
+    #   reader.parse(data)
+    #   #=> [{0=>"Arthur", 1=>"42", 2=>"Earth", 3=>""}]
+    #
+    #   reader.headers = {name: 10, age: 5, planet: 10, notes: 42}
+    #   reader.parse(data)
+    #   #=> [{name: "Arthur", age: "42", planet: "Earth", notes: ""}]
+    #
+    dsl_attr :headers
+
+    # String format used for unpacking rows
+    #
+    # This is read from the {#headers} attribute by default but can be
+    # overridden
+    #
+    # @return [String]
+    #
+    dsl_attr :skip_header_rows
+
+    # Number of file rows to skip from start of file
+    #
+    # Useful if the file starts with a preamble or header metadata
+    #
+    # @return [Fixnum]
+    #
+    dsl_attr :skip_footer_rows
+
+    # Number of file rows to skip at end of file
+    #
+    # Useful if the file ends with a summary or notice
+    #
+    # @return [Fixnum]
+    #
+    dsl_attr :row_format
+
+    # Override original file encoding
+    #
+    # @return [String]
+    #
+    dsl_attr :encoding
+
+    # @!endgroup
+
+    # Initialize a new fixed-width reader
+    #
+    # @param [Hash] opts
+    # @option opts [Array<Fixnum>, Hash<Fixnum>] headers
+    # @option opts [Fixnum] skip_header_rows
+    # @option opts [Fixnum] skip_footer_rows
+    # @option opts [String] encoding
+    #
+    def initialize(opts={}, &block)
+      super
+      @headers          = opts[:headers]
+      @skip_header_rows = opts.fetch(:skip_header_rows, 0)
+      @skip_footer_rows = opts.fetch(:skip_footer_rows, 0)
+      DSL.new(self).instance_eval(&block) if block_given?
+    end
+
+    def parse(data)
+      @column_widths, @column_keys = nil, nil
+      data.force_encoding!(encoding) if encoding
+      data = StringIO.new(data) unless data.is_a? StringIO
+      skip_header_rows.times { data.readline }
+      records = data.reduce([]) do |a, row|
+        a.tap { a << parse_row(row) unless row.strip.empty? }
+      end
+      skip_footer_rows.times { records.pop }
+      records
+    end
+
+    def row_format
+      @row_format ||= (?A << column_widths.join(?A)).freeze
+    end
+
+    private
+
+    def column_widths
+      return @column_widths if @column_widths
+      @column_widths = case headers
+      when Hash then headers.values
+      when Array then headers
+      else
+        raise "Invalid headers set for #{self.class}"
+      end
+    end
+
+    def column_keys
+      return @column_keys if @column_keys
+      @column_keys = case headers
+      when Hash then headers.keys.map(&:freeze)
+      when Array then (0 ... headers.length).to_a
+      else
+        raise "Invalid headers set for #{self.class}"
+      end
+    end
+
+    def parse_row(row)
+      Hash[column_keys.zip(row.unpack(row_format))]
+    end
+
+  end
+end

data/lib/stockboy/readers/spreadsheet.rb

@@ -0,0 +1,144 @@
+require 'stockboy/reader'
+require 'tempfile'
+require 'roo'
+
+module Stockboy::Readers
+
+  # Parse an Excel spreadsheet
+  #
+  # Backed by Roo gem. See roo for other configuration options.
+  #
+  class Spreadsheet < Stockboy::Reader
+
+    # Spreadsheet format
+    #
+    # @!attribute [rw] format
+    # @return [Symbol] +:xls+ or +:xslx+
+    #
+    dsl_attr :format
+
+    # Spreadsheet sheet number, defaults to first
+    #
+    # @!attribute [rw] sheet
+    # @return [Fixnum]
+    #
+    dsl_attr :sheet
+
+    # Line number to look for headers, starts counting at 1, like in Excel
+    #
+    # @!attribute [rw] header_row
+    # @return [Fixnum]
+    #
+    dsl_attr :header_row
+
+    # Line number of first data row, starts counting at 1, like in Excel
+    #
+    # @!attribute [rw] first_row
+    # @return [Fixnum]
+    #
+    dsl_attr :first_row
+
+    # Line number of last data row, use negative numbers to count back from end
+    #
+    # @!attribute [rw] last_row
+    # @return [Fixnum]
+    #
+    dsl_attr :last_row
+
+    # Override to set headers manually
+    #
+    # @!attribute [rw] headers
+    # @return [Array]
+    #
+    dsl_attr :headers
+
+    # @!endgroup
+
+    # Initialize a new Spreadsheet reader
+    #
+    # @param [Hash] opts
+    #
+    def initialize(opts={}, &block)
+      super
+      @format = opts[:format] || :xls
+      @sheet  = opts[:sheet]  || :first
+      @first_row = opts[:first_row]
+      @last_row  = opts[:last_row]
+      @header_row  = opts[:header_row]
+      @headers = opts[:headers]
+      @roo_options = opts[:roo_options] || {}
+      DSL.new(self).instance_eval(&block) if block_given?
+    end
+
+    def parse(content)
+      with_spreadsheet_tempfile(content) do |table|
+        headers = table_headers(table)
+
+        enum_data_rows(table).inject([]) do |rows, i|
+          rows << Hash[headers.zip(table.row(i))]
+        end
+      end
+    end
+
+    # Roo-specific options hash passed to underlying spreadsheet parser
+    #
+    # @!attribute [r] options
+    # @return [Hash]
+    #
+    def options
+      @roo_options
+    end
+
+    private
+
+    def enum_data_rows(table)
+      first_table_row(table).upto last_table_row(table)
+    end
+
+    def with_spreadsheet_tempfile(content)
+      Tempfile.open(tmp_name, Stockboy.configuration.tmp_dir) do |file|
+        file.binmode
+        file.write content
+        table = Roo::Spreadsheet.open(file.path, @roo_options)
+        table.default_sheet = sheet_number(table, @sheet)
+        table.header_line = @header_line if @header_line
+        yield table
+      end
+    end
+
+    def sheet_number(table, id)
+      case id
+      when Symbol then table.sheets.public_send id
+      when Fixnum then table.sheets[id-1]
+      when String then id
+      end
+    end
+
+    def first_table_row(table)
+      @first_row || table.first_row
+    end
+
+    def last_table_row(table)
+      if @last_row.to_i < 0
+        table.last_row + @last_row + 1
+      elsif @last_row.to_i > 0
+        @last_row
+      else
+        table.last_row
+      end
+    end
+
+    def table_headers(table)
+      return @headers if @headers
+      table.row(table_header_row(table)).map { |h| h.to_s unless h.nil? }
+    end
+
+    def table_header_row(table)
+      [table.header_line, table.first_row].max
+    end
+
+    def tmp_name
+      ['stockboy', ".#{@format}"]
+    end
+  end
+end