stockboy 0.5.0

data/lib/stockboy/mapped_record.rb

@@ -0,0 +1,59 @@
+module Stockboy
+
+  # This represents the "output" side of a {CandidateRecord}
+  #
+  # Based on the current attribute map, it will have reader methods for the
+  # output values of each attribute. This is similar to an OpenStruct, but
+  # more efficient since we cache the defined methods.
+  #
+  # @example
+  #   output = MappedRecord.new(first_name: "Zaphod")
+  #   output.first_name # => "Zaphod"
+  #
+  class MappedRecord
+
+    # This is an optimization to avoid relying on method_missing.
+    #
+    # This module holds a pool of already-defined accessor methods for sets of
+    # record attributes. Each set of methods is held in a module that gets
+    # extended into new MappedRecords.
+    #
+    # @visibility private
+    #
+    module AccessorMethods
+      def self.for(attrs)
+        @module_registry        ||= Hash.new
+        @module_registry[attrs] ||= build_module(attrs)
+      end
+
+      def self.build_module(attr_accessor_keys)
+        Module.new do
+          attr_accessor_keys.each do |key|
+            define_method key do
+              @fields[key]
+            end
+          end
+        end
+      end
+    end
+
+    # Initialize a new MappedRecord
+    #
+    # @param [Hash<Symbol>] fields
+    #   Keys map to reader methods
+    #
+    def initialize(fields)
+      mod = AccessorMethods.for(fields.keys)
+      extend mod
+      @fields = fields
+      freeze
+    end
+
+    # @return [String]
+    #
+    def to_s
+      @fields.to_s
+    end
+
+  end
+end

data/lib/stockboy/provider.rb

@@ -0,0 +1,238 @@
+require 'logger'
+require 'active_model/errors'
+require 'active_model/naming'
+require 'stockboy/dsl'
+require 'stockboy/exceptions'
+
+module Stockboy
+
+  # Provider objects handle the connection and capture of data from remote
+  # sources. This is an abstract superclass to help implement different
+  # providers.
+  #
+  # == Interface
+  #
+  # A provider object must implement the following (private) methods:
+  #
+  # [validate]
+  #   Verify the parameters required for the data source are set to
+  #   ensure a connection can be established.
+  # [fetch_data]
+  #   Populate the object's +@data+ with raw content from source. This will
+  #   usually be a raw string, and should not be parsed at this stage.
+  #   Depending on the implementation, this may involve any of:
+  #   * Establishing a connection
+  #   * Navigating to a directory
+  #   * Listing available files matching the configuration
+  #   * Picking the appropriate file
+  #   * And finally, reading/downloading data
+  #   This should also capture the timestamp of the data resource into
+  #   +@data_time+. This should be the actual created or updated time of the
+  #   data file from the source.
+  #
+  # @abstract
+  #
+  class Provider
+    extend Stockboy::DSL
+    extend ActiveModel::Naming # Required by ActiveModel::Errors
+
+    # Default logger if none is provided to the instance
+    #
+    # @return [Logger]
+    #
+    def self.logger
+      Logger.new(STDERR)
+    end
+
+    # @return [Logger]
+    #
+    attr_accessor :logger
+
+    # @return [ActiveModel::Errors]
+    #
+    attr_reader :errors
+
+    # Timestamp of the received data
+    #
+    # @return [Time]
+    #
+    attr_reader :data_time
+
+    # @return [String]
+    #
+    def inspect
+    "#<#{self.class}:#{self.object_id} data_size=#{@data_size or 'nil'} errors=#{@errors.full_messages}>"
+    end
+
+    # Must be called by subclasses via +super+ to set up dependencies
+    #
+    # @param [Hash] opts
+    # @yield DSL context for configuration
+    #
+    def initialize(opts={}, &block)
+      @logger = opts.delete(:logger) || Stockboy.configuration.logger
+      clear
+    end
+
+    # Raw input data from the source
+    #
+    # @!attribute [r] data
+    #
+    def data
+      return @data if @data
+      fetch_data if validate_config?
+      @data
+    end
+
+    # Reset received data
+    #
+    # @return [Boolean] Always true
+    #
+    def clear
+      @data = nil
+      @data_time = nil
+      @data_size = nil
+      @errors = ActiveModel::Errors.new(self)
+      true
+    end
+    alias_method :reset, :clear
+
+    # Reload provided data
+    #
+    # @return [String] Raw data
+    #
+    def reload
+      clear
+      fetch_data if validate_config?
+      @data
+    end
+
+    # Does the provider have what it needs for fetching data?
+    #
+    # @return [Boolean]
+    #
+    def valid?
+      validate
+    end
+
+    private
+
+    # Subclass should assign +@data+ with raw input, usually a string
+    #
+    # @abstract
+    #
+    def fetch_data
+      raise NoMethodError, "#{self.class}#fetch_data needs implementation"
+    end
+
+    # Use errors.add(:attribute, "Message") provided by ActiveModel
+    # for validating required provider parameters before attempting
+    # to make connections and retrieve data.
+    #
+    # @abstract
+    #
+    def validate
+      raise NoMethodError, "#{self.class}#validate needs implementation"
+    end
+
+    def validate_config?
+      unless validation = valid?
+        logger.error do
+          "Invalid #{self.class} provider configuration: #{errors.full_messages}"
+        end
+      end
+      validation
+    end
+
+    # Required by ActiveModel::Errors
+    def read_attribute_for_validation(attr)
+      send(attr)
+    end
+
+    # Required by ActiveModel::Errors
+    def self.human_attribute_name(attr, options = {})
+      attr
+    end
+
+    # Required by ActiveModel::Errors
+    def self.lookup_ancestors
+      [self]
+    end
+
+    # When picking files from a list you can supply +:first+ or +:last+ to the
+    # provider's +pick+ option, or else a block that can reduce to a single
+    # value, like:
+    #
+    #     proc do |best_match, current_match|
+    #       current_match.better_than?(best_match) ?
+    #           current_match : best_match
+    #     end
+    #
+    def pick_from(list, &block)
+      case @pick
+      when Symbol
+        list.public_send @pick
+      when Proc
+        list.reduce &@pick
+      end
+    end
+
+  end
+
+  # @!macro [new] provider.pick_validation
+  #   This validation option is applied after a matching file is picked.
+
+  # @!macro [new] provider.pick_option
+  #   @group Options
+  #
+  #   @!attribute [rw] pick
+  #     Method for choosing which file to process from potential matches.
+  #       @example
+  #         pick :last
+  #         pick :first
+  #         pick ->(list) {
+  #           list.max_by { |name| Time.strptime(name[/\d+/], "%m%d%Y").to_i }
+  #         }
+
+  # @!macro [new] provider.file_options
+  #   @group Options
+  #
+  #   @!attribute [rw] file_name
+  #     A string (glob) or regular expression matching files. E.g. one of:
+  #     @return [String, Regexp]
+  #     @example
+  #       file_name "export-latest.csv"
+  #       file_name "export-*.csv"
+  #       file_name /export-\d{4}-\d{2}-\d{2}.csv/
+  #
+  #   @!attribute [rw] file_dir
+  #     Path where data files can be found. This should be an absolute path.
+  #     @return [String]
+  #     @example
+  #       file_dir "/data"
+  #
+  #   @!attribute [rw] file_newer
+  #     Validates that the file to be processed is recent enough. To guard
+  #     against processing an old file (even if it's the latest one), this should
+  #     be set to the frequency you expect to receive new files for periodic
+  #     processing.
+  #     @macro provider.pick_validation
+  #     @return [Time, Date]
+  #     @example
+  #       since Date.today
+  #
+  #   @!attribute [rw] file_smaller
+  #     Validates the maximum data size for the matched file, in bytes
+  #     @return [Fixnum]
+  #     @macro provider.pick_validation
+  #     @example
+  #       file_smaller 1024^3
+  #
+  #   @!attribute [rw] file_larger
+  #     Validates the minimum file size for the matched file, in bytes. This can # help guard against processing zero-byte or truncated files.
+  #     @return [Fixnum]
+  #     @macro provider.pick_validation
+  #     @example
+  #       file_larger 1024
+
+end

data/lib/stockboy/providers.rb

@@ -0,0 +1,11 @@
+require 'stockboy/registry'
+
+module Stockboy
+
+  # Registry of available providers
+  #
+  module Providers
+    extend Stockboy::Registry
+  end
+
+end

data/lib/stockboy/providers/file.rb

@@ -0,0 +1,135 @@
+require 'stockboy/provider'
+
+module Stockboy::Providers
+
+  # Get data from a local file
+  #
+  # Allows for selecting the appropriate file to be read from the given
+  # directory by glob pattern or regex pattern. By default the +:last+ file in
+  # the list is used, but can be controlled by sorting and reducing with the
+  # {#pick} option.
+  #
+  # == Job template DSL
+  #
+  #   provider :file do
+  #     file_dir '/data'
+  #     file_name /report-[0-9]+\.csv/
+  #     pick ->(list) { list[-2] }
+  #   end
+  #
+  class File < Stockboy::Provider
+
+    # @!group Options
+
+    # @macro provider.file_options
+    dsl_attr :file_name
+    dsl_attr :file_dir
+    dsl_attr :file_newer, alias: :since
+    dsl_attr :file_smaller
+    dsl_attr :file_larger
+
+    # @macro provider.pick_option
+    dsl_attr :pick
+
+    # @!endgroup
+
+    # Initialize a File provider
+    #
+    def initialize(opts={}, &block)
+      super(opts, &block)
+      @file_dir     = opts[:file_dir]
+      @file_name    = opts[:file_name]
+      @file_newer   = opts[:file_newer]
+      @file_smaller = opts[:file_smaller]
+      @file_larger  = opts[:file_larger]
+      @pick         = opts[:pick] || :last
+      DSL.new(self).instance_eval(&block) if block_given?
+    end
+
+    def delete_data
+      raise Stockboy::OutOfSequence, "must confirm #matching_file or calling #data" unless picked_matching_file?
+
+      logger.info "Deleting file #{file_dir}/#{matching_file}"
+      ::File.delete matching_file
+    end
+
+    def matching_file
+      return @matching_file if @matching_file
+      files = case file_name
+      when Regexp
+        Dir.entries(file_dir)
+            .select { |i| i =~ file_name }
+            .map { |i| ::File.join(file_dir, i) }
+      when String
+        Dir[::File.join(file_dir, file_name)]
+      end
+      @matching_file = pick_file_from(files) if files.any?
+    end
+
+    def clear
+      super
+      @matching_file = nil
+      @data_size = nil
+      @data_time = nil
+    end
+
+    private
+
+    def fetch_data
+      errors.add(:base, "File #{file_name} not found") unless matching_file
+      data_file = ::File.new(matching_file, 'r') if matching_file
+      validate_file(data_file)
+      if valid?
+        logger.info "Getting file #{file_dir}/#{matching_file}"
+        @data = data_file.read
+        logger.info "Got file #{file_dir}/#{matching_file} (#{@data_size} bytes)"
+      end
+    end
+
+    def validate
+      errors.add_on_blank [:file_dir, :file_name]
+      errors.empty?
+    end
+
+    def pick_file_from(list)
+      case @pick
+      when Symbol
+        list.public_send @pick
+      when Proc
+        list.detect &@pick
+      end
+    end
+
+    def picked_matching_file?
+      !!@matching_file
+    end
+
+    def validate_file(data_file)
+      return errors.add :response, "No matching files" unless data_file
+      validate_file_newer(data_file)
+      validate_file_smaller(data_file)
+      validate_file_larger(data_file)
+    end
+
+    def validate_file_newer(data_file)
+      @data_time ||= data_file.mtime
+      if file_newer && @data_time < file_newer
+        errors.add :response, "No new files since #{file_newer}"
+      end
+    end
+
+    def validate_file_smaller(data_file)
+      @data_size ||= data_file.size
+      if file_smaller && @data_size > file_smaller
+        errors.add :response, "File size larger than #{file_smaller}"
+      end
+    end
+
+    def validate_file_larger(data_file)
+      @data_size ||= data_file.size
+      if file_larger && @data_size < file_larger
+        errors.add :response, "File size smaller than #{file_larger}"
+      end
+    end
+  end
+end

data/lib/stockboy/providers/ftp.rb

@@ -0,0 +1,205 @@
+require 'stockboy/provider'
+require 'net/ftp'
+
+module Stockboy::Providers
+
+  # Get data from a remote FTP server
+  #
+  # Allows for selecting the appropriate file to be read from the given
+  # directory by glob pattern or regex pattern (glob string is more efficient
+  # for listing files from FTP). By default the +:last+ file in the list is
+  # used, but can be controlled by sorting and reducing with the {#pick}
+  # option.
+  #
+  # == Job template DSL
+  #
+  #   provider :ftp do
+  #     host      'ftp.example.com'
+  #     username  'example'
+  #     password  '424242'
+  #     file_dir  'data/daily'
+  #     file_name /report-[0-9]+\.csv/
+  #     pick      ->(list) { list[-2] }
+  #   end
+  #
+  class FTP < Stockboy::Provider
+
+    # @!group Options
+
+    # Host name or IP address for FTP server connection
+    #
+    # @!attribute [rw] host
+    # @return [String]
+    # @example
+    #   host "ftp.example.com"
+    #
+    dsl_attr :host
+
+    # Use a passive or active connection
+    #
+    # @!attribute [rw] passive
+    # @return [Boolean]
+    # @example
+    #   passive true
+    #
+    dsl_attr :passive
+
+    # User name for connection credentials
+    #
+    # @!attribute [rw] username
+    # @return [String]
+    # @example
+    #   username "arthur"
+    #
+    dsl_attr :username
+
+    # Password for connection credentials
+    #
+    # @!attribute [rw] password
+    # @return [String]
+    # @example
+    #   password "424242"
+    #
+    dsl_attr :password
+
+    # Use binary mode for file transfers
+    #
+    # @!attribute [rw] binary
+    # @return [Boolean]
+    # @example
+    #   binary true
+    #
+    dsl_attr :binary
+
+    # @macro provider.file_options
+    dsl_attr :file_name
+    dsl_attr :file_dir
+    dsl_attr :file_newer, alias: :since
+    dsl_attr :file_smaller
+    dsl_attr :file_larger
+
+    # @macro provider.pick_option
+    dsl_attr :pick
+
+    # @!endgroup
+
+    # Initialize a new FTP provider
+    #
+    def initialize(opts={}, &block)
+      super(opts, &block)
+      @host         = opts[:host]
+      @passive      = opts[:passive]
+      @username     = opts[:username]
+      @password     = opts[:password]
+      @binary       = opts[:binary]
+      @file_dir     = opts[:file_dir]
+      @file_name    = opts[:file_name]
+      @file_newer   = opts[:file_newer]
+      @file_smaller = opts[:file_smaller]
+      @file_larger  = opts[:file_larger]
+      @pick         = opts[:pick] || :last
+      DSL.new(self).instance_eval(&block) if block_given?
+    end
+
+    def client
+      return yield @open_client if @open_client
+
+      Net::FTP.open(host, username, password) do |ftp|
+        ftp.binary = binary
+        ftp.passive = passive
+        ftp.chdir file_dir if file_dir
+        @open_client = ftp
+        response = yield ftp
+        @open_client = nil
+        response
+      end
+    rescue Net::FTPError => e
+      errors.add :response, e.message
+      logger.warn e.message
+    end
+
+    def matching_file
+      return @matching_file if @matching_file
+      client do |ftp|
+        file_listing = ftp.nlst.sort
+        @matching_file = pick_from file_listing.select(&file_name_matcher)
+      end
+    end
+
+    def delete_data
+      raise Stockboy::OutOfSequence, "must confirm #matching_file or calling #data" unless picked_matching_file?
+      client do |ftp|
+        logger.info "FTP deleting file #{host} #{file_dir}/#{matching_file}"
+        ftp.delete matching_file
+        matching_file
+      end
+    end
+
+    def clear
+      super
+      @matching_file = nil
+      @data_time = nil
+      @data_size = nil
+    end
+
+    private
+
+    def fetch_data
+      client do |ftp|
+        validate_file(matching_file)
+        if valid?
+          logger.info "FTP getting file #{host} #{file_dir}/#{matching_file}"
+          @data = ftp.get(matching_file, nil)
+          logger.info "FTP got file #{host} #{file_dir}/#{matching_file} (#{@data_size} bytes)"
+        end
+      end
+      !@data.nil?
+    end
+
+    def picked_matching_file?
+      !!@matching_file
+    end
+
+    def validate
+      errors.add_on_blank [:host, :file_name]
+      errors.empty?
+    end
+
+    def file_name_matcher
+      case file_name
+      when Regexp
+        ->(i) { i =~ file_name }
+      when String
+        ->(i) { ::File.fnmatch(file_name, i) }
+      end
+    end
+
+    def validate_file(data_file)
+      return errors.add :response, "No matching files" unless data_file
+      validate_file_newer(data_file)
+      validate_file_smaller(data_file)
+      validate_file_larger(data_file)
+    end
+
+    def validate_file_newer(data_file)
+      @data_time ||= client { |ftp| ftp.mtime(data_file) }
+      if file_newer and @data_time < file_newer
+        errors.add :response, "No new files since #{file_newer}"
+      end
+    end
+
+    def validate_file_smaller(data_file)
+      @data_size ||= client { |ftp| ftp.size(data_file) }
+      if file_smaller and @data_size > file_smaller
+        errors.add :response, "File size larger than #{file_smaller}"
+      end
+    end
+
+    def validate_file_larger(data_file)
+      @data_size ||= client { |ftp| ftp.size(data_file) }
+      if file_larger and @data_size < file_larger
+        errors.add :response, "File size smaller than #{file_larger}"
+      end
+    end
+  end
+end