stockboy 0.5.0

data/lib/stockboy/providers/http.rb

@@ -0,0 +1,123 @@
+require 'stockboy/provider'
+require 'httpi'
+
+module Stockboy::Providers
+
+  # Fetches data from an HTTP endpoint
+  #
+  # == Job template DSL
+  #
+  #   provider :http do
+  #     get "http://example.com/api/things"
+  #   end
+  #
+  class HTTP < Stockboy::Provider
+
+    # @!group Options
+
+    # Shorthand for +:method+ and +:uri+ using HTTP GET
+    #
+    # @!attribute [rw] get
+    # @return [String]
+    # @example
+    #   get 'http://example.com/api/things'
+    #
+    dsl_attr :get, attr_writer: false
+
+    # Shorthand for +:method+ and +:uri+ using HTTP POST
+    #
+    # @!attribute [rw] post
+    # @return [String]
+    # @example
+    #   post 'http://example.com/api/search'
+    #
+    dsl_attr :post, attr_writer: false
+
+    # HTTP method: +:get+ or +:post+
+    #
+    # @!attribute [rw] method
+    # @return [Symbol]
+    # @example
+    #   method :post
+    #
+    dsl_attr :method, attr_writer: false
+
+    def uri
+      return nil if @uri.nil? || @uri.empty?
+      URI(@uri).tap { |u| u.query = URI.encode_www_form(@query) }
+    end
+
+    def uri=(uri)
+      @uri = uri
+    end
+
+    # HTTP host and path to the data resource
+    #
+    # @!attribute [rw] uri
+    # @return [String]
+    # @example
+    #   uri 'http://example.com/api/things'
+    #
+    dsl_attr :uri, attr_accessor: false, alias: :url
+
+    # Hash of query options
+    #
+    # @!attribute [rw] query
+    # @return [Hash]
+    # @example
+    #   query start: 1, limit: 100
+    #
+    dsl_attr :query
+
+    def method=(http_method)
+      return @method = nil unless %w(get post).include? http_method.to_s.downcase
+      @method = http_method.to_s.downcase.to_sym
+    end
+
+    def get=(uri)
+      @method = :get
+      @uri = uri
+    end
+
+    def post=(uri)
+      @method = :post
+      @uri = uri
+    end
+
+    # @!endgroup
+
+    # Initialize an HTTP provider
+    #
+    def initialize(opts={}, &block)
+      super(opts, &block)
+      self.uri    = opts[:uri]
+      self.method = opts[:method] || :get
+      self.query  = opts[:query]  || Hash.new
+      DSL.new(self).instance_eval(&block) if block_given?
+    end
+
+    def client
+      return HTTPI unless block_given?
+      yield HTTPI
+    end
+
+    private
+
+    def validate
+      errors.add_on_blank [:uri, :method]
+      errors.empty?
+    end
+
+    def fetch_data
+      request = HTTPI::Request.new
+      request.url = uri
+      response = HTTPI.send(method, request)
+      if response.error?
+        errors.add :response, "HTTP respone error: #{response.code}"
+      else
+        @data = response.body
+      end
+    end
+
+  end
+end

data/lib/stockboy/providers/imap.rb

@@ -0,0 +1,290 @@
+require 'stockboy/provider'
+require 'net/imap'
+require 'mail'
+
+module Stockboy::Providers
+
+  # Read data from a file attachment in IMAP email
+  #
+  # == Job template DSL
+  #
+  #   provider :imap do
+  #     host "imap.example.com"
+  #     username "arthur@example.com"
+  #     password "424242"
+  #     mailbox "INBOX"
+  #     subject "Daily Report"
+  #     since Date.today
+  #     file_name /report-[0-9]+\.csv/
+  #   end
+  #
+  class IMAP < Stockboy::Provider
+
+    # Corresponds to %v mode in +DateTime#strftime+
+    VMS_DATE = /\A\d{2}-(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)-\d{2}\z/i
+
+    # @!group Options
+
+    # Host name or IP address for IMAP server connection
+    #
+    # @!attribute [rw] host
+    # @return [String]
+    # @example
+    #   host "imap.example.com"
+    #
+    dsl_attr :host
+
+    # User name for connection credentials
+    #
+    # @!attribute [rw] username
+    # @return [String]
+    # @example
+    #   username "arthur@example.com"
+    #
+    dsl_attr :username
+
+    # Password for connection credentials
+    #
+    # @!attribute [rw] password
+    # @return [String]
+    # @example
+    #   password "424242"
+    #
+    dsl_attr :password
+
+    # Where to look for email on the server (usually "INBOX")
+    #
+    # @!attribute [rw] mailbox
+    # @return [String]
+    # @example
+    #   mailbox "INBOX"
+    #
+    dsl_attr :mailbox
+
+    # Substring to find contained in matching email subject
+    #
+    # @!attribute [rw] subject
+    # @return [String]
+    # @example
+    #   subject "Daily Report"
+    #
+    dsl_attr :subject
+
+    # Email address of the sender
+    #
+    # @!attribute [rw] from
+    # @return [String]
+    # @example
+    #   from "sender+12345@example.com"
+    #
+    dsl_attr :from
+
+    # Minimum time sent for matching email
+    #
+    # @!attribute [rw] since
+    # @return [String]
+    # @example
+    #   since Date.today
+    #
+    dsl_attr :since, alias: :newer_than
+
+    # Key-value tokens for IMAP search options
+    #
+    # @!attribute [rw] search
+    # @return [String]
+    # @example
+    #   search ['FLAGGED', 'BODY', 'Report attached']
+    #
+    dsl_attr :search
+
+    # Name or pattern for matching attachment files. First matching attachment
+    # is picked, or the first attachment if not specified.
+    #
+    # @!attribute [rw] attachment
+    # @return [String, Regexp]
+    # @example
+    #   attachment "daily-report.csv"
+    #   attachment /daily-report-[0-9]+.csv/
+    #
+    dsl_attr :attachment
+
+    # Method for choosing which email message to process from potential
+    # matches. Default is last by date sent.
+    #
+    # @!attribute [rw] pick
+    # @return [Symbol, Proc]
+    # @example
+    #   pick :last
+    #   pick :first
+    #   pick ->(list) {
+    #     list.max_by { |msgid| client.fetch(msgid, 'SENTON').to_i }
+    #   }
+    #
+    dsl_attr :pick
+
+    # @!endgroup
+
+    # Library for connection, defaults to +Net::IMAP+
+    #
+    # @!attribute [rw] imap_client
+    #
+    def self.imap_client
+      @imap_client ||= Net::IMAP
+    end
+    class << self
+      attr_writer :imap_client
+    end
+
+    # Initialize a new IMAP reader
+    #
+    def initialize(opts={}, &block)
+      super(opts, &block)
+      @host         = opts[:host]
+      @username     = opts[:username]
+      @password     = opts[:password]
+      @mailbox      = opts[:mailbox]
+      @subject      = opts[:subject]
+      @from         = opts[:from]
+      @since        = opts[:since]
+      @search       = opts[:search]
+      @attachment   = opts[:attachment]
+      @file_smaller = opts[:file_smaller]
+      @file_larger  = opts[:file_larger]
+      @pick         = opts[:pick] || :last
+      DSL.new(self).instance_eval(&block) if block_given?
+    end
+
+    def client
+      raise(ArgumentError, "no block given") unless block_given?
+      return yield @open_client if @open_client
+
+      @open_client = ::Net::IMAP.new(host).tap do |i|
+        i.login(username, password)
+        i.examine(mailbox)
+      end
+      yield @open_client
+      client.disconnect
+      @open_client = nil
+    rescue ::Net::IMAP::Error => e
+      errors.add :response, "IMAP connection error"
+      client.disconnect
+      @open_client = nil
+    end
+
+    def delete_data
+      raise Stockboy::OutOfSequence, "must confirm #matching_message or calling #data" unless picked_matching_message?
+
+      logger.info "Deleting message #{username}:#{host} message_uid: #{matching_message}"
+      client do |imap|
+        imap.uid_store(matching_message, "+FLAGS", [:Deleted])
+        imap.expunge
+      end
+    end
+
+    def matching_message
+      return @matching_message if @matching_message
+      keys = fetch_imap_message_keys
+      @matching_message = pick_from(keys) unless keys.empty?
+    end
+
+    def clear
+      super
+      @matching_message = nil
+      @data_time = nil
+      @data_size = nil
+    end
+
+    private
+
+    def fetch_data
+      client do |imap|
+        return false unless matching_message
+        mail = ::Mail.new(imap.fetch(matching_message, 'RFC822')[0].attr['RFC822'])
+        if part = mail.attachments.detect { |part| validate_attachment(part) }
+          validate_file(part.decoded)
+          if valid?
+            logger.info "Getting file from #{username}:#{host} message_uid #{matching_message}"
+            @data = part.decoded
+            @data_time = normalize_imap_datetime(mail.date)
+            logger.info "Got file from #{username}:#{host} message_uid #{matching_message}"
+          end
+        end
+      end
+      !@data.nil?
+    end
+
+    def validate
+      errors.add_on_blank [:host, :username, :password]
+      errors.empty?
+    end
+
+    def fetch_imap_message_keys
+      client { |imap| imap.sort(['DATE'], search_keys, 'UTF-8') }
+    end
+
+    def picked_matching_message?
+      !!@matching_message
+    end
+
+    def validate_attachment(part)
+      case attachment
+      when String
+        part.filename == attachment
+      when Regexp
+        part.filename =~ attachment
+      else
+        true
+      end
+    end
+
+    def search_keys
+      keys = []
+      keys.concat ['SUBJECT', subject]  if subject
+      keys.concat ['FROM', from]        if from
+      keys.concat ['SINCE', date_format(since)] if since
+      keys.concat search if search
+      keys
+    end
+
+    def date_format(value)
+      case value
+      when Date, Time, DateTime
+        value.strftime('%v')
+      when Numeric
+        Time.at(value).strftime('%v')
+      when String
+        return value if value =~ VMS_DATE
+        Date.parse(value).strftime('%v')
+      end
+    end
+
+    # If activesupport is loaded, it mucks with DateTime#to_time to return
+    # self when it has a utc_offset. Handle both to always return a Time.utc.
+    #
+    def normalize_imap_datetime(datetime)
+      datetime.respond_to?(:getutc) ?
+        datetime.getutc.to_time : datetime.to_time.utc
+    end
+
+    def validate_file(data_file)
+      return errors.add :response, "No matching attachments" unless data_file
+      validate_file_smaller(data_file)
+      validate_file_larger(data_file)
+    end
+
+    def validate_file_smaller(data_file)
+      @data_size ||= data_file.bytesize
+      if file_smaller && @data_size > file_smaller
+        errors.add :response, "File size larger than #{file_smaller}"
+      end
+    end
+
+    def validate_file_larger(data_file)
+      @data_size ||= data_file.bytesize
+      if file_larger && @data_size < file_larger
+        errors.add :response, "File size smaller than #{file_larger}"
+      end
+    end
+  end
+
+end

data/lib/stockboy/providers/soap.rb

@@ -0,0 +1,120 @@
+require 'stockboy/provider'
+require 'stockboy/string_pool'
+require 'savon'
+
+module Stockboy::Providers
+
+  # Fetch data from a SOAP endpoint
+  #
+  # Backed by Savon gem, see savon for full configuration options: extra
+  # options are passed through.
+  #
+  class SOAP < Stockboy::Provider
+    include Stockboy::StringPool
+
+    # @!group Options
+    #
+    # These options correspond to Savon client options
+
+    # URL with the WSDL document
+    #
+    # @!attribute [rw] wsdl
+    # @return [String]
+    # @example
+    #   wsdl "http://example.com/api/soap?wsdl"
+    #
+    dsl_attr :wsdl
+
+    # The name of the request, see your SOAP documentation
+    #
+    # @!attribute [rw] request
+    # @return [String]
+    # @example
+    #   request "allItemsDetails"
+    #
+    dsl_attr :request
+
+    # @return [String]
+    # @!attribute [rw] namespace
+    #   Optional if specified in WSDL
+    #
+    dsl_attr :namespace
+
+    # @return [String]
+    # @!attribute [rw] namespace_id
+    #   Optional if specified in WSDL
+    #
+    dsl_attr :namespace_id
+
+    # @return [String]
+    # @!attribute [rw] endpoint
+    #   Optional if specified in WSDL
+    #
+    dsl_attr :endpoint
+
+    # Hash of message options passed in the request, often includes
+    # credentials and query options.
+    #
+    # @!attribute [rw] message
+    # @return [Hash]
+    # @example
+    #   message "clientId" => "12345", "updatedSince" => "2012-12-12"
+    #
+    dsl_attr :message
+
+    # Hash of optional HTTP request headers
+    #
+    # @!attribute [rw] headers
+    # @return [Hash]
+    # @example
+    #   headers "X-ClientKey" => "12345"
+    #
+    dsl_attr :headers
+
+    # @!endgroup
+
+    # Initialize a new SOAP provider
+    #
+    def initialize(opts={}, &block)
+      super
+      DSL.new(self).instance_eval(&block) if block_given?
+    end
+
+    # Connection object to the configured SOAP endpoint
+    #
+    # @return [Savon::Client]
+    #
+    def client
+      @client ||= Savon.client(client_options)
+      return @client unless block_given?
+      yield @client
+    end
+
+    private
+
+    def client_options
+      opts = if wsdl
+        {wsdl: wsdl}
+      elsif endpoint
+        {endpoint: endpoint}
+      end
+      opts[:convert_response_tags_to] = ->(tag) { string_pool(tag) }
+      opts[:namespace] = namespace if namespace
+      opts[:namespace_identifier] = namespace_id if namespace_id
+      opts[:headers] = headers if headers
+      opts
+    end
+
+    def validate
+      errors.add_on_blank(:endpoint) unless wsdl
+      errors.blank?
+    end
+
+    def fetch_data
+      with_string_pool do
+        @data = client.call(@request, message: message).body
+      end
+    end
+
+  end
+end