salt-parser 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ba5f5e0a409feb645962c2649710323a47534b8c
+  data.tar.gz: 43f111a2394f816e7b6905931eef337a95dd8875
+SHA512:
+  metadata.gz: 595e2860944ab5083e02b8146eaf6d1be427ad3a061f28e3ad2350eb858eb2b9fcc4dd22732888a99adadf4d5f39a8d38481b50d0fd18b419eb184e299907983
+  data.tar.gz: f909cf1b0a524d9fe9904b77c6f78baef50f3f13fb81d6cba3fe044ca5c782778be7598055acb2199d026a8f1285351801ad3067b955cf568a4a8584e3498860

data/.gitignore

@@ -0,0 +1,11 @@
+.bundle/*
+coverage/*
+vendor/bundle
+.DS_Store
+*.gem
+pkg
+.project
+
+# temporary files
+bin/**
+spec/tmp/**

data/.rspec

@@ -0,0 +1 @@
+--color

data/Gemfile

@@ -0,0 +1,15 @@
+source "https://rubygems.org"
+
+group :test do
+  gem 'timecop'
+  gem 'rspec'
+  gem 'simplecov', require: false
+end
+
+gem 'activesupport'
+gem 'chronic'
+gem 'rest-client'
+gem 'rack'
+gem 'yajl-ruby', require: 'yajl'
+gem 'nokogiri'
+gem 'pry-byebug', "~> 1.3.2"

data/Gemfile.lock

@@ -0,0 +1,88 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (4.2.3)
+      i18n (~> 0.7)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.3, >= 0.3.4)
+      tzinfo (~> 1.1)
+    byebug (2.7.0)
+      columnize (~> 0.3)
+      debugger-linecache (~> 1.2)
+    chronic (0.10.2)
+    coderay (1.1.0)
+    columnize (0.9.0)
+    debugger-linecache (1.2.0)
+    diff-lcs (1.2.5)
+    docile (1.1.5)
+    domain_name (0.5.24)
+      unf (>= 0.0.5, < 1.0.0)
+    http-cookie (1.0.2)
+      domain_name (~> 0.5)
+    i18n (0.7.0)
+    json (1.8.3)
+    method_source (0.8.2)
+    mime-types (2.6.1)
+    mini_portile (0.6.2)
+    minitest (5.8.0)
+    netrc (0.10.3)
+    nokogiri (1.6.6.2)
+      mini_portile (~> 0.6.0)
+    pry (0.10.1)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
+    pry-byebug (1.3.3)
+      byebug (~> 2.7)
+      pry (~> 0.10)
+    rack (1.6.4)
+    rest-client (1.8.0)
+      http-cookie (>= 1.0.2, < 2.0)
+      mime-types (>= 1.16, < 3.0)
+      netrc (~> 0.7)
+    rspec (3.3.0)
+      rspec-core (~> 3.3.0)
+      rspec-expectations (~> 3.3.0)
+      rspec-mocks (~> 3.3.0)
+    rspec-core (3.3.2)
+      rspec-support (~> 3.3.0)
+    rspec-expectations (3.3.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.3.0)
+    rspec-mocks (3.3.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.3.0)
+    rspec-support (3.3.0)
+    simplecov (0.10.0)
+      docile (~> 1.1.0)
+      json (~> 1.8)
+      simplecov-html (~> 0.10.0)
+    simplecov-html (0.10.0)
+    slop (3.6.0)
+    thread_safe (0.3.5)
+    timecop (0.8.0)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.7.1)
+    yajl-ruby (1.2.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activesupport
+  chronic
+  nokogiri
+  pry-byebug (~> 1.3.2)
+  rack
+  rest-client
+  rspec
+  simplecov
+  timecop
+  yajl-ruby
+
+BUNDLED WITH
+   1.10.6

data/README.rdoc

@@ -0,0 +1,39 @@
+= Salt Parser
+
+Library for parsing OFX, HBCI, QIF and SWIFT formats.
+
+TODOS:
+- Add SWIFT parser
+- Add HBCI parser
+- Add code examples
+
+== Credits
+
+Special thanks to:
+
+- @annacruz (ofx)[https://github.com/annacruz/ofx]
+- @jemmyw (qif)[https://github.com/jemmyw/Qif]
+
+
+== License
+
+(The MIT License)
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,7 @@
+require "bundler"
+Bundler::GemHelper.install_tasks
+
+require "rspec/core/rake_task"
+RSpec::Core::RakeTask.new
+
+task :default => :spec

data/lib/ofx/account.rb

@@ -0,0 +1,28 @@
+module SaltParser
+  module Ofx
+    class Account < SaltParser::Base
+      attr_accessor :balance, :bank_id, :broker_id, :currency, :id, :name,
+                    :transactions, :type, :units, :unit_price, :available_balance
+
+      def identifier
+        id
+      end
+
+      def to_hash
+        {
+          :balance           => balance ? balance.to_hash : nil,
+          :bank_id           => bank_id,
+          :broker_id         => broker_id,
+          :currency          => currency,
+          :id                => id,
+          :name              => name,
+          :transactions      => transactions.map(&:to_hash),
+          :type              => type,
+          :units             => units,
+          :unit_price        => unit_price,
+          :available_balance => available_balance ? available_balance.to_hash : nil
+        }
+      end
+    end
+  end
+end

data/lib/ofx/accounts.rb

@@ -0,0 +1,9 @@
+module SaltParser
+  module Ofx
+    class Accounts < SaltParser::Accounts
+      def find_by_transaction(transaction)
+        find(transaction.account_id)
+      end
+    end
+  end
+end

data/lib/ofx/balance.rb

@@ -0,0 +1,15 @@
+module SaltParser
+  module Ofx
+    class Balance < SaltParser::Base
+      attr_accessor :amount, :amount_in_pennies, :posted_at
+
+      def to_hash
+        {
+          :amount            => amount,
+          :amount_in_pennies => amount_in_pennies,
+          :posted_at         => posted_at
+        }
+      end
+    end
+  end
+end

data/lib/ofx/builder.rb

@@ -0,0 +1,61 @@
+module SaltParser
+  module Ofx
+    class Builder < SaltParser::Builder
+
+      def initialize(resource)
+        resource = open_resource(resource)
+        resource.rewind
+        begin
+          @content = convert_to_utf8(resource.read)
+          prepare(content)
+        rescue Exception
+          raise SaltParser::Error::UnsupportedFileError
+        end
+
+        @parser = case headers["VERSION"]
+        when /102|103/ then
+          SaltParser::Ofx::Parser::OFX102.new(:headers => headers, :body => body)
+        when /200|202|211/ then
+          SaltParser::Ofx::Parser::OFX211.new(:headers => headers, :body => body)
+        else
+          raise SaltParser::Error::UnsupportedFileError
+        end
+      end
+
+      private
+
+      def prepare(content)
+        # split headers & body
+        header_text, body_text = content.dup.split(/<OFX>/, 2)
+        header_text.gsub!("encoding=\"USASCII\"", "encoding=\"US-ASCII\"") if header_text.include?("encoding=\"USASCII\"")
+
+        raise SaltParser::Error::UnsupportedFileError unless body_text
+
+        @headers = extract_headers(header_text)
+        @body    = extract_body(body_text)
+      end
+
+      def extract_headers(header_text)
+        # Header format is different among versions. Give each
+        # parser a chance to parse the headers.
+        headers = nil
+
+        SaltParser::Ofx::Parser.constants.grep(/OFX/).each do |name|
+          headers = SaltParser::Ofx::Parser.const_get(name).parse_headers(header_text)
+          break if headers
+        end
+
+        raise SaltParser::Error::UnsupportedFileError if headers.empty?
+        headers
+      end
+
+      def extract_body(body_text)
+        # Replace body tags to parse it with Nokogiri
+        body_text.gsub!(/>\s+</m, "><")
+        body_text.gsub!(/\s+</m, "<")
+        body_text.gsub!(/>\s+/m, ">")
+        body_text.gsub!(/<(\w+?)>([^<]+)/m, '<\1>\2</\1>')
+      end
+    end
+  end
+end

data/lib/ofx/dependencies.rb

@@ -0,0 +1,13 @@
+require "nokogiri"
+require "bigdecimal"
+require "kconv"
+
+require_relative "builder"
+require_relative "parser/base"
+require_relative "parser/ofx102"
+require_relative "parser/ofx211"
+require_relative "balance"
+require_relative "account"
+require_relative "accounts"
+require_relative "sign_on"
+require_relative "transaction"

data/lib/ofx/parser/base.rb

@@ -0,0 +1,366 @@
+module SaltParser
+  module Ofx
+    module Parser
+      class Base
+        ACCOUNT_TYPES = {
+          "CHECKING"   => :checking,
+          "SAVINGS"    => :savings,
+          "CREDITCARD" => :credit_card,
+          "CREDITLINE" => :credit,
+          "INVESTMENT" => :investment,
+          "MONEYMRKT"  => :savings
+        }
+
+        TRANSACTION_TYPES = [
+          'ATM', 'CASH', 'CHECK', 'CREDIT', 'DEBIT', 'DEP', 'DIRECTDEBIT', 'DIRECTDEP', 'DIV',
+          'FEE', 'INT', 'OTHER', 'PAYMENT', 'POS', 'REPEATPMT', 'SRVCHG', 'XFER'
+        ].inject({}) { |hash, tran_type| hash[tran_type] = tran_type.downcase.to_sym; hash }
+
+        attr_reader :headers, :body, :html, :errors, :accounts, :sign_on
+
+        def initialize(options = {})
+          @headers  = options[:headers]
+          @body     = options[:body]
+          @html     = Nokogiri::HTML.parse(body)
+          @errors   = []
+          build_sign_on
+          build_accounts
+          check_for_errors
+        end
+
+        def to_hash
+          {
+            :errors => errors,
+            :sign_on => sign_on.to_hash,
+            :accounts => accounts.to_hash
+          }
+        end
+
+        def build_accounts
+          @accounts = Ofx::Accounts.new
+          build_bank_account
+          build_credit_card_account
+          build_investments_account
+        end
+
+        private
+
+        def check_for_errors
+          statuses = html.search("status").reverse
+          statuses.each do |status|
+            if status.search("severity").inner_text == "ERROR"
+              errors << SaltParser::Error::RequestError.new(["[#{status.search("code").inner_text}]", status.search("message").inner_text].join(" ").strip)
+            end
+          end
+        end
+
+        def build_bank_account
+          html.search("stmttrnrs", "bankacctinfo").each do |account|
+            begin
+              account_id = account.search("bankacctfrom > acctid").inner_text
+
+              @accounts << SaltParser::Ofx::Account.new(
+                :bank_id           => account.search("bankacctfrom > bankid").inner_text,
+                :id                => account_id,
+                :name              => account.parent.search("desc").inner_text,
+                :type              => ACCOUNT_TYPES[account.search("bankacctfrom > accttype").inner_text.to_s.upcase],
+                :transactions      => build_transactions(account.search("banktranlist > stmttrn"), account_id),
+                :balance           => build_balance(account),
+                :available_balance => build_available_balance(account),
+                :currency          => account.search("stmtrs > curdef").inner_text
+              )
+            rescue SaltParser::Error::ParseError => error
+              errors << error
+            end
+          end
+        end
+
+        def build_credit_card_account
+          html.search("ccstmttrnrs", "acctinfo").each do |account|
+            begin
+              account_id = account.search("ccacctfrom > acctid").inner_text
+              next if account_id.blank?
+              @accounts << SaltParser::Ofx::Account.new(
+                :id           => account_id,
+                :name         => account.search("desc").inner_text,
+                :type         => ACCOUNT_TYPES["CREDITCARD"],
+                :transactions => build_transactions(account.search("banktranlist > stmttrn"), account_id),
+                :balance      => build_balance(account),
+                :currency     => account.search("curdef").inner_text
+              )
+            rescue SaltParser::Error::ParseError => error
+              errors << error
+            end
+          end
+        end
+
+        def build_investments_account
+          html.search("invstmttrnrs", "acctinfo").each do |account|
+            begin
+              account_id = account.search("invacctfrom > acctid").inner_text
+              broker_id  = account.search("invacctfrom > brokerid").inner_text
+
+              next if broker_id.blank? or account_id.blank?
+              @accounts << SaltParser::Ofx::Account.new(
+                :id           => account_id,
+                :broker_id    => broker_id,
+                :type         => ACCOUNT_TYPES["INVESTMENT"],
+                :transactions => build_investment_transactions(account.search("invtranlist"), account_id),
+                :balance      => build_investment_balance(account),
+                :currency     => account.search("curdef").inner_text,
+                :units        => compute_investment_units(account),
+                :unit_price   => compute_investment_unit_price(account)
+              )
+            rescue SaltParser::Error::ParseError => error
+              errors << error
+            end
+          end
+        end
+
+        def build_transactions(transactions, account_id)
+          transactions.each_with_object([]) do |transaction, transactions|
+            begin
+              transactions << build_transaction(transaction, account_id)
+            rescue SaltParser::Error::ParseError => error
+              errors << error
+            end
+          end
+        end
+
+        def build_transaction(transaction, account_id)
+          SaltParser::Ofx::Transaction.new(
+            :amount            => build_amount(transaction),
+            :amount_in_pennies => ((build_amount(transaction) * 100).round 2).to_i,
+            :fit_id            => transaction.search("fitid").inner_text,
+            :memo              => transaction.search("memo").inner_text,
+            :name              => transaction.search("name").inner_text,
+            :payee             => transaction.search("payee").inner_text,
+            :check_number      => transaction.search("checknum").inner_text,
+            :ref_number        => transaction.search("refnum").inner_text,
+            :posted_at         => build_date(transaction.search("dtposted").inner_text),
+            :type              => build_type(transaction),
+            :sic               => transaction.search("sic").inner_text,
+            :account_id        => account_id
+          )
+        end
+
+        def build_investment_transactions(transactions_xml, account_id)
+          transactions = transactions_xml.search("stmttrn", "invtran")
+          transactions.each_with_object([]) do |transaction, transactions|
+            begin
+              if transaction.name.include?("stmttrn")
+                transactions << build_investment_transaction(transaction, account_id)
+              else
+                transactions << build_investment_transaction(transaction.parent, account_id)
+              end
+            rescue SaltParser::Error::ParseError => error
+              errors << error
+            end
+          end
+        end
+
+        def build_investment_transaction(transaction, account_id)
+          SaltParser::Ofx::Transaction.new(
+            :amount            => build_investment_amount(transaction),
+            :amount_in_pennies => ((build_investment_amount(transaction) * 100).round 2).to_i,
+            :fit_id            => transaction.search("fitid").inner_text,
+            :memo              => transaction.search("memo").inner_text,
+            :name              => transaction.search("name").inner_text,
+            :posted_at         => build_date(transaction.search("dtposted", "dttrade").inner_text),
+            :type              => build_type(transaction),
+            :ref_number        => transaction.search("refnum", "uniqueid").empty? ? "N/A" : transaction.search("refnum", "uniqueid").inner_text,
+            :account_id        => account_id,
+            :units             => parse_float(transaction.search("units").inner_text),
+            :unit_price        => parse_float(transaction.search("unitprice").inner_text)
+          )
+        end
+
+        def build_sign_on
+          @sign_on = SaltParser::Ofx::SignOn.new(
+            :language          => html.search("signonmsgsrsv1 > sonrs > language").inner_text,
+            :fi_id             => html.search("signonmsgsrsv1 > sonrs > fi > fid").inner_text,
+            :fi_name           => html.search("signonmsgsrsv1 > sonrs > fi > org").inner_text,
+            :code              => html.search("signonmsgsrsv1 > sonrs > status > code").inner_text,
+            :severity          => html.search("signonmsgsrsv1 > sonrs > status > severity").inner_text,
+            :message           => html.search("signonmsgsrsv1 > sonrs > status > message").inner_text
+          )
+        end
+
+        def build_balance(account)
+          return nil unless account.search("ledgerbal > balamt").size > 0
+
+          if account.search("ledgerbal > balamt").inner_text.match(/[\d]{14}\.[\d]+/)
+            SaltParser::Ofx::Balance.new(
+              :amount => 0.0,
+              :amount_in_pennies => 0,
+              :posted_at => build_date(account.search("ledgerbal > balamt").inner_text)
+            )
+          else
+            amount = parse_float(account.search("ledgerbal > balamt").inner_text)
+
+            SaltParser::Ofx::Balance.new(
+              :amount => amount,
+              :amount_in_pennies => ((amount * 100).round 2).to_i,
+              :posted_at => build_date(account.search("ledgerbal > dtasof").inner_text)
+            )
+          end
+        end
+
+        def build_available_balance(account)
+          return nil unless account.search("availbal").size > 0
+
+          if account.search("availbal > balamt").inner_text.match(/[\d]{14}\.[\d]+/)
+            SaltParser::Ofx::Balance.new(
+              :amount => 0.0,
+              :amount_in_pennies => 0,
+              :posted_at => build_date(account.search("availbal > balamt").inner_text)
+            )
+          else
+            amount = parse_float(account.search("availbal > balamt").inner_text)
+
+            SaltParser::Ofx::Balance.new(
+              :amount => amount,
+              :amount_in_pennies => ((amount * 100).round 2).to_i,
+              :posted_at => build_date(account.search("availbal > dtasof").inner_text)
+            )
+          end
+        end
+
+        def build_investment_balance(account)
+          if account.search("invbal > availcash").size > 0 && account.search("invpos > mktval").size < 1
+            amount = parse_float(account.search("invbal > availcash").inner_text)
+
+          elsif account.search("invbal > availcash").size < 1 && account.search("invpos > mktval").size > 0
+            amount = 0
+            account.search("invpos > mktval").map do |mktval|
+              amount += parse_float(mktval.inner_text)
+            end
+            amount
+
+          elsif account.search("invbal > availcash").size > 0 && account.search("invpos > mktval").size > 0
+            amount = 0
+            account.search("invbal > availcash").map do |availcash|
+              amount += parse_float(availcash.inner_text)
+            end
+            account.search("invpos > mktval").map do |mktval|
+              amount += parse_float(mktval.inner_text)
+            end
+            amount
+
+          else
+            return nil
+          end
+
+          SaltParser::Ofx::Balance.new(
+            :amount => amount,
+            :amount_in_pennies => ((amount * 100).round 2).to_i
+          )
+        end
+
+        def compute_investment_units(account)
+          if account.search("invpos > units").size == 1
+            parse_float(account.search("invpos > units").inner_text)
+          else
+            0.0
+          end
+        end
+
+        def compute_investment_unit_price(account)
+          if account.search("invpos > unitprice").size == 1
+            parse_float(account.search("invpos > unitprice").inner_text)
+          else
+            0.0
+          end
+        end
+
+        def build_type(element)
+          TRANSACTION_TYPES[element.search("trntype", "incometype").inner_text.to_s.upcase]
+        end
+
+        def build_amount(element)
+          parse_float(element.search("trnamt", "total").inner_text)
+        rescue TypeError => error
+          raise SaltParser::Ofx::ParseError.new(SaltParser::Ofx::ParseError::AMOUNT)
+        end
+
+        def build_investment_amount(element)
+          if element.parent.search("invbuy", "invsell").size > 0
+            -1 * parse_float(element.search("total").inner_text)
+          else
+            build_amount(element)
+          end
+        rescue TypeError => error
+          raise SaltParser::Error::ParseError.new(SaltParser::Error::ParseError::AMOUNT)
+        end
+
+        def build_date(date)
+          _, year, month, day, hour, minutes, seconds = *date.match(/(\d{4})(\d{2})(\d{2})(?:(\d{2})(\d{2})(\d{2}))?/)
+
+          date = "#{year}-#{month}-#{day} "
+          date << "#{hour}:#{minutes}:#{seconds}" if hour && minutes && seconds
+
+          Time.parse(date)
+        rescue TypeError, ArgumentError => error
+          raise SaltParser::Error::ParseError.new(SaltParser::Error::ParseError::TIME)
+        end
+
+        def parse_float(incoming, options={})
+          return incoming if incoming.is_a?(Float)
+          string = incoming.dup
+          sanitize_float_string!(string)
+
+          if options[:integral]
+            string.gsub!(",", "")
+            string.gsub!(".", "")
+            return string.to_f
+          end
+
+          indexes = {
+                      "," => string.rindex(","),
+                      "." => string.rindex(".")
+                    }
+
+          return string.to_f if indexes["."].nil? && indexes[","].nil?
+
+          if indexes["."] == nil
+            if string.scan(/,/).size > 1
+              string.gsub!(",", "")  # 123,123,123
+            else
+              string.gsub!(",", ".") # 123,123
+            end
+            return string.to_f
+          end
+
+          if indexes[","] == nil
+            string.gsub!(".", "") if string.scan(/\./).size > 1 # 123.123.123
+            return string.to_f
+          end
+
+          if indexes[","] > indexes["."]
+            # comma is decimal separator
+            string.gsub!(".", "")
+            string.gsub!(",", ".")
+          else
+            # dot is decimal separator
+            string.gsub!(",", "")
+          end
+
+          string.to_f
+        rescue => error
+          raise SaltParser::Error::ParseError.new(SaltParser::Error::ParseError::FLOAT)
+        end
+
+        def sanitize_float_string!(string)
+          # replace weird minus sign with proper minus
+          string.gsub!(8211.chr(Encoding::UTF_8), "-")
+          # replace an even weirder minus sign with proper minus
+          string.gsub!(8722.chr(Encoding::UTF_8), "-")
+          # remove everything except digits, dots, commas, '+', '-'
+          string.gsub!(/[^0-9\-+.,]/, "")
+          # remove trailing non digits
+          string.gsub!(/[-+.,]+$/, "")
+        end
+      end
+    end
+  end
+end