total_recall 0.1.0

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.own

data/Gemfile

@@ -0,0 +1,4 @@
+source 'http://rubygems.org'
+
+# Specify your gem's dependencies in total_recall.gemspec
+gemspec

data/README.md

@@ -0,0 +1,55 @@
+TotalRecall
+============
+
+Turn your bank record csv's into a [Ledger](http://ledger-cli.org/3.0/doc/ledger3.html) journal.
+
+Usage
+------------
+
+    $ total_recall
+    Tasks:
+      total_recall help [TASK]                                   # Describe available tasks or one specific task
+      total_recall ledger -i, --input=INPUT -p, --parser=PARSER  # Convert input to ledger-transactions
+      total_recall parse -i, --input=INPUT -p, --parser=PARSER   # Parses input-file and prints it
+
+    # typically you would do:
+    $ total_recall ledger -i abn.csv -p abn > abn.dat
+
+Develop
+------------
+    
+    $ git clone git://github.com/eval/total_recall.git
+    $ cd total_recall
+    $ bundle
+    $ rake spec
+
+Author
+------
+
+Gert Goet (eval) :: gert@thinkcreate.nl :: @gertgoet
+
+License
+------
+
+(The MIT license)
+
+Copyright (c) 2011 Gert Goet, ThinkCreate
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,15 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new do |t|
+  t.rspec_opts = ['--options', "\"#{File.dirname(__FILE__)}/spec/spec.opts\""]
+end
+
+desc "Run the specs"
+task :default => :spec
+
+desc 'Removes trailing whitespace'
+task :whitespace do
+  sh %{find . -name '*.rb' -exec sed -i '' 's/ *$//g' {} \\;}
+end

data/bin/total_recall

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require 'total_recall'
+
+TotalRecall::Cli.start

data/bin/total_recall.rb

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require 'total_recall'
+
+TotalRecall::Cli.start

data/lib/total_recall.rb

@@ -0,0 +1,295 @@
+require "total_recall/version"
+require "thor"
+require "mustache"
+
+module TotalRecall
+  module ParseStrategy
+    class Ing
+      require 'time'
+
+      # Expected: Hash with:
+      #:amount => Float,
+      #:currency => String,
+      #:description => String,
+      #:date => Date
+      def parse_row(row)
+        amount = row[6].sub(/,/,'.').to_f
+        {
+          :amount => (row[5] == 'Bij' ? amount : -amount),
+          :date => Date.parse(row[0]),
+          :description => [row[1], row[8]].map{|i| i.strip.gsub(/\s+/, ' ')}.join(' '),
+          :currency => 'EUR'
+        }
+      end
+
+
+      def self.options
+        {:col_sep => ",", :headers => true}
+      end
+
+      def options
+        self.class.options
+      end
+    end # /Ing
+
+    class Abn
+      require 'time'
+
+      # Expected: Hash with:
+      #:amount => Float,
+      #:currency => String,
+      #:description => String,
+      #:date => Date
+      def parse_row(row)
+        {
+          :amount => row[6].sub(/,/,'.').to_f,
+          :date => Date.parse(row[2]),
+          :description => row[7].strip.gsub(/\s+/,' '),
+          :currency => row[1]
+        }
+      end
+
+      def self.options
+        {:col_sep => "\t"}
+      end
+
+      def options
+        self.class.options
+      end
+    end # /Abn
+  end # /ParseStrategy
+
+  class Ledger < Mustache
+    attr_reader :default_account, :options
+
+    def initialize(default_account, transactions, options={})
+      @options = {:max_account_width => 80, :decimal_mark => ','}.merge(options)
+      @default_account = default_account
+      @_transactions = transactions
+    end
+
+    def transactions
+      @transactions ||= begin
+        @_transactions.map do |i|
+          res = {}
+          res[:to], res[:from] = (i[:amount] > 0 ? [default_account, i[:account]] : [i[:account], default_account])
+          if i[:tags].any?
+            tagsline = "\n  ; :%s:" % (i[:tags] * ':')
+            tagskey = i[:amount] > 0 ? :from_tags : :to_tags
+            res[tagskey] = tagsline
+          end
+          res[:amount] = fmt_amount(i[:amount].abs)
+          res[:description], res[:date], res[:currency] = i[:description], i[:date], i[:currency]
+          res[:spacer] = " " * (options[:max_account_width] - res[:to].size)
+          res
+        end
+      end
+    end
+
+
+    def self.template;<<-TEMPLATE
+{{# transactions}}
+{{date}} {{description}}
+  {{to}}  {{spacer}}{{currency}} {{amount}}{{to_tags}}
+  {{from}}{{from_tags}}
+
+{{/ transactions}}
+TEMPLATE
+    end
+
+    protected
+      def fmt_amount(amount)
+        ("%10.2f" % amount).split('.') * options[:decimal_mark]
+      end
+  end
+
+  class AccountGuesser
+    attr_reader :accounts, :tokens
+
+    def initialize
+      @accounts = {}
+      @tokens = {}
+    end
+
+    # copied from reckon(https://github.com/iterationlabs/reckon)
+    def guess(data)
+      query_tokens = tokenize(data)
+
+      search_vector = []
+      account_vectors = {}
+
+      query_tokens.each do |token|
+        idf = Math.log((accounts.keys.length + 1) / ((tokens[token] || {}).keys.length.to_f + 1))
+        tf = 1.0 / query_tokens.length.to_f
+        search_vector << tf*idf
+
+        accounts.each do |account, total_terms|
+          tf = (tokens[token] && tokens[token][account]) ? tokens[token][account] / total_terms.to_f : 0
+          account_vectors[account] ||= []
+          account_vectors[account] << tf*idf
+        end
+      end
+
+      # Should I normalize the vectors?  Probably unnecessary due to tf-idf and short documents.
+      account_vectors = account_vectors.to_a.map do |account, account_vector|
+        { :cosine => (0...account_vector.length).to_a.inject(0) { |m, i| m + search_vector[i] * account_vector[i] },
+          :account => account }
+      end
+
+      account_vectors.sort! {|a, b| b[:cosine] <=> a[:cosine] }
+      account_vectors.first && account_vectors.first[:account]
+    end
+
+    # copied from reckon(https://github.com/iterationlabs/reckon)
+    def learn(account, data)
+      accounts[account] ||= 0
+      tokenize(data).each do |token|
+        tokens[token] ||= {}
+        tokens[token][account] ||= 0
+        tokens[token][account] += 1
+        accounts[account] += 1
+      end
+    end
+
+    protected
+      def tokenize(str)
+        str.downcase.split(/[\s\-]/)
+      end
+  end
+
+  class BankParser
+    require 'csv'
+    attr_reader :strategy
+
+    def initialize(options={})
+      @strategy = options[:strategy]
+    end
+
+    # Parses csv content and returns array of hashes.
+    #
+    # @example
+    #   parser = TotalRecall::BankParser.new(:strategy => TotalRecall::ParseStrategy::Some.new)
+    #   parser.parse("12.1\n1.99") #=> [{:amount => 12.1}, {:amount => 1.99}]
+    #
+    # @param [String] str content of csv to parse.
+    # @return [Array<Hash>]
+    def parse(str, options={})
+      options = strategy.options.merge(options)
+
+      result = []
+      CSV.parse(str, options){|row| result << strategy.parse_row(row)}
+      result
+    end
+  end #/BankParser
+
+
+  class Cli < Thor
+    require "terminal-table"
+    require "highline/import"
+    require "bayes_motel"
+
+    no_tasks do
+      def self.strategies
+        {
+          'abn' => TotalRecall::ParseStrategy::Abn,
+          'ing' => TotalRecall::ParseStrategy::Ing
+        }
+      end
+
+      def highline
+        @highline ||= HighLine.new($stdin, $stderr)
+      end
+
+      def strategies
+        self.class.strategies
+      end
+
+      def parser(strategy)
+        TotalRecall::BankParser.new(:strategy => strategy)
+      end
+    end
+
+    desc "ledger", "Convert input to ledger-transactions"
+    method_option :input, :aliases => "-i", :desc => "CSV file to use for input", :required => true
+    method_option :parser, :aliases => "-p", :desc => "Parser to use (one of #{strategies.keys.inspect})", :required => true
+    def ledger
+      strategy = strategies[options[:parser]]
+      file_contents = File.read(options[:input])
+      rows = parser(strategy.new).parse(file_contents)
+
+      guesser = AccountGuesser.new
+
+      default_account = highline.ask("What is the account name of this bank account in Ledger?\n> ")
+      transactions = [] # [{<row>, :account => 'Expenses:Car'}, ...]
+      tags = []
+      start_from = 0
+      begin
+        rows.each_with_index do |row, ix|
+          if start_from && start_from != ix
+            next
+          elsif start_from
+            start_from = nil
+          end
+
+          guessed = guesser.guess(row[:description])
+          question = row[:amount] > 0 ? "What account provided this income?" : "To which account did this money go?"
+
+          $stderr.puts Terminal::Table.new(:rows => [ [ row[:date], row[:amount], row[:description] ] ])
+          account = highline.ask("#{question} (#{guessed}) or [d]one, [s]kip, [p]revious\n> ")
+          account = account.empty? ? guessed : account # defaults not working
+          case account
+          when 'd'
+            break
+          when 's'
+            next
+          when 'p'
+            transactions.pop
+            start_from = [0, (ix - 1)].max
+            raise
+          end
+
+          transaction_tags = []
+          loop do
+            tag = highline.choose do |menu|
+              menu.shell = true
+              menu.prompt = "Add tags? "
+              menu.prompt << "(#{transaction_tags.join(',')})" if transaction_tags.any?
+
+              menu.choices('Done'){ nil }
+              menu.choices(*tags - transaction_tags)
+              menu.choice('New...'){ newtag = highline.ask('Tag: ');tags << newtag; newtag }
+            end
+            break unless tag
+            transaction_tags << tag
+          end
+
+          guesser.learn(account, row[:description])
+          transactions << row.merge(:account => account, :tags => transaction_tags)
+        end
+      rescue
+        $stderr.puts "Let's do that again"
+        retry
+      end
+
+      puts Ledger.new(default_account, transactions).render
+    end
+
+    desc "parse", "Parses input-file and prints it"
+    method_option :input, :aliases => "-i", :desc => "CSV file to use for input", :required => true
+    method_option :parser, :aliases => "-p", :desc => "Parser to use (one of #{strategies.keys.join(',')})", :required => true
+    def parse
+      strategy = strategies[options[:parser]]
+      file_contents = File.read(options[:input])
+      data = parser(strategy.new).parse(file_contents)
+
+      table = Terminal::Table.new do |t|
+        t.headings = 'Date', 'Amount', 'Description'
+        data.each do |row|
+          t << [ row[:date], row[:amount], row[:description] ]
+        end
+      end
+      puts table
+    end
+
+  end
+end

data/lib/total_recall/version.rb

@@ -0,0 +1,3 @@
+module TotalRecall
+  VERSION = "0.1.0"
+end

data/spec/fixtures/abn.csv

@@ -0,0 +1,2 @@
+123456789	EUR	20101230	  100,00	   70,00	20101230	-30,00	BEA   NR:12AA34   30.12.10/22.24 iCentre CS AMSTERDAM,PAS123 
+123456789	EUR	20101231	   70,00	   50,00	20101231	-20,00	BEA   NR:12AA34   31.12.10/22.24 iCentre CS AMSTERDAM,PAS123 

data/spec/fixtures/ing.csv

@@ -0,0 +1,3 @@
+"Datum","Naam / Omschrijving","Rekening","Tegenrekening","Code","Af Bij","Bedrag (EUR)","MutatieSoort","Mededelingen"
+"20101231","Omschrijving                    ","123456789","012345678","GT","Af","100,00","Internetbankieren","mededeling eèéêë"
+"20101230","Omschrijving                    ","123456789","012345678","GT","Af","200,00","Internetbankieren","mededeling        "

data/spec/spec.opts

@@ -0,0 +1 @@
+--color

data/spec/spec_helper.rb

@@ -0,0 +1,15 @@
+require "rubygems"
+require "bundler"
+Bundler.setup
+
+$:.unshift File.expand_path("../../lib", __FILE__)
+require "total_recall"
+
+# fullpath to csv-file
+def fixture_pathname(str)
+  Pathname(%W(spec fixtures #{str}.csv).join(File::SEPARATOR)).realpath
+end
+
+def fixture_contents(str)
+  File.read(fixture_pathname(str))
+end

data/spec/total_recall/bank_parser_spec.rb

@@ -0,0 +1,52 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+describe TotalRecall::BankParser do
+  context '#parse' do
+    subject{ described_class.new(:strategy => TotalRecall::ParseStrategy::Abn.new) }
+
+    it "should return an Array" do
+      subject.parse(fixture_contents('abn')).class.should == Array
+    end
+  end
+end
+
+
+shared_examples "a parser" do
+  context '#parse_row' do
+    it "should return a hash" do
+      CSV.parse(fixture_contents(@fixture), subject.options).each do |i|
+        parsed_row = subject.parse_row(i)
+
+        parsed_row.class.should == Hash
+      end
+    end
+
+    it "should return the correct keys and classes" do
+      expected_keys_and_classes = {
+        :amount => Float,
+        :currency => String,
+        :description => String,
+        :date => Date
+      }
+
+      CSV.parse(fixture_contents(@fixture), subject.options).each do |i|
+        parsed_row = subject.parse_row(i)
+
+        parsed_row.keys.should =~ expected_keys_and_classes.keys
+        expected_keys_and_classes.each do |key, klass|
+          parsed_row.send(:[], key).class.should == klass
+        end
+      end
+    end
+  end
+end
+
+describe TotalRecall::ParseStrategy::Abn do
+  before{ @fixture = 'abn' }
+  it_behaves_like "a parser"
+end
+
+describe TotalRecall::ParseStrategy::Ing do
+  before{ @fixture = 'ing' }
+  it_behaves_like "a parser"
+end

data/total_recall.gemspec

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/total_recall/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Gert Goet"]
+  gem.email         = ["gert@thinkcreate.nl"]
+  gem.description   = %q{Turn your bank records csv's into Ledger journals}
+  gem.summary       = %q{Turn your bank records csv's into Ledger journals}
+  gem.homepage      = ""
+
+  gem.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.files         = `git ls-files`.split("\n")
+  gem.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.name          = "total_recall"
+  gem.require_paths = ["lib"]
+  gem.version       = TotalRecall::VERSION
+
+  gem.add_dependency('thor', '~> 0.14.6')
+  gem.add_dependency('terminal-table', '~> 1.4.4')
+  gem.add_dependency('highline', '~> 1.6.1')
+  gem.add_dependency('bayes_motel', '~> 0.1.0')
+  gem.add_dependency('mustache', '~> 0.99.4')
+  gem.add_development_dependency('rspec', '~> 2.7.0')
+end

metadata

@@ -0,0 +1,135 @@
+--- !ruby/object:Gem::Specification
+name: total_recall
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Gert Goet
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-01-09 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: &70281655447440 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.14.6
+  type: :runtime
+  prerelease: false
+  version_requirements: *70281655447440
+- !ruby/object:Gem::Dependency
+  name: terminal-table
+  requirement: &70281655446380 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.4.4
+  type: :runtime
+  prerelease: false
+  version_requirements: *70281655446380
+- !ruby/object:Gem::Dependency
+  name: highline
+  requirement: &70281655445700 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.6.1
+  type: :runtime
+  prerelease: false
+  version_requirements: *70281655445700
+- !ruby/object:Gem::Dependency
+  name: bayes_motel
+  requirement: &70281655444800 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: *70281655444800
+- !ruby/object:Gem::Dependency
+  name: mustache
+  requirement: &70281655444120 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.99.4
+  type: :runtime
+  prerelease: false
+  version_requirements: *70281655444120
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &70281655443400 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.7.0
+  type: :development
+  prerelease: false
+  version_requirements: *70281655443400
+description: Turn your bank records csv's into Ledger journals
+email:
+- gert@thinkcreate.nl
+executables:
+- total_recall
+- total_recall.rb
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- README.md
+- Rakefile
+- bin/total_recall
+- bin/total_recall.rb
+- lib/total_recall.rb
+- lib/total_recall/version.rb
+- spec/fixtures/.keep
+- spec/fixtures/abn.csv
+- spec/fixtures/ing.csv
+- spec/spec.opts
+- spec/spec_helper.rb
+- spec/total_recall/bank_parser_spec.rb
+- total_recall.gemspec
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: Turn your bank records csv's into Ledger journals
+test_files:
+- spec/fixtures/.keep
+- spec/fixtures/abn.csv
+- spec/fixtures/ing.csv
+- spec/spec.opts
+- spec/spec_helper.rb
+- spec/total_recall/bank_parser_spec.rb
+has_rdoc: