bank_scrap 0.0.6 → 0.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8aa6ea0f009804787942dd42adff98aef36dd601
-  data.tar.gz: a137f771dfd1eee094edbd9f71d74cbc02cce20f
+  metadata.gz: 4680376ad7f26ae34f9cf4262fa04b98b5b466ea
+  data.tar.gz: 4be7fda00b5c1a410ffb602979d20894d3511f6b
 SHA512:
-  metadata.gz: e0b08d23ebcd36680a143636d1f7eb5452b28049bee9f17b2e5b42548fa016f1627f3358216f5dcfe51c1ea7afdfa0b53129c83e76d5be4a035f8c0c0604819a
-  data.tar.gz: 558ce9735baad049a50360b48b66b13755a28365347cde5577c0625492f283522b59193a45d390af25cb6b4bd55c90f7d80b3c673ae9293a7dbaebfc52284627
+  metadata.gz: 9debe3eadab0fa8a98e3d52640cf2d2a49348f92f6a5b0d2bb5b2a9555ca19d4e61559ea3b990f28bbdcef9ceda1bf36e259d540ffe4c1e23ef25ec7ce53c250
+  data.tar.gz: d314b783e5a962669d2717f1fa2d15568284e7c4e03195e8d32073296138f7325758d2d218aa492cbfff6fe1b0b47105b951a94d05f995ba2f7fc34bcb128e70

data/.gitignore

@@ -13,5 +13,7 @@
 *.a
 mkmf.log
 
+*.gem
+
 # MAC OS X files
 .DS_Store

data/README.md

@@ -1,4 +1,4 @@
-# Bankscrap
+# BankScrap
 
 Ruby gem to extract balance and transactions from banks. You can use it either as command line tool or as a library.
 
@@ -23,7 +23,7 @@ There are two approaches to solve this problem:
 - Web scraping on the bank's site.
 - Reverse engineering the bank's mobile app to use the same API the app uses.
 
-Bankscrap uses both methods depending on the bank.
+BankScrap uses both methods depending on the bank.
 
 ## Requirements
 
@@ -52,10 +52,22 @@ Or, if you're using Bundler, just add the following to your Gemfile:
 ### From terminal
 Retrieve balance account
 
-    $ bank_scrap balance BANK_NAME --user YOUR_BANK_USER --password YOUR_BANK_PASSWORD
+##### Bankinter
 
-BANK_NAME should be in underscore case (`bankinter`, `bbva`).
+    $ bank_scrap balance bankinter --user YOUR_BANKINTER_USER --password YOUR_BANKINTER_PASSWORD
 
+##### BBVA
+
+    $ bank_scrap balance bbva --user YOUR_BBVA_USER --password YOUR_BBVA_PASSWORD
+
+##### ING Direct
+ING needs one more argument: your bithday.
+
+    $ bank_scrap balance ing --user YOUR_DNI --password YOUR_PASSWORD --extra=birthday:01/01/1980
+
+Replace 01/01/1980 with your actual birthday.
+
+---
 If you don't want to pass your user and password everytime you can define them in your .bash_profile by adding:
 
     export BANK_SCRAP_USER=YOUR_BANK_USER
@@ -63,7 +75,7 @@ If you don't want to pass your user and password everytime you can define them i
 
 ### From Ruby code
 
-You can also use this gem from your own app as library. To do so first you must initialize a Bankscrapper::Bank object
+You can also use this gem from your own app as library. To do so first you must initialize a BankScrap::Bank object
 
 ```ruby
 require 'bank_scrap'

data/bank_scrap.gemspec

@@ -22,12 +22,12 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency 'bundler', '~> 1.7'
   spec.add_development_dependency 'rake', '~> 10.0'
-  spec.add_development_dependency 'byebug', '~> 3.5.1'
+  spec.add_development_dependency 'byebug', '~> 3.5', '>= 3.5.1'
 
   spec.add_dependency 'thor', "~> 0.19"
   spec.add_dependency 'nokogiri', "~> 1.6"
   spec.add_dependency 'execjs', "~> 2.2"
-  spec.add_dependency 'curb', "~> 0.8"
+  spec.add_dependency 'mechanize', "~> 2.7.3"
   spec.add_dependency 'activesupport', "~> 4.1"
-  spec.add_dependency 'rmagick', "~> 2.2.2"
+  spec.add_dependency 'rmagick', '~> 2.2', '>= 2.2.2'
 end

data/lib/bank_scrap/bank.rb

@@ -1,66 +1,58 @@
-require 'curb'
+require 'mechanize'
+require 'logger'
 
 module BankScrap
   class Bank
 
     WEB_USER_AGENT = 'Mozilla/5.0 (Linux; Android 4.2.1; en-us; Nexus 4 Build/JOP40D) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.166 Mobile Safari/535.19'
+    attr_accessor :headers
 
     private
 
     def get(url)
-      @curl.url = url
-      @curl.get
-
-      @curl.body_str
+      @http.get(url).body
     end
 
     def post(url, fields)
-      @curl.url = url
-
-      # If hash, transform to Curl::PostField objects
-      if fields.is_a? Hash
-        fields = fields.collect {|key, value| Curl::PostField.content(key, value)}
-      end
-
-      @curl.post(fields)
-      @curl.body_str
+      @http.post(url, fields, @headers).body
     end
 
     def put(url, fields)
-      @curl.url = url
-
-      @curl.put(fields)
-      @curl.body_str
+      @http.put(url, fields, @headers).body
     end
 
-    def set_header(name, value)
-      @curl.headers[name] = value
+    # Sets temporary HTTP headers, execute a code block
+    # and resets the headers
+    def with_headers(tmp_headers)
+      current_headers = @headers
+      set_headers(tmp_headers)
+      yield
+    ensure
+      set_headers(current_headers)
     end
 
+
     def set_headers(headers)
-      headers.each { |key, value| set_header(key, value) }
+      @headers.merge! headers
+      @http.request_headers = @headers
     end
 
-    def get_headers
-      @curl.header_str
-    end
 
     def initialize_cookie(url)
       log 'Initialize cookie'
 
-      @curl.url = url
-      @curl.get
-
-      @curl.body_str
+      @http.url = url
+      @http.get(url).body
     end
 
     def initialize_connection
-      @curl = Curl::Easy.new
-      @curl.follow_location = true
-      @curl.ssl_verify_peer = false
-      @curl.verbose = true if @debug
-      @curl.enable_cookies = true
-      @curl.headers["User-Agent"] = WEB_USER_AGENT
+      @http = Mechanize.new do |mechanize|
+        mechanize.user_agent = WEB_USER_AGENT
+        mechanize.agent.http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+        mechanize.log = Logger.new(STDOUT) if @debug
+      end
+
+      @headers = {}
     end
 
     def log(msg)

data/lib/bank_scrap/banks/bbva.rb

@@ -5,7 +5,8 @@ module BankScrap
     BASE_ENDPOINT    = 'https://bancamovil.grupobbva.com'
     LOGIN_ENDPOINT   = '/DFAUTH/slod/DFServletXML'
     BALANCE_ENDPOINT = '/ENPP/enpp_mult_web_mobility_02/products/v1'
-    USER_AGENT       = 'Android;LGE;Nexus 5;1080x1776;Android;4.4.4;BMES;4.0.4'
+    # BBVA expects an identifier before the actual User Agent, but 12345 works fine
+    USER_AGENT       = '12345;Android;LGE;Nexus 5;1080x1776;Android;4.4.4;BMES;4.0.4'
 
     def initialize(user, password, log: false, debug: false, extra_args: nil)
       @user = format_user(user.dup)
@@ -24,6 +25,7 @@ module BankScrap
         'Accept-Charset'   => 'UTF-8',
         'Connection'       => 'Keep-Alive',
         'Host'             => 'bancamovil.grupobbva.com',
+        'Cookie2'          => '$Version=1'
       })
 
       login
@@ -31,7 +33,13 @@ module BankScrap
 
     def get_balance
       log 'get_balance'
-      response = post(BASE_ENDPOINT + BALANCE_ENDPOINT, nil)
+      
+      # Even if the required method is an HTTP POST
+      # the API requires a funny header that says is a GET
+      # otherwise the request doesn't work.
+      response = with_headers({'BBVA-Method' => 'GET'}) do
+        post(BASE_ENDPOINT + BALANCE_ENDPOINT, {})
+      end
 
       json = JSON.parse(response)
       json["balances"]["personalAccounts"]

data/lib/bank_scrap/banks/ing.rb

@@ -37,6 +37,10 @@ module BankScrap
       balance
     end
 
+    def raw_product_data
+      @data
+    end
+
     private
 
     def bundled_login
@@ -120,7 +124,7 @@ module BankScrap
         0.upto(9) do |j|
           pinpad_pixels_sample = single_number.get_pixels(0,0, SAMPLE_WIDTH, SAMPLE_HEIGHT)
 
-          img = Magick::ImageList.new("lib/bank_scrap/banks/ing/numbers/pinpad#{j}.png").first
+          img = Magick::ImageList.new(File.join(File.dirname(__FILE__), "/ing/numbers/pinpad#{j}.png")).first
           number_pixels_sample = img.get_pixels(0, 0, SAMPLE_WIDTH, SAMPLE_HEIGHT)
           diff = 0
           pinpad_pixels_sample.each_with_index do |pixel, index|

data/lib/bank_scrap/banks/ing_backup.rb

@@ -0,0 +1,237 @@
+require 'execjs'
+require 'pp'
+require 'json'
+require 'base64'
+require 'RMagick'
+require 'active_support'
+require 'byebug'
+require 'open-uri'
+
+module BankScrap
+  class Ing < Bank
+
+    DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.111 Safari/537.36'
+
+    BASE_ENDPOINT           = 'https://ing.ingdirect.es/'
+    FALSE_LOGIN_ENDPOINT    = BASE_ENDPOINT + 'login/'
+    CACHE_ENDPOINT          = BASE_ENDPOINT + 'login/cache.manifest'
+    DELETE_SESSION_ENDPOINT = BASE_ENDPOINT + 'genoma_api/rest/session'
+    LOGIN_ENDPOINT          = BASE_ENDPOINT + 'genoma_login/rest/session'
+    POST_AUTH_ENDPOINT      = BASE_ENDPOINT + 'genoma_api/login/auth/response'
+    CLIENT_ENDPOINT         = BASE_ENDPOINT + 'genoma_api/rest/client'
+    PRODUCTS_ENDPOINT       = BASE_ENDPOINT + 'genoma_api/rest/products'
+
+    SAMPLE_WIDTH  = 30
+    SAMPLE_HEIGHT = 30
+
+    def initialize(dni, birthday, password, log: false, debug: false)
+      @dni      = dni
+      @birthday = birthday
+      @password = password.to_s
+      @log      = log
+      @debug    = debug
+
+      initialize_connection
+
+      @curl.proxy_port = 8888
+      @curl.proxy_url = '192.168.1.21'
+
+      false_login
+      cache
+      delete_session
+      selected_positions = login
+
+      ticket = pass_pinpad(selected_positions)
+
+      post_auth(ticket)
+      call_client
+
+      get_products
+    end
+
+    private
+
+    def false_login
+      @curl.url = FALSE_LOGIN_ENDPOINT
+      @curl.headers['Host'] = 'ing.ingdirect.es'
+      @curl.headers['Connection'] = 'keep-alive'
+      @curl.headers['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
+      @curl.headers['Accept-Encoding'] = 'gzip,deflate,sdch'
+      @curl.headers['Accept-Language'] = 'en,es;q=0.8'
+
+      response = @curl.get
+    end
+
+    def cache
+      @curl.url = CACHE_ENDPOINT
+      @curl.headers['Host'] = 'ing.ingdirect.es'
+      @curl.headers['Connection'] = 'keep-alive'
+      @curl.headers['Accept-Encoding'] = 'gzip,deflate,sdch'
+      @curl.headers['Accept-Language'] = 'en,es;q=0.8'
+    end
+
+    def delete_session
+      @curl.headers['Host'] = 'ing.ingdirect.es'
+      @curl.headers['Connection'] = 'keep-alive'
+      @curl.headers['Pragma'] = 'no-cache'
+      @curl.headers['Accept'] = 'application/json, text/javascript, */*; q=0.01'
+      @curl.headers['Origin'] = 'https://ing.ingdirect.es'
+      @curl.headers['X-Requested-With'] = 'XMLHttpRequest'
+      @curl.headers['Content-Type'] = 'application/json; charset=utf-8'
+      @curl.headers['Referer'] = 'https://ing.ingdirect.es/login/'
+      @curl.headers['Accept-Encoding'] = 'zip,deflate,sdch'
+      @curl.headers['Accept-Language'] = 'n,es;q=0.8'
+      @curl.headers['Cookie'] = 's_cc=true; s_mca=Direct; s_gts=1; s_nr=1414955726141; s_sq=%5B%5BB%5D%5D'
+
+      response = @curl.delete
+      pp response
+    end
+
+    def login
+      param = '{"loginDocument":{"documentType":0,"document":"' + @dni.to_s +
+              '"},"birthday":"' + @birthday.to_s + '","companyDocument":null,"device":"desktop"}'
+      puts param
+      @curl.url = LOGIN_ENDPOINT
+      @curl.headers['Host'] = 'ing.ingdirect.es'
+      @curl.headers['Connection'] = 'keep-alive'
+      @curl.headers['Pragma'] = 'no-cache'
+
+      @curl.headers['Accept'] = 'application/json, text/javascript, */*; q=0.01'
+      @curl.headers['Origin'] = 'https://ing.ingdirect.es'
+      @curl.headers['X-Requested-With'] = 'XMLHttpRequest'
+      @curl.headers['Content-Type'] = 'application/json; charset=utf-8'
+      @curl.headers['Referer'] = 'https://ing.ingdirect.es/login/'
+      @curl.headers['Accept-Encoding'] = 'zip,deflate,sdch'
+      @curl.headers['Accept-Language'] = 'n,es;q=0.8'
+      @curl.headers['Cookie'] = 's_cc=true; s_mca=Direct; s_gts=1; s_nr=1414955726141; s_sq=%5B%5BB%5D%5D'
+
+      response = post(LOGIN_ENDPOINT, param)
+      # response = @curl.body_str
+      response = JSON.parse(response)
+      positions = response['pinPositions']
+      pinpad = response['pinpad']
+
+      save_pinpad_numbers(pinpad)
+      pinpad_numbers = recognize_pinpad_numbers
+
+      get_correct_positions(pinpad_numbers, positions)
+    end
+
+    def pass_pinpad(positions)
+      param = "{\"pinPositions\": #{positions}}"
+      @curl.url = LOGIN_ENDPOINT
+      @curl.headers['Host'] = 'ing.ingdirect.es'
+      @curl.headers['Connection'] = 'keep-alive'
+      @curl.headers['Accept'] = 'application/json, text/javascript, */*; q=0.01'
+      @curl.headers['Origin'] = 'https://ing.ingdirect.es'
+      @curl.headers['X-Requested-With'] = 'XMLHttpRequest'
+      @curl.headers['Content-Type'] = 'application/json; charset=utf-8'
+      @curl.headers['Referer'] = 'https://ing.ingdirect.es/login/?clientId=281afde24c938607e5edeac6239e8a38&continue=%2Fpfm%2F'
+      @curl.headers['Accept-Encoding'] = 'gzip,deflate,sdch'
+
+      response = put(LOGIN_ENDPOINT, param)
+      response = ActiveSupport::Gzip.decompress(response)
+      response = JSON.parse(response)
+
+      response['ticket']
+    end
+
+    def post_auth(ticket)
+        @curl.headers['Host'] = 'ing.ingdirect.es'
+        @curl.headers['Connection'] = 'keep-alive'
+        @curl.headers['Accept'] = 'application/json, text/javascript, */*; q=0.01'
+        @curl.headers['Origin'] = 'https://ing.ingdirect.es'
+        @curl.headers['X-Requested-With'] = 'XMLHttpRequest'
+        @curl.headers['Content-Type'] = 'application/x-www-form-urlencoded; charset=UTF-8'
+        @curl.headers['Referer'] = 'https://ing.ingdirect.es/login'
+        @curl.headers['Accept-Encoding'] = 'gzip,deflate'
+
+        @curl.url = POST_AUTH_ENDPOINT
+        param = "ticket=#{ticket}&device=desktop"
+        @curl.post(param)
+        response = @curl.body_str
+    end
+
+    def call_client
+      @curl.headers['Host'] = 'ing.ingdirect.es'
+      @curl.headers['Connection'] = 'keep-alive'
+      @curl.headers['Accept'] = 'application/json, text/javascript, */*; q=0.01'
+      @curl.headers['X-Requested-With'] = 'XMLHttpRequest'
+      @curl.headers['Content-Type'] = 'application/json; charset=utf-8'
+      @curl.headers['Referer'] = 'https://ing.ingdirect.es/pfm'
+      @curl.headers['Accept-Encoding'] = 'gzip,deflate,sdch'
+
+      response = get(CLIENT_ENDPOINT)
+
+      response = ActiveSupport::Gzip.decompress(response)
+      response = JSON.parse(response)
+    end
+
+    def get_products
+      @curl.headers['Host'] = 'ing.ingdirect.es'
+      @curl.headers['Connection'] = 'keep-alive'
+      @curl.headers['Accept'] = '*/*'
+      @curl.headers['X-Requested-With'] = 'XMLHttpRequest'
+      @curl.headers['Content-Type'] = 'application/json; charset=utf-8'
+      @curl.headers['Referer'] = 'https://ing.ingdirect.es/pfm'
+      @curl.headers['Accept-Encoding'] = 'gzip,deflate,sdch'
+
+      response = get(PRODUCTS_ENDPOINT)
+
+      File.open('response_raw.txt', 'w') { |file| file.write(response) }
+      response = ActiveSupport::Gzip.decompress(response)
+      File.open('response_decompressed.txt', 'w') { |file| file.write(response) }
+      File.open('response_parsed.txt', 'w') { |file| file.write(JSON.parse(response)) }
+    end
+
+    def save_pinpad_numbers(pinpad)
+      pinpad.each_with_index do |p,index|
+        File.open(build_tmp_path(index), 'wb'){ |f| f.write(Base64.decode64(p)) }
+      end
+    end
+
+    def build_tmp_path(number)
+      "tmp/original_pinpad_#{number}.png"
+    end
+
+    def recognize_pinpad_numbers
+      pinpad_numbers = []
+      0.upto(9) do |i|
+        pinpad = Magick::ImageList.new(build_tmp_path(i)).first
+
+        differences = []
+        0.upto(9) do |j|
+          pinpad_pixels_sample = pinpad.get_pixels(0,0, SAMPLE_WIDTH, SAMPLE_HEIGHT)
+
+          img = Magick::ImageList.new("numbers/pinpad#{j}.png").first
+          number_pixels_sample = img.get_pixels(0, 0, SAMPLE_WIDTH, SAMPLE_HEIGHT)
+          diff = 0
+          pinpad_pixels_sample.each_with_index do |pixel, index|
+            sample_pixel = number_pixels_sample[index]
+            diff += (pixel.red - sample_pixel.red).abs +
+                    (pixel.green - sample_pixel.green).abs +
+                    (pixel.blue - sample_pixel.blue).abs
+          end
+          differences << diff
+        end
+
+        real_number = differences.each_with_index.min.last
+        pinpad_numbers << real_number
+      end
+
+      pinpad_numbers
+    end
+
+    def get_correct_positions(pinpad_numbers, positions)
+      first_digit  = @password[positions[0] - 1]
+      second_digit = @password[positions[1] - 1]
+      third_digit  = @password[positions[2] - 1]
+
+      [
+        pinpad_numbers.index(first_digit.to_i),
+        pinpad_numbers.index(second_digit.to_i),
+        pinpad_numbers.index(third_digit.to_i)
+      ]
+    end
+  end
+end

data/lib/bank_scrap/version.rb

@@ -1,3 +1,3 @@
 module BankScrap
-  VERSION = "0.0.6"
+  VERSION = "0.0.7"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bank_scrap
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.0.7
 platform: ruby
 authors:
 - Ismael Sánchez
@@ -11,7 +11,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-11-10 00:00:00.000000000 Z
+date: 2015-01-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -46,6 +46,9 @@ dependencies:
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+    - - ">="
       - !ruby/object:Gem::Version
         version: 3.5.1
   type: :development
@@ -53,6 +56,9 @@ dependencies:
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+    - - ">="
       - !ruby/object:Gem::Version
         version: 3.5.1
 - !ruby/object:Gem::Dependency
@@ -98,19 +104,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.2'
 - !ruby/object:Gem::Dependency
-  name: curb
+  name: mechanize
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: 2.7.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: 2.7.3
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
@@ -130,6 +136,9 @@ dependencies:
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+    - - ">="
       - !ruby/object:Gem::Version
         version: 2.2.2
   type: :runtime
@@ -137,6 +146,9 @@ dependencies:
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+    - - ">="
       - !ruby/object:Gem::Version
         version: 2.2.2
 description: Command line tools to get bank account details from some banks.
@@ -169,6 +181,7 @@ files:
 - lib/bank_scrap/banks/ing/numbers/pinpad7.png
 - lib/bank_scrap/banks/ing/numbers/pinpad8.png
 - lib/bank_scrap/banks/ing/numbers/pinpad9.png
+- lib/bank_scrap/banks/ing_backup.rb
 - lib/bank_scrap/cli.rb
 - lib/bank_scrap/version.rb
 homepage: https://github.com/ismaGNU/bank_scrap
@@ -196,4 +209,3 @@ signing_key:
 specification_version: 4
 summary: Get your bank account details.
 test_files: []
-has_rdoc: