itunes-connect 0.9.0 → 0.10.0

data/bin/itunes_connect

@@ -1,6 +1,7 @@
 #!/usr/bin/env ruby
 
-require "itunes_connect"
+$:.unshift File.join(File.dirname(__FILE__), '..', 'lib')
+require 'itunes_connect'
 require "clip"
 
 ItunesConnect::Commands.usage("No command given") if ARGV.empty?

data/lib/itunes_connect/connection.rb

@@ -3,20 +3,50 @@ require "tempfile"
 require "yaml"
 require "zlib"
 require "rubygems"
-require "httpclient"
-require "nokogiri"
+gem 'mechanize'
+require "mechanize"
+
+# mechanize monkey patch
+# handle Content-Encoding of 'agzip'
+begin
+  require 'mechanize/chain/body_decoding_handler'
+
+  class Mechanize
+    class Chain
+      class BodyDecodingHandler
+        alias :orig_handle :handle
+
+        def handle(ctx, options = {})
+          response = options[:response]
+          encoding = response['Content-Encoding'] || ''
+          response['Content-Encoding'] = 'gzip' if encoding.downcase == 'agzip'
+          orig_handle(ctx, options)
+        end
+      end
+    end
+  end
+end
 
 module ItunesConnect
 
+  NETWORK_TIMEOUT = 60  # seconds
+
   # Abstracts the iTunes Connect website.
   # Implementation inspired by
   # http://code.google.com/p/itunes-connect-scraper/
   class Connection
 
-    REPORT_PERIODS = ["Monthly Free", "Weekly", "Daily"]
+    REPORT_PERIODS = ["Weekly", "Daily"]
+
+    BASE_URL = 'https://itunesconnect.apple.com'  # login base
+    REPORT_URL = 'https://reportingitc.apple.com/sales.faces'
+    LOGIN_URL = 'https://itunesconnect.apple.com/WebObjects/iTunesConnect.woa'
 
-    BASE_URL = 'https://itts.apple.com' # :nodoc:
-    REFERER_URL = 'https://itts.apple.com/cgi-bin/WebObjects/Piano.woa' # :nodoc:
+    # select ids:
+    # theForm:datePickerSourceSelectElementSales (daily)
+    # theForm:weekPickerSourceSelectElement  (weekly)
+    ID_SELECT_DAILY = "theForm:datePickerSourceSelectElementSales"
+    ID_SELECT_WEEKLY = "theForm:weekPickerSourceSelectElement"
 
     # Create a new instance with the username and password used to sign
     # in to the iTunes Connect website
@@ -24,6 +54,7 @@ module ItunesConnect
       @username, @password = username, password
       @verbose = verbose
       @debug = debug
+      @current_period = "Daily"  # default period in reportingitc interface
     end
 
     def verbose?                # :nodoc:
@@ -34,6 +65,10 @@ module ItunesConnect
       !!@debug
     end
 
+    def logged_in?              # :nodoc:
+      !!@logged_in
+    end
+
     # Retrieve a report from iTunes Connect. This method will return the
     # raw report file as a String. If specified, the <tt>date</tt>
     # parameter should be a <tt>Date</tt> instance, and the
@@ -44,118 +79,212 @@ module ItunesConnect
     # Any dates given that equal the current date or newer will cause
     # this method to raise an <tt>ArgumentError</tt>.
     #
-    def get_report(date, out, period='Daily')
+    def get_report(date, out, period = 'Daily')
       date = Date.parse(date) if String === date
       if date >= Date.today
         raise ArgumentError, "You must specify a date before today"
       end
 
-      period = 'Monthly Free' if period == 'Monthly'
       unless REPORT_PERIODS.member?(period)
         raise ArgumentError, "'period' must be one of #{REPORT_PERIODS.join(', ')}"
       end
 
-      # grab the home page
-      doc = Nokogiri::HTML(get_content(REFERER_URL))
-      login_path = (doc/"form/@action").to_s
-
-      # login
-      doc = Nokogiri::HTML(get_content(login_path, {
-                                         'theAccountName' => @username,
-                                         'theAccountPW' => @password,
-                                         '1.Continue.x' => '36',
-                                         '1.Continue.y' => '17',
-                                         'theAuxValue' => ''
-                                       }))
-
-      report_url = (doc / "//*[@name='frmVendorPage']/@action").to_s
-      report_type_name = (doc / "//*[@id='selReportType']/@name").to_s
-      date_type_name = (doc / "//*[@id='selDateType']/@name").to_s
-
-      # handle first report form
-      doc = Nokogiri::HTML(get_content(report_url, {
-                                         report_type_name => 'Summary',
-                                         date_type_name => period,
-                                         'hiddenDayOrWeekSelection' => period,
-                                         'hiddenSubmitTypeName' => 'ShowDropDown'
-                                       }))
-      report_url = (doc / "//*[@name='frmVendorPage']/@action").to_s
-      report_type_name = (doc / "//*[@id='selReportType']/@name").to_s
-      date_type_name = (doc / "//*[@id='selDateType']/@name").to_s
-      date_name = (doc / "//*[@id='dayorweekdropdown']/@name").to_s
-
-      # now get the report
-      date_str = case period
-                 when 'Daily'
-                   date.strftime("%m/%d/%Y")
-                 when 'Weekly', 'Monthly Free'
-                   date = (doc / "//*[@id='dayorweekdropdown']/option").find do |d|
-                     d1, d2 = d.text.split(' To ').map { |x| Date.parse(x) }
-                     date >= d1 and date <= d2
-                   end[:value] rescue nil
-                 end
-
-      raise ArgumentError, "No reports are available for that date" unless date_str
-
-      report = get_content(report_url, {
-                             report_type_name => 'Summary',
-                             date_type_name => period,
-                             date_name => date_str,
-                             'download' => 'Download',
-                             'hiddenDayOrWeekSelection' => date_str,
-                             'hiddenSubmitTypeName' => 'Download'
-                           })
-
-      begin
-        gunzip = Zlib::GzipReader.new(StringIO.new(report))
-        out << gunzip.read
-      rescue => e
-        doc = Nokogiri::HTML(report)
-        msg = (doc / "//font[@id='iddownloadmsg']").text.strip
-        $stderr.puts "Unable to download the report, reason:"
-        $stderr.puts msg.strip
+      login unless self.logged_in?
+
+      # fetch report
+      # (cache report page)
+      fetch_report_page unless @report_page
+
+      # requested download date
+      date_str = date.strftime("%m/%d/%Y")
+      debug_msg("download date: #{date_str}")
+
+      # determine available download options
+      @select_name = period == 'Daily' ? ID_SELECT_DAILY : ID_SELECT_WEEKLY
+      options = @report_page.search(".//select[@id='#{@select_name}']/option")
+      options = options.collect { |i| i ? i['value'] : nil } if options
+      raise "unable to determine daily report options" unless options
+
+      debug_msg("options: #{options.inspect}")
+
+      # constrain download to available reports
+      available = options.find { |i| i <= date_str } ? true : false
+
+      unless available
+        raise ArgumentError, "No #{period} reports are available for #{date_str}"
+      end
+
+      # get ajax parameter name for Daily/Weekly (<a> id)
+      report_period_link = @report_page.link_with(:text => /#{period}/)
+      @report_period_id = report_period_link.node['id']
+      raise "could not determine form period AJAX parameter" unless @report_period_id
+
+      # get ajax parameter name from <select> onchange attribute
+      # 'parameters':{'theForm:j_id_jsp_4933398_30':'theForm:j_id_jsp_4933398_30'}
+      report_date_select = @report_page.search(".//select[@id='#{@select_name}']")
+      @report_date_id = report_date_select[0]['onchange'].match(/parameters':\{'(.*?)'/)[1] rescue nil
+      raise "could not determine form date AJAX parameter" unless @report_date_id
+
+      # select report period to download (Weekly/Daily)
+      if @current_period != period
+        change_report(@report_page, date_str, @report_period_id => @report_period_id)
+        @current_period = period
       end
+
+      # select report date
+      page = change_report(@report_page, date_str, @report_date_id => @report_date_id)
+
+      # after selecting report type, recheck if selection is available.
+      # (selection options exist even when the report isn't available, so
+      #  we need to do another check here)
+      dump(client, page)
+      available = !page.body.match(/There is no report available for this selection/)
+      unless available
+        raise ArgumentError, "No #{period} reports are available for #{date_str}"
+      end
+
+      # download the report
+      page = @report_page.form_with(:name => 'theForm') do |form|
+        form['theForm:xyz'] = 'notnormal'
+        form['theForm:downloadLabel2'] = 'theForm:downloadLabel2'
+        form[@select_name] = date_str
+
+        form.delete_field!('AJAXREQUEST')
+        form.delete_field!(@report_period_id)
+        form.delete_field!(@report_date_id)
+
+        debug_form(form)
+      end.submit
+
+      dump(client, page)
+      report = page.body.to_s
+      debug_msg("report is #{report.length} bytes")
+      out << report
+      report
     end
 
     private
 
-    def client
-      @client ||= client = HTTPClient.new
+    def debug_msg(message)
+      return unless self.debug?
+      puts message
     end
 
-    def get_content(uri, query=nil, headers={ })
-      $stdout.puts "Querying #{uri} with #{query.inspect}" if self.debug?
-      if @referer
-        headers = {
-          'Referer' => @referer,
-          'User-Agent' => 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-us) AppleWebKit/531.9 (KHTML, like Gecko) Version/4.0.3 Safari/531.9'
-        }.merge(headers)
-      end
-      url = case uri
-            when /^https?:\/\//
-              uri
-            else
-              BASE_URL + uri
-            end
-
-      response = client.get(url, query, headers)
-
-      if self.debug?
-        md5 = Digest::MD5.new; md5 << url; md5 << Time.now.to_s
-        path = File.join(Dir.tmpdir, md5.to_s + ".html")
-        out = open(path, "w") do |f|
-          f << "Status: #{response.status}\n"
-          f << response.header.all.map do |name, value|
-            "#{name}: #{value}"
-          end.join("\n")
-          f << "\n\n"
-          f << response.body.dump
+    def change_report(report_page, date_str, params = {})
+      page = report_page.form_with(:name => 'theForm') do |form|
+        form.delete_field!(@report_period_id)
+        form.delete_field!(@report_date_id)
+        form.delete_field!('theForm:downloadLabel2')
+
+        params.each { |k,v| form[k] = v }
+
+        form['AJAXREQUEST'] = @ajax_id
+        form['theForm:xyz'] = 'notnormal'
+        form[@select_name] = date_str
+
+        debug_form(form)
+      end.submit
+    end
+
+    # fetch main report page (sales.faces)
+    def fetch_report_page 
+      @report_page = client.get(REPORT_URL)
+      dump(client, @report_page)
+            
+      # get ajax parameter name for AJAXREQUEST (<a> id)
+      # AJAX.Submit('theForm:j_id_jsp_4933398_2'
+      @ajax_id = @report_page.body.match(/AJAX\.Submit\('([^\']+)'/)[1] rescue nil
+      raise "could not determine form AJAX id" unless @ajax_id
+
+      @report_page
+    end
+
+    # log in and navigate to the reporting interface
+    def login
+      debug_msg("getting login page")
+      page = client.get(LOGIN_URL)
+
+      while true do
+        debug_msg("logging in")
+        page = page.form_with(:name => 'appleConnectForm') do |form|
+          raise "login form not found" unless form
+
+          form['theAccountName'] = @username
+          form['theAccountPW'] = @password
+          form['1.Continue.x'] = '35'
+          form['1.Continue.y'] = '16'
+          form['theAuxValue'] = ''
+        end.submit
+
+        dump(client, page)
+
+        # 'session expired' message sometimes appears after logging in. weird.
+        expired = page.body.match(/Your session has expired.*?href\="(.*?)"/)
+        if expired
+          debug_msg("expired session detected, retrying login")
+          page = client.get(expired[1])
+          next  # retry login
         end
-        puts "#{url} -> #{path}"
+
+        break  # done logging in
+      end
+
+      # skip past new license available notifications
+      new_license = page.body.match(/License Agreement Update/)
+      if new_license
+        debug_msg("new license detected, skipping")
+        submit_parameter = page.body.match(/input.*?type\="image".*?name="(.*?)"/)[1] rescue nil
+        raise "could not determine how to skip new license agreement" unless submit_parameter
+        page = page.form_with(:name => 'mainForm') do |form|
+          form["#{submit_parameter}.x"] = 40
+          form["#{submit_parameter}.y"] = 18
+        end.submit
       end
 
-      @referer = url
-      response.body.dump
+      # Click the sales and trends link
+      sales_link = page.link_with(:text => /Sales and Trends/)
+      raise "Sales and Trends link not found" unless sales_link
+      page2 = client.click(sales_link)
+      dump(client, page2)
+
+      @report_page = nil  # clear any cached report page
+      @logged_in = true
+    end
+
+    def client
+      return @client if @client
+      @client = Mechanize.new
+      @client.read_timeout = NETWORK_TIMEOUT
+      @client.open_timeout = NETWORK_TIMEOUT
+      @client.user_agent_alias = 'Mac FireFox'
+      @client
+    end
+
+    # dump state information to a file (debugging)
+    def dump(a, page)
+      return unless self.debug?
+
+      url = a.current_page.uri.request_uri
+      puts "current page: #{url}"
+
+      md5 = Digest::MD5.new; md5 << url; md5 << Time.now.to_s
+      path = File.join(Dir.tmpdir, md5.to_s + ".html")
+      out = open(path, "w") do |f|
+        f << "Current page: #{url}"
+        f << "Headers: #{page.header}"
+        f << page.body
+      end
+
+      puts "#{url} -> #{path}"
+    end
+
+    def debug_form(form)
+      return unless self.debug?
+
+      puts "\nsubmitting form:"
+      form.keys.each do |key|
+        puts "#{key}: #{form[key]}"
+      end
     end
 
   end

data/lib/itunes_connect/store.rb

@@ -11,7 +11,8 @@ module ItunesConnect
       @db = SQLite3::Database.new(file)
       if @db.table_info("reports").empty?
         @db.execute("CREATE TABLE reports (id INTEGER PRIMARY KEY, " +
-                    "report_date DATE, country TEXT, install_count INTEGER, " +
+                    "report_date DATE NOT NULL, country TEXT NOT NULL, " +
+                    "install_count INTEGER, " +
                     "update_count INTEGER)")
         @db.execute("CREATE UNIQUE INDEX u_reports_idx ON reports " +
                     "(report_date, country)")
@@ -25,11 +26,11 @@ module ItunesConnect
 
     # Add a record to this instance
     def add(date, country, install_count, update_count)
-      @db.execute("INSERT INTO reports (report_date, country, " +
-                  "install_count, update_count) VALUES (?, ?, ?, ?)",
-                  date, country, install_count, update_count)
+      ret = @db.execute("INSERT INTO reports (report_date, country, " +
+                        "install_count, update_count) VALUES (?, ?, ?, ?)",
+                        [format_date(date), country, install_count, update_count])
       true
-    rescue SQLite3::SQLException => e
+    rescue SQLite3::ConstraintException => e
       if e.message =~ /columns .* are not unique/
         $stdout.puts "Skipping existing row for #{country} on #{date}" if verbose?
         false
@@ -56,12 +57,12 @@ module ItunesConnect
 
       if opts[:from]
         clauses << "report_date >= ?"
-        params << opts[:from]
+        params << format_date(opts[:from])
       end
 
       if opts[:to]
         clauses << "report_date <= ?"
-        params << opts[:to]
+        params << format_date(opts[:to])
       end
 
       if opts[:country]
@@ -73,9 +74,9 @@ module ItunesConnect
       sql << clauses.join(" AND ") unless params.empty?
       sql << " ORDER BY report_date DESC"
 
-      @db.execute(sql, *params).map do |row|
+      @db.execute(sql, params).map do |row|
         OpenStruct.new({
-                         :report_date => Date.parse(row[1]),
+                         :report_date => row[1] ? Date.parse(row[1]) : nil,
                          :country => row[2],
                          :install_count => row[3].to_i,
                          :update_count => row[4].to_i
@@ -100,12 +101,12 @@ module ItunesConnect
 
       if opts[:from]
         clauses << "report_date >= ?"
-        params << opts[:from]
+        params << format_date(opts[:from])
       end
 
       if opts[:to]
         clauses << "report_date <= ?"
-        params << opts[:to]
+        params << format_date(opts[:to])
       end
 
       if opts[:country]
@@ -117,7 +118,7 @@ module ItunesConnect
       sql << clauses.join(" AND ") unless params.empty?
       sql << " GROUP BY country ORDER BY country"
 
-      @db.execute(sql, *params).map do |row|
+      @db.execute(sql, params).map do |row|
         OpenStruct.new({
                          :country => row[0],
                          :install_count => row[1].to_i,
@@ -125,5 +126,11 @@ module ItunesConnect
                        })
       end
     end
+
+    private
+
+    def format_date(date)
+      date.is_a?(Date) ? date.strftime("%Y-%m-%d") : date
+    end
   end
 end

metadata

@@ -1,7 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: itunes-connect
 version: !ruby/object:Gem::Version 
-  version: 0.9.0
+  hash: 55
+  prerelease: false
+  segments: 
+  - 0
+  - 10
+  - 0
+  version: 0.10.0
 platform: ruby
 authors: 
 - Alex Vollmer
@@ -9,69 +15,84 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-11-07 00:00:00 -08:00
+date: 2010-09-22 00:00:00 +09:30
 default_executable: itunes_connect
 dependencies: 
 - !ruby/object:Gem::Dependency 
-  name: httpclient
-  type: :runtime
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
+  name: mechanize
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        version: "2.1"
-    version: 
-- !ruby/object:Gem::Dependency 
-  name: nokogiri
+        hash: 23
+        segments: 
+        - 1
+        - 0
+        - 0
+        version: 1.0.0
   type: :runtime
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
-    requirements: 
-    - - ~>
-      - !ruby/object:Gem::Version 
-        version: "1.3"
-    version: 
+  version_requirements: *id001
 - !ruby/object:Gem::Dependency 
   name: clip
-  type: :runtime
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 21
+        segments: 
+        - 1
+        - 0
+        - 1
         version: 1.0.1
-    version: 
+  type: :runtime
+  version_requirements: *id002
 - !ruby/object:Gem::Dependency 
   name: sqlite3-ruby
-  type: :runtime
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
+        hash: 11
+        segments: 
+        - 1
+        - 2
         version: "1.2"
-    version: 
+  type: :runtime
+  version_requirements: *id003
 - !ruby/object:Gem::Dependency 
   name: rspec
-  type: :development
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
         version: "0"
-    version: 
+  type: :development
+  version_requirements: *id004
 - !ruby/object:Gem::Dependency 
   name: fakeweb
-  type: :development
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
         version: "0"
-    version: 
+  type: :development
+  version_requirements: *id005
 description: Programmatic and command-line access to iTunes Connect Reports
 email: alex.vollmer@gmail.com
 executables: 
@@ -116,21 +137,27 @@ rdoc_options:
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
       version: "0"
-  version: 
 required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
       version: "0"
-  version: 
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.5
+rubygems_version: 1.3.7
 signing_key: 
 specification_version: 3
 summary: Get your iTunes Connect Reports