adops_report_scrapper 0.1.13 → 0.1.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1ca67e7025786adde54c380f4e623e8989e9789f
-  data.tar.gz: e6fc196fdedaaa407b4f98546de5fc0320d0f147
+  metadata.gz: a8602cb3bc3c3466d5cd6f67b9cba9ed0002f75e
+  data.tar.gz: 8b37b2be038428dc7d31792a5e179aeb5cbb13f6
 SHA512:
-  metadata.gz: 6edd51ce4a9953513989ff11c8b80fa7678847288eccfeebde43d79d62210e148f90dd6e5aad2aae92cdaaa91c89e2d7a330c901d026c515962f2914be1a7827
-  data.tar.gz: aad764f4dfec0aec24418ed7d40b2534a7f4f8f50d920303ccd0fee0313428e07c63c93cca4fe68d73c9724e73d495025b64e271a647f4348ac658561d12e5d7
+  metadata.gz: 04a02dd62bbb3261fc3ec6f8039c4ed10d9386001d48d6f3b7f640b4fd45c897b8048d64148eef1b4969f3e33c9d444eacfdebd0c6c01cac388ec4038e284379
+  data.tar.gz: cc2bba6c3399ce0996b8014a8e4ce714df5313df839dfc4b1a9bcc910eef552685796c7de121fb0904ee183d24729898acb37709113b45921af2784910071114

data/lib/adops_report_scrapper/openx_client.rb

@@ -26,7 +26,7 @@ class AdopsReportScrapper::OpenxClient < AdopsReportScrapper::BaseClient
   end
 
   def request_report
-    @client.visit 'http://cmci-ui.openx.net/#/reports?tab=my_reports'
+    @client.visit "http://#{@account_prefix}.openx.net/#/reports?tab=my_reports"
     sleep 5
 
     begin

data/lib/adops_report_scrapper/sonobi_client.rb

@@ -4,12 +4,25 @@ require_relative 'base_client'
 class AdopsReportScrapper::SonobiClient < AdopsReportScrapper::BaseClient
   def date_supported?(date = nil)
     _date = date || @date
-    return true if _date >= Date.today - 2
+    return true if _date >= Date.today - 3
     false
   end
 
   private
 
+  def init_client
+    fail 'please specify sonobi key' unless @options['key']
+    fail 'please specify sonobi code' unless @options['code']
+    fail 'please specify sonobi userid' unless @options['userid']
+    @key = @options['key']
+    @code = @options['code']
+    @userid = @options['userid']
+    super
+  end
+
+  def before_quit_with_error
+  end
+
   def login
     @client.visit 'https://jetstream.sonobi.com/welcome/login.php'
     @client.fill_in 'user name', :with => @login
@@ -20,6 +33,14 @@ class AdopsReportScrapper::SonobiClient < AdopsReportScrapper::BaseClient
     rescue Exception => e
       raise e, 'Sonobi login error'
     end
+    cookies = @client.driver.cookies
+    @client.driver.close_window('0')
+
+    @client = HTTPClient.new
+    @client.cookie_manager.cookies = cookies.values.map do |cookie|
+      cookie = cookie.instance_variable_get(:@attributes)
+      HTTP::Cookie.new cookie
+    end
   end
 
   def scrap
@@ -33,74 +54,53 @@ class AdopsReportScrapper::SonobiClient < AdopsReportScrapper::BaseClient
   def request_report(country)
     date_str = @date.strftime('%Y-%m-%d')
     is_us = country == :us
-    @client.find(:xpath, '//*[text()="Reports"]').click
-    sleep 2
-    # set date
-    @client.select 'Custom'
-    sleep 1
-    @client.find(:xpath, '//input[@name="_range_start_date"]').click
-    @client.find(:xpath, "//*[@date=\"#{date_str}\"]").click
-    @client.find(:xpath, '//input[@name="_range_end_date"]').click
-    @client.find(:xpath, "//*[@date=\"#{date_str}\"]").click
 
-    # all sites
-    @client.find(:xpath, '//div[@name="_siteid"]').click
-    sleep 2
-    @client.find(:xpath, '//*[text()="Select All"]').click
+    response = @client.post('https://jetstream.sonobi.com/public/', cm: 'category.list', key: @key, code: @code, _userid: @userid, _parentid: 'locations')
+    countries = JSON.parse response.body
 
-    # select country
-    @client.find(:xpath, '//*[text()="Add New Filter"]').click
-    @client.find_all(:xpath, '//option[text()="Country"]').last.select_option
-    sleep 2
-    @client.find(:xpath, '//div[@name="_country"]').click
-    sleep 2
-    if is_us
-      @client.find(:xpath, '//*[text()="United States"]').click
-    else
-      @client.find(:xpath, '//*[text()="Select All"]').click
-      @client.find(:xpath, '//*[@class="remove_icon"][../*[text()="United States"]]').click
-    end
+    report_body = {
+      '_userid': @userid,
+      '_country' => is_us ? 'US' : 'AF,AX,AL,DZ,AS,AD,AO,AI,AQ,AG,AR,AM,AW,AP,AU,AT,AZ,BS,BH,BD,BB,BY,BE,BZ,BJ,BM,BT,BO,BA,BW,BV,BR,IO,BN,BG,BF,BI,KH,CM,CA,CV,KY,CF,TD,CL,CN,CX,CC,CO,KM,CG,CD,CK,CR,CI,HR,CU,CY,CZ,DK,DJ,DM,DO,EC,EG,SV,GQ,ER,EE,ET,EU,FK,FO,FJ,FI,FR,GF,PF,TF,GA,GM,GE,DE,GH,GI,GR,GL,GD,GP,GU,GT,GG,GN,GW,GY,HT,HM,VA,HN,HK,HU,IS,IN,ID,IR,IQ,IE,IM,IL,IT,JM,JP,JE,JO,KZ,KE,KI,KP,KR,KW,KG,LA,LV,LB,LS,LR,LY,LI,LT,LU,MO,MK,MG,MW,MY,MV,ML,MT,MH,MQ,MR,MU,YT,MX,FM,MD,MC,MN,ME,MS,MA,MZ,MM,NA,NR,NP,NL,AN,NC,NZ,NI,NE,NG,NU,NF,MP,NO,OM,PK,PW,PS,PA,PG,PY,PE,PH,PN,PL,PT,PR,QA,RE,RO,RU,RW,SH,KN,LC,PM,VC,WS,SM,ST,SA,SN,RS,SC,SL,SG,SK,SI,SB,SO,ZA,GS,ES,LK,SD,SR,SJ,SZ,SE,CH,SY,TW,TJ,TZ,TH,TL,TG,TK,TO,TT,TN,TR,TM,TC,TV,UG,UA,AE,GB,UM,UY,UZ,VU,VE,VN,VG,VI,WF,EH,YE,ZM,ZW,--',
+      'groupby' => 'day',
+      'row_per' => '_date,_placementid,_siteid',
+      'columns' => '_date,_placementid_name,_siteid_name,_impression_count,_impression_count_viewed,_impression_count_clicked,_unfilled_impressions,_revenue,_ecpm,_device_type,_placementid,_siteid',
+      'tz_offset' => 'UTC',
+      '__column_info' => '[{\"name\":\"_date\",\"label\":\"Date\",\"no_limit\":true,\"tip\":\"Shows+the+date+that+impressions+were+served+on\"},{\"name\":\"_placementid_name\",\"label\":\"Placement\",\"tip\":\"Shows+the+Placement+name+that+impressions+were+served+on\"},{\"name\":\"_siteid_name\",\"label\":\"Site\",\"tip\":\"Shows+the+name+of+the+site+that+impressions+were+served+on\"},{\"name\":\"_impression_count\",\"label\":\"Impressions\",\"format\":true,\"tip\":\"The+number+of+impressions+that+were+served\"},{\"name\":\"_impression_count_viewed\",\"label\":\"Viewable+Impressions\",\"format\":true,\"tip\":\"The+number+of+times+that+the+ad+was+viewed\"},{\"name\":\"_impression_count_clicked\",\"label\":\"Clicks\",\"format\":true,\"tip\":\"The+number+of+clicks+that+were+recorded\"},{\"name\":\"_unfilled_impressions\",\"label\":\"Unfilled+Impressions\",\"format\":true,\"tip\":\"The+number+of+impressions+could+not+be+served+due+to+ad+server+decisioning\"},{\"name\":\"_revenue\",\"label\":\"Revenue\",\"format\":\"currency\",\"pre\":\"$\",\"tip\":\"Gross+revenue+of+impressions+served\"},{\"name\":\"_ecpm\",\"label\":\"Delivered+CPM\",\"format\":\"currency\",\"pre\":\"$\",\"tip\":\"Average+CPM+of+impressions+served\"},{\"name\":\"_device_type\",\"label\":\"Device+Type\",\"tip\":\"Device+Types+include:+Desktop,+Mobile\"}]'
+    }
+
+    response = @client.post('https://jetstream.sonobi.com/public/', cm: 'report.request', key: @key, code: @code, _report: report_body.to_json, _report_type: 'publisher_report', _report_origin: 'publisher_reporting', _range: 'custom', _range_start_date: date_str, _range_end_date: date_str)
+    report = JSON.parse response.body
+    report_id = report['package']['_reportid']
+    sleep 5
 
-    # check group by
-    @client.check 'Date'
-    @client.check 'Placement'
-    @client.check 'Site'
-    @client.check 'Clicks'
-    @client.check 'Views'
-    @client.check 'Unfilled Impressions'
-    @client.check 'Device Type'
+    30.times do # pull report 30 times
+      response = @client.post('https://jetstream.sonobi.com/public/', cm: 'report.get', key: @key, code: @code, _reportid: report_id, _wait: 'true')
+      report = JSON.parse response.body
+      report_status = report['package']['status']
+      case report_status
+      when 'complete'
+        break
+      when 'processing'
+        sleep 10
+      else
+        fail 'sonobi scrapper: unknown report status'
+      end
+    end
 
-    @client.click_button 'Run Report'
-    sleep 1
-    wait_for_spin
+    @response = response
   end
 
   def extract_data_from_report(country)
-    rows = @client.find_all :xpath, '//*[@class="reports_tab_item_body"]//table/*/tr'
-    rows = rows.to_a
-    header = rows.shift
+    report = JSON.parse @response.body
+    rows = report['package']['result']
     if @data.count == 0
-      n_header = header.find_css('td,th').map { |td| td.visible_text }
-      n_header << 'Country'
-      @data << n_header
+      @data << ['Date', 'Placement', 'Site', 'Impressions', 'Clicks', 'Views', 'Unfilled Impressions', 'Revenue', 'Device Type', 'Country']
     end
-    rows.shift
     @data += rows.map do |row|
-      n_row = row.find_css('td,th').map { |td| td.visible_text }
+      n_keys = ["_date", "_placementid_name", "_siteid_name", "_impression_count", "_impression_count_clicked", "_impression_count_viewed", "_unfilled_impressions", "_revenue", "_device_type"]
+      n_row = n_keys.map { |k| row[k] }
       n_row << country.to_s.upcase
       n_row
     end
   end
-
-  def wait_for_spin
-    30.times do |_i| # wait 5 min
-      begin
-        @client.find(:css, '.circle xlarge')
-      rescue Exception => e
-        break
-      end
-      sleep 10
-    end
-    sleep 5
-  end
 end

data/lib/adops_report_scrapper/version.rb

@@ -1,3 +1,3 @@
 module AdopsReportScrapper
-  VERSION = "0.1.13"
+  VERSION = "0.1.14"
 end

data/secret.sample.yml

@@ -38,6 +38,10 @@ netseer:
 sonobi:
   login: ------
   secret: ------
+  options:
+    key: ------
+    code: ------
+    userid: ------
 nativo:
   login: ------
   secret: ------

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: adops_report_scrapper
 version: !ruby/object:Gem::Version
-  version: 0.1.13
+  version: 0.1.14
 platform: ruby
 authors:
 - Stayman Hou
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-09-02 00:00:00.000000000 Z
+date: 2016-09-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: httpclient