adops_report_scrapper 0.1.29 → 0.1.30

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ae43547932efdcbe02b3fd153a02816453999acb
-  data.tar.gz: fc4f1d79ff70f6af01b961df4d3ac1c5d4309cb3
+  metadata.gz: 7c70e5a5ebc549c6f187dc54438c39e6c1030dc5
+  data.tar.gz: b546f11751c3e55fae68847bea85a12547599875
 SHA512:
-  metadata.gz: 8f2e5caecc5d2aa2fb7917fa8e1f8b34df0d703e3f573315973bdcd1e363f0e2571dca35341740de152dac74c8b74f58412910be5a2bf3b269499eff232cb4af
-  data.tar.gz: 33a2d356602ef0f4394e9189349f114ba2ef4599ebb267406cbd0d311fe669765a0a68786f0f7c78e9bfe38bbfba56307db1cf24c7a275fbda705a120c551a66
+  metadata.gz: 463417877c7112eefe8adf41e1781232e25d8599108898d2d90ff1f4fa1e03ccb4b8767032bcf84bbdda1b946ebe0b1b5599aa9a6d545fc83bcc3716233fde62
+  data.tar.gz: 00c69ea686a7dae418294da3e3fe46857f04f1d69a8ace9f9685f70e4e0e66483b5a19e61e91d0981939a4917d12b5ca817b05cba226c50003cf6467c021547b

data/Rakefile

@@ -7,7 +7,7 @@ require 'adops_report_scrapper'
 require 'byebug'
 
 desc 'Collect all data'
-task :all => [:openx, :tremor, :brightroll, :yellowhammer, :adaptv, :fourninefive, :adx, :revcontent, :gcs, :browsi, :netseer, :sonobi, :nativo, :adsupply, :marfeel, :adsense, :criteo, :triplelift, :conversant, :liveintent, :adiply, :contentad, :facebookaudience, :adtechus, :adtomation, :rhythmone] do # openx is the most unstable one, run it first
+task :all => [:openx, :tremor, :brightroll, :yellowhammer, :adaptv, :fourninefive, :adx, :revcontent, :gcs, :browsi, :netseer, :sonobi, :nativo, :adsupply, :marfeel, :adsense, :criteo, :triplelift, :conversant, :liveintent, :adiply, :contentad, :facebookaudience, :adtechus, :adtomation, :rhythmone, :littlethings] do # openx is the most unstable one, run it first
   puts '========== You are all set'
 end
 
@@ -146,6 +146,11 @@ task :rhythmone do
   save_as_csv :rhythmone, :rhythmone
 end
 
+desc 'Collect littlethings data'
+task :littlethings do
+  save_as_csv :littlethings, :littlethings
+end
+
 def date
   @date ||= ENV['date'].nil? ? Date.today - 1 : Date.today - ENV['date'].to_i
 end

data/lib/adops_report_scrapper.rb

@@ -40,3 +40,4 @@ require 'adops_report_scrapper/adtechus_client'
 require 'adops_report_scrapper/zedo_client'
 require 'adops_report_scrapper/adtomation_client'
 require 'adops_report_scrapper/rhythmone_client'
+require 'adops_report_scrapper/littlethings_client'

data/lib/adops_report_scrapper/littlethings_client.rb

@@ -0,0 +1,50 @@
+require 'date'
+require_relative 'base_client'
+
+class AdopsReportScrapper::LittlethingsClient < AdopsReportScrapper::BaseClient
+  def date_supported?(date = nil)
+    _date = date || @date
+    return true if _date >= Date.today - 4
+    false
+  end
+
+  private
+
+  def login
+    @client.visit 'http://www.reportingthings.com'
+    @client.fill_in 'email', :with => @login
+    @client.fill_in 'password', :with => @secret
+    @client.click_button 'Sign In'
+    begin
+      @client.find :xpath, '//*[contains(text(),"Report")]'
+    rescue Exception => e
+      raise e, 'Littlethings login error'
+    end
+  end
+
+  def scrap
+    request_report
+    extract_data_from_report
+  end
+
+  def request_report
+    @client.find(:xpath, '//*[contains(text(),"Report")]').click
+    pub_id = @client.body.match(/\/report\/story\/(\d+)\/all\/all/).captures[0]
+    daterange_end_str = Date.today.strftime('%m/%d/%Y')
+    daterange_begin_str = (Date.today - 5).strftime('%m/%d/%Y')
+    daterange_str = "#{daterange_begin_str} - #{daterange_end_str}"
+    @client.visit "http://www.reportingthings.com/report/story/#{pub_id}/all/all?type=revenue&daterange=#{URI.encode(daterange_str)}"
+  end
+
+  def extract_data_from_report
+    date_str = @date.strftime('%m/%d/%Y')
+    rows = @client.find_all :xpath, '//table[@id="report-table"]/*/tr'
+    rows = rows.map { |tr| tr.find_css('td,th').map { |td| td.visible_text } }
+    header = rows.shift
+    @data = [header]
+    rows.each do |row|
+      next unless row[0] == date_str
+      @data << row
+    end
+  end
+end

data/lib/adops_report_scrapper/version.rb

@@ -1,3 +1,3 @@
 module AdopsReportScrapper
-  VERSION = "0.1.29"
+  VERSION = "0.1.30"
 end

data/secret.sample.yml

@@ -97,3 +97,6 @@ rhythmone:
     client_id: ------
     client_secret: ------
     publisher_id: ------
+littlethings:
+  login: ------
+  secret: ------

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: adops_report_scrapper
 version: !ruby/object:Gem::Version
-  version: 0.1.29
+  version: 0.1.30
 platform: ruby
 authors:
 - Stayman Hou
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-10-20 00:00:00.000000000 Z
+date: 2016-10-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: httpclient
@@ -233,6 +233,7 @@ files:
 - lib/adops_report_scrapper/criteo_client.rb
 - lib/adops_report_scrapper/facebookaudience_client.rb
 - lib/adops_report_scrapper/gcs_client.rb
+- lib/adops_report_scrapper/littlethings_client.rb
 - lib/adops_report_scrapper/liveintent_client.rb
 - lib/adops_report_scrapper/marfeel_client.rb
 - lib/adops_report_scrapper/nativo_client.rb