adops_report_scrapper 0.2.9 → 0.2.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2062161736aa199788f24a787649cc654fce4f67
-  data.tar.gz: dd424d62acb31fc097d8a4dd6951aa9b75e37d5b
+  metadata.gz: 10d81f7b1c8357521bbf4a3e9be25f6cb8e6e8d1
+  data.tar.gz: 2faf4bef4bccdc0125e666fbbf532fa3c6cf62e5
 SHA512:
-  metadata.gz: f8e3e4d325b51ff318ab3a944d2c2fbcdc29bd7583a0e5287000417fec93377fc7bc1f5ebdf5edd07894337c059bc6307efac91ec5edc88a5c3189bd203d065f
-  data.tar.gz: 9090366aabcb84f92dc3c86127059e5d29d70cac949a0a98c88b6dcca2759ad65dd29baa4042da5b555df047e13f159e329150bc05ac6e86ccb0acc4c2baba2d
+  metadata.gz: 87438a0eaf28f6a35942db7e43ec8d76f593f4d644cdf9c45913fe4f0dbd8bf1418033accf67d55c05b9f75f8bd4a29fa28fc3dfdaa3938b665f5b09e234d25e
+  data.tar.gz: a198b3cbe5db8eb61cf4ed1886c72d242f3a33ceb26b9cbf0487334b76a32ebd38887f2d742a2005046761c6a03f9e7dae753bde79dcd12b7e841eec45eff521

data/Gemfile

@@ -17,3 +17,4 @@ gem 'google-api-client', '~> 0.8.2'
 gem 'cheddar'
 gem 'roo', '~> 2.4.0'
 gem 'selenium-webdriver'
+gem 'mail'

data/Rakefile

@@ -196,6 +196,11 @@ task :spotxchange do
   save_as_csv :spotxchange, :spotxchange
 end
 
+desc 'Collect email data'
+task :mediabong do
+  save_as_csv :mediabong, :email
+end
+
 def date
   @date ||= ENV['date'].nil? ? Date.today - 1 : Date.today - ENV['date'].to_i
 end

data/adops_report_scrapper.gemspec

@@ -31,6 +31,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'cheddar', '~> 1.0'
   spec.add_dependency 'roo', '~> 2.4.0'
   spec.add_dependency 'selenium-webdriver', '~> 3.3.0'
+  spec.add_dependency 'mail', '~> 2.6.4'
 
   spec.add_development_dependency "bundler", "~> 1.8"
   spec.add_development_dependency "rake", "~> 10.0"

data/lib/adops_report_scrapper/email_client.rb

@@ -0,0 +1,72 @@
+require 'date'
+require_relative 'base_client'
+require 'net/imap'
+require 'mail'
+require 'csv'
+
+class AdopsReportScrapper::EmailClient < AdopsReportScrapper::BaseClient
+  def date_supported?(date = nil)
+    _date = date || @date
+    return true if _date >= Date.today - 30
+    false
+  end
+  private
+
+  def init_client
+    fail 'please specify email imap_server' unless @options['imap_server']
+    fail 'please specify email imap_port' unless @options['imap_port']
+    fail 'please specify email imap_ssl' unless @options['imap_ssl']
+    fail 'please specify email title' unless @options['title']
+    @imap_server = @options['imap_server']
+    @imap_port = @options['imap_port']
+    @imap_ssl = @options['imap_ssl']
+    @title = @options['title'] # supports data macro e.g. `XXX Report %Y-%m-%d` will match XXX Report `2017-04-26`
+    @date_column = @options['date_column'] # optional. supports data macro e.g. `0||%Y-%m-%d` will match rows that has `2017-04-26` for their first column
+  end
+
+  def before_quit_with_error
+  end
+
+  def scrap
+    @data = []
+    email_received_date = Net::IMAP.format_date(@date+1)
+    title = @date.strftime(@title)
+
+    imap = Net::IMAP.new(@imap_server, @imap_port, @imap_ssl)
+    imap.login(@login, @secret)
+    imap.select('INBOX')
+    report_email_ids = imap.search(['ON', email_received_date, 'SUBJECT', title])
+    if report_email_ids.count == 0
+      imap.logout
+      imap.disconnect
+      fail 'no email found with the given date and title'
+    elsif report_email_ids.count > 1
+      puts 'more than one email found with the given date and title, try to use the first one'
+    end
+    report_email_id = report_email_ids.first
+
+    body = imap.fetch(report_email_id, 'RFC822')[0].attr['RFC822']
+    mail = Mail.new(body)
+    if mail.attachments.blank?
+      imap.logout
+      imap.disconnect
+      fail 'no attachment found for the given report'
+    end
+
+    raw_data = mail.attachments.first.body.decoded
+
+    imap.logout
+    imap.disconnect
+
+    @data = CSV.parse(raw_data)
+
+    if @date_column
+      column_index, date_format_str = @date_column.split('||')
+      column_index = column_index.to_i
+      date_str = @date.strftime(date_format_str)
+      header = @data.shift
+      @data = @data.select { |row| row[column_index] == date_str }
+      @data.unshift header
+    end
+  end
+end

data/lib/adops_report_scrapper/tremor_client.rb

@@ -14,6 +14,7 @@ class AdopsReportScrapper::TremorClient < AdopsReportScrapper::BaseClient
   def login
     @client.visit 'https://console.tremorhub.com/ssp'
     @client.driver.browser.manage.window.resize_to(1366,768)
+    byebug
     @client.fill_in 'username', :with => @login
     @client.fill_in 'password', :with => @secret
     @client.click_button 'Sign In'
@@ -77,6 +78,7 @@ class AdopsReportScrapper::TremorClient < AdopsReportScrapper::BaseClient
   end
 
   def extract_data_from_report
+    byebug
     page = Nokogiri::HTML @client.html
     rows = page.xpath '//table[@id="DataTables_Table_1"]/*/tr'
     @data = rows.map { |tr| tr.css('td,th').map { |td| td.text } }

data/lib/adops_report_scrapper/version.rb

@@ -1,3 +1,3 @@
 module AdopsReportScrapper
-  VERSION = "0.2.9"
+  VERSION = "0.2.10"
 end

data/lib/adops_report_scrapper.rb

@@ -50,3 +50,4 @@ require 'adops_report_scrapper/appnexus_client'
 require 'adops_report_scrapper/sovrn_client'
 require 'adops_report_scrapper/gumgum_client'
 require 'adops_report_scrapper/spotxchange_client'
+require 'adops_report_scrapper/email_client'

data/secret.sample.yml

@@ -136,4 +136,13 @@ spotxchange:
   secret: ------
   options:
     client_id: ------
-    client_secret: ------
+    client_secret: ------
+mediabong:
+  login: ------
+  secret: ------
+  options:
+    imap_server: ------
+    imap_port: ------
+    imap_ssl: true/false
+    title: XXX Report
+    date_column: 1||%Y-%m-%d

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: adops_report_scrapper
 version: !ruby/object:Gem::Version
-  version: 0.2.9
+  version: 0.2.10
 platform: ruby
 authors:
 - Stayman Hou
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-04-05 00:00:00.000000000 Z
+date: 2017-04-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: httpclient
@@ -178,6 +178,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 3.3.0
+- !ruby/object:Gem::Dependency
+  name: mail
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.6.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.6.4
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -247,6 +261,7 @@ files:
 - lib/adops_report_scrapper/contentad_client.rb
 - lib/adops_report_scrapper/conversant_client.rb
 - lib/adops_report_scrapper/criteo_client.rb
+- lib/adops_report_scrapper/email_client.rb
 - lib/adops_report_scrapper/facebookaudience_client.rb
 - lib/adops_report_scrapper/gcs_client.rb
 - lib/adops_report_scrapper/gumgum_client.rb