adops_report_scrapper 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 368ff448654fe142f098a05d72c263f4a7c7bacc
-  data.tar.gz: 8664f20d302251678c328d37401275164476451a
+  metadata.gz: 4ed4888eb209682c4179f2320fc0c76dcc9bb266
+  data.tar.gz: b4ec13ef9772b23179c18dab0863607387bacb33
 SHA512:
-  metadata.gz: 652e2f66cc8326a1e6ebedfd5615f1b0ebaf555c80fd2c8763ac57e69a3e804ca6d2cbd7ed8d3a00b2f59471dd67117c59bc0befd6eb96386758e30eaac3d45e
-  data.tar.gz: 60ebf84d57a88314c7c922cd445ef943b572e38ba295e6c0e1ba4df100c8ad800eca2377627560898988cd0050fb4df9c8eadddfea12999972d92c2f8ed4325b
+  metadata.gz: 3bcc8a0ac2d9b7c01232a5663cd97e0eb06cdd39d1f7d767faa6a9ed6d5e75c30572dfc0abc9cfa2b04362e299521f60092d7ef4d3a4b28aaf392ee8d96bea25
+  data.tar.gz: c8056cda55d69c3993de465e19021f948bd81b5ff859ede4564f4322fcc35b9aa638da47d57b67c6a302e92869554fe3978daba32fffaad94dbe94a827250658

data/Rakefile

@@ -126,12 +126,16 @@ task :facebookaudience do
   save_as_csv :facebookaudience, :facebookaudience
 end
 
+def date
+  @date ||= ENV['date'].nil? ? Date.today - 1 : Date.today - ENV['date'].to_i
+end
+
 def get_yesterdays_file_path(adnetwork)
-  "tmp/#{Date.today.prev_day}/#{adnetwork}.csv"
+  "tmp/#{@date}/#{adnetwork}.csv"
 end
 
 def write_csv(adnet, data)
-  Dir.mkdir "tmp/#{Date.today.prev_day}" unless Dir.exists? "tmp/#{Date.today.prev_day}"
+  Dir.mkdir "tmp/#{@date}" unless Dir.exists? "tmp/#{@date}"
   CSV.open(get_yesterdays_file_path(adnet), 'w') do |csv|
     data.each do |row|
       csv << row
@@ -146,7 +150,6 @@ def save_as_csv(adnet, scrapper) # adnet and scrapper are both sym
   end
   puts "========== working on #{adnet}"
   cred = YAML.load_file('secret.yml')[adnet.to_s]
-  data = AdopsReportScrapper.get_scrapper(adnet, cred['login'], cred['secret'], cred['options']).get_data
-  byebug
+  data = AdopsReportScrapper.get_scrapper(adnet, cred['login'], cred['secret'], cred['options']).get_data(date)
   write_csv(adnet, data)
 end

data/lib/adops_report_scrapper/browsi_client.rb

@@ -3,6 +3,12 @@ require_relative 'base_client'
 require 'open-uri'
 
 class AdopsReportScrapper::BrowsiClient < AdopsReportScrapper::BaseClient
+  def date_supported?(date = nil)
+    _date = date || @date
+    return true if _date >= Date.today - 7
+    false
+  end
+
   private
 
   def login

data/lib/adops_report_scrapper/gcs_client.rb

@@ -4,6 +4,12 @@ require_relative 'base_client'
 # gcs sometimes doesn't update data in 24 hours
 
 class AdopsReportScrapper::GcsClient < AdopsReportScrapper::BaseClient
+  def date_supported?(date = nil)
+    _date = date || @date
+    return true if _date >= Date.today - 7
+    false
+  end
+
   private
 
   def login

data/lib/adops_report_scrapper/version.rb

@@ -1,3 +1,3 @@
 module AdopsReportScrapper
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: adops_report_scrapper
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Stayman Hou