adops_report_scrapper 0.2.35 → 0.2.37

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d9a80c798f022a79269546a5f1fb2f8e99921b06
-  data.tar.gz: 37cf1e17c2ad5e7837ff1e90936015fd5b232aaf
+  metadata.gz: 4cf064a3b734f43507cd24f2ad6b904bccf1225d
+  data.tar.gz: cec38e250b5689ef69913a791e05e16265419874
 SHA512:
-  metadata.gz: 3adf61abd8e28530eb3d5bcf1ff16d6c799d135c16bf03096a1a978e5e6e2dd2885c27753fdb1e60469d2e988fbb82a5d2c977811a094cc39a74000acde1438e
-  data.tar.gz: 3053718c47f72ee09e5f1d0bd5d43145a46cdd398ebe1c71916bff8ebc0157d8eb2d0fbe3fe521c7fc7fff1c9f5a23dd32b38104628dbaaf1f1c1d2d995f57d9
+  metadata.gz: 97358fa3667fb4ebf77be5b4269b5b800ede5267f09b447541e914df6b60f8136995a8c2a873d898b1e49f751eb86a69851e5faa2482de5f516d749365736213
+  data.tar.gz: 1f2e15de0eba6d7b48a71e6fb10927f40f64c4b374733cd5b1987d1b02adf1cbfeabd8e4e3c8b13436f34ab2227697e2ab2d34807e57cae40a32b91394473d6f

data/CHANGELOG

@@ -1,5 +1,11 @@
 # CHANGELOG
 
+## 0.2.37
+add ignore_receive_date option for email client to ignore the email receive date when searching for a report
+
+## 0.2.36
+add header_first_cell option for email client to match the first cell of a row and find the header/starting row of the actual data
+
 ## 0.2.35
 implement indexexchange
 

data/Rakefile

@@ -231,6 +231,11 @@ task :indexexchange do
   save_as_csv :indexexchange, :indexexchange
 end
 
+desc 'Collect yume data'
+task :yume do
+  save_as_csv :yume, :email
+end
+
 def date
   @date ||= ENV['date'].nil? ? Date.today - 1 : Date.today - ENV['date'].to_i
 end

data/lib/adops_report_scrapper/email_client.rb

@@ -22,7 +22,9 @@ class AdopsReportScrapper::EmailClient < AdopsReportScrapper::BaseClient
     @imap_ssl = @options['imap_ssl']
     @title = @options['title'] # supports data macro e.g. `XXX Report %Y-%m-%d` will match XXX Report `2017-04-26`
     @date_column = @options['date_column'] # optional. supports data macro e.g. `0||%Y-%m-%d` will match rows that has `2017-04-26` for their first column
+    @header_first_cell = @options['header_first_cell'] # optional. It will try to match the first cell text, and treat the matched row as header, ignoring all the rows above. By default, it will take the first row as header.
     @imap_ssl_verify = @options['imap_ssl_verify'].nil? ? true : @options['imap_ssl_verify']
+    @ignore_receive_date = @options['ignore_receive_date'].nil? ? false : @options['ignore_receive_date']
   end
 
   def before_quit_with_error
@@ -36,7 +38,10 @@ class AdopsReportScrapper::EmailClient < AdopsReportScrapper::BaseClient
     imap = Net::IMAP.new(@imap_server, @imap_port, @imap_ssl, nil, @imap_ssl_verify)
     imap.login(@login, @secret)
     imap.select('INBOX')
-    report_email_ids = imap.search(['ON', email_received_date, 'SUBJECT', title])
+    search_condition = []
+    search_condition += ['ON', email_received_date] unless @ignore_receive_date
+    search_condition += ['SUBJECT', title]
+    report_email_ids = imap.search search_condition
     if report_email_ids.count == 0
       imap.logout
       imap.disconnect
@@ -65,7 +70,19 @@ class AdopsReportScrapper::EmailClient < AdopsReportScrapper::BaseClient
       column_index, date_format_str = @date_column.split('||')
       column_index = column_index.to_i
       date_str = @date.strftime(date_format_str)
-      header = @data.shift
+      header = nil
+      if @header_first_cell
+        while !@data.empty?
+          row = @data.shift
+          if @header_first_cell == row[0]
+            header = row
+            break
+          end
+          fail 'empty report' if @data.length == 0
+        end
+      else
+        header = @data.shift
+      end
       @data = @data.select { |row| row[column_index] == date_str }
       @data.unshift header
     end

data/lib/adops_report_scrapper/version.rb

@@ -1,3 +1,3 @@
 module AdopsReportScrapper
-  VERSION = "0.2.35"
+  VERSION = "0.2.37"
 end

data/secret.sample.yml

@@ -147,6 +147,7 @@ mediabong:
     imap_ssl_verify: false
     title: XXX Report
     date_column: 1||%Y-%m-%d
+    ignore_receive_date: true/false
 anyclip:
   login: ------
   secret: ------

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: adops_report_scrapper
 version: !ruby/object:Gem::Version
-  version: 0.2.35
+  version: 0.2.37
 platform: ruby
 authors:
 - Stayman Hou
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-09-18 00:00:00.000000000 Z
+date: 2017-10-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: httpclient