adops_report_scrapper 0.2.35 → 0.2.37
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG +6 -0
- data/Rakefile +5 -0
- data/lib/adops_report_scrapper/email_client.rb +19 -2
- data/lib/adops_report_scrapper/version.rb +1 -1
- data/secret.sample.yml +1 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4cf064a3b734f43507cd24f2ad6b904bccf1225d
|
4
|
+
data.tar.gz: cec38e250b5689ef69913a791e05e16265419874
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 97358fa3667fb4ebf77be5b4269b5b800ede5267f09b447541e914df6b60f8136995a8c2a873d898b1e49f751eb86a69851e5faa2482de5f516d749365736213
|
7
|
+
data.tar.gz: 1f2e15de0eba6d7b48a71e6fb10927f40f64c4b374733cd5b1987d1b02adf1cbfeabd8e4e3c8b13436f34ab2227697e2ab2d34807e57cae40a32b91394473d6f
|
data/CHANGELOG
CHANGED
@@ -1,5 +1,11 @@
|
|
1
1
|
# CHANGELOG
|
2
2
|
|
3
|
+
## 0.2.37
|
4
|
+
add ignore_receive_date option for email client to ignore the email receive date when searching for a report
|
5
|
+
|
6
|
+
## 0.2.36
|
7
|
+
add header_first_cell option for email client to match the first cell of a row and find the header/starting row of the actual data
|
8
|
+
|
3
9
|
## 0.2.35
|
4
10
|
implement indexexchange
|
5
11
|
|
data/Rakefile
CHANGED
@@ -231,6 +231,11 @@ task :indexexchange do
|
|
231
231
|
save_as_csv :indexexchange, :indexexchange
|
232
232
|
end
|
233
233
|
|
234
|
+
desc 'Collect yume data'
|
235
|
+
task :yume do
|
236
|
+
save_as_csv :yume, :email
|
237
|
+
end
|
238
|
+
|
234
239
|
def date
|
235
240
|
@date ||= ENV['date'].nil? ? Date.today - 1 : Date.today - ENV['date'].to_i
|
236
241
|
end
|
@@ -22,7 +22,9 @@ class AdopsReportScrapper::EmailClient < AdopsReportScrapper::BaseClient
|
|
22
22
|
@imap_ssl = @options['imap_ssl']
|
23
23
|
@title = @options['title'] # supports data macro e.g. `XXX Report %Y-%m-%d` will match XXX Report `2017-04-26`
|
24
24
|
@date_column = @options['date_column'] # optional. supports data macro e.g. `0||%Y-%m-%d` will match rows that has `2017-04-26` for their first column
|
25
|
+
@header_first_cell = @options['header_first_cell'] # optional. It will try to match the first cell text, and treat the matched row as header, ignoring all the rows above. By default, it will take the first row as header.
|
25
26
|
@imap_ssl_verify = @options['imap_ssl_verify'].nil? ? true : @options['imap_ssl_verify']
|
27
|
+
@ignore_receive_date = @options['ignore_receive_date'].nil? ? false : @options['ignore_receive_date']
|
26
28
|
end
|
27
29
|
|
28
30
|
def before_quit_with_error
|
@@ -36,7 +38,10 @@ class AdopsReportScrapper::EmailClient < AdopsReportScrapper::BaseClient
|
|
36
38
|
imap = Net::IMAP.new(@imap_server, @imap_port, @imap_ssl, nil, @imap_ssl_verify)
|
37
39
|
imap.login(@login, @secret)
|
38
40
|
imap.select('INBOX')
|
39
|
-
|
41
|
+
search_condition = []
|
42
|
+
search_condition += ['ON', email_received_date] unless @ignore_receive_date
|
43
|
+
search_condition += ['SUBJECT', title]
|
44
|
+
report_email_ids = imap.search search_condition
|
40
45
|
if report_email_ids.count == 0
|
41
46
|
imap.logout
|
42
47
|
imap.disconnect
|
@@ -65,7 +70,19 @@ class AdopsReportScrapper::EmailClient < AdopsReportScrapper::BaseClient
|
|
65
70
|
column_index, date_format_str = @date_column.split('||')
|
66
71
|
column_index = column_index.to_i
|
67
72
|
date_str = @date.strftime(date_format_str)
|
68
|
-
header =
|
73
|
+
header = nil
|
74
|
+
if @header_first_cell
|
75
|
+
while !@data.empty?
|
76
|
+
row = @data.shift
|
77
|
+
if @header_first_cell == row[0]
|
78
|
+
header = row
|
79
|
+
break
|
80
|
+
end
|
81
|
+
fail 'empty report' if @data.length == 0
|
82
|
+
end
|
83
|
+
else
|
84
|
+
header = @data.shift
|
85
|
+
end
|
69
86
|
@data = @data.select { |row| row[column_index] == date_str }
|
70
87
|
@data.unshift header
|
71
88
|
end
|
data/secret.sample.yml
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: adops_report_scrapper
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.37
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Stayman Hou
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-10-13 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: httpclient
|