embulk-plugin-input-sfdc-event-log-files 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/README.md +6 -0
- data/lib/embulk/input_sfdc_event_log_files.rb +115 -0
- metadata +88 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 2819d1e1af9032508bf81c7485f1faa33aecf945
|
4
|
+
data.tar.gz: d4783eecda6aa5761761874dbc997cecb34f7c9c
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 72330bfce5fb1d060df9b15a2684aff8d2a56cfe3f3ae53d5df26b6ebe5e2b62420f5b39873e8ae8a9813b42d7b21ed1932ec982e453bfd2e0981d403cc38be5
|
7
|
+
data.tar.gz: 754c34759df13e87dbeabd8b52dc512b8d2fbad13eb704a1a590bdd3a55b8ae12c781e18262db1bf8e78592a1a8d6753ab75000c1560702ef629a159c4f7bea2
|
data/README.md
ADDED
@@ -0,0 +1,115 @@
|
|
1
|
+
require 'httpclient'
|
2
|
+
require 'json'
|
3
|
+
require 'time'
|
4
|
+
require 'csv'
|
5
|
+
|
6
|
+
module Embulk
|
7
|
+
|
8
|
+
class InputSfdcEventLogFiles < InputPlugin
|
9
|
+
Plugin.register_input('sfdc_event_log_files', self)
|
10
|
+
|
11
|
+
class << self
|
12
|
+
def transaction(config, &control)
|
13
|
+
oauth = config.param('oauth', :hash)
|
14
|
+
task = {
|
15
|
+
'login_url' => config.param('login_url', :string, default: 'https://login.salesforce.com'),
|
16
|
+
'oauth_client_id' => oauth['id'],
|
17
|
+
'oauth_client_secret' => oauth['secret'],
|
18
|
+
'username' => config.param('username', :string),
|
19
|
+
'password' => config.param('password', :string),
|
20
|
+
'last_log_date' => config.param('last_log_date', :string, default: '0001-01-01T00:00:00Z'),
|
21
|
+
'max_retry_times' => config.param('max_retry_times', :integer, default: 2),
|
22
|
+
}
|
23
|
+
threads = config.param('threads', :integer, default: 2)
|
24
|
+
task['client'] = client = HTTPClient.new
|
25
|
+
|
26
|
+
begin
|
27
|
+
oauth(task)
|
28
|
+
task['records'] = query(task)
|
29
|
+
|
30
|
+
reports = yield(task, [], threads)
|
31
|
+
|
32
|
+
last_log_date_report = reports.max_by { |report|
|
33
|
+
report['last_log_date']
|
34
|
+
}
|
35
|
+
config.merge(last_log_date_report)
|
36
|
+
rescue
|
37
|
+
# TODO: log
|
38
|
+
raise
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def run(task, schema, index, page_builder)
|
43
|
+
# This input plugin does not use schema
|
44
|
+
new(task, index, page_builder).run
|
45
|
+
end
|
46
|
+
|
47
|
+
private
|
48
|
+
|
49
|
+
def oauth(task)
|
50
|
+
client = task['client']
|
51
|
+
params = {
|
52
|
+
:grant_type => 'password',
|
53
|
+
:client_id => task['oauth_client_id'],
|
54
|
+
:client_secret => task['oauth_client_secret'],
|
55
|
+
:username => task['username'],
|
56
|
+
:password => task['password']
|
57
|
+
}
|
58
|
+
with_retry(task) {
|
59
|
+
res = client.post(task['login_url'] + '/services/oauth2/token', params, :Accept => 'application/json; charset=UTF-8')
|
60
|
+
parsed = JSON.parse(res.body)
|
61
|
+
client.base_url = parsed['instance_url']
|
62
|
+
client.default_header = { 'Authorization' => 'Bearer ' + parsed['access_token'] }
|
63
|
+
nil
|
64
|
+
}
|
65
|
+
end
|
66
|
+
|
67
|
+
def query(task)
|
68
|
+
client = task['client']
|
69
|
+
query = "Select LogDate, EventType, LogFile from EventLogFile Where LogDate > #{task['last_log_date']}"
|
70
|
+
with_retry(task) {
|
71
|
+
res = client.get('/services/data/v32.0/query/', :q => query)
|
72
|
+
JSON.parse(res.body)['records']
|
73
|
+
}
|
74
|
+
end
|
75
|
+
|
76
|
+
def with_retry(task)
|
77
|
+
retry_times = 0
|
78
|
+
begin
|
79
|
+
yield
|
80
|
+
rescue
|
81
|
+
# TODO: log
|
82
|
+
retry_times += 1
|
83
|
+
retry if retry_times < task['max_retry_times']
|
84
|
+
raise
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
def initialize(task, index, page_builder)
|
90
|
+
@page_builder = page_builder
|
91
|
+
@records = task['records']
|
92
|
+
@last_log_date = Time.parse(task['last_log_date'])
|
93
|
+
@client = task['client']
|
94
|
+
end
|
95
|
+
|
96
|
+
def run
|
97
|
+
@records.each do |record|
|
98
|
+
event_type = record['EventType']
|
99
|
+
@last_log_date = [@last_log_date, Time.parse(record['LogDate']).to_i].max
|
100
|
+
log_file = record['LogFile']
|
101
|
+
CSV.parse(@client.get_content(log_file), headers: true) do |row|
|
102
|
+
row['time'] = Time.parse(row['TIMESTAMP']).to_i
|
103
|
+
@page_builder.add(row)
|
104
|
+
end
|
105
|
+
end
|
106
|
+
@page_builder.finish unless @records.empty?
|
107
|
+
|
108
|
+
commit_report = {
|
109
|
+
'last_log_date' => @last_log_date.xmlschema
|
110
|
+
}
|
111
|
+
commit_report
|
112
|
+
end
|
113
|
+
end
|
114
|
+
|
115
|
+
end
|
metadata
ADDED
@@ -0,0 +1,88 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: embulk-plugin-input-sfdc-event-log-files
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Hiroshi Nakamura
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2015-01-27 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: httpclient
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: bundler
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '1.0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '1.0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rake
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: 0.9.2
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: 0.9.2
|
55
|
+
description: Embulk plugin for Salesforce.com Event Log Files input
|
56
|
+
email: nahi@ruby-lang.org
|
57
|
+
executables: []
|
58
|
+
extensions: []
|
59
|
+
extra_rdoc_files: []
|
60
|
+
files:
|
61
|
+
- lib/embulk/input_sfdc_event_log_files.rb
|
62
|
+
- README.md
|
63
|
+
homepage: https://github.com/nahi/
|
64
|
+
licenses:
|
65
|
+
- Apache 2.0
|
66
|
+
metadata: {}
|
67
|
+
post_install_message:
|
68
|
+
rdoc_options: []
|
69
|
+
require_paths:
|
70
|
+
- lib
|
71
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ">="
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
77
|
+
requirements:
|
78
|
+
- - ">="
|
79
|
+
- !ruby/object:Gem::Version
|
80
|
+
version: '0'
|
81
|
+
requirements: []
|
82
|
+
rubyforge_project:
|
83
|
+
rubygems_version: 2.0.14
|
84
|
+
signing_key:
|
85
|
+
specification_version: 4
|
86
|
+
summary: Embulk plugin for Salesforce.com Event Log Files input
|
87
|
+
test_files: []
|
88
|
+
has_rdoc:
|