fluent-plugin-sforce 0.0.4 → 0.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/.travis.yml +3 -2
- data/Makefile +13 -0
- data/README.md +9 -5
- data/VERSION +1 -1
- data/fluent-plugin-sforce.gemspec +12 -11
- data/lib/fluent/plugin/in_sforce.rb +91 -79
- data/test/helper.rb +4 -4
- data/test/plugin/test_in_sforce.rb +12 -11
- metadata +22 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 8863f1e04136df66a435721438f38036aaf395f7f4e097b060489a67c1a8ecba
|
4
|
+
data.tar.gz: 7b968ebbffa37441ce8502df1352f3f9cdb0fd003dd0abc25736c80ea21d919f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: dc7eff2c230abddec690465a64b6e68e232277f3859363418f7340c66018fb9378f3814b05fafc34a90f69f63f4c1a7a8750098cbedf1d6167fb84c92dcd1217
|
7
|
+
data.tar.gz: 1a14bc0f6d6d7a3f8b270c44f37fb5fa3ebd9bd81dc88fec0cdb80053be14fda08fdc8d2def53c70795550f6c901c8d047244c0525d002cf27fd1225929bc1e8
|
data/.gitignore
CHANGED
data/.travis.yml
CHANGED
data/Makefile
ADDED
data/README.md
CHANGED
@@ -29,14 +29,18 @@ Or install it yourself as:
|
|
29
29
|
query SELECT id, Body FROM FeedItem
|
30
30
|
polling_interval 60
|
31
31
|
# topic AllMessages
|
32
|
+
# version 43.0
|
33
|
+
# login_endpoint test.salesforce.com
|
32
34
|
</source>
|
33
35
|
```
|
34
36
|
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
37
|
+
|key|description|
|
38
|
+
|-|-|
|
39
|
+
|username|Salesforce Username for exporting data.|
|
40
|
+
|password|Salesforce User's Password.|
|
41
|
+
|query|SOQL Query.|
|
42
|
+
|polling_interval|Query Interval Time.|
|
43
|
+
|topic|PushTopic name to subscribe.|
|
40
44
|
|
41
45
|
## Contributing
|
42
46
|
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.0.
|
1
|
+
0.0.5
|
@@ -3,22 +3,23 @@ lib = File.expand_path('../lib', __FILE__)
|
|
3
3
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
4
|
|
5
5
|
Gem::Specification.new do |spec|
|
6
|
-
spec.name =
|
7
|
-
spec.version = File.read(
|
8
|
-
spec.authors = [
|
9
|
-
spec.email = [
|
6
|
+
spec.name = 'fluent-plugin-sforce'
|
7
|
+
spec.version = File.read('VERSION').strip
|
8
|
+
spec.authors = ['Makoto Tajitsu']
|
9
|
+
spec.email = ['makoto_tajitsu@hotmail.co.jp']
|
10
10
|
spec.summary = %q{Fluent Plugin to export data from Salesforce.com.}
|
11
11
|
spec.description = %q{Fluent Plugin to export data from Salesforce.com.}
|
12
|
-
spec.homepage =
|
12
|
+
spec.homepage = 'https://github.com/tzmfreedom/fluent-plugin-sforce'
|
13
13
|
|
14
14
|
spec.files = `git ls-files -z`.split("\x0")
|
15
15
|
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
16
16
|
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
17
|
-
spec.require_paths = [
|
17
|
+
spec.require_paths = ['lib']
|
18
18
|
|
19
|
-
spec.add_dependency
|
20
|
-
spec.add_dependency
|
21
|
-
spec.add_dependency
|
22
|
-
spec.add_dependency
|
23
|
-
spec.add_development_dependency
|
19
|
+
spec.add_dependency 'fluentd'
|
20
|
+
spec.add_dependency 'restforce', '~> 3.0.0'
|
21
|
+
spec.add_dependency 'faye', '~> 1.2.4'
|
22
|
+
spec.add_dependency 'nokogiri', '~> 1.8.2'
|
23
|
+
spec.add_development_dependency 'rake'
|
24
|
+
spec.add_development_dependency 'test-unit'
|
24
25
|
end
|
@@ -5,111 +5,123 @@ require 'restforce'
|
|
5
5
|
require 'date'
|
6
6
|
require 'net/http'
|
7
7
|
require 'faye'
|
8
|
+
require 'fluent/plugin/input'
|
8
9
|
|
9
10
|
module Fluent
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
end
|
11
|
+
module Plugin
|
12
|
+
class SforceInput < Input
|
13
|
+
class SforceConnectionError < StandardError; end
|
14
14
|
|
15
|
-
|
15
|
+
Fluent::Plugin.register_input('sforce', self)
|
16
16
|
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
17
|
+
config_param :query, :string, default: 'SELECT id, Body, CreatedById FROM FeedItem'
|
18
|
+
config_param :tag, :string, default: 'sforce'
|
19
|
+
config_param :polling_interval, :integer, default: 60
|
20
|
+
config_param :topic, :string, default: nil
|
21
|
+
config_param :username, :string
|
22
|
+
config_param :password, :string
|
23
|
+
config_param :version, :string, default: '43.0'
|
24
|
+
config_param :login_endpoint, :string, default: 'login.salesforce.com'
|
23
25
|
|
24
|
-
|
25
|
-
|
26
|
-
|
26
|
+
def configure(conf)
|
27
|
+
super
|
28
|
+
end
|
27
29
|
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
:instance_url => login_info["instanceUrl"]
|
33
|
-
|
34
|
-
th_low = DateTime.now().strftime("%Y-%m-%dT%H:%M:%S.000%Z")
|
35
|
-
# query
|
36
|
-
if @topic == nil then
|
37
|
-
sleep(@polling_interval)
|
38
|
-
th_high = DateTime.now().strftime("%Y-%m-%dT%H:%M:%S.000%Z")
|
39
|
-
loop do
|
40
|
-
# create soql query string
|
41
|
-
where = "CreatedDate <= #{th_high} AND CreatedDate > #{th_low}"
|
42
|
-
soql = ""
|
43
|
-
if @query =~ /^(.+)\s(where|WHERE)\s(.+)$/ then
|
44
|
-
soql = "#{$1} WHERE #{where} AND #{$3}"
|
45
|
-
elsif @query =~ /^(.+)$/ then
|
46
|
-
soql = "#{$1} WHERE #{where}"
|
47
|
-
end
|
30
|
+
def start
|
31
|
+
super
|
32
|
+
login_info = login
|
33
|
+
client = Restforce.new login_info.merge(api_version: @version)
|
48
34
|
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
35
|
+
th_low = DateTime.now.strftime('%Y-%m-%dT%H:%M:%S.000%Z')
|
36
|
+
# query
|
37
|
+
if @topic == nil then
|
38
|
+
sleep(@polling_interval)
|
39
|
+
th_high = DateTime.now.strftime('%Y-%m-%dT%H:%M:%S.000%Z')
|
40
|
+
loop do
|
41
|
+
# create soql query string
|
42
|
+
where = "CreatedDate <= #{th_high} AND CreatedDate > #{th_low}"
|
43
|
+
soql = ''
|
44
|
+
if @query =~ /^(.+)\s(where|WHERE)\s(.+)$/ then
|
45
|
+
soql = "#{$1} WHERE #{where} AND #{$3}"
|
46
|
+
elsif @query =~ /^(.+)$/ then
|
47
|
+
soql = "#{$1} WHERE #{where}"
|
48
|
+
end
|
49
|
+
|
50
|
+
begin
|
51
|
+
log.info "query: #{soql}"
|
52
|
+
records = client.query(soql)
|
53
|
+
records.each do |record|
|
54
|
+
router.emit(@tag, Fluent::Engine.now, record)
|
55
|
+
end
|
56
|
+
sleep(@polling_interval)
|
57
|
+
th_low = th_high
|
58
|
+
th_high = DateTime.now.strftime('%Y-%m-%dT%H:%M:%S.000%Z')
|
59
|
+
rescue Restforce::UnauthorizedError => e
|
60
|
+
log.error e
|
61
|
+
# retry login
|
62
|
+
login_info = login
|
63
|
+
client = Restforce.new login_info.merge(api_version: @version)
|
54
64
|
end
|
55
|
-
sleep(@polling_interval)
|
56
|
-
th_low = th_high
|
57
|
-
th_high = DateTime.now().strftime("%Y-%m-%dT%H:%M:%S.000%Z")
|
58
|
-
rescue Restforce::UnauthorizedError => e
|
59
|
-
log.error e
|
60
|
-
# retry login
|
61
|
-
login_info = login()
|
62
|
-
client = Restforce.new :oauth_token => login_info["sessionId"],
|
63
|
-
:instance_url => login_info["instanceUrl"]
|
64
65
|
end
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
66
|
+
# streaming api
|
67
|
+
else
|
68
|
+
EM.run do
|
69
|
+
log.info "suscribe: #{@topic}"
|
70
|
+
# Subscribe to the PushTopic.
|
71
|
+
client.subscribe @topic do |message|
|
72
|
+
router.emit(@tag, Fluent::Engine.now, message)
|
73
|
+
end
|
73
74
|
end
|
74
75
|
end
|
76
|
+
rescue SforceConnectionError => e
|
77
|
+
log.error e.message
|
75
78
|
end
|
76
|
-
end
|
77
79
|
|
78
|
-
|
79
|
-
|
80
|
+
def shutdown
|
81
|
+
super
|
82
|
+
end
|
80
83
|
|
81
|
-
|
84
|
+
private
|
82
85
|
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
86
|
+
def login
|
87
|
+
uri = URI(login_endpoint)
|
88
|
+
request = Net::HTTP::Post.new(uri.request_uri, initheader = {'Content-Type' =>'text/xml', 'SOAPAction' => "''"})
|
89
|
+
request.body = <<BODY
|
87
90
|
<?xml version="1.0" encoding="utf-8"?>
|
88
91
|
<env:Envelope xmlns:xsd="http://www.w3.org/2001/XMLSchema"
|
89
92
|
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
90
93
|
xmlns:env="http://schemas.xmlsoap.org/soap/envelope/">
|
91
94
|
<env:Body>
|
92
95
|
<n1:login xmlns:n1="urn:partner.soap.sforce.com">
|
93
|
-
<n1:username>#{@username}</n1:username>
|
94
|
-
<n1:password>#{@password}</n1:password>
|
96
|
+
<n1:username>#{@username.encode(xml: :text)}</n1:username>
|
97
|
+
<n1:password>#{@password.encode(xml: :text)}</n1:password>
|
95
98
|
</n1:login>
|
96
99
|
</env:Body>
|
97
100
|
</env:Envelope>
|
98
101
|
BODY
|
99
102
|
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
103
|
+
http = Net::HTTP.new(uri.host, uri.port)
|
104
|
+
http.use_ssl = true
|
105
|
+
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
106
|
+
|
107
|
+
# request login call and parse login response.
|
108
|
+
http.start do |h|
|
109
|
+
response = h.request(request)
|
110
|
+
doc = Nokogiri::XML(response.body)
|
111
|
+
fault = doc.css('faultstring').inner_text
|
112
|
+
raise SforceConnectionError, fault unless fault.empty?
|
113
|
+
|
114
|
+
session_id = doc.css('sessionId').inner_text
|
115
|
+
/^(https:\/\/.+\.salesforce\.com)\//.match(doc.css('serverUrl').inner_text)
|
116
|
+
instance_url = $1
|
117
|
+
log.info "login is successful. instance_url = '#{instance_url}'"
|
118
|
+
|
119
|
+
{oauth_token: session_id, instance_url: instance_url}
|
120
|
+
end
|
121
|
+
end
|
122
|
+
|
123
|
+
def login_endpoint
|
124
|
+
"https://#{@login_endpoint}/services/Soap/u/#{@version}"
|
113
125
|
end
|
114
126
|
end
|
115
127
|
end
|
data/test/helper.rb
CHANGED
@@ -7,11 +7,14 @@ rescue Bundler::BundlerError => e
|
|
7
7
|
$stderr.puts "Run `bundle install` to install missing gems"
|
8
8
|
exit e.status_code
|
9
9
|
end
|
10
|
-
require 'test
|
10
|
+
require 'test-unit'
|
11
11
|
|
12
12
|
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
13
13
|
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
14
|
+
|
14
15
|
require 'fluent/test'
|
16
|
+
require 'fluent/test/helpers'
|
17
|
+
|
15
18
|
unless ENV.has_key?('VERBOSE')
|
16
19
|
nulllogger = Object.new
|
17
20
|
nulllogger.instance_eval {|obj|
|
@@ -23,6 +26,3 @@ unless ENV.has_key?('VERBOSE')
|
|
23
26
|
end
|
24
27
|
|
25
28
|
require 'fluent/plugin/in_sforce'
|
26
|
-
|
27
|
-
class Test::Unit::TestCase
|
28
|
-
end
|
@@ -14,21 +14,22 @@ class SforceInputTest < Test::Unit::TestCase
|
|
14
14
|
polling_interval 60
|
15
15
|
]
|
16
16
|
|
17
|
-
def create_driver(conf = CONFIG)
|
18
|
-
Fluent::Test::InputTestDriver.new(Fluent::SforceInput).configure(conf)
|
19
|
-
end
|
20
|
-
|
21
17
|
def test_configure
|
22
18
|
d = create_driver
|
23
19
|
|
24
|
-
assert_equal
|
25
|
-
assert_equal
|
26
|
-
assert_equal
|
27
|
-
assert_equal
|
20
|
+
assert_equal 'test@example.com', d.instance.username
|
21
|
+
assert_equal 'hogefuga', d.instance.password
|
22
|
+
assert_equal 'test.access', d.instance.tag
|
23
|
+
assert_equal 'SELECT id, Name FROM Account', d.instance.query
|
28
24
|
assert_equal 60, d.instance.polling_interval
|
29
25
|
end
|
30
26
|
|
31
|
-
#
|
32
|
-
|
33
|
-
|
27
|
+
# TODO
|
28
|
+
def test_emit; end
|
29
|
+
|
30
|
+
private
|
31
|
+
|
32
|
+
def create_driver(conf = CONFIG)
|
33
|
+
Fluent::Test::InputTestDriver.new(Fluent::Plugin::SforceInput).configure(conf)
|
34
|
+
end
|
34
35
|
end
|
metadata
CHANGED
@@ -1,29 +1,29 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-sforce
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Makoto Tajitsu
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2019-01-12 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- - "
|
17
|
+
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version:
|
19
|
+
version: '0'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- - "
|
24
|
+
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version:
|
26
|
+
version: '0'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: restforce
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
@@ -80,6 +80,20 @@ dependencies:
|
|
80
80
|
- - ">="
|
81
81
|
- !ruby/object:Gem::Version
|
82
82
|
version: '0'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: test-unit
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ">="
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0'
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0'
|
83
97
|
description: Fluent Plugin to export data from Salesforce.com.
|
84
98
|
email:
|
85
99
|
- makoto_tajitsu@hotmail.co.jp
|
@@ -91,6 +105,7 @@ files:
|
|
91
105
|
- ".travis.yml"
|
92
106
|
- Gemfile
|
93
107
|
- LICENSE.txt
|
108
|
+
- Makefile
|
94
109
|
- README.md
|
95
110
|
- Rakefile
|
96
111
|
- VERSION
|
@@ -117,7 +132,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
117
132
|
version: '0'
|
118
133
|
requirements: []
|
119
134
|
rubyforge_project:
|
120
|
-
rubygems_version: 2.7.
|
135
|
+
rubygems_version: 2.7.7
|
121
136
|
signing_key:
|
122
137
|
specification_version: 4
|
123
138
|
summary: Fluent Plugin to export data from Salesforce.com.
|