i2x 0.0.4 → 0.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/i2x.gemspec +1 -0
- data/lib/i2x.rb +27 -13
- data/lib/i2x/agent.rb +19 -22
- data/lib/i2x/cashier.rb +5 -8
- data/lib/i2x/client.rb +4 -5
- data/lib/i2x/csvdetector.rb +23 -15
- data/lib/i2x/detector.rb +18 -14
- data/lib/i2x/jsondetector.rb +1 -1
- data/lib/i2x/version.rb +2 -2
- metadata +16 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5ae96c687e670ea870c7ee230796a996d6948450
|
4
|
+
data.tar.gz: cf2653409419ae955fb4b3b5f5c04a18011c28a2
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 25fcfd7a5936da1ae93be656b7855409f119e550d5cce7c60dd631b6ba001007d85b092bcd5808bf9c5df856ee2e0fa1f6aee1105f9d2320c20a77fe770565e6
|
7
|
+
data.tar.gz: 3d8f44e8e692b2ede996ed044c1c2f7110cbc817e6a03369fae9b2592dd98782aa2b3cd30791627b43ff89d648ccbe4a9b8e8484e202711ff0dc1ce98e24c588
|
data/i2x.gemspec
CHANGED
data/lib/i2x.rb
CHANGED
@@ -1,18 +1,32 @@
|
|
1
|
-
|
2
|
-
|
1
|
+
require 'i2x/cashier'
|
2
|
+
require 'i2x/helper'
|
3
|
+
require 'i2x/detector'
|
4
|
+
require 'i2x/csvdetector'
|
5
|
+
require 'i2x/jsondetector'
|
6
|
+
require 'i2x/sqldetector'
|
7
|
+
require 'i2x/xmldetector'
|
8
|
+
require 'i2x/agent'
|
9
|
+
require 'i2x/version'
|
10
|
+
require 'i2x/client'
|
3
11
|
|
4
12
|
module I2X
|
5
|
-
|
6
|
-
attr_accessor :access_token
|
7
|
-
attr_accessor :host
|
13
|
+
class Config
|
8
14
|
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
15
|
+
def self.set_host host
|
16
|
+
host << '/' unless host.end_with?('/')
|
17
|
+
@@host = host
|
18
|
+
end
|
13
19
|
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
20
|
+
def self.set_access_token api_key
|
21
|
+
@@access_token = api_key
|
22
|
+
end
|
23
|
+
|
24
|
+
def self.host
|
25
|
+
@@host
|
26
|
+
end
|
27
|
+
|
28
|
+
def self.access_token
|
29
|
+
@@access_token
|
30
|
+
end
|
31
|
+
end
|
18
32
|
end
|
data/lib/i2x/agent.rb
CHANGED
@@ -6,20 +6,18 @@
|
|
6
6
|
|
7
7
|
module I2X
|
8
8
|
class Agent
|
9
|
-
attr_accessor :content
|
10
|
-
attr_accessor :identifier
|
11
|
-
attr_accessor :publisher
|
12
|
-
attr_accessor :payload
|
13
|
-
attr_accessor :templates
|
9
|
+
attr_accessor :content, :identifier, :publisher, :payload, :templates, :seeds, :cache, :selectors
|
14
10
|
|
15
11
|
def initialize agent
|
16
12
|
begin
|
17
|
-
identifier = agent[:identifier]
|
18
|
-
publisher = agent[:publisher]
|
13
|
+
@identifier = agent[:identifier]
|
14
|
+
@publisher = agent[:publisher]
|
19
15
|
@payload = agent[:payload]
|
20
|
-
|
16
|
+
@cache = agent[:payload][:cache]
|
17
|
+
@seeds = agent[:seeds]
|
18
|
+
@selectors = agent[:payload][:selectors]
|
21
19
|
rescue Exception => e
|
22
|
-
p
|
20
|
+
p "[i2x] unable to initialize agent. #{e}"
|
23
21
|
end
|
24
22
|
|
25
23
|
end
|
@@ -31,34 +29,34 @@ module I2X
|
|
31
29
|
def execute
|
32
30
|
@checkup = {}
|
33
31
|
|
34
|
-
case publisher
|
32
|
+
case @publisher
|
35
33
|
when 'sql'
|
36
34
|
begin
|
37
|
-
@d = I2X::SQLDetector.new(
|
35
|
+
@d = I2X::SQLDetector.new(self)
|
38
36
|
rescue Exception => e
|
39
37
|
@response = {:status => 400, :error => e}
|
40
|
-
p
|
38
|
+
p "[i2x] error: #{e}"
|
41
39
|
end
|
42
40
|
when 'csv'
|
43
41
|
begin
|
44
|
-
@d = I2X::CSVDetector.new(
|
42
|
+
@d = I2X::CSVDetector.new(self)
|
45
43
|
rescue Exception => e
|
46
44
|
@response = {:status => 400, :error => e}
|
47
|
-
p
|
45
|
+
p "[i2x] error: #{e}"
|
48
46
|
end
|
49
47
|
when 'xml'
|
50
48
|
begin
|
51
|
-
@d = I2X::XMLDetector.new(
|
49
|
+
@d = I2X::XMLDetector.new(self)
|
52
50
|
rescue Exception => e
|
53
51
|
@response = {:status => 400, :error => e}
|
54
|
-
|
52
|
+
p "[i2x] error: #{e}"
|
55
53
|
end
|
56
54
|
when 'json'
|
57
55
|
begin
|
58
|
-
@d = I2X::JSONDetector.new(
|
56
|
+
@d = I2X::JSONDetector.new(self)
|
59
57
|
rescue Exception => e
|
60
58
|
@response = {:status => 400, :error => e}
|
61
|
-
p
|
59
|
+
p "[i2x] error: #{e}"
|
62
60
|
end
|
63
61
|
end
|
64
62
|
|
@@ -68,10 +66,9 @@ module I2X
|
|
68
66
|
unless content.nil? then
|
69
67
|
@d.content = content
|
70
68
|
end
|
71
|
-
update_check_at Time.now
|
72
69
|
@checkup = @d.checkup
|
73
70
|
rescue Exception => e
|
74
|
-
p
|
71
|
+
p "[i2x] error: #{e}"
|
75
72
|
end
|
76
73
|
|
77
74
|
# Start detection
|
@@ -80,7 +77,7 @@ module I2X
|
|
80
77
|
@d.detect object
|
81
78
|
end
|
82
79
|
rescue Exception => e
|
83
|
-
p
|
80
|
+
p "[i2x] error: #{e}"
|
84
81
|
end
|
85
82
|
|
86
83
|
begin
|
@@ -88,7 +85,7 @@ module I2X
|
|
88
85
|
process @checkup
|
89
86
|
end
|
90
87
|
rescue Exception => e
|
91
|
-
p
|
88
|
+
p "[i2x] error: #{e}"
|
92
89
|
end
|
93
90
|
response = {:status => @checkup[:status], :message => "[i2x][Checkup][execute] All OK."}
|
94
91
|
end
|
data/lib/i2x/cashier.rb
CHANGED
@@ -13,22 +13,19 @@ module I2X
|
|
13
13
|
# => Verify if items have already been seen in the past (on the cache).
|
14
14
|
#
|
15
15
|
# == Params
|
16
|
-
# - *
|
16
|
+
# - *cache*: the key identifier to be verified
|
17
17
|
# - *payload*: the value for matching/verification
|
18
18
|
# - *agent*: the agent performing the verification
|
19
19
|
# - *seed*: seed data (if available)
|
20
20
|
#
|
21
|
-
def self.verify
|
22
|
-
|
21
|
+
def self.verify cache, agent, payload, seed
|
22
|
+
#puts "[i2x][Cashier] verifying\n\taccess token: #{I2X::Config.access_token}\n\thost: #{I2X::Config.host}\n\tcache: #{cache}\n\tagent: #{agent}\n\tpayload: #{payload}\n\tseed: #{seed}"
|
23
23
|
begin
|
24
|
-
response = RestClient.post "#{I2X::Config.host}fluxcapacitor/verify.json", {:access_token => I2X::Config.access_token, :agent => agent, :
|
24
|
+
response = RestClient.post "#{I2X::Config.host}fluxcapacitor/verify.json", {:access_token => I2X::Config.access_token, :agent => agent[:identifier], :cache => cache, :payload => payload, :seed => seed}
|
25
25
|
rescue Exception => e
|
26
26
|
response = {:status => 400}
|
27
27
|
end
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
28
|
+
p response
|
32
29
|
response
|
33
30
|
end
|
34
31
|
end
|
data/lib/i2x/client.rb
CHANGED
@@ -13,7 +13,7 @@ module I2X
|
|
13
13
|
I2X::Config.set_host @config[:server][:host]
|
14
14
|
p '[i2x] loaded configuration'
|
15
15
|
rescue Exception => e
|
16
|
-
|
16
|
+
puts "[i2x] Failed to load configuration: #{e}"
|
17
17
|
end
|
18
18
|
|
19
19
|
end
|
@@ -24,11 +24,10 @@ module I2X
|
|
24
24
|
def validate
|
25
25
|
begin
|
26
26
|
p '[i2x] launching validation.'
|
27
|
-
|
28
|
-
out = RestClient.post "#{@config[:server][:host]}fluxcapacitor/validate_key.json", {:access_token => @config[:server][:api_key]}
|
27
|
+
out = RestClient.post "#{I2X::Config.host}fluxcapacitor/validate_key.json", {:access_token => I2X::Config.access_token}
|
29
28
|
response = {:status => 100, :response => out.to_str}
|
30
29
|
rescue Exception => e
|
31
|
-
p
|
30
|
+
p "[i2x] Failed validation: #{e}"
|
32
31
|
end
|
33
32
|
response
|
34
33
|
end
|
@@ -43,7 +42,7 @@ module I2X
|
|
43
42
|
a.execute
|
44
43
|
end
|
45
44
|
rescue Exception => e
|
46
|
-
p
|
45
|
+
p "[i2x] Failed agent processing: #{e}"
|
47
46
|
end
|
48
47
|
end
|
49
48
|
end
|
data/lib/i2x/csvdetector.rb
CHANGED
@@ -1,11 +1,5 @@
|
|
1
|
-
#require 'helper'
|
2
1
|
require 'csv'
|
3
2
|
require 'open-uri'
|
4
|
-
#require 'seedreader'
|
5
|
-
#require 'csvseedreader'
|
6
|
-
#require 'sqlseedreader'
|
7
|
-
#require 'xmlseedreader'
|
8
|
-
#require 'jsonseedreader'
|
9
3
|
|
10
4
|
module I2X
|
11
5
|
|
@@ -21,20 +15,35 @@ module I2X
|
|
21
15
|
# == Detect the changes
|
22
16
|
#
|
23
17
|
def detect object
|
24
|
-
|
25
|
-
|
18
|
+
|
19
|
+
p "[i2x][CSV] Testing #{object[:uri]}"
|
20
|
+
CSV.new(open(object[:uri]), :headers => :first_row).each do |row|
|
21
|
+
begin
|
26
22
|
unless object[:cache].nil? then
|
27
|
-
|
23
|
+
p "[i2x][CSV] with cache, verifying"
|
24
|
+
@response = Cashier.verify row[object[:cache].to_i], object, row, object[:seed]
|
25
|
+
|
28
26
|
else
|
27
|
+
p "[i2x][CSV] no cache, verifying"
|
29
28
|
@cache = Cashier.verify row[0], object, row, object[:seed]
|
29
|
+
|
30
30
|
end
|
31
|
+
rescue Exception => e
|
32
|
+
p "[i2x][CSVDetector] loading error: #{e.inspect}"
|
33
|
+
end
|
34
|
+
|
35
|
+
begin
|
36
|
+
|
37
|
+
@cache = JSON.parse(@response, {:symbolize_names => true})
|
38
|
+
@templates = @cache[:templates]
|
31
39
|
# The actual processing
|
32
40
|
#
|
33
|
-
if @cache[:status] == 100 then
|
41
|
+
if @cache[:cache][:status] == 100 then
|
34
42
|
|
35
43
|
# add row data to payload from selectors (key => key, value => column name)
|
36
44
|
payload = Hash.new
|
37
|
-
|
45
|
+
|
46
|
+
object[:selectors].each do |selector|
|
38
47
|
selector.each do |k,v|
|
39
48
|
payload[k] = row[v.to_i]
|
40
49
|
end
|
@@ -42,12 +51,11 @@ module I2X
|
|
42
51
|
# add payload object to payloads list
|
43
52
|
@payloads.push payload
|
44
53
|
end
|
54
|
+
|
55
|
+
rescue Exception => e
|
56
|
+
p "[i2x][CSVDetector] processing error: #{e.inspect}"
|
45
57
|
end
|
46
|
-
rescue Exception => e
|
47
|
-
I2X::Slog.exception e
|
48
58
|
end
|
49
59
|
end
|
50
|
-
|
51
|
-
|
52
60
|
end
|
53
61
|
end
|
data/lib/i2x/detector.rb
CHANGED
@@ -13,14 +13,16 @@ module I2X
|
|
13
13
|
# Main change detection class, to be inherited by SQL, CSV, JSON and XML detectors (and others to come).
|
14
14
|
#
|
15
15
|
class Detector
|
16
|
-
attr_accessor :identifier, :agent, :objects, :payloads, :content
|
16
|
+
attr_accessor :identifier, :agent, :objects, :payloads, :content, :templates
|
17
17
|
|
18
|
-
def initialize
|
18
|
+
def initialize agent
|
19
19
|
begin
|
20
|
-
@agent =
|
20
|
+
@agent = agent
|
21
21
|
@payloads = Array.new
|
22
22
|
@objects = Array.new
|
23
23
|
@help = I2X::Helper.new
|
24
|
+
@templates = {}
|
25
|
+
puts "Loaded new detector: #{agent.identifier}"
|
24
26
|
rescue Exception => e
|
25
27
|
|
26
28
|
end
|
@@ -31,15 +33,14 @@ module I2X
|
|
31
33
|
# == Start original source detection process
|
32
34
|
#
|
33
35
|
def checkup
|
34
|
-
# update checkup time
|
35
|
-
@agent.update_check_at @help.datetime
|
36
36
|
|
37
37
|
begin
|
38
38
|
|
39
|
+
|
39
40
|
##
|
40
41
|
# => Process seed data, if available.
|
41
42
|
#
|
42
|
-
|
43
|
+
unless @agent.seeds.nil? then
|
43
44
|
@agent.seeds.each do |seed|
|
44
45
|
case seed[:publisher]
|
45
46
|
when 'csv'
|
@@ -73,33 +74,36 @@ module I2X
|
|
73
74
|
@objects.push read
|
74
75
|
end
|
75
76
|
rescue Exception => e
|
76
|
-
|
77
|
+
p "[i2x] error: #{e}"
|
77
78
|
end
|
78
79
|
end
|
79
80
|
|
80
81
|
else
|
81
82
|
##
|
82
83
|
# no seeds, simply copy agent data
|
83
|
-
object = @help.deep_copy @agent
|
84
|
-
object[:identifier] = @agent
|
84
|
+
object = @help.deep_copy @agent.payload
|
85
|
+
object[:identifier] = @agent.identifier
|
86
|
+
object[:cache] = @agent.cache
|
85
87
|
object[:seed] = object[:identifier]
|
86
|
-
|
88
|
+
object[:selectors] = @agent.selectors
|
89
|
+
p "\n\tSelectors: #{object[:selectors]}"
|
90
|
+
unless self.content.nil? then
|
87
91
|
object[:content] = self.content
|
88
92
|
end
|
89
93
|
@objects.push object
|
90
94
|
end
|
91
95
|
rescue Exception => e
|
92
96
|
@response = {:status => 404, :message => "[i2x][Detector] failed to load doc, #{e}"}
|
93
|
-
|
97
|
+
p "[i2x] error: #{e}"
|
94
98
|
end
|
95
99
|
|
96
100
|
begin
|
97
101
|
# increase detected events count
|
98
|
-
|
99
|
-
@response = { :payload => @payloads, :status => 100}
|
102
|
+
|
103
|
+
@response = { :payload => @payloads, :templates => @templates, :status => 100}
|
100
104
|
rescue Exception => e
|
101
105
|
@response = {:status => 404, :message => "[i2x][Detector] failed to process queries, #{e}"}
|
102
|
-
|
106
|
+
p "[i2x] error: #{e}"
|
103
107
|
end
|
104
108
|
@response
|
105
109
|
end
|
data/lib/i2x/jsondetector.rb
CHANGED
data/lib/i2x/version.rb
CHANGED
@@ -1,3 +1,3 @@
|
|
1
1
|
module I2X
|
2
|
-
VERSION = "0.0.
|
3
|
-
end
|
2
|
+
VERSION = "0.0.5"
|
3
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: i2x
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Pedro Lopes
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-01-
|
11
|
+
date: 2014-01-21 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -66,6 +66,20 @@ dependencies:
|
|
66
66
|
- - ">="
|
67
67
|
- !ruby/object:Gem::Version
|
68
68
|
version: '0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: jsonpath
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - "~>"
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :runtime
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - "~>"
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
69
83
|
description: 'i2x: integrate everything. Automated real-time integration framework.'
|
70
84
|
email:
|
71
85
|
- hello@pedrolopes.net
|