i2x 0.0.4 → 0.0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: aeb7e0cf94104c029d30e1202b2dedd3241613b7
4
- data.tar.gz: 25dbf9cc1bf9ea0f1ffcadc8c69d2676d6e11569
3
+ metadata.gz: 5ae96c687e670ea870c7ee230796a996d6948450
4
+ data.tar.gz: cf2653409419ae955fb4b3b5f5c04a18011c28a2
5
5
  SHA512:
6
- metadata.gz: 1e4a6bf452b40dc35d13e7f1d79d0b829c0b490d26582b28b6f2360aa34abea28dd4c7c71d7e2d71ac2418d411e6f16f80c955271510173ec02f7075e5267be6
7
- data.tar.gz: f5d5f737ae6fdfb520d1e5e8275a8f34bf9833365367d0d97e02189f3f6dbbde5bb1e0f369e76695253649afbdcc84fc39e8540dc4baaa61d76fd9fd3ec29980
6
+ metadata.gz: 25fcfd7a5936da1ae93be656b7855409f119e550d5cce7c60dd631b6ba001007d85b092bcd5808bf9c5df856ee2e0fa1f6aee1105f9d2320c20a77fe770565e6
7
+ data.tar.gz: 3d8f44e8e692b2ede996ed044c1c2f7110cbc817e6a03369fae9b2592dd98782aa2b3cd30791627b43ff89d648ccbe4a9b8e8484e202711ff0dc1ce98e24c588
data/i2x.gemspec CHANGED
@@ -22,4 +22,5 @@ Gem::Specification.new do |spec|
22
22
 
23
23
  spec.add_runtime_dependency 'rest-client', '~> 0'
24
24
  spec.add_runtime_dependency 'nokogiri'
25
+ spec.add_runtime_dependency 'jsonpath', '~> 0'
25
26
  end
data/lib/i2x.rb CHANGED
@@ -1,18 +1,32 @@
1
- current_folder = File.expand_path('../', __FILE__)
2
- Dir["#{current_folder}/i2x/*.rb"].each {|f| require f}
1
+ require 'i2x/cashier'
2
+ require 'i2x/helper'
3
+ require 'i2x/detector'
4
+ require 'i2x/csvdetector'
5
+ require 'i2x/jsondetector'
6
+ require 'i2x/sqldetector'
7
+ require 'i2x/xmldetector'
8
+ require 'i2x/agent'
9
+ require 'i2x/version'
10
+ require 'i2x/client'
3
11
 
4
12
  module I2X
5
- class Config
6
- attr_accessor :access_token
7
- attr_accessor :host
13
+ class Config
8
14
 
9
- def self.set_host host
10
- host << '/' unless host.ends_with?('/')
11
- @@host = host
12
- end
15
+ def self.set_host host
16
+ host << '/' unless host.end_with?('/')
17
+ @@host = host
18
+ end
13
19
 
14
- def self.set_access_token api_key
15
- @@access_token = api_key
16
- end
17
- end
20
+ def self.set_access_token api_key
21
+ @@access_token = api_key
22
+ end
23
+
24
+ def self.host
25
+ @@host
26
+ end
27
+
28
+ def self.access_token
29
+ @@access_token
30
+ end
31
+ end
18
32
  end
data/lib/i2x/agent.rb CHANGED
@@ -6,20 +6,18 @@
6
6
 
7
7
  module I2X
8
8
  class Agent
9
- attr_accessor :content
10
- attr_accessor :identifier
11
- attr_accessor :publisher
12
- attr_accessor :payload
13
- attr_accessor :templates
9
+ attr_accessor :content, :identifier, :publisher, :payload, :templates, :seeds, :cache, :selectors
14
10
 
15
11
  def initialize agent
16
12
  begin
17
- identifier = agent[:identifier]
18
- publisher = agent[:publisher]
13
+ @identifier = agent[:identifier]
14
+ @publisher = agent[:publisher]
19
15
  @payload = agent[:payload]
20
- templates = agent[:template]
16
+ @cache = agent[:payload][:cache]
17
+ @seeds = agent[:seeds]
18
+ @selectors = agent[:payload][:selectors]
21
19
  rescue Exception => e
22
- p '[i2x] unable to initialize agent. ' + e.to_str
20
+ p "[i2x] unable to initialize agent. #{e}"
23
21
  end
24
22
 
25
23
  end
@@ -31,34 +29,34 @@ module I2X
31
29
  def execute
32
30
  @checkup = {}
33
31
 
34
- case publisher
32
+ case @publisher
35
33
  when 'sql'
36
34
  begin
37
- @d = I2X::SQLDetector.new(identifier)
35
+ @d = I2X::SQLDetector.new(self)
38
36
  rescue Exception => e
39
37
  @response = {:status => 400, :error => e}
40
- p '[i2x] error: ' + e.to_str
38
+ p "[i2x] error: #{e}"
41
39
  end
42
40
  when 'csv'
43
41
  begin
44
- @d = I2X::CSVDetector.new(identifier)
42
+ @d = I2X::CSVDetector.new(self)
45
43
  rescue Exception => e
46
44
  @response = {:status => 400, :error => e}
47
- p '[i2x] error: ' + e.to_str
45
+ p "[i2x] error: #{e}"
48
46
  end
49
47
  when 'xml'
50
48
  begin
51
- @d = I2X::XMLDetector.new(identifier)
49
+ @d = I2X::XMLDetector.new(self)
52
50
  rescue Exception => e
53
51
  @response = {:status => 400, :error => e}
54
- p '[i2x] error: ' + e.to_str
52
+ p "[i2x] error: #{e}"
55
53
  end
56
54
  when 'json'
57
55
  begin
58
- @d = I2X::JSONDetector.new(identifier)
56
+ @d = I2X::JSONDetector.new(self)
59
57
  rescue Exception => e
60
58
  @response = {:status => 400, :error => e}
61
- p '[i2x] error: ' + e.to_str
59
+ p "[i2x] error: #{e}"
62
60
  end
63
61
  end
64
62
 
@@ -68,10 +66,9 @@ module I2X
68
66
  unless content.nil? then
69
67
  @d.content = content
70
68
  end
71
- update_check_at Time.now
72
69
  @checkup = @d.checkup
73
70
  rescue Exception => e
74
- p '[i2x] error: ' + e.to_str
71
+ p "[i2x] error: #{e}"
75
72
  end
76
73
 
77
74
  # Start detection
@@ -80,7 +77,7 @@ module I2X
80
77
  @d.detect object
81
78
  end
82
79
  rescue Exception => e
83
- p '[i2x] error: ' + e.to_str
80
+ p "[i2x] error: #{e}"
84
81
  end
85
82
 
86
83
  begin
@@ -88,7 +85,7 @@ module I2X
88
85
  process @checkup
89
86
  end
90
87
  rescue Exception => e
91
- p '[i2x] error: ' + e.to_str
88
+ p "[i2x] error: #{e}"
92
89
  end
93
90
  response = {:status => @checkup[:status], :message => "[i2x][Checkup][execute] All OK."}
94
91
  end
data/lib/i2x/cashier.rb CHANGED
@@ -13,22 +13,19 @@ module I2X
13
13
  # => Verify if items have already been seen in the past (on the cache).
14
14
  #
15
15
  # == Params
16
- # - *memory*: the key identifier to be verified
16
+ # - *cache*: the key identifier to be verified
17
17
  # - *payload*: the value for matching/verification
18
18
  # - *agent*: the agent performing the verification
19
19
  # - *seed*: seed data (if available)
20
20
  #
21
- def self.verify memory, agent, payload, seed
22
-
21
+ def self.verify cache, agent, payload, seed
22
+ #puts "[i2x][Cashier] verifying\n\taccess token: #{I2X::Config.access_token}\n\thost: #{I2X::Config.host}\n\tcache: #{cache}\n\tagent: #{agent}\n\tpayload: #{payload}\n\tseed: #{seed}"
23
23
  begin
24
- response = RestClient.post "#{I2X::Config.host}fluxcapacitor/verify.json", {:access_token => I2X::Config.access_token, :agent => agent, :memory => memory, :payload => payload, :seed => seed}
24
+ response = RestClient.post "#{I2X::Config.host}fluxcapacitor/verify.json", {:access_token => I2X::Config.access_token, :agent => agent[:identifier], :cache => cache, :payload => payload, :seed => seed}
25
25
  rescue Exception => e
26
26
  response = {:status => 400}
27
27
  end
28
-
29
-
30
-
31
-
28
+ p response
32
29
  response
33
30
  end
34
31
  end
data/lib/i2x/client.rb CHANGED
@@ -13,7 +13,7 @@ module I2X
13
13
  I2X::Config.set_host @config[:server][:host]
14
14
  p '[i2x] loaded configuration'
15
15
  rescue Exception => e
16
- p '[i2x] Failed to load configuration' + e.to_str
16
+ puts "[i2x] Failed to load configuration: #{e}"
17
17
  end
18
18
 
19
19
  end
@@ -24,11 +24,10 @@ module I2X
24
24
  def validate
25
25
  begin
26
26
  p '[i2x] launching validation.'
27
- @config[:server][:host] << '/' unless @config[:server][:host].ends_with?('/')
28
- out = RestClient.post "#{@config[:server][:host]}fluxcapacitor/validate_key.json", {:access_token => @config[:server][:api_key]}
27
+ out = RestClient.post "#{I2X::Config.host}fluxcapacitor/validate_key.json", {:access_token => I2X::Config.access_token}
29
28
  response = {:status => 100, :response => out.to_str}
30
29
  rescue Exception => e
31
- p '[i2x] validation failed. ' + e.to_str
30
+ p "[i2x] Failed validation: #{e}"
32
31
  end
33
32
  response
34
33
  end
@@ -43,7 +42,7 @@ module I2X
43
42
  a.execute
44
43
  end
45
44
  rescue Exception => e
46
- p '[i2x] agent processing failed. ' + e.to_str
45
+ p "[i2x] Failed agent processing: #{e}"
47
46
  end
48
47
  end
49
48
  end
@@ -1,11 +1,5 @@
1
- #require 'helper'
2
1
  require 'csv'
3
2
  require 'open-uri'
4
- #require 'seedreader'
5
- #require 'csvseedreader'
6
- #require 'sqlseedreader'
7
- #require 'xmlseedreader'
8
- #require 'jsonseedreader'
9
3
 
10
4
  module I2X
11
5
 
@@ -21,20 +15,35 @@ module I2X
21
15
  # == Detect the changes
22
16
  #
23
17
  def detect object
24
- begin
25
- CSV.new(open(object[:uri]), :headers => :first_row).each do |row|
18
+
19
+ p "[i2x][CSV] Testing #{object[:uri]}"
20
+ CSV.new(open(object[:uri]), :headers => :first_row).each do |row|
21
+ begin
26
22
  unless object[:cache].nil? then
27
- @cache = Cashier.verify row[object[:cache].to_i], object, row, object[:seed]
23
+ p "[i2x][CSV] with cache, verifying"
24
+ @response = Cashier.verify row[object[:cache].to_i], object, row, object[:seed]
25
+
28
26
  else
27
+ p "[i2x][CSV] no cache, verifying"
29
28
  @cache = Cashier.verify row[0], object, row, object[:seed]
29
+
30
30
  end
31
+ rescue Exception => e
32
+ p "[i2x][CSVDetector] loading error: #{e.inspect}"
33
+ end
34
+
35
+ begin
36
+
37
+ @cache = JSON.parse(@response, {:symbolize_names => true})
38
+ @templates = @cache[:templates]
31
39
  # The actual processing
32
40
  #
33
- if @cache[:status] == 100 then
41
+ if @cache[:cache][:status] == 100 then
34
42
 
35
43
  # add row data to payload from selectors (key => key, value => column name)
36
44
  payload = Hash.new
37
- JSON.parse(object[:selectors]).each do |selector|
45
+
46
+ object[:selectors].each do |selector|
38
47
  selector.each do |k,v|
39
48
  payload[k] = row[v.to_i]
40
49
  end
@@ -42,12 +51,11 @@ module I2X
42
51
  # add payload object to payloads list
43
52
  @payloads.push payload
44
53
  end
54
+
55
+ rescue Exception => e
56
+ p "[i2x][CSVDetector] processing error: #{e.inspect}"
45
57
  end
46
- rescue Exception => e
47
- I2X::Slog.exception e
48
58
  end
49
59
  end
50
-
51
-
52
60
  end
53
61
  end
data/lib/i2x/detector.rb CHANGED
@@ -13,14 +13,16 @@ module I2X
13
13
  # Main change detection class, to be inherited by SQL, CSV, JSON and XML detectors (and others to come).
14
14
  #
15
15
  class Detector
16
- attr_accessor :identifier, :agent, :objects, :payloads, :content
16
+ attr_accessor :identifier, :agent, :objects, :payloads, :content, :templates
17
17
 
18
- def initialize identifier
18
+ def initialize agent
19
19
  begin
20
- @agent = Agent.find_by! identifier: identifier
20
+ @agent = agent
21
21
  @payloads = Array.new
22
22
  @objects = Array.new
23
23
  @help = I2X::Helper.new
24
+ @templates = {}
25
+ puts "Loaded new detector: #{agent.identifier}"
24
26
  rescue Exception => e
25
27
 
26
28
  end
@@ -31,15 +33,14 @@ module I2X
31
33
  # == Start original source detection process
32
34
  #
33
35
  def checkup
34
- # update checkup time
35
- @agent.update_check_at @help.datetime
36
36
 
37
37
  begin
38
38
 
39
+
39
40
  ##
40
41
  # => Process seed data, if available.
41
42
  #
42
- if @agent.seeds.size != 0 then
43
+ unless @agent.seeds.nil? then
43
44
  @agent.seeds.each do |seed|
44
45
  case seed[:publisher]
45
46
  when 'csv'
@@ -73,33 +74,36 @@ module I2X
73
74
  @objects.push read
74
75
  end
75
76
  rescue Exception => e
76
-
77
+ p "[i2x] error: #{e}"
77
78
  end
78
79
  end
79
80
 
80
81
  else
81
82
  ##
82
83
  # no seeds, simply copy agent data
83
- object = @help.deep_copy @agent[:payload]
84
- object[:identifier] = @agent[:identifier]
84
+ object = @help.deep_copy @agent.payload
85
+ object[:identifier] = @agent.identifier
86
+ object[:cache] = @agent.cache
85
87
  object[:seed] = object[:identifier]
86
- unless self.content.nil? then
88
+ object[:selectors] = @agent.selectors
89
+ p "\n\tSelectors: #{object[:selectors]}"
90
+ unless self.content.nil? then
87
91
  object[:content] = self.content
88
92
  end
89
93
  @objects.push object
90
94
  end
91
95
  rescue Exception => e
92
96
  @response = {:status => 404, :message => "[i2x][Detector] failed to load doc, #{e}"}
93
- I2X::Slog.exception e
97
+ p "[i2x] error: #{e}"
94
98
  end
95
99
 
96
100
  begin
97
101
  # increase detected events count
98
- @agent.increment!(:events_count, @payloads.size)
99
- @response = { :payload => @payloads, :status => 100}
102
+
103
+ @response = { :payload => @payloads, :templates => @templates, :status => 100}
100
104
  rescue Exception => e
101
105
  @response = {:status => 404, :message => "[i2x][Detector] failed to process queries, #{e}"}
102
- I2X::Slog.exception e
106
+ p "[i2x] error: #{e}"
103
107
  end
104
108
  @response
105
109
  end
@@ -1,7 +1,7 @@
1
1
  #require 'helper'
2
2
  require 'open-uri'
3
3
  require 'jsonpath'
4
- require 'rest-client'
4
+ require 'rest_client'
5
5
  require 'csv'
6
6
  require 'json'
7
7
  #require 'seedreader'
data/lib/i2x/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module I2X
2
- VERSION = "0.0.4"
3
- end
2
+ VERSION = "0.0.5"
3
+ end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: i2x
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.4
4
+ version: 0.0.5
5
5
  platform: ruby
6
6
  authors:
7
7
  - Pedro Lopes
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2014-01-20 00:00:00.000000000 Z
11
+ date: 2014-01-21 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -66,6 +66,20 @@ dependencies:
66
66
  - - ">="
67
67
  - !ruby/object:Gem::Version
68
68
  version: '0'
69
+ - !ruby/object:Gem::Dependency
70
+ name: jsonpath
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - "~>"
74
+ - !ruby/object:Gem::Version
75
+ version: '0'
76
+ type: :runtime
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - "~>"
81
+ - !ruby/object:Gem::Version
82
+ version: '0'
69
83
  description: 'i2x: integrate everything. Automated real-time integration framework.'
70
84
  email:
71
85
  - hello@pedrolopes.net