i2x 0.0.4 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: aeb7e0cf94104c029d30e1202b2dedd3241613b7
-  data.tar.gz: 25dbf9cc1bf9ea0f1ffcadc8c69d2676d6e11569
+  metadata.gz: 5ae96c687e670ea870c7ee230796a996d6948450
+  data.tar.gz: cf2653409419ae955fb4b3b5f5c04a18011c28a2
 SHA512:
-  metadata.gz: 1e4a6bf452b40dc35d13e7f1d79d0b829c0b490d26582b28b6f2360aa34abea28dd4c7c71d7e2d71ac2418d411e6f16f80c955271510173ec02f7075e5267be6
-  data.tar.gz: f5d5f737ae6fdfb520d1e5e8275a8f34bf9833365367d0d97e02189f3f6dbbde5bb1e0f369e76695253649afbdcc84fc39e8540dc4baaa61d76fd9fd3ec29980
+  metadata.gz: 25fcfd7a5936da1ae93be656b7855409f119e550d5cce7c60dd631b6ba001007d85b092bcd5808bf9c5df856ee2e0fa1f6aee1105f9d2320c20a77fe770565e6
+  data.tar.gz: 3d8f44e8e692b2ede996ed044c1c2f7110cbc817e6a03369fae9b2592dd98782aa2b3cd30791627b43ff89d648ccbe4a9b8e8484e202711ff0dc1ce98e24c588

data/i2x.gemspec

@@ -22,4 +22,5 @@ Gem::Specification.new do |spec|
   
   spec.add_runtime_dependency 'rest-client', '~> 0'
   spec.add_runtime_dependency 'nokogiri'
+  spec.add_runtime_dependency 'jsonpath', '~> 0'
 end

data/lib/i2x.rb

@@ -1,18 +1,32 @@
-current_folder = File.expand_path('../', __FILE__) 
-Dir["#{current_folder}/i2x/*.rb"].each {|f| require f}
+require 'i2x/cashier'
+require 'i2x/helper'
+require 'i2x/detector'
+require 'i2x/csvdetector'
+require 'i2x/jsondetector'
+require 'i2x/sqldetector'
+require 'i2x/xmldetector'
+require 'i2x/agent'
+require 'i2x/version'
+require 'i2x/client'
 
 module I2X
-  class Config
-  	attr_accessor :access_token
-  	attr_accessor :host
+	class Config
 
-  	def self.set_host host
-  		host << '/' unless host.ends_with?('/')
-  		@@host = host
-  	end
+		def self.set_host host
+			host << '/' unless host.end_with?('/')
+			@@host = host
+		end
 
-  	def self.set_access_token api_key
-  		@@access_token = api_key
-  	end
-  end
+		def self.set_access_token api_key
+			@@access_token = api_key
+		end
+
+		def self.host
+			@@host
+		end
+
+		def self.access_token
+			@@access_token
+		end
+	end
 end

data/lib/i2x/agent.rb

@@ -6,20 +6,18 @@
 
 module I2X
   class Agent 
-    attr_accessor :content
-    attr_accessor :identifier
-    attr_accessor :publisher
-    attr_accessor :payload
-    attr_accessor :templates
+    attr_accessor :content, :identifier, :publisher, :payload, :templates, :seeds, :cache, :selectors
 
     def initialize agent
       begin
-        identifier = agent[:identifier]
-        publisher = agent[:publisher]
+        @identifier = agent[:identifier]
+        @publisher = agent[:publisher]
         @payload = agent[:payload]
-        templates = agent[:template]
+        @cache = agent[:payload][:cache]
+        @seeds = agent[:seeds]
+        @selectors = agent[:payload][:selectors]
       rescue Exception => e
-        p '[i2x] unable to initialize agent. ' + e.to_str
+        p "[i2x] unable to initialize agent. #{e}"
       end
 
     end
@@ -31,34 +29,34 @@ module I2X
     def execute
       @checkup = {}
 
-      case publisher
+      case @publisher
       when 'sql'
         begin
-          @d = I2X::SQLDetector.new(identifier)
+          @d = I2X::SQLDetector.new(self)
         rescue Exception => e
           @response = {:status => 400, :error => e}
-          p '[i2x] error: ' + e.to_str
+          p "[i2x] error: #{e}"
         end
       when 'csv'
         begin
-          @d = I2X::CSVDetector.new(identifier)
+          @d = I2X::CSVDetector.new(self)
         rescue Exception => e
           @response = {:status => 400, :error => e}
-          p '[i2x] error: ' + e.to_str
+          p "[i2x] error: #{e}"
         end
       when 'xml'
         begin
-          @d = I2X::XMLDetector.new(identifier)
+          @d = I2X::XMLDetector.new(self)
         rescue Exception => e
           @response = {:status => 400, :error => e}
-          p '[i2x] error: ' + e.to_str
+         p "[i2x] error: #{e}"
         end
       when 'json'
         begin
-          @d = I2X::JSONDetector.new(identifier)
+          @d = I2X::JSONDetector.new(self)
         rescue Exception => e
           @response = {:status => 400, :error => e}
-          p '[i2x] error: ' + e.to_str
+          p "[i2x] error: #{e}"
         end
       end
 
@@ -68,10 +66,9 @@ module I2X
           unless content.nil? then
             @d.content = content
           end
-          update_check_at Time.now
           @checkup = @d.checkup
         rescue Exception => e
-          p '[i2x] error: ' + e.to_str
+          p "[i2x] error: #{e}"
         end
 
         # Start detection
@@ -80,7 +77,7 @@ module I2X
             @d.detect object
           end
         rescue Exception => e
-          p '[i2x] error: ' + e.to_str
+          p "[i2x] error: #{e}"
         end
 
         begin
@@ -88,7 +85,7 @@ module I2X
             process @checkup
           end
         rescue Exception => e
-          p '[i2x] error: ' + e.to_str
+          p "[i2x] error: #{e}"
         end
         response = {:status => @checkup[:status], :message => "[i2x][Checkup][execute] All OK."}     
       end

data/lib/i2x/cashier.rb

@@ -13,22 +13,19 @@ module I2X
     # => Verify if items have already been seen in the past (on the cache).
     #
     # == Params
-    # - *memory*: the key identifier to be verified
+    # - *cache*: the key identifier to be verified
     # - *payload*: the value for matching/verification
     # - *agent*: the agent performing the verification
     # - *seed*: seed data (if available)
     #
-    def self.verify memory, agent, payload, seed
-     
+    def self.verify cache, agent, payload, seed
+      #puts "[i2x][Cashier] verifying\n\taccess token: #{I2X::Config.access_token}\n\thost: #{I2X::Config.host}\n\tcache: #{cache}\n\tagent: #{agent}\n\tpayload: #{payload}\n\tseed: #{seed}"
       begin
-        response = RestClient.post "#{I2X::Config.host}fluxcapacitor/verify.json", {:access_token => I2X::Config.access_token, :agent => agent, :memory => memory, :payload => payload, :seed => seed}
+        response = RestClient.post "#{I2X::Config.host}fluxcapacitor/verify.json", {:access_token => I2X::Config.access_token, :agent => agent[:identifier], :cache => cache, :payload => payload, :seed => seed}
       rescue Exception => e
         response = {:status => 400}
       end
-      
-      
-      
-
+      p response
       response
     end
   end

data/lib/i2x/client.rb

@@ -13,7 +13,7 @@ module I2X
 				I2X::Config.set_host @config[:server][:host]
 				p '[i2x] loaded configuration'
 			rescue Exception => e
-				p '[i2x] Failed to load configuration' + e.to_str
+				puts "[i2x] Failed to load configuration: #{e}"
 			end
 			
 		end
@@ -24,11 +24,10 @@ module I2X
 		def validate
 			begin
 				p '[i2x] launching validation.'
-				@config[:server][:host] << '/' unless @config[:server][:host].ends_with?('/')
-				out = RestClient.post "#{@config[:server][:host]}fluxcapacitor/validate_key.json", {:access_token => @config[:server][:api_key]}	
+				out = RestClient.post "#{I2X::Config.host}fluxcapacitor/validate_key.json", {:access_token => I2X::Config.access_token}	
 				response = {:status => 100, :response => out.to_str}
 			rescue Exception => e
-				p '[i2x] validation failed. ' + e.to_str
+				p "[i2x] Failed validation: #{e}"
 			end
 			response
 		end
@@ -43,7 +42,7 @@ module I2X
 					a.execute
 				end
 			rescue Exception => e
-				p '[i2x] agent processing failed. ' + e.to_str
+				p "[i2x] Failed agent processing: #{e}"
 			end
 		end
 	end

data/lib/i2x/csvdetector.rb

@@ -1,11 +1,5 @@
-#require 'helper'
 require 'csv'
 require 'open-uri'
-#require 'seedreader'
-#require 'csvseedreader'
-#require 'sqlseedreader'
-#require 'xmlseedreader'
-#require 'jsonseedreader'
 
 module I2X
 
@@ -21,20 +15,35 @@ module I2X
     # == Detect the changes
     #
     def detect object
-      begin
-        CSV.new(open(object[:uri]), :headers => :first_row).each do |row|
+
+      p "[i2x][CSV] Testing #{object[:uri]}"
+      CSV.new(open(object[:uri]), :headers => :first_row).each do |row|
+        begin
           unless object[:cache].nil? then
-            @cache = Cashier.verify row[object[:cache].to_i], object, row, object[:seed]
+            p "[i2x][CSV] with cache, verifying"
+            @response = Cashier.verify row[object[:cache].to_i], object, row, object[:seed]
+            
           else
+            p "[i2x][CSV] no cache, verifying"
             @cache = Cashier.verify row[0], object, row, object[:seed]
+            
           end
+        rescue Exception => e
+          p "[i2x][CSVDetector] loading error: #{e.inspect}"
+        end
+
+        begin
+
+          @cache = JSON.parse(@response, {:symbolize_names => true})
+          @templates = @cache[:templates]
           # The actual processing
           #
-          if @cache[:status] == 100 then
+          if @cache[:cache][:status] == 100 then
 
             # add row data to payload from selectors (key => key, value => column name)
             payload = Hash.new
-            JSON.parse(object[:selectors]).each do |selector|
+            
+            object[:selectors].each do |selector|
               selector.each do |k,v|
                 payload[k] = row[v.to_i]
               end
@@ -42,12 +51,11 @@ module I2X
             # add payload object to payloads list
             @payloads.push payload
           end
+
+        rescue Exception => e
+          p "[i2x][CSVDetector] processing error: #{e.inspect}"
         end
-      rescue Exception => e
-        I2X::Slog.exception e
       end
     end
-
-    
   end
 end

data/lib/i2x/detector.rb

@@ -13,14 +13,16 @@ module I2X
   # Main change detection class, to be inherited by SQL, CSV, JSON and XML detectors (and others to come).
   #
   class Detector
-    attr_accessor :identifier, :agent, :objects, :payloads, :content
+    attr_accessor :identifier, :agent, :objects, :payloads, :content, :templates
 
-    def initialize identifier
+    def initialize agent
       begin
-        @agent = Agent.find_by! identifier: identifier
+        @agent = agent
         @payloads = Array.new
         @objects = Array.new
         @help = I2X::Helper.new
+        @templates = {}
+        puts "Loaded new detector: #{agent.identifier}"
       rescue Exception => e
         
       end
@@ -31,15 +33,14 @@ module I2X
     # == Start original source detection process
     #
     def checkup
-      # update checkup time
-      @agent.update_check_at @help.datetime
 
       begin
 
+
         ##
         # => Process seed data, if available.
         #
-        if @agent.seeds.size != 0 then
+        unless @agent.seeds.nil? then
           @agent.seeds.each do |seed|
             case seed[:publisher]
             when 'csv'
@@ -73,33 +74,36 @@ module I2X
                 @objects.push read
               end
             rescue Exception => e
-              
+              p "[i2x] error: #{e}"
             end
           end
 
         else
           ##
           # no seeds, simply copy agent data
-          object = @help.deep_copy @agent[:payload]
-          object[:identifier] = @agent[:identifier]
+          object = @help.deep_copy @agent.payload 
+          object[:identifier] = @agent.identifier
+          object[:cache] = @agent.cache
           object[:seed] = object[:identifier]
-          unless self.content.nil? then
+          object[:selectors] = @agent.selectors
+          p "\n\tSelectors: #{object[:selectors]}"
+           unless self.content.nil? then
             object[:content] = self.content
           end
           @objects.push object
         end
       rescue Exception => e
         @response = {:status => 404, :message => "[i2x][Detector] failed to load doc, #{e}"}
-        I2X::Slog.exception e
+        p "[i2x] error: #{e}"
       end
 
       begin
         # increase detected events count
-        @agent.increment!(:events_count, @payloads.size)
-        @response = { :payload => @payloads, :status => 100}
+        
+        @response = { :payload => @payloads, :templates => @templates, :status => 100}
       rescue Exception => e
         @response = {:status => 404, :message => "[i2x][Detector] failed to process queries, #{e}"}
-        I2X::Slog.exception e
+        p "[i2x] error: #{e}"
       end
       @response
     end

data/lib/i2x/jsondetector.rb

@@ -1,7 +1,7 @@
 #require 'helper'
 require 'open-uri'
 require 'jsonpath'
-require 'rest-client'
+require 'rest_client'
 require 'csv'
 require 'json'
 #require 'seedreader'

data/lib/i2x/version.rb

@@ -1,3 +1,3 @@
 module I2X
-  VERSION = "0.0.4"
-end
+  VERSION = "0.0.5"
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: i2x
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors:
 - Pedro Lopes
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-01-20 00:00:00.000000000 Z
+date: 2014-01-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -66,6 +66,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: jsonpath
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 'i2x: integrate everything. Automated real-time integration framework.'
 email:
 - hello@pedrolopes.net