data_collector 0.41.1 → 0.42.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7129e581e83ed3d1d2ac2a7df0f755d55b278c99a3caf83ee1d1b93ff143b440
-  data.tar.gz: 2914af4cd633fa592a770d579e8d27d5d69a9c0559829161ff210a26040fe498
+  metadata.gz: 882f493242fb171054818f0afe724c6389920c1a9590777a3d693a5638a3a0c3
+  data.tar.gz: 28b47fcd8de7c19d407cfeb49d7ca24a2f9199f88dfa167dce39fbb86fb3744c
 SHA512:
-  metadata.gz: 2629bcf0144699d9196a28884d30af0db03a0afd28392114e66df80e924a2e1945005f0c0ead108e015487c78050d7485d03a123a6822635413462f09c3f3930
-  data.tar.gz: 927b22389fcebad4e66f446a5ed131b85e05fde2aa6fa3343f3382112c5eed45f4a6c787f4d845126fc9c46f66cf4a7c9bb9abec170ec49f200fda41b3b36274
+  metadata.gz: 91c0b362b95077a286ad47ed526889e10a053dd93c691963f2957350e9d5b956d4703d893bcce507db6f60feb35f5a8026d260e90c83bbcd7ceb657143be6df9
+  data.tar.gz: 6a4c1340f09916a83a351330ae7ede7f4b0538a5f8aae84ea365409dafb66a033df421baf27282dc22ccedee7c165f0a5e1365a69e680c127b2c6bbacfb7a045

data/lib/data_collector/ext/base.rb

@@ -1,37 +1,41 @@
-require 'connection_pool'
+require 'securerandom'
+#require 'connection_pool'
 
 module BunnyBurrow
-  class Connection
-    include Singleton
-    attr_reader :connection
-    attr_accessor :verify_peer, :connection_name, :rabbitmq_url
-
-    def connection
-      unless @connection
-        @connection = Bunny.new(@rabbitmq_url, verify_peer: @verify_peer, connection_name: @connection_name)
-        @connection.start
-      end
-
-      @connection.start unless @connection.connected? || @connection.closed?
-      #@connection.start if @connection.closed?
-
-      #pp @connection.status
-
-      @connection
-    end
-
-    def channel
-      @channel = connection.create_channel unless @channel && @channel.open?
-
-      @channel
-    end
-  end
+  # class Connection
+  #   include Singleton
+  #   attr_reader :connection
+  #   attr_accessor :verify_peer, :connection_name, :rabbitmq_url
+  #
+  #   def connection
+  #     unless @connection
+  #       @connection = Bunny.new(@rabbitmq_url, verify_peer: @verify_peer, connection_name: @connection_name)
+  #       @connection.start
+  #     end
+  #
+  #     @connection.start unless @connection.connected? || @connection.closed?
+  #     #@connection.start if @connection.closed?
+  #
+  #     #pp @connection.status
+  #
+  #     @connection
+  #   end
+  #
+  #   def channel
+  #     @channel = connection.create_channel unless @channel && @channel.open?
+  #
+  #     @channel
+  #   end
+  # end
 
   class Base
     attr_accessor(
       :connection_name
     )
 
+    def initialize
+      @connection_name = "#{Process.pid}-#{SecureRandom.uuid}"
+    end
     # private
     #
     # def connection

data/lib/data_collector/ext/nokogiri.rb

@@ -0,0 +1,36 @@
+require 'nokogiri'
+require 'active_support/core_ext/hash/indifferent_access'
+
+# Mix-in of the Nokogiri XML Node Class that implements methods to convert
+# nodes to a hash.
+# Author: Mario "Kuroir" Ricalde (http://kuroir.com)
+class Nokogiri::XML::Node
+  # Convert a selected node to a Hash. It accepts a CSS3 Selector as an attribute.
+  # Returns the hash.
+  def to_hash(selector = 'body > *')
+    hash = []
+    self.css(selector).each do |node|
+      hash << node.collect_nodes
+    end
+    # Return the hash
+    hash.map{|m| m.is_a?(Hash) ? m.with_indifferent_access : m}
+  end
+
+  # Recursive method to collect nodes.
+  # We add the children symbol always to keep the array structure constant.
+  def collect_nodes
+    { self.name.to_sym => self.collect_attributes.merge({:children => collect_children }) }
+  end
+
+  # Collect Attributes of a given node.
+  def collect_attributes
+    output = {}
+    self.attributes.each { |name, value| output = output.merge({ name.to_sym => value.to_s.split(/\s+/) }) }
+    output
+  end
+
+  # Priest method.
+  def collect_children
+    self.element_children.collect { |child| child.collect_nodes } || []
+  end
+end

data/lib/data_collector/input.rb

@@ -2,6 +2,7 @@
 require 'http'
 require 'open-uri'
 require 'nokogiri'
+require_relative 'ext/nokogiri'
 require 'linkeddata'
 require 'nori'
 require 'uri'
@@ -15,6 +16,7 @@ require 'csv'
 require_relative 'input/dir'
 require_relative 'input/queue'
 require_relative 'input/rpc'
+require 'base64'
 
 # require_relative 'ext/xml_utility_node'
 module DataCollector
@@ -166,11 +168,16 @@ module DataCollector
             data = xml_to_hash(data, options)
           when 'text/xml'
             data = xml_to_hash(data, options)
+          when 'text/html'
+            data = html_to_hash(data, options)
           when 'text/turtle'
             graph = RDF::Graph.new do |graph|
               RDF::Turtle::Reader.new(data) {|reader| graph << reader}
             end
             data = JSON.parse(graph.dump(:jsonld, validate: false, standard_prefixes: true))
+          when /^image/
+            options['file_type'] = file_type
+            data = image_to_data(data, options)
           else
             data = xml_to_hash(data, options)
           end
@@ -196,6 +203,8 @@ module DataCollector
       data = nil
       uri = normalize_uri(uri)
       absolute_path = File.absolute_path(uri)
+      file_type = MIME::Types.type_for(uri).first.to_s
+      options['file_type'] = file_type
       raise DataCollector::Error, "#{uri.to_s} not found" unless File.exist?("#{absolute_path}")
       unless options.has_key?('raw') && options['raw'] == true
         @raw = data = File.read("#{absolute_path}")
@@ -206,6 +215,8 @@ module DataCollector
           data = JSON.parse(data)
         when '.xml'
           data = xml_to_hash(data, options)
+        when '.html'
+          data = html_to_hash(data, options)
         when '.gz'
           tar_data = []
           Minitar.open(Zlib::GzipReader.new(File.open("#{absolute_path}", 'rb'))) do |i|
@@ -232,6 +243,8 @@ module DataCollector
           end #tar
         when '.csv'
           data = csv_to_hash(data, options)
+        when '.jpg', '.png', '.gif'
+          data = image_to_data(data, options)
         else
           raise "Do not know how to process #{uri.to_s}"
         end
@@ -252,6 +265,11 @@ module DataCollector
       DataCollector::Input::Rpc.new(uri, options)
     end
 
+    def image_to_data(data, options = {})
+      file_type = options['file_type']
+      "data:#{file_type};#{Base64.encode64(data)}"
+    end
+
     def xml_to_hash(data, options = {})
       # gsub('&lt;\/', '&lt; /') outherwise wrong XML-parsing (see records lirias1729192 )
       return unless data.is_a?(String)
@@ -264,6 +282,12 @@ module DataCollector
       nori.parse(data)
     end
 
+    def html_to_hash(data, options = {})
+      return unless data.is_a?(String)
+      html_data = Nokogiri::HTML(data)
+      html_data.to_hash
+    end
+
     def csv_to_hash(data, options = {})
       csv_option_keys = options.keys & CSV::DEFAULT_OPTIONS.keys
       all_cvs_options = {headers: true, header_converters: [:downcase, :symbol]}

data/lib/data_collector/version.rb

@@ -1,4 +1,4 @@
 # encoding: utf-8
 module DataCollector
-  VERSION = "0.41.1"
+  VERSION = "0.42.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: data_collector
 version: !ruby/object:Gem::Version
-  version: 0.41.1
+  version: 0.42.0
 platform: ruby
 authors:
 - Mehmet Celik
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-12-04 00:00:00.000000000 Z
+date: 2024-01-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -311,6 +311,7 @@ files:
 - lib/data_collector/config_file.rb
 - lib/data_collector/core.rb
 - lib/data_collector/ext/base.rb
+- lib/data_collector/ext/nokogiri.rb
 - lib/data_collector/ext/xml_utility_node.rb
 - lib/data_collector/input.rb
 - lib/data_collector/input/dir.rb