data_collector 0.41.2 → 0.43.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6e47e1ddfa862a19d96aab23718b06aae7b0a84c9719a0662ea18ecae0f09c6b
-  data.tar.gz: 115717172899307b66783f1b8aa32615590d9fdf44d3d944cf532a5d9a941be2
+  metadata.gz: b2c8102350695f8cfc598856372d9efd4bfa8b9cf4c3f84110b6dfbfdff3ee62
+  data.tar.gz: 042bf4690116ba07f34459068e3afbc7e63f483c7e0518ecb1581833cb85b3ae
 SHA512:
-  metadata.gz: 3ee069f8134c3d90bc3832aaafd09812182f4b6e13f38dbbc18776d18f15f9de04f1d2d049ae61bb3b77fb2777a8fd33ce4ca933be546d83eac77a5d18baeae6
-  data.tar.gz: 83dfc12faa94aeb9aeecfd40be6c1ed74296f704f4650334dfc9099f6201671d4796c2166220160004c7fbb5a060ef09e5f29e9380c15fe953cae29931df8441
+  metadata.gz: ff4a4c20f156dadbd665ff330f9c9f057a20391b2fc4fbab45afe59bb3071fd69aaaf6c67521b37a3d5d61b581761fcb21432be405d402a3571c32e47ce86f1b
+  data.tar.gz: a52836f02b727c1ed679520492921bfed09892d79155bf56fa2b823792a04512cfae267d15c22210acc32a460a8f2ed2a350e6ece388e860a41b0fbb473e778d

data/lib/data_collector/ext/nokogiri.rb

@@ -0,0 +1,36 @@
+require 'nokogiri'
+require 'active_support/core_ext/hash/indifferent_access'
+
+# Mix-in of the Nokogiri XML Node Class that implements methods to convert
+# nodes to a hash.
+# Author: Mario "Kuroir" Ricalde (http://kuroir.com)
+class Nokogiri::XML::Node
+  # Convert a selected node to a Hash. It accepts a CSS3 Selector as an attribute.
+  # Returns the hash.
+  def to_hash(selector = 'body > *')
+    hash = []
+    self.css(selector).each do |node|
+      hash << node.collect_nodes
+    end
+    # Return the hash
+    hash.map{|m| m.is_a?(Hash) ? m.with_indifferent_access : m}
+  end
+
+  # Recursive method to collect nodes.
+  # We add the children symbol always to keep the array structure constant.
+  def collect_nodes
+    { self.name.to_sym => self.collect_attributes.merge({:children => collect_children }) }
+  end
+
+  # Collect Attributes of a given node.
+  def collect_attributes
+    output = {}
+    self.attributes.each { |name, value| output = output.merge({ name.to_sym => value.to_s.split(/\s+/) }) }
+    output
+  end
+
+  # Priest method.
+  def collect_children
+    self.element_children.collect { |child| child.collect_nodes } || []
+  end
+end

data/lib/data_collector/input.rb

@@ -2,6 +2,7 @@
 require 'http'
 require 'open-uri'
 require 'nokogiri'
+require_relative 'ext/nokogiri'
 require 'linkeddata'
 require 'nori'
 require 'uri'
@@ -15,6 +16,7 @@ require 'csv'
 require_relative 'input/dir'
 require_relative 'input/queue'
 require_relative 'input/rpc'
+require 'base64'
 
 # require_relative 'ext/xml_utility_node'
 module DataCollector
@@ -166,11 +168,16 @@ module DataCollector
             data = xml_to_hash(data, options)
           when 'text/xml'
             data = xml_to_hash(data, options)
+          when 'text/html'
+            data = html_to_hash(data, options)
           when 'text/turtle'
             graph = RDF::Graph.new do |graph|
               RDF::Turtle::Reader.new(data) {|reader| graph << reader}
             end
             data = JSON.parse(graph.dump(:jsonld, validate: false, standard_prefixes: true))
+          when /^image/
+            options['file_type'] = file_type
+            data = image_to_data(data, options)
           else
             data = xml_to_hash(data, options)
           end
@@ -196,6 +203,8 @@ module DataCollector
       data = nil
       uri = normalize_uri(uri)
       absolute_path = File.absolute_path(uri)
+      file_type = MIME::Types.type_for(uri).first.to_s
+      options['file_type'] = file_type
       raise DataCollector::Error, "#{uri.to_s} not found" unless File.exist?("#{absolute_path}")
       unless options.has_key?('raw') && options['raw'] == true
         @raw = data = File.read("#{absolute_path}")
@@ -206,6 +215,8 @@ module DataCollector
           data = JSON.parse(data)
         when '.xml'
           data = xml_to_hash(data, options)
+        when '.html'
+          data = html_to_hash(data, options)
         when '.gz'
           tar_data = []
           Minitar.open(Zlib::GzipReader.new(File.open("#{absolute_path}", 'rb'))) do |i|
@@ -232,6 +243,8 @@ module DataCollector
           end #tar
         when '.csv'
           data = csv_to_hash(data, options)
+        when '.jpg', '.png', '.gif'
+          data = image_to_data(data, options)
         else
           raise "Do not know how to process #{uri.to_s}"
         end
@@ -252,6 +265,11 @@ module DataCollector
       DataCollector::Input::Rpc.new(uri, options)
     end
 
+    def image_to_data(data, options = {})
+      file_type = options['file_type']
+      "data:#{file_type};#{Base64.encode64(data)}"
+    end
+
     def xml_to_hash(data, options = {})
       # gsub('&lt;\/', '&lt; /') outherwise wrong XML-parsing (see records lirias1729192 )
       return unless data.is_a?(String)
@@ -264,6 +282,12 @@ module DataCollector
       nori.parse(data)
     end
 
+    def html_to_hash(data, options = {})
+      return unless data.is_a?(String)
+      html_data = Nokogiri::HTML(data)
+      html_data.to_hash
+    end
+
     def csv_to_hash(data, options = {})
       csv_option_keys = options.keys & CSV::DEFAULT_OPTIONS.keys
       all_cvs_options = {headers: true, header_converters: [:downcase, :symbol]}

data/lib/data_collector/output.rb

@@ -49,7 +49,7 @@ module DataCollector
             if v.is_a?(Array) # merge with array
               @data[k] = [@data[k]] + v
             else
-              @data[k] = v
+              @data[k] = [@data[k], v]
             end
           end
         else

data/lib/data_collector/version.rb

@@ -1,4 +1,4 @@
 # encoding: utf-8
 module DataCollector
-  VERSION = "0.41.2"
+  VERSION = "0.43.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: data_collector
 version: !ruby/object:Gem::Version
-  version: 0.41.2
+  version: 0.43.0
 platform: ruby
 authors:
 - Mehmet Celik
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-12-04 00:00:00.000000000 Z
+date: 2024-01-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -311,6 +311,7 @@ files:
 - lib/data_collector/config_file.rb
 - lib/data_collector/core.rb
 - lib/data_collector/ext/base.rb
+- lib/data_collector/ext/nokogiri.rb
 - lib/data_collector/ext/xml_utility_node.rb
 - lib/data_collector/input.rb
 - lib/data_collector/input/dir.rb