data_collector 0.2.2 → 0.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 67dc86adcebab3a6deba3ca2adf6c41c5925c03fc0dca627b030a39ff889d796
-  data.tar.gz: a4283972b9035c9e48370bcd07f3a375fb5312de584ed757f3572cc4c0b47164
+  metadata.gz: bc7b7aa820d36ad017732ae695c841013cce3a826e9042e19a2995442847a39b
+  data.tar.gz: 6e2e5012d09ea7cf08c967093242f2e5af7389e62cd3d631b213ba948b019b8a
 SHA512:
-  metadata.gz: 65fee9ff5f15c521afc0f8181a67cebe5d7d3668a72e5cf792572745b9f7279565972cfea1568fed08d1173b33686c7fd9038c244afbc37891d25a25b6d3962d
-  data.tar.gz: 5cb4b0144a4993a00af0fc9d9637f9bc1c1fb0b72fd082e30c9f7ffbd319fbd2ad28ce95ae9b0fe54020bcb0e3ec0251380830ff4525d7a80c33966355a1ac0a
+  metadata.gz: fb68056b1e72c17f51de9df99487f85b01dc64fb17951419eebb7a5e35d07906f4b47093092ecf6afda5b1a415a51e12d5ad7c47fda9bae39c783b0187eb52fc
+  data.tar.gz: 3d3a3b35c1b011fb55bec52ea98a2291a2d3fa2d270ce68948879d4c4b1978af4f15a0bb524e5b5fa20a924252d349ea1907f25d1df99674885a23a9d4fd2a52

data/README.md

@@ -3,12 +3,26 @@ Convinience module to Extract, Transform and Load your data.
 
 #### input    
 Read input from an URI
+
+**Public methods**
+```ruby
+  from_uri(source, options = {:raw, :content_type})
+```
+- source: an uri with a scheme of http, https, file
+- options:
+    - raw: _boolean_ do not parse
+    - content_type: _string_ force a content_type if the 'Content-Type' returned by the http server is incorrect 
+
 example:
 ```ruby  
     input.from_uri("http://www.libis.be")
     input.from_uri("file://hello.txt")
+    input.from_uri("http://www.libis.be/record.jsonld", content_type: 'application/ld+json')
 ```
 
+
+
+
 Inputs can be JSON, XML or CSV or XML in a TAR.GZ file
 
  ### output 
@@ -67,19 +81,18 @@ Into a temp directory
 ```    
    
 #### filter
-filter data from a hash using [JsonPath](http://goessner.net/articles/JsonPath/index.html)
+filter data from a hash using [JSONPath](http://goessner.net/articles/JsonPath/index.html)
 
 ```ruby
     filtered_data = filter(data, "$..metadata.record")
 ```
 
 #### rules
-
-rules allows you to define a simple structure to run against a JSONPath filter
+Allows you to define a simple lambda structure to run against a JSONPath filter
 
 A rule is made up of a Hash the key is the map key field its value is a Hash with a JSONPath filter and options to apply a convert method on the filtered results.
 Available convert methods are: time, map, each, call, suffix
-  - time: Parses a given time/date string into a Time object
+  - time: parses a given time/date string into a Time object
   - map: applies a mapping to a filter
   - suffix: adds a suffix to a result
   - call: executes a lambda on the filter

data/data_collector.gemspec

@@ -9,8 +9,8 @@ Gem::Specification.new do |spec|
   spec.authors       = ["Mehmet Celik"]
   spec.email         = ["mehmet@celik.be"]
 
-  spec.summary       = %q{ETL library}
-  spec.description   = %q{INPUT, FILTER, OUTPUT data}
+  spec.summary       = %q{ETL helper library}
+  spec.description   = %q{INPUT, FILTER, OUTPUT data with RULES and code}
   spec.homepage      = "https://github.com/mehmetc/data_collector"
   spec.license       = "MIT"
 
@@ -44,6 +44,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "mime-types", "~> 3.2"
   spec.add_runtime_dependency "minitar", "= 0.9"
   spec.add_runtime_dependency "activesupport", "~> 5.2"
+  spec.add_runtime_dependency "json-ld", "~> 3.1"
 
   spec.add_development_dependency "bundler", "~> 2.0"
   spec.add_development_dependency "rake", "~> 10.0"

data/lib/data_collector/input.rb

@@ -2,7 +2,7 @@
 require 'http'
 require 'open-uri'
 require 'nokogiri'
-require 'json'
+require 'json/ld'
 require 'nori'
 require 'uri'
 require 'logger'
@@ -15,20 +15,20 @@ require 'csv'
 
 #require_relative 'ext/xml_utility_node'
 module DataCollector
-class Input
-  attr_reader :raw
+  class Input
+    attr_reader :raw
 
-  def initialize
-    @logger = Logger.new(STDOUT)
-  end
+    def initialize
+      @logger = Logger.new(STDOUT)
+    end
 
-  def from_uri(source, options = {})
-    source = CGI.unescapeHTML(source)
-    @logger.info("Loading #{source}")
-    uri = URI(source)
-    begin
-      data = nil
-      case uri.scheme
+    def from_uri(source, options = {})
+      source = CGI.unescapeHTML(source)
+      @logger.info("Loading #{source}")
+      uri = URI(source)
+      begin
+        data = nil
+        case uri.scheme
         when 'http'
           data = from_http(uri, options)
         when 'https'
@@ -37,42 +37,43 @@ class Input
           data = from_file(uri, options)
         else
           raise "Do not know how to process #{source}"
-      end
+        end
 
-      data = data.nil? ? 'no data found' : data
+        data = data.nil? ? 'no data found' : data
 
-      if block_given?
-        yield data
-      else
-        data
+        if block_given?
+          yield data
+        else
+          data
+        end
+      rescue => e
+        @logger.info(e.message)
+        puts e.backtrace.join("\n")
+        nil
       end
-    rescue => e
-      @logger.info(e.message)
-      puts e.backtrace.join("\n")
-      nil
     end
-  end
 
-  private
-  def from_http(uri, options = {})
-    from_https(uri, options)
-  end
-
-  def from_https(uri, options = {})
-    data = nil
-    http = HTTP
+    private
 
-    if options.keys.include?(:user) && options.keys.include?(:password)
-      user = options[:user]
-      password = options[:password]
-      http = HTTP.basic_auth(user: user, pass: password)
-    else
-      @logger.warn ("User or Password parameter not found")
+    def from_http(uri, options = {})
+      from_https(uri, options)
     end
 
-    http_response = http.get(escape_uri(uri))
+    def from_https(uri, options = {})
+      data = nil
+      http = HTTP
+
+      if options.keys.include?(:user) && options.keys.include?(:password)
+        user = options[:user]
+        password = options[:password]
+        http = HTTP.basic_auth(user: user, pass: password)
+      else
+        @logger.warn ("User or Password parameter not found")
+      end
 
-    case http_response.code
+      http_response = http.get(escape_uri(uri))
+
+      case http_response.code
       when 200
         @raw = data = http_response.body.to_s
 
@@ -80,21 +81,24 @@ class Input
         #   f.puts data
         # end
 
-        file_type = file_type_from(http_response.headers)
+        file_type = options.with_indifferent_access.has_key?(:content_type) ? options.with_indifferent_access[:content_type] : file_type_from(http_response.headers)
 
         unless options.with_indifferent_access.has_key?(:raw) && options.with_indifferent_access[:raw] == true
           case file_type
-            when 'application/json'
-              data = JSON.parse(data)
-            when 'application/atom+xml'
-              data = xml_to_hash(data)
-            when 'text/csv'
-              data = csv_to_hash(data)
-            when 'application/xml'
-            when 'text/xml'
-              data = xml_to_hash(data)
-            else
-              data = xml_to_hash(data)
+          when 'application/ld+json'
+            data = JSON.parse(data)
+          when 'application/json'
+            data = JSON.parse(data)
+          when 'application/atom+xml'
+            data = xml_to_hash(data)
+          when 'text/csv'
+            data = csv_to_hash(data)
+          when 'application/xml'
+            data = xml_to_hash(data)
+          when 'text/xml'
+            data = xml_to_hash(data)
+          else
+            data = xml_to_hash(data)
           end
         end
       when 401
@@ -103,17 +107,19 @@ class Input
         raise 'Not found'
       else
         raise "Unable to process received status code = #{http_response.code}"
-    end
+      end
 
-    data
-  end
+      data
+    end
 
-  def from_file(uri, options = {})
-    data = nil
-    absolute_path = File.absolute_path("#{uri.host}#{uri.path}")
-    unless options.has_key?('raw') && options['raw'] == true
-      @raw = data = File.read("#{absolute_path}")
-      case File.extname(absolute_path)
+    def from_file(uri, options = {})
+      data = nil
+      absolute_path = File.absolute_path("#{uri.host}#{uri.path}")
+      unless options.has_key?('raw') && options['raw'] == true
+        @raw = data = File.read("#{absolute_path}")
+        case File.extname(absolute_path)
+        when '.jsonld'
+          data = JSON.parse(data)
         when '.json'
           data = JSON.parse(data)
         when '.xml'
@@ -129,44 +135,43 @@ class Input
           data = csv_to_hash(data)
         else
           raise "Do not know how to process #{uri.to_s}"
+        end
       end
+
+      data
     end
 
-    data
-  end
+    def xml_to_hash(data)
+      #gsub('<\/', '< /') outherwise wrong XML-parsing (see records lirias1729192 )
+      data = data.gsub /</, '< /'
+      nori = Nori.new(parser: :nokogiri, strip_namespaces: true, convert_tags_to: lambda { |tag| tag.gsub(/^@/, '_') })
+      nori.parse(data)
+      #JSON.parse(nori.parse(data).to_json)
+    end
 
-  private
-  def xml_to_hash(data)
-    #gsub('<\/', '< /') outherwise wrong XML-parsing (see records lirias1729192 )
-    data = data.gsub /</, '< /'
-    nori = Nori.new(parser: :nokogiri, strip_namespaces: true, convert_tags_to: lambda {|tag| tag.gsub(/^@/, '_')})
-    nori.parse(data)
-    #JSON.parse(nori.parse(data).to_json)
-  end
+    def csv_to_hash(data)
+      csv = CSV.parse(data, headers: true, header_converters: [:downcase, :symbol])
 
-  def csv_to_hash(data)
-    csv = CSV.parse(data, headers: true, header_converters: [:downcase, :symbol])
+      csv.collect do |record|
+        record.to_hash
+      end
+    end
 
-    csv.collect do |record|
-      record.to_hash
+    def escape_uri(uri)
+      #"#{uri.to_s.gsub(uri.query, '')}#{CGI.escape(CGI.unescape(uri.query))}"
+      uri.to_s
     end
-  end
 
-  def escape_uri(uri)
-    #"#{uri.to_s.gsub(uri.query, '')}#{CGI.escape(CGI.unescape(uri.query))}"
-    uri.to_s
-  end
+    def file_type_from(headers)
+      file_type = 'application/octet-stream'
+      file_type = if headers.include?('Content-Type')
+                    headers['Content-Type'].split(';').first
+                  else
+                    MIME::Types.of(filename_from(headers)).first.content_type
+                  end
 
-  def file_type_from(headers)
-    file_type = 'application/octet-stream'
-    file_type = if headers.include?('Content-Type')
-                  headers['Content-Type'].split(';').first
-                else
-                  MIME::Types.of(filename_from(headers)).first.content_type
-                end
+      return file_type
+    end
 
-    return file_type
   end
-
-end
 end

data/lib/data_collector/version.rb

@@ -1,4 +1,4 @@
 # encoding: utf-8
 module DataCollector
-  VERSION = "0.2.2"
+  VERSION = "0.2.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: data_collector
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.3
 platform: ruby
 authors:
 - Mehmet Celik
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-12-02 00:00:00.000000000 Z
+date: 2020-01-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -122,6 +122,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '5.2'
+- !ruby/object:Gem::Dependency
+  name: json-ld
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -164,7 +178,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '5.0'
-description: INPUT, FILTER, OUTPUT data
+description: INPUT, FILTER, OUTPUT data with RULES and code
 email:
 - mehmet@celik.be
 executables: []
@@ -214,5 +228,5 @@ requirements: []
 rubygems_version: 3.0.2
 signing_key: 
 specification_version: 4
-summary: ETL library
+summary: ETL helper library
 test_files: []