data_collector 0.34.0 → 0.36.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2dfc0690cc1200254a8d5af979048c4b1049d481a7ae31dd8984634a820e1db3
-  data.tar.gz: c9cfdea232374ccb3490b8a02e60d3ab8e418ae5300cf367956482d5888f775c
+  metadata.gz: 649b618a7676b2153451b2bc3053726dd332ccc95a58ce0fe7115f4c9cc5a7eb
+  data.tar.gz: 625ca0e9da4e6b986d7ead973c80fc04a87a5539f0f7263efc672ee0f3ef7eb1
 SHA512:
-  metadata.gz: 649feb684c3449321da8611f525f09319ab592a5f810dfb7da2258d120519a32c4ac80072669d525af3ff935373842d9db03dedb27e04c16dcfd25a7dfc4435d
-  data.tar.gz: 292f698256ed9e8ab5e1ddc2903edcb0ab17a7471de5d2537fa8f099ed9259030afb576040485e0f5f899c456e0360dff69b7bae7f0907f1be079661cdb7a60d
+  metadata.gz: 2372b50b9fc9e6f5a17575c1755e169f0ddbaef44ed96847d8f5c87b28a3e30e46276cf47f6e506fe843a2293190b54fd556978ea175f937693abd4f83a637dd
+  data.tar.gz: 7a417a9ee3a70140a3601d20613159222f6c2d6d0d7feaa99c447c7e31306f14c6f08c50543cb8fda85315b5fe62abde202e2b3bda51ac421f6a517f7b6be740

data/README.md

@@ -84,6 +84,8 @@ A push happens when new data is created in a directory, message queue, ...
     - content_type: _string_ force a content_type if the 'Content-Type' returned by the http server is incorrect 
     - headers: request headers
     - cookies: session cookies etc.
+    - method: http verb one of [GET, POST] defaul('GET')
+    - body: http post body
 
 ###### example:
 ```ruby  
@@ -91,6 +93,8 @@ A push happens when new data is created in a directory, message queue, ...
     input.from_uri("http://www.libis.be")
     input.from_uri("file://hello.txt")
     input.from_uri("http://www.libis.be/record.jsonld", content_type: 'application/ld+json')
+    input.from_uri("https://www.w3.org/TR/rdf12-turtle/examples/example1.ttl")
+    input.from_uri("https://dbpedia.org/sparql", body: "query=SELECT * WHERE {?sub ?pred ?obj} LIMIT 10", method:"POST", headers: {accept: "text/turtle"})
 
 # read data from a RabbitMQ queue
     listener = input.from_uri('amqp://user:password@localhost?channel=hello&queue=world')
@@ -399,6 +403,13 @@ Should give as output
 </data>
 ```
 
+You can provide options to input.from_uri for better reading CSV formats these 
+are the same the Ruby [CSV](https://docs.ruby-lang.org/en/master/CSV.html#class-CSV-label-Options) class
+
+Loading a CSV file with **;** as the row seperator
+```ruby
+i = input.from_uri('https://support.staffbase.com/hc/en-us/article_attachments/360009197031/username.csv', col_sep: ';')
+```
 
 ## Installation
 

data/data_collector.gemspec

@@ -39,7 +39,6 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'activesupport', '~> 7.0'
   spec.add_runtime_dependency 'http', '~> 5.1'
   spec.add_runtime_dependency 'json', '~> 2.6'
-  spec.add_runtime_dependency 'json-ld', '~> 3.3'
   spec.add_runtime_dependency 'jsonpath', '~> 1.1'
   spec.add_runtime_dependency 'mime-types', '~> 3.5'
   spec.add_runtime_dependency 'minitar', '= 0.9'
@@ -51,6 +50,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'bunny_burrow', '~> 1.5'
   spec.add_runtime_dependency 'builder', '~> 3.2'
   spec.add_runtime_dependency 'parse-cron', '~> 0.1'
+  spec.add_runtime_dependency 'linkeddata', '~> 3.3'
 
   spec.add_development_dependency 'bundler', '~> 2.3'
   spec.add_development_dependency 'minitest', '~> 5.18'

data/lib/data_collector/input.rb

@@ -2,7 +2,7 @@
 require 'http'
 require 'open-uri'
 require 'nokogiri'
-require 'json/ld'
+require 'linkeddata'
 require 'nori'
 require 'uri'
 require 'logger'
@@ -122,15 +122,24 @@ module DataCollector
         http = http.headers(options[:headers])
       end
 
+      ctx = nil
+      http_query_options = {}
       if options.key?(:verify_ssl) && uri.scheme.eql?('https')
         @logger.warn "Disabling SSL verification. "
         # shouldn't use this but we all do ...
         ctx = OpenSSL::SSL::SSLContext.new
         ctx.verify_mode = OpenSSL::SSL::VERIFY_NONE
 
-        http_response = http.follow.get(escape_uri(uri), ssl_context: ctx)
+        http_query_options[:ssl_context] = ctx
+      end
+
+      if options.key?(:method) && options[:method].downcase.eql?('post')
+        raise DataCollector::InputError, "No body found, a POST request needs a body" unless options.key?(:body)
+        http_query_options[:body] = options[:body]
+
+        http_response = http.follow.post(escape_uri(uri), http_query_options)
       else
-        http_response = http.follow.get(escape_uri(uri))
+        http_response = http.follow.get(escape_uri(uri), http_query_options)
       end
 
       case http_response.code
@@ -152,11 +161,16 @@ module DataCollector
           when 'application/atom+xml'
             data = xml_to_hash(data, options)
           when 'text/csv'
-            data = csv_to_hash(data)
+            data = csv_to_hash(data, options)
           when 'application/xml'
             data = xml_to_hash(data, options)
           when 'text/xml'
             data = xml_to_hash(data, options)
+          when 'text/turtle'
+            graph = RDF::Graph.new do |graph|
+              RDF::Turtle::Reader.new(data) {|reader| graph << reader}
+            end
+            data = JSON.parse(graph.dump(:jsonld, validate: false, standard_prefixes: true))
           else
             data = xml_to_hash(data, options)
           end
@@ -171,7 +185,7 @@ module DataCollector
       when 404
         raise DataCollector::InputError, 'Not found'
       else
-        raise DataCollector::InputError, "Unable to process received status code = #{http_response.code}"
+        raise DataCollector::InputError, "Unable to process received status code = #{http_response.code} error= #{http_response.body.to_s}"
       end
 
       #[data, http_response.code]
@@ -217,7 +231,7 @@ module DataCollector
             end #entry
           end #tar
         when '.csv'
-          data = csv_to_hash(data)
+          data = csv_to_hash(data, options)
         else
           raise "Do not know how to process #{uri.to_s}"
         end
@@ -250,8 +264,15 @@ module DataCollector
       nori.parse(data)
     end
 
-    def csv_to_hash(data)
-      csv = CSV.parse(data, headers: true, header_converters: [:downcase, :symbol])
+    def csv_to_hash(data, options = {})
+      csv_option_keys = options.keys & CSV::DEFAULT_OPTIONS.keys
+      all_cvs_options = {headers: true, header_converters: [:downcase, :symbol]}
+
+      csv_option_keys.each do |k|
+        all_cvs_options[k] = options[k]
+      end
+
+      csv = CSV.parse(data, **all_cvs_options)
 
       csv.collect do |record|
         record.to_hash

data/lib/data_collector/version.rb

@@ -1,4 +1,4 @@
 # encoding: utf-8
 module DataCollector
-  VERSION = "0.34.0"
+  VERSION = "0.36.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: data_collector
 version: !ruby/object:Gem::Version
-  version: 0.34.0
+  version: 0.36.0
 platform: ruby
 authors:
 - Mehmet Celik
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-11-09 00:00:00.000000000 Z
+date: 2023-11-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -52,20 +52,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.6'
-- !ruby/object:Gem::Dependency
-  name: json-ld
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.3'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.3'
 - !ruby/object:Gem::Dependency
   name: jsonpath
   requirement: !ruby/object:Gem::Requirement
@@ -220,6 +206,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.1'
+- !ruby/object:Gem::Dependency
+  name: linkeddata
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -334,7 +334,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.10
+rubygems_version: 3.4.21
 signing_key:
 specification_version: 4
 summary: ETL helper library