data_collector 0.34.0 → 0.35.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2dfc0690cc1200254a8d5af979048c4b1049d481a7ae31dd8984634a820e1db3
-  data.tar.gz: c9cfdea232374ccb3490b8a02e60d3ab8e418ae5300cf367956482d5888f775c
+  metadata.gz: 1b9d99a5bf660ca9feb5bea5dd13f12c2539dcb224d6689b7a1181af32d0ab20
+  data.tar.gz: 84fa315a8e2fa932caf84cad75d9c86c643c8d3b1faa389b383125de23022630
 SHA512:
-  metadata.gz: 649feb684c3449321da8611f525f09319ab592a5f810dfb7da2258d120519a32c4ac80072669d525af3ff935373842d9db03dedb27e04c16dcfd25a7dfc4435d
-  data.tar.gz: 292f698256ed9e8ab5e1ddc2903edcb0ab17a7471de5d2537fa8f099ed9259030afb576040485e0f5f899c456e0360dff69b7bae7f0907f1be079661cdb7a60d
+  metadata.gz: 82cb1c183a2d7a8ea950a181f2ae9571720533b6defd378bc0cab91fc84d876c75fc71029cd1f46c993c0b004eee7ecf32635bff808cda4419b3236ea7206c42
+  data.tar.gz: 3bf761a214120ed1d12aad599e8137ce9a9b2028c757a7e59117d0ac3300e307237160cfb359032d8506db8efc37aee3669604bba0cbac59b60b51e064fc79e8

data/README.md

@@ -84,6 +84,8 @@ A push happens when new data is created in a directory, message queue, ...
     - content_type: _string_ force a content_type if the 'Content-Type' returned by the http server is incorrect 
     - headers: request headers
     - cookies: session cookies etc.
+    - method: http verb one of [GET, POST] defaul('GET')
+    - body: http post body
 
 ###### example:
 ```ruby  
@@ -91,6 +93,8 @@ A push happens when new data is created in a directory, message queue, ...
     input.from_uri("http://www.libis.be")
     input.from_uri("file://hello.txt")
     input.from_uri("http://www.libis.be/record.jsonld", content_type: 'application/ld+json')
+    input.from_uri("https://www.w3.org/TR/rdf12-turtle/examples/example1.ttl")
+    input.from_uri("https://dbpedia.org/sparql", body: "query=SELECT * WHERE {?sub ?pred ?obj} LIMIT 10", method:"POST", headers: {accept: "text/turtle"})
 
 # read data from a RabbitMQ queue
     listener = input.from_uri('amqp://user:password@localhost?channel=hello&queue=world')

data/data_collector.gemspec

@@ -39,7 +39,6 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'activesupport', '~> 7.0'
   spec.add_runtime_dependency 'http', '~> 5.1'
   spec.add_runtime_dependency 'json', '~> 2.6'
-  spec.add_runtime_dependency 'json-ld', '~> 3.3'
   spec.add_runtime_dependency 'jsonpath', '~> 1.1'
   spec.add_runtime_dependency 'mime-types', '~> 3.5'
   spec.add_runtime_dependency 'minitar', '= 0.9'
@@ -51,6 +50,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'bunny_burrow', '~> 1.5'
   spec.add_runtime_dependency 'builder', '~> 3.2'
   spec.add_runtime_dependency 'parse-cron', '~> 0.1'
+  spec.add_runtime_dependency 'linkeddata', '~> 3.3'
 
   spec.add_development_dependency 'bundler', '~> 2.3'
   spec.add_development_dependency 'minitest', '~> 5.18'

data/lib/data_collector/input.rb

@@ -2,7 +2,7 @@
 require 'http'
 require 'open-uri'
 require 'nokogiri'
-require 'json/ld'
+require 'linkeddata'
 require 'nori'
 require 'uri'
 require 'logger'
@@ -122,15 +122,24 @@ module DataCollector
         http = http.headers(options[:headers])
       end
 
+      ctx = nil
+      http_query_options = {}
       if options.key?(:verify_ssl) && uri.scheme.eql?('https')
         @logger.warn "Disabling SSL verification. "
         # shouldn't use this but we all do ...
         ctx = OpenSSL::SSL::SSLContext.new
         ctx.verify_mode = OpenSSL::SSL::VERIFY_NONE
 
-        http_response = http.follow.get(escape_uri(uri), ssl_context: ctx)
+        http_query_options[:ssl_context] = ctx
+      end
+
+      if options.key?(:method) && options[:method].downcase.eql?('post')
+        raise DataCollector::InputError, "No body found, a POST request needs a body" unless options.key?(:body)
+        http_query_options[:body] = options[:body]
+
+        http_response = http.follow.post(escape_uri(uri), http_query_options)
       else
-        http_response = http.follow.get(escape_uri(uri))
+        http_response = http.follow.get(escape_uri(uri), http_query_options)
       end
 
       case http_response.code
@@ -157,6 +166,11 @@ module DataCollector
             data = xml_to_hash(data, options)
           when 'text/xml'
             data = xml_to_hash(data, options)
+          when 'text/turtle'
+            graph = RDF::Graph.new do |graph|
+              RDF::Turtle::Reader.new(data) {|reader| graph << reader}
+            end
+            data = JSON.parse(graph.dump(:jsonld, validate: false, standard_prefixes: true))
           else
             data = xml_to_hash(data, options)
           end
@@ -171,7 +185,7 @@ module DataCollector
       when 404
         raise DataCollector::InputError, 'Not found'
       else
-        raise DataCollector::InputError, "Unable to process received status code = #{http_response.code}"
+        raise DataCollector::InputError, "Unable to process received status code = #{http_response.code} error= #{http_response.body.to_s}"
       end
 
       #[data, http_response.code]

data/lib/data_collector/version.rb

@@ -1,4 +1,4 @@
 # encoding: utf-8
 module DataCollector
-  VERSION = "0.34.0"
+  VERSION = "0.35.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: data_collector
 version: !ruby/object:Gem::Version
-  version: 0.34.0
+  version: 0.35.0
 platform: ruby
 authors:
 - Mehmet Celik
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-11-09 00:00:00.000000000 Z
+date: 2023-11-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -52,20 +52,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.6'
-- !ruby/object:Gem::Dependency
-  name: json-ld
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.3'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.3'
 - !ruby/object:Gem::Dependency
   name: jsonpath
   requirement: !ruby/object:Gem::Requirement
@@ -220,6 +206,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.1'
+- !ruby/object:Gem::Dependency
+  name: linkeddata
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -334,7 +334,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.10
+rubygems_version: 3.4.21
 signing_key:
 specification_version: 4
 summary: ETL helper library