data_collector 0.53.0 → 0.55.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 749914ea43bad2734c79a131eb5db2f1ef2b953fa386a9cb3eb50698e7d2f1f4
-  data.tar.gz: 0402100bbfe402ad1c96cc0b767c608a917402e748ea4f57b9bf298ca9510dba
+  metadata.gz: a5cc4cf83290713c75860b1cf70e90ae019e423b53f6909b425ba0f134af9672
+  data.tar.gz: eb8d383233e5d8492c3112582cb820bc4b28eeed0bea33d0bd715532456c24c1
 SHA512:
-  metadata.gz: b00c092e0670a26a85bbe1299c687a132c65af7950c5b0f74ff537a8ba948af27617164bd8beebc3b072d424fafdd58865e8cafbaa03d5e6aafb8e682806ba43
-  data.tar.gz: 4072e26cfd9a2ef57b094b3ea7cc340f0d735e8ca4356d6e346f725b2486e7284b624936724c0545677571d7ff2a0c447785374778cb55080e69cf54bc7df1a3
+  metadata.gz: 14b5ef4e812c16fb05a1ac75ab38d365eed90ce9883d09053cfe656df8d2d3c5101bc81e8d439a7ceaa65de1552b451d64bdfd5c614844502ee6841ef25c3ef2
+  data.tar.gz: ecc4b0d595804845ec423f4270578f053ffc5c987ed6c082ad9a1bc068f2a4ebe9ae0b5eb36b55826bf82a0caf7f2e8797b2ff7d9a9a597848357c29912c6bd7

data/README.md

@@ -95,6 +95,7 @@ A push happens when new data is created in a directory, message queue, ...
     input.from_uri("http://www.libis.be/record.jsonld", content_type: 'application/ld+json')
     input.from_uri("https://www.w3.org/TR/rdf12-turtle/examples/example1.ttl")
     input.from_uri("https://dbpedia.org/sparql", body: "query=SELECT * WHERE {?sub ?pred ?obj} LIMIT 10", method:"POST", headers: {accept: "text/turtle"})
+    input.from_uri(StringIO.new(File.read('myrecords.xml')), content_type: 'application/xml' )
 
 # read data from a RabbitMQ queue
     listener = input.from_uri('amqp://user:password@localhost?channel=hello&queue=world')

data/data_collector.gemspec

@@ -52,6 +52,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'parse-cron', '~> 0.1'
   spec.add_runtime_dependency 'linkeddata', '~> 3.3'
   spec.add_runtime_dependency 'connection_pool', '~> 2.4'
+  spec.add_runtime_dependency 'open-uri-and-write', '~> 0.1.0'
   #spec.add_runtime_dependency 'grpc', '~> 1.61'
 
 

data/lib/data_collector/ext/file.rb

@@ -0,0 +1,60 @@
+# Extensions and modifications (monkeypatching) to the File class:
+
+class File
+
+  class << self
+    alias original_delete delete
+    alias original_open open
+    alias original_exist? exist?
+  end
+
+  def self.exist?(name)
+    if(name[/https?:\/\//])
+      dav = OpenUriAndWrite::CredentialsStore.get_connection_for_url(name)
+      dav.exist?(name)
+    else
+      self.original_exist?(name)
+    end
+  end
+
+  def self.delete(names)
+    filenames = []
+    if(names.class == String)
+      filenames << names
+    elsif(names.class = Array)
+      filenames = names
+    end
+    filenames.each do |filename|
+      if(filename[/^(https?):\/\//])
+        dav = OpenUriAndWrite::CredentialsStore.get_connection_for_url(filename)
+        dav.delete(filename)
+      else
+        self.original_delete(filename)
+      end
+    end
+  end
+
+  def self.open(name, *rest, &block)
+    if name.respond_to?(:open)
+      name.open(*rest, &block)
+    elsif name.respond_to?(:to_s) and name[/^(https?):\/\//] and rest.size > 0 and rest.first.to_s[/^w/]
+      webdav_agent = OpenUriAndWrite::Handle.new(name, rest)
+      if(block)
+        yield webdav_agent
+      else
+        return webdav_agent
+      end
+    else
+      rest.map! do |m|
+        if m.is_a?(Hash)
+          {}.store( m.keys.first.to_s.to_sym, m.values.first)
+        else
+          m
+        end
+      end
+
+      self.original_open(name, *rest, &block)
+    end
+  end
+
+end

data/lib/data_collector/ext/x-open-uri-and-write.rb

@@ -0,0 +1,25 @@
+require 'stringio'
+require 'uri'
+require 'open-uri'
+require 'net/dav'
+require 'highline/import'
+
+require 'open-uri-and-write/handle'
+require 'open-uri-and-write/usernames'
+require 'open-uri-and-write/credentials_store'
+require_relative 'file'
+require 'open-uri-and-write/dir_extensions'
+require 'open-uri-and-write/kernel_extensions'
+
+module OpenUriAndWrite
+  class Handle < StringIO
+    def write(string)
+      if(@filemode[/^r/])
+        raise IOError.new(true), "not opened for writing"
+      end
+
+      super(string)
+      @dav.put_string(@url, string)
+    end
+  end
+end

data/lib/data_collector/input.rb

@@ -180,10 +180,7 @@ module DataCollector
           when 'text/html'
             data = html_to_hash(data, options)
           when 'text/turtle'
-            graph = RDF::Graph.new do |graph|
-              RDF::Turtle::Reader.new(data) { |reader| graph << reader }
-            end
-            data = JSON.parse(graph.dump(:jsonld, validate: false, standard_prefixes: true))
+            data = rdf_to_hash(data, options)
           when /^image/
             options['file_type'] = file_type
             data = image_to_data(data, options)
@@ -208,6 +205,13 @@ module DataCollector
       data
     end
 
+    def rdf_to_hash(data, options = {})
+      graph = RDF::Graph.new do |graph|
+        RDF::Turtle::Reader.new(data) { |reader| graph << reader }
+      end
+      data = JSON.parse(graph.dump(:jsonld, validate: false, standard_prefixes: true))
+    end
+
     def from_stringio(sio, options = {}, &block)
       raise DataCollector::InputError, "No IO input" unless sio.is_a?(StringIO)
       raise DataCollector::InputError, "content_type option not supplied" unless options.key?(:content_type)
@@ -271,6 +275,8 @@ module DataCollector
           data = csv_to_hash(data, options)
         when '.jpg', '.png', '.gif'
           data = image_to_data(data, options)
+        when '.ttl'
+          data = rdf_to_hash(data, options)
         else
           raise "Do not know how to process #{uri.to_s}"
         end

data/lib/data_collector/version.rb

@@ -1,4 +1,4 @@
 # encoding: utf-8
 module DataCollector
-  VERSION = "0.53.0"
+  VERSION = "0.55.0"
 end

data/lib/proxy_logger.rb

@@ -1,3 +1,5 @@
+require_relative 'data_collector/ext/x-open-uri-and-write'
+
 class ProxyLogger
   attr_reader :targets
   def initialize(*targets)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: data_collector
 version: !ruby/object:Gem::Version
-  version: 0.53.0
+  version: 0.55.0
 platform: ruby
 authors:
 - Mehmet Celik
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-07-05 00:00:00.000000000 Z
+date: 2025-01-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -234,6 +234,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.4'
+- !ruby/object:Gem::Dependency
+  name: open-uri-and-write
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -311,7 +325,9 @@ files:
 - lib/data_collector/config_file.rb
 - lib/data_collector/core.rb
 - lib/data_collector/ext/base.rb
+- lib/data_collector/ext/file.rb
 - lib/data_collector/ext/nokogiri.rb
+- lib/data_collector/ext/x-open-uri-and-write.rb
 - lib/data_collector/ext/xml_utility_node.rb
 - lib/data_collector/input.rb
 - lib/data_collector/input/dir.rb