data_collector 0.20.0 → 0.21.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9be0a896bc07779a96233f57ca14de24c78caff9c74b9ccc1b5d41e21a048920
-  data.tar.gz: 9461c9b07a972af2324a67dbbf0858ca89c3d231ad60b18d192a22fa896b3242
+  metadata.gz: 919f205875cd4bdd02d951b50fba36833743c96eb887d8b183bdb4fabcff62d2
+  data.tar.gz: 6feefeb606971a467dbf1cd47d39a3e76b5e23563fbd81c7be37f07f9c2f4bc2
 SHA512:
-  metadata.gz: 8cf41185c86361d0ac0e487334e8fdbef16b697494c855220223550bdbbd4b8574955c1c68c16d108d399532e4cdbe4a32db352450a1e8ce2c7aa95f597362f5
-  data.tar.gz: 16dfd97b0a1ffcb8238ec93df50dfa13dca1be5dedb6c29ce9efacee379fae290208dedda985f0e716b728ca4eb33780cd98c07f8e3403d6039389be1e7c9129
+  metadata.gz: d51898627739f4047234e617ed10e8686cbcdc76782ea9cdf78ed5c566ed18801de0938d0abe6d57d734297a066904043a29027d33b3de6ef35fbe255cf19c2b
+  data.tar.gz: cd05871afe0d9d9bcb698abb96292d7ed4780d41c3ddeb74db649abb7512f4c06f3991c11211cb2d574d7636de68a4fa1b23fa817943f894666fa64ff4aa31f8

data/README.md

@@ -1,15 +1,13 @@
 # DataCollector
-Convenience module to Extract, Transform and Load your data in a Pipeline.
+Convenience module to Extract, Transform and Load data in a Pipeline.
 The 'INPUT', 'OUTPUT' and 'FILTER' object will help you to read, transform and output your data.
-Support objects like CONFIG, LOG, ERROR, RULES. Will help you to write manageable rules to transform and log your data.
+Support objects like CONFIG, LOG, ERROR, RULES help you to write manageable rules to transform and log your data.
 Include the DataCollector::Core module into your application gives you access to these objects.
 ```ruby
 include DataCollector::Core
 ```
-
 Every object can be used on its own.
 
-
 #### Pipeline
 Allows you to create a simple pipeline of operations to process data. With a data pipeline, you can collect, process, and transform data, and then transfer it to various systems and applications.
 
@@ -28,16 +26,29 @@ executed in the [ISO8601 duration format](https://www.digi.com/resources/documen
  - .on_message: handle to run every time a trigger event happens
 ###### example:
 ```ruby
-#create a pipline scheduled to run every 10 minutes
+#create a pipeline scheduled to run every 10 minutes
 pipeline = Pipeline.new(schedule: 'PT10M')
 
 pipeline.on_message do |input, output|
-  # logic
+  data = input.from_uri("https://dummyjson.com/comments?limit=10")
+  # process data
 end
 
 pipeline.run
 ```
 
+```ruby
+#create a pipeline to listen and process files in a directory
+extract = DataCollector::Pipeline.new(name: 'extract', uri: 'file://./data/in')
+
+extract.on_message do |input, output, filename|
+  data = input.from_uri("file://#{filename}")
+  # process data
+end
+
+extract.run
+```
+
 #### input
 The input component is part of the processing logic. All data is converted into a Hash, Array, ... accessible using plain Ruby or JSONPath using the filter object.  
 The input component can fetch data from various URIs, such as files, URLs, directories, queues, ...  
@@ -60,18 +71,18 @@ A push happens when new data is created in a directory, message queue, ...
     input.from_uri("http://www.libis.be/record.jsonld", content_type: 'application/ld+json')
 
 # read data from a RabbitMQ queue
-    listener = input.from_uri('amqp://user:password@localhost?channel=hello')
+    listener = input.from_uri('amqp://user:password@localhost?channel=hello&queue=world')
     listener.on_message do |input, output, message| 
       puts message
     end
-    listener.start
+    listener.run
 
 # read data from a directory
     listener = input.from_uri('file://this/is/directory')
     listener.on_message do |input, output, filename|
       puts filename
     end
-    listener.start
+    listener.run
 ```
 
 Inputs can be JSON, XML or CSV or XML in a TAR.GZ file   
@@ -94,6 +105,30 @@ Output is an object you can store key/value pairs that needs to be written to an
     output[:last_name] = 'Doe'
 ```    
 
+```ruby
+# get all keys from the output object
+    output.keys
+    output.key?(:name)
+    output.each do |k,v|
+      puts "#{k}:#{v}"      
+    end
+```
+```ruby
+# add hash to output
+    output << { age: 22 }
+
+    puts output[:age]
+# # 22
+```
+```ruby
+# add array to output
+    output << [1,2,3,4]
+    puts output.keys
+# # datap
+    puts output['datap']
+# # [1, 2, 3, 4]
+```
+
 Write output to a file, string use an ERB file as a template
 example:
 ___test.erb___
@@ -116,25 +151,34 @@ will produce
 Into a variable
 ```ruby
     result = output.to_s("test.erb")
+#template is optional
+    result = output.to_s
 ```  
 
-Into a file stored in records dir
+Into a file
 ```ruby
-    output.to_file("test.erb")
+    output.to_uri("file://data.xml", {template: "test.erb", content_type: "application/xml"})
+#template is optional
+    output.to_uri("file://data.json", {content_type: "application/json"})
 ``` 
 
 Into a tar file stored in data
 ```ruby
-    output.to_file("test.erb", "my_data.tar.gz")
+# create a tar file with a random name
+    data = output.to_uri("file://data.json", {content_type: "application/json", tar:true})
+#choose
+    data = output.to_uri("file://./test.json", {template: "test.erb", content_type: 'application/json', tar_name: "test.tar.gz"}) 
 ```    
 
 Other output methods
 ```ruby
 output.raw
 output.clear
-output.to_tmp_file("test.erb", "tmp_data")
-output.to_jsonfile(data, "test")
+output.to_xml(template: 'test.erb', root: 'record') # root defaults to 'data'
+output.to_json
 output.flatten
+output.crush
+output.keys
 ```
 
 Into a temp directory

data/data_collector.gemspec

@@ -43,11 +43,13 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'jsonpath', '~> 1.1'
   spec.add_runtime_dependency 'mime-types', '~> 3.4'
   spec.add_runtime_dependency 'minitar', '= 0.9'
-  spec.add_runtime_dependency 'nokogiri', '~> 1.14'
+  spec.add_runtime_dependency 'nokogiri', '~> 1.15'
   spec.add_runtime_dependency 'nori', '~> 2.6'
   spec.add_runtime_dependency 'iso8601', '~> 0.13'
   spec.add_runtime_dependency 'listen', '~> 3.8'
   spec.add_runtime_dependency 'bunny', '~> 2.20'
+  spec.add_runtime_dependency 'bunny_burrow', '~> 1.5'
+  spec.add_runtime_dependency 'builder', '~> 3.2'
 
   spec.add_development_dependency 'bundler', '~> 2.3'
   spec.add_development_dependency 'minitest', '~> 5.18'

data/lib/data_collector/core.rb

@@ -137,6 +137,11 @@ module DataCollector
     end
     module_function :error
 
+    def logger
+      @logger ||= Logger.new(STDOUT)
+    end
+    module_function :logger
+
   end
 
 end

data/lib/data_collector/input/dir.rb

@@ -4,7 +4,7 @@ require 'listen'
 module DataCollector
   class Input
     class Dir < Generic
-      def initialize(uri, options)
+      def initialize(uri, options = {})
         super
       end
 
@@ -18,7 +18,7 @@ module DataCollector
         @listener ||= Listen.to("#{@uri.host}/#{@uri.path}", @options) do |modified, added, _|
           files = added | modified
           files.each do |filename|
-            handle_on_message(input, output, filename)
+            handle_on_message(@input, @output, filename)
           end
         end
       end

data/lib/data_collector/input/generic.rb

@@ -3,19 +3,21 @@ require 'listen'
 module DataCollector
   class Input
     class Generic
-      def initialize(uri, options)
+      def initialize(uri, options = {})
         @uri = uri
         @options = options
+        @running = false
 
         @input = DataCollector::Input.new
         @output = DataCollector::Output.new
 
-        @listener = create_listener
+        @name = options[:name] || "input-#{Time.now.to_i}-#{rand(10000)}"
+        create_listener
       end
 
       def run(should_block = false, &block)
         raise DataCollector::Error, 'Please supply a on_message block' if @on_message_callback.nil?
-        @listener.start
+        @running = true
 
         if should_block
           while running?
@@ -37,11 +39,11 @@ module DataCollector
       end
 
       def running?
-        @listener.running?
+        @running
       end
 
       def stopped?
-        @listener.stopped?
+        @running == false
       end
 
       def paused?

data/lib/data_collector/input/queue.rb

@@ -1,15 +1,15 @@
 require_relative 'generic'
 require 'bunny'
 require 'active_support/core_ext/hash'
+require 'ostruct'
 
 module DataCollector
   class Input
     class Queue < Generic
-      def initialize(uri, options)
+      def initialize(uri, options = {})
         super
 
         if running?
-          create_channel unless @channel
           create_queue unless @queue
         end
       end
@@ -18,9 +18,9 @@ module DataCollector
         @listener.open?
       end
 
-      def send(message)
+      def send(route, message)
         if running?
-          @queue.publish(message)
+          @exchange.publish(message, routing_key: route)
         end
       end
 
@@ -37,6 +37,15 @@ module DataCollector
                       end
       end
 
+      def create_exchange
+        @exchange ||= begin
+                        options = CGI.parse(@uri.query).with_indifferent_access
+                        raise DataCollector::Error, '"channel" query parameter missing from uri.' unless options.include?(:channel)
+                        create_channel
+                        @channel.topic(options[:channel].first, auto_delete: true)
+                      end
+      end
+
       def create_channel
         raise DataCollector::Error, 'Connection to RabbitMQ is closed' if @listener.closed?
         @channel ||= @listener.create_channel
@@ -45,11 +54,12 @@ module DataCollector
       def create_queue
         @queue ||= begin
                      options = CGI.parse(@uri.query).with_indifferent_access
-                     raise DataCollector::Error, '"channel" query parameter missing from uri.' unless options.include?(:channel)
-                     queue = @channel.queue(options[:channel].first)
+                     raise DataCollector::Error, '"queue" query parameter missing from uri.' unless options.include?(:queue)
+                     create_exchange
+                     queue = @channel.queue(options[:queue].first, auto_delete: true).bind(@exchange, routing_key: "#{options[:queue].first}.#")
 
-                     queue.subscribe do |delivery_info, metadata, payload|
-                       handle_on_message(input, output, payload)
+                     queue.subscribe(consumer_tag: @name) do |delivery_info, metadata, payload|
+                       handle_on_message(@input, @output, OpenStruct.new(info: delivery_info, properties: metadata, body: payload))
                      end if queue
 
                      queue

data/lib/data_collector/input/rpc.rb

@@ -0,0 +1,73 @@
+require_relative 'generic'
+require 'bunny_burrow'
+require 'active_support/core_ext/hash'
+require 'ostruct'
+require 'securerandom'
+require 'thread'
+
+module DataCollector
+  class Input
+    class Rpc < Generic
+      def initialize(uri, options = {})
+        super
+      end
+
+      def running?
+        @running
+      end
+
+      def stop
+        if running?
+          @listener.shutdown
+          @running = false
+        end
+      end
+
+      def pause
+        raise "PAUSE not implemented."
+      end
+
+
+
+      def run(should_block = false, &block)
+          @listener.subscribe(@bunny_queue) do |payload|
+            payload = JSON.parse(payload)
+            response = BunnyBurrow::Server.create_response
+            response[:data] = handle_on_message(@input, @output, payload)
+
+            response
+          end
+          @running = true
+
+          if should_block
+            while running?
+              yield block if block_given?
+              @listener.wait
+            end
+          else
+            yield block if block_given?
+          end
+      end
+
+      private
+      def create_listener
+        @listener ||= BunnyBurrow::Server.new do |server|
+          parse_uri
+          server.rabbitmq_url = @bunny_uri.to_s
+          server.rabbitmq_exchange = @bunny_channel
+          server.logger = DataCollector::Core.logger
+        end
+      end
+
+      def parse_uri
+        raise 'URI must be of format rpc+amqp://user:password@host/exchange/queue' unless @uri.path =~ /\// && @uri.path.split('/').length == 3
+
+        @bunny_channel = @uri.path.split('/')[1]
+        @bunny_queue   = @uri.path.split('/')[2]
+        @bunny_uri = @uri.clone
+        @bunny_uri.path=''
+        @bunny_uri.scheme='amqp'
+      end
+    end
+  end
+end

data/lib/data_collector/input.rb

@@ -14,6 +14,7 @@ require 'minitar'
 require 'csv'
 require_relative 'input/dir'
 require_relative 'input/queue'
+require_relative 'input/rpc'
 
 #require_relative 'ext/xml_utility_node'
 module DataCollector
@@ -26,7 +27,7 @@ module DataCollector
 
     def from_uri(source, options = {})
       source = CGI.unescapeHTML(source)
-      @logger.info("Loading #{source}")
+      @logger.info("Reading #{source}")
       uri = URI(source)
       begin
         data = nil
@@ -43,8 +44,12 @@ module DataCollector
             raise DataCollector::Error, "#{uri.host}/#{uri.path} not found" unless File.exist?("#{uri.host}/#{uri.path}")
             data = from_file(uri, options)
           end
-        when 'amqp'
-          data = from_queue(uri,options)
+        when /amqp/
+          if uri.scheme =~ /^rpc/
+            data = from_rpc(uri, options)
+          else
+            data = from_queue(uri, options)
+          end
         else
           raise "Do not know how to process #{source}"
         end
@@ -103,7 +108,7 @@ module DataCollector
       end
 
       case http_response.code
-      when 200
+      when 200..299
         @raw = data = http_response.body.to_s
 
         # File.open("#{rand(1000)}.xml", 'wb') do |f|
@@ -130,14 +135,20 @@ module DataCollector
             data = xml_to_hash(data)
           end
         end
+
+        raise '206 Partial Content' if http_response.code ==206
+
       when 401
         raise 'Unauthorized'
+      when 403
+        raise 'Forbidden'
       when 404
         raise 'Not found'
       else
         raise "Unable to process received status code = #{http_response.code}"
       end
 
+      #[data, http_response.code]
       data
     end
 
@@ -178,6 +189,10 @@ module DataCollector
       DataCollector::Input::Queue.new(uri, options)
     end
 
+    def from_rpc(uri, options = {})
+      DataCollector::Input::Rpc.new(uri, options)
+    end
+
     def xml_to_hash(data)
       #gsub('<\/', '< /') outherwise wrong XML-parsing (see records lirias1729192 )
       data = data.gsub /</, '< /'

data/lib/data_collector/output/generic.rb

@@ -0,0 +1,38 @@
+require 'bunny_burrow'
+require 'json'
+
+module DataCollector
+  class Output
+    class Generic
+      def initialize(uri, options = {})
+        @uri = uri
+        @options = options
+        @running = false
+
+        create_producer
+      end
+
+      def send(message)
+        raise DataCollector::Error, 'Please implement a producer'
+      end
+
+      def running?
+        @running
+      end
+
+      def stopped?
+        @running == false
+      end
+
+      def stop
+        @running = false
+      end
+
+      private
+      def create_producer
+        raise DataCollector::Error, 'Please implement a producer'
+      end
+
+    end
+  end
+end

data/lib/data_collector/output/rpc.rb

@@ -0,0 +1,43 @@
+require_relative './generic'
+
+module DataCollector
+  class Output
+    class Rpc < Generic
+      def initialize(uri, options = {})
+        super
+      end
+
+      def send(message)
+        raise DataCollector::Error, "No client found" if @producer.nil? || stopped?
+        JSON.parse(@producer.publish(message, @bunny_queue))
+      end
+
+      def stop
+        if @producer && @running
+          @running = false
+          @producer.shutdown
+        end
+      end
+
+      private
+      def create_producer
+        @producer ||= BunnyBurrow::Client.new do |client|
+          parse_uri
+          client.rabbitmq_url = @bunny_uri.to_s
+          client.rabbitmq_exchange = @bunny_channel
+          client.logger = DataCollector::Core.logger
+          @running = true
+        end
+      end
+      def parse_uri
+        raise 'URI must be of format rpc+amqp://user:password@host/exchange/queue' unless @uri.path =~ /\// && @uri.path.split('/').length == 3
+
+        @bunny_channel = @uri.path.split('/')[1]
+        @bunny_queue   = @uri.path.split('/')[2]
+        @bunny_uri = @uri.clone
+        @bunny_uri.path=''
+        @bunny_uri.scheme='amqp'
+      end
+    end
+  end
+end

data/lib/data_collector/output.rb

@@ -1,4 +1,4 @@
-#encoding: UTF-8
+# encoding: UTF-8
 require 'nokogiri'
 require 'erb'
 require 'date'
@@ -7,48 +7,51 @@ require 'zlib'
 require 'cgi'
 require 'active_support/core_ext/hash'
 require 'fileutils'
+require_relative './output/rpc'
 
 module DataCollector
   class Output
     include Enumerable
-    attr_reader :data, :tar_file
+    attr_reader :data
 
     def initialize(data = {})
-      @data = data
+      @data = HashWithIndifferentAccess.new(data)
       @logger = Logger.new(STDOUT)
     end
 
-    def each
-      @data.each do |d|
-        yield d
+    def each(&block)
+      if block_given?
+        @data.each(&block) if @data
+      else
+        to_enum(:each)
       end
     end
 
     def [](k, v = nil)
-      data[k]
+      @data[k]
     end
 
     def []=(k, v = nil)
       unless v.nil?
-        if data.has_key?(k)
-          if data[k].is_a?(Array) then
+        if @data.has_key?(k)
+          if @data[k].is_a?(Array) then
             if v.is_a?(Array)
-              data[k] += v
+              @data[k] += v
             else
-              data[k] << v
+              @data[k] << v
             end
           else
-            data[k] = v
+            @data[k] = v
             # HELP: why am I creating an array here?
             # t = data[k]
             # data[k] = Array.new([t, v])
           end
         else
-          data[k] = v
+          @data[k] = v
         end
       end
 
-      data
+      @data
     end
 
     def <<(input_data)
@@ -57,7 +60,7 @@ module DataCollector
           self[k] = input_data[k]
         end
       elsif input_data.is_a?(Array)
-        data["datap"] = [] unless @data.has_key?("datap")
+        @data["datap"] = [] unless @data.has_key?("datap")
         d = @data["datap"].flatten.compact
         d += input_data
         @data["datap"] = d.compact.flatten
@@ -84,6 +87,14 @@ module DataCollector
       @data
     end
 
+    def flatten()
+      out = Hash.new
+      @data.each do |m|
+        out[m[0]] = m[1]
+      end
+      out
+    end
+
     def crush
       data = @data
       @data = deep_compact(data)
@@ -91,11 +102,10 @@ module DataCollector
 
     def clear
       @data = {}
-      #GC.start(full_mark: true, immediate_sweep: true)
+      # GC.start(full_mark: true, immediate_sweep: true)
       GC.start
     end
 
-
     def to_s(erb_file = nil)
       data = @data
 
@@ -149,104 +159,174 @@ module DataCollector
 
       data[:response_date] = DateTime.now.xmlschema
 
-      result = ERB.new(File.read(erb_file), 0, '>').result(binding)
-
-      result
+      ERB.new(File.read(erb_file), 0, '>').result(binding)
     rescue Exception => e
       raise "unable to transform to text: #{e.message}"
     end
 
     def to_tmp_file(erb_file, records_dir)
-      id = data[:id].first rescue 'unknown'
-      result = to_s(erb_file)
-      xml_result = Nokogiri::XML(result, nil, 'UTF-8') do |config|
-        config.noblanks
-      end
-
-      unless File.directory?(records_dir)
-        FileUtils.mkdir_p(records_dir)
-      end
+      raise '[DEPRECATED] `to_tmp_file` deprecated. Please use `to_uri("file://abc.xml", {template: "template.erb", content_type: "application/xml"})` instead'
+    rescue Exception => e
+      raise "unable to save to file: #{e.message}"
+    end
 
-      file_name = "#{records_dir}/#{id}_#{rand(1000)}.xml"
+    def to_tar_file(erb_file, tar_file_name = nil)
+      raise '[DEPRECATED] `to_tar_file` deprecated. Please use `to_uri("file://abc.xml", {content_type: "application/xml", tar: true})` instead'
+    rescue Exception => e
+      raise "unable to save to file: #{e.message}"
+    end
 
-      File.open(file_name, 'wb:UTF-8') do |f|
-        f.puts xml_result.to_xml
-      end
-      return file_name
+    def to_jsonfile (jsondata, jsonfile)
+      raise '[DEPRECATED] `to_jsonfile` deprecated. Please use `to_uri("file://abc.json", {template: "template.erb", content_type: "application/json"})` instead'
+    rescue Exception => e
+      raise "unable to save to jsonfile: #{e.message}"
     end
 
-    def to_file(erb_file, tar_file_name = nil)
-      id = data[:id].first rescue 'unknown'
-      result = to_s(erb_file)
+    def to_uri(destination, options = {})
+      destination = CGI.unescapeHTML(destination)
+      @logger.info("writing #{destination}")
+      uri = URI(destination)
+      begin
+        data = nil
+        case uri.scheme
+        when 'http'
+          data = to_http(uri, options)
+        when 'https'
+          data = to_https(uri, options)
+        when 'file'
+          data = to_file(uri, options)
+        when /amqp/
+          if uri.scheme =~ /^rpc/
+            data = to_rpc(uri, options)
+          else
+            data = to_queue(uri, options)
+          end
+        else
+          raise "Do not know how to process #{source}"
+        end
 
-      xml_result = Nokogiri::XML(result, nil, 'UTF-8') do |config|
-        config.noblanks
-      end
+        data = data.nil? ? 'no data found' : data
 
-      if tar_file_name.nil?
-        file_name = "records/#{id}_#{rand(1000)}.xml"
-        File.open(file_name, 'wb:UTF-8') do |f|
-          f.puts xml_result.to_xml
+        if block_given?
+          yield data
+        else
+          data
         end
+      rescue => e
+        @logger.info(e.message)
+        puts e.backtrace.join("\n")
+        nil
+      end
+    end
 
-        return file_name
+    def to_xml(options = {})
+      if options.key?(:template)
+        result = to_s(options[:template])
+        xml_result = Nokogiri::XML(result, nil, 'UTF-8') do |config|
+          config.noblanks
+        end
       else
-
-        Minitar::Output.open(Zlib::GzipWriter.new(File.open("records/#{tar_file_name}", 'wb:UTF-8'))) do |f|
-          xml_data = xml_result.to_xml
-          f.tar.add_file_simple("#{id}_#{rand(1000)}.xml", data: xml_data, size: xml_data.size, mtime: Time.now.to_i)
+        xml_root = options[:root] || 'data'
+        xml_result = Nokogiri::XML(@data.to_xml(root: xml_root), nil, 'UTF-8') do |config|
+          config.noblanks
         end
-
-        return tar_file_name
       end
 
-    rescue Exception => e
-      raise "unable to save to file: #{e.message}"
+      xml_result.to_s
     end
 
-
-    def to_jsonfile (jsondata, jsonfile)
-      file_name = "records/#{jsonfile}_#{Time.now.to_i}_#{rand(1000)}.json"
-      File.open(file_name, 'wb') do |f|
-        f.puts jsondata.to_json
+    def to_json(options = {})
+      if options.key?(:template)
+        result = to_s(options[:template])
+      else
+        result = @data
       end
-    rescue Exception => e
-      raise "unable to save to jsonfile: #{e.message}"
+
+      result.to_json
     end
 
-    def flatten()
-      out = Hash.new
-      @data.each do |m|
-        out[m[0]] = m[1]
-      end
-      out
+    def to_rpc(uri, options = {})
+      DataCollector::Output::Rpc.new(uri, options)
     end
 
+    def to_queueto_rpc(uri, options = {})
+      raise "to be implemented"
+    end
+
+    private
+
     def deep_compact( data )
       if data.is_a?(Hash)
-        #puts " - Hash - #{data}"
+        # puts " - Hash - #{data}"
         data.compact!
         data.each { |k, v| data[k] = deep_compact(v) }
         data.compact!
-        data
       elsif data.is_a?(Array)
-        #puts " - Array - #{data}"
-        data.each { |v| deep_compact(v) }
-        data.empty? ? nil : data
+        # puts " - Array - #{data}"
+        data.map! { |v| deep_compact(v) }
+        data.compact!
         #puts " - Array size- #{data.size}"
         data.size == 1 ? data[0] : data
       elsif data.is_a?(String)
-        #puts " - String - #{data}"
-        data.blank? ? nil : data
+        # puts " - String - #{data}"
+        data.strip.blank? ? nil : data
       else
         data
       end
     end
-  
-    private
 
-    def tar_file(tar_file_name)
-      @tar_file ||= Minitar::Output.open(File.open("records/#{tar_file_name}", "a+b"))
+    def to_http(uri, options)
+      to_https(uri, options)
     end
+
+    def to_https(uri, options)
+
+      raise 'TODO'
+    end
+
+    def to_file(uri, options)
+      file_type = options[:content_type] || 'application/octet-stream'
+      file_name = options[:name] || "#{uri.host}#{uri.path}" || nil
+      tar_file_name = options[:tar_name] || "#{Time.now.to_i}_#{rand(1000)}.tar.gz"
+      tar = options[:tar] || options.key?(:tar_name) || false
+      result = ''
+
+      case file_type
+      when 'application/json'
+        result = to_json(options)
+        file_name = "#{Time.now.to_i}_#{rand(1000)}.json" if file_name.nil?
+      when 'application/xml'
+        result = to_xml(options)
+        file_name = "#{Time.now.to_i}_#{rand(1000)}.xml" if file_name.nil?
+      else
+        file_name = "#{Time.now.to_i}_#{rand(1000)}.txt" if file_name.nil?
+        result = @data.to_json
+      end
+
+      if tar
+        #tar_file = Zlib::GzipWriter.new(File.open("#{tar_file_name}", 'wb'))
+        tar_file = File.open("#{tar_file_name}", 'wb')
+
+        Minitar::Output.tar(tar_file) do |f|
+          f.add_file_simple("#{file_name}", {size: result.size, mtime: Time.now.to_i, data: result})
+        end
+      else
+        file_name_absolute_path = File.absolute_path(file_name)
+        file_directory = File.dirname(file_name_absolute_path)
+
+        unless File.directory?(file_directory)
+          FileUtils.mkdir_p(file_directory)
+        end
+
+        File.open(file_name_absolute_path, 'wb:UTF-8') do |f|
+          f.puts result
+        end
+      end
+
+      result
+    rescue StandardError => e
+      raise "Unable to save data: #{e.message}"
+    end
+
   end
-end
+end

data/lib/data_collector/pipeline.rb

@@ -12,7 +12,7 @@ module DataCollector
       @run_count = 0
 
       @schedule = options[:schedule] || {}
-      @name = options[:name] || "#{Time.now.to_i}-#{rand(10000)}"
+      @name = options[:name] || "pipeline-#{Time.now.to_i}-#{rand(10000)}"
       @options = options
       @listeners = []
     end

data/lib/data_collector/version.rb

@@ -1,4 +1,4 @@
 # encoding: utf-8
 module DataCollector
-  VERSION = "0.20.0"
+  VERSION = "0.21.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: data_collector
 version: !ruby/object:Gem::Version
-  version: 0.20.0
+  version: 0.21.0
 platform: ruby
 authors:
 - Mehmet Celik
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-05-17 00:00:00.000000000 Z
+date: 2023-07-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -114,14 +114,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.14'
+        version: '1.15'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.14'
+        version: '1.15'
 - !ruby/object:Gem::Dependency
   name: nori
   requirement: !ruby/object:Gem::Requirement
@@ -178,6 +178,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.20'
+- !ruby/object:Gem::Dependency
+  name: bunny_burrow
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: builder
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -259,7 +287,10 @@ files:
 - lib/data_collector/input/dir.rb
 - lib/data_collector/input/generic.rb
 - lib/data_collector/input/queue.rb
+- lib/data_collector/input/rpc.rb
 - lib/data_collector/output.rb
+- lib/data_collector/output/generic.rb
+- lib/data_collector/output/rpc.rb
 - lib/data_collector/pipeline.rb
 - lib/data_collector/rules.rb
 - lib/data_collector/rules.rb.depricated