data_collector 0.35.0 → 0.38.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1b9d99a5bf660ca9feb5bea5dd13f12c2539dcb224d6689b7a1181af32d0ab20
-  data.tar.gz: 84fa315a8e2fa932caf84cad75d9c86c643c8d3b1faa389b383125de23022630
+  metadata.gz: 7c584cb29547a8e6e2caf026d8860228366e2cae387ac4111c125e5450a90586
+  data.tar.gz: ea87c6fec5817023b653472f6566bf48e951755e16b8e5ab08378c4600ca01fe
 SHA512:
-  metadata.gz: 82cb1c183a2d7a8ea950a181f2ae9571720533b6defd378bc0cab91fc84d876c75fc71029cd1f46c993c0b004eee7ecf32635bff808cda4419b3236ea7206c42
-  data.tar.gz: 3bf761a214120ed1d12aad599e8137ce9a9b2028c757a7e59117d0ac3300e307237160cfb359032d8506db8efc37aee3669604bba0cbac59b60b51e064fc79e8
+  metadata.gz: c6b8fbe6975c800cd2ca179d6cc97555aa6b2cfa279d223f9cb52c03fa48186e336f6129701b820630eba912901c7d4b2a60f58678f282742cba36f4b46e0b3c
+  data.tar.gz: f08929aea4bb836e55a8ac0203cf14048ccfe1c60d1ed7af871625938877c0b2f38619cf2923da48bac9e002fa9dd37967639713390f0bc4c011dcd54f7eb50d

data/README.md

@@ -403,6 +403,13 @@ Should give as output
 </data>
 ```
 
+You can provide options to input.from_uri for better reading CSV formats these 
+are the same the Ruby [CSV](https://docs.ruby-lang.org/en/master/CSV.html#class-CSV-label-Options) class
+
+Loading a CSV file with **;** as the row seperator
+```ruby
+i = input.from_uri('https://support.staffbase.com/hc/en-us/article_attachments/360009197031/username.csv', col_sep: ';')
+```
 
 ## Installation
 

data/data_collector.gemspec

@@ -51,6 +51,8 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'builder', '~> 3.2'
   spec.add_runtime_dependency 'parse-cron', '~> 0.1'
   spec.add_runtime_dependency 'linkeddata', '~> 3.3'
+  spec.add_runtime_dependency 'connection_pool', '~> 2.4'
+
 
   spec.add_development_dependency 'bundler', '~> 2.3'
   spec.add_development_dependency 'minitest', '~> 5.18'

data/lib/data_collector/ext/base.rb

@@ -0,0 +1,58 @@
+require 'connection_pool'
+
+module BunnyBurrow
+  class Connection
+    include Singleton
+    attr_reader :connection, :started
+    attr_accessor :verify_peer, :name, :rabbitmq_url
+
+    def initialize
+      super
+      @started = false
+    end
+    def connection
+      @connection ||= Bunny.new(@rabbitmq_url, verify_peer: @verify_peer, connection_name: @name)
+      unless @started
+        @connection.start
+        @started = true
+      end
+
+      @connection
+    end
+
+    def channel
+      @channel ||= ConnectionPool::Wrapper.new do
+        connection.create_channel
+      end
+    end
+  end
+
+  class Base
+    attr_accessor(
+      :name
+    )
+
+    private
+
+    def connection
+      Connection.instance.name = @name
+      Connection.instance.verify_peer = @verify_peer
+      Connection.instance.rabbitmq_url = @rabbitmq_url
+
+      unless @connection
+        @connection = Connection.instance.connection
+      end
+
+      @connection
+    end
+
+    def channel
+      Connection.instance.name = @name
+      Connection.instance.verify_peer = @verify_peer
+      Connection.instance.rabbitmq_url = @rabbitmq_url
+
+      @channel ||= Connection.instance.channel
+    end
+  end
+end
+

data/lib/data_collector/input/generic.rb

@@ -1,4 +1,7 @@
+require 'bunny_burrow'
+require_relative '../ext/base'
 require 'listen'
+require 'active_support/core_ext/hash'
 
 module DataCollector
   class Input

data/lib/data_collector/input/rpc.rb

@@ -1,9 +1,4 @@
 require_relative 'generic'
-require 'bunny_burrow'
-require 'active_support/core_ext/hash'
-require 'ostruct'
-require 'securerandom'
-require 'thread'
 
 module DataCollector
   class Input
@@ -64,6 +59,7 @@ module DataCollector
           server.rabbitmq_url = @bunny_uri.to_s
           server.rabbitmq_exchange = @bunny_channel
 
+          #server.connection_name = @name
           server.logger = DataCollector::Core.logger if log
         end
 

data/lib/data_collector/input.rb

@@ -161,7 +161,7 @@ module DataCollector
           when 'application/atom+xml'
             data = xml_to_hash(data, options)
           when 'text/csv'
-            data = csv_to_hash(data)
+            data = csv_to_hash(data, options)
           when 'application/xml'
             data = xml_to_hash(data, options)
           when 'text/xml'
@@ -231,7 +231,7 @@ module DataCollector
             end #entry
           end #tar
         when '.csv'
-          data = csv_to_hash(data)
+          data = csv_to_hash(data, options)
         else
           raise "Do not know how to process #{uri.to_s}"
         end
@@ -264,8 +264,15 @@ module DataCollector
       nori.parse(data)
     end
 
-    def csv_to_hash(data)
-      csv = CSV.parse(data, headers: true, header_converters: [:downcase, :symbol])
+    def csv_to_hash(data, options = {})
+      csv_option_keys = options.keys & CSV::DEFAULT_OPTIONS.keys
+      all_cvs_options = {headers: true, header_converters: [:downcase, :symbol]}
+
+      csv_option_keys.each do |k|
+        all_cvs_options[k] = options[k]
+      end
+
+      csv = CSV.parse(data, **all_cvs_options)
 
       csv.collect do |record|
         record.to_hash

data/lib/data_collector/output/generic.rb

@@ -1,5 +1,7 @@
 require 'bunny_burrow'
+require_relative '../ext/base'
 require 'json'
+require 'active_support/core_ext/hash'
 
 module DataCollector
   class Output

data/lib/data_collector/output/rpc.rb

@@ -20,12 +20,14 @@ module DataCollector
       end
 
       private
-      def create_producer
+      def create_producer(log = false)
         @producer ||= BunnyBurrow::Client.new do |client|
           parse_uri
           client.rabbitmq_url = @bunny_uri.to_s
           client.rabbitmq_exchange = @bunny_channel
-          client.logger = DataCollector::Core.logger
+          #client.connection_name = @name
+
+          client.logger = DataCollector::Core.logger if log
           @running = true
         end
       end

data/lib/data_collector/version.rb

@@ -1,4 +1,4 @@
 # encoding: utf-8
 module DataCollector
-  VERSION = "0.35.0"
+  VERSION = "0.38.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: data_collector
 version: !ruby/object:Gem::Version
-  version: 0.35.0
+  version: 0.38.0
 platform: ruby
 authors:
 - Mehmet Celik
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-11-13 00:00:00.000000000 Z
+date: 2023-11-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -220,6 +220,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.3'
+- !ruby/object:Gem::Dependency
+  name: connection_pool
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.4'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -296,6 +310,7 @@ files:
 - lib/data_collector.rb
 - lib/data_collector/config_file.rb
 - lib/data_collector/core.rb
+- lib/data_collector/ext/base.rb
 - lib/data_collector/ext/xml_utility_node.rb
 - lib/data_collector/input.rb
 - lib/data_collector/input/dir.rb