data_collector 0.31.0 → 0.33.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ec0485c6ec0c9c429f8dcef45b7357874f3f6de8449902f54f38a1fb36e2548a
-  data.tar.gz: 57820299fe1718fa07e561dc933c14d04aa2fbfe5ad264dfafa2db861c162aed
+  metadata.gz: 38cc404597c0961fb94fae9421a00b6e6a18de26f2e1ce568928eb2e46d2395a
+  data.tar.gz: 849195568397a3a553d14e882bdb18662eddc0f55c6af0a16d5983c4f68979bd
 SHA512:
-  metadata.gz: aea723ca8aee53554b06259ad463b7bc460b0363bfce702dd3d7292519c185660fd86270a3d9b6c84db49133ed74d57eebcf27fcf974184465506967b96de56b
-  data.tar.gz: ffc8d3e780aab83742fc8fefdabf3f10fbdcbed5fdcddce1a69c81754e84f951e7b7ccd896037f91f82f7da23d8fd650c67d6982281ab80f23e5bb54ec0c9537
+  metadata.gz: b63028d59533473eb875184049f98137147db027d837729f1de5a203a41d6c12631264e17ddc626f7366d32e8183adc81bb9036c931fd39f8c77398394e59ce3
+  data.tar.gz: 922f2cba4f14321eee7cff191c9e775cde4595e78a97515dde6583a2d25b84b52a0c4b13db7304c40b0f2d50a80c1fd0a003b991dbb899395ee776a6b00bb52d

data/README.md

@@ -46,7 +46,7 @@ pipeline.run
 
 ```ruby
 #create a pipeline scheduled to run every morning at 06:00 am
-pipeline = Pipeline.new(schedule: '0 6 * * *')
+pipeline = Pipeline.new(cron: '0 6 * * *')
 
 pipeline.on_message do |input, output|
   data = input.from_uri("https://dummyjson.com/comments?limit=10")
@@ -336,10 +336,22 @@ Log to stdout
     log("hello world")
 ```
 #### error
-Log an error
+Log an error to stdout
 ```ruby
     error("if you have an issue take a tissue")
 ```
+### logger
+Logs are by default written to Standard OUT. If you want to change where to log to.  
+```ruby
+f = File.open('/tmp/data.log', 'w')
+f.sync = true # do not buffer
+# add multiple log outputs
+logger(STDOUT, f)
+
+#write to both STDOUT and /tmp/data.log
+log('Hello world')
+```
+
 ## Example
 Input data ___test.csv___
 ```csv

data/lib/data_collector/config_file.rb

@@ -6,9 +6,19 @@ module DataCollector
   class ConfigFile
     @config = {}
     @config_file_path = ''
+    @config_file_name = 'config.yml'
+    @mtime = nil
 
     def self.version
-      '0.0.1'
+      '0.0.3'
+    end
+
+    def self.name
+      @config_file_name
+    end
+
+    def self.name=(config_file_name)
+      @config_file_name = config_file_name
     end
 
     def self.path
@@ -37,19 +47,27 @@ module DataCollector
       @config.include?(key)
     end
 
+    def self.keys
+      init
+      @config.keys
+    end
 
-    private_class_method def self.init
+    def self.init
       discover_config_file_path
-      if @config.empty?
-        config = YAML::load_file("#{path}/config.yml")
+      raise Errno::ENOENT, "#{@config_file_path}/config.yml Not Found. Set path to config.yml" unless File.exist?("#{@config_file_path}/config.yml")
+
+      ftime = File.exist?("#{@config_file_path}/config.yml") ? File.mtime("#{@config_file_path}/config.yml") : nil
+      if @config.empty? || @mtime != ftime
+        config = YAML::load_file("#{@config_file_path}/config.yml")
         @config = process(config)
       end
     end
 
-
-    private_class_method def self.discover_config_file_path
+    def self.discover_config_file_path
       if @config_file_path.nil? || @config_file_path.empty?
-        if File.exist?('config.yml')
+        if ENV.key?('CONFIG_FILE_PATH')
+          @config_file_path = ENV['CONFIG_FILE_PATH']
+        elsif File.exist?('config.yml')
           @config_file_path = '.'
         elsif File.exist?("config/config.yml")
           @config_file_path = 'config'
@@ -57,7 +75,7 @@ module DataCollector
       end
     end
 
-    private_class_method def self.process(config)
+    def self.process(config)
       new_config = {}
       config.each do |k, v|
         if config[k].is_a?(Hash)
@@ -68,5 +86,10 @@ module DataCollector
 
       new_config
     end
+
+    private_class_method :new
+    private_class_method :init
+    private_class_method :discover_config_file_path
+    private_class_method :process
   end
 end

data/lib/data_collector/core.rb

@@ -1,6 +1,6 @@
 # encoding: utf-8
 require 'jsonpath'
-require 'logger'
+require 'proxy_logger'
 
 require_relative 'input'
 require_relative 'output'
@@ -114,7 +114,7 @@ module DataCollector
 
       filtered
     rescue StandardError => e
-      @logger ||= Logger.new(STDOUT)
+      @logger ||= self.logger
       @logger.error("#{filter_path} failed: #{e.message}")
       []
     end
@@ -126,19 +126,25 @@ module DataCollector
     module_function :config
 
     def log(message)
-      @logger ||= Logger.new(STDOUT)
+      @logger ||= self.logger
       @logger.info(message)
     end
     module_function :log
 
     def error(message)
-      @logger ||= Logger.new(STDOUT)
+      @logger ||= self.logger
       @logger.error(message)
     end
     module_function :error
 
-    def logger
-      @logger ||= Logger.new(STDOUT)
+    def logger(*destinations)
+      @logger ||= begin
+                    destinations = STDOUT if destinations.nil? || destinations.empty?
+                    Logger.new(ProxyLogger.new(destinations))
+                  rescue StandardError => e
+                    puts "Unable to instantiate ProxyLogger: #{e.message}"
+                    Logger.new(STDOUT)
+                  end
     end
     module_function :logger
 

data/lib/data_collector/input/rpc.rb

@@ -14,6 +14,8 @@ module DataCollector
 
       def running?
         @running
+      rescue StandardError => e
+        DataCollector::Core.error(e.message)
       end
 
       def stop
@@ -21,10 +23,14 @@ module DataCollector
           @listener.shutdown
           @running = false
         end
+      rescue StandardError => e
+        DataCollector::Core.error(e.message)
       end
 
       def pause
         raise "PAUSE not implemented."
+      rescue StandardError => e
+        DataCollector::Core.error(e.message)
       end
 
 
@@ -47,16 +53,23 @@ module DataCollector
           else
             yield block if block_given?
           end
+      rescue StandardError => e
+        DataCollector::Core.error(e.message)
       end
 
       private
-      def create_listener
+      def create_listener(log = false)
         @listener ||= BunnyBurrow::Server.new do |server|
           parse_uri
           server.rabbitmq_url = @bunny_uri.to_s
           server.rabbitmq_exchange = @bunny_channel
-          #server.logger = DataCollector::Core.logger
+
+          server.logger = DataCollector::Core.logger if log
         end
+
+        @listener
+      rescue StandardError => e
+        DataCollector::Core.error(e.message)
       end
 
       def parse_uri

data/lib/data_collector/output.rb

@@ -6,9 +6,11 @@ require 'minitar'
 require 'zlib'
 require 'cgi'
 require 'active_support/core_ext/hash'
+require 'active_support/core_ext/array'
 require "active_support/isolated_execution_state"
 require 'active_support/xml_mini'
 require 'fileutils'
+
 require_relative './output/rpc'
 
 module DataCollector
@@ -43,10 +45,11 @@ module DataCollector
               @data[k] << v
             end
           else
-            @data[k] = v
-            # HELP: why am I creating an array here?
-            # t = data[k]
-            # data[k] = Array.new([t, v])
+            if v.is_a?(Array) # merge with array
+              @data[k] = [@data[k]] + v
+            else
+              @data[k] = v
+            end
           end
         else
           @data[k] = v
@@ -251,7 +254,7 @@ module DataCollector
       DataCollector::Output::Rpc.new(uri, options)
     end
 
-    def to_queueto_rpc(uri, options = {})
+    def to_queue(uri, options = {})
       raise "to be implemented"
     end
 
@@ -263,6 +266,7 @@ module DataCollector
         data.compact!
         data.each { |k, v| data[k] = deep_compact(v) }
         data.compact!
+        data
       elsif data.is_a?(Array)
         # puts " - Array - #{data}"
         data.map! { |v| deep_compact(v) }

data/lib/data_collector/version.rb

@@ -1,4 +1,4 @@
 # encoding: utf-8
 module DataCollector
-  VERSION = "0.31.0"
+  VERSION = "0.33.0"
 end

data/lib/proxy_logger.rb

@@ -0,0 +1,16 @@
+class ProxyLogger
+  attr_reader :targets
+  def initialize(*targets)
+    @targets = targets.flatten
+  end
+
+  def write(*args)
+    @targets.each do |t|
+      t.write(*args)
+    end
+  end
+
+  def close
+    @targets.each(&:close)
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: data_collector
 version: !ruby/object:Gem::Version
-  version: 0.31.0
+  version: 0.33.0
 platform: ruby
 authors:
 - Mehmet Celik
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-09-18 00:00:00.000000000 Z
+date: 2023-11-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -311,6 +311,7 @@ files:
 - lib/data_collector/rules_ng.rb
 - lib/data_collector/runner.rb
 - lib/data_collector/version.rb
+- lib/proxy_logger.rb
 homepage: https://github.com/mehmetc/data_collector
 licenses:
 - MIT
@@ -333,7 +334,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.19
+rubygems_version: 3.4.10
 signing_key:
 specification_version: 4
 summary: ETL helper library