data_collector 0.32.0 → 0.33.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a236c17cf807b2c641b9da834633dc9ed7a0d297b5638367726554c7f09f37b2
-  data.tar.gz: 95282bdbc813b58e9f15b4b4e24bb2f56ff9933ea8c209c60bbb9c368c0fd7a6
+  metadata.gz: 38cc404597c0961fb94fae9421a00b6e6a18de26f2e1ce568928eb2e46d2395a
+  data.tar.gz: 849195568397a3a553d14e882bdb18662eddc0f55c6af0a16d5983c4f68979bd
 SHA512:
-  metadata.gz: 4cd169de59382390b9fba943bfa726d7f4c276ce4b7a18a8ab6cbbf115a040acf6c90ecc55b8367f4888962fe9829b2b952976e3911fcedfb253415939a16aac
-  data.tar.gz: d5b0c8f43c20b3870445dea95fc72819e5fc016aabf39e37c99b74207bdf289d84b2096a37e7e88599664015153d82dd3fbd38d524e92e5c3df5877b5fcb2538
+  metadata.gz: b63028d59533473eb875184049f98137147db027d837729f1de5a203a41d6c12631264e17ddc626f7366d32e8183adc81bb9036c931fd39f8c77398394e59ce3
+  data.tar.gz: 922f2cba4f14321eee7cff191c9e775cde4595e78a97515dde6583a2d25b84b52a0c4b13db7304c40b0f2d50a80c1fd0a003b991dbb899395ee776a6b00bb52d

data/README.md

@@ -46,7 +46,7 @@ pipeline.run
 
 ```ruby
 #create a pipeline scheduled to run every morning at 06:00 am
-pipeline = Pipeline.new(schedule: '0 6 * * *')
+pipeline = Pipeline.new(cron: '0 6 * * *')
 
 pipeline.on_message do |input, output|
   data = input.from_uri("https://dummyjson.com/comments?limit=10")
@@ -336,10 +336,22 @@ Log to stdout
     log("hello world")
 ```
 #### error
-Log an error
+Log an error to stdout
 ```ruby
     error("if you have an issue take a tissue")
 ```
+### logger
+Logs are by default written to Standard OUT. If you want to change where to log to.  
+```ruby
+f = File.open('/tmp/data.log', 'w')
+f.sync = true # do not buffer
+# add multiple log outputs
+logger(STDOUT, f)
+
+#write to both STDOUT and /tmp/data.log
+log('Hello world')
+```
+
 ## Example
 Input data ___test.csv___
 ```csv

data/lib/data_collector/config_file.rb

@@ -6,9 +6,19 @@ module DataCollector
   class ConfigFile
     @config = {}
     @config_file_path = ''
+    @config_file_name = 'config.yml'
+    @mtime = nil
 
     def self.version
-      '0.0.1'
+      '0.0.3'
+    end
+
+    def self.name
+      @config_file_name
+    end
+
+    def self.name=(config_file_name)
+      @config_file_name = config_file_name
     end
 
     def self.path
@@ -37,19 +47,27 @@ module DataCollector
       @config.include?(key)
     end
 
+    def self.keys
+      init
+      @config.keys
+    end
 
-    private_class_method def self.init
+    def self.init
       discover_config_file_path
-      if @config.empty?
-        config = YAML::load_file("#{path}/config.yml")
+      raise Errno::ENOENT, "#{@config_file_path}/config.yml Not Found. Set path to config.yml" unless File.exist?("#{@config_file_path}/config.yml")
+
+      ftime = File.exist?("#{@config_file_path}/config.yml") ? File.mtime("#{@config_file_path}/config.yml") : nil
+      if @config.empty? || @mtime != ftime
+        config = YAML::load_file("#{@config_file_path}/config.yml")
         @config = process(config)
       end
     end
 
-
-    private_class_method def self.discover_config_file_path
+    def self.discover_config_file_path
       if @config_file_path.nil? || @config_file_path.empty?
-        if File.exist?('config.yml')
+        if ENV.key?('CONFIG_FILE_PATH')
+          @config_file_path = ENV['CONFIG_FILE_PATH']
+        elsif File.exist?('config.yml')
           @config_file_path = '.'
         elsif File.exist?("config/config.yml")
           @config_file_path = 'config'
@@ -57,7 +75,7 @@ module DataCollector
       end
     end
 
-    private_class_method def self.process(config)
+    def self.process(config)
       new_config = {}
       config.each do |k, v|
         if config[k].is_a?(Hash)
@@ -68,5 +86,10 @@ module DataCollector
 
       new_config
     end
+
+    private_class_method :new
+    private_class_method :init
+    private_class_method :discover_config_file_path
+    private_class_method :process
   end
 end

data/lib/data_collector/core.rb

@@ -1,6 +1,6 @@
 # encoding: utf-8
 require 'jsonpath'
-require 'logger'
+require 'proxy_logger'
 
 require_relative 'input'
 require_relative 'output'
@@ -114,7 +114,7 @@ module DataCollector
 
       filtered
     rescue StandardError => e
-      @logger ||= Logger.new(STDOUT)
+      @logger ||= self.logger
       @logger.error("#{filter_path} failed: #{e.message}")
       []
     end
@@ -126,19 +126,25 @@ module DataCollector
     module_function :config
 
     def log(message)
-      @logger ||= Logger.new(STDOUT)
+      @logger ||= self.logger
       @logger.info(message)
     end
     module_function :log
 
     def error(message)
-      @logger ||= Logger.new(STDOUT)
+      @logger ||= self.logger
       @logger.error(message)
     end
     module_function :error
 
-    def logger
-      @logger ||= Logger.new(STDOUT)
+    def logger(*destinations)
+      @logger ||= begin
+                    destinations = STDOUT if destinations.nil? || destinations.empty?
+                    Logger.new(ProxyLogger.new(destinations))
+                  rescue StandardError => e
+                    puts "Unable to instantiate ProxyLogger: #{e.message}"
+                    Logger.new(STDOUT)
+                  end
     end
     module_function :logger
 

data/lib/data_collector/output.rb

@@ -6,9 +6,11 @@ require 'minitar'
 require 'zlib'
 require 'cgi'
 require 'active_support/core_ext/hash'
+require 'active_support/core_ext/array'
 require "active_support/isolated_execution_state"
 require 'active_support/xml_mini'
 require 'fileutils'
+
 require_relative './output/rpc'
 
 module DataCollector
@@ -252,7 +254,7 @@ module DataCollector
       DataCollector::Output::Rpc.new(uri, options)
     end
 
-    def to_queueto_rpc(uri, options = {})
+    def to_queue(uri, options = {})
       raise "to be implemented"
     end
 
@@ -264,6 +266,7 @@ module DataCollector
         data.compact!
         data.each { |k, v| data[k] = deep_compact(v) }
         data.compact!
+        data
       elsif data.is_a?(Array)
         # puts " - Array - #{data}"
         data.map! { |v| deep_compact(v) }

data/lib/data_collector/version.rb

@@ -1,4 +1,4 @@
 # encoding: utf-8
 module DataCollector
-  VERSION = "0.32.0"
+  VERSION = "0.33.0"
 end

data/lib/proxy_logger.rb

@@ -0,0 +1,16 @@
+class ProxyLogger
+  attr_reader :targets
+  def initialize(*targets)
+    @targets = targets.flatten
+  end
+
+  def write(*args)
+    @targets.each do |t|
+      t.write(*args)
+    end
+  end
+
+  def close
+    @targets.each(&:close)
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: data_collector
 version: !ruby/object:Gem::Version
-  version: 0.32.0
+  version: 0.33.0
 platform: ruby
 authors:
 - Mehmet Celik
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-10-13 00:00:00.000000000 Z
+date: 2023-11-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -311,6 +311,7 @@ files:
 - lib/data_collector/rules_ng.rb
 - lib/data_collector/runner.rb
 - lib/data_collector/version.rb
+- lib/proxy_logger.rb
 homepage: https://github.com/mehmetc/data_collector
 licenses:
 - MIT
@@ -333,7 +334,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.19
+rubygems_version: 3.4.10
 signing_key:
 specification_version: 4
 summary: ETL helper library