data_collector 0.18.0 → 0.19.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9b2dda800a0c468ee0db8c4a4546f98c8baa005f0cff2df603e613d404021315
-  data.tar.gz: c505eb5354999645eb5ea9fbb5200ce100a37d9f3e0eac85bf9416d21cd3514a
+  metadata.gz: 35d57ff2998ab1343a4d6e906bcd76bd67951a0eae9a6db69387e4de7dbba285
+  data.tar.gz: 702a6447c28533d2dcdce237cc209417963ea2827eaed4f4ad0ab56a62c42783
 SHA512:
-  metadata.gz: a44557a687028b74b495236a47b4d802a4a6e130526a639ddf63b7b6a8a07b090f5197c23a36b2b4c9628bcfa33a0d38e2451c1a3224a45fa63d388f6922624e
-  data.tar.gz: b98a223f063f24b8f78e1358faeb02e33e365edd77b0fba2d28649fa0ad17d79f386ff216326040f3ec87390cb595f41382733ea042c5357c9cf48a23481d8c7
+  metadata.gz: 80e487e0d8bfa19cec43a607b3c58698c37e23fd6385be3102d3ca87584348d585241f1794f848c460c02751f29f3a28d1365c472b8e3a532a922f9104fb2e06
+  data.tar.gz: 0366f4350e54e1bf985f68d3d0532b6fb00394aad23a3641b07fd65594e7e0b16ddf5da90574251c75b83c63fe09455aa988b7395706fe76e995b17fc79cb2fe

data/README.md

@@ -269,8 +269,11 @@ Log to stdout
 ```ruby
     log("hello world")
 ```
-
-
+#### error
+Log an error
+```ruby
+    error("if you have an issue take a tissue")
+```
 ## Example
 Input data ___test.csv___
 ```csv

data/lib/data_collector/input.rb

@@ -37,8 +37,10 @@ module DataCollector
           data = from_https(uri, options)
         when 'file'
           if File.directory?("#{uri.host}/#{uri.path}")
+            raise DataCollector::Error, "#{uri.host}/#{uri.path} not found" unless File.exist?("#{uri.host}/#{uri.path}")
             return from_dir(uri, options)
           else
+            raise DataCollector::Error, "#{uri.host}/#{uri.path} not found" unless File.exist?("#{uri.host}/#{uri.path}")
             data = from_file(uri, options)
           end
         when 'amqp'

data/lib/data_collector/pipeline.rb

@@ -13,6 +13,8 @@ module DataCollector
 
       @schedule = options[:schedule] || {}
       @name = options[:name] || "#{Time.now.to_i}-#{rand(10000)}"
+      @options = options
+      @listeners = []
     end
 
     def on_message(&block)
@@ -22,6 +24,9 @@ module DataCollector
     def run
       if paused? && @running
         @paused = false
+        @listeners.each do |listener|
+          listener.run if listener.paused?
+        end
       end
 
       @running = true
@@ -42,8 +47,20 @@ module DataCollector
         end
       else # run once
         @run_count += 1
-        DataCollector::Core.log("PIPELINE running once")
-        handle_on_message(@input, @output)
+        if @options.key?(:uri)
+          listener = Input.new.from_uri(@options[:uri], @options)
+          listener.on_message do |input, output, filename|
+            DataCollector::Core.log("PIPELINE triggered by #{filename}")
+            handle_on_message(@input, @output, filename)
+          end
+          @listeners << listener
+
+          listener.run(true)
+
+        else
+          DataCollector::Core.log("PIPELINE running once")
+          handle_on_message(@input, @output)
+        end
       end
     rescue StandardError => e
       DataCollector::Core.error("PIPELINE run failed: #{e.message}")
@@ -54,10 +71,18 @@ module DataCollector
     def stop
       @running = false
       @paused = false
+      @listeners.each do |listener|
+        listener.stop if listener.running?
+      end
     end
 
     def pause
-      @paused = !@paused if @running
+      if @running
+      @paused = !@paused
+        @listeners.each do |listener|
+          listener.pause if listener.running?
+        end
+      end
     end
 
     def running?
@@ -74,11 +99,11 @@ module DataCollector
 
     private
 
-    def handle_on_message(input, output)
+    def handle_on_message(input, output, filename = nil)
       if (callback = @on_message_callback)
         timing = Time.now
         begin
-          callback.call(input, output)
+          callback.call(input, output, filename)
         rescue StandardError => e
           DataCollector::Core.error("PIPELINE #{e.message}")
         ensure

data/lib/data_collector/rules_ng.rb

@@ -51,7 +51,7 @@ module DataCollector
 
       data = apply_filtered_data_on_payload(data, rule_payload, options)
 
-      output_data << {tag.to_sym => data} unless data.nil? || (data.is_a?(Array) && data.empty?)
+      output_data << { tag.to_sym => data } unless data.nil? || (data.is_a?(Array) && data.empty?)
     rescue StandardError => e
       # puts "error running rule '#{tag}'\n\t#{e.message}"
       # puts e.backtrace.join("\n")
@@ -61,15 +61,26 @@ module DataCollector
     def apply_filtered_data_on_payload(input_data, payload, options = {})
       return nil if input_data.nil?
 
-      normalized_options = options.select{|k,v| k !~ /^_/ }.with_indifferent_access
+      normalized_options = options.select { |k, v| k !~ /^_/ }.with_indifferent_access
       output_data = nil
       case payload.class.name
       when 'Proc'
         data = input_data.is_a?(Array) ? input_data : [input_data]
         output_data = if normalized_options.empty?
-                        data.map { |d| payload.call(d) }
+                        # data.map { |d| payload.curry.call(d).call(d) }
+                        data.map { |d|
+                          loop do
+                            payload_result = payload.curry.call(d)
+                            break payload_result unless payload_result.is_a?(Proc)
+                          end
+                        }
                       else
-                        data.map { |d| payload.call(d, normalized_options) }
+                        data.map { |d|
+                          loop do
+                            payload_result = payload.curry.call(d, normalized_options)
+                            break payload_result unless payload_result.is_a?(Proc)
+                          end
+                        }
                       end
       when 'Hash'
         input_data = [input_data] unless input_data.is_a?(Array)
@@ -77,9 +88,9 @@ module DataCollector
           output_data = input_data.map do |m|
             if payload.key?('suffix')
               if (m.is_a?(Hash))
-                m.transform_values{|v| v.is_a?(String) ? "#{v}#{payload['suffix']}" : v}
+                m.transform_values { |v| v.is_a?(String) ? "#{v}#{payload['suffix']}" : v }
               elsif m.is_a?(Array)
-                m.map{|n| n.is_a?(String) ? "#{n}#{payload['suffix']}": n}
+                m.map { |n| n.is_a?(String) ? "#{n}#{payload['suffix']}" : n }
               elsif m.methods.include?(:to_s)
                 "#{m}#{payload['suffix']}"
               else
@@ -102,8 +113,8 @@ module DataCollector
       output_data.compact! if output_data.is_a?(Array)
       output_data.flatten! if output_data.is_a?(Array)
       if output_data.is_a?(Array) &&
-         output_data.size == 1 &&
-         (output_data.first.is_a?(Array) || output_data.first.is_a?(Hash))
+        output_data.size == 1 &&
+        (output_data.first.is_a?(Array) || output_data.first.is_a?(Hash))
         output_data = output_data.first
       end
 

data/lib/data_collector/version.rb

@@ -1,4 +1,4 @@
 # encoding: utf-8
 module DataCollector
-  VERSION = "0.18.0"
+  VERSION = "0.19.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: data_collector
 version: !ruby/object:Gem::Version
-  version: 0.18.0
+  version: 0.19.0
 platform: ruby
 authors:
 - Mehmet Celik
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-04-18 00:00:00.000000000 Z
+date: 2023-05-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport