data_collector 0.32.0 → 0.33.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: a236c17cf807b2c641b9da834633dc9ed7a0d297b5638367726554c7f09f37b2
4
- data.tar.gz: 95282bdbc813b58e9f15b4b4e24bb2f56ff9933ea8c209c60bbb9c368c0fd7a6
3
+ metadata.gz: 38cc404597c0961fb94fae9421a00b6e6a18de26f2e1ce568928eb2e46d2395a
4
+ data.tar.gz: 849195568397a3a553d14e882bdb18662eddc0f55c6af0a16d5983c4f68979bd
5
5
  SHA512:
6
- metadata.gz: 4cd169de59382390b9fba943bfa726d7f4c276ce4b7a18a8ab6cbbf115a040acf6c90ecc55b8367f4888962fe9829b2b952976e3911fcedfb253415939a16aac
7
- data.tar.gz: d5b0c8f43c20b3870445dea95fc72819e5fc016aabf39e37c99b74207bdf289d84b2096a37e7e88599664015153d82dd3fbd38d524e92e5c3df5877b5fcb2538
6
+ metadata.gz: b63028d59533473eb875184049f98137147db027d837729f1de5a203a41d6c12631264e17ddc626f7366d32e8183adc81bb9036c931fd39f8c77398394e59ce3
7
+ data.tar.gz: 922f2cba4f14321eee7cff191c9e775cde4595e78a97515dde6583a2d25b84b52a0c4b13db7304c40b0f2d50a80c1fd0a003b991dbb899395ee776a6b00bb52d
data/README.md CHANGED
@@ -46,7 +46,7 @@ pipeline.run
46
46
 
47
47
  ```ruby
48
48
  #create a pipeline scheduled to run every morning at 06:00 am
49
- pipeline = Pipeline.new(schedule: '0 6 * * *')
49
+ pipeline = Pipeline.new(cron: '0 6 * * *')
50
50
 
51
51
  pipeline.on_message do |input, output|
52
52
  data = input.from_uri("https://dummyjson.com/comments?limit=10")
@@ -336,10 +336,22 @@ Log to stdout
336
336
  log("hello world")
337
337
  ```
338
338
  #### error
339
- Log an error
339
+ Log an error to stdout
340
340
  ```ruby
341
341
  error("if you have an issue take a tissue")
342
342
  ```
343
+ ### logger
344
+ Logs are by default written to Standard OUT. If you want to change where to log to.
345
+ ```ruby
346
+ f = File.open('/tmp/data.log', 'w')
347
+ f.sync = true # do not buffer
348
+ # add multiple log outputs
349
+ logger(STDOUT, f)
350
+
351
+ #write to both STDOUT and /tmp/data.log
352
+ log('Hello world')
353
+ ```
354
+
343
355
  ## Example
344
356
  Input data ___test.csv___
345
357
  ```csv
@@ -6,9 +6,19 @@ module DataCollector
6
6
  class ConfigFile
7
7
  @config = {}
8
8
  @config_file_path = ''
9
+ @config_file_name = 'config.yml'
10
+ @mtime = nil
9
11
 
10
12
  def self.version
11
- '0.0.1'
13
+ '0.0.3'
14
+ end
15
+
16
+ def self.name
17
+ @config_file_name
18
+ end
19
+
20
+ def self.name=(config_file_name)
21
+ @config_file_name = config_file_name
12
22
  end
13
23
 
14
24
  def self.path
@@ -37,19 +47,27 @@ module DataCollector
37
47
  @config.include?(key)
38
48
  end
39
49
 
50
+ def self.keys
51
+ init
52
+ @config.keys
53
+ end
40
54
 
41
- private_class_method def self.init
55
+ def self.init
42
56
  discover_config_file_path
43
- if @config.empty?
44
- config = YAML::load_file("#{path}/config.yml")
57
+ raise Errno::ENOENT, "#{@config_file_path}/config.yml Not Found. Set path to config.yml" unless File.exist?("#{@config_file_path}/config.yml")
58
+
59
+ ftime = File.exist?("#{@config_file_path}/config.yml") ? File.mtime("#{@config_file_path}/config.yml") : nil
60
+ if @config.empty? || @mtime != ftime
61
+ config = YAML::load_file("#{@config_file_path}/config.yml")
45
62
  @config = process(config)
46
63
  end
47
64
  end
48
65
 
49
-
50
- private_class_method def self.discover_config_file_path
66
+ def self.discover_config_file_path
51
67
  if @config_file_path.nil? || @config_file_path.empty?
52
- if File.exist?('config.yml')
68
+ if ENV.key?('CONFIG_FILE_PATH')
69
+ @config_file_path = ENV['CONFIG_FILE_PATH']
70
+ elsif File.exist?('config.yml')
53
71
  @config_file_path = '.'
54
72
  elsif File.exist?("config/config.yml")
55
73
  @config_file_path = 'config'
@@ -57,7 +75,7 @@ module DataCollector
57
75
  end
58
76
  end
59
77
 
60
- private_class_method def self.process(config)
78
+ def self.process(config)
61
79
  new_config = {}
62
80
  config.each do |k, v|
63
81
  if config[k].is_a?(Hash)
@@ -68,5 +86,10 @@ module DataCollector
68
86
 
69
87
  new_config
70
88
  end
89
+
90
+ private_class_method :new
91
+ private_class_method :init
92
+ private_class_method :discover_config_file_path
93
+ private_class_method :process
71
94
  end
72
95
  end
@@ -1,6 +1,6 @@
1
1
  # encoding: utf-8
2
2
  require 'jsonpath'
3
- require 'logger'
3
+ require 'proxy_logger'
4
4
 
5
5
  require_relative 'input'
6
6
  require_relative 'output'
@@ -114,7 +114,7 @@ module DataCollector
114
114
 
115
115
  filtered
116
116
  rescue StandardError => e
117
- @logger ||= Logger.new(STDOUT)
117
+ @logger ||= self.logger
118
118
  @logger.error("#{filter_path} failed: #{e.message}")
119
119
  []
120
120
  end
@@ -126,19 +126,25 @@ module DataCollector
126
126
  module_function :config
127
127
 
128
128
  def log(message)
129
- @logger ||= Logger.new(STDOUT)
129
+ @logger ||= self.logger
130
130
  @logger.info(message)
131
131
  end
132
132
  module_function :log
133
133
 
134
134
  def error(message)
135
- @logger ||= Logger.new(STDOUT)
135
+ @logger ||= self.logger
136
136
  @logger.error(message)
137
137
  end
138
138
  module_function :error
139
139
 
140
- def logger
141
- @logger ||= Logger.new(STDOUT)
140
+ def logger(*destinations)
141
+ @logger ||= begin
142
+ destinations = STDOUT if destinations.nil? || destinations.empty?
143
+ Logger.new(ProxyLogger.new(destinations))
144
+ rescue StandardError => e
145
+ puts "Unable to instantiate ProxyLogger: #{e.message}"
146
+ Logger.new(STDOUT)
147
+ end
142
148
  end
143
149
  module_function :logger
144
150
 
@@ -6,9 +6,11 @@ require 'minitar'
6
6
  require 'zlib'
7
7
  require 'cgi'
8
8
  require 'active_support/core_ext/hash'
9
+ require 'active_support/core_ext/array'
9
10
  require "active_support/isolated_execution_state"
10
11
  require 'active_support/xml_mini'
11
12
  require 'fileutils'
13
+
12
14
  require_relative './output/rpc'
13
15
 
14
16
  module DataCollector
@@ -252,7 +254,7 @@ module DataCollector
252
254
  DataCollector::Output::Rpc.new(uri, options)
253
255
  end
254
256
 
255
- def to_queueto_rpc(uri, options = {})
257
+ def to_queue(uri, options = {})
256
258
  raise "to be implemented"
257
259
  end
258
260
 
@@ -264,6 +266,7 @@ module DataCollector
264
266
  data.compact!
265
267
  data.each { |k, v| data[k] = deep_compact(v) }
266
268
  data.compact!
269
+ data
267
270
  elsif data.is_a?(Array)
268
271
  # puts " - Array - #{data}"
269
272
  data.map! { |v| deep_compact(v) }
@@ -1,4 +1,4 @@
1
1
  # encoding: utf-8
2
2
  module DataCollector
3
- VERSION = "0.32.0"
3
+ VERSION = "0.33.0"
4
4
  end
@@ -0,0 +1,16 @@
1
+ class ProxyLogger
2
+ attr_reader :targets
3
+ def initialize(*targets)
4
+ @targets = targets.flatten
5
+ end
6
+
7
+ def write(*args)
8
+ @targets.each do |t|
9
+ t.write(*args)
10
+ end
11
+ end
12
+
13
+ def close
14
+ @targets.each(&:close)
15
+ end
16
+ end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: data_collector
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.32.0
4
+ version: 0.33.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Mehmet Celik
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2023-10-13 00:00:00.000000000 Z
11
+ date: 2023-11-07 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activesupport
@@ -311,6 +311,7 @@ files:
311
311
  - lib/data_collector/rules_ng.rb
312
312
  - lib/data_collector/runner.rb
313
313
  - lib/data_collector/version.rb
314
+ - lib/proxy_logger.rb
314
315
  homepage: https://github.com/mehmetc/data_collector
315
316
  licenses:
316
317
  - MIT
@@ -333,7 +334,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
333
334
  - !ruby/object:Gem::Version
334
335
  version: '0'
335
336
  requirements: []
336
- rubygems_version: 3.4.19
337
+ rubygems_version: 3.4.10
337
338
  signing_key:
338
339
  specification_version: 4
339
340
  summary: ETL helper library