datapipe2 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 33537af6d0ecafd5f50fc7b4d1a9dd9687c68966
4
+ data.tar.gz: 138fd72d0c33d055fd3da2cc401ac39e44f8e765
5
+ SHA512:
6
+ metadata.gz: f6f9739676a60053cbdc07a5aa2866553ae15f9cd9db8da9ad945226528b81b55226d579c236bbfa72e9ad00e09ed82dee2ca4490cb86805d9e7b4e0db9c21b9
7
+ data.tar.gz: 63046abefaeeacb871ff19d7a12abf7048005f39372a74d88fc5c23c6325e4a74829ef059590a452bc97de1608b871cee62dd208ed1375a1f0764682a0906aab
data/bin/datapipe2 ADDED
@@ -0,0 +1,17 @@
1
+ #!/usr/bin/env ruby
2
+ #
3
+ #
4
+ # The application 'datapipe' is installed as part of a gem, and
5
+ # this file is here to facilitate running it.
6
+ #
7
+ # $LOAD_PATH.unshift '/guyirvine.com/datapipe2/lib'
8
+
9
+ require 'rubygems'
10
+ require 'datapipe2'
11
+
12
+ if ARGV.length == 0
13
+ DataPipe2::Host.new.run
14
+ else
15
+ method_name = ARGV.shift
16
+ send(method_name, *ARGV)
17
+ end
data/lib/datapipe2.rb ADDED
@@ -0,0 +1,18 @@
1
+ # Don't buffer stdout
2
+ $stdout.sync = true
3
+
4
+ # Don't buffer stdout
5
+ module DataPipe2
6
+ require 'helper_functions'
7
+ require 'jobs'
8
+ require 'host'
9
+
10
+ require 'fns/db_to_csv'
11
+ require 'fns/db_to_json'
12
+ require 'fns/csv_to_db'
13
+
14
+ class DataPipe2lineError < StandardError
15
+ end
16
+ class EnvironmentVariableNotFoundError < DataPipe2lineError
17
+ end
18
+ end
@@ -0,0 +1,16 @@
1
+ require 'csv'
2
+
3
+ def csv_to_db(file_name, table_name)
4
+ db_env_name = 'DB'
5
+ db = DataPipe2.get_fluid_db(db_env_name)
6
+ db.execute("TRUNCATE TABLE #{table_name}", [])
7
+
8
+ csv = CSV.read(file_name)
9
+ columns = csv.shift
10
+ sql = "INSERT INTO #{table_name} ( #{columns.join(',')} )
11
+ VALUES (#{Array.new(columns.length, '?').join(',')})"
12
+ csv.each do |row|
13
+ db.execute(sql, row)
14
+ end
15
+ DataPipe2.log "#{table_name}: #{csv.length}", true
16
+ end
@@ -0,0 +1,16 @@
1
+ require 'csv'
2
+
3
+ def db_to_csv(sql, path, name)
4
+ db_env_name = 'DB'
5
+ db = DataPipe2.get_fluid_db(db_env_name)
6
+
7
+ file_path = "#{path}/#{name}"
8
+ File.delete(file_path) if File.exist?(file_path)
9
+
10
+ rst = db.queryForResultset(sql, [])
11
+
12
+ CSV.open(file_path, 'w') do |csv|
13
+ csv << rst[0].keys
14
+ rst.each { |r| csv << r.values }
15
+ end
16
+ end
@@ -0,0 +1,14 @@
1
+ require 'FluidDb'
2
+ require 'json'
3
+
4
+ def db_to_json(sql, path, name)
5
+ # Dir.mkdir(path) unless Dir.exists?(path)
6
+ db_env_name = 'DB'
7
+ db = DataPipe2.get_fluid_db(db_env_name)
8
+
9
+ file_path = "#{path}/#{name}"
10
+ File.delete(file_path) if File.exist?(file_path)
11
+
12
+ rst = db.queryForResultset(sql, [])
13
+ File.write(file_path, rst.to_json)
14
+ end
@@ -0,0 +1,27 @@
1
+ require 'FluidDb'
2
+
3
+ # DataPipe2 Helpers
4
+ module DataPipe2
5
+ def self.log(string, verbose = false)
6
+ type = verbose ? 'VERB' : 'INFO'
7
+ if !ENV['VERBOSE'].nil? || !verbose
8
+ timestamp = Time.new.strftime('%Y-%m-%d %H:%M:%S')
9
+ puts "[#{type}] #{timestamp} :: #{string}"
10
+ end
11
+ end
12
+
13
+ def self.log_dsl(name, string, verbose = false)
14
+ log "name: #{name}, #{string}", verbose
15
+ end
16
+
17
+ def self.get_env_var(name)
18
+ fail EnvironmentVariableNotFoundError, name if ENV[name].nil?
19
+ ENV[name]
20
+ end
21
+
22
+ def self.get_fluid_db(env_name)
23
+ uri_string = get_env_var(env_name)
24
+ log "uri: #{uri_string}", true
25
+ FluidDb::Db(uri_string)
26
+ end
27
+ end
data/lib/host.rb ADDED
@@ -0,0 +1,44 @@
1
+ require 'rubygems'
2
+ require 'rack'
3
+ require 'thin'
4
+
5
+ module DataPipe2
6
+ # Host
7
+ class Host
8
+ def run
9
+ libs = ENV['LIB'] ||= './lib'
10
+ libs.split(';').each do |path|
11
+ DataPipe2.log "Adding libdir: #{path}"
12
+ $LOAD_PATH.unshift path
13
+ end
14
+
15
+ @dsl_paths = ENV['DSL'] ||= './dsl'
16
+ DataPipe2.log "dsl_paths: #{@dsl_paths}"
17
+ @hash = {}
18
+
19
+ @hash['jobs'] = Jobs.new
20
+ long_run
21
+ end
22
+
23
+ def long_run
24
+ Kernel.loop do
25
+ begin
26
+ single_run
27
+
28
+ sleep 0.5
29
+ rescue SystemExit, Interrupt
30
+ puts 'Exiting on request ...'
31
+ break
32
+ end
33
+ end
34
+ end
35
+
36
+ def single_run
37
+ @dsl_paths.split(';').each do |dsl_dir|
38
+ Dir.glob("#{dsl_dir}/*.dsl").each do |dsl_path|
39
+ @hash['jobs'].call dsl_path
40
+ end
41
+ end
42
+ end
43
+ end
44
+ end
data/lib/jobs.rb ADDED
@@ -0,0 +1,83 @@
1
+ # Doc
2
+ module DataPipe2
3
+ require 'parse-cron'
4
+
5
+ # Hold a single job definition
6
+ class Job
7
+ attr_reader :name, :next, :errorList
8
+
9
+ def initialize(path)
10
+ @path = path
11
+ @name = File.basename(path, '.dsl')
12
+ @cron_string = ''
13
+
14
+ @error_list = []
15
+ set_cron
16
+ end
17
+
18
+ # Job Error -> Time, Exception Class Name, nsg, backtrace
19
+ def add_error(e)
20
+ @error_list << "#{e.class.name}: #{e.message}\n#{e.backtrace.join("\n")}"
21
+ end
22
+
23
+ def clear_error
24
+ @error_list = []
25
+ end
26
+
27
+ def run_now
28
+ @next = Time.now - 1
29
+ end
30
+
31
+ def set_cron
32
+ tmp = ENV["#{@name}_CRON"] ||= '0 0 * * *'
33
+ return if tmp == @cron_string
34
+
35
+ @cron_string = tmp
36
+ @cron = CronParser.new(@cron_string)
37
+ end
38
+
39
+ def call
40
+ run if Time.now > @next
41
+ end
42
+
43
+ def run
44
+ begin
45
+ DataPipe2.log "path: #{@path}", true
46
+ DataPipe2.log "dsl: #{@name}"
47
+ load @path
48
+ clear_error
49
+ rescue SystemExit, Interrupt
50
+ raise
51
+ rescue StandardError => e
52
+ string = e.message
53
+ p e.backtrace
54
+ DataPipe2.log_dsl @name, string
55
+ add_error(e)
56
+ end
57
+
58
+ set_cron
59
+ @next = @cron.next(Time.now)
60
+ end
61
+ end
62
+
63
+ # Hold all jobs
64
+ class Jobs
65
+ attr_reader :hash, :byName
66
+
67
+ def initialize
68
+ @hash = {}
69
+ @by_name = {}
70
+ end
71
+
72
+ def call(path)
73
+ if @hash[path].nil?
74
+ j = Job.new(path)
75
+ @hash[path] = j
76
+ @by_name[j.name.downcase] = j
77
+ j.run
78
+ else
79
+ @hash[path].call
80
+ end
81
+ end
82
+ end
83
+ end
metadata ADDED
@@ -0,0 +1,51 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: datapipe2
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - Guy Irvine
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2016-05-02 00:00:00.000000000 Z
12
+ dependencies: []
13
+ description: Helping to move data around your system
14
+ email: guy@guyirvine.com
15
+ executables:
16
+ - datapipe2
17
+ extensions: []
18
+ extra_rdoc_files: []
19
+ files:
20
+ - lib/datapipe2.rb
21
+ - lib/fns/csv_to_db.rb
22
+ - lib/fns/db_to_csv.rb
23
+ - lib/fns/db_to_json.rb
24
+ - lib/helper_functions.rb
25
+ - lib/host.rb
26
+ - lib/jobs.rb
27
+ - bin/datapipe2
28
+ homepage: http://rubygems.org/gems/datapipe2
29
+ licenses: []
30
+ metadata: {}
31
+ post_install_message:
32
+ rdoc_options: []
33
+ require_paths:
34
+ - lib
35
+ required_ruby_version: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - '>='
38
+ - !ruby/object:Gem::Version
39
+ version: '0'
40
+ required_rubygems_version: !ruby/object:Gem::Requirement
41
+ requirements:
42
+ - - '>='
43
+ - !ruby/object:Gem::Version
44
+ version: '0'
45
+ requirements: []
46
+ rubyforge_project:
47
+ rubygems_version: 2.0.14.1
48
+ signing_key:
49
+ specification_version: 4
50
+ summary: DataPipe2
51
+ test_files: []