datapipe2 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/bin/datapipe2 +17 -0
- data/lib/datapipe2.rb +18 -0
- data/lib/fns/csv_to_db.rb +16 -0
- data/lib/fns/db_to_csv.rb +16 -0
- data/lib/fns/db_to_json.rb +14 -0
- data/lib/helper_functions.rb +27 -0
- data/lib/host.rb +44 -0
- data/lib/jobs.rb +83 -0
- metadata +51 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 33537af6d0ecafd5f50fc7b4d1a9dd9687c68966
|
4
|
+
data.tar.gz: 138fd72d0c33d055fd3da2cc401ac39e44f8e765
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: f6f9739676a60053cbdc07a5aa2866553ae15f9cd9db8da9ad945226528b81b55226d579c236bbfa72e9ad00e09ed82dee2ca4490cb86805d9e7b4e0db9c21b9
|
7
|
+
data.tar.gz: 63046abefaeeacb871ff19d7a12abf7048005f39372a74d88fc5c23c6325e4a74829ef059590a452bc97de1608b871cee62dd208ed1375a1f0764682a0906aab
|
data/bin/datapipe2
ADDED
@@ -0,0 +1,17 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
#
|
3
|
+
#
|
4
|
+
# The application 'datapipe' is installed as part of a gem, and
|
5
|
+
# this file is here to facilitate running it.
|
6
|
+
#
|
7
|
+
# $LOAD_PATH.unshift '/guyirvine.com/datapipe2/lib'
|
8
|
+
|
9
|
+
require 'rubygems'
|
10
|
+
require 'datapipe2'
|
11
|
+
|
12
|
+
if ARGV.length == 0
|
13
|
+
DataPipe2::Host.new.run
|
14
|
+
else
|
15
|
+
method_name = ARGV.shift
|
16
|
+
send(method_name, *ARGV)
|
17
|
+
end
|
data/lib/datapipe2.rb
ADDED
@@ -0,0 +1,18 @@
|
|
1
|
+
# Don't buffer stdout
|
2
|
+
$stdout.sync = true
|
3
|
+
|
4
|
+
# Don't buffer stdout
|
5
|
+
module DataPipe2
|
6
|
+
require 'helper_functions'
|
7
|
+
require 'jobs'
|
8
|
+
require 'host'
|
9
|
+
|
10
|
+
require 'fns/db_to_csv'
|
11
|
+
require 'fns/db_to_json'
|
12
|
+
require 'fns/csv_to_db'
|
13
|
+
|
14
|
+
class DataPipe2lineError < StandardError
|
15
|
+
end
|
16
|
+
class EnvironmentVariableNotFoundError < DataPipe2lineError
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
require 'csv'
|
2
|
+
|
3
|
+
def csv_to_db(file_name, table_name)
|
4
|
+
db_env_name = 'DB'
|
5
|
+
db = DataPipe2.get_fluid_db(db_env_name)
|
6
|
+
db.execute("TRUNCATE TABLE #{table_name}", [])
|
7
|
+
|
8
|
+
csv = CSV.read(file_name)
|
9
|
+
columns = csv.shift
|
10
|
+
sql = "INSERT INTO #{table_name} ( #{columns.join(',')} )
|
11
|
+
VALUES (#{Array.new(columns.length, '?').join(',')})"
|
12
|
+
csv.each do |row|
|
13
|
+
db.execute(sql, row)
|
14
|
+
end
|
15
|
+
DataPipe2.log "#{table_name}: #{csv.length}", true
|
16
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
require 'csv'
|
2
|
+
|
3
|
+
def db_to_csv(sql, path, name)
|
4
|
+
db_env_name = 'DB'
|
5
|
+
db = DataPipe2.get_fluid_db(db_env_name)
|
6
|
+
|
7
|
+
file_path = "#{path}/#{name}"
|
8
|
+
File.delete(file_path) if File.exist?(file_path)
|
9
|
+
|
10
|
+
rst = db.queryForResultset(sql, [])
|
11
|
+
|
12
|
+
CSV.open(file_path, 'w') do |csv|
|
13
|
+
csv << rst[0].keys
|
14
|
+
rst.each { |r| csv << r.values }
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
require 'FluidDb'
|
2
|
+
require 'json'
|
3
|
+
|
4
|
+
def db_to_json(sql, path, name)
|
5
|
+
# Dir.mkdir(path) unless Dir.exists?(path)
|
6
|
+
db_env_name = 'DB'
|
7
|
+
db = DataPipe2.get_fluid_db(db_env_name)
|
8
|
+
|
9
|
+
file_path = "#{path}/#{name}"
|
10
|
+
File.delete(file_path) if File.exist?(file_path)
|
11
|
+
|
12
|
+
rst = db.queryForResultset(sql, [])
|
13
|
+
File.write(file_path, rst.to_json)
|
14
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
require 'FluidDb'
|
2
|
+
|
3
|
+
# DataPipe2 Helpers
|
4
|
+
module DataPipe2
|
5
|
+
def self.log(string, verbose = false)
|
6
|
+
type = verbose ? 'VERB' : 'INFO'
|
7
|
+
if !ENV['VERBOSE'].nil? || !verbose
|
8
|
+
timestamp = Time.new.strftime('%Y-%m-%d %H:%M:%S')
|
9
|
+
puts "[#{type}] #{timestamp} :: #{string}"
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
def self.log_dsl(name, string, verbose = false)
|
14
|
+
log "name: #{name}, #{string}", verbose
|
15
|
+
end
|
16
|
+
|
17
|
+
def self.get_env_var(name)
|
18
|
+
fail EnvironmentVariableNotFoundError, name if ENV[name].nil?
|
19
|
+
ENV[name]
|
20
|
+
end
|
21
|
+
|
22
|
+
def self.get_fluid_db(env_name)
|
23
|
+
uri_string = get_env_var(env_name)
|
24
|
+
log "uri: #{uri_string}", true
|
25
|
+
FluidDb::Db(uri_string)
|
26
|
+
end
|
27
|
+
end
|
data/lib/host.rb
ADDED
@@ -0,0 +1,44 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'rack'
|
3
|
+
require 'thin'
|
4
|
+
|
5
|
+
module DataPipe2
|
6
|
+
# Host
|
7
|
+
class Host
|
8
|
+
def run
|
9
|
+
libs = ENV['LIB'] ||= './lib'
|
10
|
+
libs.split(';').each do |path|
|
11
|
+
DataPipe2.log "Adding libdir: #{path}"
|
12
|
+
$LOAD_PATH.unshift path
|
13
|
+
end
|
14
|
+
|
15
|
+
@dsl_paths = ENV['DSL'] ||= './dsl'
|
16
|
+
DataPipe2.log "dsl_paths: #{@dsl_paths}"
|
17
|
+
@hash = {}
|
18
|
+
|
19
|
+
@hash['jobs'] = Jobs.new
|
20
|
+
long_run
|
21
|
+
end
|
22
|
+
|
23
|
+
def long_run
|
24
|
+
Kernel.loop do
|
25
|
+
begin
|
26
|
+
single_run
|
27
|
+
|
28
|
+
sleep 0.5
|
29
|
+
rescue SystemExit, Interrupt
|
30
|
+
puts 'Exiting on request ...'
|
31
|
+
break
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def single_run
|
37
|
+
@dsl_paths.split(';').each do |dsl_dir|
|
38
|
+
Dir.glob("#{dsl_dir}/*.dsl").each do |dsl_path|
|
39
|
+
@hash['jobs'].call dsl_path
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
data/lib/jobs.rb
ADDED
@@ -0,0 +1,83 @@
|
|
1
|
+
# Doc
|
2
|
+
module DataPipe2
|
3
|
+
require 'parse-cron'
|
4
|
+
|
5
|
+
# Hold a single job definition
|
6
|
+
class Job
|
7
|
+
attr_reader :name, :next, :errorList
|
8
|
+
|
9
|
+
def initialize(path)
|
10
|
+
@path = path
|
11
|
+
@name = File.basename(path, '.dsl')
|
12
|
+
@cron_string = ''
|
13
|
+
|
14
|
+
@error_list = []
|
15
|
+
set_cron
|
16
|
+
end
|
17
|
+
|
18
|
+
# Job Error -> Time, Exception Class Name, nsg, backtrace
|
19
|
+
def add_error(e)
|
20
|
+
@error_list << "#{e.class.name}: #{e.message}\n#{e.backtrace.join("\n")}"
|
21
|
+
end
|
22
|
+
|
23
|
+
def clear_error
|
24
|
+
@error_list = []
|
25
|
+
end
|
26
|
+
|
27
|
+
def run_now
|
28
|
+
@next = Time.now - 1
|
29
|
+
end
|
30
|
+
|
31
|
+
def set_cron
|
32
|
+
tmp = ENV["#{@name}_CRON"] ||= '0 0 * * *'
|
33
|
+
return if tmp == @cron_string
|
34
|
+
|
35
|
+
@cron_string = tmp
|
36
|
+
@cron = CronParser.new(@cron_string)
|
37
|
+
end
|
38
|
+
|
39
|
+
def call
|
40
|
+
run if Time.now > @next
|
41
|
+
end
|
42
|
+
|
43
|
+
def run
|
44
|
+
begin
|
45
|
+
DataPipe2.log "path: #{@path}", true
|
46
|
+
DataPipe2.log "dsl: #{@name}"
|
47
|
+
load @path
|
48
|
+
clear_error
|
49
|
+
rescue SystemExit, Interrupt
|
50
|
+
raise
|
51
|
+
rescue StandardError => e
|
52
|
+
string = e.message
|
53
|
+
p e.backtrace
|
54
|
+
DataPipe2.log_dsl @name, string
|
55
|
+
add_error(e)
|
56
|
+
end
|
57
|
+
|
58
|
+
set_cron
|
59
|
+
@next = @cron.next(Time.now)
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
# Hold all jobs
|
64
|
+
class Jobs
|
65
|
+
attr_reader :hash, :byName
|
66
|
+
|
67
|
+
def initialize
|
68
|
+
@hash = {}
|
69
|
+
@by_name = {}
|
70
|
+
end
|
71
|
+
|
72
|
+
def call(path)
|
73
|
+
if @hash[path].nil?
|
74
|
+
j = Job.new(path)
|
75
|
+
@hash[path] = j
|
76
|
+
@by_name[j.name.downcase] = j
|
77
|
+
j.run
|
78
|
+
else
|
79
|
+
@hash[path].call
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
metadata
ADDED
@@ -0,0 +1,51 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: datapipe2
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Guy Irvine
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2016-05-02 00:00:00.000000000 Z
|
12
|
+
dependencies: []
|
13
|
+
description: Helping to move data around your system
|
14
|
+
email: guy@guyirvine.com
|
15
|
+
executables:
|
16
|
+
- datapipe2
|
17
|
+
extensions: []
|
18
|
+
extra_rdoc_files: []
|
19
|
+
files:
|
20
|
+
- lib/datapipe2.rb
|
21
|
+
- lib/fns/csv_to_db.rb
|
22
|
+
- lib/fns/db_to_csv.rb
|
23
|
+
- lib/fns/db_to_json.rb
|
24
|
+
- lib/helper_functions.rb
|
25
|
+
- lib/host.rb
|
26
|
+
- lib/jobs.rb
|
27
|
+
- bin/datapipe2
|
28
|
+
homepage: http://rubygems.org/gems/datapipe2
|
29
|
+
licenses: []
|
30
|
+
metadata: {}
|
31
|
+
post_install_message:
|
32
|
+
rdoc_options: []
|
33
|
+
require_paths:
|
34
|
+
- lib
|
35
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
36
|
+
requirements:
|
37
|
+
- - '>='
|
38
|
+
- !ruby/object:Gem::Version
|
39
|
+
version: '0'
|
40
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
41
|
+
requirements:
|
42
|
+
- - '>='
|
43
|
+
- !ruby/object:Gem::Version
|
44
|
+
version: '0'
|
45
|
+
requirements: []
|
46
|
+
rubyforge_project:
|
47
|
+
rubygems_version: 2.0.14.1
|
48
|
+
signing_key:
|
49
|
+
specification_version: 4
|
50
|
+
summary: DataPipe2
|
51
|
+
test_files: []
|