sqlload 0.3.0beta
Sign up to get free protection for your applications and to get access to all the features.
- data/bin/sqlload +109 -0
- data/lib/sqlload.rb +127 -0
- data/lib/sqlload/database.rb +52 -0
- metadata +66 -0
data/bin/sqlload
ADDED
@@ -0,0 +1,109 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
# Copyright 2012 Ludovico Fischer
|
4
|
+
#
|
5
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
6
|
+
# you may not use this file except in compliance with the License.
|
7
|
+
# You may obtain a copy of the License at
|
8
|
+
#
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
10
|
+
#
|
11
|
+
# Unless required by applicable law or agreed to in writing, software
|
12
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
13
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
14
|
+
# See the License for the specific language governing permissions and
|
15
|
+
# limitations under the License.
|
16
|
+
require 'sqlload'
|
17
|
+
|
18
|
+
|
19
|
+
module Tasks
|
20
|
+
BASE_DIRECTORY = 'SQL'
|
21
|
+
def self.get_datasets()
|
22
|
+
datasets = []
|
23
|
+
Dir[File.join(BASE_DIRECTORY, '*')].each do |d|
|
24
|
+
if File.directory? d
|
25
|
+
datasets << DataSet.new(d)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
datasets
|
29
|
+
end
|
30
|
+
|
31
|
+
def self.list_available_datasets()
|
32
|
+
self.get_datasets().each do |d|
|
33
|
+
puts File.basename(d.directory)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
def self.load_dataset(dataset_name, user_options)
|
38
|
+
if dataset_name.nil? then abort 'You must specify a dataset to load' end
|
39
|
+
dataset = DataSet.new(File.join(BASE_DIRECTORY, dataset_name), user_options)
|
40
|
+
dataset.load
|
41
|
+
end
|
42
|
+
|
43
|
+
def self.delete_dataset(dataset_name, user_options)
|
44
|
+
if dataset_name.nil? then abort 'You must specify a dataset to reset' end
|
45
|
+
dataset = DataSet.new(File.join(BASE_DIRECTORY, dataset_name), user_options)
|
46
|
+
dataset.delete
|
47
|
+
end
|
48
|
+
|
49
|
+
def self.reset_dataset(dataset_name, user_options)
|
50
|
+
if dataset_name.nil? then abort 'You must specify a dataset to reset' end
|
51
|
+
dataset = DataSet.new(File.join(BASE_DIRECTORY, dataset_name), user_options)
|
52
|
+
dataset.reset
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
module User
|
57
|
+
def self.get_commandline_options(command_string)
|
58
|
+
require 'optparse'
|
59
|
+
|
60
|
+
options = {}
|
61
|
+
|
62
|
+
optparse = OptionParser.new do |opts|
|
63
|
+
opts.banner = 'Usage: sqlload [options] list|load|delete|reset [dataset]'
|
64
|
+
|
65
|
+
opts.version = '0.2.1'
|
66
|
+
|
67
|
+
opts.on('-U', '--user USERNAME', 'Specify username') do |user|
|
68
|
+
options[:user] = user
|
69
|
+
end
|
70
|
+
|
71
|
+
opts.on('-W', '--password PASSWORD', 'Specify password') do |password|
|
72
|
+
options[:password] = password
|
73
|
+
end
|
74
|
+
|
75
|
+
opts.on('-d', '--database DBNAME', 'Specify database name') do |dbname|
|
76
|
+
options[:dbname] = dbname
|
77
|
+
end
|
78
|
+
|
79
|
+
opts.on('-p', '--port PORT', 'Specify port') do |port|
|
80
|
+
options[:port] = port
|
81
|
+
end
|
82
|
+
|
83
|
+
opts.on_tail('-h', '--help', 'Displays this message') do
|
84
|
+
puts opts
|
85
|
+
exit
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
optparse.parse!(command_string)
|
90
|
+
options
|
91
|
+
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
user_options = User.get_commandline_options(ARGV)
|
96
|
+
dataset_name = ARGV[1]
|
97
|
+
|
98
|
+
case ARGV[0]
|
99
|
+
when 'list'
|
100
|
+
Tasks.list_available_datasets
|
101
|
+
when 'load'
|
102
|
+
Tasks.load_dataset(dataset_name, user_options)
|
103
|
+
when 'delete'
|
104
|
+
Tasks.delete_dataset(dataset_name, user_options)
|
105
|
+
when 'reset'
|
106
|
+
Tasks.reset_dataset(dataset_name, user_options)
|
107
|
+
|
108
|
+
else abort 'You must specify one of list, load, delete or reset.'
|
109
|
+
end
|
data/lib/sqlload.rb
ADDED
@@ -0,0 +1,127 @@
|
|
1
|
+
# Copyright 2012 Ludovico Fischer
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
require 'sqlload/database'
|
16
|
+
require 'pty'
|
17
|
+
require 'expect'
|
18
|
+
|
19
|
+
##
|
20
|
+
# A collection of data that can be inserted together into a database
|
21
|
+
class DataSet
|
22
|
+
include DBConfig, DB
|
23
|
+
attr_accessor :ups, :downs, :raws, :config, :directory
|
24
|
+
|
25
|
+
##
|
26
|
+
# Creates a dataset using the SQL and the configuration contained
|
27
|
+
# in directory. The supplied user configuration overrides
|
28
|
+
# the configuration contained in the directory
|
29
|
+
def initialize(directory, user_config = {})
|
30
|
+
@ups = []
|
31
|
+
@downs = []
|
32
|
+
@raws = []
|
33
|
+
@directory = directory
|
34
|
+
populate()
|
35
|
+
@config = load_config(directory).merge(user_config)
|
36
|
+
if insufficient(config)
|
37
|
+
raise ArgumentError, 'No database name'
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
##
|
42
|
+
# If there are reset scripts available,
|
43
|
+
# executes them before executing once again the regular scripts
|
44
|
+
def reset
|
45
|
+
if @downs.empty?
|
46
|
+
raise ArgumentError, 'There are no reset scripts to run'
|
47
|
+
end
|
48
|
+
delete
|
49
|
+
load
|
50
|
+
end
|
51
|
+
|
52
|
+
##
|
53
|
+
# Executes the SQL scripts associated with the dataset.
|
54
|
+
def load
|
55
|
+
get_db_connection(@config) do |db_connection|
|
56
|
+
@ups.each do |s|
|
57
|
+
result = db_connection.exec(s.statement)
|
58
|
+
puts "#{s.filename}: #{result.cmd_status}"
|
59
|
+
end
|
60
|
+
end
|
61
|
+
psql_count = 0
|
62
|
+
@raws.each do |s|
|
63
|
+
system({'PGPASSWORD' => @config[:password]}, "psql -h localhost -U #{@config[:user]} -d #{@config[:dbname]} < #{s.filename}")
|
64
|
+
psql_count += 1
|
65
|
+
end
|
66
|
+
puts "psql invoked #{psql_count} time(s)"
|
67
|
+
end
|
68
|
+
|
69
|
+
##
|
70
|
+
# Executes the SQL scripts marked as reset scripts
|
71
|
+
def delete
|
72
|
+
get_db_connection(@config) do |db_connection|
|
73
|
+
@downs.each do |s|
|
74
|
+
result = db_connection.exec(s.statement)
|
75
|
+
puts "#{s.filename}: #{result.cmd_status}"
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
def to_s
|
81
|
+
File.basename(@directory)
|
82
|
+
end
|
83
|
+
|
84
|
+
private
|
85
|
+
|
86
|
+
DataPiece = Struct.new(:filename, :statement)
|
87
|
+
|
88
|
+
# Fills the list of statements to execute from the information
|
89
|
+
# found by inspecting the directory contents
|
90
|
+
def populate()
|
91
|
+
require 'find'
|
92
|
+
Find.find(@directory) do |path|
|
93
|
+
if File.file?(path) && File.extname(path) == '.sql'
|
94
|
+
data = DataPiece.new(path, File.read(path))
|
95
|
+
if is_raws path
|
96
|
+
@raws << data
|
97
|
+
elsif is_downs path
|
98
|
+
@downs << data
|
99
|
+
elsif is_ups path
|
100
|
+
@ups << data
|
101
|
+
end
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
@ups.sort! do |x, y|
|
106
|
+
File.basename(x.filename) <=> File.basename(y.filename)
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
110
|
+
# Returns true if the configuration does not contain enough information
|
111
|
+
# to connect to a database
|
112
|
+
def insufficient(config)
|
113
|
+
config[:dbname].nil?
|
114
|
+
end
|
115
|
+
|
116
|
+
def is_ups(path)
|
117
|
+
File.extname(path) == '.sql' && File.basename(path) != 'reset.sql'
|
118
|
+
end
|
119
|
+
|
120
|
+
def is_downs(path)
|
121
|
+
File.basename(path) == 'reset.sql'
|
122
|
+
end
|
123
|
+
|
124
|
+
def is_raws(path)
|
125
|
+
File.fnmatch?('*psql*', path)
|
126
|
+
end
|
127
|
+
end
|
@@ -0,0 +1,52 @@
|
|
1
|
+
# Copyright 2012 Ludovico Fischer
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
module DB
|
16
|
+
require 'pg'
|
17
|
+
##
|
18
|
+
# Yields a database connection configured with the supplied options.
|
19
|
+
# This method will attempt to close the connection once the block terminates.
|
20
|
+
def get_db_connection(options)
|
21
|
+
dbname = options[:dbname]
|
22
|
+
unless dbname
|
23
|
+
abort "You must specify a database name"
|
24
|
+
end
|
25
|
+
user = options.fetch(:user, dbname)
|
26
|
+
password = options[:password]
|
27
|
+
port = options.fetch(:port, 5432)
|
28
|
+
begin
|
29
|
+
db_connection = PG.connect(:host => 'localhost', :port => port, :dbname => dbname, :user => user, :password => password)
|
30
|
+
yield db_connection
|
31
|
+
ensure
|
32
|
+
db_connection.finish unless db_connection.nil?
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
|
38
|
+
module DBConfig
|
39
|
+
##
|
40
|
+
# Generates a configuration hash from a JSON file
|
41
|
+
#
|
42
|
+
# [path] the path to the dataset to load a config for
|
43
|
+
def load_config(path)
|
44
|
+
require 'json'
|
45
|
+
|
46
|
+
config = JSON.parse(File.read(self.get_config_path(path)), :symbolize_names => true)
|
47
|
+
end
|
48
|
+
|
49
|
+
def get_config_path(dataset_path)
|
50
|
+
File.join(dataset_path, 'config.json')
|
51
|
+
end
|
52
|
+
end
|
metadata
ADDED
@@ -0,0 +1,66 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: sqlload
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.3.0beta
|
5
|
+
prerelease: 5
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Ludovico Fischer
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2012-04-24 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: pg
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ~>
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: 0.13.0
|
22
|
+
type: :runtime
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ~>
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: 0.13.0
|
30
|
+
description: ! 'sqlload inspects a directory hierarchy: for each subdirectory, sqlload
|
31
|
+
offers to execute any SQL file, using the database specified in the JSON configuration.'
|
32
|
+
email: ludovico.fischer@lunatech.com
|
33
|
+
executables:
|
34
|
+
- sqlload
|
35
|
+
extensions: []
|
36
|
+
extra_rdoc_files: []
|
37
|
+
files:
|
38
|
+
- lib/sqlload.rb
|
39
|
+
- lib/sqlload/database.rb
|
40
|
+
- bin/sqlload
|
41
|
+
homepage: https://github.com/ludovicofischer/sqlloader
|
42
|
+
licenses:
|
43
|
+
- Apache License, Version 2.0
|
44
|
+
post_install_message:
|
45
|
+
rdoc_options: []
|
46
|
+
require_paths:
|
47
|
+
- lib
|
48
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
49
|
+
none: false
|
50
|
+
requirements:
|
51
|
+
- - ! '>='
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: '0'
|
54
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
55
|
+
none: false
|
56
|
+
requirements:
|
57
|
+
- - ! '>'
|
58
|
+
- !ruby/object:Gem::Version
|
59
|
+
version: 1.3.1
|
60
|
+
requirements: []
|
61
|
+
rubyforge_project:
|
62
|
+
rubygems_version: 1.8.23
|
63
|
+
signing_key:
|
64
|
+
specification_version: 3
|
65
|
+
summary: Load and execute SQL from a directory hierarchy. Only supports PostgreSQL
|
66
|
+
test_files: []
|