sc_analytics 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/LICENSE.txt +21 -0
- data/README.md +41 -0
- data/Rakefile +6 -0
- data/bin/console +15 -0
- data/bin/run_report.rb +22 -0
- data/bin/setup +7 -0
- data/example/db_configs_sample.yml +23 -0
- data/lib/sc_analytics/connections/connection.rb +28 -0
- data/lib/sc_analytics/connections/connection_pool.rb +28 -0
- data/lib/sc_analytics/connections/credentials.rb +31 -0
- data/lib/sc_analytics/connections/oracle.rb +58 -0
- data/lib/sc_analytics/connections.rb +14 -0
- data/lib/sc_analytics/exceptions.rb +4 -0
- data/lib/sc_analytics/query.rb +281 -0
- data/lib/sc_analytics/version.rb +3 -0
- data/lib/sc_analytics.rb +76 -0
- data/spec/sc_analytics/query/query_spec.rb +11 -0
- data/spec/sc_analytics_spec.rb +7 -0
- data/spec/spec_helper.rb +2 -0
- metadata +137 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 174fbe2000c4f79d76fc86330eac07a14d9f4807
|
4
|
+
data.tar.gz: 51f8eb358ef21862d6ae311e0c8bc45671e45045
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 3ae18c5a68627339343f6d15c6076c161d30dfe87bf98a656a76cd1fc0c01d094a35ec011fbe17b2e242aa0114676a82bc745a9953277bfe960b9ec5b2fc3f19
|
7
|
+
data.tar.gz: 101e0b4cb37f6cda59b7131ee7ac536f1132677f5cb339796ea0f865a3dd8c8e2bdde37668260680e5a492377a0091fed783ef8dd4c4793c8956e3ef65f48f4e
|
data/LICENSE.txt
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2015 Dave Rowan
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
13
|
+
all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,41 @@
|
|
1
|
+
# ScAnalytics
|
2
|
+
|
3
|
+
Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/sc_analytics`. To experiment with that code, run `bin/console` for an interactive prompt.
|
4
|
+
|
5
|
+
TODO: Delete this and the text above, and describe your gem
|
6
|
+
|
7
|
+
## Installation
|
8
|
+
|
9
|
+
Add this line to your application's Gemfile:
|
10
|
+
|
11
|
+
```ruby
|
12
|
+
gem 'sc_analytics'
|
13
|
+
```
|
14
|
+
|
15
|
+
And then execute:
|
16
|
+
|
17
|
+
$ bundle
|
18
|
+
|
19
|
+
Or install it yourself as:
|
20
|
+
|
21
|
+
$ gem install sc_analytics
|
22
|
+
|
23
|
+
## Usage
|
24
|
+
|
25
|
+
TODO: Write usage instructions here
|
26
|
+
|
27
|
+
## Development
|
28
|
+
|
29
|
+
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
30
|
+
|
31
|
+
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
32
|
+
|
33
|
+
## Contributing
|
34
|
+
|
35
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/sc_analytics.
|
36
|
+
|
37
|
+
|
38
|
+
## License
|
39
|
+
|
40
|
+
The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
|
41
|
+
|
data/Rakefile
ADDED
data/bin/console
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
#!C:/Ruby21-x64/bin/ruby.exe
|
2
|
+
|
3
|
+
require "bundler/setup"
|
4
|
+
require "sc_analytics"
|
5
|
+
include SCAnalytics
|
6
|
+
|
7
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
8
|
+
# with your gem easier. You can also use a different console, if you like.
|
9
|
+
|
10
|
+
# (If you use this, don't forget to add pry to your Gemfile!)
|
11
|
+
# require "pry"
|
12
|
+
# Pry.start
|
13
|
+
|
14
|
+
require "irb"
|
15
|
+
IRB.start
|
data/bin/run_report.rb
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
require 'sc_analytics'
|
2
|
+
include SCAnalytics
|
3
|
+
|
4
|
+
report = ARGV[0]
|
5
|
+
unless report
|
6
|
+
puts "Please enter the name of the report you wish to run"
|
7
|
+
report = STDIN.gets.strip
|
8
|
+
raise unless report
|
9
|
+
end
|
10
|
+
|
11
|
+
*report_path, report_name = report.split("/")
|
12
|
+
report_name = report_name.sub(/(\.rb)*$/, ".rb")
|
13
|
+
|
14
|
+
puts
|
15
|
+
job_start = alert("BEGINNING JOB #{report_name.gsub(/\.rb$/,'')}")
|
16
|
+
|
17
|
+
Dir.chdir(report_path.join("/")) unless report_path.empty?
|
18
|
+
|
19
|
+
instructions = File.read(report_name)
|
20
|
+
SCAnalytics.instance_eval(instructions, report_name)
|
21
|
+
|
22
|
+
alert("COMPLETED JOB #{report_name} in #{(Time.now - job_start)/60} minutes")
|
data/bin/setup
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
#
|
2
|
+
#
|
3
|
+
#
|
4
|
+
# db_name:
|
5
|
+
# connection:
|
6
|
+
# type:
|
7
|
+
# host:
|
8
|
+
# port:
|
9
|
+
# service:
|
10
|
+
# credentials:
|
11
|
+
# username:
|
12
|
+
# password:
|
13
|
+
####################################################################
|
14
|
+
---
|
15
|
+
db_name:
|
16
|
+
connection:
|
17
|
+
type: oracle
|
18
|
+
host: scanalytics.com
|
19
|
+
port: 63011
|
20
|
+
service: scanalytics.service.com
|
21
|
+
credentials:
|
22
|
+
username: user
|
23
|
+
password: pass
|
@@ -0,0 +1,28 @@
|
|
1
|
+
module SCAnalytics
|
2
|
+
module Connections
|
3
|
+
|
4
|
+
Struct.new("QueryResult",:headers,:rows)
|
5
|
+
|
6
|
+
class Connection
|
7
|
+
attr_reader :host, :port, :service, :username
|
8
|
+
|
9
|
+
def initialize(connection_name, options)
|
10
|
+
@connection_name = connection_name
|
11
|
+
@host = options[:connection][:host]
|
12
|
+
@port = options[:connection][:port]
|
13
|
+
@service = options[:connection][:service]
|
14
|
+
@username = options[:credentials][:username]
|
15
|
+
@password = options[:credentials][:password]
|
16
|
+
end
|
17
|
+
|
18
|
+
private
|
19
|
+
|
20
|
+
def connect
|
21
|
+
alert "Connecting to #{@connection_name.to_s} as user #{@username.to_s} . . . "
|
22
|
+
yield
|
23
|
+
alert "Successfully connected to #{@connection_name.to_s} as user #{@username.to_s}!"
|
24
|
+
end
|
25
|
+
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
module SCAnalytics
|
2
|
+
module Connections
|
3
|
+
module ConnectionPool
|
4
|
+
@active_connections = {}
|
5
|
+
|
6
|
+
def self.get_connection_to(server_name)
|
7
|
+
establish_connection_to server_name.downcase.to_sym unless @active_connections[server_name.downcase.to_sym]
|
8
|
+
|
9
|
+
@active_connections[server_name.downcase.to_sym]
|
10
|
+
end
|
11
|
+
|
12
|
+
def self.establish_connection_to(server_name)
|
13
|
+
config = Credentials.connection_details server_name
|
14
|
+
@active_connections[server_name] = case Credentials.connection_type(server_name)
|
15
|
+
when "oracle"
|
16
|
+
require 'sc_analytics/connections/oracle'
|
17
|
+
Oracle.new server_name, config
|
18
|
+
when "mysql"
|
19
|
+
Mysql.new server_name, config
|
20
|
+
else
|
21
|
+
raise
|
22
|
+
end
|
23
|
+
|
24
|
+
@active_connections[server_name]
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
module SCAnalytics
|
2
|
+
module Connections
|
3
|
+
module Credentials
|
4
|
+
include SCAnalytics
|
5
|
+
require 'yaml'
|
6
|
+
|
7
|
+
CONFIGS_FILE = "G:/Rowan/db_configs.yml"
|
8
|
+
@credentials = YAML.load(File.read(CONFIGS_FILE))
|
9
|
+
|
10
|
+
def self.symbolize_keys_for(hsh)
|
11
|
+
hsh = hsh.inject({}) {|new_hsh,(k,v)| new_hsh[k.downcase.to_sym] = v; new_hsh}
|
12
|
+
|
13
|
+
hsh.keys.each do |k|
|
14
|
+
hsh[k] = symbolize_keys_for hsh[k] if hsh[k].is_a? Hash
|
15
|
+
end
|
16
|
+
|
17
|
+
hsh
|
18
|
+
end
|
19
|
+
|
20
|
+
@credentials = symbolize_keys_for @credentials
|
21
|
+
|
22
|
+
def self.connection_type(db_name)
|
23
|
+
@credentials[db_name.downcase.to_sym][:connection][:type].downcase
|
24
|
+
end
|
25
|
+
|
26
|
+
def self.connection_details(db_name)
|
27
|
+
@credentials[db_name.downcase.to_sym]
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,58 @@
|
|
1
|
+
module SCAnalytics
|
2
|
+
module Connections
|
3
|
+
class Oracle < Connection
|
4
|
+
require 'oci8'
|
5
|
+
OCI8::BindType::Mapping[:number_unknown_prec] = OCI8::BindType::BasicNumberType
|
6
|
+
|
7
|
+
def initialize(connection_name, options)
|
8
|
+
super
|
9
|
+
|
10
|
+
parse_db_url
|
11
|
+
connect
|
12
|
+
end
|
13
|
+
|
14
|
+
def define_col_types_for(query, cursor)
|
15
|
+
(query.columns_to_cast || {}).each_pair do |type, cols|
|
16
|
+
cols.each do |col|
|
17
|
+
cursor.define(col, type)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def get_cursor_for(query)
|
23
|
+
cursor = @client.parse(query.sql)
|
24
|
+
end
|
25
|
+
|
26
|
+
def run(query, cursor)
|
27
|
+
cursor.exec
|
28
|
+
define_col_types_for query, cursor
|
29
|
+
|
30
|
+
headers = cursor.get_col_names
|
31
|
+
results = []
|
32
|
+
|
33
|
+
while row = cursor.fetch
|
34
|
+
results << row
|
35
|
+
end
|
36
|
+
|
37
|
+
Struct::QueryResult.new(headers, results)
|
38
|
+
end
|
39
|
+
|
40
|
+
def connect
|
41
|
+
super do
|
42
|
+
@client = OCI8.new @username, @password, @db_url
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
def parse_db_url
|
47
|
+
url = []
|
48
|
+
url << "//#{@host}" if @host
|
49
|
+
url << ":#{@port}" if @port
|
50
|
+
url << "/" if @host
|
51
|
+
url << @service
|
52
|
+
|
53
|
+
@db_url = url.join
|
54
|
+
end
|
55
|
+
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
module SCAnalytics
|
2
|
+
module Connections
|
3
|
+
require 'sc_analytics/connections/credentials'
|
4
|
+
require 'sc_analytics/connections/connection_pool'
|
5
|
+
require 'sc_analytics/connections/connection'
|
6
|
+
|
7
|
+
def self.use(server_name)
|
8
|
+
ConnectionPool.get_connection_to server_name
|
9
|
+
end
|
10
|
+
|
11
|
+
def self.test_connections
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
@@ -0,0 +1,281 @@
|
|
1
|
+
module SCAnalytics
|
2
|
+
|
3
|
+
class Query
|
4
|
+
require 'erb'
|
5
|
+
|
6
|
+
attr_reader :connection, :results, :query_name, :bind_variables, :csv_file, :combined_csv_file, :csv_dir, :csv_combined_dir, :cursor, :columns_to_cast, :start_time, :end_time
|
7
|
+
attr_accessor :sql, :database, :auto_cast, :concurrent
|
8
|
+
alias_method :result, :results
|
9
|
+
alias_method :name, :query_name
|
10
|
+
alias_method :csv_files, :csv_file
|
11
|
+
|
12
|
+
REQUIRED_PARAMS = [:query_name, :database, :sql]
|
13
|
+
DEFAULT_DATA_DIR = :data_files
|
14
|
+
DEFAULT_SQL_DIR = :sql_queries
|
15
|
+
|
16
|
+
@@total_queries = 0
|
17
|
+
|
18
|
+
def initialize(&block)
|
19
|
+
yield self if block_given?
|
20
|
+
|
21
|
+
check_params
|
22
|
+
fork_query if @should_fork
|
23
|
+
end
|
24
|
+
|
25
|
+
def bind_variables(var_hash = {})
|
26
|
+
@bind_variables||= {}
|
27
|
+
var_hash.each_pair do |bind_var, value|
|
28
|
+
@bind_variables[bind_var] = value
|
29
|
+
end
|
30
|
+
end
|
31
|
+
alias_method :bind_variable, :bind_variables
|
32
|
+
|
33
|
+
def check_params
|
34
|
+
missing_params = []
|
35
|
+
|
36
|
+
REQUIRED_PARAMS.each do |param|
|
37
|
+
missing_params << param unless instance_variable_get("@#{param}")
|
38
|
+
end
|
39
|
+
|
40
|
+
raise SCAnalytics::QueryError, "must set query parameter#{"s" if missing_params.length > 1 }: #{missing_params.join(", ")}" unless missing_params.empty?
|
41
|
+
end
|
42
|
+
|
43
|
+
def cast_columns(cols, type)
|
44
|
+
@columns_to_cast||= {}
|
45
|
+
@columns_to_cast[type]||= []
|
46
|
+
|
47
|
+
@columns_to_cast[type] << cols
|
48
|
+
@columns_to_cast[type].flatten!
|
49
|
+
|
50
|
+
return nil
|
51
|
+
end
|
52
|
+
|
53
|
+
def database=(*db)
|
54
|
+
@database = db.flatten
|
55
|
+
@should_fork = (@database.size > 1)
|
56
|
+
end
|
57
|
+
|
58
|
+
def default_file_name(combined_file = false)
|
59
|
+
file_names = []
|
60
|
+
|
61
|
+
if combined_file == :combined
|
62
|
+
file_names = "#{@database.join("_")}_#{self.name.gsub(" ","_")}_combined"
|
63
|
+
else
|
64
|
+
@database.each do |db|
|
65
|
+
file_names << "#{db}_#{self.name.gsub(" ","_")}"
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
file_names
|
70
|
+
end
|
71
|
+
alias_method :default_file_names, :default_file_name
|
72
|
+
|
73
|
+
def forked?
|
74
|
+
@should_fork
|
75
|
+
end
|
76
|
+
|
77
|
+
def export_csv
|
78
|
+
CSV.open(@csv_file.first,"w") do |csv|
|
79
|
+
csv << @results.headers
|
80
|
+
|
81
|
+
@results.rows.each do |row|
|
82
|
+
csv << row
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
def export_csv_combined
|
88
|
+
CSV.open(@combined_csv_file,"w") do |csv|
|
89
|
+
csv << @results.values.first.headers
|
90
|
+
end
|
91
|
+
|
92
|
+
@results.each_value do |qry|
|
93
|
+
CSV.open(@combined_csv_file,"a") do |csv|
|
94
|
+
qry.rows.each do |row|
|
95
|
+
csv << row
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
def name=(query_name)
|
102
|
+
@query_name = query_name.to_s
|
103
|
+
end
|
104
|
+
|
105
|
+
def run
|
106
|
+
unless @should_fork
|
107
|
+
@connection = Connections.use @database.first
|
108
|
+
@cursor = @connection.get_cursor_for self
|
109
|
+
bind_vars_to_cursor if @bind_variables
|
110
|
+
|
111
|
+
@start_time = Time.now
|
112
|
+
alert("Running query #{self.name}")
|
113
|
+
|
114
|
+
@results = @connection.run(self, @cursor)
|
115
|
+
|
116
|
+
@end_time = Time.now
|
117
|
+
alert("Completed query #{self.name} in #{time_elapsed/60} minutes")
|
118
|
+
|
119
|
+
@@total_queries += 1
|
120
|
+
|
121
|
+
export_csv if @to_csv
|
122
|
+
else
|
123
|
+
@results = run_forked_queries
|
124
|
+
export_csv_combined if @to_csv_combined
|
125
|
+
end
|
126
|
+
end
|
127
|
+
|
128
|
+
def sql
|
129
|
+
@sql.result(binding)
|
130
|
+
end
|
131
|
+
|
132
|
+
def sql=(arg)
|
133
|
+
raise SCAnalytics::QueryError, "a SQL statement is already bound to query #{self.name}" if @sql
|
134
|
+
|
135
|
+
file_name = apply_format arg, :suggest => {:ext => :sql}
|
136
|
+
file_name_in_subfolder = apply_format file_name, :suggest => {:dir => DEFAULT_SQL_DIR}
|
137
|
+
|
138
|
+
sql = case
|
139
|
+
when File.exists?(file_name)
|
140
|
+
File.read(file_name)
|
141
|
+
when File.exists?(file_name_in_subfolder)
|
142
|
+
File.read(file_name_in_subfolder)
|
143
|
+
else
|
144
|
+
# assume that the argument is a sql statement if no file is found
|
145
|
+
arg
|
146
|
+
end
|
147
|
+
|
148
|
+
# remove semi-colon from end of sql statement (if present) because ruby-oci8 does not accept these.
|
149
|
+
# also remove all trailing new lines and white space to make sure we find any trailing semi-colons.
|
150
|
+
while sql.chomp! || sql.chomp!(' ') || sql.chomp!(';'); end
|
151
|
+
@sql = ERB.new(sql)
|
152
|
+
ensure
|
153
|
+
@results = nil
|
154
|
+
end
|
155
|
+
|
156
|
+
def time_elapsed
|
157
|
+
if @start_time && @end_time
|
158
|
+
@end_time - @start_time
|
159
|
+
else
|
160
|
+
nil
|
161
|
+
end
|
162
|
+
end
|
163
|
+
|
164
|
+
def to_csv(*file_names)
|
165
|
+
@to_csv = true
|
166
|
+
@csv_dir = File.expand_path((file_names.last.is_a?(Hash) ? file_names.pop[:csv_dir] : DEFAULT_DATA_DIR).to_s)
|
167
|
+
file_names = default_file_names if file_names.empty?
|
168
|
+
|
169
|
+
@csv_file = file_names.flatten.map{|file_name| apply_format file_name, :enforce => {:ext => :csv}, :suggest => {:dir => @csv_dir}}
|
170
|
+
|
171
|
+
raise SCAnalytics::QueryError, "must supply one csv file name for each database bound to query #{self.name}" unless @csv_file.length == @database.length
|
172
|
+
end
|
173
|
+
|
174
|
+
def to_csv_combined(*file_name)
|
175
|
+
raise ArgumentError, "too many arguments - #{file_name.length} for 2" if file_name.length > 2
|
176
|
+
raise ArgumentError, "optional second argument must be of type Hash" if file_name.length == 2 && ! file_name.last.is_a?(Hash)
|
177
|
+
@to_csv_combined = true
|
178
|
+
@csv_combined_dir = File.expand_path((file_name.last.is_a?(Hash) ? file_name.pop[:csv_dir] : DEFAULT_DATA_DIR).to_s)
|
179
|
+
file_name = file_name.empty? ? default_file_name(:combined) : file_name.first
|
180
|
+
|
181
|
+
@combined_csv_file = apply_format file_name, :enforce => {:ext => :csv}, :suggest => {:dir => @csv_combined_dir}
|
182
|
+
end
|
183
|
+
|
184
|
+
def self.total_queries
|
185
|
+
@@total_queries
|
186
|
+
end
|
187
|
+
|
188
|
+
private
|
189
|
+
|
190
|
+
def bind_vars_to_cursor
|
191
|
+
@bind_variables.each_pair do |bind_var, value|
|
192
|
+
@cursor.bind_param(":#{bind_var.to_s}", value)
|
193
|
+
end
|
194
|
+
end
|
195
|
+
|
196
|
+
def apply_format(file_name, options = {})
|
197
|
+
unknown_options = []
|
198
|
+
options.keys.each do |option|
|
199
|
+
unknown_options << option unless [:enforce, :suggest].include?(option.to_sym)
|
200
|
+
end
|
201
|
+
raise "unknown format option#{"s" if unknown_options.length > 1}: #{unknown_options.join(", ")}" unless unknown_options.empty?
|
202
|
+
|
203
|
+
options.each_pair do |option, settings|
|
204
|
+
settings.each_pair do |setting, format|
|
205
|
+
file_name = send("#{option.to_s}_#{setting.to_s}".to_sym, file_name, format)
|
206
|
+
end
|
207
|
+
end
|
208
|
+
|
209
|
+
file_name
|
210
|
+
end
|
211
|
+
|
212
|
+
def enforce_ext(file_name, ext)
|
213
|
+
return file_name if file_name.match(/\.#{ext}$/)
|
214
|
+
|
215
|
+
# remove any file extension provided and make sure
|
216
|
+
# that the file_name ends with required extension
|
217
|
+
file_name = file_name.sub(/\.\w+$(?!\.)/i, "")
|
218
|
+
file_name + ".#{ext}"
|
219
|
+
end
|
220
|
+
|
221
|
+
def enforce_dir(file_name, dir)
|
222
|
+
return file_name if File.dirname(file_name) == dir.to_s
|
223
|
+
|
224
|
+
FileUtils.mkdir_p(dir.to_s)
|
225
|
+
file_name = file_name.sub(File.dirname(file_name), "") unless File.dirname(file_name) == "."
|
226
|
+
"#{dir}/" + file_name
|
227
|
+
end
|
228
|
+
|
229
|
+
def suggest_ext(file_name, ext)
|
230
|
+
return file_name if file_name.match(/\.\w+$(?!\.)/i)
|
231
|
+
|
232
|
+
file_name + ".#{ext}"
|
233
|
+
end
|
234
|
+
|
235
|
+
def suggest_dir(file_name, dir)
|
236
|
+
return file_name unless File.dirname(file_name) == "."
|
237
|
+
|
238
|
+
FileUtils.mkdir_p(dir.to_s)
|
239
|
+
"#{dir}/" + file_name
|
240
|
+
end
|
241
|
+
|
242
|
+
def fork_query
|
243
|
+
@tines = {}
|
244
|
+
|
245
|
+
@database.each_with_index do |db, i|
|
246
|
+
forked_query = self.dup
|
247
|
+
forked_query.database = db
|
248
|
+
forked_query.to_csv(@csv_file[i], :csv_dir => @csv_dir) if @to_csv
|
249
|
+
forked_query.name = "#{db.to_s}_#{self.name}"
|
250
|
+
@tines[db] = forked_query
|
251
|
+
end
|
252
|
+
end
|
253
|
+
|
254
|
+
def run_forked_queries
|
255
|
+
@start_time = Time.now
|
256
|
+
|
257
|
+
results = {}
|
258
|
+
threads = []
|
259
|
+
@tines.each_value do |query|
|
260
|
+
threads << Thread.new do
|
261
|
+
Thread.stop
|
262
|
+
query.run
|
263
|
+
results[query.database.first] = query.results
|
264
|
+
end
|
265
|
+
end
|
266
|
+
|
267
|
+
threads.each{|thread| until thread.stop?; end}
|
268
|
+
|
269
|
+
threads.each do |thread|
|
270
|
+
thread.run
|
271
|
+
thread.join if @concurrent == false
|
272
|
+
end
|
273
|
+
|
274
|
+
threads.each{|thread| thread.join }
|
275
|
+
|
276
|
+
@end_time = Time.now
|
277
|
+
return results
|
278
|
+
end
|
279
|
+
|
280
|
+
end
|
281
|
+
end
|
data/lib/sc_analytics.rb
ADDED
@@ -0,0 +1,76 @@
|
|
1
|
+
require 'sc_analytics/version'
|
2
|
+
require 'sc_analytics/query'
|
3
|
+
require 'sc_analytics/connections'
|
4
|
+
require 'sc_analytics/exceptions'
|
5
|
+
|
6
|
+
require 'date'
|
7
|
+
require 'fileutils'
|
8
|
+
require 'csv'
|
9
|
+
require 'win32ole'
|
10
|
+
require 'thread'
|
11
|
+
|
12
|
+
begin
|
13
|
+
# for development environment
|
14
|
+
require 'pry'
|
15
|
+
rescue LoadError
|
16
|
+
end
|
17
|
+
|
18
|
+
module SCAnalytics
|
19
|
+
def run_query(*queries)
|
20
|
+
queries.each{|qry| qry.run }
|
21
|
+
end
|
22
|
+
alias :run_queries :run_query
|
23
|
+
alias :run_in_sequence :run_query
|
24
|
+
|
25
|
+
def run_in_parallel(*queries)
|
26
|
+
queries.each { |query| raise "cannot run forked query #{qry.name} in parallel with other queries" if query.forked? && queries.length > 1 }
|
27
|
+
|
28
|
+
threads = []
|
29
|
+
queries.each do |query|
|
30
|
+
threads << Thread.new do
|
31
|
+
query.run
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
threads.each{|thread| thread.join }
|
36
|
+
end
|
37
|
+
|
38
|
+
def alert(status)
|
39
|
+
curr_time = Time.now
|
40
|
+
puts "#{curr_time.strftime("%m/%d/%Y %H:%M:%S")} #{status}"
|
41
|
+
puts
|
42
|
+
return curr_time
|
43
|
+
end
|
44
|
+
|
45
|
+
def open_excel_file(file_name)
|
46
|
+
excel_instance.Workbooks.open("#{Dir.pwd}/#{file_name}")
|
47
|
+
nil
|
48
|
+
end
|
49
|
+
|
50
|
+
def run_excel_macro(macro_name, arg = nil)
|
51
|
+
if arg
|
52
|
+
excel_instance.run(macro_name, arg)
|
53
|
+
else
|
54
|
+
excel_instance.run(macro_name)
|
55
|
+
end
|
56
|
+
|
57
|
+
nil
|
58
|
+
end
|
59
|
+
|
60
|
+
def quit_excel
|
61
|
+
excel_instance.quit
|
62
|
+
nil
|
63
|
+
end
|
64
|
+
|
65
|
+
private
|
66
|
+
|
67
|
+
def excel_instance
|
68
|
+
unless @excel
|
69
|
+
@excel = WIN32OLE.new('Excel.Application')
|
70
|
+
@excel.visible = true
|
71
|
+
#at_exit {@excel.quit}
|
72
|
+
end
|
73
|
+
|
74
|
+
@excel
|
75
|
+
end
|
76
|
+
end
|
data/spec/spec_helper.rb
ADDED
metadata
ADDED
@@ -0,0 +1,137 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: sc_analytics
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Dave Rowan
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2015-10-30 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: bundler
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.10'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1.10'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rake
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '10.0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '10.0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rspec
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: pry
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: ruby-oci8
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ">="
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :runtime
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
83
|
+
description:
|
84
|
+
email:
|
85
|
+
- DRowan99@gmail.com
|
86
|
+
executables: []
|
87
|
+
extensions: []
|
88
|
+
extra_rdoc_files: []
|
89
|
+
files:
|
90
|
+
- LICENSE.txt
|
91
|
+
- README.md
|
92
|
+
- Rakefile
|
93
|
+
- bin/console
|
94
|
+
- bin/run_report.rb
|
95
|
+
- bin/setup
|
96
|
+
- example/db_configs_sample.yml
|
97
|
+
- lib/sc_analytics.rb
|
98
|
+
- lib/sc_analytics/connections.rb
|
99
|
+
- lib/sc_analytics/connections/connection.rb
|
100
|
+
- lib/sc_analytics/connections/connection_pool.rb
|
101
|
+
- lib/sc_analytics/connections/credentials.rb
|
102
|
+
- lib/sc_analytics/connections/oracle.rb
|
103
|
+
- lib/sc_analytics/exceptions.rb
|
104
|
+
- lib/sc_analytics/query.rb
|
105
|
+
- lib/sc_analytics/version.rb
|
106
|
+
- spec/sc_analytics/query/query_spec.rb
|
107
|
+
- spec/sc_analytics_spec.rb
|
108
|
+
- spec/spec_helper.rb
|
109
|
+
homepage:
|
110
|
+
licenses:
|
111
|
+
- MIT
|
112
|
+
metadata:
|
113
|
+
allowed_push_host: https://rubygems.org
|
114
|
+
post_install_message:
|
115
|
+
rdoc_options: []
|
116
|
+
require_paths:
|
117
|
+
- lib
|
118
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
119
|
+
requirements:
|
120
|
+
- - ">="
|
121
|
+
- !ruby/object:Gem::Version
|
122
|
+
version: '0'
|
123
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
124
|
+
requirements:
|
125
|
+
- - ">="
|
126
|
+
- !ruby/object:Gem::Version
|
127
|
+
version: '0'
|
128
|
+
requirements: []
|
129
|
+
rubyforge_project:
|
130
|
+
rubygems_version: 2.2.3
|
131
|
+
signing_key:
|
132
|
+
specification_version: 4
|
133
|
+
summary: A gem for easily automating database jobs.
|
134
|
+
test_files:
|
135
|
+
- spec/sc_analytics/query/query_spec.rb
|
136
|
+
- spec/sc_analytics_spec.rb
|
137
|
+
- spec/spec_helper.rb
|