hsql 0.1.0 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile +2 -0
- data/README.md +70 -18
- data/Rakefile +3 -3
- data/bin/console +5 -7
- data/bin/hsql +56 -0
- data/examples/simple.sql +18 -0
- data/hsql.gemspec +17 -15
- data/lib/hsql.rb +9 -4
- data/lib/hsql/file.rb +11 -10
- data/lib/hsql/query.rb +20 -0
- data/lib/hsql/version.rb +2 -2
- metadata +29 -12
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 28ac7b37a61c29be9bf0bcda436fd0c493e9700b
|
4
|
+
data.tar.gz: 64551381b85a0906d715ad66bb877fd2abc9f435
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 73bfea62d54d190aa8057789da2099d38929b910b491140e9011d7f059b2495d0f0370d5e2bc78c058f3c1da0a9a65db94b4c180ced8714dda77489111e99e75
|
7
|
+
data.tar.gz: 1c19e860682b308ebc8e979117aecdc254069e4a07a142f3157c2ed7996b2f2c689bcb1c4602a9522500fdce162a6a55746150d986688e8f56e223091833d530
|
data/Gemfile
CHANGED
data/README.md
CHANGED
@@ -1,31 +1,83 @@
|
|
1
1
|
# HSQL
|
2
2
|
|
3
|
-
"
|
4
|
-
This allows
|
5
|
-
|
3
|
+
"Hash (of data) and SQL" is a library that parses `.sql` files with YAML
|
4
|
+
[front matter](http://jekyllrb.com/docs/frontmatter/). This allows
|
5
|
+
analysts and other non-developers to write and develop ETLs without
|
6
|
+
having to write source code but still giving them the power of
|
6
7
|
specifying variables to interpolate into the SQL and other metadata that
|
7
8
|
the program executing the SQL can use.
|
8
9
|
|
9
10
|
## How to use this
|
10
11
|
|
11
12
|
Rather than specifying variables and metadata for a set of database
|
12
|
-
queries in a
|
13
|
+
queries in a `.rb`,`.py` or other programming language source file the queries
|
13
14
|
should be written to a .sql file directly.
|
14
15
|
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
UPDATE summaries_performed SET complete = 1 {{{update_condition}}};
|
16
|
+
# filename: daily_summary.sql
|
17
|
+
owner: jackdanger
|
18
|
+
schedule: hourly
|
19
|
+
data:
|
20
|
+
production:
|
21
|
+
output_table: summaries
|
22
|
+
update_condition:
|
23
|
+
development:
|
24
|
+
output_table: jackdanger_summaries
|
25
|
+
update_condition: WHERE 1 <> 1
|
26
|
+
---
|
27
|
+
INSERT INTO {{{output_table}}} SELECT * FROM interesting_information;
|
28
|
+
UPDATE summaries_performed SET complete = 1 {{{update_condition}}};
|
29
29
|
|
30
30
|
The above is a SQL file and any text editor will allow analysts to use
|
31
31
|
code completion and syntax highlighting for their queries.
|
32
|
+
|
33
|
+
The `data` hash in the YAML front matter lists a set of variables, by
|
34
|
+
environment, that can be interpolated into the SQL queries. To render
|
35
|
+
the queries an environment must be provided.
|
36
|
+
|
37
|
+
```bash
|
38
|
+
$ hsql daily_summary.sql development
|
39
|
+
USE some_database;
|
40
|
+
INSERT INTO jackdanger_summaries SELECT * FROM interesting_information;
|
41
|
+
UPDATE summaries_performed SET complete = 1 WHERE 1 <> 1;
|
42
|
+
```
|
43
|
+
|
44
|
+
The `hsql` command-line utility allows these SQL source files to be
|
45
|
+
easily run in the context of some other application that understands
|
46
|
+
when and where to execute the queries.
|
47
|
+
|
48
|
+
To access the metadata directly there is a simple programmatic API:
|
49
|
+
|
50
|
+
```ruby
|
51
|
+
>> file = HSQL.parse_file('./daily_summary.sql', 'development')
|
52
|
+
>> file.queries
|
53
|
+
=> [
|
54
|
+
"USE some_database;",
|
55
|
+
"INSERT INTO jackdanger_summaries SELECT * FROM interesting_information;",
|
56
|
+
"UPDATE summaries_performed SET complete = 1 WHERE 1 <> 1;"
|
57
|
+
]
|
58
|
+
```
|
59
|
+
|
60
|
+
The object returned from `HSQL.parse_file` provides you access to both
|
61
|
+
the rendered queries and the data specified in the front matter. You can
|
62
|
+
use this to schedule the queries, to run them and send failure notices
|
63
|
+
to a list of watchers. It's a general-purpose store of data for the
|
64
|
+
query author to configure whatever system you use to run ETL queries.
|
65
|
+
|
66
|
+
```ruby
|
67
|
+
>> file = HSQL.parse_file('./daily_summary.sql', 'development')
|
68
|
+
>> file.yaml
|
69
|
+
=> {
|
70
|
+
'owner' => 'jackdanger',
|
71
|
+
'schedule' => 'hourly',
|
72
|
+
'data' => {
|
73
|
+
'production' => {
|
74
|
+
'output_table' => 'summaries',
|
75
|
+
'update_condition' => nil,
|
76
|
+
},
|
77
|
+
'development' => {
|
78
|
+
'output_table' => 'jackdanger_summaries',
|
79
|
+
'update_condition' => 'WHERE 1 <> 1',
|
80
|
+
},
|
81
|
+
}
|
82
|
+
}
|
83
|
+
```
|
data/Rakefile
CHANGED
data/bin/console
CHANGED
@@ -1,14 +1,12 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
|
3
|
-
require
|
4
|
-
|
3
|
+
require 'bundler/setup'
|
4
|
+
$LOAD_PATH << File.expand_path('../../lib', __FILE__)
|
5
|
+
require 'hsql'
|
5
6
|
|
6
7
|
# You can add fixtures and/or initialization code here to make experimenting
|
7
8
|
# with your gem easier. You can also use a different console, if you like.
|
8
9
|
|
9
10
|
# (If you use this, don't forget to add pry to your Gemfile!)
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
require "irb"
|
14
|
-
IRB.start
|
11
|
+
require 'pry'
|
12
|
+
Pry.start
|
data/bin/hsql
ADDED
@@ -0,0 +1,56 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'optparse'
|
4
|
+
require 'yaml'
|
5
|
+
require 'json'
|
6
|
+
|
7
|
+
options = {}
|
8
|
+
option_parser = OptionParser.new do |opts|
|
9
|
+
opts.banner = <<-BANNER
|
10
|
+
USAGE: #{__FILE__} file environment [--yaml]
|
11
|
+
|
12
|
+
file: Any *.sql file that has a YAML header ending in three hyphens.
|
13
|
+
environment: Must be one of the keys of the "data" hash in the YAML header of the .sql file.
|
14
|
+
--yaml: Outputs only the YAML metadata from the file, not the SQL queries themselves.
|
15
|
+
|
16
|
+
for details, run:
|
17
|
+
open https://github.com/JackDanger/hsql
|
18
|
+
BANNER
|
19
|
+
|
20
|
+
opts.on('-y', '--yaml', 'Output metadata for this file as YAML') do |option|
|
21
|
+
options[:meta_only] = 'yaml'
|
22
|
+
end
|
23
|
+
opts.on('-j', '--json', 'Output metadata for this file as JSON') do |option|
|
24
|
+
options[:meta_only] = 'json'
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
option_parser.parse!
|
29
|
+
|
30
|
+
filename, env = ARGV
|
31
|
+
|
32
|
+
unless filename && env
|
33
|
+
puts option_parser.banner
|
34
|
+
exit 1
|
35
|
+
end
|
36
|
+
|
37
|
+
begin
|
38
|
+
file = File.new(filename)
|
39
|
+
rescue Errno::ENOENT
|
40
|
+
puts "#{filename.inspect}: No such file"
|
41
|
+
exit 1
|
42
|
+
end
|
43
|
+
|
44
|
+
require_relative '../lib/hsql'
|
45
|
+
|
46
|
+
file = HSQL.parse_file(file, env)
|
47
|
+
if 'yaml' == options[:meta_only]
|
48
|
+
puts file.metadata.to_yaml
|
49
|
+
elsif 'json' == options[:meta_only]
|
50
|
+
puts file.metadata.to_json
|
51
|
+
else
|
52
|
+
file.queries.each do |query|
|
53
|
+
# Runs the query through the parser and then deparses it
|
54
|
+
puts query
|
55
|
+
end
|
56
|
+
end
|
data/examples/simple.sql
ADDED
@@ -0,0 +1,18 @@
|
|
1
|
+
# filename: daily_summary.sql
|
2
|
+
owner: jackdanger
|
3
|
+
schedule: hourly
|
4
|
+
requires:
|
5
|
+
- daily_payments
|
6
|
+
- hourly_users
|
7
|
+
- some_other_value
|
8
|
+
data:
|
9
|
+
production:
|
10
|
+
output_table: summaries
|
11
|
+
update_condition:
|
12
|
+
development:
|
13
|
+
output_table: jackdanger_summaries
|
14
|
+
update_condition: WHERE 1 <> 1
|
15
|
+
---
|
16
|
+
INSERT INTO {{{output_table}}} -- this query is joined to one line
|
17
|
+
SELECT COUNT(*) FROM interesting_information; -- and the comments get stripped
|
18
|
+
UPDATE summaries_performed SET complete = 1 {{{update_condition}}};
|
data/hsql.gemspec
CHANGED
@@ -4,24 +4,26 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
4
|
require 'hsql/version'
|
5
5
|
|
6
6
|
Gem::Specification.new do |spec|
|
7
|
-
spec.name =
|
8
|
-
spec.version =
|
9
|
-
spec.authors = [
|
10
|
-
spec.email = [
|
7
|
+
spec.name = 'hsql'
|
8
|
+
spec.version = HSQL::VERSION
|
9
|
+
spec.authors = ['Jack Danger Canty']
|
10
|
+
spec.email = ['gems@jackcanty.com']
|
11
11
|
|
12
|
-
spec.summary =
|
13
|
-
spec.description =
|
14
|
-
spec.homepage =
|
15
|
-
spec.license =
|
12
|
+
spec.summary = 'Store a hash of data with your SQL queries.'
|
13
|
+
spec.description = 'Write SQL queries in a .sql format and ship them with metadata about how they should be executed'
|
14
|
+
spec.homepage = 'https://github.com/JackDanger/hsql'
|
15
|
+
spec.license = 'MIT'
|
16
16
|
|
17
17
|
spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
18
|
-
spec.bindir =
|
18
|
+
spec.bindir = 'exe'
|
19
19
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
20
|
-
spec.require_paths = [
|
20
|
+
spec.require_paths = ['lib']
|
21
21
|
|
22
|
-
spec.add_dependency
|
23
|
-
spec.
|
24
|
-
spec.add_development_dependency
|
25
|
-
spec.add_development_dependency
|
26
|
-
spec.add_development_dependency
|
22
|
+
spec.add_dependency 'mustache', '> 0'
|
23
|
+
# spec.add_dependency 'pg_query', '>= 0.6.2'
|
24
|
+
spec.add_development_dependency 'bundler', '~> 1.10'
|
25
|
+
spec.add_development_dependency 'rubocop', '~> 0.3'
|
26
|
+
spec.add_development_dependency 'rake', '> 0'
|
27
|
+
spec.add_development_dependency 'rspec', '~> 3.3'
|
28
|
+
spec.add_development_dependency 'pry-byebug', '> 3'
|
27
29
|
end
|
data/lib/hsql.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
|
-
require_relative
|
2
|
-
require_relative
|
3
|
-
require_relative
|
1
|
+
require_relative 'hsql/version'
|
2
|
+
require_relative 'hsql/template'
|
3
|
+
require_relative 'hsql/file'
|
4
|
+
require_relative 'hsql/query'
|
4
5
|
require 'mustache'
|
5
6
|
|
6
7
|
module HSQL
|
@@ -12,7 +13,11 @@ module HSQL
|
|
12
13
|
# example) this will return a HSQL::File object providing access to the parts
|
13
14
|
# of that file.
|
14
15
|
def self.parse(string, environment)
|
15
|
-
|
16
|
+
fail ArgumentError, 'The environment argument is required' unless environment
|
16
17
|
File.new(string, environment).parse!
|
17
18
|
end
|
19
|
+
|
20
|
+
def self.parse_file(file, environment)
|
21
|
+
parse(file.read, environment)
|
22
|
+
end
|
18
23
|
end
|
data/lib/hsql/file.rb
CHANGED
@@ -1,15 +1,16 @@
|
|
1
1
|
# HSQL::File parses the input file and provides reader methods to the hash of
|
2
2
|
# YAML data from the front matter section and a list of the queries in the SQL
|
3
3
|
# portion.
|
4
|
+
require 'yaml'
|
5
|
+
require_relative 'query'
|
4
6
|
module HSQL
|
5
7
|
class File < Struct.new(:string, :environment)
|
6
|
-
def
|
7
|
-
@
|
8
|
+
def metadata
|
9
|
+
@metadata ||= ::YAML.load(@front_matter)
|
8
10
|
end
|
9
11
|
|
10
12
|
def queries
|
11
|
-
|
12
|
-
@queries ||= @rendered_sql.lines.map(&:chomp).reject(&:empty?)
|
13
|
+
@queries ||= Query.parse(@rendered_sql)
|
13
14
|
end
|
14
15
|
|
15
16
|
def parse!
|
@@ -24,7 +25,7 @@ module HSQL
|
|
24
25
|
@split ||= begin
|
25
26
|
@front_matter, divider, @sql = string.partition(/^---$/)
|
26
27
|
unless divider == '---'
|
27
|
-
|
28
|
+
fail FormatError, 'The YAML front matter is required, otherwise this is just a SQL file'
|
28
29
|
end
|
29
30
|
true
|
30
31
|
end
|
@@ -32,11 +33,11 @@ module HSQL
|
|
32
33
|
|
33
34
|
def data
|
34
35
|
@data ||= begin
|
35
|
-
if
|
36
|
-
unless
|
37
|
-
|
36
|
+
if metadata['data']
|
37
|
+
unless metadata['data'].key?(environment)
|
38
|
+
fail ArgumentError, "The environment #{environment.inspect} is not specified"
|
38
39
|
end
|
39
|
-
|
40
|
+
metadata['data'][environment]
|
40
41
|
end || {}
|
41
42
|
end
|
42
43
|
end
|
@@ -45,7 +46,7 @@ module HSQL
|
|
45
46
|
template = Template.new(@sql)
|
46
47
|
template.variable_names.each do |name|
|
47
48
|
unless data.key?(name)
|
48
|
-
|
49
|
+
fail FormatError, "#{name.inspect} is not set in #{environment.inspect} environment"
|
49
50
|
end
|
50
51
|
end
|
51
52
|
|
data/lib/hsql/query.rb
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
# PgQuery uses Postgres' own parser to parse and then deparse each query.
|
2
|
+
require 'pg_query'
|
3
|
+
module HSQL
|
4
|
+
class Query < Struct.new(:ast)
|
5
|
+
# Returns a list of queries found in the source SQL
|
6
|
+
def self.parse(source)
|
7
|
+
# Splits on semicolons at the end of the line, eliding any comment that
|
8
|
+
# might be there.
|
9
|
+
PgQuery.parse(source).parsetree.map do |ast|
|
10
|
+
Query.new(ast)
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
# Show the parsed query as reconstructed SQL
|
15
|
+
def to_s
|
16
|
+
PgQuery.deparse ast
|
17
|
+
end
|
18
|
+
alias_method :to_sql, :to_s
|
19
|
+
end
|
20
|
+
end
|
data/lib/hsql/version.rb
CHANGED
@@ -1,3 +1,3 @@
|
|
1
|
-
module
|
2
|
-
VERSION =
|
1
|
+
module HSQL
|
2
|
+
VERSION = '0.2.0'
|
3
3
|
end
|
metadata
CHANGED
@@ -1,27 +1,27 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: hsql
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jack Danger Canty
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-08-
|
11
|
+
date: 2015-08-17 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: mustache
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- - "
|
17
|
+
- - ">"
|
18
18
|
- !ruby/object:Gem::Version
|
19
19
|
version: '0'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- - "
|
24
|
+
- - ">"
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '0'
|
27
27
|
- !ruby/object:Gem::Dependency
|
@@ -39,17 +39,31 @@ dependencies:
|
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '1.10'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
|
-
name:
|
42
|
+
name: rubocop
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
45
|
- - "~>"
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version: '0'
|
47
|
+
version: '0.3'
|
48
48
|
type: :development
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
52
|
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0.3'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: rake
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">"
|
53
67
|
- !ruby/object:Gem::Version
|
54
68
|
version: '0'
|
55
69
|
- !ruby/object:Gem::Dependency
|
@@ -58,28 +72,28 @@ dependencies:
|
|
58
72
|
requirements:
|
59
73
|
- - "~>"
|
60
74
|
- !ruby/object:Gem::Version
|
61
|
-
version: '
|
75
|
+
version: '3.3'
|
62
76
|
type: :development
|
63
77
|
prerelease: false
|
64
78
|
version_requirements: !ruby/object:Gem::Requirement
|
65
79
|
requirements:
|
66
80
|
- - "~>"
|
67
81
|
- !ruby/object:Gem::Version
|
68
|
-
version: '
|
82
|
+
version: '3.3'
|
69
83
|
- !ruby/object:Gem::Dependency
|
70
84
|
name: pry-byebug
|
71
85
|
requirement: !ruby/object:Gem::Requirement
|
72
86
|
requirements:
|
73
|
-
- - "
|
87
|
+
- - ">"
|
74
88
|
- !ruby/object:Gem::Version
|
75
|
-
version: '
|
89
|
+
version: '3'
|
76
90
|
type: :development
|
77
91
|
prerelease: false
|
78
92
|
version_requirements: !ruby/object:Gem::Requirement
|
79
93
|
requirements:
|
80
|
-
- - "
|
94
|
+
- - ">"
|
81
95
|
- !ruby/object:Gem::Version
|
82
|
-
version: '
|
96
|
+
version: '3'
|
83
97
|
description: Write SQL queries in a .sql format and ship them with metadata about
|
84
98
|
how they should be executed
|
85
99
|
email:
|
@@ -98,10 +112,13 @@ files:
|
|
98
112
|
- README.md
|
99
113
|
- Rakefile
|
100
114
|
- bin/console
|
115
|
+
- bin/hsql
|
101
116
|
- bin/setup
|
117
|
+
- examples/simple.sql
|
102
118
|
- hsql.gemspec
|
103
119
|
- lib/hsql.rb
|
104
120
|
- lib/hsql/file.rb
|
121
|
+
- lib/hsql/query.rb
|
105
122
|
- lib/hsql/template.rb
|
106
123
|
- lib/hsql/version.rb
|
107
124
|
homepage: https://github.com/JackDanger/hsql
|