hsql 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b36d377513ad3f4e47233608a6c6a3d10abc5400
-  data.tar.gz: cfa80fcbd40a0f23e715d84274d6f60be9ae56b9
+  metadata.gz: 28ac7b37a61c29be9bf0bcda436fd0c493e9700b
+  data.tar.gz: 64551381b85a0906d715ad66bb877fd2abc9f435
 SHA512:
-  metadata.gz: 0ceda7a4890b22ce3a751b249d4d6a13b2db52093086ab6b5b7cdefd045e77c99779676a39954ba14b14ce2e64556110e4a39a89b71bcbfd097ad5f5438c4e1c
-  data.tar.gz: 46ff5428b17d28372906f0d92e7f04846337165e36a5a447c6b2036a2ef87133f62024e9e80aa38758d4d498fe5e3dc5cd7accfccb78f6cfa4001b13546e8a3e
+  metadata.gz: 73bfea62d54d190aa8057789da2099d38929b910b491140e9011d7f059b2495d0f0370d5e2bc78c058f3c1da0a9a65db94b4c180ced8714dda77489111e99e75
+  data.tar.gz: 1c19e860682b308ebc8e979117aecdc254069e4a07a142f3157c2ed7996b2f2c689bcb1c4602a9522500fdce162a6a55746150d986688e8f56e223091833d530

data/Gemfile

@@ -2,3 +2,5 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in hsql.gemspec
 gemspec
+
+gem 'pg_query', git: 'https://github.com/lfittl/pg_query.git'

data/README.md

@@ -1,31 +1,83 @@
 # HSQL
 
-".SQL ETL" is a library that parses `.sql` files with YAML front matter.
-This allows analysts and other non-developers to write and develop ETLs
-without having to write source code but still giving them the power of
+"Hash (of data) and SQL" is a library that parses `.sql` files with YAML
+[front matter](http://jekyllrb.com/docs/frontmatter/). This allows
+analysts and other non-developers to write and develop ETLs without
+having to write source code but still giving them the power of
 specifying variables to interpolate into the SQL and other metadata that
 the program executing the SQL can use.
 
 ## How to use this
 
 Rather than specifying variables and metadata for a set of database
-queries in a .rb or other programming language source file the queries
+queries in a `.rb`,`.py` or other programming language source file the queries
 should be written to a .sql file directly.
 
-    # filename: daily_summary.sql
-    owner: jackdanger
-    schedule: hourly
-    data:
-      production:
-        output_table: summaries
-        update_condition:
-      development:
-        output_table: jackdanger_summaries
-        update_condition: WHERE 1 <> 1
-    ---
-    USE some_database;
-    INSERT INTO {{{output_table}}} SELECT * FROM interesting_information;
-    UPDATE summaries_performed SET complete = 1 {{{update_condition}}};
+        # filename: daily_summary.sql
+        owner: jackdanger
+        schedule: hourly
+        data:
+          production:
+            output_table: summaries
+            update_condition:
+          development:
+            output_table: jackdanger_summaries
+            update_condition: WHERE 1 <> 1
+        ---
+        INSERT INTO {{{output_table}}} SELECT * FROM interesting_information;
+        UPDATE summaries_performed SET complete = 1 {{{update_condition}}};
 
 The above is a SQL file and any text editor will allow analysts to use
 code completion and syntax highlighting for their queries.
+
+The `data` hash in the YAML front matter lists a set of variables, by
+environment, that can be interpolated into the SQL queries. To render
+the queries an environment must be provided.
+
+```bash
+$ hsql daily_summary.sql development
+USE some_database;
+INSERT INTO jackdanger_summaries SELECT * FROM interesting_information;
+UPDATE summaries_performed SET complete = 1 WHERE 1 <> 1;
+```
+
+The `hsql` command-line utility allows these SQL source files to be
+easily run in the context of some other application that understands
+when and where to execute the queries.
+
+To access the metadata directly there is a simple programmatic API:
+
+```ruby
+>> file = HSQL.parse_file('./daily_summary.sql', 'development')
+>> file.queries
+=> [
+  "USE some_database;",
+  "INSERT INTO jackdanger_summaries SELECT * FROM interesting_information;",
+  "UPDATE summaries_performed SET complete = 1 WHERE 1 <> 1;"
+]
+```
+
+The object returned from `HSQL.parse_file` provides you access to both
+the rendered queries and the data specified in the front matter. You can
+use this to schedule the queries, to run them and send failure notices
+to a list of watchers. It's a general-purpose store of data for the
+query author to configure whatever system you use to run ETL queries.
+
+```ruby
+>> file = HSQL.parse_file('./daily_summary.sql', 'development')
+>> file.yaml
+=> {
+  'owner' => 'jackdanger',
+  'schedule' => 'hourly',
+  'data' => {
+    'production' => {
+      'output_table' => 'summaries',
+      'update_condition' => nil,
+    },
+    'development' => {
+      'output_table' => 'jackdanger_summaries',
+      'update_condition' => 'WHERE 1 <> 1',
+    },
+  }
+}
+```

data/Rakefile

@@ -1,6 +1,6 @@
-require "bundler/gem_tasks"
-require "rspec/core/rake_task"
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
 
 RSpec::Core::RakeTask.new(:spec)
 
-task :default => :spec
+task default: :spec

data/bin/console

@@ -1,14 +1,12 @@
 #!/usr/bin/env ruby
 
-require "bundler/setup"
-require "hsql"
+require 'bundler/setup'
+$LOAD_PATH << File.expand_path('../../lib', __FILE__)
+require 'hsql'
 
 # You can add fixtures and/or initialization code here to make experimenting
 # with your gem easier. You can also use a different console, if you like.
 
 # (If you use this, don't forget to add pry to your Gemfile!)
-# require "pry"
-# Pry.start
-
-require "irb"
-IRB.start
+require 'pry'
+Pry.start

data/bin/hsql

@@ -0,0 +1,56 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+require 'yaml'
+require 'json'
+
+options = {}
+option_parser = OptionParser.new do |opts|
+  opts.banner = <<-BANNER
+USAGE: #{__FILE__} file environment [--yaml]
+
+file:        Any *.sql file that has a YAML header ending in three hyphens.
+environment: Must be one of the keys of the "data" hash in the YAML header of the .sql file.
+--yaml:      Outputs only the YAML metadata from the file, not the SQL queries themselves.
+
+for details, run:
+  open https://github.com/JackDanger/hsql
+BANNER
+
+  opts.on('-y', '--yaml', 'Output metadata for this file as YAML') do |option|
+    options[:meta_only] = 'yaml'
+  end
+  opts.on('-j', '--json', 'Output metadata for this file as JSON') do |option|
+    options[:meta_only] = 'json'
+  end
+end
+
+option_parser.parse!
+
+filename, env = ARGV
+
+unless filename && env
+  puts option_parser.banner
+  exit 1
+end
+
+begin
+  file = File.new(filename)
+rescue Errno::ENOENT
+  puts "#{filename.inspect}: No such file"
+  exit 1
+end
+
+require_relative '../lib/hsql'
+
+file = HSQL.parse_file(file, env)
+if 'yaml' == options[:meta_only]
+  puts file.metadata.to_yaml
+elsif 'json' == options[:meta_only]
+  puts file.metadata.to_json
+else
+  file.queries.each do |query|
+    # Runs the query through the parser and then deparses it
+    puts query
+  end
+end

data/examples/simple.sql

@@ -0,0 +1,18 @@
+# filename: daily_summary.sql
+owner: jackdanger
+schedule: hourly
+requires:
+  - daily_payments
+  - hourly_users
+  - some_other_value
+data:
+  production:
+    output_table: summaries
+    update_condition:
+  development:
+    output_table: jackdanger_summaries
+    update_condition: WHERE 1 <> 1
+---
+INSERT INTO {{{output_table}}} -- this query is joined to one line
+  SELECT COUNT(*) FROM interesting_information; -- and the comments get stripped
+UPDATE summaries_performed SET complete = 1 {{{update_condition}}};

data/hsql.gemspec

@@ -4,24 +4,26 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'hsql/version'
 
 Gem::Specification.new do |spec|
-  spec.name          = "hsql"
-  spec.version       = Setl::VERSION
-  spec.authors       = ["Jack Danger Canty"]
-  spec.email         = ["gems@jackcanty.com"]
+  spec.name          = 'hsql'
+  spec.version       = HSQL::VERSION
+  spec.authors       = ['Jack Danger Canty']
+  spec.email         = ['gems@jackcanty.com']
 
-  spec.summary       = %q{Store a hash of data with your SQL queries.}
-  spec.description   = %q{Write SQL queries in a .sql format and ship them with metadata about how they should be executed}
-  spec.homepage      = "https://github.com/JackDanger/hsql"
-  spec.license       = "MIT"
+  spec.summary       = 'Store a hash of data with your SQL queries.'
+  spec.description   = 'Write SQL queries in a .sql format and ship them with metadata about how they should be executed'
+  spec.homepage      = 'https://github.com/JackDanger/hsql'
+  spec.license       = 'MIT'
 
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
-  spec.bindir        = "exe"
+  spec.bindir        = 'exe'
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
-  spec.require_paths = ["lib"]
+  spec.require_paths = ['lib']
 
-  spec.add_dependency "mustache", '~> 0'
-  spec.add_development_dependency "bundler", "~> 1.10"
-  spec.add_development_dependency "rake", '~> 0'
-  spec.add_development_dependency "rspec", '~> 0'
-  spec.add_development_dependency "pry-byebug", '~> 0'
+  spec.add_dependency 'mustache', '> 0'
+  #  spec.add_dependency 'pg_query', '>= 0.6.2'
+  spec.add_development_dependency 'bundler', '~> 1.10'
+  spec.add_development_dependency 'rubocop', '~> 0.3'
+  spec.add_development_dependency 'rake', '> 0'
+  spec.add_development_dependency 'rspec', '~> 3.3'
+  spec.add_development_dependency 'pry-byebug', '> 3'
 end

data/lib/hsql.rb

@@ -1,6 +1,7 @@
-require_relative "hsql/version"
-require_relative "hsql/template"
-require_relative "hsql/file"
+require_relative 'hsql/version'
+require_relative 'hsql/template'
+require_relative 'hsql/file'
+require_relative 'hsql/query'
 require 'mustache'
 
 module HSQL
@@ -12,7 +13,11 @@ module HSQL
   # example) this will return a HSQL::File object providing access to the parts
   # of that file.
   def self.parse(string, environment)
-    raise ArgumentError, "The environment argument is required" unless environment
+    fail ArgumentError, 'The environment argument is required' unless environment
     File.new(string, environment).parse!
   end
+
+  def self.parse_file(file, environment)
+    parse(file.read, environment)
+  end
 end

data/lib/hsql/file.rb

@@ -1,15 +1,16 @@
 # HSQL::File parses the input file and provides reader methods to the hash of
 # YAML data from the front matter section and a list of the queries in the SQL
 # portion.
+require 'yaml'
+require_relative 'query'
 module HSQL
   class File < Struct.new(:string, :environment)
-    def yaml
-      @yaml ||= YAML.load(@front_matter)
+    def metadata
+      @metadata ||= ::YAML.load(@front_matter)
     end
 
     def queries
-      # TODO: allow multi-line queries!
-      @queries ||= @rendered_sql.lines.map(&:chomp).reject(&:empty?)
+      @queries ||= Query.parse(@rendered_sql)
     end
 
     def parse!
@@ -24,7 +25,7 @@ module HSQL
       @split ||= begin
         @front_matter, divider, @sql = string.partition(/^---$/)
         unless divider == '---'
-          raise FormatError, "The YAML front matter is required, otherwise this is just a SQL file"
+          fail FormatError, 'The YAML front matter is required, otherwise this is just a SQL file'
         end
         true
       end
@@ -32,11 +33,11 @@ module HSQL
 
     def data
       @data ||= begin
-        if yaml['data']
-          unless yaml['data'].key?(environment)
-            raise ArgumentError, "The environment #{environment.inspect} is not specified"
+        if metadata['data']
+          unless metadata['data'].key?(environment)
+            fail ArgumentError, "The environment #{environment.inspect} is not specified"
           end
-          yaml['data'][environment]
+          metadata['data'][environment]
         end || {}
       end
     end
@@ -45,7 +46,7 @@ module HSQL
       template = Template.new(@sql)
       template.variable_names.each do |name|
         unless data.key?(name)
-          raise FormatError, "#{name.inspect} is not set in #{environment.inspect} environment"
+          fail FormatError, "#{name.inspect} is not set in #{environment.inspect} environment"
         end
       end
 

data/lib/hsql/query.rb

@@ -0,0 +1,20 @@
+# PgQuery uses Postgres' own parser to parse and then deparse each query.
+require 'pg_query'
+module HSQL
+  class Query < Struct.new(:ast)
+    # Returns a list of queries found in the source SQL
+    def self.parse(source)
+      # Splits on semicolons at the end of the line, eliding any comment that
+      # might be there.
+      PgQuery.parse(source).parsetree.map do |ast|
+        Query.new(ast)
+      end
+    end
+
+    # Show the parsed query as reconstructed SQL
+    def to_s
+      PgQuery.deparse ast
+    end
+    alias_method :to_sql, :to_s
+  end
+end

data/lib/hsql/version.rb

@@ -1,3 +1,3 @@
-module Setl
-  VERSION = "0.1.0"
+module HSQL
+  VERSION = '0.2.0'
 end

metadata

@@ -1,27 +1,27 @@
 --- !ruby/object:Gem::Specification
 name: hsql
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Jack Danger Canty
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-08-03 00:00:00.000000000 Z
+date: 2015-08-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mustache
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">"
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">"
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
@@ -39,17 +39,31 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.10'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
@@ -58,28 +72,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.3'
 - !ruby/object:Gem::Dependency
   name: pry-byebug
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3'
 description: Write SQL queries in a .sql format and ship them with metadata about
   how they should be executed
 email:
@@ -98,10 +112,13 @@ files:
 - README.md
 - Rakefile
 - bin/console
+- bin/hsql
 - bin/setup
+- examples/simple.sql
 - hsql.gemspec
 - lib/hsql.rb
 - lib/hsql/file.rb
+- lib/hsql/query.rb
 - lib/hsql/template.rb
 - lib/hsql/version.rb
 homepage: https://github.com/JackDanger/hsql