hsql 0.3.8 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 92a6d1e3fd48b97e097cd313b872e3b4846099b5
-  data.tar.gz: cf1ccc9268ec22de5e1f4c1b30919a91bf1a4aa4
+  metadata.gz: 8f0f814ecf9e392ba9fdaf7b9ca4189cf5d938ce
+  data.tar.gz: 03b44002a2c25d8ae9aad92662e942af90f30d4c
 SHA512:
-  metadata.gz: da5c1bdd69680ddaa11ed50cf55f30fd69ecdb60f55c8ff04cce8b0eeb28451cb4e1faf09d59226ddaa516757e080b1fd5d0407b19c9cd91ea3a7ce0a75b375f
-  data.tar.gz: 480d4571781e006096bdcc4834c3ae77cf481ba0d99e56a086b5dba3cb8a32131569cf10b957f5417af48e096b781d6c46704aa8201f9bcd007c2d672fb9fd00
+  metadata.gz: f403e66309574a45a882bfb52150273a83d15ed2868b382bc89520943128459d7888b36f4fdfcbed03f8ca39be6adb6b1ff19845baf82035f2c7da388e7843b4
+  data.tar.gz: d789ce8019449d9b7cf20d61ee84cca6f4584dba60750074d1d9a1b288c6109e706b0e93c4eb2a4e9834a78b798078a0ad54685f95754632d82588c68b5f92f7

data/README.md

@@ -11,13 +11,13 @@ the program executing the SQL can use.
 
 Rather than specifying variables and metadata for a set of database
 queries in a `.rb`,`.py` or other programming language source file the queries
-should be written to a .sql file directly.
+should be written into a .sql file directly.
 
 ```sql
 # filename: daily_summary.sql
 owner: jackdanger
 schedule: hourly
-data:
+environments:
   production:
     output_table: summaries
     update_condition:
@@ -29,57 +29,168 @@ INSERT INTO {{{output_table}}} SELECT * FROM interesting_information;
 UPDATE summaries_performed SET complete = 1 {{{update_condition}}};
 ```
 
-The above is a SQL file and any text editor will allow analysts to use
+The above is a `.sql` file and any text editor will allow analysts to use
 code completion and syntax highlighting for their queries.
 
 The `data` hash in the YAML front matter lists a set of variables, by
 environment, that can be interpolated into the SQL queries. To render
-the queries an environment must be provided.
+the queries an environment must be provided. The templating system used
+for interpolating data is [Mustache](https://mustache.github.io/)
+(though theoretically we could use any templating system).
 
+### Rendering SQL on the command line
 ```bash
-$ hsql daily_summary.sql development
+$ hsql daily_summary.sql -env development
 USE some_database;
 INSERT INTO jackdanger_summaries SELECT * FROM interesting_information;
 UPDATE summaries_performed SET complete = 1 WHERE 1 <> 1;
 ```
 
-The `hsql` command-line utility allows these SQL source files to be
-easily run in the context of some other application that understands
-when and where to execute the queries.
+The `hsql` command-line utility outputs valid Postgres SQL with any
+template variables filled in from the YAML data.
+To access the metadata directly use the `--yaml` or `--json` flag
 
-To access the metadata directly there is a simple programmatic API:
+```bash
+$ hsql daily_summary.sql -env development --json
+{"owner":"jackdanger","schedule":"hourly","output_table":"jackdanger_summaries","update_condition":"WHERE 1 <> 1"}
+$ hsql daily_summary.sql -env development --yaml
+---
+owner: jackdanger
+schedule: hourly
+output_table: jackdanger_summaries
+update_condition: WHERE 1 <> 1
+```
 
-```ruby
->> file = HSQL.parse_file('./daily_summary.sql', 'development')
->> file.queries
-=> [
-  "USE some_database;",
-  "INSERT INTO jackdanger_summaries SELECT * FROM interesting_information;",
-  "UPDATE summaries_performed SET complete = 1 WHERE 1 <> 1;"
-]
+### Working with times and dates
+By default when you run the `hsql` command it will set the template
+variable `{{{ now }}}` to the current moment (as per the local
+machine's clock). You can modify this by setting a `--timestamp`
+command line flag to any Ruby-parseable time or date string and that
+will be used to establish the value of `{{{ now }}}`.
+
+To avoid having to do date math in SQL all of the following are also
+available in every template, relative to the value of `{{{ now }}}`:
+
+```
+                              "now" => "'2015-10-06 12:34:55 -0700'",
+                "beginning_of_hour" => "'2015-10-06 12:00:00 -0700'",
+                 "beginning_of_day" => "'2015-10-06 00:00:00 -0700'",
+                "beginning_of_week" => "'2015-10-05 00:00:00 -0700'",
+               "beginning_of_month" => "'2015-10-01 00:00:00 -0700'",
+             "beginning_of_quarter" => "'2015-10-01 00:00:00 -0700'",
+                "beginning_of_year" => "'2015-01-01 00:00:00 -0800'",
+                      "end_of_hour" => "'2015-10-06 12:59:59 -0700'",
+                       "end_of_day" => "'2015-10-06 23:59:59 -0700'",
+                      "end_of_week" => "'2015-10-11 23:59:59 -0700'",
+                     "end_of_month" => "'2015-10-31 23:59:59 -0700'",
+                   "end_of_quarter" => "'2015-12-31 23:59:59 -0800'",
+                      "end_of_year" => "'2015-12-31 23:59:59 -0800'",
+       "beginning_of_previous_hour" => "'2015-10-06 11:00:00 -0700'",
+             "end_of_previous_hour" => "'2015-10-06 11:59:59 -0700'",
+        "beginning_of_previous_day" => "'2015-10-05 00:00:00 -0700'",
+              "end_of_previous_day" => "'2015-10-05 23:59:59 -0700'",
+       "beginning_of_previous_week" => "'2015-09-28 00:00:00 -0700'",
+             "end_of_previous_week" => "'2015-10-04 23:59:59 -0700'",
+      "beginning_of_previous_month" => "'2015-09-01 00:00:00 -0700'",
+            "end_of_previous_month" => "'2015-09-30 23:59:59 -0700'",
+    "beginning_of_previous_quarter" => "'2015-07-01 00:00:00 -0700'",
+          "end_of_previous_quarter" => "'2015-09-30 23:59:59 -0700'",
+       "beginning_of_previous_year" => "'2014-01-01 00:00:00 -0800'",
+             "end_of_previous_year" => "'2014-12-31 23:59:59 -0800'"
 ```
 
-The object returned from `HSQL.parse_file` provides you access to both
-the rendered queries and the data specified in the front matter. You can
-use this to schedule the queries, to run them and send failure notices
-to a list of watchers. It's a general-purpose store of data for the
-query author to configure whatever system you use to run ETL queries.
+### Programmatic Ruby API
+
+If you'd like to manipulate the queries in some advanced way
+(e.g. to modify the AST of a parsed query) you can use the gem as a
+dependency.
 
 ```ruby
->> file = HSQL.parse_file('./daily_summary.sql', 'development')
->> file.yaml
+>> require 'hsql'
+>> file = HSQL::File.parse_file('./simple.sql', 'development')
+>> query = file.queries.first;
+>> query.to_s
+=> "INSERT INTO jackdanger_summaries SELECT count(*) FROM interesting_information"
+>> query.ast
 => {
-  'owner' => 'jackdanger',
-  'schedule' => 'hourly',
-  'data' => {
-    'production' => {
-      'output_table' => 'summaries',
-      'update_condition' => nil,
-    },
-    'development' => {
-      'output_table' => 'jackdanger_summaries',
-      'update_condition' => 'WHERE 1 <> 1',
-    },
-  }
+    "INSERT INTO" => {
+             "relation" => {
+            "RANGEVAR" => {
+                    "schemaname" => nil,
+                       "relname" => "jackdanger_summaries",
+                        "inhOpt" => 2,
+                "relpersistence" => "p",
+                         "alias" => nil,
+                      "location" => 13
+            }
+        },
+                 "cols" => nil,
+           "selectStmt" => {
+            "SELECT" => {
+                "distinctClause" => nil,
+                    "intoClause" => nil,
+                    "targetList" => [
+                    [0] {
+                        "RESTARGET" => {
+                                   "name" => nil,
+                            "indirection" => nil,
+                                    "val" => {
+                                "FUNCCALL" => {
+                                            "funcname" => [
+                                        [0] "count"
+                                    ],
+                                                "args" => nil,
+                                           "agg_order" => nil,
+                                          "agg_filter" => nil,
+                                    "agg_within_group" => false,
+                                            "agg_star" => true,
+                                        "agg_distinct" => false,
+                                       "func_variadic" => false,
+                                                "over" => nil,
+                                            "location" => 79
+                                }
+                            },
+                               "location" => 79
+                        }
+                    }
+                ],
+                    "fromClause" => [
+                    [0] {
+                        "RANGEVAR" => {
+                                "schemaname" => nil,
+                                   "relname" => "interesting_information",
+                                    "inhOpt" => 2,
+                            "relpersistence" => "p",
+                                     "alias" => nil,
+                                  "location" => 95
+                        }
+                    }
+                ],
+                   "whereClause" => nil,
+                   "groupClause" => nil,
+                  "havingClause" => nil,
+                  "windowClause" => nil,
+                   "valuesLists" => nil,
+                    "sortClause" => nil,
+                   "limitOffset" => nil,
+                    "limitCount" => nil,
+                 "lockingClause" => nil,
+                    "withClause" => nil,
+                            "op" => 0,
+                           "all" => false,
+                          "larg" => nil,
+                          "rarg" => nil
+            }
+        },
+        "returningList" => nil,
+           "withClause" => nil
+    }
 }
+>> query.ast['INSERT INTO']['relation']['RANGEVAR']['relname'] = 'othertable'
+=> "othertable"
+>> query.to_s
+=> "INSERT INTO othertable SELECT count(*) FROM interesting_information"
 ```
+
+Please don't hesitate to open a PR or issue for any reason. New
+contributors and bug fixes are welcome. Forks of this project will be celebrated.

data/bin/hsql

@@ -19,7 +19,7 @@ end
 
 require_relative '../lib/hsql'
 
-file = HSQL::File.parse_file(file, options)
+file = HSQL::File.parse_file(filename, options)
 if 'yaml' == options[:meta_only]
   puts file.to_yaml
 elsif 'json' == options[:meta_only]

data/examples/simple.sql

@@ -5,7 +5,7 @@ requires:
   - daily_payments
   - hourly_users
   - some_other_value
-data:
+environments:
   production:
     output_table: summaries
     update_condition:

data/lib/hsql/file.rb

@@ -17,15 +17,15 @@ module HSQL
       @verbose = options[:verbose]
     end
 
-    # Given the contents of a SQL file with YAML front matter (see README for an
-    # example) this will return a HSQL::File object providing access to the parts
-    # of that file.
-    def self.parse(string, options)
-      new(string, options).parse!
+    # Given a SQL file with YAML front matter (see README for an
+    # example) this will return a HSQL::File object providing access to
+    # the parts of that file.
+    def self.parse_file(filename, options)
+      parse(::File.read(filename), options)
     end
 
-    def self.parse_file(file, options)
-      parse(file.read, options)
+    def self.parse(source, options)
+      new(source, options).parse!
     end
 
     def to_yaml
@@ -37,7 +37,11 @@ module HSQL
     end
 
     def metadata
-      @metadata ||= @front_matter ? ::YAML.load(@front_matter) : {}
+      @metadata ||= begin
+        hash = @front_matter ? ::YAML.load(@front_matter) : {}
+        environments = hash.delete('environments') || {}
+        hash.merge(environments[environment] || {})
+      end
     end
 
     def queries
@@ -46,12 +50,18 @@ module HSQL
 
     def parse!
       split!
-      interpolate_data!
+      interpolate_metadata!
       self
     end
 
     private
 
+    # Insert the `environments:` data for the given environment into our
+    # SQL queries.
+    def template_data
+      @data ||= metadata.merge(Data.for_machines(timestamp))
+    end
+
     def split!
       @split ||= begin
         top_half, divider, rest = string.partition(/^---$/)
@@ -65,17 +75,8 @@ module HSQL
       end
     end
 
-    def data
-      @data ||= begin
-        hash = metadata['data'] || {}
-        hash = hash.merge(hash[environment] || {})
-        hash.merge(Data.for_machines(timestamp))
-      end
-    end
-
-    def interpolate_data!
-      # Insert the `data:` section of YAML for the given environment into our SQL queries.
-      @rendered_sql = Template.new(@sql, @verbose).render(data)
+    def interpolate_metadata!
+      @rendered_sql = Template.new(@sql, @verbose).render(template_data)
     end
   end
 end

data/lib/hsql/version.rb

@@ -1,4 +1,4 @@
 # http://semver.org/
 module HSQL
-  VERSION = '0.3.8'
+  VERSION = '0.4.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: hsql
 version: !ruby/object:Gem::Version
-  version: 0.3.8
+  version: 0.4.0
 platform: ruby
 authors:
 - Jack Danger Canty
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-02 00:00:00.000000000 Z
+date: 2015-10-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mustache