pg_query 0.0.2

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    ZTM4ODg0YjI2Mzk1MmZhODhiOGQwNDJlZDNhMGJiZTViYjRlNzNmMQ==
+  data.tar.gz: !binary |-
+    NjUxZDg4NGIxY2NkN2U2ZDczMTA3ZjRmMzIxYmEwNWI5ZGE5OGIwOQ==
+SHA512:
+  metadata.gz: !binary |-
+    NWY3NGEyMmNlNDJjMGM3NDU2YTE3ZDVmYTdiYjNiYmU3OTYxNmQxYThiMTEy
+    NzBlMTljZjE1NmU3OGIwM2FiZmRjMmQ1MjQ0ZDJkMDVkZTk2ZWJhMzgzNTBm
+    NGI0NjUyNDM3ZWY3OTNjM2Y3NzVmNTBkOWZhZmU2MzM1MTAxODg=
+  data.tar.gz: !binary |-
+    Nzc3YTAxODJiYjdhMGEwMTUyNmZlMmUyNDRkNWEwZjkzODI0Y2ZiYWJmMzFm
+    NWM3YzAyMTFkOTU4YmVjYzllMGU2ODA3Y2YzZWQ2MGIxOTNhNGJmMDI1Yjlk
+    MWFmNWM1NGE2Njk1MWQzNGE0ZjI0MDYyZTc5NTMzMTI4YjA0ZWM=

data/LICENSE

@@ -0,0 +1,7 @@
+Copyright (c) 2014, pganalyze
+
+Permission to use, copy, modify, and distribute this software and its documentation for any purpose, without fee, and without a written agreement is hereby granted, provided that the above copyright notice and this paragraph and the following two paragraphs appear in all copies.
+
+IN NO EVENT SHALL pganalyze BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF pganalyze HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+pganalyze SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON AN "AS IS" BASIS, AND pganalyze HAS NO OBLIGATIONS TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.

data/Rakefile

@@ -0,0 +1,14 @@
+require "bundler/gem_tasks"
+require "rake/extensiontask"
+require "rspec/core/rake_task"
+
+Rake::ExtensionTask.new "pg_query" do |ext|
+  ext.lib_dir = "lib/pg_query"
+end
+
+RSpec::Core::RakeTask.new
+
+task spec: :compile
+
+task default: :spec
+task test: :spec

data/ext/pg_query/extconf.rb

@@ -0,0 +1,35 @@
+require 'mkmf'
+
+workdir = Dir.pwd
+pgdir = File.join(workdir, "postgres")
+
+# Download & compile PostgreSQL if we don't have it yet
+#
+# Note: We intentionally use a patched version that fixes bugs in outfuncs.c
+if !Dir.exists?(pgdir)
+  unless File.exists?("#{workdir}/postgres.zip")
+    system("curl https://codeload.github.com/pganalyze/postgres/zip/more-outfuncs -o #{workdir}/postgres.zip") || raise("ERROR")
+  end
+  system("unzip -q #{workdir}/postgres.zip -d #{workdir}") || raise("ERROR")
+  system("mv #{workdir}/postgres-more-outfuncs #{pgdir}") || raise("ERROR")
+  system("cd #{pgdir}; ./configure") || raise("ERROR")
+  system("cd #{pgdir}; make") || raise("ERROR")
+end
+
+$objs = `find #{pgdir}/src/backend -name '*.o' | egrep -v '(main/main\.o|snowball|libpqwalreceiver|conversion_procs)' | xargs echo`
+$objs += " #{pgdir}/src/timezone/localtime.o #{pgdir}/src/timezone/strftime.o #{pgdir}/src/timezone/pgtz.o"
+$objs += " #{pgdir}/src/common/libpgcommon_srv.a #{pgdir}/src/port/libpgport_srv.a"
+$objs = $objs.split(" ")
+
+$objs << File.join(File.dirname(__FILE__), "pg_query.o")
+
+$CFLAGS << " -I #{pgdir}/src/include"
+
+SYMFILE = File.join(File.dirname(__FILE__), "pg_query.sym")
+if RUBY_PLATFORM =~ /darwin/
+  $DLDFLAGS << "-exported_symbols_list #{SYMFILE}"
+else
+  $DLDFLAGS << "-export-symbols #{SYMFILE}"
+end
+
+create_makefile 'pg_query/pg_query'

data/ext/pg_query/pg_query.c

@@ -0,0 +1,107 @@
+#include "postgres.h"
+#include "utils/memutils.h"
+#include "parser/parser.h"
+#include "nodes/print.h"
+
+#include <unistd.h>
+#include <fcntl.h>
+
+#include <ruby.h>
+
+const char* progname = "pg_query";
+
+static void raise_parse_error(ErrorData* error)
+{
+	VALUE cPgQuery, cParseError;
+	VALUE exc, args[2];
+	
+	cPgQuery = rb_const_get(rb_cObject, rb_intern("PgQuery"));
+	cParseError = rb_const_get_at(cPgQuery, rb_intern("ParseError"));
+	
+	args[0] = rb_tainted_str_new_cstr(error->message);
+	args[1] = INT2NUM(error->cursorpos);
+	
+	exc = rb_class_new_instance(2, args, cParseError);
+	
+	rb_exc_raise(exc);
+}
+
+#define STDERR_BUFFER_LEN 4096
+
+static VALUE pg_query_raw_parse(VALUE self, VALUE input)
+{
+	Check_Type(input, T_STRING);
+	
+	MemoryContext ctx = NULL;
+	VALUE result;
+	ErrorData* error = NULL;
+	char stderr_buffer[STDERR_BUFFER_LEN + 1] = {0};
+	int stderr_global;
+	int stderr_pipe[2];
+
+	ctx = AllocSetContextCreate(TopMemoryContext,
+								"RootContext",
+								ALLOCSET_DEFAULT_MINSIZE,
+								ALLOCSET_DEFAULT_INITSIZE,
+								ALLOCSET_DEFAULT_MAXSIZE);
+	MemoryContextSwitchTo(ctx);
+	
+	// Setup pipe for stderr redirection
+	if (pipe(stderr_pipe) != 0)
+		rb_raise(rb_eRuntimeError, "PgQuery._raw_parse: Could not allocate pipe for stderr redirection");
+
+	fcntl(stderr_pipe[0], F_SETFL, fcntl(stderr_pipe[0], F_GETFL) | O_NONBLOCK);
+	
+	// Redirect stderr to the pipe
+	stderr_global = dup(STDERR_FILENO);
+	dup2(stderr_pipe[1], STDERR_FILENO);
+	close(stderr_pipe[1]);
+	
+	// Parse it!
+	PG_TRY();
+	{
+		List *tree;
+		char *str;
+		
+		str = StringValueCStr(input);
+		tree = raw_parser(str);
+		
+		str = nodeToString(tree);
+	
+		// Save stderr for result
+		read(stderr_pipe[0], stderr_buffer, STDERR_BUFFER_LEN);
+	
+		result = rb_ary_new();
+		rb_ary_push(result, rb_tainted_str_new_cstr(str));
+		rb_ary_push(result, rb_str_new2(stderr_buffer));
+	
+		pfree(str);
+	}
+	PG_CATCH();
+	{
+		error = CopyErrorData();
+		FlushErrorState();
+	}
+	PG_END_TRY();
+	
+	// Restore stderr & return to previous PostgreSQL memory context
+	dup2(stderr_global, STDERR_FILENO);
+	MemoryContextSwitchTo(TopMemoryContext);
+	MemoryContextDelete(ctx);
+	
+	// If we got an error, throw a ParseError exception
+	if (error) raise_parse_error(error);
+	
+	return result;
+}
+
+void Init_pg_query(void)
+{
+	VALUE cPgQuery;
+	
+	MemoryContextInit();
+
+	cPgQuery = rb_const_get(rb_cObject, rb_intern("PgQuery"));
+
+	rb_define_singleton_method(cPgQuery, "_raw_parse", pg_query_raw_parse, 1);
+}

data/ext/pg_query/pg_query.sym

@@ -0,0 +1 @@
+_Init_pg_query

data/lib/pg_query/parse.rb

@@ -0,0 +1,181 @@
+require 'json'
+
+class PgQuery
+  def self.parse(query)
+    parsetree, stderr = _raw_parse(query)
+    
+    parsetree = [] if parsetree == '<>'
+    
+    if !parsetree.nil? && !parsetree.empty?
+      parsetree = parsetree_to_json(parsetree)
+      parsetree = JSON.parse(parsetree, max_nesting: 1000)
+    end
+    
+    warnings = []
+    stderr.each_line do |line|
+      next unless line[/^WARNING/]
+      warnings << line.strip
+    end
+    
+    PgQuery.new(query, parsetree, warnings)
+  end
+
+  attr_reader :query
+  attr_reader :parsetree
+  attr_reader :warnings
+  def initialize(query, parsetree, warnings = [])
+    @query = query
+    @parsetree = parsetree
+    @warnings = warnings
+  end
+
+protected
+  def self.parsetree_to_json(str)
+    str.strip!
+    control_chars = '(){}: '
+    location = nil # :hashname, :key, :value
+    structure_stack = [] # :hash, :array (when delimiter opens we push, when delimiter closes we pop)
+    open_string = false
+    escaped_string = false
+    next_char_is_escaped = false
+    double_hash_close_in = 0 # This is used to ask for an additional closing delimiter (added when x = 1 and we reach a closing delimiter)
+    out = ""
+  
+    i = 0
+    loop do
+      break if i > str.size-1
+    
+      c = str[i]
+      last_location = location
+      char_is_escaped = next_char_is_escaped
+      next_char_is_escaped = false
+      if control_chars.include?(c) && !char_is_escaped && (!open_string || !escaped_string)
+        # Space is not always a control character, skip in those cases
+        if c == ' ' && last_location == :hashname && str[i+1] != ':'
+          out += c
+          i += 1
+          next
+        end
+        
+        # Keep empty nodes as empty hashes (e.g. nodes that can't be output)
+        if c == '{' && str[i+1] == '}'
+          out += "{}"
+          out += ", " if i+2 < str.size && !'})'.include?(str[i+2])
+          i += 2 # Skip {}
+          next
+        end
+      
+        location = nil # All control characters reset location
+      
+        # This should never happen, but if it does, catch it
+        if open_string
+          out += '"' unless escaped_string
+          open_string = false
+          escaped_string = false
+        end
+      
+        # Write out JSON control characters
+        if last_location == :hashname
+          out += ': {'
+        elsif last_location == :key
+          out += ': '
+        end
+      
+        case c
+        when '('
+          out += '['
+        when ')'
+          out += ']'
+        when '{'
+          out += '{'
+        when '}'
+          out += '}}'
+        when ':'
+          # No JSON equivalent
+        end
+        
+        # Handle double hash closes (required for out-of-place nodes like ANY)
+        case c
+        when '{'  
+          if double_hash_close_in > 0
+            double_hash_close_in += 1
+          end
+        when '}'
+          if double_hash_close_in == 1
+            out += '}}'
+            double_hash_close_in = 0
+          elsif double_hash_close_in > 1
+            double_hash_close_in -= 1
+          end
+        end
+      
+        # Write out delimiter if needed
+        if (last_location == :value || '})'.include?(c)) && i+1 < str.size && !'})'.include?(str[i+1])
+          out += ', '
+        end
+      
+        # Determine new location
+        case c
+        when '{'
+          structure_stack << :hash
+          location = :hashname
+        when '('
+          structure_stack << :array
+          location = :value if !control_chars.include?(str[i+1])
+        when '}', ')'
+          structure_stack.pop
+        when ':'
+          location = :key
+        when ' '
+          location = :value if [:value, :key].include?(last_location) && !control_chars.include?(str[i+1])
+        end
+      else
+        if char_is_escaped
+          case c
+          when '"'
+            out += "\\\""
+          when '\\'
+            out += "\\\\"
+          else
+            out += c
+          end
+        elsif str[i] == '<' && str[i+1] == '>' && control_chars.include?(str[i+2])
+          # Make <> into null values
+          i += 1
+          out += "null"
+        elsif c == '\\'
+          next_char_is_escaped = true # For next round
+          # Ignore all other cases
+        elsif c[/[A-Z]/] && !open_string && last_location != :value && last_location != :hashname && structure_stack.last == :hash
+          # We were not expecting a node name here, but this can happen (e.g. with ANY), try to construct into valid expression
+          location = :hashname
+          double_hash_close_in = 1
+          open_string = true
+          out += '"lexpr": {"'
+          out += c
+        elsif control_chars.include?(str[i-1]) && !open_string && !'0123456789'.include?(c)
+          open_string = true
+          escaped_string = true if c == '"'
+          out += '"' unless escaped_string
+          out += c
+          c = nil # To avoid close string taking care of us...
+        else
+          out += c
+        end
+      
+        # Close string if next element is control character
+        if open_string && !char_is_escaped && c != '\\' &&
+           ((last_location == :hashname && ((str[i+1] == '}') || (str[i+1] == ' ' && str[i+2] == ':') || (str[i+1] == ' ' && str[i+2] == ' ' && str[i+3] == ':'))) ||
+            (last_location != :hashname && !escaped_string && control_chars.include?(str[i+1])) || 
+            (escaped_string && c == '"'))
+          out += '"' unless escaped_string
+          open_string = false
+          escaped_string = false
+        end
+      end
+
+      i += 1
+    end
+    out
+  end
+end

data/lib/pg_query/parse_error.rb

@@ -0,0 +1,9 @@
+class PgQuery
+  class ParseError < ArgumentError
+    attr_reader :location
+    def initialize(message, location)
+      super(message)
+      @location = location
+    end
+  end
+end

data/lib/pg_query/version.rb

@@ -0,0 +1,3 @@
+class PgQuery
+  VERSION = '0.0.2'
+end

data/lib/pg_query.rb

@@ -0,0 +1,5 @@
+require 'pg_query/version'
+require 'pg_query/parse_error'
+
+require 'pg_query/pg_query'
+require 'pg_query/parse'

metadata

@@ -0,0 +1,95 @@
+--- !ruby/object:Gem::Specification
+name: pg_query
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+platform: ruby
+authors:
+- Lukas Fittl
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-05-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake-compiler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.8'
+description: Parses SQL queries using a copy of the PostgreSQL server query parser
+email: lukas@fittl.com
+executables: []
+extensions:
+- ext/pg_query/extconf.rb
+extra_rdoc_files: []
+files:
+- LICENSE
+- Rakefile
+- ext/pg_query/extconf.rb
+- ext/pg_query/pg_query.c
+- ext/pg_query/pg_query.sym
+- lib/pg_query.rb
+- lib/pg_query/parse.rb
+- lib/pg_query/parse_error.rb
+- lib/pg_query/version.rb
+homepage: http://github.com/pganalyze/pg_query
+licenses:
+- PostgreSQL
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: PostgreSQL query parsing and normalization library
+test_files: []