filter_lexer 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 4eb20adfd043c015d21dbc1639c4669945131cef
+  data.tar.gz: a0f2855582ae1094e6ef329b82d04f4b8b03fcec
+SHA512:
+  metadata.gz: ccc25baabaa2ca2af95d5b7653dd996ab674c6822708bef2ff92db4fd26d6c7b25dad79c1c2fd0e71cd472763c05abbf78c2af6b5f7c8693b0837e325c7c9d65
+  data.tar.gz: 71f924d03e1af187be0e270c0ceab349f092cb02cb8ff0fcdcfe3ba09864f82c693595ddd4b01c82fb821e5a7a1e2ec6755558a31a0699eeff14f065b12330a8

data/.gitignore

@@ -0,0 +1,35 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalisation:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+Gemfile.lock
+.ruby-version
+.ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+rvm:
+  - 2.2.3
+before_install: gem install bundler -v 1.10.6

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in filter_lexer.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Michon van Dooren
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Michon van Dooren
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,70 @@
+# FilterLexer
+
+This is a simple treetop implementation for a basic SQL-like filtering syntax.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'filter_lexer'
+```
+
+And then execute:
+
+```sh
+bundle
+```
+
+Or install it yourself as:
+
+```sh
+gem install filter_lexer
+```
+
+## Usage
+
+To parse a filter, simply do the following:
+
+```ruby
+tree = FilterLexer::Parser.parse('foo == "BAR"')
+```
+
+If the parsing succeeds, a tree is returned. This tree can be output (for development purposes) using `FilterLexer::Parser.output_tree`.
+
+If the parsing fails, a `ParseException` is raised containing the detials of the failure.
+
+Simple complete example showing success and failure:
+
+```ruby
+examples = [
+	'foo == "BAR"',
+	'foo == "BAR" &&',
+]
+
+examples.each do |example|
+	begin
+		tree = FilterLexer::Parser.parse(example)
+		puts "Parsed #{example}"
+		FilterLexer::Parser.output_tree(tree)
+	rescue FilterLexer::ParseException
+		puts $!.message
+		puts $!.context
+	end
+end
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rspec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/MaienM/FilterLexer.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "filter_lexer"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/filter_lexer.gemspec

@@ -0,0 +1,32 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'filter_lexer/version'
+
+Gem::Specification.new do |spec|
+	spec.name          = 'filter_lexer'
+	spec.version       = FilterLexer::VERSION
+	spec.authors       = ['Michon van Dooren']
+	spec.email         = ['michon1992@gmail.com']
+
+	spec.summary       = %q{A basic lexer that supports basic sql-like filtering logic.}
+	spec.description   = %q{Simple treetop lexer that supports writing filters in an SQL-like manner.}
+	spec.homepage      = 'http://github.com/MaienM/FilterLexer'
+	spec.license       = 'MIT'
+
+	if spec.respond_to?(:metadata)
+		spec.metadata['allowed_push_host'] = 'https://rubygems.org'
+	else
+		raise 'RubyGems 2.0 or newer is required to protect against public gem pushes.'
+	end
+
+	spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+	spec.bindir        = 'exe'
+	spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+	spec.require_paths = ['lib']
+
+	spec.add_development_dependency 'bundler', '~> 1.10'
+	spec.add_development_dependency 'rake', '~> 10.0'
+	spec.add_development_dependency 'rspec'
+	spec.add_dependency 'treetop', '~> 1.6'
+end

data/lib/filter_lexer/exceptions.rb

@@ -0,0 +1,28 @@
+module FilterLexer
+	class ParseException < Exception
+		def initialize(parser)
+			@index = parser.failure_index
+			@input = parser.input
+		end
+
+		def message
+			return "Parse error at index #{@index}"
+		end
+
+		def context
+			i1 = @index - 40
+			i1 = 0 if i1 < 0
+			i2 = @index + 40
+			i2 = @input.size if i2 > @input.size
+
+			context = @input.slice(i1..i2)
+			if i1 > 0
+				context = "...#{context}"
+				i1 -= 3
+			end
+			context = "#{context}..." if i2 < @input.size 
+
+			return context + "\r\n" + ' ' * (@index - i1) + '^'
+		end
+	end
+end

data/lib/filter_lexer/formatters/sql.rb

@@ -0,0 +1,143 @@
+module FilterLexer
+	class Expression
+		def to_query
+			return "#{query_string} #{query_variables}"
+		end
+
+		def query_string
+			return self.elements.map { |e| e.query_string }.join(' ')
+		end
+
+		def query_variables
+			return self.elements.map { |e| e.query_variables }.flatten
+		end
+	end
+
+	class Group
+		def query_string
+			return "(#{self.elements.first.query_string})"
+		end
+
+		def query_variables
+			return self.elements.first.query_variables
+		end
+	end
+
+	class Filter
+		def query_string
+			return "#{elements[0].sql} #{elements[1].sql} ?"
+		end
+
+		def query_variables 
+			return [elements[2].sql]
+		end
+	end
+
+	class Operator
+		def query_string
+			return self.sql
+		end
+
+		def query_variables 
+			return []
+		end
+	end
+
+	class OrOperator
+		def sql
+			return 'OR'
+		end
+	end
+
+	class AndOperator
+		def sql
+			return 'AND'
+		end
+	end
+
+	class EQOperator
+		def sql
+			return 'IS' if parent.value_class < ValueSpecial
+			return '='
+		end
+	end
+
+	class NEQOperator
+		def sql
+			return 'IS NOT' if parent.value_class < ValueSpecial
+			return '<>'
+		end
+	end
+
+	class LTOperator
+		def sql
+			return '<'
+		end
+	end
+
+	class LEOperator
+		def sql
+			return '<='
+		end
+	end
+
+	class GTOperator
+		def sql
+			return '>='
+		end
+	end
+
+	class GEOperator
+		def sql
+			return '>='
+		end
+	end
+
+	class NotLikeOperator
+		def sql
+			return 'NOT LIKE'
+		end
+	end
+
+	class LikeOperator
+		def sql
+			return 'LIKE'
+		end
+	end
+
+	class BooleanLiteralFalse
+		def sql
+			return 'FALSE'
+		end
+	end
+
+	class BooleanLiteralTrue
+		def sql
+			return 'TRUE'
+		end
+	end
+
+	class NullLiteral
+		def sql
+			return 'NULL'
+		end
+	end
+
+	class StringLiteral
+		def sql
+			return self.text_value
+		end
+	end
+
+	class NumberLiteral
+		def sql
+			return self.text_value
+		end
+	end
+
+	class Identifier
+		def sql
+			return self.text_value
+		end
+	end
+end

data/lib/filter_lexer/nodes.rb

@@ -0,0 +1,92 @@
+module FilterLexer
+	class Expression < Treetop::Runtime::SyntaxNode
+	end
+
+	class Group < Treetop::Runtime::SyntaxNode
+	end
+
+	class Filter < Treetop::Runtime::SyntaxNode
+		def identifier
+			return elements[0].sql
+		end
+
+		def operator
+			return elements[1].sql
+		end
+
+		def value
+			return elements[2].sql
+		end
+
+		def value_class
+			return elements[2].class
+		end
+	end
+
+	# Operators
+
+	class Operator < Treetop::Runtime::SyntaxNode
+	end
+
+	class OrOperator < Operator
+	end
+
+	class AndOperator < Operator
+	end
+
+	class EQOperator < Operator
+	end
+
+	class NEQOperator < Operator
+	end
+
+	class LTOperator < Operator
+	end
+
+	class LEOperator < Operator
+	end
+
+	class GTOperator < Operator
+	end
+
+	class GEOperator < Operator
+	end
+
+	class NotLikeOperator < Operator
+	end
+
+	class LikeOperator < Operator
+	end
+
+	# Values
+
+	class Value < Treetop::Runtime::SyntaxNode
+	end
+
+	class ValueSpecial < Value
+	end
+
+	class ValueScalar < Value
+	end
+
+	class BooleanLiteralFalse < ValueSpecial
+	end
+
+	class BooleanLiteralTrue < ValueSpecial
+	end
+
+	class NullLiteral < ValueSpecial
+	end
+
+	class StringLiteral < ValueScalar
+	end
+
+	class NumberLiteral < ValueScalar
+	end
+
+	# Identifier
+
+	class Identifier < Treetop::Runtime::SyntaxNode
+	end
+end
+

data/lib/filter_lexer/syntax.treetop

@@ -0,0 +1,169 @@
+grammar FilterLexer
+	# Complex combinations
+
+	rule expression
+		space? (expression_compound / body) space? <Expression>
+	end
+
+	rule expression_compound
+		body space? combine_operator space? expression <Expression>
+	end
+
+	rule body
+		group / filter
+	end
+
+	rule group
+		'(' expression ')' <Group>
+	end
+
+	# Filters
+
+	rule filter 
+		filter_equality / filter_comparison / filter_like
+	end
+
+	rule filter_equality
+		identifier space? equality_operator space? value <Filter>
+	end
+
+	rule filter_comparison
+		identifier space? comparison_operator space? number <Filter>
+	end
+
+	rule filter_like
+		identifier space like_operator space string <Filter>
+	end
+
+	# Logic
+
+	rule combine_operator
+		or_operator / and_operator
+	end
+
+	rule or_operator
+		'' ('||' / 'OR' / 'or') <OrOperator>
+	end
+
+	rule and_operator
+		'' ('&&' / 'AND' / 'and') <AndOperator>
+	end
+
+	# Operators
+
+	rule equality_operator
+		eq_operator / neq_operator
+	end
+
+	rule comparison_operator
+		le_operator / lt_operator / ge_operator / gt_operator
+	end
+
+	rule like_operator
+		yes_like_operator / not_like_operator
+	end
+
+	rule eq_operator
+		'==' <EQOperator>
+	end
+
+	rule neq_operator
+		'' ('!=' / '<>') <NEQOperator>
+	end
+
+	rule lt_operator
+		'<' <LTOperator>
+	end
+
+	rule le_operator
+		'<=' <LEOperator>
+	end
+
+	rule gt_operator
+		'>' <GTOperator>
+	end
+
+	rule ge_operator
+		'>=' <GEOperator>
+	end
+
+	rule yes_like_operator
+		'' ('like' / 'LIKE' / '~=') <LikeOperator>
+	end
+
+	rule not_like_operator
+		'' ('!~=' / 'NOT LIKE' / 'not like') <NotLikeOperator>
+	end
+
+	# Literals
+
+	rule value
+		value_special / number / string
+	end
+
+	rule value_special
+		null / boolean 
+	end
+
+	rule number
+		number_sign? (number_float / number_integer) number_exponent? <NumberLiteral>
+	end
+
+	rule number_integer
+		number_base
+	end
+
+	rule number_float
+		number_base '.' number_base
+	end
+
+	rule number_sign
+		'+' / '-'
+	end
+
+	rule number_exponent
+		'e' number_base
+	end
+
+	rule number_base
+		[0-9]+
+	end
+
+	rule boolean
+		boolean_true / boolean_false
+	end
+
+	rule boolean_true
+		'' ('true' / 'TRUE' / 'on' / 'ON' / 'yes' / 'YES') <BooleanLiteralTrue>
+	end
+
+	rule boolean_false
+		'' ('false' / 'FALSE' / 'off' / 'OFF' / 'no' / 'NO') <BooleanLiteralFalse>
+	end
+
+	rule null
+		'' ('null' / 'NULL' / 'nil' / 'NIL' / 'nul' / 'NUL') <NullLiteral>
+	end
+
+	rule string
+		string_single / string_double
+	end
+
+	rule string_single
+		"'" ([^'\\] / "\\" . )* "'" <StringLiteral>
+	end
+
+	rule string_double
+		'"' ([^"\\] / "\\" . )* '"' <StringLiteral>
+	end
+
+	rule identifier
+		[a-zA-Z] [a-zA-Z0-9_]* <Identifier>
+	end
+
+	# Whitespace
+
+	rule space
+		[\s]+
+	end
+end

data/lib/filter_lexer/version.rb

@@ -0,0 +1,3 @@
+module FilterLexer
+	VERSION = "0.1.0"
+end

data/lib/filter_lexer.rb

@@ -0,0 +1,57 @@
+require 'treetop'
+
+require "filter_lexer/version"
+require "filter_lexer/exceptions"
+require "filter_lexer/nodes"
+require 'filter_lexer/syntax.treetop'
+
+module FilterLexer
+	class Parser
+		@@parser = FilterLexerParser.new
+
+		class << self
+			def parse(data)
+				# Pass the data over to the parser instance
+				tree = @@parser.parse(data)
+
+				# If the AST is nil then there was an error during parsing
+				# we need to report a simple error message to help the user
+				if tree.nil?
+					raise ParseException, @@parser
+				end
+
+				# Cleanup the tree
+				clean_tree(tree)
+
+				return tree
+			end
+
+			def output_tree(element, indent = '')
+				puts "#{indent}#{element.class}: #{element.text_value}"
+				(element.elements || []).each do |e|
+					output_tree(e, "#{indent}  ")
+				end
+			end
+
+			private
+
+			def is_syn(node)
+				return node.class.name == 'Treetop::Runtime::SyntaxNode'
+			end
+
+			def node_elem(node)
+				return node.elements || []
+			end
+
+			def clean_tree(root_node)
+				return if root_node.elements.nil?
+				# Clean child elements
+				root_node.elements.each { |node| clean_tree(node) }
+				# Remove empty syntax elements
+				root_node.elements.reject! { |node| node_elem(node).empty? && is_syn(node) }
+				# Replace syntax elements with just one child with that child
+				root_node.elements.map! { |node| (node_elem(node).size == 1 && is_syn(node)) ? node.elements.first : node }
+			end
+		end
+	end
+end

metadata

@@ -0,0 +1,117 @@
+--- !ruby/object:Gem::Specification
+name: filter_lexer
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Michon van Dooren
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2015-11-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: treetop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+description: Simple treetop lexer that supports writing filters in an SQL-like manner.
+email:
+- michon1992@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- LICENSE
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- filter_lexer.gemspec
+- lib/filter_lexer.rb
+- lib/filter_lexer/exceptions.rb
+- lib/filter_lexer/formatters/sql.rb
+- lib/filter_lexer/nodes.rb
+- lib/filter_lexer/syntax.treetop
+- lib/filter_lexer/version.rb
+homepage: http://github.com/MaienM/FilterLexer
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5.1
+signing_key: 
+specification_version: 4
+summary: A basic lexer that supports basic sql-like filtering logic.
+test_files: []