structured_search 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: d44d9c424f6c8016aa9f87f520eb13fa78597259
+  data.tar.gz: efcd6f229905161abdccde0eb5740c94baefdbcf
+SHA512:
+  metadata.gz: ca7d8fde1e85b7cd55d862b98f90142cb27d2914b464ccd1d5692f3fa2f43df31e6372c7f69048581c964a3f1f851df90a88762c2261465e4b089d593c5ec852
+  data.tar.gz: 7640c8320709de35fe5e4c92b017e68280214f6b02cc22f522863c4dbdec49178c2755dd36b6d287a477d0c1171731b00ee070277a8eaa8768bd08b7f525678a

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright 2014 YOURNAME
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,3 @@
+= StructuredSearch
+
+This project rocks and uses MIT-LICENSE.

data/Rakefile

@@ -0,0 +1,23 @@
+begin
+  require 'bundler/setup'
+rescue LoadError
+  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+end
+
+require 'rdoc/task'
+
+RDoc::Task.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'StructuredSearch'
+  rdoc.options << '--line-numbers'
+  rdoc.rdoc_files.include('README.rdoc')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+# route :default to :spec for travis ci
+require 'rspec/core/rake_task'
+task :default => :spec
+RSpec::Core::RakeTask.new
+
+Bundler::GemHelper.install_tasks
+

data/lib/structured_search.rb

@@ -0,0 +1,25 @@
+require 'structured_search/lexer'
+require 'structured_search/parser'
+require 'structured_search/evaluator'
+require 'structured_search/errors'
+require 'structured_search/patterns'
+require 'structured_search/token'
+require 'structured_search/base_node'
+
+module StructuredSearch
+  class << self
+
+    # Creates an evaluator instance, with a given input and provider hash
+    # and returns the evaluator result.
+    # Params:
+    # +input+:: Input string to parse and evaluate.
+    # +providers+:: The search provider keys and classes.
+    def evaluate(input, providers)
+      parser = StructuredSearch::Parser.new(input, providers)
+      parser.parse_to_end
+      @evaluator = StructuredSearch::Evaluator.new(parser.statements)
+      @evaluator.eval if @evaluator
+    end
+
+  end
+end

data/lib/structured_search/base_node.rb

@@ -0,0 +1,23 @@
+module StructuredSearch
+  module Tree
+
+    # The base of any node in the AST, it stores the line, 
+    # column and type of token.
+    class BaseNode
+
+      #+line+:: The line this AST node was found
+      #+column+:: The column this AST node was found
+      #+type+:: Holds the type of node for fast lookup
+      attr_accessor :line, :column, :type
+
+      # sends each token value to it's respective attribute
+      def initialize(topts = {})
+        topts.each { |key, val| send "#{key}=", val }
+      end
+    end
+
+    # require all parse tree files
+    Dir[File.expand_path("../tree/*", __FILE__)].each { |f| require f }
+
+  end
+end

data/lib/structured_search/errors.rb

@@ -0,0 +1,22 @@
+module StructuredSearch
+
+  # Raised when an error occurs during lexical analysis, 
+  # such as an incorrect character in the input
+  class LexicalError < StandardError
+  end
+
+  # Raised when an error occurs during parsing, such as 
+  # an invalid token in the stream (FROM before a SELECT etc.)
+  class SyntaxError < StandardError
+  end
+
+  # Raised when a provider is referenced, but no provider
+  # class is given to the evaluator
+  class UnregisteredProviderError < StandardError
+  end
+
+  # Raised when a provider does not have a method required
+  # by StructuredSearch (e.g. doesn't respond to search() etc.)
+  class ProviderNotAvailableError < StandardError
+  end
+end

data/lib/structured_search/evaluator.rb

@@ -0,0 +1,89 @@
+require 'structured_search/errors'
+
+module StructuredSearch
+
+  # Evaluates one or more statements that make up a query.
+  # Each statement is made up of a series of AST nodes produced 
+  # by the parser, from input generated by the lexer.
+  class Evaluator
+
+    # statements to be evaluated
+    attr_accessor :statements
+
+    #:nodoc:
+    def initialize(statements = [])
+      @statements = statements
+    end
+
+    # Evaluate every statement given to the initializer. 
+    # Returns:
+    # +result+:: An array containing the result of each statement
+    def eval
+      results = []
+
+      @statements.each do |s|
+        st_res = eval_statement(s)
+        results << st_res if st_res
+      end
+      results
+    end
+  
+    # Evaluates a single statement.
+    # Returns:
+    # +result+:: An array containing the result of the statement
+    def eval_statement(statement)
+      search_terms = results = []
+
+      statement.each do |node|
+        
+        case node.type
+        when :SELECT
+          search_terms = node.search_terms
+        when :FROM
+          results = provider_lookup(node, search_terms)
+        when :WHERE
+          node.constraints.each do
+            # results = send :method, results (or something like that)
+          end
+        end
+      end
+
+      results
+    end
+
+private
+
+    # Performs a search operation, looking up each search term 
+    # on each provider (by way of Provider#search), and storing it in a Hash.
+    # Parameters:
+    # +node+:: The FROM node that contains search provider sources
+    # +search_terms+:: The search terms from a SELECT node, as an array
+    # Returns:
+    # +provider_results+::A Hash that contains the provider as a key, and the
+    # search term as a value, which can be used to lookup the result via 
+    # 'result[:provider][:search_term]'.
+    def provider_lookup(node, search_terms)
+      provider_results = {}
+      search_results = {}
+
+      node.sources.each do |provider, prov_class|
+        if prov_class.respond_to? :search
+          search_terms.each do |search_term|
+            search_result = prov_class.send :search, search_term
+            search_results[search_term] = search_result if search_result
+          end
+
+          provider_results[provider] = search_results
+
+        else
+          raise ProviderNotAvailableError, "Method 'search' not available for provider #{provider}"
+        end
+      end
+
+      provider_results
+    end
+
+
+
+  end
+end

data/lib/structured_search/lexer.rb

@@ -0,0 +1,89 @@
+require 'structured_search/token'
+require 'structured_search/errors'
+
+module StructuredSearch
+
+  # Converts the input into a token stream, that can be worked 
+  # by the syntax parser.
+  class Lexer
+
+    # +input+:: Input string to parse
+    # +column+:: Current column position
+    # +line+:: Current position in the line
+    # +lexer_offset+:: Current character position in the input string
+    attr_accessor :input, :column, :line, :lexer_offset
+
+    # Returns the current state of the lexer, by way of input, 
+    # current line and column.
+    def state
+      { input: input, column: column, line: line }
+    end
+
+    # Sets the state of the lexer
+    def state=(state)
+      @input = state[:input]
+      @column = state[:column]
+      @line = state[:line]
+    end
+
+    # Creates a new instance of the Lexer.
+    # Params:
+    # +input+:: The SQL input that will be parsed.
+    def initialize(input)
+      @input = input
+      @column = 1
+      @line = 1
+      @lexer_offset = 0
+    end
+
+    # Scans the input, matching each token that appears and 
+    # returns the token. Supports both read and peek operations
+    # determined by the state of the peek flag.
+    # Params:
+    # +is_peek+:: Whether the lexer will consume the token, or 
+    #             remain in it's current position (false by default)
+    # Returns:
+    # +token+:: A StructuredSeach::Token is returned to the caller.
+    def scan(is_peek = false)
+      PATTERNS.each do |pattern|
+        match = pattern[1].match(@input, @lexer_offset)
+        if match
+          token_data = { token:  pattern[0],
+                         lexeme: pattern[2] ? pattern[2].call(match) : '',
+                         line: @line, column: @column }
+          token = Token.new(token_data)
+
+          # increment line and col position if a read op:
+          if !is_peek
+            tok_length = match[0].size
+            newline_count = match[0].count("\n")
+            @lexer_offset += tok_length
+            @line += newline_count
+            @column = 1 if newline_count
+            @column += tok_length - (match[0].rindex("\n") || 0)
+          end
+
+          # clear any whitespace
+          if pattern[0] == :WHITESPACE
+            @lexer_offset += match[0].size
+            return scan(is_peek)
+          else
+            return token
+          end
+
+        end
+      end
+
+      # have we underrun the input due to lex error?:
+      if @lexer_offset < @input.size
+        raise LexicalError, "Unexpected character \"#{@input[@lexer_offset+1]}\"  at (Line #{@line}, Column #{@column})"
+      end
+
+      nil
+    end
+
+  end
+end
+
+
+

data/lib/structured_search/parser.rb

@@ -0,0 +1,134 @@
+module StructuredSearch
+
+  # Parses a token stream, returning an array of StructuredSearch::Statement
+  class Parser
+
+    # stores all parse tree nodes
+    attr_reader :statements, :providers
+
+    # Creates a new instance of the Parser, taking a Lexer and a hash of
+    # providers (An identifier and class that contains the search method)
+    # Params:
+    # +lexer+:: A StructuredSearch::Lexer object
+    # +providers+:: A Hash of provider names and their classes 
+    def initialize(input, providers)
+      @lexer = StructuredSearch::Lexer.new(input)
+      @providers = Hash.new
+      providers.each { |k,v| @providers[k.downcase] = v }
+      
+      @nodes, @statements = [], []
+    end
+
+    # Reads the next token in the stream
+    def read_token
+      @lexer.scan
+    end
+
+    # Peeks at the next token in stream
+    def peek_token
+      @lexer.scan(true)
+    end
+
+    # Parses the token stream into statements until there
+    # are no more tokens left
+    def parse_to_end
+      while peek_token
+        new_node = parse
+        @nodes << new_node if new_node
+      end
+
+      # flush token stream to a statement if no semicolon
+      new_statement if @nodes.length > 0
+    end
+
+    # Parses the next token in the token stream into an AST node
+    # Returns an AST node
+    def parse
+      @current_token = read_token
+      
+      case @current_token.token
+        when :SEMICOLON
+          new_statement
+        else
+          send "new_#{@current_token.token.downcase}"
+      end
+    end
+
+    # Basic options given to BaseNode when creating a new instance
+    # of a node, including the line, column and type
+    def basic_options
+      { line: @current_token.line, column: @current_token.column, type: @current_token.token }
+    end
+
+private
+
+    # Creates a new Tree::Select
+    def new_select
+      quant_list = [ :ALL, :DISTINCT ]
+      select_list = [ :ASTERISK, :STRING ]
+
+      select_tok = Tree::Select.new(basic_options)
+
+      # handle an optional set quantifier (ALL or DISTINCT)
+      if quant_list.include? peek_token.token
+        select_tok.set_quantifier = read_token.token
+      end
+
+      # handle a select list (ASTERISK or search terms)
+      if select_list.include? peek_token.token
+
+        # read in all search terms from the query:
+        while peek_token.token == :STRING
+          select_tok.add_search_term(read_token.lexeme)
+          read_token if peek_token.token == :COMMA
+        end
+
+      else
+        raise SyntaxError "No valid select list given (#{error_location})"
+      end
+      
+      select_tok
+    end
+
+    # Creates a new Tree::From
+    def new_from
+      source_tokens = [ :STRING ]
+      from_tok = Tree::From.new(basic_options)
+
+      # read in all the derived columns - search sources:
+      while peek_token and source_tokens.include? peek_token.token
+        src_token = read_token
+        # check if the provider is registered in the given list:
+        if provider_exists? src_token.lexeme.downcase
+          from_tok.sources[src_token.lexeme] = @providers[src_token.lexeme.downcase.to_sym]
+        else raise UnregisteredProviderError, "#{src_token.lexeme} is not a registered provider"
+        end
+      end
+
+      if from_tok.sources.count == 0
+        raise SyntaxError, "No search sources given (#{error_location})"
+      end
+
+      from_tok
+    end
+
+    # Creates a new Tree::Statement
+    def new_statement
+      @statements.push Tree::Statement.new(@nodes)
+      @nodes = [] # reset node array
+      nil
+    end
+
+    # Checks whether a search provider class exists
+    def provider_exists?(source)
+      @providers.has_key? source.to_sym
+    end
+
+    # Returns a string that will inform the user where an error
+    # has occurred.
+    def error_location
+      "Line #{basic_options[:line]}, Column #{basic_options[:column]}"
+    end
+    
+  end
+end

data/lib/structured_search/patterns.rb

@@ -0,0 +1,45 @@
+module StructuredSearch
+
+  class Lexer
+    
+    # SQL reserved words list
+    RESERVED = %w{
+      SELECT ALL DISTINCT FROM WHERE ASC DESC
+      }
+
+    # Pattern hash of token keys and regex values
+    PATTERNS = [
+
+      [:WHITESPACE, /[\r\v\f\t ]+/],
+      [:TERMINATOR, /[\r\n]/],
+
+      # intern reserved words and their patterns
+      *RESERVED.map { |rw| [rw.intern, /#{rw}(?=[^A-z0-9_])/] },
+      
+      # match single / double quoted strings
+      [:STRING, /(['"])(\\n|\\.|((?!\1)(?!\\)|.)*?((?!\1)(?!\\).)?)\1/, -> match { match[2] } ],
+      [:L_PAREN,    /\(/],
+      [:R_PAREN,    /\)/],
+      [:L_BRACKET,  /\[/],
+      [:R_BRACKET,  /\]/],
+      [:L_BRACE,    /\{/],
+      [:R_BRACE,    /\}/],
+      [:PERCENT,    /%/ ],
+      [:AMPERSAND,  /&/ ],
+      [:ASTERISK,   /\*/],
+      [:PLUS,       /\+/],
+      [:MINUS,      /-/],
+      [:COMMA,      /,/ ],
+      [:PERIOD,     /\./],
+      [:COLON,      /:/ ],
+      [:SEMICOLON,  /;/ ],
+      [:LEQ,        /<=/],
+      [:EQUALS,     /=/ ],
+      [:GEQ,        />=/],
+      [:QUESTION,   /\?/],
+      [:CIRCUMFLEX, /\^/],
+      [:UNDERSCORE, /_/ ],
+      [:PIPE,       /\|/]
+    ].map { |pattern| [pattern[0], /\G#{pattern[1]}/m, pattern[2]] }
+  end
+end

data/lib/structured_search/token.rb

@@ -0,0 +1,27 @@
+module StructuredSearch
+
+  # A token generated by the lexer
+  class Token
+
+    #+token+:: The token's type
+    #+lexeme+:: The matched characters that make up the token
+    #+column+:: Column this token was located
+    #+line+:: Line this token was located
+    attr_accessor :token, :lexeme, :column, :line
+
+    #:nodoc:
+    def initialize(tok = {})
+      #send val to key setter
+      tok.each do |k, v|
+        send "#{k}=", v
+      end
+    end
+
+    # Returns a string representation of the token
+    def to_s
+      "#{@token}: '#{@lexeme}' (Line #{@line}, Column #{@column})"
+    end
+  end
+
+end
+

data/lib/structured_search/tree/asterisk.rb

@@ -0,0 +1,16 @@
+module StructuredSearch
+  module Tree
+
+    # Asterisk (*) node
+    class Asterisk < BaseNode
+
+      #:nodoc:
+      def initialize(*argv)
+        super *argv
+      end
+
+
+    end
+
+  end
+end

data/lib/structured_search/tree/from.rb

@@ -0,0 +1,23 @@
+module StructuredSearch
+  module Tree
+
+    # FROM reserved word node
+    class From < BaseNode
+
+      # in sql spec, this is where we store all the
+      # derived columns, such as Google, Omniref etc
+      #
+      # This is a hash containing the symbol reference
+      # and a class reference to what generates the query
+      attr_accessor :sources
+
+      #:nodoc:
+      def initialize(*argv)
+        @sources = {}
+        super *argv
+      end
+
+    end
+
+  end
+end

data/lib/structured_search/tree/select.rb

@@ -0,0 +1,33 @@
+module StructuredSearch
+  module Tree
+
+    ##
+    # SELECT reserved word node
+    class Select < BaseNode
+
+      # +set_quantifier+:: Whether this search uses ALL or DISTINCT
+      # +search_terms+:: The search terms we're looking for
+      attr_accessor :set_quantifier, :search_terms
+
+      # Sets the set quantifier (either ALL or DISTINCT) for this statement
+      def set_quantifier=(quantifier)
+        if [:ALL, :DISTINCT].include? quantifier
+          @set_quantifier = quantifier
+        end
+      end
+
+      # Adds a search term to the list of terms
+      def add_search_term(term)
+        @search_terms.push term
+      end
+
+      #:nodoc:
+      def initialize(*argv)
+        @search_terms = []
+        @set_quantifier = :ALL
+        super *argv
+      end
+
+    end
+  end
+end

data/lib/structured_search/tree/statement.rb

@@ -0,0 +1,13 @@
+module StructuredSearch::Tree
+  
+  # A statement is a collection of nodes that make up a 
+  # single query, such as "SELECT * FROM 'Here', 'There';"
+  class Statement < Array
+
+    #:nodoc:
+    def initialize(nodes)
+      self.concat(nodes)
+    end
+
+  end
+end

data/lib/structured_search/tree/string.rb

@@ -0,0 +1,15 @@
+module StructuredSearch
+  module Tree
+
+    # Represents a string node, 'Google', 'SearchThisString' etc.
+    class String < BaseNode
+
+      #:nodoc:
+      def initialize(*argv)
+        super *argv
+      end
+
+    end
+
+  end
+end

data/lib/structured_search/tree/where.rb

@@ -0,0 +1,14 @@
+module StructuredSearch
+  module Tree
+
+    # WHERE reserved word node
+    class Where < BaseNode
+
+      #:nodoc:
+      def initialize(*argv)
+        super *argv
+      end
+
+    end
+  end
+end

data/lib/structured_search/version.rb

@@ -0,0 +1,4 @@
+module StructuredSearch
+  #:nodoc:
+  VERSION = "0.0.1"
+end

data/lib/tasks/structured_search_tasks.rake

@@ -0,0 +1,4 @@
+# desc "Explaining what the task does"
+# task :structured_search do
+#   # Task goes here
+# end

metadata

@@ -0,0 +1,175 @@
+--- !ruby/object:Gem::Specification
+name: structured_search
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Matt Ryder
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-11-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: coveralls
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec-rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard-spork
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: google_custom_search_api
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Use SQL to query search engines and APIs.
+email:
+- matt@mattryder.co.uk
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- MIT-LICENSE
+- README.rdoc
+- Rakefile
+- lib/structured_search.rb
+- lib/structured_search/base_node.rb
+- lib/structured_search/errors.rb
+- lib/structured_search/evaluator.rb
+- lib/structured_search/lexer.rb
+- lib/structured_search/parser.rb
+- lib/structured_search/patterns.rb
+- lib/structured_search/token.rb
+- lib/structured_search/tree/asterisk.rb
+- lib/structured_search/tree/from.rb
+- lib/structured_search/tree/select.rb
+- lib/structured_search/tree/statement.rb
+- lib/structured_search/tree/string.rb
+- lib/structured_search/tree/where.rb
+- lib/structured_search/version.rb
+- lib/tasks/structured_search_tasks.rake
+homepage: http://www.mattryder.co.uk
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.7
+signing_key: 
+specification_version: 4
+summary: Use SQL to query search engines and APIs.
+test_files: []