simply_stored 0.1.4

data/test/vendor/simplerdb-0.2/lib/simplerdb/query_language.rb

@@ -0,0 +1,266 @@
+require 'rubygems'
+require 'dhaka'
+require 'set'
+
+module SimplerDB
+  # Uses the lexer/parser/evaluator to perform the query and do simple paging
+  class QueryExecutor
+    ERROR_MARKER = ">>>"
+  
+    def initialize
+      @lexer = Dhaka::Lexer.new(QueryLexerSpec)
+      @parser = Dhaka::Parser.new(QueryGrammar)
+    end
+    
+    # Execute the query
+    def do_query(query, domain, max = 100, token = 0)
+      parse_result = @parser.parse(@lexer.lex(query))
+      token = 0 if token.nil?
+      
+      case parse_result
+        when Dhaka::TokenizerErrorResult
+          raise tokenize_error_message(parse_result.unexpected_char_index, query)
+        when Dhaka::ParseErrorResult
+          raise parse_error_message(parse_result.unexpected_token, query) 
+      end
+  
+      items = QueryEvaluator.new(domain).evaluate(parse_result)
+      results = []
+      count = 0
+      items.each do |item|
+        break if results.size == max
+        results << item if count >= token
+        count += 1
+      end
+      
+      if (count == items.size)
+        return results,nil
+      else
+        return results,count
+      end
+    end
+    
+    # From dhaka examples
+    def parse_error_message(unexpected_token, program)
+      if unexpected_token.symbol_name == Dhaka::END_SYMBOL_NAME
+        "Unexpected end of file."
+      else
+        "Unexpected token #{unexpected_token.symbol_name}:\n#{program.dup.insert(unexpected_token.input_position - 1, ERROR_MARKER)}"
+      end
+    end
+    
+    def tokenize_error_message(unexpected_char_index, program)
+      "Unexpected character #{program[unexpected_char_index - 1].chr}:\n#{program.dup.insert(unexpected_char_index - 1, ERROR_MARKER)}"
+    end
+    
+    def evaluation_error_message(evaluation_result, program)
+      "#{evaluation_result.exception}:\n#{program.dup.insert(evaluation_result.node.tokens[0].input_position, ERROR_MARKER)}"
+    end
+  
+  end
+  
+  # The SimpleDB query language grammar, as defined at
+  # http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/SDB_API_Query.html
+  class QueryGrammar < Dhaka::Grammar
+    for_symbol(Dhaka::START_SYMBOL_NAME) do
+      start                %w| predicates |
+    end
+  
+    for_symbol('predicates') do
+      single_predicate     %w| predicate |
+      not_predicate        %w| not predicate |
+      intersection         %w| predicate intersection predicates |
+      not_intersection     %w| not predicate intersection predicates |
+      union                %w| predicate union predicates |
+      not_union            %w| not predicate union predicates |
+    end
+    
+    for_symbol('predicate') do
+      attribute_comparison %w| [ attribute_comparison ] |
+    end
+    
+    for_symbol('attribute_comparison') do
+      single_comparison    %w| identifier comp_op constant |
+      not_comparison       %w| not identifier comp_op constant |
+      and_comparison       %w| identifier comp_op constant and attribute_comparison |
+      not_and_comparison   %w| not identifier comp_op constant and attribute_comparison |
+      or_comparison        %w| identifier comp_op constant or attribute_comparison |
+      not_or_comparison    %w| not identifier comp_op constant or attribute_comparison |
+    end
+    
+    for_symbol('comp_op') do
+      equal                %w| = |
+      greater_than         %w| > |
+      less_than            %w| < |
+      greater_or_equal     %w| >= |
+      less_or_equal        %w| <= |
+      not_equal            %w| != |
+      starts_with          %w| starts-with |
+    end
+    
+    for_symbol('identifier') do
+      identifier           %w| quoted_string |
+    end
+    
+    for_symbol('constant') do
+      constant            %w| quoted_string |
+    end
+  
+  end
+  
+  # The lexer for the query language.
+  class QueryLexerSpec < Dhaka::LexerSpecification
+    
+    %w| = > < >= <= != starts-with |.each do |op|
+      for_pattern(op) do
+        create_token(op)
+      end
+    end
+    
+    for_pattern('\[') do
+      create_token('[')
+    end
+    
+    for_pattern('\]') do
+      create_token(']')
+    end
+    
+    for_pattern('\s+') do
+      # ignore whitespace
+    end
+    
+    KEYWORDS = %w| not and or union intersection |
+    KEYWORDS.each do |keyword|
+      for_pattern(keyword) do
+        create_token(keyword)
+      end
+    end
+    
+    for_pattern("'(\\\\'|[^'])+'") do
+      create_token 'quoted_string'
+    end
+    
+  end
+  
+  # The query evaluator. This class acts on the parse tree and will return
+  # a list of item names that match the query from the evaluate method.
+  class QueryEvaluator < Dhaka::Evaluator
+    self.grammar = QueryGrammar
+    
+    def initialize(domain = nil)
+      @domain = domain
+      @predicate_identifier = nil
+      @all_items = nil
+    end
+    
+    define_evaluation_rules do
+     
+      for_single_predicate do
+        evaluate(child_nodes[0])
+      end
+      
+      for_not_predicate do
+        results = evaluate(child_nodes[1])
+        all_items.difference(results)
+      end
+      
+      for_intersection do
+        results = evaluate(child_nodes[0])
+        results.intersection(evaluate(child_nodes[2]))
+      end
+      
+      # TODO Nots are probably not handled correctly. Need to play with AWS to find out for sure.
+      for_not_intersection do
+        results = evaluate(child_nodes[1])
+        all_items.difference(results.intersection(evaluate(child_nodes[3])))
+      end
+      
+      for_union do
+        results = evaluate(child_nodes[0])
+        results.union(evaluate(child_nodes[2]))
+      end
+      
+      for_not_union do
+        results = evaluate(child_nodes[1])
+        all_items.difference(results.union(evaluate(child_nodes[3])))
+      end
+      
+      for_attribute_comparison do
+        evaluate(child_nodes[1])
+      end
+      
+      for_single_comparison do
+        do_comparison(evaluate(child_nodes[0]), evaluate(child_nodes[1]), evaluate(child_nodes[2]))
+      end
+      
+      for_not_comparison do
+        do_comparison(evaluate(child_nodes[1]), evaluate(child_nodes[2]), evaluate(child_nodes[3]), true)
+      end
+      
+      for_and_comparison do
+        results = do_comparison(evaluate(child_nodes[0]), evaluate(child_nodes[1]), evaluate(child_nodes[2]))
+        results.intersection(evaluate(child_nodes[4]))
+      end
+      
+      for_not_and_comparison do
+        results = do_comparison(evaluate(child_nodes[1]), evaluate(child_nodes[2]), evaluate(child_nodes[3]), true)
+        results.intersection(evaluate(child_nodes[5]))
+      end
+      
+      for_or_comparison do
+        results = do_comparison(evaluate(child_nodes[0]), evaluate(child_nodes[1]), evaluate(child_nodes[2]))
+        results.union(evaluate(child_nodes[4]))
+      end
+      
+      for_not_or_comparison do
+        results = do_comparison(evaluate(child_nodes[1]), evaluate(child_nodes[2]), evaluate(child_nodes[3]), true)
+        results.union(evaluate(child_nodes[5]))
+      end
+      
+      for_equal { lambda { |v1, v2| v1 == v2 } }
+      for_greater_than { lambda { |v1, v2| v1 > v2 } }
+      for_less_than { lambda { |v1, v2| v1 < v2 } }
+      for_greater_or_equal { lambda { |v1, v2| v1 >= v2 } }
+      for_less_or_equal { lambda { |v1, v2| v1 <= v2 } }
+      # TODO ['a1' != 'v2'] should return false if a1 has values v1 AND v2
+      for_not_equal { lambda { |v1, v2| v1 != v2 } }
+      for_starts_with { lambda { |v1, v2| v2[0...v1.length] == v1 } }
+      
+      for_identifier { val(child_nodes[0]) }
+      for_constant { val(child_nodes[0]) }
+    end
+  
+    def val(node)
+      node.token.value.to_s[1..-2]
+    end
+  
+    def all_items
+      unless @all_items
+        @all_items = @domain.items.collect { |k,v| k }.to_set
+      end
+      
+      return @all_items
+    end
+  
+    # Apply the given comparison params to every item in the domain
+    def do_comparison(identifier, op, constant, negate = false)  
+      results = Set.new
+      
+      if @domain
+        @domain.items.each_value do |item|
+          attrs = item.attributes[identifier]
+          
+          attrs.each do |attr|
+            match = op.call(constant, attr.value)
+            if (match && !negate) || (negate && !match)
+              results << item
+              break
+            end
+          end
+        end
+      end
+      
+      results
+    end
+  end
+end

data/test/vendor/simplerdb-0.2/lib/simplerdb/server.rb

@@ -0,0 +1,33 @@
+require 'rubygems'
+require 'simplerdb/servlet'
+require 'webrick'
+
+# Runs the SimplerDB server
+module SimplerDB
+  class Server
+    
+    # Create the server running on the given port
+    def initialize(port = nil)
+      @port = port || 8087
+    end
+    
+    # Run the server on the configured port.
+    def start
+      config = { :Port => @port, :AccessLog => [] }
+      @server = WEBrick::HTTPServer.new(config)
+      @server.mount("/", SimplerDB::RESTServlet)
+      
+      ['INT', 'TERM'].each do |signal| 
+        trap(signal) { @server.shutdown }
+      end
+      
+      @server.start
+    end
+  
+    # Shut down the server
+    def shutdown
+      @server.shutdown
+    end
+    
+  end
+end

data/test/vendor/simplerdb-0.2/lib/simplerdb/servlet.rb

@@ -0,0 +1,191 @@
+require 'rubygems'
+require 'webrick'
+require 'builder'
+require 'simplerdb/db'
+require 'simplerdb/client_exception'
+
+module SimplerDB
+  
+  # A WEBrick servlet to handle API requests over REST
+  class RESTServlet < WEBrick::HTTPServlet::AbstractServlet
+      
+    def do_GET(req, res)
+      action = req.query["Action"]
+      
+      begin
+        if action.nil?
+          # Raise an error
+          raise ClientException.new(:MissingAction, "No action was supplied with this request")
+        else
+          # Process the action appropriately
+          xml = send("do_#{action.downcase}", req)
+          res.body = xml
+          res.status = 200
+        end
+      rescue ClientException => e
+        res.body = error_xml(e.code, e.msg)
+        res.status = 400 # What is the right status?
+      end
+      
+      res['content-type'] = 'text/xml'
+    end
+
+    alias do_POST do_GET
+    alias do_DELETE do_GET
+    
+    # Handle CreateDomain requests
+    def do_createdomain(req)
+      name = req.query["DomainName"]
+      DB.instance.create_domain(name)
+      build_result("CreateDomain")
+    end
+    
+    # Handle ListDomains requests
+    def do_listdomains(req)
+      max = req.query["MaxNumberOfDomains"]
+      next_token = req.query["NextToken"]
+      
+      max = max.to_i if max
+      domains,token = DB.instance.list_domains(max, next_token)
+      
+      build_result("ListDomains") do |doc|
+        doc.ListDomainsResult do
+          if domains
+            domains.each do |domain|
+              doc.DomainName domain
+            end
+          end
+          
+          if token
+            doc.NextToken token
+          end
+        end
+      end
+    end
+    
+    
+    # DeleteDomains requests
+    def do_deletedomain(req)
+      name = req.query["DomainName"]
+      DB.instance.delete_domain(name)
+      build_result("DeleteDomain")  
+    end
+    
+    # PutAttributes requests
+    def do_putattributes(req)
+      domain_name = req.query["DomainName"]
+      item_name = req.query["ItemName"]
+      attrs = parse_attribute_args(req)
+      DB.instance.put_attributes(domain_name, item_name, attrs)
+      build_result("PutAttributes")
+    end
+    
+    # DeleteAttributes requests
+    def do_deleteattributes(req)
+      domain_name = req.query["DomainName"]
+      item_name = req.query["ItemName"]
+      attrs = parse_attribute_args(req)
+      DB.instance.delete_attributes(domain_name, item_name, attrs)
+      build_result("DeleteAttributes")
+    end
+    
+    # GetAttributes request
+    def do_getattributes(req)
+      domain_name = req.query["DomainName"]
+      item_name = req.query["ItemName"]
+      attr_name = req.query["AttributeName"]
+      attrs = DB.instance.get_attributes(domain_name, item_name)
+      
+      build_result("GetAttributes") do |doc|
+        doc.GetAttributesResult do
+          if attrs
+            attrs.each do |attr|
+              doc.Attribute do
+                doc.Name attr.name
+                doc.Value attr.value
+              end
+            end
+          end
+        end
+      end
+    end
+    
+    # Query request
+    def do_query(req)
+      domain_name = req.query["DomainName"]
+      max_items = req.query["MaxItems"].to_i
+      max_items = 100 if (max_items < 1 || max_items > 250)
+      next_token = req.query["NextToken"]
+      query = req.query["QueryExpression"]
+      
+      results,token = DB.instance.query(domain_name, query, max_items, next_token)
+      build_result("Query") do |doc|
+        doc.QueryResult do
+          if results
+            results.each do |res|
+              doc.ItemName res.name
+            end
+          end
+        
+          if token
+            doc.NextToken token
+          end
+        end
+      end
+    end
+    
+    # Build a result for the given action.
+    # The given block will add action-specific return fields.
+    def build_result(action)
+      xml = ''
+      doc = Builder::XmlMarkup.new(:target => xml)
+        doc.tag!("#{action}Response", :xmlns => "http://sdb.amazonaws.com/doc/2007-11-07") do
+          if block_given?
+            yield doc
+          end
+          
+          doc.ResponseMetadata do
+            doc.RequestId "1234"
+            doc.BoxUsage "0"
+          end
+        end
+        
+      xml
+    end
+    
+    # Return the error response for the given error code and message
+    def error_xml(code, msg)
+      xml = ''
+      doc = Builder::XmlMarkup.new(:target => xml)
+      
+      doc.Response do
+        doc.Errors do
+          doc.Error do
+            doc.Code code.to_s
+            doc.Message msg
+            doc.BoxUsage "0"
+          end
+        end
+        
+        doc.RequestID  "1234"
+      end
+      
+      xml
+    end 
+    
+    def parse_attribute_args(req)
+      args = []
+      for i in (0...100)
+        name = req.query["Attribute.#{i}.Name"]
+        value = req.query["Attribute.#{i}.Value"]
+        replace = (req.query["Attribute.#{i}.Replace"] == "true")
+        if name && value
+          args << AttributeParam.new(name, value, replace)
+        end
+      end
+      
+      args
+    end
+  end
+  
+end

data/test/vendor/simplerdb-0.2/lib/simplerdb.rb

@@ -0,0 +1,3 @@
+class SimplerDB
+  VERSION = '0.2'
+end

data/test/vendor/simplerdb-0.2/test/functional_test.rb

@@ -0,0 +1,81 @@
+require "test/unit"
+require 'rubygems'
+require 'right_aws'
+require 'simplerdb/server'
+
+# Test the live, running service over HTTP
+class FunctionalTest < Test::Unit::TestCase
+  
+  def setup
+    @server = SimplerDB::Server.new(8087)
+    @thread = Thread.new { @server.start }
+    @sdb = RightAws::SdbInterface.new('access','secret',
+                                     {:server => 'localhost', :port => 8087, :protocol => 'http'})
+
+  end
+  
+  def teardown
+    @server.shutdown
+    @thread.join
+  end
+  
+  def test_domains
+    # Create/list domains
+    @sdb.create_domain("d1")
+    @sdb.create_domain("d2")
+    result = @sdb.list_domains
+    assert result[:domains].index("d1")
+    assert result[:domains].index("d2")
+    
+    # List domains with paging
+    domains = []
+    count = 0
+    @sdb.list_domains(1) do |result|
+      domains += result[:domains]
+      count += 1
+      true
+    end
+    
+    assert count == 2
+    assert domains.index("d1")
+    assert domains.index("d2")
+  end
+  
+  def test_items
+    # Create some bands with albums + genre
+    @sdb.create_domain("bands")
+    
+    attrs = {:albums => ['Being There', 'Summer Teeth'], :genre => "rock"}
+    @sdb.put_attributes("bands", "Wilco", attrs)
+    
+    attrs = {:albums => ['OK Computer', 'Kid A'], :genre => "alternative"}
+    @sdb.put_attributes("bands", "Radiohead", attrs)
+    
+    attrs = {:albums => ['The Soft Bulletin'], :genre => "alternative"}
+    @sdb.put_attributes("bands", "The Flaming Lips", attrs)
+    
+    # Read the attributes back
+    results = @sdb.get_attributes("bands", "Wilco")
+    assert results[:attributes]["albums"].sort == ['Being There', 'Summer Teeth']
+    assert results[:attributes]["genre"] == ["rock"]
+    
+    # Query the bands domain
+    results = @sdb.query("bands", "['genre' = 'alternative']")
+    assert results[:items].sort == ["Radiohead", "The Flaming Lips"]
+    
+    results = @sdb.query("bands", "['albums' = 'Being There']")
+    assert results[:items] == ["Wilco"]
+    
+    results = @sdb.query("bands", "['albums' starts-with 'OK' or 'albums' = 'The Soft Bulletin']")
+    assert results[:items].sort == ["Radiohead", "The Flaming Lips"]
+  
+    # Modify and re-read attributes
+    @sdb.put_attributes("bands", "The Flaming Lips", {:albums => "Yoshimi Battles the Pink Robots"})
+    @sdb.put_attributes("bands", "The Flaming Lips", {:genre => "rock"}, :replace)
+    
+    results = @sdb.get_attributes("bands", "The Flaming Lips")
+    assert results[:attributes]["albums"].sort == ["The Soft Bulletin", "Yoshimi Battles the Pink Robots"]
+    assert results[:attributes]["genre"] == ["rock"]
+  end
+  
+end

data/test/vendor/simplerdb-0.2/test/query_evaluator_test.rb

@@ -0,0 +1,73 @@
+require "test/unit"
+require 'simplerdb/query_language'
+require 'simplerdb/db'
+
+# Tests the query evaluator
+class QueryEvaluatorTest < Test::Unit::TestCase
+  include SimplerDB
+  
+  def setup
+    @db = DB.instance
+    @db.create_domain("test")
+  end
+  
+  def teardown
+    @db.reset
+  end
+  
+  def test_single_predicate
+    bulk_insert('i1' => [["a1", "v1"], ["a2", "v2"], ["a3", "bcd"]],
+                'i2' => [["a1", "vv1"]])
+    
+    assert_items 'i1', @db.query("test", "['a1' = 'v1']")
+    assert_items 'i1', @db.query("test", "['a1' != 'v2']")
+    assert_items 'i1', @db.query("test", "['a1' >= 'v1']")
+    assert_items 'i1', @db.query("test", "['a1' <= 'v1']")
+    assert_items 'i1', @db.query("test", "['a3' starts-with 'bc']")
+    assert_items 'i1', @db.query("test", "['a3' > 'cde']")
+    assert_items 'i1', @db.query("test", "['a3' < 'abc']")
+    assert_items ['i1', 'i2'], @db.query("test", "['a1' = 'v1' or 'a1' = 'vv1']")
+    assert_items ['i1', 'i2'], @db.query("test", "['a1' <= 'v1']")
+    
+    assert_empty @db.query("test", "['a2' != 'v2']")
+    assert_empty @db.query("test", "['a2' > 'v2']")
+    assert_empty @db.query("test", "['a2' = 'v1']")
+    assert_empty @db.query("testxxxxx", "['a2' = 'v2']")
+    assert_empty @db.query("test", "['a2' = 'v1']")
+  end
+  
+  def test_ops
+    
+  end
+  
+  def bulk_insert(items)
+    items.each do |name, attrs|
+      params = []
+      for attr in attrs
+        params << AttributeParam.new(attr[0], attr[1])
+      end
+      
+      @db.put_attributes("test", name, params)
+    end
+  end
+  
+  def assert_empty(results)
+    assert results[0].empty?
+  end
+  
+  def assert_items(items, results)
+    results = results[0]
+    for item in items
+      found = false
+      for result in results
+        if result.name == item
+          found = true
+          break
+        end
+      end
+      
+      assert found, "Missing item #{item}"
+    end
+  end
+  
+end