rethinkdb 1.2.0.1

data/lib/base_classes.rb

@@ -0,0 +1,39 @@
+# Copyright 2010-2012 RethinkDB, all rights reserved.
+#TODO: toplevel doc
+module RethinkDB
+  class RQL_Query; end
+  class Read_Query < RQL_Query # :nodoc:
+  end
+  class Write_Query < RQL_Query # :nodoc:
+  end
+  class Meta_Query < RQL_Query # :nodoc:
+  end
+
+  module Sequence; end
+  class JSON_Expression < Read_Query; include Sequence; end
+  class Single_Row_Selection < JSON_Expression; end
+  class Stream_Expression < Read_Query; include Sequence; end
+  class Multi_Row_Selection < Stream_Expression; end
+
+  class Database; end
+  class Table; end
+
+  class Connection; end
+  class Query_Results; include Enumerable; end
+
+  module RQL; end
+
+  # Shortcuts to easily build RQL queries.
+  module Shortcuts
+    # The only shortcut right now.  May be used as a wrapper to create
+    # RQL values or as a shortcut to access function in the RQL
+    # namespace.  For example, the following are equivalent:
+    #   r.add(1, 2)
+    #   RethinkDB::RQL.add(1, 2)
+    #   r(1) + r(2)
+    #   r(3)
+    def r(*args)
+      (args == [] ) ? RethinkDB::RQL : RQL.expr(*args)
+    end
+  end
+end

data/lib/bt.rb

@@ -0,0 +1,148 @@
+# Copyright 2010-2012 RethinkDB, all rights reserved.
+module RethinkDB
+  module BT_Mixin
+    attr_accessor :highlight, :line
+
+    def sanitize_context(context)
+      if __FILE__ =~ /^(.*\/)[^\/]+.rb$/
+        prefix = $1;
+        context.reject{|x| x =~ /^#{prefix}/}
+      else
+        context
+      end
+    end
+
+    def format_highlights(str)
+      str.chars.map{|x| x == "\000" ? "^" : " "}.join.rstrip
+    end
+
+    def force_raise(query, debug=false)
+      obj = query.run
+      #maybe_reformat_err(query, obj);
+      PP.pp obj if debug
+      if obj.class == Query_Results
+        obj = obj.to_a
+      elsif obj.class == Hash
+        raise RuntimeError,obj['first_error'] if obj['first_error']
+      end
+      obj
+    end
+
+    def maybe_reformat_err(query, obj)
+      if obj.class == Hash && (err = obj['first_error'])
+        obj['first_error'] = format_err(query, err)
+      end
+    end
+
+    def format_err(query, err)
+      parts = err.split("\nBacktrace:\n")
+      errline = parts[0] || ""
+      bt = (parts[1] && parts[1].split("\n")) || []
+      errline+"\n"+query.print_backtrace(bt)
+    end
+
+    def set(sym,val)
+      @map = {} if not @map
+      res, @map[sym] = @map[sym], val
+      return res
+    end
+    def consume sym
+      @map = {} if not @map
+      res, @map[sym] = @map[sym], nil
+      return res.nil? ? 0 : res
+    end
+    def expand arg
+      case arg
+      when 'attr'           then []
+      when 'attrname'       then []
+      when 'base'           then [1+consume(:reduce)]
+      when 'body'           then [4+consume(:reduce)]
+      when 'datacenter'     then []
+      when 'db_name'        then []
+      when 'expr'           then [2]
+      when 'false'          then [3]
+      when 'group_mapping'  then [1, 1, 1]
+      when 'key'            then [3]
+      when 'keyname'        then []
+      when 'lowerbound'     then [1, 2]
+      when 'mapping'        then [1, 2]
+      when 'modify_map'     then [2, 1]
+      when 'order_by'       then []
+      when 'point_map'      then [4, 1]
+      when 'predicate'      then [1, 2]
+      when 'reduce'         then [1]
+      when 'reduction'      then set(:reduce, -1); [1, 3]
+      when 'stream'         then [1]
+      when 'table_name'     then []
+      when 'table_ref'      then []
+      when 'test'           then [1]
+      when 'true'           then [2]
+      when 'upperbound'     then [1, 3]
+      when 'value_mapping'  then [1, 2, 1]
+      when 'view'           then [1]
+      when /arg:([0-9]+)/   then [2, $1.to_i]
+      when /attrs:([0-9]+)/ then []
+      when /elem:([0-9]+)/  then [$1.to_i+1]
+      when /bind:(.+)$/     then [1, $1]
+      when /key:(.+)$/      then [1..-1, $1]
+      when /query:([0-9]+)/ then [3, $1.to_i]
+      when /term:([0-9]+)/  then [2, $1.to_i]
+      else  [:error]
+      end
+    end
+
+    def recur(arr, lst, val)
+      if lst[0].class == String
+        entry = arr[0..-1].find{|x| x[0].to_s == lst[0]}
+        raise RuntimeError if not entry
+        mark(entry[1], lst[1..-1], val)
+      elsif lst[0].class == Fixnum || lst[0].class == Range
+        mark(arr[lst[0]], lst[1..-1], val) if lst != []
+      else
+        raise RuntimeError
+      end
+    end
+    def mark(query, lst, val)
+      #PP.pp [lst, query.class, query]
+      if query.class == Array
+        recur(query, lst, val) if lst != []
+      elsif query.kind_of? RQL_Query
+        if lst == []
+          @line = sanitize_context(query.context)[0]
+          val,query.marked = query.marked, val
+          val
+        else
+          recur(query.body, lst, val)
+        end
+      else raise RuntimeError
+      end
+    end
+
+    def with_marked_error(query, bt)
+      @map = {}
+      bt = bt.map{|x| expand x}.flatten
+      raise RuntimeError if bt.any?{|x| x == :error}
+      old = mark(query, bt, :error)
+      res = yield
+      mark(query, bt, old)
+      return res
+    end
+
+    def with_highlight
+      @highlight = true
+      str = yield
+      @highlight = false
+      str.chars.map{|x| x == "\000" ? "^" : " "}.join.rstrip
+    end
+
+    def alt_inspect(query, &block)
+      class << query
+        attr_accessor :str_block
+        def inspect; real_inspect({:str => @str_block.call(self)}); end
+      end
+      query.str_block = block
+      query
+    end
+  end
+  module BT; extend BT_Mixin; end
+end

data/lib/data_collectors.rb

@@ -0,0 +1,35 @@
+# Copyright 2010-2012 RethinkDB, all rights reserved.
+module RethinkDB
+  # Data collectors are ways of aggregating data (for use with
+  # Sequence#groupby).  You can define your own; all they are is a hash
+  # containing a <b>+mapping+</b> to be applied to row, a
+  # <b>+base+</b>/<b>+reduction+</b> to reduce over the mappings, and an
+  # optional <b>+finalizer+</b> to call on the output of the reduction.  You can
+  # expand the builtin data collectors below for examples.
+  #
+  # All of the builtin collectors can also be accessed using the <b>+r+</b>
+  # shortcut (like <b>+r.sum+</b>).
+  module Data_Collectors
+    # Count the number of rows in each group.
+    def self.count
+      { :mapping => lambda{|row| 1},
+        :base => 0,
+        :reduction => lambda{|acc, val| acc + val} }
+    end
+
+    # Sum a particular attribute of the rows in each group.
+    def self.sum(attr)
+      { :mapping => lambda{|row| row[attr]},
+        :base => 0,
+        :reduction => lambda{|acc, val| acc + val} }
+    end
+
+    # Average a particular attribute of the rows in each group.
+    def self.avg(attr)
+      { :mapping => lambda{|row| [row[attr], 1]},
+        :base => [0, 0],
+        :reduction => lambda{|acc, val| [acc[0] + val[0], acc[1] + val[1]]},
+        :finalizer => lambda{|res| res[0] / res[1]} }
+    end
+  end
+end

data/lib/jsons.rb

@@ -0,0 +1,136 @@
+# Copyright 2010-2012 RethinkDB, all rights reserved.
+module RethinkDB
+  # A query returning a JSON expression.  Most of the functions that you
+  # can run on a JSON object are found in RethinkDB::RQL and accessed
+  # with the <b>+r+</b> shortcut.  For convenience, may of the
+  # functions in Rethinkdb::RQL can be run as if they were instance
+  # methods of JSON_Expression.  For example, the following are
+  # equivalent:
+  #   r.add(1, 2)
+  #   r(1).add(2)
+  #   r(3)
+  #
+  # Running a JSON_Expression query will return a literal Ruby
+  # representation of the resulting JSON, rather than a stream or a
+  # string.  For example, the following are equivalent:
+  #   r.add(1,2).run
+  #   3
+  # As are:
+  #   r({:a => 1, :b => 2}).pick(:a).run
+  #   {'a' => 1}
+  # (Note that the symbol keys were coerced into string keys in the
+  # object.  JSON doesn't distinguish between symbols and strings.)
+  class JSON_Expression
+    # If <b>+ind+</b> is a symbol or a string, gets the corresponding
+    # attribute of an object.  For example, the following are equivalent:
+    #   r({:id => 1})[:id]
+    #   r({:id => 1})['id']
+    #   r(1)
+    # Otherwise, if <b>+ind+</b> is a number or a range, invokes RethinkDB::Sequence#[]
+    def [](ind)
+      if ind.class == Symbol || ind.class == String
+        BT.alt_inspect(JSON_Expression.new [:call, [:getattr, ind], [self]]) {
+          "#{self.inspect}[#{ind.inspect}]"
+        }
+      else
+        super
+      end
+    end
+
+    # Append a single element to an array.  The following are equivalent:
+    #   r([1,2,3,4])
+    #   r([1,2,3]).append(4)
+    def append(el)
+      JSON_Expression.new [:call, [:arrayappend], [self, S.r(el)]]
+    end
+
+    # Get an attribute of a JSON object (the same as
+    # JSON_Expression#[]).  The following are equivalent.
+    #   r({:id => 1}).getattr(:id)
+    #   r({:id => 1})[:id]
+    #   r(1)
+    def getattr(attrname)
+      JSON_Expression.new [:call, [:getattr, attrname], [self]]
+    end
+
+    # Check whether a JSON object has a particular attribute.  The
+    # following are equivalent:
+    #   r({:id => 1}).contains(:id)
+    #   r(true)
+    def contains(attrname)
+      JSON_Expression.new [:call, [:contains, attrname], [self]]
+    end
+
+    # Construct a JSON object that has a subset of the attributes of
+    # another JSON object by specifying which to keep.  The following are equivalent:
+    #   r({:a => 1, :b => 2, :c => 3}).pick(:a, :c)
+    #   r({:a => 1, :c => 3})
+    def pick(*attrnames)
+      JSON_Expression.new [:call, [:pick, *attrnames], [self]]
+    end
+
+    # Construct a JSON object that has a subset of the attributes of
+    # another JSON object by specifying which to drop.  The following are equivalent:
+    #   r({:a => 1, :b => 2, :c => 3}).without(:a, :c)
+    #   r({:b => 2})
+    def unpick(*attrnames)
+      JSON_Expression.new [:call, [:unpick, *attrnames], [self]]
+    end
+
+    # Convert from an array to a stream.  While most sequence functions are polymorphic
+    # and handle both arrays and streams, when arrays or streams need to be
+    # combined (e.g. via <b>+union+</b>) you need to explicitly convert between
+    # the types.  This is mostly for safety, but also because which type you're
+    # working with affects error handling.  The following are equivalent:
+    #   r([1,2,3]).arraytostream
+    #   r([1,2,3]).to_stream
+    def array_to_stream(); Stream_Expression.new [:call, [:array_to_stream], [self]]; end
+
+    # Prefix numeric -.  The following are equivalent:
+    #   -r(1)
+    #   r(-1)
+    def -@; JSON_Expression.new [:call, [:subtract], [self]]; end
+    # Prefix numeric +.  The following are equivalent:
+    #   +r(-1)
+    #   r(-1)
+    def +@; JSON_Expression.new [:call, [:add], [self]]; end
+  end
+
+  # A query representing a variable.  Produced by e.g. RQL::letvar.  This
+  # is its own class because it needs to behave correctly when spliced
+  # into a string (see RQL::js).
+  class Var_Expression < JSON_Expression
+    # Convert from an RQL query representing a variable to the name of that
+    # variable.  Used e.g. in constructing javascript functions.
+    def to_s
+      if @body.class == Array and @body[0] == :var
+      then @body[1]
+      else raise TypeError, 'Can only call to_s on RQL_Queries representing variables.'
+      end
+    end
+  end
+
+  # Like Multi_Row_Selection, but for a single row.  E.g.:
+  #   table.get(0)
+  # yields a Single_Row_Selection
+  class Single_Row_Selection < JSON_Expression
+    # Analagous to Multi_Row_Selection#update
+    def update(variant=nil)
+      S.with_var {|vname,v|
+        Write_Query.new [:pointupdate, *(@body[1..-1] + [[vname, S.r(yield(v))]])]
+      }.apply_variant(variant)
+    end
+
+    # Analagous to Multi_Row_Selection#mutate
+    def replace(variant=nil)
+      S.with_var {|vname,v|
+        Write_Query.new [:pointreplace, *(@body[1..-1] + [[vname, S.r(yield(v))]])]
+      }.apply_variant(variant)
+    end
+
+    # Analagous to Multi_Row_Selection#delete
+    def delete
+      Write_Query.new [:pointdelete, *(@body[1..-1])]
+    end
+  end
+end

data/lib/net.rb

@@ -0,0 +1,275 @@
+# Copyright 2010-2012 RethinkDB, all rights reserved.
+require 'socket'
+require 'thread'
+require 'json'
+
+module RethinkDB
+  module Faux_Abort # :nodoc:
+    class Abort # :nodoc:
+    end
+  end
+  # The result of a calling Connection#run on a query that returns a stream.
+  # This class is <b>+Enumerable+</b>.  You can find documentation on Enumerable
+  # classes at http://ruby-doc.org/core-1.9.3/Enumerable.html .
+  #
+  # <b>NOTE:</b> unlike most enumerable objects, you can only iterate over Query
+  # results once.  The results are fetched lazily from the server in chunks
+  # of 1000.  If you need to access values multiple times, use the <b>+to_a+</b>
+  # instance method to get an Array, and then work with that.
+  class Query_Results
+    def out_of_date # :nodoc:
+      @conn.conn_id != @conn_id
+    end
+
+    def inspect # :nodoc:
+      state = @run ? "(exhausted)" : "(enumerable)"
+      extra = out_of_date ? " (Connection #{@conn.inspect} reset!)" : ""
+      "#<RethinkDB::Query_Results:#{self.object_id} #{state}#{extra}: #{@query.inspect}>"
+    end
+
+    def initialize(query, connection, token) # :nodoc:
+      @query = query
+      @run = false
+      @conn_id = connection.conn_id
+      @conn = connection
+      @token = token
+    end
+
+    def each (&block) # :nodoc:
+      raise RuntimeError, "Can only iterate over Query_Results once!" if @run
+      raise RuntimeError, "Connection has been reset!" if out_of_date
+      @conn.token_iter(@query, @token, &block)
+      @run = true
+      return self
+    end
+  end
+
+  # TODO: Make sure tokens don't overflow.
+
+  # A connection to the RethinkDB
+  # cluster.  You need to create at least one connection before you can run
+  # queries.  After creating a connection <b>+conn+</b> and constructing a
+  # query <b>+q+</b>, the following are equivalent:
+  #  conn.run(q)
+  #  q.run
+  # (This is because by default, invoking the <b>+run+</b> instance method on a
+  # query runs it on the most-recently-opened connection.)
+  #
+  # ==== Attributes
+  # * +default_db+ - The current default database (can be set with Connection#use).  Defaults to 'test'.
+  # * +conn_id+ - You probably don't care about this.  Used in some advanced situations where you care about whether a connection object has reconnected.
+  class Connection
+    def inspect # :nodoc:
+      properties = "(#{@host}:#{@port}) (Default Database: '#{@default_db}')"
+      state = @listener ? "(listening)" : "(closed)"
+      "#<RethinkDB::Connection:#{self.object_id} #{properties} #{state}>"
+    end
+
+    @@last = nil
+    @@magic_number = 0xaf61ba35
+
+    def debug_socket # :nodoc:
+      @socket;
+    end
+
+    # Reconnect to the server.  This will interrupt all queries on the
+    # server and invalidate all outstanding enumerables on the client.
+    def reconnect
+      @socket.close if @socket
+      @socket = TCPSocket.open(@host, @port)
+      @waiters = {}
+      @data = {}
+      @mutex = Mutex.new
+      @conn_id += 1
+      start_listener
+      self
+    end
+
+    def dispatch msg # :nodoc:
+      PP.pp msg if $DEBUG
+      payload = msg.serialize_to_string
+      #File.open('sexp_payloads.txt', 'a') {|f| f.write(payload.inspect+"\n")}
+      packet = [payload.length].pack('L<') + payload
+      @socket.send(packet, 0)
+      return msg.token
+    end
+
+    def wait token # :nodoc:
+      res = nil
+      raise RuntimeError, "Connection closed by server!" if not @listener
+      @mutex.synchronize do
+        (@waiters[token] = ConditionVariable.new).wait(@mutex) if not @data[token]
+        res = @data.delete token if @data[token]
+      end
+      raise RuntimeError, "Connection closed by server!" if !@listener or !res
+      return res
+    end
+
+    def continue token # :nodoc:
+      msg = Query.new
+      msg.type = Query::QueryType::CONTINUE
+      msg.token = token
+      dispatch msg
+    end
+
+    def error(query,protob,err=RuntimeError) # :nodoc:
+      bt = protob.backtrace ? protob.backtrace.frame : []
+      #PP.pp bt
+      msg = "RQL: #{protob.error_message}\n" + query.print_backtrace(bt)
+      raise err,msg
+    end
+
+    def token_iter(query, token) # :nodoc:
+      done = false
+      data = []
+      loop do
+        if (data == [])
+          break if done
+
+          begin
+            protob = wait token
+          rescue @abort_module::Abort => e
+            print "\nAborting query and reconnecting...\n"
+            self.reconnect()
+            raise e
+          end
+
+          case protob.status_code
+          when Response::StatusCode::SUCCESS_JSON then
+            yield JSON.parse('['+protob.response[0]+']')[0]
+            return false
+          when Response::StatusCode::SUCCESS_PARTIAL then
+            continue token
+            data.push *protob.response
+          when Response::StatusCode::SUCCESS_STREAM then
+            data.push *protob.response
+            done = true
+          when Response::StatusCode::SUCCESS_EMPTY then
+            return false
+          when Response::StatusCode::BAD_QUERY then error query,protob,ArgumentError
+          when Response::StatusCode::RUNTIME_ERROR then error query,protob,RuntimeError
+          else error query,protob
+          end
+        end
+        #yield JSON.parse("["+data.shift+"]")[0] if data != []
+        yield JSON.parse('['+data.shift+']')[0] if data != []
+        #yield data.shift if data != []
+      end
+      return true
+    end
+
+    # Create a new connection to <b>+host+</b> on port <b>+port+</b>.
+    # You may also optionally provide a default database to use when
+    # running queries over that connection.  Example:
+    #   c = Connection.new('localhost', 28015, 'default_db')
+    def initialize(host='localhost', port=28015, default_db='test')
+      begin
+        @abort_module = ::IRB
+      rescue NameError => e
+        @abort_module = Faux_Abort
+      end
+      @@last = self
+      @host = host
+      @port = port
+      @default_db = default_db
+      @conn_id = 0
+      reconnect
+    end
+    attr_reader :default_db, :conn_id
+
+    # Change the default database of a connection.
+    def use(new_default_db)
+      @default_db = new_default_db
+    end
+
+    # Run a query over the connection.  If you run a query that returns a JSON
+    # expression (e.g. a reduce), you get back that JSON expression.  If you run
+    # a query that returns a stream of results (e.g. filtering a table), you get
+    # back an enumerable object of class RethinkDB::Query_Results.
+    #
+    # You may also provide some options as an optional second
+    # argument.  The only useful option right now is
+    # <b>+:use_outdated+</b>, which specifies whether tables can use
+    # outdated information.  (If you specified this on a per-table
+    # basis in your query, specifying it again here won't override the
+    # original choice.)
+    #
+    # <b>NOTE:</b> unlike most enumerable objects, you can only iterate over the
+    # result once.  See RethinkDB::Query_Results for more details.
+    def run (query, opts={:use_outdated => false})
+      #File.open('sexp.txt', 'a') {|f| f.write(query.sexp.inspect+"\n")}
+      is_atomic = (query.kind_of?(JSON_Expression) ||
+                   query.kind_of?(Meta_Query) ||
+                   query.kind_of?(Write_Query))
+      map = {}
+      map[:default_db] = @default_db if @default_db
+      S.check_opts(opts, [:use_outdated])
+      map[S.conn_outdated] = !!opts[:use_outdated]
+      protob = query.query(map)
+      if is_atomic
+        a = []
+        singular = token_iter(query, dispatch(protob)){|row| a.push row}
+        a.each{|o| BT.maybe_reformat_err(query, o)}
+        singular ? a : a[0]
+      else
+        return Query_Results.new(query, self, dispatch(protob))
+      end
+    end
+
+    # Close the connection.
+    def close
+      @listener.terminate! if @listener
+      @listener = nil
+      @socket.close
+      @socket = nil
+    end
+
+    # Return the last opened connection, or throw if there is no such
+    # connection.  Used by e.g. RQL_Query#run.
+    def self.last_connection
+      return @@last if @@last
+      raise RuntimeError, "No last connection.  Use RethinkDB::Connection.new."
+    end
+
+    def start_listener # :nodoc:
+      class << @socket
+        def read_exn(len) # :nodoc:
+          buf = read len
+          raise RuntimeError,"Connection closed by server." if !buf or buf.length != len
+          return buf
+        end
+      end
+      @socket.send([@@magic_number].pack('L<'), 0)
+      @listener.terminate! if @listener
+      @listener = Thread.new do
+        loop do
+          begin
+            response_length = @socket.read_exn(4).unpack('L<')[0]
+            response = @socket.read_exn(response_length)
+          rescue RuntimeError => e
+            @mutex.synchronize do
+              @listener = nil
+              @waiters.each {|kv| kv[1].signal}
+            end
+            Thread.current.terminate!
+            abort("unreachable")
+          end
+          #TODO: Recovery
+          begin
+            protob = Response.new.parse_from_string(response)
+          rescue
+            p response
+            abort("Bad Protobuf.")
+          end
+          @mutex.synchronize do
+            @data[protob.token] = protob
+            if (@waiters[protob.token])
+              cond = @waiters.delete protob.token
+              cond.signal
+            end
+          end
+        end
+      end
+    end
+  end
+end