haplocheirus-client 0.6.5

data/LICENSE

@@ -0,0 +1,13 @@
+Haplocheirus Client is Copyright (C) 2010 Twitter, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not
+use this work except in compliance with the License. You may obtain a copy of
+the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations under
+the License.

data/README.md

@@ -0,0 +1,70 @@
+Haplocheirus Client
+-------------------
+
+Haplocheirus is a highly available, partitioned storage service for
+vectors of heterogenous blobs. This is a Ruby client library for that service.
+
+
+INSTALLATION
+------------
+
+    gem install haplocheirus-client
+
+
+USAGE
+-----
+
+Attach a new client to a Haplo service:
+
+    >> client = Haplocheirus.new # Defaults to localhost:7666
+
+Store a vector under the id of '0':
+
+    >> client.store '0', ['foo', 'bar']
+
+Find the first 2 entries, starting at index 0:
+
+    >> client.get '0', 0, 2
+    ['bar', 'foo'] # note the reverse order
+
+Append an entry:
+
+    >> client.append '0', 'baz'
+    >> client.get '0', 0, 3
+    ['baz', 'bar', 'foo']
+
+Merge that vector with another:
+
+    >> client.merge '0', ['bat', 'quux']
+    >> client.get '0', 0, 5
+    ['quux', 'baz', 'bat', 'bar', 'foo']
+
+
+Remove an entry:
+
+    >> client.remove 'bat', '0'
+    >> client.get '0', 0, 4
+    ['quux', 'baz', 'bar', 'foo']
+
+Remove a set of entries:
+
+    >> client.unmerge '0' ['foo', 'bar']
+    >> client.get '0', 0, 2
+    ['quux', 'baz']
+
+Delete the vector:
+    >> client.delete '0'
+
+
+CONTRIBUTORS
+------------
+
+Brandon Mitchell
+
+
+LICENSE
+-------
+
+Copyright (C) 2010 Twitter, Inc.
+
+This work is licensed under the Apache License, Version 2.0. See LICENSE for details.

data/lib/haplocheirus.rb

@@ -0,0 +1,40 @@
+require 'thrift'
+require 'thrift_client'
+
+require 'haplocheirus/thrift/timeline_store'
+require 'haplocheirus/service'
+require 'haplocheirus/client'
+
+module Haplocheirus
+  autoload :MockService, 'haplocheirus/mock_service'
+
+  # Convenience method for:
+  #
+  #    s = Haplocheirus::Service.new(*args)
+  #    Haplocheirus::Client.new(s)
+  #
+  def self.new(*args)
+    service = Haplocheirus::Service.new(*args)
+    Haplocheirus::Client.new(service)
+  rescue Haplocheirus::Service::ServiceDisabled
+    nil
+  end
+
+  # Nice-to-haves...
+  class TimelineSegment #:nodoc:
+
+    def hit?
+      state == TimelineSegmentState::HIT
+    end
+
+    def miss?
+      state == TimelineSegmentState::MISS
+    end
+
+    def timeout?
+      state == TimelineSegmentState::TIMEOUT
+    end
+
+  end
+
+end

data/lib/haplocheirus/client.rb

@@ -0,0 +1,177 @@
+class Haplocheirus::Client
+
+  attr_accessor :service
+
+  # ==== Parameters
+  # service<ThriftClient>
+  #
+  def initialize(service)
+    @service = service
+  end
+
+  # Appends an entry to a set of timelines given by
+  # timeline_ids. Appends will do nothing if the timeline has not been
+  # created using #store.
+  #
+  # ==== Parameters
+  # entry
+  # prefix<String>:: Prefix to prepend to each id
+  # timeline_ids<Array[Integer], Integer>
+  #
+  def append(entry, prefix, *timeline_ids)
+    @service.append entry, prefix, timeline_ids.flatten
+  end
+
+  # Removes an entry from a set of timlines given by timeline_ids
+  #
+  # ==== Paramaters
+  # entry
+  # prefix<String>:: Prefix to prepend to each id
+  # timeline_ids<Array[Integer]>
+  #
+  def remove(entry, prefix, timeline_ids)
+    @service.remove entry, prefix, timeline_ids
+  end
+
+  # Gets entries on the timeline given by timeline_id, optionally
+  # beginning at offset and limited by length. Timelines are stored in
+  # recency order - an offset of 0 is the latest entry. Returns nil if
+  # the timeline_id does not exist.
+  #
+  # ==== Parameters
+  # timeline_id<String>
+  # offset<Integer>
+  # length<Integer>
+  # dedupe<Boolean>:: Optional. Defaults to false.
+  #
+  # ==== Returns
+  # TimelineSegment
+  #
+  # NOTE: The #size of the returned segment is computed *before* dupes
+  # are removed.
+  #
+  def get(timeline_id, offset, length, dedupe = false)
+    @service.get timeline_id, offset, length, dedupe
+  rescue Haplocheirus::TimelineStoreException
+    nil
+  end
+
+  # ==== Parameters
+  # timeline_queries<Array[TimelineQuery]>:: list of query structs
+  #
+  # ==== Returns
+  # Array[TimelineSegment]
+  #
+  # NOTE: Because there is no identifying information in the returned
+  # TimelineSegments, there is a strict ordering relationship between
+  # the query and returned segment lists.
+  #
+  def get_multi(timeline_queries)
+    @service.get_multi timeline_queries
+  end
+  
+  # Gets a range of entries from the timeline given by timeline_id
+  # since from_id (exclusive). This may include entries that were inserted out
+  # of order. from_id and to_id are treated as a 8 byte prefixes. If
+  # to_id is <= 0, results are not bounded by a maximum value. Returns
+  # nil if the timeline_id does not exist.
+  #
+  # ==== Parameters
+  # timeline_id<String>
+  # from_id<Integer>
+  # to_id<Integer>:: Optional. Defaults to 0.
+  # dedupe<Integer>:: Optional. Defaults to false.
+  #
+  # ==== Returns
+  # TimelineSegment
+  #
+  # NOTE: The #size of the returned segment is computed *before* dupes
+  # are removed.
+  #
+  def range(timeline_id, from_id, to_id = 0, dedupe = false)
+    @service.get_range timeline_id, from_id, to_id, dedupe
+  rescue Haplocheirus::TimelineStoreException
+    nil
+  end
+
+  # Atomically stores a set of entries into a timeline given by
+  # timeline_id. The entries are stored in the order provided.
+  #
+  # ==== Parameters
+  # timeline_id<String>
+  # entries<Array>
+  #
+  def store(timeline_id, entries)
+    @service.store timeline_id, entries
+  end
+
+  # Returns the intersection of entries with the current contents of
+  # the timeline given by timeline_id. Returns nil if the
+  # timeline_id does not exist.
+  #
+  # ==== Parameters
+  # timeline_id<String>
+  # entries<Array>
+  #
+  def filter(timeline_id, *entries)
+    # FIXME: Expose max search depth
+    @service.filter timeline_id, entries.flatten, -1
+  rescue Haplocheirus::TimelineStoreException
+    []
+  end
+
+  # Merges the entries into the timeline given by timeline_id. Merges
+  # will do nothing if the timeline hasn't been created using
+  # #store. Entries should be byte arrays of at least 8B per entry.
+  #
+  # ==== Parameters
+  # timeline_id<String>
+  # entries<Array>
+  #
+  def merge(timeline_id, entries)
+    @service.merge timeline_id, entries
+  end
+
+  # Merges entries in the timeline given by source_id into the
+  # timeline given by dest_id. Does nothing if source_id does not exist.
+  #
+  # ==== Parameters
+  # dest_id<String>
+  # source_id<String>
+  #
+  def merge_indirect(dest_id, source_id)
+    @service.merge_indirect dest_id, source_id
+  end
+
+  # Remove a list of entries from a timeline. Unmerges will do nothing
+  # if the timeline hasn't been created using #store. Entries should
+  # be byte arrays of at least 8B per entry.
+  #
+  # ==== Parameters
+  # timeline_id<String>
+  # entries<Array>
+  #
+  def unmerge(timeline_id, entries)
+    @service.unmerge timeline_id, entries
+  end
+
+  # Removes entries in the timeline given by source_id from the
+  # timeline given by dest_id. Does nothing if source_id does not exist.
+  #
+  # ==== Parameters
+  # dest_id<String>
+  # source_id<String>
+  #
+  def unmerge_indirect(dest_id, source_id)
+    @service.unmerge_indirect dest_id, source_id
+  end
+
+  # Removes the timeline from the backend store
+  #
+  # ==== Parameters
+  # timeline_id<String>
+  #
+  def delete(timeline_id)
+    @service.delete_timeline timeline_id
+  end
+end

data/lib/haplocheirus/mock_service.rb

@@ -0,0 +1,166 @@
+require 'set'
+
+class Haplocheirus::MockService #:nodoc:
+
+  class MockNode < Struct.new(:status_id, :secondary_id, :bitfield)
+    include Comparable
+
+    RETWEET_BIT = 31
+
+    def self.unpack(string)
+      new *string.unpack("QQI")
+    end
+
+    def initialize(*args)
+      super
+      self.bitfield ||= 0
+    end
+
+    def <=>(other)
+      status_id <=> other.status_id
+    end
+
+    def is_share?
+      bitfield[RETWEET_BIT] == 1
+    end
+  end
+
+  def initialize
+    @timelines = {}
+  end
+
+  def append(e, p, is)
+    is.each do |i|
+      key = p + i.to_s
+      next unless @timelines.key?(key)
+      # NOTE: This check occurs on read, server-side
+      @timelines[key].unshift(e) unless @timelines[key].include?(e)
+    end
+  end
+
+  def remove(e, p, is)
+    is.each do |i|
+      key = p + i.to_s
+      next unless @timelines.key?(key)
+      @timelines[key].reject! { |i| i == e }
+      @timelines.delete(key) if @timelines[key].empty?
+    end
+  end
+
+  def get(i, o, l, d = false)
+    raise Haplocheirus::TimelineStoreException unless @timelines.key?(i)
+    t = @timelines[i].to_a[o..(o+l)]
+    t.sort! { |a, b| a[0,8].unpack("Q") <=> b[0,8].unpack("Q") }
+    t = dedupe(t) if d
+    Haplocheirus::TimelineSegment.new(:entries => t.reverse.map{ |tt| tt.dup },
+                                      :size => t.length,
+                                      :state => Haplocheirus::TimelineSegmentState::HIT)
+  end
+
+  def get_multi(qs)
+    qs.map do |q|
+      begin
+        get q.timeline_id, q.offset, q.length, q.dedupe
+      rescue Haplocheirus::TimelineStoreException
+        Haplocheirus::TimelineSegment.new(:entries => [],
+                                          :size => 0,
+                                          :state => Haplocheirus::TimelineSegmentState::MISS)
+      end
+    end
+  end
+  
+  def get_range(i, f, t = 0, d = false)
+    raise Haplocheirus::TimelineStoreException unless @timelines.key?(i)
+    min = @timelines[i].index([f].pack("Q"))
+    max = t > 0 ? @timelines[i].index([t].pack("Q")) : 0
+    t = min ? @timelines[i][max..min-1] : @timelines[i]
+    t.sort! { |a, b| a[0,8].unpack("Q") <=> b[0,8].unpack("Q") }
+    t = dedupe(t) if d
+    Haplocheirus::TimelineSegment.new(:entries => t.reverse,
+                                      :size => @timelines[i].length,
+                                      :state => Haplocheirus::TimelineSegmentState::HIT)
+  end
+
+  def store(i, e)
+    @timelines[i] = []
+    e.reverse.each { |n| append n, '', [i] }
+  end
+
+  def filter(i, e, depth = -1)
+    raise Haplocheirus::TimelineStoreException unless @timelines.key?(i)
+
+    haystack = @timelines[i].map do |ea|
+      node = MockNode.unpack(ea)
+      if node.is_share?
+        node.secondary_id
+      else
+        node.status_id
+      end
+    end.uniq
+
+    # FIXME: Only send the first 8 bytes for the needles
+    e.select do |packed|
+      node = MockNode.unpack(packed)
+      haystack.include?(node.status_id)
+    end
+  end
+
+  def merge(i, e)
+    return unless @timelines.key?(i)
+
+    e.each do |el|
+      o = 0
+      o += 1 while @timelines[i][0] <= el
+      @timelines[i].insert(o + 1, el)
+    end
+  end
+
+  def merge_indirect(d, s)
+    merge(d, @timelines[s]) if @timelines.key?(s)
+  end
+
+  def unmerge(i, e)
+    return unless @timelines.key?(i)
+    @timelines[i].reject! { |o| e.find { |el| MockNode.unpack(el) == MockNode.unpack(o) } }
+  end
+
+  def unmerge_indirect(d, s)
+    unmerge(d, @timelines[s]) if @timelines.key?(s)
+  end
+
+  def delete_timeline(i)
+    @timelines.delete(i)
+  end
+
+  # This is not part of Haplo, but is useful for test harnesses
+  def reset!
+    @timelines = {}
+  end
+
+  private
+
+  def dedupe(t)
+    # I can't wait until Array#uniq takes a block...
+    seen = { }
+    nodes = []
+
+    t.each do |i|
+      node = MockNode.unpack(i)
+      next if seen.key?(node.status_id)
+      
+      if node.is_share?
+        next if seen.key?(node.secondary_id)
+
+        seen[node.status_id] = i
+        seen[node.secondary_id] = true
+        nodes << node.status_id
+      else
+        seen[node.status_id] = i
+        nodes << node.status_id
+      end
+    end
+
+    seen.values_at(*nodes.sort!)
+  end
+
+end