flockdb 0.3.0

data/.gitignore

@@ -0,0 +1 @@
+pkg/*

data/LICENSE

@@ -0,0 +1,13 @@
+FlockDB Client is Copyright (C) 2010 Twitter, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not
+use this work except in compliance with the License. You may obtain a copy of
+the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations under
+the License.

data/README.md

@@ -0,0 +1,60 @@
+FlockDB ruby client
+
+FlockDB is a distributed graph database capable of storing millions of
+nodes, billions of edges, and handling tens of thousands of operations
+per second. Interact with it in ruby with this client.
+
+
+INSTALLATION
+------------
+
+    gem install flockdb
+
+FlockDB depends on thrift and thrift_client, which will install as
+part of the gem's dependencies. Note that thrift has a native C
+extension which it will need to compile.
+
+
+USAGE
+-----
+
+Instantiate a client:
+
+    flockdb = Flock.new 'localhost:7915, :graphs => { :follows => 1, :blocks => 2 }
+
+Flock.new is a convenience shortcut for Flock::Client.new. Flock.new
+expects a list (or a single) servers as the first argument, and an
+options hash. Flock::Client.new takes the same options as ThriftClient
+for configuring the raw thrift client, as well as :graphs for mapping
+ruby symbols to the corresponding graph ids. See [thrift_client's
+documentation](http://github.com/fauna/thrift_client) for information
+on the other options it takes.
+
+Edge manipulation:
+
+    flockdb.add(1, :follows, 2)
+    flockdb.remove(1, :blocks, 2)
+
+Perform a query:
+
+    flockdb.select(1, :follows, nil).to_a
+
+The client supports a rich query api to build up complex set operations:
+
+    flockdb.select(nil, :follows, 1).difference(flockdb.select(1, :follows, nil).intersect(2, :follows, nil)).to_a
+
+
+CONTRIBUTORS
+------------
+
+Nick Kallen  
+Matt Freels  
+Rael Dornfest  
+
+
+LICENSE
+-------
+
+Copyright (C) 2010 Twitter, Inc.
+
+This work is licensed under the Apache License, Version 2.0. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,34 @@
+ROOT_DIR = File.expand_path(File.dirname(__FILE__))
+
+require 'rubygems' rescue nil
+require 'rake'
+require 'spec/rake/spectask'
+
+task :default => :spec
+
+desc "Run all specs in spec directory."
+Spec::Rake::SpecTask.new(:spec) do |t|
+  t.spec_opts = ['--options', "\"#{ROOT_DIR}/spec/spec.opts\""]
+  t.spec_files = FileList['spec/**/*_spec.rb']
+end
+
+# gemification with jeweler
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gemspec|
+    gemspec.name = "flockdb"
+    gemspec.summary = "Ruby Flock client"
+    gemspec.description = "Get your flock on in Ruby."
+    gemspec.email = "freels@twitter.com"
+    gemspec.homepage = "http://twitter.com"
+    gemspec.authors = ["Matt Freels", "Rael Dornfest", "Nick Kallen"]
+    gemspec.add_dependency 'thrift', '>= 0.2.0'
+    gemspec.add_dependency 'thrift_client', '>= 0.4.1'
+
+    # development
+    gemspec.add_development_dependency 'rspec'
+    gemspec.add_development_dependency 'rr'
+  end
+rescue LoadError
+  puts "Jeweler not available. Install it with: gem install jeweler"
+end

data/VERSION

@@ -0,0 +1 @@
+0.3.0

data/flockdb.gemspec

@@ -0,0 +1,82 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{flockdb}
+  s.version = "0.3.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Matt Freels", "Rael Dornfest", "Nick Kallen"]
+  s.date = %q{2010-04-19}
+  s.description = %q{Get your flock on in Ruby.}
+  s.email = %q{freels@twitter.com}
+  s.extra_rdoc_files = [
+    "LICENSE",
+     "README.md"
+  ]
+  s.files = [
+    ".gitignore",
+     "LICENSE",
+     "README.md",
+     "Rakefile",
+     "VERSION",
+     "flockdb.gemspec",
+     "lib/flock.rb",
+     "lib/flock/client.rb",
+     "lib/flock/mixins/sizeable.rb",
+     "lib/flock/mock_service.rb",
+     "lib/flock/operation.rb",
+     "lib/flock/operations/complex_operation.rb",
+     "lib/flock/operations/execute_operation.rb",
+     "lib/flock/operations/execute_operations.rb",
+     "lib/flock/operations/query_term.rb",
+     "lib/flock/operations/select_operation.rb",
+     "lib/flock/operations/simple_operation.rb",
+     "lib/flock/service.rb",
+     "lib/flock/thrift/edges.rb",
+     "lib/flock/thrift/edges_types.rb",
+     "lib/flock/thrift/flock_constants.rb",
+     "lib/flock/thrift/flock_types.rb",
+     "lib/flock/thrift/shards.rb",
+     "lib/flockdb.rb",
+     "spec/flock_spec.rb",
+     "spec/mock_service_spec.rb",
+     "spec/spec.opts",
+     "spec/spec_helper.rb"
+  ]
+  s.homepage = %q{http://twitter.com}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.6}
+  s.summary = %q{Ruby Flock client}
+  s.test_files = [
+    "spec/flock_spec.rb",
+     "spec/mock_service_spec.rb",
+     "spec/spec_helper.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<thrift>, [">= 0.2.0"])
+      s.add_runtime_dependency(%q<thrift_client>, [">= 0.4.1"])
+      s.add_development_dependency(%q<rspec>, [">= 0"])
+      s.add_development_dependency(%q<rr>, [">= 0"])
+    else
+      s.add_dependency(%q<thrift>, [">= 0.2.0"])
+      s.add_dependency(%q<thrift_client>, [">= 0.4.1"])
+      s.add_dependency(%q<rspec>, [">= 0"])
+      s.add_dependency(%q<rr>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<thrift>, [">= 0.2.0"])
+    s.add_dependency(%q<thrift_client>, [">= 0.4.1"])
+    s.add_dependency(%q<rspec>, [">= 0"])
+    s.add_dependency(%q<rr>, [">= 0"])
+  end
+end
+

data/lib/flock.rb

@@ -0,0 +1,40 @@
+# external dependencies
+require 'thrift'
+require 'thrift_client'
+
+require 'flock/mixins/sizeable'
+
+# thrift sources. load order is important.
+require 'flock/thrift/flock_types'
+require 'flock/thrift/flock_constants'
+require 'flock/thrift/shards'
+require 'flock/thrift/edges_types'
+require 'flock/thrift/edges'
+
+require 'flock/operation'
+require 'flock/operations/query_term'
+require 'flock/operations/select_operation'
+require 'flock/operations/complex_operation'
+require 'flock/operations/simple_operation'
+require 'flock/operations/execute_operation'
+require 'flock/operations/execute_operations'
+require 'flock/service'
+require 'flock/client'
+
+module Flock
+  autoload :MockService, 'flock/mock_service'
+
+  class UnknownGraphError < FlockException
+    def initialize(graph)
+      super("Unable to look up id for graph #{graph.inspect}. Register graphs with Flock.graphs = <graph list>.")
+    end
+  end
+
+  class << self
+    attr_accessor :default_service_class, :graphs
+
+    def new(*args)
+      Flock::Client.new(*args)
+    end
+  end
+end

data/lib/flock/client.rb

@@ -0,0 +1,124 @@
+class Flock::Client
+
+  attr_accessor :graphs
+  attr_reader :service
+
+  class << self
+    def service_class; Flock.default_service_class ||= Flock::Service end
+  end
+
+  # takes arguments a list of servers and an options hash to pass to the default service_class,
+  # or a service itself
+  def initialize(servers = nil, options = {})
+    if graphs = (options.delete(:graphs) || Flock.graphs)
+      @graphs = graphs
+    end
+
+    @service =
+      if servers.nil? or servers.is_a? Array or servers.is_a? String
+        self.class.service_class.new(servers, options)
+      else
+        servers
+      end
+  end
+
+
+  # local results cache
+
+  def cache_locally
+    @cache = {}
+    yield
+  ensure
+    @cache = nil
+  end
+
+  def _cache(op, query)
+    return yield unless @cache
+    @cache[[op, query]] ||= yield
+  end
+
+  def _cache_clear
+    @cache = {} if @cache
+  end
+
+
+  # queries
+
+  def select(*query)
+    Flock::SimpleOperation.new(self, _query_args(query))
+  end
+
+  def contains(*query)
+    _cache :contains, query do
+      select(*query).any?
+    end
+  end
+
+  def size(*query)
+    _cache :size, query do
+      select(*query).size
+    end
+  end
+  alias count size
+
+
+  # edge manipulation
+
+  def update(method, source_id, graph, destination_id, priority = Flock::Priority::High)
+    _cache_clear
+    ops = current_transaction || Flock::ExecuteOperations.new(@service, priority)
+    ops.send(method, *_query_args([source_id, graph, destination_id]))
+    ops.apply unless in_transaction?
+  end
+
+  [:add, :remove, :archive, :unarchive].each do |method|
+    class_eval "def #{method}(*args); update(#{method.inspect}, *args) end", __FILE__, __LINE__
+  end
+
+  def transaction(priority = Flock::Priority::High, &block)
+    new_transaction = !in_transaction?
+
+    ops =
+      if new_transaction
+        Thread.current[:edge_transaction] = Flock::ExecuteOperations.new(@service, priority)
+      else
+        current_transaction
+      end
+
+    result = yield self if block.arity == 1
+    ops.apply if new_transaction
+    result
+
+  ensure
+    Thread.current[:edge_transaction] = nil if new_transaction
+  end
+
+  def current_transaction
+    Thread.current[:edge_transaction]
+  end
+
+  def in_transaction?
+    !!Thread.current[:edge_transaction]
+  end
+
+
+  # graph name lookup utility methods
+
+  def _lookup_graph(key)
+    if @graphs.nil? or key.is_a? Integer
+      key
+    else
+      @graphs[key] or raise UnknownGraphError.new(key)
+    end
+  end
+
+  def _query_args(args)
+    source, graph, destination = *((args.length == 1) ? args.first : args)
+    [_node_arg(source), _lookup_graph(graph), _node_arg(destination)]
+  end
+
+  def _node_arg(node)
+    return node.map {|n| n.to_i if n } if node.respond_to? :map
+    node.to_i if node
+  end
+end

data/lib/flock/mixins/sizeable.rb

@@ -0,0 +1,17 @@
+module Flock
+  module Mixins
+    module Sizeable
+      def any?(&block)
+        if !block_given?
+          size > 0
+        else
+          map(&block).any?
+        end
+      end
+      
+      def empty?
+        size == 0
+      end
+    end
+  end
+end

data/lib/flock/mock_service.rb

@@ -0,0 +1,178 @@
+module Flock
+  module MockService
+    extend self
+
+    attr_accessor :timeout, :fixtures
+
+    def clear
+      @sources = @destinations = nil
+    end
+
+    def load(fixtures = nil)
+      fixtures ||= self.fixtures or raise "No flock fixtures specified. either pass fixtures to load, or set Flock::MockService.fixtures."
+
+      clear
+
+      fixtures.each do |fixture|
+        file, graph_id, source, destination = fixture.values_at(:file, :graph, :source, :destination)
+
+        fixtures_data = YAML::load(ERB.new(File.open(file, 'r').read).result(binding)).sort
+        fixtures_data.each do |key, value|
+          sources[[value[destination], graph_id]] << value[source]
+          destinations[[value[source], graph_id]] << value[destination]
+        end
+      end
+    end
+
+    def inspect
+      "Flock::MockService: " + @sources.inspect + " - " + @destinations.inspect
+    end
+
+    def execute(operations)
+      operations = operations.operations
+      operations.each do |operation|
+        term = operation.term
+        backward_data_source, forward_data_source = term.is_forward ? [sources, destinations] : [destinations, sources]
+        backward_archived_data_source, forward_archived_data_source = term.is_forward ? [archived_sources, archived_destinations] : [archived_destinations, archived_sources]
+        source_id, graph_id = term.source_id, term.graph_id
+        destination_ids = term.destination_ids && term.destination_ids.unpack("Q*")
+        case operation.operation_type
+        when Edges::ExecuteOperationType::Add
+          if destination_ids.nil?
+            backward_archived_data_source.delete([source_id, graph_id]).to_a.each do |n|
+              (forward_data_source[[n, graph_id]] << source_id).uniq!
+              (backward_data_source[[source_id, graph_id]] << n).uniq!
+              forward_archived_data_source[[n, graph_id]].delete(source_id)
+            end
+          else
+            destination_ids.each do |destination_id|
+              backward_data_source[[destination_id, graph_id]] << source_id
+              forward_data_source[[source_id, graph_id]] << destination_id
+            end
+          end
+        when Edges::ExecuteOperationType::Remove
+          destination_ids.each do |destination_id|
+            backward_data_source[[destination_id, graph_id]].delete(source_id)
+            forward_data_source[[source_id, graph_id]].delete(destination_id)
+          end
+        when Edges::ExecuteOperationType::Archive
+          if destination_ids.nil?
+            backward_data_source.delete([source_id, graph_id]).to_a.each do |n|
+              forward_archived_data_source[[n, graph_id]] << source_id
+              backward_archived_data_source[[source_id, graph_id]] << n
+              forward_data_source[[n, graph_id]].delete(source_id)
+            end
+          else
+            raise "not yet implemented"
+          end
+        end
+      end
+    end
+
+    def select(select_operations, page)
+      iterate(select_query(select_operations), page)
+    end
+
+    def count(select_operations)
+      select_query(select_operations).size
+    end
+
+    # FIXME
+    def counts_of_sources_for(destination_ids, graph_id)
+      destination_ids.unpack("Q*").map do |destination_id|
+        (sources[[destination_id, graph_id]] || []).size
+      end.pack("I*")
+    end
+
+    private
+    def select_query(select_operations)
+      stack = []
+      select_operations.each do |select_operation|
+        case select_operation.operation_type
+        when Edges::SelectOperationType::SimpleQuery
+          query_term = select_operation.term
+          data_source = query_term.is_forward ? destinations : sources
+          data = data_source[[query_term.source_id, query_term.graph_id]]
+          if query_term.destination_ids
+            data &= query_term.destination_ids.unpack("Q*")
+          end
+          stack.push(data)
+        when Edges::SelectOperationType::Intersection
+          stack.push(stack.pop & stack.pop)
+        when Edges::SelectOperationType::Union
+          stack.push(stack.pop | stack.pop)
+        when Edges::SelectOperationType::Difference
+          operand2 = stack.pop
+          operand1 = stack.pop
+          stack.push(operand1 - operand2)
+        end
+      end
+      stack.pop
+    end
+
+    private
+
+    def iterate(data, page)
+      return empty_result if page.cursor == Flock::CursorEnd
+
+      start = if page.cursor < Flock::CursorStart
+                [-page.cursor - page.count, 0].max
+              else
+                page.cursor == Flock::CursorStart ? 0 : page.cursor
+              end
+      rv = data.slice(start, page.count)
+      next_cursor = (start + page.count >= data.size) ? Flock::CursorEnd : start + page.count
+      prev_cursor = (start <= 0) ? Flock::CursorEnd : -start
+
+      result = Flock::Results.new
+      result.ids = Array(rv).pack("Q*")
+      result.next_cursor = next_cursor
+      result.prev_cursor = prev_cursor
+      result
+    end
+
+    def empty_result
+      @empty_result ||= begin
+                          empty_result = Flock::Results.new
+                          empty_result.ids = ""
+                          empty_result.next_cursor = Flock::CursorEnd
+                          empty_result.prev_cursor = Flock::CursorEnd
+                          empty_result
+                        end
+    end
+
+    def sources
+      @sources ||= Hash.new do |h, k|
+        h[k] = []
+      end
+    end
+
+    def destinations
+      @destinations ||= Hash.new do |h, k|
+        h[k] = []
+      end
+    end
+
+    def archived_sources
+      @archived_sources ||= Hash.new do |h, k|
+        h[k] = []
+      end
+    end
+
+    def archived_destinations
+      @archived_destinations ||= Hash.new do |h, k|
+        h[k] = []
+      end
+    end
+
+    # deprecated
+    public
+    def offset_sources_for(destination_id, graph_id, offset, count)
+      (sources[[destination_id, graph_id]].slice(offset, count) || []).pack("Q*")
+    end
+
+    def offset_destinations_for(source_id, graph_id, offset, count)
+      (destinations[[source_id, graph_id]].slice(offset, count) || []).pack("Q*")
+    end
+  end
+end