pregel 0.0.1

data/.gitignore

@@ -0,0 +1,3 @@
+pkg/*
+*.gem
+.bundle

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://rubygems.org"
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,26 @@
+PATH
+  remote: .
+  specs:
+    pregel (0.0.1)
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    diff-lcs (1.1.2)
+    rspec (2.0.1)
+      rspec-core (~> 2.0.1)
+      rspec-expectations (~> 2.0.1)
+      rspec-mocks (~> 2.0.1)
+    rspec-core (2.0.1)
+    rspec-expectations (2.0.1)
+      diff-lcs (>= 1.1.2)
+    rspec-mocks (2.0.1)
+      rspec-core (~> 2.0.1)
+      rspec-expectations (~> 2.0.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  pregel!
+  rspec (~> 2.0.0)

data/README.md

@@ -0,0 +1,57 @@
+# Pregel
+
+Single-node implementation of Google's Pregel framework for large-scale graph processing. It does not provide any of the distributed components, but implements the core functional pieces within a single Ruby VM such that you can develop and run iterative graph algorithms as if you had the full power of Pregel at your disposal.
+
+To learn more about Pregel see following resources:
+
+ * [Pregel, a system for large-scale graph processing](http://portal.acm.org/citation.cfm?id=1582716.1582723)
+ * [Large-scale graph computing at Google](http://googleresearch.blogspot.com/2009/06/large-scale-graph-computing-at-google.html)
+ * [Phoebus](http://github.com/xslogic/phoebus) is a distributed Erlang implementation of Pregel
+
+# PageRank example with Pregel
+To run a PageRank computation on an arbitrary graph, you simply specify the vertices & edges, and then define a compute function for each vertex. The coordinator then partitions the work between a specified number of workers (Ruby threads, in our case), and iteratively executes "supersteps" until we converge on a result. At each superstep, the vertex can read and process incoming messages and then send messages to other vertices. Hence, the full PageRank implementation is:
+
+    class PageRankVertex < Vertex
+      def compute
+        if superstep >= 1
+          sum = messages.inject(0) {|total,msg| total += msg; total }
+          @value = (0.15 / num_nodes) + 0.85 * sum
+        end
+
+        if superstep < 30
+          deliver_to_all_neighbors(@value / neighbors.size)
+        else
+          halt
+        end
+      end
+    end
+
+The above algorithm will run for 30 iterations, at which point all vertices will mark themselves as inactive and the coordinator will terminate our computation.
+
+ * [Computing PageRank for a simple circular graph](https://github.com/igrigorik/pregel/blob/master/spec/coordinator_spec.rb#L52)
+ * [Computing PageRank for a more complex grap](https://github.com/igrigorik/pregel/blob/master/spec/coordinator_spec.rb#L70)
+
+# License
+
+(The MIT License)
+
+Copyright (c) 2010 Ilya Grigorik
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,3 @@
+require 'bundler'
+
+Bundler::GemHelper.install_tasks

data/autotest/discover.rb

@@ -0,0 +1 @@
+Autotest.add_discovery { 'rspec2' }

data/lib/pregel.rb

@@ -0,0 +1,30 @@
+require 'pregel/vertex'
+require 'pregel/worker'
+require 'pregel/coordinator'
+
+require 'singleton'
+
+class PostOffice
+  include Singleton
+
+  def initialize
+    @mailboxes = Hash.new
+    @mutex = Mutex.new
+  end
+
+  def deliver(to, msg)
+    @mutex.synchronize do
+      if @mailboxes[to]
+        @mailboxes[to].push msg
+      else
+        @mailboxes[to] = [msg]
+      end
+    end
+  end
+
+  def read(box)
+    @mutex.synchronize do
+      @mailboxes.delete(box) || []
+    end
+  end
+end

data/lib/pregel/coordinator.rb

@@ -0,0 +1,33 @@
+module Pregel
+  class Coordinator
+    attr_reader :workers
+
+    def initialize(graph, options = {})
+      raise "empty graph" if graph.empty?
+
+      @workers = []
+      @options = {
+        :partitions => 1
+      }.merge(options)
+
+      partition(graph) do |subgraph|
+        @workers << Worker.new(subgraph)
+      end
+    end
+
+    def partition(graph)
+      size = (graph.size.to_f / @options[:partitions]).ceil
+      graph.each_slice(size) { |slice| yield slice }
+    end
+
+    def run
+      loop do
+        # execute a superstep and wait for workers to complete
+        step = @workers.select {|w| w.active > 0}.collect {|w| w.superstep }
+        step.each {|t| t.join}
+
+        break if @workers.select {|w| w.active > 0}.size.zero?
+      end
+    end
+  end
+end

data/lib/pregel/version.rb

@@ -0,0 +1,3 @@
+module Pregel
+  VERSION = "0.0.1"
+end

data/lib/pregel/vertex.rb

@@ -0,0 +1,41 @@
+module Pregel
+  class Vertex
+    attr_reader :id
+    attr_accessor :value, :messages
+
+    def initialize(id, value, *outedges)
+      @id = id
+      @value = value
+      @outedges = outedges
+      @messages = []
+      @active = true
+      @superstep = 0
+    end
+
+    def edges
+      block_given? ? @outedges.each {|e| yield e} : @outedges
+    end
+
+    def deliver_to_all_neighbors(msg)
+      edges.each {|e| deliver(e, msg)}
+    end
+
+    def deliver(to, msg)
+      PostOffice.instance.deliver(to, msg)
+    end
+
+    def step
+      @superstep += 1
+      compute
+    end
+
+    def halt;     @active = false;  end
+    def active!;  @active = true;   end
+    def active?;  @active;          end
+
+    def superstep; @superstep; end
+    def neighbors; @outedges; end
+
+    def compute; end
+  end
+end

data/lib/pregel/worker.rb

@@ -0,0 +1,25 @@
+module Pregel
+  class Worker
+    attr_reader :vertices, :active
+
+    def initialize(graph = [])
+      raise 'empty worker graph' if graph.empty?
+      @vertices = graph
+      @active   = graph.size
+    end
+
+    def superstep
+      Thread.new do
+        @vertices.each do |v|
+          v.messages = PostOffice.instance.read(v.id)
+          v.active! if v.messages.size > 0
+        end
+
+        active = @vertices.select {|v| v.active?}
+        active.each {|v| v.step}
+
+        @active = active.select {|v| v.active?}.size
+      end
+    end
+  end
+end

data/pregel.gemspec

@@ -0,0 +1,22 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "pregel/version"
+
+Gem::Specification.new do |s|
+  s.name        = "pregel"
+  s.version     = Pregel::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Ilya Grigorik"]
+  s.email       = ["ilya@igvita.com"]
+  s.homepage    = "http://github.com/igrigorik/pregel"
+  s.summary     = "Single-node implementation of Google's Pregel framework for large-scale graph processing."
+  s.description = s.summary
+  s.rubyforge_project = "pregel"
+
+  s.add_development_dependency "rspec", '~> 2.0.0'
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+end

data/spec/coordinator_spec.rb

@@ -0,0 +1,95 @@
+require 'helper'
+
+describe Coordinator do
+  it 'should not allow empty graphs' do
+    lambda { Coordinator.new([]) }.should raise_error
+  end
+
+  let(:graph) do
+    [
+      AddVertex.new(:igvita,     1,  :wikipedia),
+      AddVertex.new(:wikipedia,  2,  :google),
+      AddVertex.new(:google,     1,  :wikipedia)
+    ]
+  end
+
+  it 'should partition graphs with variable worker sizes' do
+    c = Coordinator.new(graph)
+    c.workers.size.should == 1
+
+    c = Coordinator.new(graph, partitions: 2)
+    c.workers.size.should == 2
+  end
+
+  it 'should schedule workers to run until there are no active vertices' do
+    c = Coordinator.new(graph)
+    c.run
+
+    c.workers.each do |w|
+      w.vertices.each do |v|
+        v.value.should == 5
+      end
+    end
+  end
+
+  context 'PageRank' do
+    class PageRankVertex < Vertex
+      def compute
+        if superstep >= 1
+          sum = messages.inject(0) {|total,msg| total += msg; total }
+          @value = (0.15 / 3) + 0.85 * sum
+        end
+
+        if superstep < 30
+          deliver_to_all_neighbors(@value / neighbors.size)
+        else
+          halt
+        end
+      end
+    end
+
+
+    it 'should calculate PageRank of a circular graph' do
+      graph = [
+        #                   name     value  out-edges
+        PageRankVertex.new(:igvita,     1,  :wikipedia),
+        PageRankVertex.new(:wikipedia,  1,  :google),
+        PageRankVertex.new(:google,     1,  :igvita)
+      ]
+
+      c = Coordinator.new(graph)
+      c.run
+
+      c.workers.each do |w|
+        w.vertices.each do |v|
+          (v.value * 100).to_i.should == 33
+        end
+      end
+    end
+
+    it 'should calculate PageRank of arbitrary graph' do
+      graph = [
+        # page 1 -> page 1, page 2  (0.18)
+        # page 2 -> page 1, page 3  (0.13)
+        # page 3 -> page 3          (0.69)
+
+        #                   name     value  out-edges
+        PageRankVertex.new(:igvita,     1,  :igvita, :wikipedia),
+        PageRankVertex.new(:wikipedia,  1,  :igvita, :google),
+        PageRankVertex.new(:google,     1,  :google)
+      ]
+
+      c = Coordinator.new(graph)
+      c.run
+
+      c.workers.each do |w|
+        (w.vertices.find {|v| v.id == :igvita }.value * 100).ceil.to_i.should == 19
+        (w.vertices.find {|v| v.id == :wikipedia }.value * 100).ceil.to_i.should == 13
+        (w.vertices.find {|v| v.id == :google }.value * 100).to_i.should == 68
+      end
+    end
+  end
+
+  it 'should parition nodes by hashing the node id'
+  it 'should allow scheduling multiple partitions to a single worker'
+end

data/spec/helper.rb

@@ -0,0 +1,10 @@
+require 'lib/pregel'
+
+include Pregel
+
+class AddVertex < Vertex
+  def compute
+    @value += 1
+    halt if @value >= 5
+  end
+end

data/spec/vertex_spec.rb

@@ -0,0 +1,56 @@
+require 'helper'
+
+describe Vertex do
+  it 'should create a vertex with id, value, and edges' do
+    lambda { Vertex.new(:a, 10, :b) }.should_not raise_error
+    lambda { Vertex.new(:a, 10, :b, :c) }.should_not raise_error
+  end
+
+  let(:v) { Vertex.new(:a, 10, :b, :c) }
+
+  it 'should report new vertex as active' do
+    v.active?.should be_true
+
+    v.halt
+    v.active?.should be_false
+  end
+
+  it 'should contain a modifiable value' do
+    v.value = 20
+    v.value.should == 20
+  end
+
+  it 'should keep track of the current superstep' do
+    v.superstep.should == 0
+    v.step
+    v.superstep.should == 1
+  end
+
+  it 'should allow iterating over out-edges' do
+    v.edges.size.should == 2
+    v.edges.each {|e| [:b, :c].should include e }
+  end
+
+  it 'should allow arbitrary type for edge' do
+    lambda do
+      Vertex.new(:a, 10, {id: 1, weight:10}, {id: 2, weight:20})
+    end.should_not raise_error
+  end
+
+  it 'should allow a user defined compute' do
+    class V < Vertex
+      def compute; @value = 20; end
+    end
+
+    v = V.new(:a, 10)
+    v.compute
+    v.value.should == 20
+  end
+
+  it 'should have an inbox for incoming messages' do
+    v.messages.size.should == 0
+    v.messages = [:a, :b]
+    v.messages.size.should == 2
+  end
+
+end

data/spec/worker_spec.rb

@@ -0,0 +1,61 @@
+require 'helper'
+
+describe Worker do
+
+  let(:graph) do
+    [
+      AddVertex.new(:igvita,     1,  :wikipedia),
+      AddVertex.new(:wikipedia,  2,  :google),
+      AddVertex.new(:google,     1,  :wikipedia)
+    ]
+  end
+
+  let(:worker) do
+    Worker.new(graph)
+  end
+
+  it 'should accept non-zero number of nodes' do
+    lambda { Worker.new([]) }.should raise_error
+  end
+
+  it 'should partition graphs with variable worker sizes' do
+    graph = [
+      Vertex.new(:igvita,     1,  :wikipedia),
+      Vertex.new(:wikipedia,  2,  :google),
+      Vertex.new(:google,     1,  :wikipedia)
+    ]
+
+    c = Coordinator.new(graph)
+    c.workers.size.should == 1
+
+    c = Coordinator.new(graph, partitions: 2)
+    c.workers.size.should == 2
+  end
+
+  it 'should execute an async superstep' do
+    # TODO: simulate async message delivery to worker by returning
+    # a thread per message
+    worker.superstep.should be_an_instance_of Thread
+  end
+
+  it 'should perform single iteration on the graph' do
+    # TODO: reuse a threadpool within worker for each partition
+    worker.superstep.join
+    worker.vertices.first.value.should == 2
+  end
+
+  it 'should return the number of active vertices for next superstep' do
+    worker.superstep.join
+    worker.active.should > 0
+  end
+
+  it 'should deliver messages to vertices at beginning of each superstep' do
+    PostOffice.instance.deliver(:igvita, 'hello')
+    worker.superstep.join
+
+    ig = worker.vertices.find {|v| v.id == :igvita }
+    ig.messages.size.should == 1
+    ig.messages.first.should == 'hello'
+  end
+
+end

metadata

@@ -0,0 +1,97 @@
+--- !ruby/object:Gem::Specification 
+name: pregel
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Ilya Grigorik
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-10-30 00:00:00 -04:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 2
+        - 0
+        - 0
+        version: 2.0.0
+  type: :development
+  version_requirements: *id001
+description: Single-node implementation of Google's Pregel framework for large-scale graph processing.
+email: 
+- ilya@igvita.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- Gemfile
+- Gemfile.lock
+- README.md
+- Rakefile
+- autotest/discover.rb
+- lib/pregel.rb
+- lib/pregel/coordinator.rb
+- lib/pregel/version.rb
+- lib/pregel/vertex.rb
+- lib/pregel/worker.rb
+- pregel.gemspec
+- spec/coordinator_spec.rb
+- spec/helper.rb
+- spec/vertex_spec.rb
+- spec/worker_spec.rb
+has_rdoc: true
+homepage: http://github.com/igrigorik/pregel
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: pregel
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Single-node implementation of Google's Pregel framework for large-scale graph processing.
+test_files: 
+- spec/coordinator_spec.rb
+- spec/helper.rb
+- spec/vertex_spec.rb
+- spec/worker_spec.rb