keen 0.0.2 → 0.0.4

data/Gemfile

@@ -1 +1,3 @@
 source :rubygems
+gem 'rspec'
+gem 'fakeweb'

data/bin/keen_send

@@ -1,11 +1,11 @@
 #!/usr/bin/env ruby
-$LOAD_PATH << File.join(File.dirname(__FILE__), '..', 'lib')
-require 'rubygems'
-require 'optparse'
-require 'keen'
+$LOAD_PATH << File.join(File.dirname(__FILE__), "..", "lib")
+require "rubygems"
+require "optparse"
+require "keen"
 
 options = {
-  :env => 'production',
+  :env => "production",
 }
 
 required = [

data/examples.rb

@@ -1,3 +1,4 @@
+require 'rubygems'
 require 'keen'
  
 # Get these from the keen.io website:

data/examples/rails_2/CoolForums/app/controllers/application_controller.rb

@@ -1,3 +1,6 @@
+require 'json'
+require 'keen'
+
 # Filters added to this controller apply to all controllers in the application.
 # Likewise, all the methods added will be available for all controllers.
 

data/keen.gemspec

@@ -22,9 +22,10 @@ Gem::Specification.new do |s|
   # s.add_development_dependency "rspec"
   # s.add_runtime_dependency "rest-client"
   
-  s.add_dependency('multi_json', '>= 1.0.3')
+  s.add_dependency('json', '>= 1.6.5')
+  s.add_dependency('fakeweb', '>= 1.3.0')
+  s.add_dependency('rspec', '>= 2.9.0')
   s.add_dependency('system_timer', '>= 1.2.4')
-  s.add_dependency('httparty', '>= 0.8.1')
   s.add_dependency('redis', '>= 2.2.2')
 
   # took these from Twilio library:

data/lib/keen.rb

@@ -1,3 +1,6 @@
 require 'keen/client'
 require 'keen/version'
-require 'keen/storage/flat_file_handler'
+require 'keen/utils'
+require 'keen/async/job'
+require 'keen/async/worker'
+require 'keen/async/storage/redis_handler'

data/lib/keen/async/job.rb

@@ -0,0 +1,70 @@
+require 'keen/async/storage/redis_handler'
+
+module Keen
+  module Async
+    class Job
+      # Represents one job.
+      #
+      #
+
+      attr_accessor :project_id, :auth_token, :collection_name, :event_body
+      
+      def to_json(options=nil)
+        @definition.to_json
+      end
+
+
+      def to_s
+        self.to_json
+      end
+      
+      def initialize(handler, definition={})
+
+        load_definition(definition)
+        @handler = handler
+        
+      end
+
+      def load_definition(definition)
+
+        definition = Keen::Utils.symbolize_keys(definition)
+
+        # define some key lists:
+        required_keys = [:project_id, :auth_token, :collection_name, :event_body]
+        optional_keys = []
+        all_keys = required_keys + optional_keys
+
+
+        # don't allow them to send nil values for anything
+        definition.each do |key, value|
+          # reject unrecognized keys:
+          raise "Unrecognized key: #{key}" unless all_keys.include? key.to_sym
+        end
+
+
+        required_keys.each do |key|
+          value = definition[key]
+
+          raise "You sent a nil value for the #{key}." if value.nil?
+        end
+
+
+        all_keys.each do |key|
+          value = definition[key]
+          self.instance_variable_set("@#{key}", value)
+        end
+
+        @definition = definition
+
+
+      end
+
+      def save
+        @handler.record_job(self)
+      end
+
+    end
+  end
+end
+
+

data/lib/keen/async/storage/flat_file_handler.rb

@@ -0,0 +1,57 @@
+require 'fileutils'
+require 'base64'
+require 'zlib'
+
+
+
+module Keen
+  module Async
+    module Storage
+      class FlatFileHandler
+
+        # Paths
+        # -----
+        
+        # Where new events go as they come in:
+        ACTIVE = "active.txt"
+
+        # The temporary file that will store records during processing:
+        PROCESSING = "processing.txt"
+
+        # Records are put here if processing fails:
+        FAILED = "failed.txt"
+
+        def initialize(filepath)
+          @filepath = filepath
+        
+          is_directory?
+          is_writable?
+          is_readable?
+        end
+
+        def dump_contents
+          # move
+        end
+
+        def is_writable?
+          if not FileTest.writable? @filepath
+            raise "Can't write to file: " + @filepath
+          end
+        end
+        
+        def is_readable?
+          if not FileTest.readable? @filepath
+            raise "Can't read from file: " + @filepath
+          end
+        end
+
+        def is_directory?
+          if not FileTest.directory? @filepath
+            raise "Can't read from file: " + @filepath
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/keen/async/storage/redis_handler.rb

@@ -0,0 +1,136 @@
+require 'fileutils'
+require 'redis'
+require 'json'
+require 'time'
+
+module Keen
+  module Async
+    module Storage
+      class RedisHandler
+
+        # Keys
+        # ----
+
+        def global_key_prefix
+          "keen_415" + Keen::VERSION
+        end
+        
+        def active_queue_key
+          "#{global_key_prefix}.active_queue_key"
+        end
+
+        def failed_queue_key
+          "#{global_key_prefix}.failed_queue_key"
+        end
+
+        def add_to_active_queue(value)
+          @redis.lpush active_queue_key, value
+          puts "added #{value} to active queue; length is now #{@redis.llen active_queue_key}"
+        end
+
+        def record_job(job)
+          add_to_active_queue JSON.generate(job)
+        end
+
+        def handle_prior_failures
+          # TODO consume the failed_queue and do something with it (loggly? retry? flat file?)
+        end
+
+        def initialize
+          @redis = Redis.new
+        end
+
+        def count_active_queue
+          @redis.llen active_queue_key
+        end
+
+        def get_collated_jobs(how_many)
+
+          # Returns a hash of the next `how_many` jobs, indexed on project_id and then collection_name.
+          #
+          # It looks like this:
+          #
+          #  collated = {
+          #    "4f5775ad163d666a6100000e" => {
+          #      "clicks" => [
+          #        Keen::Storage::Job.new({
+          #          :project_id => "4f5775ad163d666a6100000e",
+          #          :auth_token => "a5d4eaf432914823a94ecd7e0cb547b9",
+          #          :collection_name => "clicks",
+          #          :event_body => {:user_id => "12345"},
+          #        }),
+          #        Keen::Storage::Job.new({
+          #          :project_id => "4f5775ad163d666a6100000e",
+          #          :auth_token => "a5d4eaf432914823a94ecd7e0cb547b9",
+          #          :collection_name => "clicks",
+          #          :event_body => {:user_id => "12345"},
+          #        }),
+          #        Keen::Storage::Job.new({
+          #          :project_id => "4f5775ad163d666a6100000e",
+          #          :auth_token => "a5d4eaf432914823a94ecd7e0cb547b9",
+          #          :collection_name => "clicks",
+          #          :event_body => {:user_id => "12345"},
+          #        }),
+          #      ],
+          #      "purchases" => [
+          #        Keen::Storage::Job.new({
+          #          :project_id => "4f5775ad163d666a6100000e",
+          #          :auth_token => "a5d4eaf432914823a94ecd7e0cb547b9",
+          #          :collection_name => "purchases",
+          #          :event_body => {:user_id => "12345"},
+          #        }),
+          #        Keen::Storage::Job.new({
+          #          :project_id => "4f5775ad163d666a6100000e",
+          #          :auth_token => "a5d4eaf432914823a94ecd7e0cb547b9",
+          #          :collection_name => "purchases",
+          #          :event_body => {:user_id => "12345"},
+          #        }),
+          #        Keen::Storage::Job.new({
+          #          :project_id => "4f5775ad163d666a6100000e",
+          #          :auth_token => "a5d4eaf432914823a94ecd7e0cb547b9",
+          #          :collection_name => "purchases",
+          #          :event_body => {:user_id => "12345"},
+          #        }),
+          #      ],
+          #    }
+          #  }
+
+          handle_prior_failures
+
+          key = active_queue_key
+
+          jobs = []
+
+          how_many.times do
+            this = @redis.lpop key
+            jobs.push JSON.parse this
+          end
+
+          collate_jobs(jobs)
+        end
+
+        def collate_jobs(queue)
+          collated = {}
+
+          queue.each do |job_hash|
+
+            job = Keen::Async::Job.new(self, job_hash)
+
+            if not collated.has_key? job.project_id
+              collated[job.project_id] = {}
+            end
+
+            if not collated[job.project_id].has_key? job.collection_name
+              collated[job.project_id][job.collection_name] = []
+            end
+
+            collated[job.project_id][job.collection_name].push(job)
+          end
+
+          collated
+        end
+
+      end
+    end
+  end
+end

data/lib/keen/async/worker.rb

@@ -0,0 +1,81 @@
+require "keen/async/storage/redis_handler"
+require "net/http"
+require "net/https"
+
+
+
+module Keen
+
+  module Async
+
+    # How many events should we send over the wire at a time?
+    BATCH_SIZE = 100
+    SSL_CA_FILE = File.dirname(__FILE__) + '../../../conf/cacert.pem'
+     
+    class Worker
+
+      def initialize(handler)
+        @handler = handler
+      end
+
+      def batch_url(project_id)
+        if not project_id
+          raise "Missing project_id."
+        end
+        "https://api.keen.io/1.0/projects/#{project_id}/_events"
+      end
+
+      def process_queue
+
+        queue_length = @handler.count_active_queue
+
+        batch_size = Keen::Async::BATCH_SIZE
+
+        num_batches = queue_length / batch_size
+
+        num_batches.times do
+          collated = @handler.get_collated_jobs(batch_size)
+          collated.each do |project_id, batch|
+            send_batch(project_id, batch)
+          end
+        end
+      end
+      
+      def send_batch(project_id, batch)
+        if not batch
+          return
+        end
+        
+        first_key = batch.keys[0]
+        job_list = batch[first_key]
+        auth_token = job_list[0].auth_token
+        
+        uri = URI.parse(batch_url(project_id))
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = true
+        http.ca_file = Keen::Async::SSL_CA_FILE
+        http.verify_mode = OpenSSL::SSL::VERIFY_PEER
+        http.verify_depth = 5
+
+        request = Net::HTTP::Post.new(uri.path)
+        request.body = batch.to_json
+        request["Content-Type"] = "application/json"
+        request["Authorization"] = auth_token
+
+        response = http.request(request)
+
+        #response = Net::HTTP.start(uri.host, uri.port) {|http|
+          #http.request(request)
+        #}
+
+        puts response
+        # TODO: If something fails, we should move the job to the 
+        # prior_failures queue by calling, perhaps:
+        #
+        # @handler.log_failed_job(job)
+      end
+
+    end
+  end
+
+end

data/lib/keen/client.rb

@@ -1,123 +1,79 @@
-require 'keen/storage/redis_handler'
-require 'json'
-require "net/http"
+require "keen/async/storage/redis_handler"
+require "keen/async/job"
+require "json"
 require "uri"
+require "time"
 
 
 module Keen
+
   class Client
 
-    def self.batch_url(project_id)
-      "http://api.keen.io/1.0/projects/#{project_id}/_events"
-    end
-    
-    def initialize(project_id, auth_token)
-    end
+    attr_accessor :storage_handler, :project_id, :auth_token
 
-    def add_event(collection_name, event_body)
-      validate_collection_name(collection_name)
+    def initialize(project_id, auth_token, options = {})
 
-    end
+      raise "project_id must be string" unless project_id.kind_of? String
+      raise "auth_token must be string" unless auth_token.kind_of? String
 
-    def self.process_queue(options)
-      mode = options[:storage_mode].to_sym
-      case mode
-      when :flat_file
-        self.process_queue_from_flat_file(options)
-      when :redis
-        self.process_queue_from_redis(options)
-      else
-        raise "Unknown storage_mode sent: `#{mode}`"
-      end
+      default_options = {
+        :storage_mode => :redis,
+      }
+      
+      options = default_options.update(options)
+
+      @project_id = project_id
+      @auth_token = auth_token
+      @storage_mode = options[:storage_mode]
     end
 
-    def self.process_queue_from_redis(options)
-      require 'keen/storage/redis_handler'
-      handler = Keen::Storage::RedisHandler.new
-      collated = handler.process_queue
-
-      # TODO: remove this mock:
-      collated = {
-        "4f5775ad163d666a6100000e" => {
-          "clicks" => [
-            Keen::Storage::Item.new({
-              :project_id => "4f5775ad163d666a6100000e",
-              :auth_token => "a5d4eaf432914823a94ecd7e0cb547b9",
-              :collection_name => "clicks",
-              :event_body => {:user_id => "12345"},
-            }),
-            Keen::Storage::Item.new({
-              :project_id => "4f5775ad163d666a6100000e",
-              :auth_token => "a5d4eaf432914823a94ecd7e0cb547b9",
-              :collection_name => "clicks",
-              :event_body => {:user_id => "12345"},
-            }),
-            Keen::Storage::Item.new({
-              :project_id => "4f5775ad163d666a6100000e",
-              :auth_token => "a5d4eaf432914823a94ecd7e0cb547b9",
-              :collection_name => "clicks",
-              :event_body => {:user_id => "12345"},
-            }),
-          ],
-          "purchases" => [
-            Keen::Storage::Item.new({
-              :project_id => "4f5775ad163d666a6100000e",
-              :auth_token => "a5d4eaf432914823a94ecd7e0cb547b9",
-              :collection_name => "purchases",
-              :event_body => {:user_id => "12345"},
-            }),
-            Keen::Storage::Item.new({
-              :project_id => "4f5775ad163d666a6100000e",
-              :auth_token => "a5d4eaf432914823a94ecd7e0cb547b9",
-              :collection_name => "purchases",
-              :event_body => {:user_id => "12345"},
-            }),
-            Keen::Storage::Item.new({
-              :project_id => "4f5775ad163d666a6100000e",
-              :auth_token => "a5d4eaf432914823a94ecd7e0cb547b9",
-              :collection_name => "purchases",
-              :event_body => {:user_id => "12345"},
-            }),
-          ],
-        }
-      }
+    def handler
+
+      unless @storage_handler
+        mode = @storage_mode
+
+        case mode
+        when :redis
+          @storage_handler = Keen::Async::Storage::RedisHandler.new
+        else
+          raise "Unknown storage_mode sent to client: `#{mode}`"
+        end
 
-      collated.each do |project_id, batch|
-        self.send_batch(project_id, batch)
       end
+
+      @storage_handler
     end
-    
-    def self.send_batch(project_id, batch)
-      if not batch
-        return
-      end
 
-      first_key = batch.keys[0]
-      item_list = batch[first_key]
-      puts item_list[0].class
-      auth_token = item_list[0].auth_token
-      
-      uri = URI.parse(self.batch_url(project_id))
+    def add_event(collection_name, event_body, timestamp=nil)
+      #
+      # `collection_name` should be a string
+      #
+      # `event_body` should be a JSON-serializable hash
+      #
+      # `timestamp` is optional. If sent, it should be a Time instance.  
+      #  If it's not sent, we'll use the current time.
 
-      request = Net::HTTP::Post.new(uri.path)
-      request.body = batch.to_json
-      request["Content-Type"] = "application/json"
-      request["Authorization"] = auth_token
+      validate_collection_name(collection_name)
 
-      response = Net::HTTP.start(uri.host, uri.port) {|http|
-        http.request(request)
-      }
+      unless timestamp
+        timestamp = Time.now
+      end
 
-      puts response
+      event_body[:_timestamp] = timestamp.utc.iso8601
 
-      # TODO DK:  need to send this batch of Keen::Storage::Item instances
-      # to API!  we can just use the first auth_token we find on an Item.
-      # If something fails, stick the item into the prior_failures queue
-      # using push
+      job = Keen::Async::Job.new(handler, {
+        :project_id => @project_id,
+        :auth_token => @auth_token,
+        :collection_name => collection_name,
+        :event_body => event_body,
+      })
+
+      job.save
     end
 
-    def self.process_queue_from_flat_file(options)
-      raise "this feature isn't supported yet!!!"
+    def validate_collection_name(collection_name)
+      # TODO
     end
+
   end
 end

data/lib/keen/version.rb

@@ -1,3 +1,3 @@
 module Keen
-  VERSION = "0.0.2"
+  VERSION = "0.0.4"
 end

data/test/keen_spec.rb

@@ -1,7 +1,7 @@
-$LOAD_PATH << File.join(File.dirname(__FILE__), '..', 'lib')
+$LOAD_PATH << File.join(File.dirname(__FILE__), "..", "lib")
 
-require 'keen'
-require 'fakeweb'
+require "keen"
+require "fakeweb"
 
 describe Keen::Client do
 
@@ -9,6 +9,24 @@ describe Keen::Client do
   before :all do
     FakeWeb.register_uri(:any, %r/https:\/\/api.keen.io\//, :body => '{"message": "You tried to reach Keen"}')
   end
+
+  describe "#add_event" do
+    project_id = "4f5775ad163d666a6100000e"
+    auth_token = "a5d4eaf432914823a94ecd7e0cb547b9"
+
+    keen = Keen::Client.new(project_id, auth_token, :storage_mode => :redis)
+
+    310.times do
+      keen.add_event("rspec_clicks", {
+        :hi => "you",
+      })
+    end
+
+    worker = Keen::Async::Worker.new(handler = keen.storage_handler)
+
+    worker.process_queue
+
+  end
   
   # TODO spec it out, lazy bones!
   #

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: keen
 version: !ruby/object:Gem::Version 
-  hash: 27
+  hash: 23
   prerelease: 
   segments: 
   - 0
   - 0
-  - 2
-  version: 0.0.2
+  - 4
+  version: 0.0.4
 platform: ruby
 authors: 
 - Kyle Wild
@@ -15,61 +15,77 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-03-16 00:00:00 -05:00
+date: 2012-03-19 00:00:00 -05:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
-  name: multi_json
+  name: json
   prerelease: false
   requirement: &id001 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 17
+        hash: 5
         segments: 
         - 1
-        - 0
-        - 3
-        version: 1.0.3
+        - 6
+        - 5
+        version: 1.6.5
   type: :runtime
   version_requirements: *id001
 - !ruby/object:Gem::Dependency 
-  name: system_timer
+  name: fakeweb
   prerelease: false
   requirement: &id002 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 23
+        hash: 27
         segments: 
         - 1
-        - 2
-        - 4
-        version: 1.2.4
+        - 3
+        - 0
+        version: 1.3.0
   type: :runtime
   version_requirements: *id002
 - !ruby/object:Gem::Dependency 
-  name: httparty
+  name: rspec
   prerelease: false
   requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 61
+        hash: 43
         segments: 
+        - 2
+        - 9
         - 0
-        - 8
-        - 1
-        version: 0.8.1
+        version: 2.9.0
   type: :runtime
   version_requirements: *id003
 - !ruby/object:Gem::Dependency 
-  name: redis
+  name: system_timer
   prerelease: false
   requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 23
+        segments: 
+        - 1
+        - 2
+        - 4
+        version: 1.2.4
+  type: :runtime
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: redis
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -81,7 +97,7 @@ dependencies:
         - 2
         version: 2.2.2
   type: :runtime
-  version_requirements: *id004
+  version_requirements: *id005
 description: See the github repo or examples.rb for usage information.
 email: 
 - kyle@keen.io
@@ -175,9 +191,11 @@ files:
 - examples/rails_2/Gemfile
 - keen.gemspec
 - lib/keen.rb
+- lib/keen/async/job.rb
+- lib/keen/async/storage/flat_file_handler.rb
+- lib/keen/async/storage/redis_handler.rb
+- lib/keen/async/worker.rb
 - lib/keen/client.rb
-- lib/keen/storage/flat_file_handler.rb
-- lib/keen/storage/redis_handler.rb
 - lib/keen/version.rb
 - test/keen_spec.rb
 has_rdoc: true

data/lib/keen/storage/flat_file_handler.rb

@@ -1,55 +0,0 @@
-require 'fileutils'
-require 'base64'
-require 'zlib'
-
-
-
-module Keen
-  module Storage
-    class FlatFileHandler
-
-      # Paths
-      # -----
-      
-      # Where new events go as they come in:
-      ACTIVE = "active.txt"
-
-      # The temporary file that will store records during processing:
-      PROCESSING = "processing.txt"
-
-      # Records are put here if processing fails:
-      FAILED = "failed.txt"
-
-      def initialize(filepath)
-        @filepath = filepath
-      
-        is_directory?
-        is_writable?
-        is_readable?
-      end
-
-      def dump_contents
-        # move
-      end
-
-      def is_writable?
-        if not FileTest.writable? @filepath
-          raise "Can't write to file: " + @filepath
-        end
-      end
-      
-      def is_readable?
-        if not FileTest.readable? @filepath
-          raise "Can't read from file: " + @filepath
-        end
-      end
-
-      def is_directory?
-        if not FileTest.directory? @filepath
-          raise "Can't read from file: " + @filepath
-        end
-      end
-
-    end
-  end
-end

data/lib/keen/storage/redis_handler.rb

@@ -1,155 +0,0 @@
-require 'fileutils'
-require 'redis'
-require 'json'
-
-module Keen
-  module Storage
-    class Item
-      # Represents one item in the Redis queue.
-      
-      def to_json(options)
-        @event_body.to_json
-      end
-
-      def to_s
-        self.to_json
-      end
-
-      def auth_token
-        @auth_token
-      end
-      
-      def initialize(definition={})
-        @project_id = definition[:project_id]
-        @auth_token = definition[:auth_token]
-        @collection_name = definition[:collection_name]
-        @event_body = definition[:event_body]
-        
-        # TODO: type checking?
-      end
-
-      def save
-        handler = Keen::Storage::RedisHandler.new
-        handler.add_event(self)
-      end
-
-    end
-
-    class ProjectBatch
-      # Represents a batch of records (for a given project)
-      def initialize(project_id, auth_token)
-        @project_id = project_id
-        @auth_token = auth_token
-        @items = []
-      end
-
-      def add_item(item)
-        @items.push(item)
-      end
-    end
-
-    class RedisHandler
-
-      # Keys
-      # ----
-
-      def global_key_prefix
-        "keen-" + Keen::VERSION
-      end
-      
-      def active_queue_key
-        "#{global_key_prefix}.active_queue_key"
-      end
-
-      def processing_queue_key
-        "#{global_key_prefix}.processing_queue_key"
-      end
-
-      def failed_queue_key
-        "#{global_key_prefix}.failed_queue_key"
-      end
-
-      def lock_active_queue_key
-        "lock" + active_queue_key
-      end
-
-      def lock_active_queue
-        # TODO: add locking
-        key = lock_active_queue_key
-      end
-
-      def unlock_active_queue
-        # TODO: add locking
-        key = lock_active_queue_key
-      end
-
-      def record_event(collection_name, event_body)
-        # TODO this is the main public method!
-        # must write..
-      end
-
-      def handle_prior_failures
-        # TODO consume the failed_queue and do something with it (loggly? retry? flat file?)
-      end
-
-      def initialize
-        @redis = Redis.new
-      end
-
-      def get_active_queue_contents
-        # get the list of jsonified hashes back:
-        list = @redis.get active_queue_key
-
-        if not list
-          []
-        end
-      end
-
-      def flush_active_queue
-        @redis.del active_queue_key
-      end
-
-      def set_processing_queue(queue)
-      end
-
-      def process_queue
-        handle_prior_failures
-
-        lock_active_queue
-
-        queue = get_active_queue_contents
-
-        flush_active_queue
-
-        set_processing_queue(queue)
-
-        unlock_active_queue
-
-        # translate the queue strings into Items
-        items = queue.map {|json| Keen::Storage::Item.new(JSON.parse json)}.compact
-
-        collated = collate_items(items)
-      end
-
-      def collate_items(queue)
-        collated = {}
-
-        # traverse backwards so the most recent auth tokens take precedent:
-        queue.reverse_each do |item_hash|
-          if not collated.has_key? item.project_id
-            collated[item.project_id] = {}
-          end
-
-          if not collated[item.project_id].has_key? item.collection_name
-            collated[item.project_id][item.collection_name] = Keen::Storage::ProjectBatch.new
-          end
-
-          collated[item.project_id][item.collection_name].add_item(item)
-        end
-
-        collated
-      end
-
-    end
-  end
-end