json_proxy 0.5.1

data/History.txt

@@ -0,0 +1,6 @@
+=== 1.0.0 / 2008-09-29
+
+* 1 major enhancement
+
+  * Birthday!
+

data/README.rdoc

@@ -0,0 +1,66 @@
+= Readme
+
+Json Proxy is a small server/dsl intending for serving JSON documents, and in particular JSON documents created from external webservices.
+JsonProxy supports
+* Caching - with either CouchDb or a simple Filesystem based cache
+* Asynchronous Operation - Json Proxy supports asynchronous data loading via a HTTP polling mechanism
+
+== Prequisites
+
+* Couchdb (For caching, alternative FileCache exists)
+* Starling (For queue)
+* Rack (For server)
+
+== DSL
+
+The JsonProxy DSL is used for creating new services
+
+  np_namespace "echo" do |ns|
+    ns.route 'echo', [:message] do |message|
+      {:message => message}
+    end
+  end
+
+Each route should return an object with a #to_json method.
+
+== Cache
+
+All queries are cached, the cache can be controlled by setting the cache_expiry and cache_key options.
+
+== JSON Envelope
+
+The response is always a JSON object, this object is an envelope around the data.  
+The data can be accessed through the data property of the response, the other properties
+of the response are used for meta-information such as the response code and response message.
+
+  http://localhost/echo/echo?message=blah&jsonp=callback  
+    => {status : 200, statusMessage : "Ok", data : {message : "blah" } }
+
+  
+== JSON-P 
+
+If the request has an argument 'jsonp' the server will prefix the response with the argument value, allowing the 
+response to be used to trigger callbacks.
+
+  http://localhost/echo/echo?message=blah&jsonp=callback  
+    => callback({status : 200, statusMessage : "Ok", data : {message : "blah" } })   
+
+== Asynchronous Operation
+
+JSON Proxy supports asynchronous operation by returning immediately if the data is not in the cache.  
+The server returns a 202 status code in this scenario.  The operation is then performed in the background, 
+while the client polls the server.  Once the operation has completed the data will be available in the cache and will be
+sent to the client on the next request.
+
+  Loop while status is 202
+    http://localhost/echo/echo?message=blah  
+      => {status : 202, statusMessage : "Processing. Please retry shortly" }
+  end
+  http://localhost/echo/echo?message=blah  
+    => {status : 200, statusMessage : "Ok", data : {message : "blah" } } 
+  
+Background processing is performed by using a shared queue (implemented using starling).  When a cache miss occurs the 
+server adds the current url to a queue including an extra force parameter.  The QueueDaemon gets urls from the queue and 
+fetches them.  The force parameter ensures that the operation is processed synchronously.
+
+

data/Rakefile

@@ -0,0 +1,29 @@
+require 'rubygems'
+begin
+  require 'rake'
+rescue LoadError
+  puts 'This script should only be accessed via the "rake" command.'
+  puts 'Installation: gem install rake -y'
+  exit
+end
+require 'rake'
+require 'rake/clean'
+require 'rake/packagetask'
+require 'rake_remote_task'
+require 'spec/rake/spectask'
+
+$:.unshift File.dirname(__FILE__) + "/lib"
+require 'json_proxy'
+
+$:.unshift File.dirname(__FILE__) + "/tasks/lib"
+
+role :app, "gandrew.com"
+role :rubyforge, "rubyforge.org"
+
+DEPLOY_ROOT = "/var/web/projects/json_proxy"
+RUBYFORGE_ROOT = "/var/www/gforge-projects/json-proxy/"
+
+Dir['tasks/**/*.rake'].each { |rake| load rake }
+
+
+

data/bin/json_proxy

@@ -0,0 +1,117 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'rack'
+require 'activesupport'
+require 'configatron'
+require 'choice'
+
+$:.unshift(File.dirname(__FILE__) + "/../lib") 
+require 'json_proxy'
+
+
+Choice.options do
+
+  footer ''
+  footer '  Example: json_proxy -p 8080 -c config/config.rb --config=lib/*.rb'
+  footer ''
+
+  
+  option :port do
+    short '-p'
+    long '--port=PORT'
+    desc 'The default port to listen on (default 4567)'
+    cast Integer
+    default 4567
+  end
+  
+  option :config do
+    short '-c'
+    long '--config=CONFIG'
+    desc 'Specify the configuration file'
+  end
+  
+  option :services do
+    short '-s'
+    long '--services=*SERVICES'
+    desc 'Specify the services you would like to run'
+  end
+
+  option :help do
+    long '--help'
+    desc 'Show this message'
+  end
+
+  option :version do
+    short '-v'
+    long '--version'
+    desc 'Show version'
+    action do
+      puts "#{JsonProxy::NAME}-#{JsonProxy::VERSION}"
+      exit
+    end
+  end
+  
+  option :workers do
+    short '-w'
+    long '--workers'
+    default 1
+  end
+end
+
+if Choice.choices[:config] 
+  require Choice.choices[:config]
+end
+
+if Choice.choices[:services] 
+  include Server::DSL
+  Choice.choices[:services].each do |service|
+    require service
+  end
+end
+
+
+appThread = Thread.new do
+
+  # Rack::ShowExceptions.new
+  app = Rack::Reloader.new Server::Server.new 
+
+  Rack::Handler::Mongrel.run(app, :Port => Choice.choices[:port]) do |server|
+    puts "==  json_proxy running on port #{Choice.choices[:port]}"
+    trap("INT") do
+      puts "\n== Good night."
+      exit()
+    end
+  end
+
+end
+
+workers = Choice.choices[:workers].to_i
+if workers > 0 
+  workers.times do 
+    
+    workerThread = Thread.new do
+      begin
+        worker = UrlQueue::QueueDaemon.new
+        puts "Worker started"
+        sleeping = false
+        loop do
+          processed = worker.process
+          if processed == 0 && !sleeping
+            sleeping = true
+        #    puts "Queue empty \n"
+          elsif processed > 0
+        #    puts "Processed #{processed}\n"
+            sleeping = false
+          end
+          sleep(1)
+        end
+      rescue Exception =>e
+        puts "Worker Exception: " + e
+        exit(1)
+      end
+    end
+  end
+end
+
+appThread.join

data/bin/json_proxy_worker

@@ -0,0 +1,28 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'rack'
+require 'activesupport'
+require 'configatron'
+require 'choice'
+
+$:.unshift(File.dirname(__FILE__) + "/../lib") 
+require 'json_proxy'
+
+worker = UrlQueue::QueueDaemon.new
+
+sleeping = false
+
+puts "Worker started"
+
+loop do
+  processed = worker.process
+  if processed == 0 && !sleeping
+    sleeping = true
+#    puts "Queue empty \n"
+  elsif processed > 0
+#    puts "Processed #{processed}\n"
+    sleeping = false
+  end
+  sleep(1)
+end

data/bin/json_proxy_worker~

@@ -0,0 +1,24 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'rack'
+require 'activesupport'
+require 'configatron'
+require 'choice'
+
+$:.unshift(File.dirname(__FILE__) + "/../lib") 
+require 'json_proxy'
+
+ puts "Worker started"
+
+loop do
+  processed = worker.process
+  if processed == 0 && !sleeping
+    sleeping = true
+#    puts "Queue empty \n"
+  elsif processed > 0
+#    puts "Processed #{processed}\n"
+    sleeping = false
+  end
+  sleep(1)
+end

data/bin/json_proxy~

@@ -0,0 +1,117 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'rack'
+require 'activesupport'
+require 'configatron'
+require 'choice'
+
+$:.unshift(File.dirname(__FILE__) + "/../lib") 
+require 'json_proxy'
+
+
+Choice.options do
+
+  footer ''
+  footer '  Example: json_proxy -p 8080 -c config/config.rb --config=lib/*.rb'
+  footer ''
+
+  
+  option :port do
+    short '-p'
+    long '--port=PORT'
+    desc 'The default port to listen on (default 4567)'
+    cast Integer
+    default 4567
+  end
+  
+  option :config do
+    short '-c'
+    long '--config=CONFIG'
+    desc 'Specify the configuration file'
+  end
+  
+  option :services do
+    short '-s'
+    long '--services=*SERVICES'
+    desc 'Specify the services you would like to run'
+  end
+
+  option :help do
+    long '--help'
+    desc 'Show this message'
+  end
+
+  option :version do
+    short '-v'
+    long '--version'
+    desc 'Show version'
+    action do
+      puts "#{JsonProxy::NAME}-#{JsonProxy::VERSION}"
+      exit
+    end
+  end
+  
+  option :workers do
+    short '-w'
+    long '--workers'
+    default 1
+  end
+end
+
+if Choice.choices[:config] 
+  require Choice.choices[:config]
+end
+
+if Choice.choices[:services] 
+  include Server::DSL
+  Choice.choices[:services].each do |service|
+    require service
+  end
+end
+
+
+appThread = Thread.new do
+
+  # Rack::ShowExceptions.new
+  app = Rack::Reloader.new Server::Server.new 
+
+  Rack::Handler::Mongrel.run(app, :Port => Choice.choices[:port]) do |server|
+    puts "==  json_proxy running on port #{Choice.choices[:port]}"
+    trap("INT") do
+      puts "\n== Good night."
+      exit()
+    end
+  end
+
+end
+
+workers = Choice.choices[:workers].to_i
+if workers > 0 
+  workers.times do 
+    
+    workerThread = Thread.new do
+      begin
+        worker = UrlQueue::QueueDaemon.new
+        puts "Worker started"
+        sleeping = false
+        loop do
+          processed = worker.process
+          if processed == 0 && !sleeping
+            sleeping = true
+        #    puts "Queue empty \n"
+          elsif processed > 0
+        #    puts "Processed #{processed}\n"
+            sleeping = false
+          end
+          sleep(1)
+        end
+      rescue Exception =>e
+        puts e
+        exit(1)
+      end
+    end
+  end
+end
+
+appThread.join

data/test/functional/server_spec.rb

@@ -0,0 +1,138 @@
+require 'rubygems'
+require 'net/http'
+require 'active_support'
+
+require File.dirname(__FILE__) + '/spec_helper'
+include Server::DSL
+require File.dirname(__FILE__) + '/test_service.rb'
+
+
+
+# Test basic server operation using echo server
+
+describe "Server" do
+
+  def start_server
+    app = Server::Server.new 
+    Thread.new {
+       @acc = Rack::Handler::Mongrel.run(app, :Port => 4567) do |server|
+        puts "Server running"     
+       end
+     }
+    sleep 2
+    
+  end
+  
+  def start_worker
+    Thread.new {
+      puts "Worker Started"
+      worker = UrlQueue::QueueDaemon.new
+      sleeping = false
+      loop do
+        processed = worker.process
+        if processed == 0 && !sleeping
+          sleeping = true
+          puts "Queue empty \n"
+        elsif processed > 0
+          puts "Processed #{processed}\n"
+          sleeping = false
+        end
+       sleep(1)
+      end
+    }
+    sleep 1
+  end
+
+  before(:all) do
+    start_server
+    start_worker
+  end
+  
+  
+  before(:each) do
+  end
+  
+  def get(msg)
+    Net::HTTP.get_response(URI.parse('http://localhost:4567/echo/echo.js?message='+msg))
+  end
+  
+  def force_get(msg)
+    Net::HTTP.get_response(URI.parse('http://localhost:4567/echo/echo.js?message='+msg+'&force=true'))
+  end
+  
+  def random_msg
+    "random-#{rand(9999999)}"
+  end
+  
+  describe "with missing service" do
+    it "response should be 404" do
+      response = Net::HTTP.get_response(URI.parse('http://localhost:4567/missing'))
+      response.code.should =="200"
+      response.body.should =="Handler not found"
+    end
+  end
+
+  describe "with echo service" do
+
+    describe "with cached message" do
+      before(:each) do
+          force_get "cached"
+      end
+    
+      it "response should == message " do
+        response = get("cached")
+        response.code.should == "200"
+        body = ActiveSupport::JSON.decode(response.body)
+        body.should be :kind_of, Hash
+        body['status'].should == 200
+        body['data'].should be :kind_of, Hash
+        body['data']['message'].should == "cached" 
+      end
+   
+    end
+    
+    describe "with uncached message" do
+    
+      it "response should be 202 - Processing until complete then success" do
+        msg = random_msg
+        response = get(msg)
+        response.code.should == "202"
+        response.body.should_not be(:empty)
+        body = ActiveSupport::JSON.decode(response.body)
+        body.should be :kind_of, Hash
+        body['status'].should == 202
+        count = 1;
+        maxCount = 10;
+        while response = get(msg)
+          puts "Retry #{count} \n"
+          if (response.code != "202" || count > maxCount)
+            break
+          end
+          count += 1
+          sleep(1)
+        end
+        response.code.should == "200"
+        body = ActiveSupport::JSON.decode(response.body)
+        body['status'].should == 200
+        body['data'].should be :kind_of, Hash
+        body['data']['message'].should == msg 
+      end
+   
+    end
+    
+    describe "with forced query" do
+
+      it "response should == message " do
+        msg = random_msg
+        response = force_get(msg)
+        response.code.should == "200"
+        body = ActiveSupport::JSON.decode(response.body)
+        body['status'].should == 200
+        body['data'].should be :kind_of, Hash
+        body['data']['message'].should == msg 
+      end
+    
+    end
+
+  end
+end