canonicurl 0.0.2

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in canonicurl.gemspec
+gemspec

data/Rakefile

@@ -0,0 +1,2 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks

data/canonicurl.gemspec

@@ -0,0 +1,23 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "canonicurl/version"
+
+Gem::Specification.new do |s|
+  s.name        = "canonicurl"
+  s.version     = Canonicurl::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["David Dai"]
+  s.email       = ["david.github@gmail.com"]
+  s.homepage    = "https://github.com/newtonapple/canonicurl"
+  s.summary     = %q{A Canonical URL cache using Redis}
+  s.description = %q{}
+
+  s.rubyforge_project = "canonicurl"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+  s.add_dependency 'redis', '>= 2.2.0'
+  s.add_dependency 'em-http-request', '>= 1.0.0.beta.3'
+end

data/lib/canonicurl/cache.rb

@@ -0,0 +1,123 @@
+require 'digest/md5'
+require 'redis'
+require 'em-http'
+
+module Canonicurl  
+  class Cache
+    CANONICAL  = 'C'
+    ERROR      = 'E'
+    LOCKED     = 'L'
+    RESOLVING  = 'R'
+
+    TTL = 60 * 60 * 24 * 90 # 90 days ~ 3 months
+    REDIRECTS = 5
+    CONNECTION_TIMEOUT = 5
+    KEY_PREFIX = 'curl:'
+
+    attr_accessor :db, :ttl, :timeout, :redirects
+    attr_reader :key_prefix
+
+    def self.url(code_or_url)
+      code_or_url && code_or_url.size > 1 ? code_or_url : url
+    end
+
+
+    def initialize(options={})
+      @db         = options[:db] || Redis.connect
+      @ttl        = options[:ttl] || TTL
+      @timeout    = options[:timeout] || CONNECTION_TIMEOUT
+      @redirects  = options[:redirects] || REDIRECTS
+      @key_prefix = options[:key_prefix] || KEY_PREFIX
+    end
+
+
+    def get(url)
+      @db.get key(url)
+    end
+
+
+    def fetch(url, callbacks={})
+      k = key(url)
+      @db.setnx(k, LOCKED) #  lock it if key doesn't exist
+
+      result = @db.get(k)
+      if !result.nil? && result.size > 1
+        return result
+
+      end
+
+      case result
+      when CANONICAL
+        yield url
+      when LOCKED
+        resolve(url, k, callbacks)
+        RESOLVING
+      else
+        result
+      end
+    end
+
+
+    def set(url, canonical_url, url_key=nil)
+      url_key = url_key || key(url)
+      if url == canonical_url
+        @db.setex(url_key, @ttl, CANONICAL)
+      else
+        @db.setex(url_key, @ttl, canonical_url)
+        @db.setex(key(canonical_url), @ttl, CANONICAL) # preemptively set the canonical_url
+      end
+    end
+
+
+    def key(url)
+      "#{@key_prefix}#{Digest::MD5.hexdigest(url)}"
+    end
+
+
+    private
+
+      def resolve(url, url_key, callbacks)
+        em_already_running = true
+        @db.set(url_key, RESOLVING)
+        em do |running|
+          em_already_running = running
+          http = EM::HttpRequest.new(url,
+                  :connection_timeout => @timeout,
+                  :inactivity_timeout => @timeout * 2).get(:redirects => @redirects)
+          http.callback {
+            status = http.response_header.status.to_i
+            case status
+            when 200...300
+              canonical_url = http.last_effective_url.to_s
+              set url, canonical_url, url_key
+              callbacks[:resolved].call(canonical_url, http) if callbacks[:resolved]
+            else
+              @db.set url_key, (status / 100).to_s
+              callbacks[:failed].call(http) if callbacks[:failed]
+            end
+            EM.stop unless em_already_running
+          }    
+          http.errback  {
+            @db.set(url_key, ERROR) 
+            callbacks[:error].call(http) if callbacks[:error]
+            EM.stop unless em_already_running
+          }
+        end
+      rescue Exception => e
+        @db.set(url_key, ERROR)
+        callbacks[:exception].call(e) if callbacks[:exception]
+        EM.stop unless em_already_running
+      end
+
+
+      def em
+        if EM.reactor_running? 
+          yield true
+        else
+          EM.run do
+            yield false
+          end
+        end
+      end
+  end
+end

data/lib/canonicurl/version.rb

@@ -0,0 +1,3 @@
+module Canonicurl
+  VERSION = "0.0.2"
+end

data/lib/canonicurl.rb

@@ -0,0 +1,5 @@
+module Canonicurl
+end
+
+require 'canonicurl/cache'
+require 'canonicurl/version'

metadata

@@ -0,0 +1,105 @@
+--- !ruby/object:Gem::Specification 
+name: canonicurl
+version: !ruby/object:Gem::Version 
+  hash: 27
+  prerelease: 
+  segments: 
+  - 0
+  - 0
+  - 2
+  version: 0.0.2
+platform: ruby
+authors: 
+- David Dai
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-04-06 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: redis
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 7
+        segments: 
+        - 2
+        - 2
+        - 0
+        version: 2.2.0
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: em-http-request
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 62196357
+        segments: 
+        - 1
+        - 0
+        - 0
+        - beta
+        - 3
+        version: 1.0.0.beta.3
+  type: :runtime
+  version_requirements: *id002
+description: ""
+email: 
+- david.github@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- Gemfile
+- Rakefile
+- canonicurl.gemspec
+- lib/canonicurl.rb
+- lib/canonicurl/cache.rb
+- lib/canonicurl/version.rb
+homepage: https://github.com/newtonapple/canonicurl
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: canonicurl
+rubygems_version: 1.7.2
+signing_key: 
+specification_version: 3
+summary: A Canonical URL cache using Redis
+test_files: []
+