concord_cacher 0.0.1

data/Manifest

@@ -0,0 +1,19 @@
+README.textile
+Rakefile
+lib/concord_cacher.rb
+lib/concord/cacher.rb
+lib/concord/diy_local_cacher.rb
+lib/concord/java_proxy_cacher.rb
+spec/data/element_reference.otml
+spec/data/empty.otml
+spec/data/recursion.otml
+spec/data/resources/chart_line.png
+spec/data/resources/delete.png
+spec/data/resources/recurse1.otml
+spec/data/resources/recurse2.otml
+spec/data/resources/text.txt
+spec/data/standard_uri.otml
+spec/diy_local_cacher_spec.rb
+spec/helpers/cache_helper.rb
+spec/java_proxy_cacher_spec.rb
+Manifest

data/README.textile

@@ -0,0 +1,27 @@
+h1. concord_cacher
+
+The concord_cacher gem provides support for locally caching a resource and all referenced resources in multiple different ways.
+It is intended for using with other Concord Consortium projects and not necessarily for outside projects.
+
+h3. install
+
+<pre><code>
+$ gem install concord_cacher
+</code></pre>
+
+h3. example
+
+<pre><code>
+$ irb
+> require 'rubygems'
+> require 'concord_cacher'
+>
+> cacher = Concord::JavaProxyCacher.new(:url => "http://some/url/to/cache", :cache_dir => "/some/local/dir/to/store/things")
+> cacher.cache
+</code></pre>
+
+h3. tests
+
+<pre><code>
+$ rake spec
+</code></pre>

data/Rakefile

@@ -0,0 +1,35 @@
+require 'rubygems'
+
+require 'rake'
+require 'spec/rake/spectask'
+
+require './lib/concord_cacher.rb'
+
+require 'echoe'
+Echoe.new('concord_cacher', '0.0.1') do |p|
+  p.description    = "concord_cacher provides support for locally caching a resource and all referenced resources in multiple different ways. It is intended for using with other Concord Consortium projects and not necessarily for outside projects."
+  p.summary        = "Support for locally caching a resource and all referenced resources in multiple different ways"
+  p.url            = "http://github.com/psndcsrv/concord_cacher"
+  p.author         = "Aaron Unger"
+  p.email          = "aunger @nospam@ concord.org"
+  p.ignore_pattern = ["tmp/*","pkg/*"]
+  p.development_dependencies = []
+end
+
+task :default => :spec
+Spec::Rake::SpecTask.new do |t|
+  t.spec_files = FileList["spec/**/*_spec.rb"]
+end
+
+namespace :hudson do
+  task :spec => ["hudson:setup:rspec", 'rake:spec']
+
+  namespace :setup do
+    task :pre_ci do
+      ENV["CI_REPORTS"] = 'spec/reports/'
+      gem 'ci_reporter'
+      require 'ci/reporter/rake/rspec'
+    end
+    task :rspec => [:pre_ci, "ci:setup:rspec"]
+  end
+end

data/concord_cacher.gemspec

@@ -0,0 +1,32 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{concord_cacher}
+  s.version = "0.0.1"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Aaron Unger"]
+  s.cert_chain = ["/Users/aunger/gem-public_cert.pem"]
+  s.date = %q{2010-03-25}
+  s.description = %q{concord_cacher provides support for locally caching a resource and all referenced resources in multiple different ways. It is intended for using with other Concord Consortium projects and not necessarily for outside projects.}
+  s.email = %q{aunger @nospam@ concord.org}
+  s.extra_rdoc_files = ["README.textile", "lib/concord_cacher.rb", "lib/concord/cacher.rb", "lib/concord/diy_local_cacher.rb", "lib/concord/java_proxy_cacher.rb"]
+  s.files = ["README.textile", "Rakefile", "lib/concord_cacher.rb", "lib/concord/cacher.rb", "lib/concord/diy_local_cacher.rb", "lib/concord/java_proxy_cacher.rb", "spec/data/element_reference.otml", "spec/data/empty.otml", "spec/data/recursion.otml", "spec/data/resources/chart_line.png", "spec/data/resources/delete.png", "spec/data/resources/recurse1.otml", "spec/data/resources/recurse2.otml", "spec/data/resources/text.txt", "spec/data/standard_uri.otml", "spec/diy_local_cacher_spec.rb", "spec/helpers/cache_helper.rb", "spec/java_proxy_cacher_spec.rb", "Manifest", "concord_cacher.gemspec"]
+  s.homepage = %q{http://github.com/psndcsrv/concord_cacher}
+  s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Concord_cacher", "--main", "README.textile"]
+  s.require_paths = ["lib"]
+  s.rubyforge_project = %q{concord_cacher}
+  s.rubygems_version = %q{1.3.6}
+  s.signing_key = %q{/Users/aunger/gem-private_key.pem}
+  s.summary = %q{Support for locally caching a resource and all referenced resources in multiple different ways}
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+    else
+    end
+  else
+  end
+end

data/lib/concord/cacher.rb

@@ -0,0 +1,234 @@
+class ::Concord::Cacher
+  require 'rubygems'
+  require 'open-uri'
+  require 'cgi'
+  require 'rexml/document'
+  
+  DEBUG = false
+
+  #   scan for anything that matches (http://[^'"]+)
+  URL_REGEX = /(http[s]?:\/\/[^'"]+)/i
+  # the imageBytes can be referenced by a OTImage object
+  SRC_REGEX = /(?:src|href|imageBytes)[ ]?=[ ]?['"]([^'"]+)/i
+  NLOGO_REGEX = /import-drawing "([^"]+)"/i
+  ALWAYS_SKIP_REGEX = /^(mailto|jres)/i   # (resourceFile =~ /^mailto/) || (resourceFile =~ /^jres/)
+  RECURSE_ONCE_REGEX = /html$/i  # (resourceFile =~ /otml$/ || resourceFile =~ /html/)
+  RECURSE_FOREVER_REGEX = /(otml|cml|mml|nlogo)$/i
+  
+  attr_reader :otml_url, :cache_dir, :uuid, :errors
+  
+  def initialize(opts = {})
+    defaults = {:rewrite_urls => false, :verbose => false, :cache_headers => true, :create_map => true}
+    opts = defaults.merge(opts)
+    raise ArgumentError, "Must include :url, and :cache_dir in the options hash." unless opts[:url] && opts[:cache_dir]
+    @rewrite_urls = opts[:rewrite_urls]
+    @cache_dir = opts[:cache_dir]
+    @verbose = opts[:verbose]
+    @cache_headers = opts[:cache_headers]
+    @create_map = opts[:create_map]
+    url = opts[:url]
+    @filename = File.basename(url, ".otml")
+    @content = ""
+    open(url) do |r|
+      @content_headers = r.respond_to?("meta") ? r.meta : {}
+      @content_headers['_http_version'] = "HTTP/1.1 #{r.respond_to?("status") ? r.status.join(" ") : "200 OK"}"
+      @content = r.read
+    end
+    @uuid = generate_uuid
+    if (URI.parse(url).kind_of?(URI::HTTP))
+      @otml_url = url
+    else
+      # this probably references something on the local fs. we need to extract the document's codebase, if there is ony
+      if @content =~ /<otrunk[^>]+codebase[ ]?=[ ]?['"]([^'"]+)/
+        # @otml_url = "#{$1}/#{@filename}.otml"
+        @otml_url = "#{$1}"
+        @content.sub!(/codebase[ ]?=[ ]?['"][^'"]+['"]/,"")
+      else
+        @otml_url = url
+      end
+    end
+    
+    @otml_url.sub!(/[^\/]+$/,"")
+
+  	@errors = {}
+  	
+  	@url_to_hash_map = {}
+	end
+	
+	def cache
+	  copy_otml_to_local_cache
+  	
+  	write_url_to_hash_map if @create_map
+	end
+	
+	def generate_main_filename
+	  raise NotImplementedError, "You should be using this class through one of its sub-classes!"
+	end
+	
+	def generate_filename(opts = {})
+	  raise NotImplementedError, "You should be using this class through one of its sub-classes!"
+  end
+  
+  def generate_uuid
+    raise NotImplementedError, "You should be using this class through one of its sub-classes!"
+  end
+  
+  def copy_otml_to_local_cache
+    # save the file in the local server directories
+    filename = generate_main_filename
+
+    # open the otml file from the specified url or grab the embedded content
+    uri = URI.parse(@otml_url)
+    if uri.relative?
+      # we need the main URI to be absolute so that we can use it to resolve references
+      file_root = URI.parse("file:///")
+      uri = file_root.merge(uri)
+    end
+    @content = parse_file("#{@cache_dir}#{@filename}", @content, @cache_dir, uri, true)
+    
+    write_resource(@cache_dir + filename, @content)
+    write_property_map(@cache_dir + filename + ".hdrs", @content_headers) if @cache_headers
+    @url_to_hash_map[@otml_url + @filename + ".otml"] = filename
+
+    puts "\nThere were #{@errors.length} artifacts with errors.\n" if @verbose
+    @errors.each do |k,v|
+    	puts "In #{k}:" if @verbose
+    	v.uniq.each do |e|
+        puts "    #{e}" if @verbose
+      end
+    end
+  end
+  
+  def parse_file(orig_filename, content, cache_dir, parent_url, recurse)
+    short_filename = /\/([^\/]+)$/.match(orig_filename)[1]
+    print "\n#{short_filename}: " if @verbose
+    processed_lines = []
+    lines = content.split("\n")
+    lines.each do |line|
+      line = CGI.unescapeHTML(line)
+      match_indexes = []
+      while ( ((match = URL_REGEX.match(line)) && (! match_indexes.include?(match.begin(1)))) ||
+              ((match = SRC_REGEX.match(line)) && (! match_indexes.include?(match.begin(1)))) ||
+              (/.*\.nlogo/.match(short_filename) && (match = NLOGO_REGEX.match(line)) && (! match_indexes.include?(match.begin(1))))  )
+        print "\nMatched url: #{match[1]}: " if DEBUG
+        match_indexes << match.begin(1)
+        #   get the resource from that location, save it locally
+        match_url = match[1].gsub(/\s+/,"").gsub(/[\?\#&;=\+\$,<>"\{\}\|\\\^\[\]].*$/,"")
+        # puts("pre: #{match[1]}, post: #{match_url}") if DEBUG
+        begin
+          resource_url = URI.parse(CGI.unescapeHTML(match_url))
+        rescue
+          @errors[parent_url] ||= []
+        @errors[parent_url] << "Bad URL: '#{CGI.unescapeHTML(match_url)}', skipping."
+          print 'x' if @verbose
+          next
+        end
+        if (resource_url.relative?)
+          # relative URL's need to have their parent document's codebase appended before trying to download
+          resource_url = parent_url.merge(resource_url.to_s)
+        end
+        resourceFile = match_url
+        resourceFile = resourceFile.gsub(/http[s]?:\/\//,"")
+        resourceFile = resourceFile.gsub(/\/$/,"")
+
+        if (resourceFile.length < 1) || ALWAYS_SKIP_REGEX.match(resourceFile)
+          print "S" if @verbose
+          next
+        end
+        
+      	begin
+          resource_content = ""
+          resource_headers = {}
+          open(resource_url.scheme == 'file' ? resource_url.path : resource_url.to_s) do |r|
+            resource_headers = r.respond_to?("meta") ? r.meta : {}
+            resource_headers['_http_version'] = "HTTP/1.1 #{r.respond_to?("status") ? r.status.join(" ") : "200 OK"}"
+            resource_content = r.read
+          end
+				rescue OpenURI::HTTPError, Timeout::Error, Errno::ENOENT => e
+          @errors[parent_url] ||= []
+          @errors[parent_url] << "Problem getting file: #{resource_url.to_s},   Error: #{e}"
+          print 'X' if @verbose
+					next
+				end
+
+        localFile = generate_filename(:content => resource_content, :url => resource_url)
+        @url_to_hash_map[resource_url.to_s] = localFile
+        line.sub!(match_url.to_s,localFile.to_s) if @rewrite_urls
+        
+        
+        # skip downloading already existing files.
+        # because we're working with sha1 hashes we can be reasonably certain the content is a complete match
+        if File.exist?(cache_dir + localFile)
+          print 's' if @verbose
+        else
+          # if it's an otml/html file, we should parse it too (only one level down)
+          if (recurse && (RECURSE_ONCE_REGEX.match(resourceFile) || RECURSE_FOREVER_REGEX.match(resourceFile)))
+							puts "recursively parsing '#{resource_url.to_s}'" if DEBUG
+							recurse_further = false
+							if RECURSE_FOREVER_REGEX.match(resourceFile)
+							  recurse_further = true
+						  end
+							begin
+                resource_content = parse_file(cache_dir + resourceFile, resource_content, cache_dir, resource_url, recurse_further)
+							rescue OpenURI::HTTPError => e
+                @errors[parent_url] ||= []
+                @errors[parent_url] << "Problem getting or writing file: #{resource_url.to_s},   Error: #{e}"
+                print 'X' if @verbose
+								next
+							end
+          end
+          begin
+            write_resource(cache_dir + localFile, resource_content)
+            write_property_map(cache_dir + localFile + ".hdrs", resource_headers) if @cache_headers
+            print "." if @verbose
+          rescue Exception => e
+            @errors[parent_url] ||= []
+            @errors[parent_url] << "Problem getting or writing file: #{resource_url.to_s},   Error: #{e}"
+            print 'X' if @verbose
+          end
+        end
+      end
+      processed_lines << line
+    end
+
+    print ".\n" if @verbose
+    return processed_lines.join("\n")
+  end
+  
+  def write_resource(filename, content)
+    f = File.new(filename, "w")
+    f.write(content)
+    f.flush
+    f.close
+  end
+  
+  def write_url_to_hash_map
+    load_existing_map if (File.exists?(@cache_dir + "url_map.xml"))
+    write_property_map(@cache_dir + "url_map.xml", @url_to_hash_map)
+  end
+    
+  def write_property_map(filename, hash_map)
+    File.open(filename, "w") do |f|
+      f.write('<?xml version="1.0" encoding="UTF-8"?>' + "\n")
+      f.write('<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">' + "\n")
+      f.write('<properties>' + "\n")
+      hash_map.each do |url,hash|
+        f.write("<entry key='#{CGI.escapeHTML(url)}'>#{hash}</entry>\n")
+      end
+      f.write('</properties>' + "\n")
+      f.flush
+    end
+  end
+  
+  def load_existing_map
+    map_content = ::REXML::Document.new(File.new(@cache_dir + "url_map.xml")).root
+    map_content.elements.each("entry") do |entry|
+      k = entry.attributes["key"]
+      if ! (@url_to_hash_map.include? k)
+        val = entry.text
+        @url_to_hash_map[k] = val
+        # puts "Adding previously defined url: #{k}  =>  #{val}" if DEBUG
+      end
+    end
+  end
+end

data/lib/concord/diy_local_cacher.rb

@@ -0,0 +1,31 @@
+class ::Concord::DiyLocalCacher < ::Concord::Cacher
+  require 'uri'
+  require 'digest/sha1'
+  
+  def initialize(opts = {})
+    raise InvalidArgumentError, "Must include :activity in the options hash." unless opts[:activity]
+    @activity = opts[:activity]
+    opts[:cache_headers] ||= false
+    opts[:create_map] ||= false
+    opts[:rewrite_urls] ||= true
+    super
+  end
+  
+  def generate_main_filename
+    "#{generate_uuid}.otml"
+  end
+  
+  def generate_uuid
+    @activity.uuid
+  end
+  
+  def generate_filename(opts = {})
+    raise InvalidArgumentError, "Must include :url key in opts" unless opts[:url]
+    url = opts[:url]
+    if url.kind_of?(::URI) && url.scheme == 'file'
+      url = url.path
+    end
+    url = url.to_s
+    return ::Digest::SHA1.hexdigest(url)
+  end
+end

data/lib/concord/java_proxy_cacher.rb

@@ -0,0 +1,16 @@
+class ::Concord::JavaProxyCacher < ::Concord::Cacher
+  require 'digest/sha1'
+  
+  def generate_main_filename
+    generate_filename(:content => @content)
+  end
+  
+  def generate_uuid
+    generate_filename(:content => @content)
+  end
+  
+  def generate_filename(opts = {})
+    raise InvalidArgumentError, "Must include :content key in opts" unless opts[:content]
+    ::Digest::SHA1.hexdigest(opts[:content])
+  end
+end

data/lib/concord_cacher.rb

@@ -0,0 +1,8 @@
+$:.unshift(File.dirname(__FILE__)) unless
+  $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
+
+module Concord
+  require 'concord/cacher'
+  require 'concord/diy_local_cacher'
+  require 'concord/java_proxy_cacher'
+end

data/spec/data/element_reference.otml

@@ -0,0 +1,22 @@
+<otrunk id="67daad67-8762-4f5a-8c7c-50b7c3544c0b">
+  <imports>
+    <import class="org.concord.otrunk.OTSystem" />
+  </imports>
+  <objects>  
+    <OTSystem local_id="system">
+      <root>
+        <OTText>
+          <!-- Absolute references -->
+          <a href="http://loops.diy.concord.org/">Visit the Concord Website</a>
+          <img src="http://portal.concord.org/images/icons/chart_bar.png" />
+          <OTBlob imageBytes="http://portal.concord.org/images/icons/chart_pie.png" />
+          
+          <!-- relative references -->
+          <a href="resources/text.txt">Read some text</a>
+          <img src="resources/delete.png" />
+          <OTBlob imageBytes="resources/chart_line.png" />
+        </OTText>
+      </root>
+    </OTSystem>
+  </objects>
+</otrunk>

data/spec/data/empty.otml

@@ -0,0 +1,15 @@
+<!--
+  This is a pseudo-otml file which holds all sorts of ways to reference a file.
+  It won't actually load in OTrunk, and is only really useful to test positive results -
+  i.e. that the cacher correctly finds and caches all of the various urls represented here.
+  It won't allow us to detect if the cacher is caching unexpected URLS.
+-->
+<otrunk id="de74ee26-53a6-4edd-b4b1-2f2c5ac7ec21">
+  <imports>
+    <import class="org.concord.otrunk.OTSystem" />
+  </imports>
+  <objects>  
+    <OTSystem local_id="system">
+    </OTSystem>
+  </objects>
+</otrunk>

data/spec/data/recursion.otml

@@ -0,0 +1,14 @@
+<otrunk id="6b230c1d-8816-4ef3-9c7a-b84a0da0b35f">
+  <imports>
+    <import class="org.concord.otrunk.OTSystem" />
+  </imports>
+  <objects>  
+    <OTSystem local_id="system">
+      <root>
+        <OTText>
+          <a href="resources/recurse1.otml">Recursive otml</a>
+        </OTText>
+      </root>
+    </OTSystem>
+  </objects>
+</otrunk>

data/spec/data/resources/recurse1.otml

@@ -0,0 +1,15 @@
+<otrunk id="6b230c1d-8816-4ef3-9c7a-b84a0da0b35f">
+  <imports>
+    <import class="org.concord.otrunk.OTSystem" />
+  </imports>
+  <objects>  
+    <OTSystem local_id="system">
+      <root>
+        <OTText>
+          <a href="recurse2.otml">Recursive otml</a>
+          <img src="delete.png" />
+        </OTText>
+      </root>
+    </OTSystem>
+  </objects>
+</otrunk>

data/spec/data/resources/recurse2.otml

@@ -0,0 +1,14 @@
+<otrunk id="6b230c1d-8816-4ef3-9c7a-b84a0da0b35f">
+  <imports>
+    <import class="org.concord.otrunk.OTSystem" />
+  </imports>
+  <objects>  
+    <OTSystem local_id="system">
+      <root>
+        <OTText>
+          <OTBlob imageBytes="chart_line.png" />
+        </OTText>
+      </root>
+    </OTSystem>
+  </objects>
+</otrunk>

data/spec/data/resources/text.txt

@@ -0,0 +1 @@
+This is some text.

data/spec/data/standard_uri.otml

@@ -0,0 +1,17 @@
+<otrunk id="6b230c1d-8816-4ef3-9c7a-b84a0da0b35f">
+  <imports>
+    <import class="org.concord.otrunk.OTSystem" />
+  </imports>
+  <objects>  
+    <OTSystem local_id="system">
+      <root>
+        <OTText>
+          <!-- normal http -->
+          http://portal.concord.org/images/icons/delete.png
+          <!-- https -->
+          https://mail.google.com/mail/images/2/5/mountains/base/gmail_solid_white.png
+        </OTText>
+      </root>
+    </OTSystem>
+  </objects>
+</otrunk>

data/spec/diy_local_cacher_spec.rb

@@ -0,0 +1,225 @@
+begin
+  require 'concord_cacher'
+rescue LoadError
+  require File.join(File.dirname(__FILE__), '..','lib','concord.rb')
+end
+
+require File.join(File.dirname(__FILE__),'helpers','cache_helper.rb')
+
+require 'fileutils'
+
+include FileUtils
+
+require 'openssl'
+module OpenSSL
+  module SSL
+	  remove_const :VERIFY_PEER
+	end
+end
+OpenSSL::SSL::VERIFY_PEER = OpenSSL::SSL::VERIFY_NONE
+
+SPEC_ROOT = File.expand_path(File.dirname(__FILE__))
+
+describe 'DIY Local Cacher' do
+  include CacheHelper
+  
+  before(:each) do
+    @klass = Concord::DiyLocalCacher
+    @cache = File.join(SPEC_ROOT, '..', 'tmp','diy_local')
+    mkdir_p(@cache)
+    @cache += '/'
+  end
+  
+  def mockup(file)
+    return mock('activity',{:uuid => 'hash', :url => file})
+  end
+  
+  after(:each) do
+    rm_rf(@cache)
+  end
+  
+  describe 'empty otml' do
+    it 'should not create a url map xml file' do
+      cache('empty.otml', :activity => mockup('empty.otml'))
+      does_not_exist?('url_map.xml')
+    end
+  
+    it 'should create a cached file of the original url' do
+      url = File.join(SPEC_ROOT,'data','empty.otml')
+      cache('empty.otml', :activity => mockup('empty.otml'))
+      exists?('hash.otml')
+    end
+  
+    it 'should not create a cached header of the original url' do
+      url = File.join(SPEC_ROOT,'data','empty.otml')
+      expected_filename = 'hash.otml'
+      cache('empty.otml', :activity => mockup('empty.otml'))
+      does_not_exist?("#{expected_filename}.hdrs")
+    end
+  end
+  
+  describe 'standard uri syntax' do
+    it 'should cache 2 referenced files' do
+      expected_files = []
+      expected_files << 'hash.otml' # standard_uri.otml
+      expected_files << ::Digest::SHA1.hexdigest('http://portal.concord.org/images/icons/delete.png')
+      expected_files << ::Digest::SHA1.hexdigest('https://mail.google.com/mail/images/2/5/mountains/base/gmail_solid_white.png')
+
+      cache('standard_uri.otml', :activity => mockup('standard_uri.otml'))
+      
+      cache_size.should == 3
+      expected_files.each do |f|
+        exists?(f)
+      end
+      
+      
+    end
+    
+    it 'should rewrite the urls in the main otml file' do
+      cache('standard_uri.otml', :activity => mockup('standard_uri.otml'))
+      
+      file_content = File.read(File.join(@cache,'hash.otml'))
+      
+      file_content.should_not match(/http:/)
+      file_content.should match(::Digest::SHA1.hexdigest('http://portal.concord.org/images/icons/delete.png'))
+      file_content.should match(::Digest::SHA1.hexdigest('https://mail.google.com/mail/images/2/5/mountains/base/gmail_solid_white.png'))
+    end
+  end
+  
+  describe 'element references syntax' do
+    it 'should cache 6 referenced files' do
+      expected_files = []
+      expected_files << 'hash.otml' # element_reference.otml
+      expected_files << ::Digest::SHA1.hexdigest('http://loops.diy.concord.org/')
+      expected_files << ::Digest::SHA1.hexdigest('http://portal.concord.org/images/icons/chart_bar.png')
+      expected_files << ::Digest::SHA1.hexdigest('http://portal.concord.org/images/icons/chart_pie.png')
+      expected_files << ::Digest::SHA1.hexdigest(File.join(SPEC_ROOT,'data','resources','text.txt'))
+      expected_files << ::Digest::SHA1.hexdigest(File.join(SPEC_ROOT,'data','resources','delete.png'))
+      expected_files << ::Digest::SHA1.hexdigest(File.join(SPEC_ROOT,'data','resources','chart_line.png'))
+      
+      cache('element_reference.otml', :activity => mockup('element_reference.otml'))
+      
+      cache_size.should == 7
+      expected_files.each do |f|
+        exists?(f)
+      end
+    end
+    
+    it 'should rewrite the urls in the main otml file' do
+      expected_urls = []
+      unexpected_urls = []
+      unexpected_urls << 'http://loops.diy.concord.org/'
+      unexpected_urls << 'http://portal.concord.org/images/icons/chart_bar.png'
+      unexpected_urls << 'http://portal.concord.org/images/icons/chart_pie.png'
+      unexpected_urls << File.join('resources','text.txt')
+      unexpected_urls << File.join('resources','delete.png')
+      unexpected_urls << File.join('resources','chart_line.png')
+      
+      unexpected_urls.each do |url|
+        if url =~ /^http/
+          expected_urls << ::Digest::SHA1.hexdigest(url)
+        else
+          expected_urls << ::Digest::SHA1.hexdigest(File.join(SPEC_ROOT,'data',url))
+        end
+      end
+      
+      cache('element_reference.otml', :activity => mockup('element_reference.otml'))
+      
+      file_content = File.read(File.join(@cache,'hash.otml'))
+      
+      unexpected_urls.each do |url|
+        file_content.should_not match(Regexp.new(url))
+      end
+      
+      expected_urls.each do |url|
+        file_content.should match(Regexp.new(url))
+      end
+    end
+  end
+  
+  describe 'recursive references' do
+    it 'should cache 4 referenced files in otml files' do
+      expected_files = []
+      expected_files << 'hash.otml' # recursion.otml
+      expected_files << ::Digest::SHA1.hexdigest(File.join(SPEC_ROOT,'data','resources','recurse1.otml'))
+      expected_files << ::Digest::SHA1.hexdigest(File.join(SPEC_ROOT,'data','resources','delete.png'))
+      expected_files << ::Digest::SHA1.hexdigest(File.join(SPEC_ROOT,'data','resources','recurse2.otml'))
+      expected_files << ::Digest::SHA1.hexdigest(File.join(SPEC_ROOT,'data','resources','chart_line.png'))
+      
+      cache('recursion.otml', :activity => mockup('recursion.otml'))
+      
+      cache_size.should == 5
+      expected_files.each do |f|
+        exists?(f)
+      end
+    end
+    
+    it 'should rewrite urls in first level recursion otml' do
+      recurse_otml = ::Digest::SHA1.hexdigest(File.join(SPEC_ROOT,'data','resources','recurse1.otml'))
+      
+      expected_urls = []
+      unexpected_urls = []
+      
+      unexpected_urls << File.join('resources','recurse2.otml')
+      unexpected_urls << File.join('resources','delete.png')
+      
+      unexpected_urls.each do |url|
+        expected_urls << ::Digest::SHA1.hexdigest(File.join(SPEC_ROOT,'data',url))
+      end
+      
+      cache('recursion.otml', :activity => mockup('recursion.otml'))
+      
+      file_content = File.read(File.join(@cache,recurse_otml))
+      
+      unexpected_urls.each do |url|
+        file_content.should_not match(Regexp.new(url))
+      end
+      
+      expected_urls.each do |url|
+        file_content.should match(Regexp.new(url))
+      end
+    end
+    
+    it 'should rewrite urls in second level recursion otml' do
+      recurse_otml = ::Digest::SHA1.hexdigest(File.join(SPEC_ROOT,'data','resources','recurse2.otml'))
+      
+      expected_urls = []
+      unexpected_urls = []
+
+      unexpected_urls << File.join('resources','chart_line.png')
+      
+      unexpected_urls.each do |url|
+        expected_urls << ::Digest::SHA1.hexdigest(File.join(SPEC_ROOT,'data',url))
+      end
+      
+      cache('recursion.otml', :activity => mockup('recursion.otml'))
+      
+      file_content = File.read(File.join(@cache,recurse_otml))
+      
+      unexpected_urls.each do |url|
+        file_content.should_not match(Regexp.new(url))
+      end
+      
+      expected_urls.each do |url|
+        file_content.should match(Regexp.new(url))
+      end
+    end
+  end
+  
+  describe 'embedded nlogo files' do
+    it 'should correctly download resources referenced from within netlogo model files'
+  end
+  
+  describe 'embedded mw files' do
+    it 'should correctly download resources referenced from within mw model files'
+  end
+  
+  describe 'never cache' do
+    it 'should always skip mailto and jres references'
+  end
+  
+  describe 'recursion limits' do
+    it 'should only recurse html files once'
+    it 'should recurse otml,cml,mml and nlogo files forever'
+  end
+end

data/spec/helpers/cache_helper.rb

@@ -0,0 +1,21 @@
+module CacheHelper
+  def cache(file, opts = {})
+    options = {:url => File.join(SPEC_ROOT,'data',file), :cache_dir => @cache, :verbose => false}.merge(opts)
+    cacher = @klass.new(options)
+    cacher.cache
+  end
+
+  def exists?(file)
+    f = File.join(@cache,file)
+    File.should be_exists(f)
+  end
+  
+  def does_not_exist?(file)
+    f = File.join(@cache,file)
+    File.should_not be_exists(f)
+  end
+
+  def cache_size
+    Dir.glob(@cache + "/*").size
+  end
+end

data/spec/java_proxy_cacher_spec.rb

@@ -0,0 +1,138 @@
+begin
+  require 'concord_cacher'
+rescue LoadError
+  require File.join(File.dirname(__FILE__), '..','lib','concord.rb')
+end
+
+require File.join(File.dirname(__FILE__),'helpers','cache_helper.rb')
+
+require 'fileutils'
+
+include FileUtils
+
+require 'openssl'
+module OpenSSL
+  module SSL
+	  remove_const :VERIFY_PEER
+	end
+end
+OpenSSL::SSL::VERIFY_PEER = OpenSSL::SSL::VERIFY_NONE
+
+SPEC_ROOT = File.expand_path(File.dirname(__FILE__))
+
+
+describe 'Java Proxy Cacher' do
+  include CacheHelper
+  
+  before(:each) do
+    @klass = Concord::JavaProxyCacher
+    @cache = File.join(SPEC_ROOT, "..", 'tmp','java_proxy')
+    mkdir_p(@cache)
+    @cache += '/'
+  end
+  
+  after(:each) do
+    rm_rf(@cache)
+  end
+  
+  describe 'empty otml' do
+    it 'should create a url map xml file' do
+      cache('empty.otml')
+      exists?('url_map.xml')
+    end
+  
+    it 'should create a cached file of the original url' do
+      url = File.join(SPEC_ROOT,'data','empty.otml')
+      expected_filename = ::Digest::SHA1.hexdigest(File.read(url))
+      cache('empty.otml')
+      exists?(expected_filename)
+    end
+  
+    it 'should create a cached header of the original url' do
+      url = File.join(SPEC_ROOT,'data','empty.otml')
+      expected_filename = ::Digest::SHA1.hexdigest(File.read(url))
+      cache('empty.otml')
+      exists?("#{expected_filename}.hdrs")
+    end
+  end
+  
+  describe 'standard uri syntax' do
+    it 'should cache 2 referenced files' do
+      expected_files = []
+      expected_files << 'e954312036485d3ca1894265922d9bd9491bf59e' # standard_uri.otml
+      expected_files << '8f0ebcb45d7ba71a541d4781329f4a6900c7ee65' # http://portal.concord.org/images/icons/delete.png
+      expected_files << '21b8b442e4449f642fcbd6796f4f0f937ec6c70d' # https://mail.google.com/mail/images/2/5/mountains/base/gmail_solid_white.png
+      expected_files << expected_files.collect{|f| f+".hdrs" } # headers for each file
+      expected_files.flatten!
+      expected_files << 'url_map.xml'
+
+      cache('standard_uri.otml')
+      
+      cache_size.should == 7
+      expected_files.each do |f|
+        exists?(f)
+      end
+    end
+  end
+  
+  describe 'element references syntax' do
+    it 'should cache 6 referenced files' do
+      expected_files = []
+      expected_files << '9f945e576290efa874842b4ee07ab437d9d94a67' # element_reference.otml
+      expected_files << 'd9a2565586307e2924c953dfe788154749e93799' # http://loops.diy.concord.org/
+      expected_files << '4e9576a56db3d142113b8905d7aa93e31c9f441b' # http://portal.concord.org/images/icons/chart_bar.png
+      expected_files << '41f082b7e69a399679a47acfdcd7e7a204e49745' # http://portal.concord.org/images/icons/chart_pie.png
+      expected_files << 'cbe7ac86926fd3b8aa8659842a1d8c299d8966a7' # resources/text.txt
+      expected_files << '8f0ebcb45d7ba71a541d4781329f4a6900c7ee65' # resources/delete.png
+      expected_files << 'd1cea238486aeeba9215d56bf71efc243754fe48' # resources/chart_line.png
+      expected_files << expected_files.collect{|f| f+".hdrs" } # headers for each file
+      expected_files.flatten!
+      expected_files << 'url_map.xml'
+      
+      cache('element_reference.otml')
+      
+      cache_size.should == 15
+      expected_files.each do |f|
+        exists?(f)
+      end
+    end
+  end
+  
+  describe 'recursive references' do
+    it 'should cache 4 referenced files in otml files' do
+      expected_files = []
+      expected_files << 'dbbd46b446a205047cfbf32e7af350a73c38848d' # recursion.otml
+      expected_files << 'cdc3d425b0ac9c3e89e1b79e0ad8a07c09bcedbd' # resources/recurse1.otml
+      expected_files << '8f0ebcb45d7ba71a541d4781329f4a6900c7ee65' # resources/delete.png
+      expected_files << '10f39c75f40386e8fbbb9320b6e77f3bd12b0f1d' # resources/recurse2.otml
+      expected_files << 'd1cea238486aeeba9215d56bf71efc243754fe48' # resources/chart_line.png
+      expected_files << expected_files.collect{|f| f+".hdrs" } # headers for each file
+      expected_files.flatten!
+      expected_files << 'url_map.xml'
+      
+      cache('recursion.otml')
+      
+      cache_size.should == 11
+      expected_files.each do |f|
+        exists?(f)
+      end
+    end
+  end
+  
+  describe 'embedded nlogo files' do
+    it 'should correctly download resources referenced from within netlogo model files'
+  end
+  
+  describe 'embedded mw files' do
+    it 'should correctly download resources referenced from within mw model files'
+  end
+  
+  describe 'never cache' do
+    it 'should always skip mailto and jres references'
+  end
+  
+  describe 'recursion limits' do
+    it 'should only recurse html files once'
+    it 'should recurse otml,cml,mml and nlogo files forever'
+  end
+end

data.tar.gz.sig

@@ -0,0 +1 @@
+��j��oZ=y��/��A���d@��ǃ($��*]>Zv�r���	�&V�c��Csأ���n��f�
'���9�yw�d��G�������.��E�����âf�I�J�єe)V��N[1�

metadata

@@ -0,0 +1,112 @@
+--- !ruby/object:Gem::Specification 
+name: concord_cacher
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Aaron Unger
+autorequire: 
+bindir: bin
+cert_chain: 
+- |
+  -----BEGIN CERTIFICATE-----
+  MIIDMjCCAhqgAwIBAgIBADANBgkqhkiG9w0BAQUFADA/MQ8wDQYDVQQDDAZhdW5n
+  ZXIxFzAVBgoJkiaJk/IsZAEZFgdjb25jb3JkMRMwEQYKCZImiZPyLGQBGRYDb3Jn
+  MB4XDTEwMDMyNTE1NDM0MloXDTExMDMyNTE1NDM0MlowPzEPMA0GA1UEAwwGYXVu
+  Z2VyMRcwFQYKCZImiZPyLGQBGRYHY29uY29yZDETMBEGCgmSJomT8ixkARkWA29y
+  ZzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMcObwOz2qlPRc/6PVHq
+  7rrPxt+Ltc9shUtXrIjD0mQMqZtln6TkFnilPAXUd/NbxLrvZqrpPL5GeE5SlKA6
+  Hy2J5cj1A0c3qHmskaPnVma6NpnHmlWkIj1u//yCjz2ktaFYV7dHN2IlRK9M8t7F
+  Wt8DeVclSOeq3jOhjoOsSX5rIN8/MsWZcq42deHHiHQ0R5htLb1rmZTFKJynKA+m
+  Is6vytLGO/2iDeuiW8FQxBUwDEc7LNpfANCNk4WudUGi6K1WeteqTpyzJcso2yVZ
+  aMLpviD8WUxUZahz226ThwVLoTJB70GM8BVOFXApNdvbM0b45d1neyeHyU2f2ZeF
+  9g8CAwEAAaM5MDcwCQYDVR0TBAIwADALBgNVHQ8EBAMCBLAwHQYDVR0OBBYEFGQn
+  5Lsh1f/5AryTzU2cmoKsh6XqMA0GCSqGSIb3DQEBBQUAA4IBAQAfrvFWSUZPQnWg
+  5JxdAaHDMdVRlJGOTTpL/2L9GcDr8jaG32AOMcRDzhcCwKE5oBQZixUCidcvlh1y
+  8diuKpwKZrFtZT3RVwZm3wNHk9TaSprCJkhjhATLWZ2h9bGoTA2FH7VLpacUcG9r
+  L7W8Y8jL7G4+TZYCLcnH5O3XQ7AkAYVYogt4DLYN9Ma7xwN3MAFfuCRJhY49XLYd
+  Wv+hK/ewN+n0Uub+jPpW1Ahi7Ah67RVIRpsWfNpEjx+CB4hPY2GwjpmIKhgt4Fyg
+  c90zsqVWrMWBRgRiSl0yYsmuGSPmllNn2m9Vq4ZAHpjCBv5E0AObBv0g/WQgiMjX
+  8kT2T2VF
+  -----END CERTIFICATE-----
+
+date: 2010-03-25 00:00:00 -04:00
+default_executable: 
+dependencies: []
+
+description: concord_cacher provides support for locally caching a resource and all referenced resources in multiple different ways. It is intended for using with other Concord Consortium projects and not necessarily for outside projects.
+email: aunger @nospam@ concord.org
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README.textile
+- lib/concord_cacher.rb
+- lib/concord/cacher.rb
+- lib/concord/diy_local_cacher.rb
+- lib/concord/java_proxy_cacher.rb
+files: 
+- README.textile
+- Rakefile
+- lib/concord_cacher.rb
+- lib/concord/cacher.rb
+- lib/concord/diy_local_cacher.rb
+- lib/concord/java_proxy_cacher.rb
+- spec/data/element_reference.otml
+- spec/data/empty.otml
+- spec/data/recursion.otml
+- spec/data/resources/chart_line.png
+- spec/data/resources/delete.png
+- spec/data/resources/recurse1.otml
+- spec/data/resources/recurse2.otml
+- spec/data/resources/text.txt
+- spec/data/standard_uri.otml
+- spec/diy_local_cacher_spec.rb
+- spec/helpers/cache_helper.rb
+- spec/java_proxy_cacher_spec.rb
+- Manifest
+- concord_cacher.gemspec
+has_rdoc: true
+homepage: http://github.com/psndcsrv/concord_cacher
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --line-numbers
+- --inline-source
+- --title
+- Concord_cacher
+- --main
+- README.textile
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 1
+      - 2
+      version: "1.2"
+requirements: []
+
+rubyforge_project: concord_cacher
+rubygems_version: 1.3.6
+signing_key: 
+specification_version: 3
+summary: Support for locally caching a resource and all referenced resources in multiple different ways
+test_files: []
+