facets 1.1.0 → 1.2.0

data/lib/facets/more/dictionary.rb

@@ -0,0 +1,436 @@
+#--
+# Dictionary
+#
+# Copyright (c) 2005 Thomas Sawyer, Jan Molic
+# ported from OrderHash 2.0, Copyright (c) 2005 jan molic
+#
+# Ruby License
+#
+# This module is free software. You may use, modify, and/or redistribute this
+# software under the same terms as Ruby.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+# ==========================================================================
+#  History
+# --------------------------------------------------------------------------
+#  2005-12-09 trans
+#  * Added #order_by, with which one can set internal sort order.
+#  * Added #<<, takes a two element [k,v] array and inserts.
+#  * Added ::auto which creates Dictionay sub-entries as needed.
+#  * Added ::alpha which creates a new Dictionary sorted by key.
+#
+#  2005-10-26  trans
+#  * changed name of class to Dictionary
+#  * fixed #update method
+#  * added #dup and #insert
+#
+#  2055-02-01  trans
+#  * ported to calibre from jan molic's OrderHash
+# ==========================================================================
+#++
+
+#:title: Dictionary
+#
+# The Dictionary class is a Hash that preserves order.
+# So it has some array-like extensions also. By defualt
+# a Dictionary object preserves insertion order, but any 
+# order can be specified including alphabetical key order.
+#
+# == Usage
+#
+# Just require this file and use Dictionary instead of Hash.
+#
+#   # You can do simply
+#   hsh = Dictionary.new
+#   hsh['z'] = 1
+#   hsh['a'] = 2
+#   hsh['c'] = 3
+#   p hsh.keys     #=> ['z','a','c']
+#
+#   # or using Dictionary[] method
+#   hsh = Dictionary['z', 1, 'a', 2, 'c', 3] 
+#   p hsh.keys     #=> ['z','a','c']
+#
+#   # but this don't preserve order
+#   hsh = Dictionary['z'=>1, 'a'=>2, 'c'=>3]
+#   p hsh.keys     #=> ['a','c','z']
+#
+#   # Dictionary has useful extensions: push, pop and unshift
+#   p hsh.push('to_end', 15)       #=> true, key added
+#   p hsh.push('to_end', 30)       #=> false, already - nothing happen
+#   p hsh.unshift('to_begin', 50)  #=> true, key added
+#   p hsh.unshift('to_begin', 60)  #=> false, already - nothing happen
+#   p hsh.keys                     #=> ["to_begin", "a", "c", "z", "to_end"]
+#   p hsh.pop                      #=> ["to_end", 15], if nothing remains, return nil
+#   p hsh.keys                     #=> ["to_begin", "a", "c", "z"]
+#   p hsh.shift                    #=> ["to_begin", 30], if nothing remains, return nil
+#
+# == Author(s)
+#
+# * jan molic
+# * trans
+#
+# == Thanks
+#
+# Andrew Johnson for his suggestions and fixes of Hash[], merge,
+# to_a, inspect and shift.
+#
+
+class Dictionary < Hash
+
+  class << self
+
+    def []( *args )
+      hsh = new
+      if Hash === args[0]
+        hsh.replace( args[0] )
+      elsif (args.size % 2) != 0
+        raise ArgumentError, "odd number of elements for Hash"
+      else
+        hsh[args.shift] = args.shift while args.size > 0
+      end
+      hsh
+    end
+
+    # Like #new but the block sets the order.
+    #
+    def new_by( *args, &blk )
+      new(*args).order_by(&blk)
+    end
+
+    # Alternate to #new which creates a dictionary sorted by key.
+    #
+    #   d = Dictionary.key_new
+    #   d["z"] = 1
+    #   d["y"] = 2
+    #   d["x"] = 3
+    #   d  #=> {"x"=>3,"y"=>2,"z"=>2}
+    #
+    # This is equivalent to:
+    #
+    #   Dictionary.new.order_by { |key,value| key }
+    #
+    # An alias #alpha is also provided.
+    def key_new( *args, &block )
+      new(*args, &block).order_by{ |k,v| k }
+    end
+    alias_method :alpha, :key_new
+
+    # Alternate constructor which sorts by value.
+    #
+    #   d = Dictionary.value_new
+    #   d["z"] = 1
+    #   d["y"] = 2
+    #   d["x"] = 3
+    #   d  #=> {"x"=>3,"y"=>2,"z"=>2}
+    #
+    # This is equivalent to:
+    #
+    #   Dictionary.new.order_by { |key,value| value }
+    #
+    def value_new( *args, &block )
+      new(*args, &block).order_by{ |k,v| v }
+    end
+
+    # Alternate to #new which auto-creates sub-dictionaries as needed.
+    #
+    #   d = Dictionary.auto
+    #   d["a"]["b"]["c"] = "abc"  #=> { "a"=>{"b"=>{"c"=>"abc"}}}
+    #
+    def auto( *args )
+      leet = lambda { |hsh, key| hsh[key] = new( &leet ) }
+      new(*args, &leet)
+    end
+
+    def key_auto
+      leet = lambda { |hsh, key| hsh[key] = new( &leet ) }
+      new(*args, &leet).order_by{ |k,v| k }
+    end
+
+  end
+
+  attr_accessor :order
+
+  def initialize( *args, &blk )
+    @order = []
+    @order_by = nil
+    super
+  end
+
+  def order
+    reorder if @order_by
+    @order
+  end
+
+  def order_by( &block )
+    @order_by = block
+    order
+    self
+  end
+
+  def reorder
+    if @order_by
+      assoc = @order.collect{ |k| [k,self[k]] }.sort_by( &@order_by )
+      @order = assoc.collect{ |k,v| k }
+    end
+    @order
+  end
+
+  def store_only( a,b )
+    store( a,b )
+  end
+
+  alias_method :orig_store, :store
+  protected :orig_store
+
+  def insert( i,k,v )
+    @order.insert( i,k )
+    orig_store( k,v )
+  end
+
+  alias_method :unordered_store, :store
+  private :unordered_store
+
+  def store( a,b )
+    @order.push( a ) unless has_key? a
+    super( a,b )
+  end
+  alias []= store
+
+  def ==( hsh2 )
+    return false if @order != hsh2.order
+    super hsh2
+  end
+
+  def clear
+    @order = []
+    super
+  end
+
+  def delete( key )
+    @order.delete key
+    super
+  end
+
+  def each_key
+    order.each { |k| yield( k ) }
+    self
+  end
+
+  def each_value
+    order.each { |k| yield( self[k] ) }
+    self
+  end
+
+  #alias_method :unordered_each, :each
+
+  def each
+    order.each { |k| yield( k,self[k] ) }
+    self
+  end
+  alias each_pair each
+
+  def delete_if
+    order.clone.each { |k| delete k if yield }
+    self
+  end
+
+  def values
+    ary = []
+    order.each { |k| ary.push self[k] }
+    ary
+  end
+
+  def keys
+    order
+  end
+
+  def invert
+    hsh2 = self.class.new
+    order.each { |k| hsh2[self[k]] = k }
+    hsh2
+  end
+
+  def reject( &block )
+    self.dup.delete_if &block
+  end
+
+  def reject!( &block )
+    hsh2 = reject &block
+    self == hsh2 ? nil : hsh2
+  end
+
+  def replace( hsh2 )
+    @order = hsh2.keys
+    super hsh2
+  end
+
+  def shift
+    key = order.first
+    key ? [key,delete(key)] : super
+  end
+
+  def unshift( k,v )
+    unless self.include? k
+      @order.unshift k
+      orig_store(k,v)
+      true
+    else
+      false
+    end
+  end
+
+  def <<(kv)
+    push *kv
+  end
+
+  def push( k,v )
+    unless self.include? k
+      @order.push k
+      orig_store(k,v)
+      true
+    else
+      false
+    end
+  end
+
+  def pop
+    key = order.last
+    key ? [key,delete(key)] : nil
+  end
+
+  def to_a
+    ary = []
+    each { |k,v| ary << [k,v] }
+    ary
+  end
+
+  def to_s
+    self.to_a.to_s
+  end
+
+  def inspect
+    ary = []
+    each {|k,v| ary << k.inspect + "=>" + v.inspect}
+    '{' + ary.join(", ") + '}'
+  end
+
+  def dup
+    self.class[*to_a.flatten]
+  end
+
+  def update( hsh2 )
+    hsh2.each { |k,v| self[k] = v }
+    reorder
+    self
+  end
+  alias :merge! update
+
+  def merge hsh2
+    self.dup.update(hsh2)
+  end
+
+  def select
+    ary = []
+    each { |k,v| ary << [k,v] if yield k,v }
+    ary
+  end
+
+end
+
+
+
+#  _____         _
+# |_   _|__  ___| |_
+#   | |/ _ \/ __| __|
+#   | |  __/\__ \ |_
+#   |_|\___||___/\__|
+#
+
+=begin testing
+
+  require 'test/unit'
+
+  class TC_Dictionary < Test::Unit::TestCase
+
+    def test_create
+      hsh = Dictionary['z', 1, 'a', 2, 'c', 3] 
+      assert_equal( ['z','a','c'], hsh.keys )
+    end
+
+    def test_op_store
+      hsh = Dictionary.new
+      hsh['z'] = 1
+      hsh['a'] = 2
+      hsh['c'] = 3
+      assert_equal( ['z','a','c'], hsh.keys )
+    end
+
+    def test_push
+      hsh = Dictionary['a', 1, 'c', 2, 'z', 3]
+      assert( hsh.push('end', 15) )
+      assert_equal( 15, hsh['end'] )
+      assert( ! hsh.push('end', 30) )
+      assert( hsh.unshift('begin', 50) )
+      assert_equal( 50, hsh['begin'] )
+      assert( ! hsh.unshift('begin', 60) )
+      assert_equal( ["begin", "a", "c", "z", "end"], hsh.keys )
+      assert_equal( ["end", 15], hsh.pop )
+      assert_equal( ["begin", "a", "c", "z"], hsh.keys )
+      assert_equal( ["begin", 50], hsh.shift )
+    end
+
+    def test_insert
+      # front
+      h = Dictionary['a', 1, 'b', 2, 'c', 3]
+      r = Dictionary['d', 4, 'a', 1, 'b', 2, 'c', 3]
+      assert_equal( 4, h.insert(0,'d',4) )
+      assert_equal( r, h )
+      # back
+      h = Dictionary['a', 1, 'b', 2, 'c', 3]
+      r = Dictionary['a', 1, 'b', 2, 'c', 3, 'd', 4]
+      assert_equal( 4, h.insert(-1,'d',4) )
+      assert_equal( r, h )
+    end
+
+    def test_update
+      # with other orderred hash
+      h1 = Dictionary['a', 1, 'b', 2, 'c', 3]
+      h2 = Dictionary['d', 4]
+      r = Dictionary['a', 1, 'b', 2, 'c', 3, 'd', 4]
+      assert_equal( r, h1.update(h2) )
+      assert_equal( r, h1 )
+      # with other hash
+      h1 = Dictionary['a', 1, 'b', 2, 'c', 3]
+      h2 = { 'd' => 4 }
+      r = Dictionary['a', 1, 'b', 2, 'c', 3, 'd', 4]
+      assert_equal( r, h1.update(h2) )
+      assert_equal( r, h1 )
+    end
+
+    def test_merge
+      # with other orderred hash
+      h1 = Dictionary['a', 1, 'b', 2, 'c', 3]
+      h2 = Dictionary['d', 4]
+      r = Dictionary['a', 1, 'b', 2, 'c', 3, 'd', 4]
+      assert_equal( r, h1.merge(h2) )
+      # with other hash
+      h1 = Dictionary['a', 1, 'b', 2, 'c', 3]
+      h2 = { 'd' => 4 }
+      r = Dictionary['a', 1, 'b', 2, 'c', 3, 'd', 4]
+      assert_equal( r, h1.merge(h2) )
+    end
+
+    def test_order_by
+      h1 = Dictionary['a', 3, 'b', 2, 'c', 1]
+      h1.order_by{ |k,v| v }
+      assert_equal( [1,2,3], h1.values )
+      assert_equal( ['c','b','a'], h1.keys )
+    end
+
+  end
+
+=end

data/lib/facets/more/downloader.rb

@@ -0,0 +1,272 @@
+#--
+# Downloader
+#
+# Copyright (c) 2005 Thomas Sawyer
+#
+# Ruby License
+#
+# This module is free software. You may use, modify, and/or redistribute this
+# software under the same terms as Ruby.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+# ==========================================================================
+# REVISION HISTORY
+# ==========================================================================
+#  2005-09-28 trans
+#  Moved to Calibre from Baker project.
+# ==========================================================================
+#
+#++
+
+require 'digest/md5'
+require 'open-uri'
+
+require 'facet/string/unbracket'
+require 'calib/progressbar'
+
+# :title: Downloader
+#
+# Tool module for downloading files and extracting
+# archive files. Currently this is console-based
+# but in the future will have SOC for any interface.
+
+class Downloader
+
+  #class DownloadError < StandardError ; end
+  class Checksum < StandardError ; end
+
+  class Mirror
+    def initialize( url, region=nil )
+      @url = url
+      @region = region
+      @interface = nil
+    end
+    attr_accessor :url, :region
+  end
+
+  # delegate to an interface (under development)
+
+  def interface ; @interface; end
+  def interface=(iface)
+    @interface = iface
+  end
+
+  def initialize( destination, mirrors=nil, region=nil, &config )
+    @destination = destination
+    @mirrors = []
+    add_mirrors( mirrors ) if mirrors
+    @region = (region || 'US').to_s
+    config.call(self) if config
+  end
+
+  attr_accessor :destination, :region
+
+  def add_mirror( url, region=nil )
+    @mirrors << Mirror.new( url, region )
+  end
+
+  def add_mirrors( mirrors )
+    mirrors.each { |mirror|
+      case mirror
+      when String
+        url, rgn = *mirror.strip.split(' ')
+        add_mirror( url, rgn.unbracket )
+      when Array
+        add_mirror( *mirror )
+      when Hash
+        add_mirror( mirror[:url], mirror[:region] )
+      when Mirror
+        @mirrors << mirror
+      else
+        raise "unrecogized mirror definition #{mirror.inspect}"
+      end
+    }
+  end
+
+  def mirrors
+    @mirrors.sort{ |a,b| a.region == region ? 1 : ( b.region == region ? -1 : 0 ) }
+  end
+
+  # fetch
+
+  def fetch( file, checksum=0, est_size=0, force=false )
+    urls = mirrors.collect { |m| "#{m.url.chomp('/')}/#{file}" }
+    urls = prioritize_urls( urls )
+    filepath = "#{destination.chomp('/')}/#{file}"
+    monitored_download( urls, filepath, checksum, est_size, force )
+  end
+
+#   def download_verbose
+#     @download_verbose ||= $VERBOSE
+#     true
+#   end
+# 
+#   def download_verbose=(x)
+#     @download_verbose = x ? true : false
+#   end
+
+  # regional_urls - array of arrays of [ url, region, md5, expected_size ]
+  # local_region - region of the user's system
+  # to_dir - where to store downloaded file (full path)
+  # force - download even if file already exists locally
+
+  def monitored_download( urls, filepath, checksum, est_size=0, force=false )
+
+    checksum = checksum.to_s.strip
+    est_size = nil if est_size == 0
+
+    success=nil
+
+    # source file exists and passes checksum then we need not fetch
+    #file_path = File.join(to_dir,File.basename(url[0]))
+    if File.exists?(filepath)
+      if compute_checksum(filepath) == checksum and ! force
+        interface.report("File has already been fetched and passes checksum.")
+        success = filepath
+      else
+        File.delete(filepath)
+      end
+    end
+
+    unless success
+      # download
+      urls.each do |url|
+        begin
+          #file_path = File.join(to_dir,File.basename(url[0]))
+          #file_checksum = url[2].to_s.strip
+          #file_size = url[3].to_i
+          success = self.download( url, filepath, checksum, est_size )
+          break if success
+        rescue
+          next
+        end
+      end
+    end
+
+    return success
+  end
+
+  # In the future we may test each connection for fastest download
+
+  def prioritize_urls( urls )
+    urls
+    # put local region first
+    #prioritized_urls = regional_urls.find_all { |a| a[1] == local_region }
+    #prioritized_urls.concat regional_urls.find_all { |a| a[1] != local_region }
+    #return prioritized_urls
+  end
+
+  # currently can only download a single compressed file
+  # does not handle downloading an uncompressed directory tree (should it? doubt it)
+  #
+  # currently this displays progress to STDOUT; either their should
+  # be a way to activate/deactivate or preferably use ducktype singletons
+  # (more on that later, see google://_whytheluckystiff if interested)
+  # of course I prefer chain messaging but matz said no :(
+
+  def download( url, filepath, checksum='', est_size=nil )
+
+    checksum = checksum.to_s.strip
+    est_size = nil if est_size == 0
+
+    download_complete = nil
+
+    if interface
+      interface.preparing_to_download( File.basename( filepath ), url, est_size )
+    end
+
+    progress_total = est_size ? est_size : 100000000  # pretend 100MB if no size
+    pbar = Console::ProgressBar.new( "Status", progress_total, STDOUT )
+    pbar.bar_mark = "="
+    pbar.format = "%-6s %3d%% %s %s"
+    pbar.file_transfer_mode if est_size
+
+    progress_proc = proc { |posit| pbar.set(posit) }
+
+    STDOUT.sync = true
+    begin
+      local_file = File.open( filepath, 'w' )
+      remote_file = open( url, :progress_proc => progress_proc )
+      local_file << remote_file.read
+    rescue
+      pbar.halt
+      download_complete = nil
+      raise
+    else
+      pbar.finish
+      download_complete = filepath
+    ensure
+      remote_file.close unless remote_file.nil?
+      local_file.close unless local_file.nil?
+      STDOUT.sync = false
+    end
+
+    unless checksum.empty?
+      raise ChecksumError if compute_checksum(filepath) != checksum
+    end
+
+    if interface
+      if checksum.empty?
+        interface.lacks_checksum( compute_checksum(filepath), :md5 )
+      end
+      unless est_size
+        interface.lacks_size( File.size(filepath) )
+      end
+    end
+
+    if download_complete
+      if interface
+        interface.downloaded( filepath )
+      end
+    end
+
+    return download_complete
+  end
+
+  # compute_checksum
+
+  def compute_checksum( local_path )
+    if File.exists?( local_path )
+      File.open( local_path ) do |local_file|
+        return Digest::MD5.new( local_file.read ).hexdigest
+      end
+    end
+  end
+
+  # extract
+
+  def extract( local_path )
+    success = false
+    local_dir = File.dirname(local_path)
+    local_file = File.basename(local_path)
+    current_dir = Dir.getwd
+    begin
+      Dir.chdir(local_dir)
+      case local_file
+        when /.*gz$/
+          system "tar -xzf #{local_file}"
+        when /.*bz2$/
+          system "tar -xjf #{local_file}"
+        when /.zip$/
+          system "unzip #{local_file}"
+        else
+          success = false
+      end
+    rescue
+      success = false
+    else
+      success = true
+    ensure
+      Dir.chdir(current_dir)
+    end
+    if interface
+      interface.extracted( local_path)
+    end
+    return success
+  end
+
+end
+