cloudlib 0.3.10

data/bin/cloudlib-web

@@ -0,0 +1,216 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'digest/sha1'
+require 'sinatra'  # sinatra gem
+require 'tempfile'
+require 'fileutils'
+require 'cloudlib'
+require 'highline/import'       # highline gem
+
+# check that required environment variables are set
+envvars = ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "CLOUDLIB_LIBRARY_NAME", "CLOUDLIB_WEB_USERNAME", "CLOUDLIB_WEB_PASSWORD"]
+envvars.each do |var|
+  unless ENV[var]
+    ENV[var] = ask("#{var}: ", String) { |q| q.echo = if var == "AWS_SECRET_ACCESS_KEY" || var == "CLOUDLIB_WEB_PASSWORD" then "*" else true end }
+  end
+end
+Cloudlib::Entry.connect(ENV['CLOUDLIB_LIBRARY_NAME'], ENV['AWS_ACCESS_KEY_ID'], ENV['AWS_SECRET_ACCESS_KEY'])
+
+use Rack::Auth::Basic do |username, password|
+  username == ENV['CLOUDLIB_WEB_USERNAME'] &&
+  password == ENV['CLOUDLIB_WEB_PASSWORD']
+end
+
+get '/stylesheet.css' do
+  content_type 'text/css', :charset => 'utf-8'
+  sass :stylesheet
+end
+
+get '/' do
+  @token, @entries = "", []
+  @query = ""
+  haml :index
+end
+
+post '/' do
+  if params[:query]
+     @query = params[:query]
+     @token, @entries = Cloudlib::Entry.query(@query, 10, params[:token])
+  else
+     @token, @entries = "", []
+     @query = ""
+  end
+  haml :index
+end
+
+get '/upload' do
+  @error = params[:error]
+  haml :upload
+end
+
+post '/upload' do
+  tempfile = params[:fileToUpload][:tempfile]
+  tempfilepath = tempfile.path
+  tempfile.close
+  origname = params[:fileToUpload][:filename] || ""
+  if origname == ""
+    redirect "/upload?error=Please+select+a+file+to+upload."
+  else
+    entry = Cloudlib::Entry.from_file(tempfilepath, params[:fileToUpload][:filename])
+    FileUtils.rm tempfilepath
+    set_attributes_from_form(entry)
+    entry.save
+    redirect "/"
+  end
+end
+
+get '/*/bibtex' do
+  @entry = Cloudlib::Entry.find_by_name(params[:splat][0])
+  content_type 'text/plain', :charset => 'utf-8'
+  @entry.to_bibtex
+end
+
+get '/*' do
+  name = "#{params[:splat][0]}"
+  @entry = Cloudlib::Entry.find_by_name(name)
+  haml :modify
+end
+
+post '/*' do
+  entry = Cloudlib::Entry.find_by_name(params[:splat][0])
+  set_attributes_from_form(entry)
+  entry.save
+  redirect '/'
+end
+
+delete '/*' do
+  entry = Cloudlib::Entry.find_by_name(params[:splat][0])
+  entry.delete
+  redirect '/'
+end
+
+def field_for_type?(field, type)
+  Cloudlib::Entry.fields(type).member?(field)
+end
+
+def show_fields(type)
+  cmds = Cloudlib::Entry.fields.map do |field|
+    "document.getElementById('#{field.to_s}').setAttribute('style', 'display: #{if field_for_type?(field, type) then 'all' else 'none' end}'); "
+  end
+  return cmds.join
+end
+
+def set_attributes_from_form(entry)
+  entry.attributes['entry_type'] = params['entry_type']
+  Cloudlib::Entry.fields.each do |field|
+    if field_for_type?(field, params['entry_type']) && params[field]
+       entry.set_attribute(field.to_s, params[field])
+    else
+       entry.set_attribute(field.to_s, '')
+    end
+  end
+end
+
+use_in_file_templates!
+
+__END__
+
+@@ layout
+!!! Strict
+%head
+  %link{:href => '/stylesheet.css', :type => 'text/css', :media => 'all', :rel => 'stylesheet'}
+  %title
+    = ENV['CLOUDLIB_LIBRARY_NAME']
+%body
+  %h1
+    %a{:href => '/'}
+      = ENV['CLOUDLIB_LIBRARY_NAME']
+  %div#content
+    = yield
+  %div#footer
+    powered by
+    %a{:href => 'http://github.com/jgm/cloudlib/tree/master'}cloudlib
+
+@@ index
+%div.queryform
+  %form{:method => 'POST', :action => '/'}
+    %input{:type => 'text', :name => 'query', :value => @query, :size => '30'}
+    %input{:type => 'submit', :value => 'Search'}
+  %ol
+    - @entries.each do |i|
+      %li
+        = i.to_s
+        %a{:href => "/#{i.name}/bibtex"}bibtex
+        %span.separator •
+        %a{:href => "/#{i.name}"}modify
+        %span.separator •
+        %a{:href => i.url}download
+  - if not @token.empty?
+    %form{:method => 'POST', :action => '/'}
+      %input{:type => 'text', :name => 'query', :value => @query, :style => 'display: none'}
+      %input{:type => 'text', :name => 'token', :value => @token, :style => 'display: none'}
+      %input{:type => 'submit', :value => 'More matches...'}
+  %a{:href => "/upload"}upload
+
+@@ upload
+%form{:method => 'POST', :action => '/upload', :enctype => 'multipart/form-data'}
+  %p.error
+    = @error
+  %label Select file to upload:
+  %br
+  %input{:type => 'file', :name => 'fileToUpload', :size => 40}
+  = haml :metadata, :layout => false
+  %input{:type => 'submit', :value => 'Add file to library'}
+
+@@ metadata
+%table
+  %tr
+    %td
+      %label Type:
+    %td
+      %select{:name => 'entry_type'}
+        - ['','article','book','incollection','chapter','unpublished'].each do |type|
+          %option{:onChange => show_fields(type), :selected => (@entry && @entry.show_attribute('entry_type') == type) || type.empty?}
+            = type
+  - Cloudlib::Entry.fields.each do |field|
+    %tr{:style => (@entry && field_for_type?(field, @entry.show_attribute('entry_type'))) || 'display: none;', :id => field.to_s}
+      %td
+        %label
+          = field.to_s.capitalize + ':'
+      %td
+        %input{:type => 'text', :name => field.to_s, :value => (@entry && @entry.show_attribute(field.to_s)) || '', :size => 50}
+
+@@ modify
+%div.detail
+%form{:method => 'POST', :action => "/#{@entry.name}"}
+  = haml :metadata, :layout => false
+  %p
+    %input{:type => 'submit', :value => 'Update metadata'}
+%form{:method => 'POST', :action => "/#{@entry.name}"}
+  %p
+    %input{:type => 'text', :name => '_method', :value => 'delete', :style => 'display: none;'}
+    %input{:type => 'submit', :value => 'Delete this entry'}
+
+@@ stylesheet
+body
+  font-size: small
+  padding: 10px
+h1
+  border-top: 1px solid gray
+  border-bottom: 1px solid gray
+h1 a
+  color: #7a7a7a
+  text-decoration: none
+  &:visited
+    color: #7a7a7a
+li
+  padding-bottom: 0.3em
+#footer
+  border-top: 1px solid gray
+  margin-top: 1em
+  padding-top: 1em
+  font-size: x-small
+  text-align: center
+p.error
+  color: red
+  font-weight: bold

data/cloudlib.gemspec

@@ -0,0 +1,28 @@
+Gem::Specification.new do |s|
+  s.name     = "cloudlib"
+  s.version  = "0.3.10"
+  s.date     = "2010-10-21"
+  s.summary  = "Tools for maintaining a library of books and articles in Amazon S3 and SimpleDB"
+  s.email    = "jgm@berkeley.edu"
+  s.homepage = "http://github.com/jgm/cloudlib"
+  s.description = "Cloudlib is a ruby library and commands for maintaining a library of books and articles on the Amazon 'cloud': S3 and SimpleDB."
+  s.has_rdoc = true
+  s.authors  = ["John MacFarlane"]
+  s.bindir   = "bin"
+  s.executables = ["cloudlib", "cloudlib-web"]
+  s.default_executable = "cloudlib"
+  s.files    = [ "README",
+        "LICENSE",
+        "cloudlib.gemspec", 
+        "lib/cloudlib.rb",
+        "bin/cloudlib",
+        "bin/cloudlib-web" ]
+  s.test_files = []
+  s.rdoc_options = ["--main", "README", "--inline-source"]
+  s.extra_rdoc_files = ["README"]
+  s.add_dependency("aws-s3", [">= 0.5.1"])
+  s.add_dependency("aws-sdb", [">= 0.3.1"])
+  s.add_dependency("sinatra", [">= 0.3.2"])
+  s.add_dependency("highline", [">= 1.2.9"])
+end
+

data/lib/cloudlib.rb

@@ -0,0 +1,386 @@
+# This library provides the means for maintaining a database of
+# documents on Amazon's S3 file store, with searchable metadata in
+# Amazon's SimpleDB database. Think of it as a filing cabinet or library
+# that can be extended indefinitely and accessed from anywhere in the
+# world: a library that lives "in the cloud."
+
+# In order to use this library, you need to sign up for
+# Amazon's S3 and SimpleDB services:
+#
+# * Amazon SimpleDB: http://aws.amazon.com/simpledb/
+# * Amazon S3: http://aws.amazon.com/s3/
+#
+# Simple usage example:
+#
+#   require 'rubygems'
+#   require 'cloudlib'
+#   include Cloudlib
+#   Entry.connect('xxx_key_id_xxx', 'xxx_secret_access_key_xxx', 'my_aws_library')
+#   logic_entries = Entry.query('logic')
+#   logic_entries.each {|entry| puts entry.to_s}
+#
+# For more examples of the use of the library, see the programs cloudlib.rb
+# and cloudlib-web.rb, included in the gem.
+
+# Author::    John MacFarlane (jgm at berkeley dot edu)
+# Copyright:: Copyright (c) 2008 John MacFarlane
+# License::   GPL v2
+
+require 'rubygems'
+require 'readline'
+require 'aws/s3'   # aws-s3 gem
+require 'aws_sdb'  # aws-sdb gem
+require 'open-uri'
+require 'fileutils'
+
+module Cloudlib
+
+# A library entry, including content and metadata. An entry has a name
+# (which is also the key of the associated S3 object) and an attributes
+# hash. The name is of the form "sha1.ext", where sha1 is a SHA1 hash of
+# the contents of the file, and ext is the file extension. This makes
+# it impossible to have entries with duplicate contents. The attributes
+# hash contains the following fields:
+#
+# * extension      - file extension including .
+# * size           - size of contents (bytes)
+# * date-added     - date entry was added to library
+# * entry_type     - article, book, chapter, incollection, unpublished
+# * authors        - list of authors
+# * editors        - list of editors
+# * title          - title of entry
+# * booktitle      - title of book containing entry
+# * year           - publication year of entry
+# * publisher      - publisher of book
+# * address        - publication address
+# * journal        - journal containing entry
+# * volume         - volume number of journal
+# * pages          - page range of entry in book or journal
+# * keywords       - keywords
+# * doi            - DOI for entry
+# * url            - URL for entry
+# * comments       - miscellaneous comments
+# * *_lowercase    - lowercase version of *
+# * *_words        - lowercase version of *, split into a list of words
+# * all_words      - list of words in title, authors, editors, booktitle, keywords
+
+class Entry
+
+  attr_accessor :name, :attributes
+
+  # Establish connections to the S3 file store and the SimpleDB database.
+  # If values are not supplied for the parameters, they will default to
+  # the values of the environment variables CLOUDLIB_LIBRARY_NAME,
+  # AWS_ACCESS_KEY_ID, and AWS_SECRET_ACCESS_KEY. Note that library_name
+  # is the name of both the S3 bucket that will hold the contents of
+  # the entries and the SimpleDB domain that will hold the metadata.
+  def self.connect(library_name=ENV['CLOUDLIB_LIBRARY_NAME'],
+                   aws_access_key_id=ENV['AWS_ACCESS_KEY_ID'],
+                   aws_secret_access_key=ENV['AWS_SECRET_ACCESS_KEY'],
+                   debug = false)
+    @@aws_access_key_id = aws_access_key_id
+    @@aws_secret_access_key = aws_secret_access_key
+    AWS::S3::Base.establish_connection!(:access_key_id => @@aws_access_key_id, :secret_access_key => @@aws_secret_access_key, :use_ssl => true)
+    @@bucket = library_name
+    logger = Logger.new(STDERR)
+    logger.level = if debug then Logger::DEBUG else Logger::WARN end
+    @@db = AwsSdb::Service.new(:access_key_id => @@aws_access_key_id, :secret_access_key => @@aws_secret_access_key, :use_ssl => true, :logger => logger)
+  end
+
+  # Creates a new entry object.  To create an entry with contents,
+  # use Entry.from_file.
+  def initialize(name, attributes={'all_words' => []})
+    @name = name
+    @attributes = attributes
+  end
+
+  # Create the S3 bucket and SimpleDB domain that will store the library entries.
+  # This method should be run once to create the library.
+  def self.create_library
+    AWS::S3::Bucket.create(@@bucket)
+    @@db.create_domain(@@bucket)
+  end
+
+  # Delete the S3 bucket and SimpleDB domain that store the library entries.
+  # All data will be lost.
+  def self.delete_library
+    AWS::S3::Bucket.delete(@@bucket, :force => true)
+    @@db.delete_domain(@@bucket)
+  end
+
+  # Creates and saves an entry from a file, using attributes supplied.
+  # Returns the entry.
+  def self.from_file(path, filename=path, attributes={'all_words' => []})
+    sha1 = Digest::SHA1.file(path).hexdigest
+    ext  = File.extname(filename)
+    name = "#{sha1}#{ext}"
+    attributes['size'] = File.size(path).to_s
+    attributes['date-added'] = Date.today.to_s
+    entry = Entry.new(name, attributes)
+    AWS::S3::S3Object.store(name, open(path), @@bucket)
+    @@db.put_attributes(@@bucket, name, attributes, replace=true)
+    return entry
+  end
+
+  # Return an entry with the specified name.  Raises an error if not found.
+  def self.find_by_name(name)
+    attributes = @@db.get_attributes(@@bucket, name)
+    if attributes == {} then raise "Item not found." end
+    Entry.new(name, attributes)
+  end
+
+  # Queries the database and returns a list [token, entries].  entries is
+  # a list of up to numitems Entry objects that match the query. If
+  # there are more entries than numitems, token will be nonempty, and
+  # can be passed in on a subsequent calls for the remaining entries.
+  #
+  # The query string can contain one or more words.  If a word is
+  # preceded by ti=, only entries that match it in the title will be
+  # returned.  Similarly, au= searches authors, jo= journals, pu=
+  # publishers, ad= addresses, ed= editors, bo= booktitle (for collections),
+  # and ye= years.  ye> and # ye< may also be used.
+  # The form ti='word1 word2' may also be used; entries will only match
+  # if their titles contain both word1 and word2.
+  def self.query(query_string, numitems=10, token=nil)
+    query_parts = query_string.downcase.scan(/((ti(?:tle)?|au(?:thors?)?|jo(?:urnal)?|bo(?:ooktitle)?|pu(?:blisher)?|ad(?:ddress)?|ed(?:itors?)?|ye(?:ar)?)\s*([<=>])\s*('[^']*'|"[^"]*"|\S*)|\S+)\s*/)
+    query = query_parts.reject {|part| part[0] == '*'}.map do |part|
+      whole, key, comparison, val = part
+      if val then val = val.gsub(/^['"](.*)['"]$/, "\\1") end
+      if not val then val = whole end
+      key_full = if key
+                   case key[0..1]
+                   when 'ti'
+                    'title'
+                   when 'au'
+                    'authors'
+                   when 'jo'
+                    'journal'
+                   when 'pu'
+                    'publisher'
+                   when 'ad'
+                    'address'
+                   when 'ed'
+                    'editors'
+                   when 'ye'
+                    'year'
+                   else 'all'
+                   end
+                 else
+                   'all'
+                 end
+      # split hyphenated names into components, since a query might just have one
+      vals = val.split(/[-[:space:]]+/)
+      vals.map do |v|
+        if key_full == 'year'    # there is no year_words field
+           "['year' #{comparison} '#{v}']"
+        else
+           v_escaped = v.gsub(/\\/,"\\\\\\\\").gsub(/'/,"\\\\'")
+           "['#{key_full}_words' = '#{v_escaped}']"
+        end
+      end.join(" intersection ")
+    end.join(" intersection ")
+    # note: query has to include year in order to sort by year
+    # hence this dummy search
+    if query.empty?
+       query = "['year' starts-with ''] sort 'year'"
+    else
+       query += " intersection ['year' starts-with ''] sort 'year'"
+    end
+    names, token = if token
+                     @@db.query(@@bucket, query, numitems, token)
+                   else
+                     @@db.query(@@bucket, query, numitems)
+                   end
+    entries = names.map do |name|
+      attributes = @@db.get_attributes(@@bucket, name)
+      Entry.new(name, attributes)
+    end
+    return token, entries
+  end
+
+  # Returns a human-friendly filename for the entry, constructed from
+  # authors and title.
+  def friendly_filename
+    authornames = self.attributes['authors'].map {|a| last_name(a)}.join('_')
+    title = self.show_attribute('title').gsub(/[,.\/[:space:]]+/,'_')
+    ext = File.extname(self.name)
+    return "#{authornames}_#{title}#{ext}"
+  end
+
+  # Deletes the entry.
+  def delete
+    AWS::S3::S3Object.delete(self.name, @@bucket)
+    @@db.delete_attributes(@@bucket, self.name)
+  end
+
+  # Saves the entry (metadata only; contents are saved by the from_file
+  # method).
+  def save
+    @@db.put_attributes(@@bucket, self.name, self.attributes, replace=true)
+  end
+
+  # Downloads the entry and saves as filename.
+  def download(path)
+    if File.exist?(path)
+      STDERR.puts "Backing up existing #{path} as #{path}~"
+      FileUtils.copy_file(path, "#{path}~", preserve=true)
+    end
+    open(path, 'w') do |outfile|
+      open(self.url, 'r') do |source|
+        FileUtils.copy_stream(source, outfile)
+      end
+    end
+    return path
+  end
+
+  # Returns a bibtex entry for the entry.
+  def to_bibtex
+    pairs = self.fields.map do |field|
+      if self.attributes[field.to_s]
+         sprintf("  %-15s= {%s}", field.to_s, self.show_attribute(field.to_s))
+      else
+         nil
+      end
+    end
+    pairs += [sprintf("  %-15s= {%s}", "file", self.name)]
+    authornames = self.attributes['authors'].map {|a| last_name(a)}.join('.')
+    year = self.attributes['year']
+    entry_type = self.show_attribute('entry_type') || 'unknown'
+    if entry_type == 'chapter' then entry_type = 'inbook' end
+    entry_key = "#{authornames}:#{year}"
+    "@#{entry_type.upcase}{#{entry_key},\n#{pairs.join(",\n")}\n}"
+  end
+
+  # Returns a string representation of the entry's metadata.
+  def to_s
+    authors = self.show_attribute('authors')
+    unless authors.empty?
+      authors = "#{authors}, "
+    end
+    title = "#{self.show_attribute('title')}"
+    year = self.show_attribute('year')
+    titleyear = if year.empty?
+                   title + ". "
+                else
+                   title + " (#{year}). "
+                end
+    pubaddr = [self.show_attribute('address'),
+               self.show_attribute('publisher')].reject {|x| x.empty?}.join(": ")
+    chapter = self.show_attribute('chapter')
+    pages = self.show_attribute('pages')
+    booktitle = self.show_attribute('booktitle')
+    editors = self.show_attribute('editors')
+    journal = self.show_attribute('journal')
+    volume = self.show_attribute('volume')
+    rest = case self.show_attribute('entry_type')
+           when 'article'
+             if journal.empty?
+                ""
+             else
+                "#{journal} #{volume}" +
+                if pages.empty? then "." else ", #{pages}." end
+             end
+           when 'book'
+             if pubaddr.empty? then "" else "#{pubaddr}." end
+           when 'chapter'
+             if pubaddr.empty? then "" else "#{pubaddr}." end +
+             if chapter.empty? then "" else " Chapter #{chapter}." end +
+             if pages.empty? then "" else " #{pages}." end
+           when 'incollection'
+             "In " +
+             if editors.empty? then "" else editors + " (eds.), " end +
+             booktitle +
+             if pubaddr.empty? then "" else " (#{pubaddr})." end +
+             if chapter.empty? then "" else " Chapter #{chapter}." end +
+             if pages.empty? then "" else " #{pages}." end
+           when 'unpublished'
+             " (unpublished)."
+           else ""
+           end
+    return authors + titleyear + rest
+  end
+
+  # Sets the specified metadata attribute to ans.  ans is assumed to be a regular string.
+  # It will be split by " and " for authors and editors, or by spaces for keywords.
+  def set_attribute(attribute, ans)
+    index = ['title', 'authors', 'editors', 'booktitle'].member?(attribute)
+    if ans.nil? || ans.empty?
+      self.attributes[attribute] = nil
+    else
+      newval = if attribute == 'editors' || attribute == 'authors'
+                  ans.split(" and ").map {|a| a.strip}
+               elsif attribute == 'keywords'
+                  ans.split
+               else
+                  [ans.strip]
+               end
+      self.attributes[attribute] = newval
+      unless ['url', 'doi', 'keywords'].member?(attribute)
+        self.attributes[attribute + "_lowercase"] = newval.map {|a| a.downcase}
+        self.attributes[attribute + "_words"] = self.attributes[attribute + "_lowercase"].map {|a| a.split(/[[:punct:]]*[[:space:]]+|-+/)}.flatten.reject {|a| a.empty?}
+      end
+      # recalculate all_words
+      tit_auth_words = ['title', 'authors', 'editors', 'booktitle'].map {|att| self.attributes[att + "_words"] || []}.flatten
+      keywords = self.attributes['keywords'] || []
+      self.attributes['all_words'] = keywords + tit_auth_words
+    end
+  end
+
+  # Returns a string representation of an attribute.
+  def show_attribute(attribute)
+    value = self.attributes[attribute]
+    if value.nil?
+       ""
+    elsif attribute == 'keywords'
+       value.join(' ')
+    elsif attribute == 'editors' || attribute == 'authors'
+       value.join(' and ')
+    else
+       value[0]
+    end
+  end
+
+  # Returns an array of the field keywords appropriate for a type of entry.
+  def self.fields(entry_type='*')
+    fields = [:title, :authors, :year]
+    case entry_type
+    when 'article'
+      fields += [:journal, :volume, :pages]
+    when 'book'
+      fields += [:publisher, :address]
+    when 'chapter'
+      fields += [:booktitle, :chapter, :publisher, :address, :pages]
+    when 'incollection'
+      fields += [:booktitle, :chapter, :publisher, :address, :editors, :pages]
+    when '*'
+      fields += [:journal, :volume, :booktitle, :editors, :chapter,
+                 :publisher, :address, :pages]
+    end
+    fields += [:keywords, :url, :doi, :comments]
+    return fields
+  end
+
+  # Returns the fields appropriate for an entry.
+  def fields
+    entry_type = self.show_attribute('entry_type')
+    Entry.fields(entry_type)
+  end
+
+  def url(minutes = 10)
+    AWS::S3::S3Object.url_for(self.name, @@bucket, :expires_in => 60 * minutes)
+  end
+
+  private
+  # Returns the author's last name.
+  def last_name(author)
+    if author =~ /,/
+      author =~ /([^ ,]+),/
+    else
+      author =~ /([^ \t]+)$/
+    end
+    return $1
+  end
+
+end
+
+end