larsklevan-attachment_fu_app_engine 0.2.0

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2008 Lars Klevan
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README

@@ -0,0 +1,62 @@
+AttachmentFuAppEngine
+=====================
+
+Extension for AttachmentFu (http://github.com/technoweenie/attachment_fu) which uses the Google App Engine for storage and image resizing.  Removes the need to install ImageMagick and allows you to add or change thumbnail sizes without migrating previous data.
+
+The Google App Engine backend code is included so you can run it in your own App Engine instance.
+
+Example
+=======
+
+class Photo < ActiveRecord::Base
+  has_attachment :storage => :app_engine, :thumbnails => {:small_square => "45x45!"}
+end
+
+AttachmentFuAppEngine attempts to reproduce the resize format from ImageMagick.
+
+* "100x100"  - scale the image to fit within a 100x100 box (the larger dimension will be 100, such as 100x75), preserving the aspect ratio.  Will scale up small images.
+* "100x100!" - scale the image to be exactly 100x100 by scaling proportionately and cropping off the edges.  Will not distort the image but may hide part of it.
+* "100x100>" - scale the image to be at most 100x100 - smaller images will not be enlarged.
+
+In addition to the standard thumbnail specification:
+Image.find(1).public_filename(:small_square)
+
+you can also use:
+Image.find(1).public_filename('40x40!')
+
+
+Installation
+============
+
+gem install larsklevan-attachment_fu_app_engine -s http://gems.github.com 
+OR
+script/plugin install git://github.com/larsklevan/attachment_fu_app_engine.git
+
+
+Migration
+=========
+
+You can use the provided rake task to move data from S3 to the App Engine.  Just run:
+rake ATTACHMENT_CLASS=Image migrate_s3_to_app_engine
+
+
+Configuration
+=============
+
+You can use your own instance of the App Engine code to store images so you have your own quotas.  Use
+the provided app engine code and configure the URL as follows:
+
+Technoweenie::AttachmentFu::Backends::AppEngineBackend.base_url = "http://attachment-fu-gae.appspot.com"
+
+
+Limitations
+===========
+
+* Tested with Rails 2.1 only
+* Google App Engine is in early release, so if you go past your daily storage/bandwidth/CPU limit you are out of luck.
+* Google App Engine does not support HTTPS.
+* Google App Engine currently has a limit of 1MB per reocrd
+
+For more information about the status of the App Engine see http://googleappengine.blogspot.com/2008/04/were-up-and-running.html
+
+Copyright (c) 2008 Lars Klevan, released under the MIT license

data/Rakefile

@@ -0,0 +1,22 @@
+require 'rake'
+require 'rake/testtask'
+require 'rake/rdoctask'
+
+desc 'Default: run unit tests.'
+task :default => :test
+
+desc 'Test the attachment_fu_app_engine plugin.'
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'lib'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = true
+end
+
+desc 'Generate documentation for the attachment_fu_app_engine plugin.'
+Rake::RDocTask.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'AttachmentFuAppEngine'
+  rdoc.options << '--line-numbers' << '--inline-source'
+  rdoc.rdoc_files.include('README')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/app_engine/attachment_fu_app_engine/app.yaml

@@ -0,0 +1,8 @@
+application: attachment-fu-gae
+version: 1
+runtime: python
+api_version: 1
+
+handlers:
+- url: /.*
+  script: attachment.py

data/app_engine/attachment_fu_app_engine/attachment.py

@@ -0,0 +1,219 @@
+import os
+import cgi
+import logging
+from StringIO import StringIO
+import struct
+
+import wsgiref.handlers
+from datetime import date
+
+from google.appengine.api import images
+from google.appengine.api import memcache
+from google.appengine.ext import db
+from google.appengine.ext import webapp
+from google.appengine.ext.webapp import template
+
+class Attachment(db.Model):
+  path = db.StringProperty()
+
+  filename = db.StringProperty()
+  uploaded_data = db.BlobProperty()
+  content_type = db.StringProperty()
+  height = db.IntegerProperty()
+  width = db.IntegerProperty()
+  size = db.IntegerProperty()
+
+  # FROM http://www.google.com/codesearch?hl=en&q=+getImageInfo+show:RjgT7H1iBVM:V39CptbrGJ8:XcXNaKeZR3k&sa=N&cd=2&ct=rc&cs_p=http://www.zope.org/Products/Zope3/3.0.0final/ZopeX3-3.0.0.tgz&cs_f=ZopeX3-3.0.0/Dependencies/zope.app.file-ZopeX3-3.0.0/zope.app.file/image.py#l88
+  def extract_image_attributes(self,data):
+    data = str(data)
+    size = len(data)
+    height = -1
+    width = -1
+    content_type = ''
+
+    # handle GIFs
+    if (size >= 10) and data[:6] in ('GIF87a', 'GIF89a'):
+        # Check to see if content_type is correct
+        content_type = 'image/gif'
+        w, h = struct.unpack("<HH", data[6:10])
+        width = int(w)
+        height = int(h)
+
+    # See PNG v1.2 spec (http://www.cdrom.com/pub/png/spec/)
+    # Bytes 0-7 are below, 4-byte chunk length, then 'IHDR'
+    # and finally the 4-byte width, height
+    elif ((size >= 24) and data.startswith('\211PNG\r\n\032\n') and (data[12:16] == 'IHDR')):
+        content_type = 'image/png'
+        w, h = struct.unpack(">LL", data[16:24])
+        width = int(w)
+        height = int(h)
+
+    # Maybe this is for an older PNG version.
+    elif (size >= 16) and data.startswith('\211PNG\r\n\032\n'):
+        # Check to see if we have the right content type
+        content_type = 'image/png'
+        w, h = struct.unpack(">LL", data[8:16])
+        width = int(w)
+        height = int(h)
+
+    # handle JPEGs
+    elif (size >= 2) and data.startswith('\377\330'):
+        content_type = 'image/jpeg'
+        jpeg = StringIO(data)
+        jpeg.read(2)
+        b = jpeg.read(1)
+        try:
+            while (b and ord(b) != 0xDA):
+                while (ord(b) != 0xFF): b = jpeg.read(1)
+                while (ord(b) == 0xFF): b = jpeg.read(1)
+                if (ord(b) >= 0xC0 and ord(b) <= 0xC3):
+                    jpeg.read(3)
+                    h, w = struct.unpack(">HH", jpeg.read(4))
+                    break
+                else:
+                    jpeg.read(int(struct.unpack(">H", jpeg.read(2))[0])-2)
+                b = jpeg.read(1)
+            width = int(w)
+            height = int(h)
+        except struct.error:
+            pass
+        except ValueError:
+            pass
+
+    return height,width
+
+  def update_uploaded_data(self, data, content_type):
+    if content_type.startswith('image'):
+      self.height, self.width = self.extract_image_attributes(data)
+      if not self.height:
+        #if we can't determine the image attributes in the original format try converting it to a PNG with a no-op rotate
+        image = images.Image(data)
+        image.rotate(0)
+        self.height, self.width = self.extract_image_attributes(image.execute_transforms(output_encoding=images.PNG))
+
+    self.content_type = content_type
+    self.uploaded_data = data
+    self.size = len(data)
+  
+  # I'm attempting to replicate the resize format from http://www.imagemagick.org/Usage/resize/#resize
+  # at least enough to be usable for avatar or photo gallery thumbnails
+  def resize(self, format):
+    preserve_aspect_ratio = True
+    allow_scale_up = True
+    if format.endswith("!"):
+      preserve_aspect_ratio = False
+      format = format.rstrip("!")
+    elif format.endswith(">"):
+      allow_scale_up = False
+      format = format.rstrip(">")
+
+    width,height = format.split('x')
+
+    img = images.Image(self.uploaded_data)
+    if not preserve_aspect_ratio:
+      requested_aspect = float(height)/float(width)
+      aspect = float(self.height)/float(self.width)
+
+      ratio = requested_aspect / aspect
+      if (ratio < 1):
+        left_x = 0.0
+        right_x = 1.0
+        top_y = 0.5 - (ratio / 2)
+        bottom_y = 0.5 + (ratio / 2)
+      else:
+        top_y = 0.0
+        bottom_y = 1.0
+        left_x = 0.5 - ((1/ratio) / 2)
+        right_x = 0.5 + ((1/ratio) / 2)
+
+      # seem to have issues with small rounding errors for larger images - request for 2000x2000 can end up at 1998x2000
+      # presumably rounding errors - the 0-1 scale for cropping is weird...
+      img.crop(left_x=left_x,top_y=top_y,right_x=right_x,bottom_y=bottom_y)
+
+    if allow_scale_up or int(width) < self.width or int(height) < self.height:
+      img.resize(width=int(width), height=int(height))
+
+    output_encoding, content_type = images.PNG, 'image/png'
+    if self.content_type == 'image/jpeg' or self.content_type == 'image/jpg':
+      output_encoding, content_type = images.JPEG, 'image/jpeg'
+
+    img.rotate(0) #no-op so that we don't break if we haven't done any transforms
+    return img.execute_transforms(output_encoding), content_type
+
+class UploadAttachmentPage(webapp.RequestHandler):
+  def get(self):
+    path = os.path.join(os.path.dirname(__file__), 'new.html')
+    self.response.out.write(template.render(path, {}))
+
+class AttachmentPage(webapp.RequestHandler):
+  def get(self):
+    attachment = None
+    try:
+      id = self.request.path.split('/')[-1]
+      attachment = Attachment.get(db.Key(id))
+    except:
+      None
+
+    if not attachment:
+      attachment = db.Query(Attachment).filter("path =", self.request.path[1::]).get()
+
+    if not attachment:
+      # Either "id" wasn't provided, or there was no attachment with that ID
+      # in the datastore.
+      self.error(404)
+      return
+
+    today = date.today()
+    self.response.headers.add_header("Expires", date(year=today.year + 1,month=today.month, day=today.day).ctime())
+    format = self.request.get("resize")
+    if format:
+      memcache_client = memcache.Client()
+      cache_key = "attachment-" + str(attachment.key()) + "-" + format
+      result = memcache_client.get(cache_key)
+      if not result:
+        data, content_type = attachment.resize(format)
+        memcache_client.set(cache_key, [data, content_type])
+      else:
+        data, content_type = result[0], result[1]
+      self.response.headers['Content-Type'] = content_type
+      self.response.out.write(data)
+    else:
+      self.response.headers['Content-Type'] = str(attachment.content_type)
+      self.response.out.write(attachment.uploaded_data)
+
+  def post(self):
+    form = cgi.FieldStorage()
+    
+    path = form.getvalue('path')
+    attachment = db.Query(Attachment).filter("path =", path).get()
+    if not attachment:
+      attachment = Attachment()
+    attachment.path = path
+    
+    uploaded_data = form['uploaded_data']
+    attachment.filename = uploaded_data.filename
+
+    attachment.update_uploaded_data(uploaded_data.value, uploaded_data.type)
+    attachment.put()
+    
+    
+    logging.debug('Added attachment with path: ' + attachment.path + ' id: ' + str(attachment.key()))
+    self.redirect('/attachments/' + str(attachment.key()))
+
+
+class RedirectPage(webapp.RequestHandler):
+  def get(self):
+    self.redirect('/attachments/new')
+
+def main():
+  logging.getLogger().setLevel(logging.DEBUG)
+
+  application = webapp.WSGIApplication(
+    [('/attachments/new', UploadAttachmentPage),
+      ('/attachments.*', AttachmentPage),
+      ('/.*', RedirectPage)],
+    debug=True)
+  wsgiref.handlers.CGIHandler().run(application)
+
+if __name__ == "__main__":
+  main()

data/app_engine/attachment_fu_app_engine/index.yaml

@@ -0,0 +1,17 @@
+indexes:
+
+# AUTOGENERATED
+
+# This index.yaml is automatically updated whenever the dev_appserver
+# detects that a new type of query is run.  If you want to manage the
+# index.yaml file manually, remove the above marker line (the line
+# saying "# AUTOGENERATED").  If you want to manage some indexes
+# manually, move them above the marker line.  The index.yaml file is
+# automatically uploaded to the admin console when you next deploy
+# your application using appcfg.py.
+
+# Unused in query history -- copied from input.
+- kind: Attachment
+  properties:
+  - name: path
+    direction: desc

data/app_engine/attachment_fu_app_engine/new.html

@@ -0,0 +1,17 @@
+<html>
+  <body>
+    Upload a file:
+    <br/>
+    <br/>
+    <form action="/attachments" method="post" enctype="multipart/form-data">
+      <label for="file">Select a file:</label>
+      <br/>
+      <input type="file" name="uploaded_data" />
+      <br/>
+      <label for="file">Path:</label>
+      <input type="text" name="path" />
+      <br/>
+      <input type="submit" value="Submit" />
+    </form>
+  </body>
+</html>

data/attachment_fu_app_engine.gemspec

@@ -0,0 +1,13 @@
+Gem::Specification.new do |s|
+  s.name     = "attachment_fu_app_engine"
+  s.version  = "0.2.0"
+  s.date     = "2008-08-02"
+  s.summary  = "Extension for AttachmentFu which uses the Google App Engine for storage"
+  s.email    = "tastybyte@gmail.com"
+  s.homepage = "http://github.com/larsklevan/attachment_fu_app_engine"
+  s.description = "Extension for AttachmentFu (http://github.com/technoweenie/attachment_fu) which uses the Google App Engine for storage and image resizing."
+  s.has_rdoc = true
+  s.authors  = ["Lars Klevan"]
+  s.files    = %w{app_engine/attachment_fu_app_engine/app.yaml app_engine/attachment_fu_app_engine/attachment.py app_engine/attachment_fu_app_engine/index.yaml app_engine/attachment_fu_app_engine/new.html app_engine/attachment_fu_app_engine/photo.pyc attachment_fu_app_engine.gemspec init.rb initializer.rb.tpl install.rb lib/multipart_post.rb lib/tasks/migrate.rake lib/technoweenie/attachment_fu/backends/app_engine_backend.rb MIT-LICENSE Rakefile README}
+end
+

data/initializer.rb.tpl

@@ -0,0 +1,6 @@
+# if you're using this in production I'd recommend creating your own app engine app with the provided backend code
+# see http://code.google.com/appengine/
+Technoweenie::AttachmentFu::Backends::AppEngineBackend.base_url = "http://attachment-fu-gae.appspot.com"
+
+# storage prefix prevents collision between multiple apps using the same app engine for storage
+Technoweenie::AttachmentFu::Backends::AppEngineBackend.storage_prefix = "<%= rand(10e12) %>"

data/install.rb

@@ -0,0 +1,10 @@
+require 'erb'
+
+initializer = File.dirname(__FILE__) + '/../../../config/initializers/app_engine_backend.rb'
+
+unless File.exist? initializer
+  initializer_template = IO.read(File.dirname(__FILE__) + '/initializer.rb.tpl')
+  File.open(initializer, 'w') { |f| f << ERB.new(initializer_template).result }
+end
+
+puts IO.read(File.join(File.dirname(__FILE__), 'README'))

data/lib/multipart_post.rb

@@ -0,0 +1,30 @@
+class MultipartPost
+  # see http://www.realityforge.org/articles/2006/03/02/upload-a-file-via-post-with-net-http for file upload with http
+  def self.post(uri, params=[])
+    chunks = []    
+    params.each do |param|
+      param[:name]
+      chunks << if param[:mime_type]
+        "Content-Disposition: form-data; name=\"#{CGI::escape(param[:name])}\"; filename=\"#{param[:filename]}\"\r\n" +
+               "Content-Transfer-Encoding: binary\r\n" +
+               "Content-Type: #{param[:mime_type]}\r\n" +
+               "\r\n#{param[:value]}\r\n"
+      else
+        "Content-Disposition: form-data; name=\"#{CGI::escape(param[:name])}\"\r\n" +
+               "\r\n#{param[:value]}\r\n"
+      end
+    end
+    boundary = "349832898984244898448024464570528145"
+    post_body = ""
+    chunks.each do |chunk|
+      post_body << "--#{boundary}\r\n"
+      post_body << chunk
+    end
+    post_body << "--#{boundary}--\r\n"
+    
+    uri = URI.parse(uri)
+    Net::HTTP.new(uri.host, uri.port).start do |http|
+      http.request_post(uri.path, post_body, "Content-type" => "multipart/form-data; boundary=" + boundary)
+    end
+  end
+end

data/lib/tasks/migrate.rake

@@ -0,0 +1,37 @@
+# move from s3 to gae
+
+desc 'Migrate file data from s3 storage to app engine'
+task :migrate_s3_to_app_engine => :environment do
+  require 'open-uri'
+  require 'multipart_post'
+
+  raise 'You should specify the class to migrate with ATTACHMENT_CLASS=Image'
+  clazz = ENV['ATTACHMENT_CLASS'].constantize
+
+  raise 'You should set the :storage option from your class to :s3 when you run this migration' unless clazz.attachment_options[:storage].to_s == 's3'
+
+  app_engine_config = Technoweenie::AttachmentFu::Backends::AppEngineBackend
+
+  failures = []
+  clazz.find(:all).each do |o|
+    next if o.respond_to?(:parent) && o.parent
+    begin
+      puts "Migrating #{o.public_filename}"
+  
+      temp_data = open(o.public_filename).read
+
+      app_engine_path = ['attachments', app_engine_config.storage_prefix, clazz.attachment_options[:path_prefix], o.id, o.filename].compact.join('/')
+
+      response = MultipartPost.post(app_engine_config.base_url + '/attachments', [
+        {:name => 'uploaded_data', :filename => o.filename, :mime_type => o.content_type, :value => temp_data},
+        {:name => 'path', :value => app_engine_path}
+      ])
+      raise response.body unless response.is_a? Net::HTTPRedirection
+    rescue => err
+      puts "Failed to migrate #{o.public_filename} - #{err.message}"
+      failures << o.id
+      next
+    end
+  end
+  puts "#{failures.size} files failed to migrate" unless failures.empty?
+end

data/lib/technoweenie/attachment_fu/backends/app_engine_backend.rb

@@ -0,0 +1,72 @@
+require 'net/http'
+require 'multipart_post'
+
+module Technoweenie # :nodoc:
+  module AttachmentFu # :nodoc:
+    module Backends
+      # store in Google App Engine
+      module AppEngineBackend
+        mattr_accessor :base_url, :storage_prefix
+        @@base_url = "http://attachment-fu-gae.appspot.com"
+        @@storage_prefix = nil
+
+        def public_filename(thumbnail = nil)
+          thumbnails = HashWithIndifferentAccess.new(attachment_options[:thumbnails])
+
+          query = if thumbnail && thumbnails[thumbnail]
+            "?resize=#{thumbnails[thumbnail]}"
+          elsif thumbnail.is_a?(String)
+            "?resize=#{thumbnail}"
+          elsif attachment_options[:resize]
+            "?resize=#{attachment_options[:resize]}"
+          else
+            ''
+          end
+          "#{AppEngineBackend.base_url}/#{full_filename}#{query}"
+        end
+
+        def create_temp_file
+          write_to_temp_file current_data
+        end
+
+        # Gets the current data from the database
+        def current_data
+          uri = URI.parse(AppEngineBackend.base_url)
+          Net::HTTP.new(uri.host, uri.port).start do |http|
+            http.get(filename).response_body
+          end
+        end
+
+        #TODO: HACK??
+        def process_attachment
+          @saved_attachment = true
+        end
+
+        # The full path to the file relative to the bucket name
+        # Example: <tt>:table_name/:id/:filename</tt>
+        def full_filename
+          ['attachments', storage_prefix, attachment_options[:path_prefix].gsub('public/', ''), id.to_s, filename].compact.join('/')
+        end
+
+        def create_or_update_thumbnail(*args)
+          #ignore
+        end
+        protected
+          def destroy_file
+            #ignore
+          end
+
+          def save_to_storage
+            if save_attachment?
+              response = MultipartPost.post(AppEngineBackend.base_url + '/attachments', [
+                {:name => 'uploaded_data', :value => temp_data, :mime_type => content_type, :filename => filename},
+                {:name => 'path', :value => full_filename}
+              ])
+              raise response.body unless response.is_a? Net::HTTPRedirection
+            end
+            true
+          end
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,67 @@
+--- !ruby/object:Gem::Specification 
+name: larsklevan-attachment_fu_app_engine
+version: !ruby/object:Gem::Version 
+  version: 0.2.0
+platform: ruby
+authors: 
+- Lars Klevan
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2008-08-02 00:00:00 -07:00
+default_executable: 
+dependencies: []
+
+description: Extension for AttachmentFu (http://github.com/technoweenie/attachment_fu) which uses the Google App Engine for storage and image resizing.
+email: tastybyte@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- app_engine/attachment_fu_app_engine/app.yaml
+- app_engine/attachment_fu_app_engine/attachment.py
+- app_engine/attachment_fu_app_engine/index.yaml
+- app_engine/attachment_fu_app_engine/new.html
+- app_engine/attachment_fu_app_engine/photo.pyc
+- attachment_fu_app_engine.gemspec
+- init.rb
+- initializer.rb.tpl
+- install.rb
+- lib/multipart_post.rb
+- lib/tasks/migrate.rake
+- lib/technoweenie/attachment_fu/backends/app_engine_backend.rb
+- MIT-LICENSE
+- Rakefile
+- README
+has_rdoc: true
+homepage: http://github.com/larsklevan/attachment_fu_app_engine
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.2.0
+signing_key: 
+specification_version: 2
+summary: Extension for AttachmentFu which uses the Google App Engine for storage
+test_files: []
+