alula 0.2.3 → 0.4.0b

data/lib/alula/plugin.rb

@@ -0,0 +1,32 @@
+require 'hashie/mash'
+
+module Alula
+  class Plugin
+    def self.register(name, klass); plugins[name.to_s] = klass; end
+    def self.plugins; @@plugins ||= {}; end
+    def plugins; self.class.plugins; end
+    
+    def self.load(name, options)
+      if plugins[name] and !(!!options == options and !options)
+        plugin = plugins[name]
+        return plugin if plugin.install(options || Hashie::Mash.new)
+      end
+    end
+    
+    def self.addons; @@addons ||= Hash.new {|hash, key| hash[key] = []}; end
+    def self.addon(type, content_or_block); addons[type] << content_or_block; end
+    
+    def self.script_load_mode=(mode)
+      @@script_load_mode = case mode
+      when :sync
+        :sync
+      when :defer
+        self.script_load_mode == :sync ? :sync : :defer
+      end
+    end
+    
+    def self.script_load_mode
+      @@script_load_mode ||= :async
+    end
+  end
+end

data/lib/alula/processor.rb

@@ -0,0 +1,86 @@
+require 'mimemagic'
+require 'mini_exiftool'
+
+module Alula
+  class Processor
+    attr_reader :item
+    attr_reader :options
+    attr_reader :site
+    attr_reader :attachments
+    
+    def self.mimetype(*mimetypes)
+      (class << self; self; end).send(:define_method, "mimetypes") do
+        mimetypes.collect do |m|
+          if m.kind_of?(String)
+            Regexp.new("^#{m}$")
+          else
+            m
+          end
+        end
+      end
+    end
+    
+    def self.available?(options)
+      true
+    end
+    
+    def self.process?(item, opts)
+      if self.respond_to?(:mimetypes)
+        mimetype = if opts[:fast]
+          MimeMagic.by_extension(item.extension)
+        else
+          MimeMagic.by_magic(File.open(item.filepath))
+        end
+        
+        return !(self.mimetypes.select{|re| re.match(mimetype.type)}.empty?)
+      end
+      
+      return false
+    end
+    
+    def initialize(item, opts)
+      @item = item
+      @site = opts.delete(:site)
+      @attachments = opts.delete(:attachments)
+      
+      @options = opts.delete(:options)
+      
+      # Networked processors, create global 'queues' to prevent multiple simultanous upload/downloads
+      @@upload ||= Mutex.new
+      @@download ||= Mutex.new
+    end
+    
+    def cleanup
+      @item = nil
+    end
+    
+    def process
+    end
+    
+    def asset_path(name, type)
+      asset_name = self.attachments.asset_name(name, type.to_s)
+      output = File.join(self.site.storage.path(:cache, "attachments"), asset_name)
+      # Make sure our directory exists
+      output_dir = self.site.storage.path(:cache, "attachments", File.dirname(asset_name))
+      
+      output
+    end
+    
+    def info
+      @info ||= begin
+        info = Dimensions.dimensions(self.item.filepath)
+        info ||= begin
+          _info = MiniExiftool.new self.item.filepath
+          [_info.imagewidth, _info.imageheight]
+        end
+        Hashie::Mash.new({
+          width: info[0],
+          height: info[1],
+        })
+      end
+    end
+    
+  end
+end
+
+Dir[File.dirname(__FILE__) + '/processors/*.rb'].each { |f| require f }

data/lib/alula/processors/dummy.rb

@@ -0,0 +1,24 @@
+module Alula
+  class DummyProcessor < Processor
+    
+    def process
+      super
+        
+      asset_name = self.attachments.asset_name(item.name)
+      output = File.join(self.site.storage.path(:cache, "attachments"), asset_name)
+      # Make sure our directory exists
+      output_dir = self.site.storage.path(:cache, "attachments", File.dirname(asset_name))
+      
+      # Skip processing if output already exists...
+      unless File.exists?(output)
+        # Just simply copy attachement
+        FileUtils.cp(item.filepath, output)
+      end
+      
+      # Cleanup ourself
+      cleanup
+    end
+  end
+end
+
+Alula::AttachmentProcessor.register('dummy', Alula::DummyProcessor)

data/lib/alula/processors/image.rb

@@ -0,0 +1,52 @@
+module Alula
+  class ImageProcessor < Processor
+    def process
+      super
+      
+      sizes.each do |size|
+        width, height = size[:size]
+        # output_dir = self.site.storage.path(:cache, "attachments", size[:type].to_s)
+        
+        # Generate attachment hash
+        name = File.join size[:type].to_s, if size[:hires]
+          ext = File.extname(item.name)
+          item.name.gsub(/#{ext}$/, "-hires#{ext}")
+        else
+          item.name
+        end
+        
+        output = asset_path(name, size[:type].to_s)
+        # asset_name = self.attachments.asset_name(name, size[:type].to_s)
+        # output = File.join(self.site.storage.path(:cache, "attachments"), asset_name)
+        # # Make sure our directory exists
+        # output_dir = self.site.storage.path(:cache, "attachments", File.dirname(asset_name))
+        
+        # Skip image processing if output already exists...
+        next if File.exists?(output)
+        
+        # Skip if our original resolution isn't enough for hires images
+        next if (width > self.info.width and height > self.info.height) and size[:hires]
+        
+        # Generate resized image to output path
+        resize_image(output: output, size: size[:size])
+      end
+      
+      # Cleanup ourself
+      cleanup
+    end
+    
+    def sizes
+      @sizes ||= begin
+        sizes = []
+        sizes << { type: :image,  hires: false, size: self.site.config.attachments["image"]["size"].split("x").collect{|i| i.to_i} }
+        sizes << { type: :thumbnail, hires: false, size: self.site.config.attachments["image"]["thumbnail"].split("x").collect{|i| i.to_i} }
+        
+        if self.site.config.attachments["image"]["hires"]
+          sizes += sizes.collect {|s| { type: s[:type], hires: true, size: s[:size].collect{|x| x*2} } }
+        end
+        
+        sizes
+      end
+    end
+  end
+end

data/lib/alula/processors/magick.rb

@@ -0,0 +1,83 @@
+require 'alula/processors/image'
+require 'hashie/mash'
+require 'RMagick'
+require 'mini_exiftool'
+
+module Alula
+  class Magick < ImageProcessor
+    # Register mimetypes
+    mimetype "image/jpeg", "image/png", "image/gif"
+    
+    def resize_image(opts)
+      output = opts.delete(:output)
+      width, height = opts.delete(:size)
+      
+      # Generate resized image
+      resized = self.image.resize_to_fit(width, height)
+      # Make it progressive
+      resized.interlace = ::Magick::PlaneInterlace
+      
+      # Strip unwanted properties
+      tags = Hash[*(self.site.config.attachments["image"]["keep_tags"].collect{|t| [t, self.exif[t]]}).flatten]
+      resized.strip!
+      
+      resized.write(output)
+      
+      # Save our EXIF info
+      exif = MiniExiftool.new output
+      tags.each {|key, value| exif[key] = value }
+      exif.save
+    end
+    
+    def cleanup
+      super
+      
+      @info = nil
+      @image = nil
+      @exif = nil
+    end
+    
+    def info
+      @info ||= begin
+        info = ::Magick::Image.ping(self.item.filepath).first
+        Hashie::Mash.new({
+          width: info.columns,
+          height: info.rows,
+        })
+      end
+    end
+    
+    def exif
+      @exif ||= MiniExiftool.new self.item.filepath
+    end
+    
+    def image
+      @image ||= begin
+        image = ::Magick::Image.read(self.item.filepath).first
+        unless self.options[:no_rotate]
+          case image.orientation.to_i
+            when 2
+              image.flop!
+            when 3
+              image.rotate!(180)
+            when 4
+              image.flip!
+            when 5
+              image.transpose!
+            when 6
+              image.rotate!(90)
+            when 7
+              image.transverse!
+            when 8
+              image.rotate!(270)
+          end
+          
+          image
+        end
+      end
+    end
+    
+  end
+end
+
+Alula::AttachmentProcessor.register('magick', Alula::Magick)

data/lib/alula/processors/video.rb

@@ -0,0 +1,97 @@
+module Alula
+  class VideoProcessor < Processor
+    def process
+      # Select variants we need to generate
+      generate = variants.select do |name, format|
+        size = format[:size].split("x").collect{|i| i.to_i}
+        
+        # Check if video requires rotating
+        if self.info.rotation == 90 or self.info.rotation == 270
+          size.reverse!
+        end
+        width, height = size
+        
+        # Generate attachment name hash
+        ext = (format[:mobile] ? "-mobile" : "") + (format[:hires] ? "-hires" : "") + ".#{format[:format]}"
+        name = File.join "video", item.name.gsub(/\.#{item.extension}$/, "#{ext}")
+        
+        output = asset_path(name, :video)
+        
+        format[:output] = output
+        
+        # Skip video processing if output already exists...
+        !File.exists?(output) and !((width > self.info.width and height > self.info.height) and format[:hires])
+      end
+      
+      # Detect if thumbnail is required
+      size = self.site.config.attachments.video["thumbnail"].split("x").collect{|x|x.to_i}
+      thumbnails = ([{size: size, hires: false}] + [{size: size.collect{|x| x*2}, hires: true}])
+        .collect {|tn|
+          if self.info.rotation == 90 or self.info.rotation == 270
+            tn[:size].reverse!
+          end
+          tn[:size] = tn[:size].join("x")
+          name = File.join "thumbnail", item.name.gsub(/\.#{item.extension}$/, "#{tn[:hires] ? "-hires" : ""}.png")
+          tn[:output] = asset_path(name, :thumbnail)
+          tn[:label] = "thumbnail#{tn[:hires] ? "-hires" : ""}"
+          tn
+        }
+        .select {|tn| !tn[:hires] or (tn[:hires] and self.site.config.attachments.image.hires)}
+        .select{|tn|
+          width, height = tn[:size].split("x").collect{|i| i.to_i};
+          !File.exists?(tn[:output]) and !((width > self.info.width and height > self.info.height) and format[:hires])
+        }
+      
+      thumbnails = Hash[thumbnails.collect{|tn| [tn[:label], tn]}]
+      
+      encode(generate, thumbnails)
+    end
+    
+    private
+    def variants
+      @variants ||= begin
+        # Collect all formats
+        variants = Hash[
+          self.site.config.attachments.video.formats.collect { |format|
+            [format, {
+              format: format,
+              size: self.site.config.attachments.video["size-sd"],
+              mobile: false,
+              hires: false }]
+          }
+        ]
+        # Generate mobile variants?
+        if self.site.config.attachments.video.mobile
+          variants.merge!(Hash[ variants.collect {|name, fmt| ["#{name}-mobile", fmt.merge({
+            size: self.site.config.attachments.video["size-mobile-sd"],
+            mobile: true,
+          })] } ])
+        end
+        
+        # Generate HD versions
+        if self.site.config.attachments.video.hires
+          variants.merge!(Hash[ variants.collect {|name, fmt| ["#{name}-hires", fmt.merge({
+            size: (self.site.config.attachments.video[(fmt[:mobile] ? "size-mobile-hd" : "size-hd")]),
+            hires: true,
+          })] } ])
+        end
+        
+        
+        # Sort by preferred order
+        formats = self.site.config.attachments.video.formats
+        variants.sort {|a, b|
+          # Sort by preferred format order
+          c = formats.index(a.last[:format]) <=> formats.index(b.last[:format])
+
+          # Sort HD videos on top
+          c == 0 and c = (a.last[:hires] == b.last[:hires]) ? 0 : (a.last[:hires] ? -1 : 1)
+          
+          # Put mobile low
+          c == 0 and c = (a.last[:mobile] == b.last[:mobile]) ? 0 : (a.last[:mobile] ? 1 : -1)
+          
+          c
+        }
+      end
+    end
+  end
+end

data/lib/alula/processors/zencoder.rb

@@ -0,0 +1,199 @@
+require 'alula/processors/video'
+require 'aws-sdk'
+require 'zencoder'
+
+module Alula
+  class Zencoder < VideoProcessor
+    # Register mimetypes
+    mimetype /^video\//
+    
+    def self.available?(options)
+      options.has_key?("token") and options.has_key?("key_id") and options.has_key?("access_key")
+    end
+    
+    def initialize(item, opts)
+      super
+      
+      ::Zencoder.api_key = options.token
+      @s3 = ::AWS::S3.new({
+        :access_key_id     => options.key_id,
+        :secret_access_key => options.access_key,
+      })
+    end
+    
+    def cleanup
+      super
+      
+      @s3 = nil
+      @bucket = nil
+      @object = nil
+      ::Zencoder.api_key = nil
+    end
+    
+    def encode(variants, thumbnails)
+      return if variants.empty? and thumbnails.empty?
+      
+      @variants = variants
+      @thumbnails = thumbnails
+      
+      # Upload attachment to S3 bucket
+      upload_item unless item_uploaded
+      
+      # Create encoding profiles
+      job = {input: "s3://#{@bucket.name}/#{@object.key}", outputs: [], test: !!self.site.config.testing}
+      
+      profiles.each { |name, profile| job[:outputs] << profile }
+      
+      job = zencoder_encode(job)
+      success = zencoder_download(job)
+    end
+    
+    private
+    def item_uploaded
+      @bucket ||= @s3.buckets[self.site.config.attachments.zencoder.bucket]
+      unless @bucket.exists?
+        @bucket = @s3.buckets.create(self.site.config.attachments.zencoder.bucket)
+      end
+      
+      folder = @bucket.objects["#{File.dirname(item.name)}/"]
+      unless folder.exists?
+        folder = @bucket.objects["#{File.dirname(item.name)}/"].write(nil)
+      end
+      
+      @object = @bucket.objects[item.name]
+      return @object.exists?
+    end
+    
+    def upload_item
+      # Fetch upload lock to guarantee that we're onlyones to upload
+      @@upload.synchronize do
+        self.site.progress.create :upload, title: "Uploading #{item.name}", total: File.size(item.filepath)
+        self.site.progress.set_file_transfer(:upload)
+
+        min_chunk_size = 5 * 1024 * 1024  # S3 minimum chunk size (5Mb)
+        @object.multipart_upload do |upload|
+          io = File.open(item.filepath)
+          
+          parts = []
+          
+          bufsize = (io.size > 2 * min_chunk_size) ? min_chunk_size : io.size
+          while buf = io.read(bufsize)
+            md5 = Digest::MD5.base64digest(buf)
+            
+            part = upload.add_part(buf)
+            parts << part
+            
+            self.site.progress.set(:upload, io.pos)
+            if (io.size - (io.pos + bufsize)) < bufsize
+              bufsize = (io.size - io.pos) if (io.size - io.pos) > 0
+            end
+          end
+          
+          upload.complete(parts)
+        end
+        
+        self.site.progress.finish(:upload)
+        
+        # Give Zencoder rights to read file
+        @object.acl.change do |acl|
+          acl.grant(:read).to(:canonical_user_id => '6c8583d84664a381db0c6af0e79b285ede571885fbe768e7ea50e5d3760597dd')
+        end
+      end
+    end
+    
+    def zencoder_encode(job)
+      response = ::Zencoder::Job.create(job)
+      return false if response.code != "201"
+      
+      job_id = response.body["id"]
+      self.site.progress.create "encode-#{job_id}", title: "Encoding #{item.name}", total: 100
+      while (%w{pending waiting processing}.include?((job = ::Zencoder::Job.progress(job_id)).body['state']))
+        self.site.progress.set("encode-#{job_id}", job.body['progress'].to_f)
+        sleep(1)
+      end
+      self.site.progress.finish "encode-#{job_id}"
+      
+      job = ::Zencoder::Job.details(job_id)
+      if job.body['job']['state'] != "finished"
+        return false
+      end
+      return job
+    end
+    
+    def zencoder_download(job)
+      @@download.synchronize do
+        unless @variants.empty?
+          job.body['job']['output_media_files'].each_with_index do |output, idx|
+            output_name = profiles[output['label']][:filename]
+            output_io = File.open(output_name, 'w')
+        
+            open(output['url'], {
+              :content_length_proc => lambda {|t|
+                if t && 0 < t
+                  self.site.progress.create output['label'],
+                    title: "Downloading (%i/%i)" % [idx + 1, job.body['job']['output_media_files'].count],
+                    total: t
+                  self.site.progress.set_file_transfer(output['label'])
+                end
+              },
+              :progress_proc => lambda {|s|
+                  self.site.progress.set(output['label'], s)
+              }
+            }) do |io|
+              output_io.write(io.read)
+            end
+            self.site.progress.finish(output['label'])
+          end
+        end
+        
+        if job.body['job']['thumbnails']
+          # Download thumbnails
+          job.body['job']['thumbnails'].each do |tn|
+            output_file = @thumbnails[tn['group_label']][:output]
+
+            open(tn['url']) { |i| File.open(output_file, 'w') {|o| o.write(i.read) } }
+          end
+        end
+        
+        return true
+      end
+    end
+    
+    def profiles
+      @profiles ||= begin
+        profiles = Hash[@variants.collect { |variant, opts|
+          profile = {
+            label: variant,
+            filename: opts[:output],
+            format: opts[:format],
+            size: opts[:size],
+          }
+          if opts[:mobile]
+            profile[:device_profile] = opts[:hires] ? "mobile/advanced" : "mobile/baseline"
+          end
+          [variant, profile]
+        }]
+        
+        unless profiles.empty?
+          profiles[profiles.keys.first][:thumbnails] = thumbnail_profiles
+        end
+        if profiles.empty? and !thumbnail_profiles.empty?
+          profiles = {thumbnails: { thumbnails: thumbnail_profiles }}
+        end
+        profiles
+      end
+    end
+    
+    def thumbnail_profiles
+      @thumbnail_profiles ||= @thumbnails.collect do |label, tn|
+        {
+          label: tn[:label],
+          number: 1,
+          size: tn[:size],
+        }
+      end
+    end
+  end
+end
+
+Alula::AttachmentProcessor.register('zencoder', Alula::Zencoder)