smugsync 0.2

data/lib/smugsync/fetch.rb

@@ -0,0 +1,95 @@
+
+module Smug
+
+# TODO: maybe rename
+class FetchCommand < Command
+
+    def exec
+        optparser = Trollop::Parser.new do
+            banner <<-END
+Usage: smug fetch [<options>]
+Show the status of local SmugMug folder.
+
+Options:
+            END
+            opt :force,
+                "Force full refresh of albums and images list",
+                :short => :f
+        end
+
+        opts = Trollop::with_standard_exception_handling(optparser) do
+            optparser.parse(ARGV)
+        end
+
+        authenticate
+
+        status_message "Refreshing albums cache"
+        refresh_cache(:all_albums, opts) { |album, cache_status| status_message "." }
+        status_message " done\n"
+    end
+
+    # block is executed for each album with two argumens: album and cache_status
+    # cache_status is:
+    # - :refreshed      album cache was refreshed
+    # - :fresh          album in cache is the same as in server
+    # options can be:
+    # - :force => true  list of album's images is always refreshed
+    #   this is necessary for refreshing cache after uploads and here's why:
+    #   - smugmug stores LastUpdated data for albums with 1 second precision
+    #   - upload can be fast enough to create album and add image to the album
+    #     in one second
+    def refresh_cache(albums_to_refresh, options = {}, &block)
+        authenticate
+
+        old_cache = if File.exist?(Config::config_file_name("cache"))
+            JSON::parse(Config::config_file("cache", "r").read)
+        else
+            []
+        end
+
+        # - to refresh all albums we need to reconstruct the cache from scratch
+        #   to make sure that deleted albums are not left in the cache
+        # - to refresh selected albums we start from existing cache
+        #   and replace only refreshed albums in that cache
+        albums_on_server = smugmug_albums_get(:Heavy => true)["Albums"]
+        if albums_to_refresh == :all_albums
+            albums_to_refresh = albums_on_server
+            new_cache = []
+        else
+            albums_to_refresh_ids = albums_to_refresh.map { |a| a["id"] }
+            # refetch albums metadata
+            albums_to_refresh = albums_on_server.select { |a| albums_to_refresh_ids.include?(a["id"]) }
+            # leave albums that should not be refreshed in the cache
+            new_cache = old_cache.reject { |a| albums_to_refresh_ids.include?(a["id"]) }
+        end
+
+        albums_to_refresh.each do |album|
+            cached_album = old_cache.find { |a| a["id"] == album["id"] }
+
+            if !options[:force] and cached_album and cached_album["LastUpdated"] == album["LastUpdated"]
+                # album is in cache and is not changed: copy images from old cache
+                # because it can take a long time to get the list of images
+                # from server for large albums
+                album["Images"] = cached_album["Images"]
+                yield(album, :skipped) if block_given?
+            else
+                # album was changed, get the list of images
+                album["Images"] = smugmug_images_get(
+                    :AlbumID => album["id"],
+                    :AlbumKey => album["Key"],
+                    :Heavy => true
+                )["Album"]["Images"]
+                yield(album, :refreshed) if block_given?
+            end
+
+            new_cache << album
+        end
+
+        cache_file = Config::config_file("cache", "w+")
+        cache_file.puts JSON::pretty_generate(new_cache)
+        cache_file.close
+    end
+
+end
+
+end

data/lib/smugsync/init.rb

@@ -0,0 +1,40 @@
+require 'oauth'
+
+module Smug
+
+class InitCommand < Command
+
+    def exec
+        # need to request access token from the user
+        request_token = oauth_consumer.get_request_token
+        # TODO: explain what happens better
+        puts <<-EOS
+Authorize app at:
+#{request_token.authorize_url}&Access=Full&Permissions=Modify
+Press Enter when finished
+        EOS
+        gets
+        access_token = nil
+        begin
+            access_token = request_token.get_access_token
+        rescue OAuth::Unauthorized
+            $stderr.puts "Fatal: Could not authorize with SmugMug. Run 'smug init' again."
+            exit(-1)
+        end
+
+        config = {
+            :access_token => {
+                :secret => access_token.secret,
+                :token => access_token.token
+            }
+        }
+        Smug::Config::create_config_dir
+        File.open(Smug::Config::config_file_name(Smug::Config::ACCESS_TOKEN_CONFIG_FILE), "w+") do |f|
+            f.puts JSON.pretty_generate(config)
+        end
+        puts "Initialized SmugMug folder and authorized with SmugMug"
+    end
+
+end
+
+end

data/lib/smugsync/status.rb

@@ -0,0 +1,123 @@
+
+module Smug
+
+class StatusCommand < Command
+
+    include Utils
+
+    ALBUM_STATUS_FORMAT = "%20s\t%s/(%s files)"
+    IMAGE_STATUS_FORMAT = "%20s\t%s/%s"
+
+    def exec
+        authenticate
+
+        # TODO: support categories and nested categories
+        # for now comparison supports only flat list of albums
+        status = current_dir_status
+        status.each do |album_status|
+            if album_status[:status] != :not_modified
+                puts sprintf(ALBUM_STATUS_FORMAT, album_status[:status], album_status[:album], album_status[:images].length)
+            else
+                album_status[:images].each do |image_status|
+                    puts sprintf(IMAGE_STATUS_FORMAT, image_status[:status], album_status[:album], image_status[:image])
+                end
+            end
+        end
+    end
+
+    def current_dir_status
+        current_dir = Config::relative_to_root(Dir.pwd).to_s
+        if current_dir == '.'
+            albums_status
+        else
+            Dir::chdir('..')
+            status = [album_status(
+                :local => current_dir,
+                :remote => albums.find { |a| a["Title"] == current_dir }
+            )]
+            Dir::chdir(current_dir)
+            status
+        end
+    end
+
+    def albums_status
+        status = []
+
+        local_albums = Pathname::pwd.children(false).find_all { |a| a.directory? and a.basename.to_s != ".smug" }.map { |a| a.basename.to_s }
+        remote_albums = albums.map { |a| a["Title"] }
+
+        status += albums.map do |album|
+            album_status(
+                :remote => album,
+                :local => local_albums.find { |a| a == album["Title"] }
+            )
+        end
+        status += (local_albums - remote_albums).map do |local_album|
+            album_status(:remote => nil, :local => local_album)
+        end
+
+        status
+    end
+
+    def album_status(options = { :local => nil, :remote => nil })
+        local = options[:local]
+        remote = options[:remote]
+
+        if !local and !remote
+            raise "album_status: requires either local or remote album"
+        elsif local and !remote
+            {
+                :album => local,
+                :status => :not_uploaded,
+                :images => local_images(local).map { |img| { :image => img, :status => :not_uploaded} }
+            }
+        elsif !local and remote
+            {
+                :album => remote["Title"],
+                :status => :not_downloaded,
+                :images => remote_images(remote).map { |img| { :image => img, :status => :not_downloaded} }
+            }
+        else
+            # TODO: assure that local and remote titles are the same
+            # TODO: document case insensitivity
+            not_uploaded = local_images(local) - remote_images(remote)
+            not_downloaded = remote_images(remote) - local_images(local)
+
+            {
+                :album => remote["Title"],
+                :status => :not_modified,
+                :images => not_uploaded.map { |img| { :image => img, :status => :not_uploaded} } + not_downloaded.map { |img| { :image => img, :status => :not_downloaded} }
+            }
+        end
+    end
+
+private
+
+    def albums
+        # TODO: status doesn't work without cache
+        @albums ||= JSON::parse Config::config_file("cache", "r").read
+    end
+
+    def local_images(album_path)
+        ignore_list = []
+
+        ignore_file_name = Pathname.new(album_path).join(".smugignore")
+        if File.exists?(ignore_file_name)
+            ignore_list = File.read(ignore_file_name).split($/)
+        end
+
+        Pathname.new(album_path).children(false).reject do |p|
+            filename = p.basename.to_s
+            filename == '.smugignore' or ignore_list.include? filename
+        end.map do |p|
+            p.basename.to_s.gsub(/^\d\d\d\./, '')
+        end
+    end
+
+    def remote_images(album)
+        album["Images"].map {|p| p["FileName"] }
+    end
+
+end
+
+end

data/lib/smugsync/upload.rb

@@ -0,0 +1,117 @@
+require 'system_timer'
+require 'md5'
+require 'set'
+
+module Smug
+
+class UploadCommand < Command
+
+    # current upload algorithm:
+    # 1. get the list not uploaded stuff starting from current dir and below
+    # 2. chdir to root (for simplicity)
+    # 3. for each album
+    #   3.1. create it if it doesn't exist
+    #   3.2. upload files
+    def exec
+        optparser = Trollop::Parser.new do
+            banner <<-END
+Usage: smug upload [<options>]
+Upload files to SmugMug.
+
+Options:
+            END
+            opt :timeout,
+                "Timeout to upload one file in seconds",
+                :short => :t,
+                :default => 24*3600
+        end
+
+        opts = Trollop::with_standard_exception_handling(optparser) do
+            optparser.parse(ARGV)
+        end
+
+        authenticate
+
+        status = StatusCommand.new.current_dir_status
+        Dir::chdir(Config::root_dir)
+
+        modified_albums = Set.new
+
+        status.each_with_index do |album_status, i|
+            if album_status[:status] == :not_uploaded
+                # create album
+                puts "Creating album #{album_status[:album]}"
+                result = smugmug_albums_create(
+                    "Title" => album_status[:album]
+                )
+                raise "Cannot create album" unless result["stat"] == "ok"
+
+                # refresh cache for the newly created album
+                album = smugmug_albums_getInfo(
+                    :AlbumID => result["Album"]["id"],
+                    :AlbumKey => result["Album"]["Key"]
+                )["Album"]
+
+                FetchCommand.new.refresh_cache([album])
+                @albums = nil # TODO: hack to force reparsing of cache file
+            elsif album_status[:status] == :not_downloaded
+                next
+            end
+
+            album = albums.find { |a| a["Title"] == album_status[:album] }
+
+            files_to_upload = album_status[:images].find_all { |i| i[:status] == :not_uploaded }.map { |i| "#{album["Title"]}/#{i[:image]}" }
+
+            File.open("upload.log", "w+") { |f| f.puts "start" }
+            num_files = files_to_upload.length
+            files_to_upload.each_with_index do |filename, i|
+                modified_albums << album
+
+                begin
+                    SystemTimer.timeout_after(opts[:timeout]) do
+                        puts "Uploading #{filename} (#{Time.now.to_s}) (#{i}/#{num_files})"
+
+                        data = File.open(filename, "rb") { |f| f.read }
+
+                        req = {}
+                        req['Content-Length'] = File.size(filename).to_s
+                        req['Content-MD5'] = MD5.hexdigest(data)
+                        req['X-Smug-AlbumID'] = album["id"].to_s
+                        req['X-Smug-Version'] = '1.3.0'
+                        req['X-Smug-FileName'] = filename
+                        req['X-Smug-ResponseType'] = "JSON"
+
+                        results = put("http://upload.smugmug.com/#{filename}", data, req)
+                        puts " => #{results["stat"]} (#{Time.now.to_s})"
+                    end
+                rescue Timeout::Error
+                    puts " => timed out"
+                    File.open("upload.log", "a") { |f| f.puts filename }
+
+                    # TODO: delete image from server
+                rescue Exception => e
+                    puts " => error #{e.message}"
+                    puts e.backtrace.join("\n")
+                    File.open("upload.log", "a") { |f| f.puts filename }
+
+                    # TODO: delete image from server
+                end
+            end
+        end
+
+        # refresh cache for modified albums only
+        puts "Refreshing albums cache"
+        FetchCommand.new.refresh_cache(modified_albums, :force => true)
+    end
+
+private
+
+    def albums
+        # TODO: upload doesn't work without cache
+        @albums ||= JSON::parse Config::config_file("cache", "r").read
+    end
+
+end
+
+end
+

data/lib/smugsync/utils.rb

@@ -0,0 +1,22 @@
+
+module Smug
+
+module Utils
+
+    def status_message(message)
+        $stdout.print message
+        $stdout.flush
+    end
+
+    def method_missing(method, *args, &block)
+        # if method looks like smugmug API call, call get()
+        if method.to_s =~ /^smugmug_/
+            get(method.to_s.gsub('_', '.'), *args, &block)
+        else
+            super(method, *args, &block)
+        end
+    end
+
+end
+
+end

data/lib/smugsync.rb

@@ -0,0 +1,11 @@
+require 'json'
+require 'trollop'
+require 'smugsync/config'
+require 'smugsync/utils'
+require 'smugsync/command'
+require 'smugsync/albums'
+require 'smugsync/upload'
+require 'smugsync/init'
+require 'smugsync/status'
+require 'smugsync/fetch'
+require 'smugsync/download'

data/smugsync.gemspec

@@ -0,0 +1,27 @@
+require 'rubygems'
+
+SPEC = Gem::Specification.new do |s|
+    s.name      = "smugsync"
+    s.version   = "0.2"
+    s.author    = "Alexander Dymo"
+    s.email     = "adymo@kdevelop.org"
+    s.homepage  = "http://github.com/adymo/smugsync"
+    s.platform  = Gem::Platform::RUBY
+    s.summary   = "."
+
+    s.add_development_dependency('bundler',     '>= 1.0.0')
+    s.add_development_dependency('assert_same', '>= 0.1.0')
+
+    s.add_dependency('trollop',         '>= 1.16.0')
+    s.add_dependency('json',            '>= 1.6.0')
+    s.add_dependency('oauth',           '>= 0.4.0')
+    s.add_dependency('system_timer',    '>= 1.2.0')
+
+    s.files         = `git ls-files`.split("\n")
+    s.test_files    = `git ls-files -- test/*`.split("\n")
+    s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+
+    s.require_path      = "lib"
+    s.autorequire       = "smugsync"
+    s.has_rdoc          = false
+end

data/smugsync.kdev4

@@ -0,0 +1,3 @@
+[Project]
+Manager=KDevGenericManager
+Name=smugsync

data/test/local/.gitignore

@@ -0,0 +1 @@
+*

data/test/mock/command.rb

@@ -0,0 +1,15 @@
+
+class Smug::Command
+
+    alias_method :get_without_mock, :get
+    def get(method, params = {})
+        mocked_method = method.gsub(".", "_")
+        server = SmugServer.new
+        if server.respond_to?(mocked_method)
+            JSON.parse(server.send(mocked_method, params))
+        else
+            get_without_mock(method, params)
+        end
+    end
+
+end

data/test/mock/oauth/tokens/request_token.rb

@@ -0,0 +1,11 @@
+
+class OAuth::RequestToken
+
+    TEST_ACCESS_TOKEN = {
+        "access_token" => {
+            :secret => "f186f4ec22d33f1ffda956cd2fdcfa247c0282686cfe5fd80611dd9d62f20a12",
+            :token => "fc5eed66ecef208453842516b1cac024"
+        }
+    }
+
+end

data/test/mock/smug_server.rb

@@ -0,0 +1,196 @@
+
+class SmugServer
+
+    attr_reader :server_albums
+
+    def initialize
+        @server_albums = get_server_albums_info
+    end
+
+    def smugmug_albums_get(params = { :Heavy => false })
+        result = {
+            "stat" => "ok",
+            "method" => "smugmug.albums.get",
+            "Albums" => []
+        }
+
+        server_albums.each do |album|
+            if params[:Heavy]
+                result["Albums"] << album.delete_if { |key, value| key == 'Images' }
+            else
+                result["Albums"] << album.delete_if do |key, value|
+                    not ["id", "Key", "Category", "SubCategory", "Title"].include?(key)
+                end
+            end
+        end
+
+        result.to_json
+    end
+
+    def smugmug_images_get(params = { :AlbumID => nil, :AlbumKey => nil, :Heavy => false })
+        result = {
+            "stat" => "ok",
+            "method" => "smugmug.images.get",
+        }
+
+        album = server_albums.find { |album| album["id"] == params[:AlbumID] and album["Key"] == params[:AlbumKey] }
+        return result unless album
+
+        result["Album"] = {
+            "id" => params[:AlbumID],
+            "Key" => params[:AlbumKey],
+            "ImageCount" => 0,
+            "Images" => []
+        }
+
+        album["Images"].each do |image|
+            result["Album"]["ImageCount"] += 1
+            if params[:Heavy]
+                result["Album"]["Images"] << image
+            else
+                result["Album"]["Images"] << image.delete_if do |key, value|
+                    not ["id", "Key"].include?(key)
+                end
+            end
+        end
+
+        result.to_json
+    end
+
+private
+
+    def get_server_albums_info
+        albums = []
+
+        server_dir = Dir.new(SmugTestCase.server_dir)
+        album_id = 0
+        image_id = 0
+        server_dir.each do |album_name|
+            next if ['.', '..', '.gitignore'].include?(album_name)
+
+            album_full_path = File.join(SmugTestCase.server_dir, album_name)
+            raise "Server directory contains file #{album_name} outside of album" unless File.directory?(album_full_path)
+
+            # nice name in smugmug is the name users see in url
+            # currently we do nothing with that but let's keep it
+            # different from album name
+            album_nice_name = album_name.gsub(" ", "_")
+            album_id += 1
+            album_key = "key#{album_id.to_s(27).tr("0-9a-q", "A-Z")}"
+
+            album = {
+                # default settings for smugmug albums that we do not care about
+                "WordSearchable" => true,
+                "Theme" => {
+                    "Name" => "default",
+                    "id" => 1
+                },
+                "SortMethod" => "Position",
+                "ImageCount" => 1,
+                "Highlight" => {
+                    "id" => 1725930094,
+                    "Key" => "b9fk6m6",
+                    "Type" => "Random"
+                },
+                "External" => true,
+                "Comments" => true,
+                "Clean" => false,
+                "X3Larges" => true,
+                "Keywords" => "",
+                "UnsharpAmount" => 0.2,
+                "Public" => true,
+                "Position" => 1,
+                "UnsharpRadius" => 1,
+                "SortDirection" => false,
+                "Filenames" => true,
+                "ColorCorrection" => 2,
+                "SquareThumbs" => true,
+                "Originals" => true,
+                "SmugSearchable" => true,
+                "Header" => false,
+                "X2Larges" => true,
+                "FamilyEdit" => false,
+                "EXIF" => true,
+                "UnsharpThreshold" => 0.05,
+                "Template" => {
+                    "id" => 0
+                },
+                "HideOwner" => false,
+                "CanRank" => true,
+                "UnsharpSigma" => 1,
+                "Share" => true,
+                "Protected" => false,
+                "Printable" => true,
+                "UploadKey" => "",
+                "Geography" => true,
+                "FriendEdit" => false,
+                "Category" => {
+                    "Name" => "TestCategory",
+                    "id" => 1
+                },
+                "Passworded" => false,
+                "PasswordHint" => "",
+                "Password" => "",
+                "Description" => "",
+
+                # albums settings that we currently care about
+                "id" => album_id,
+                "Key" => album_key,
+                "NiceName" => album_nice_name,
+                "URL" => "http://user.smugmug.com/TestCategory/#{album_nice_name}/#{album_id}_#{album_key}",
+                "Title" => album_name,
+                "LastUpdated" => "2012-02-26 02:40:44",
+                "Images" => []
+            }
+
+            album_dir = Dir.new(album_full_path)
+            album_dir.each do |image_name|
+                next if ['.', '..'].include?(image_name)
+
+                image_id += 1
+                image_key = "key#{image_id.to_s(27).tr("0-9a-q", "A-Z")}"
+
+                album["Images"] << {
+                    # image urls, for now not used/tested
+                    "URL" => "http://url",
+                    "OriginalURL" => "http://original_url",
+                    "X3LargeURL" => "http://xl3_url",
+                    "X2LargeURL" => "http://xl2_url",
+                    "XLargeURL" => "http://xl_url",
+                    "LargeURL" => "http://large_url",
+                    "MediumURL" => "http://medium_url",
+                    "SmallURL" => "http://small_url",
+                    "TinyURL" => "http://tiny_url",
+                    "ThumbURL" => "http://thumb_url",
+                    "LightboxURL" => "http://lightbox_url",
+
+                    # settings that we don't care about
+                    "Status" => "Open",
+                    "Keywords" => "",
+                    "Hidden" => false,
+                    "Serial" => 1,
+                    "Position" => 1,
+                    "Size" => 91144,
+                    "Width" => 638,
+                    "Height" => 638,
+                    "Format" => "JPG",
+                    "Date" => "2012-02-26 02:40:44",
+                    "Watermark" => false,
+                    "MD5Sum" => "d63af258e5ec78db084af9164a8f3581",
+                    "Caption" => "",
+                    "LastUpdated" => "2012-02-26 02:42:01",
+
+                    # albums settings that we currently care about
+                    "FileName" => image_name,
+                    "id" => image_id,
+                    "Key" => image_key,
+                    "Type" => "Album",
+                }
+            end
+
+            albums << album
+        end
+        albums
+    end
+
+end

data/test/server/.gitignore

@@ -0,0 +1 @@
+*