redsync 0.1.2 → 0.2.0

data/config.yml.dist

@@ -1,6 +1,8 @@
 --- 
 :url: "http://your.redmine.url/"
-:project_slug: "project_identifier"
+:projects:
+  - "project_one"
+  - "project_two"
 :username: "username"
 :password: "password"
 :data_dir: "~/redsync"

data/lib/datetime_nil_compare.rb

@@ -0,0 +1,7 @@
+require 'date'
+
+class << nil
+  def to_datetime
+    DateTime.civil
+  end
+end

data/lib/redsync.rb

@@ -8,61 +8,69 @@ require 'date'
 require 'iconv'
 require 'mechanize'
 require 'active_support/all'
+require 'ir_b'
 
 require 'redsync/cli'
-require 'redsync/sync_stat'
+require 'redsync/wiki'
+require 'redsync/wiki_page'
 
 
 class Redsync
 
-  class Result
-    DOWNLOADED = 1
-    SKIPPED_UNKNOWN = 2
-    SKIPPED_OLD = 4
-    UPLOADED = 8
-    CREATED = 16
-    ERROR_ON_UPLOAD = 32
-    ERROR_ON_CREATE = 64
-  end
-
-
+  attr_reader :url,
+              :projects,
+              :username,
+              :data_dir,
+              :extension,
+              :wikis
+
+  # Valid options:
+  #   :url => Redmine's base URL. Required.
+  #   :projects => List of target projects. Required.
+  #   :username => Redmine username. Required.
+  #   :password => Redmine password. Required.
+  #   :data_dir => Directory to read/write. Required.
+  #   :extension => Filename extensions. Defaults to "txt"
   def initialize(options)
-    @config = {}
-    @config.merge! options
-    @config[:data_dir] = File.expand_path(@config[:data_dir])
-    @config[:conflicts_dir] = File.join(@config[:data_dir], "__redsync_conflicts__")
-    puts "Using data dir: #{@config[:data_dir]}"
-
-    @config[:url] = @config[:url].match(/(.*?)\/?$/)[1]
-    @config[:login_url] = @config[:url] + "/login"
-    @config[:wiki_base_url] = @config[:url] + "/projects/" + @config[:project_slug] + "/wiki"
+    options = {
+      :extension => "txt",
+    }.merge(options)
+    
+    @url = options[:url].match(/(.*?)\/?$/)[1]
+    @projects = options[:projects]
+    @username = options[:username]
+    @password = options[:password]
+    @data_dir = File.expand_path(options[:data_dir])
+    @extension = options[:extension]
+
+    @login_url = @url + "/login"
 
     initialize_system_files
 
     @agent = Mechanize.new
-    @syncstat = SyncStat.new(@config, @agent)
-
-    @logged_in = false
   end
 
 
   def initialize_system_files
-    unless File.exist? @config[:data_dir]
-      puts "Creating data dir"
-      FileUtils.mkdir(@config[:data_dir]) 
+    if File.exist? @data_dir
+      puts "Using data dir: #{@data_dir}"
+    else
+      puts "Creating #{@data_dir}"
+      FileUtils.mkdir(@data_dir) 
     end
   end
 
 
   def login
-    puts "Logging in as #{@config[:username]} to #{@config[:login_url]}..."
-    page = @agent.get(@config[:login_url])
+    puts "Logging in as #{@username} to #{@login_url}..."
+    page = @agent.get(@login_url)
     login_form = page.form_with(:action => "/login")
-    login_form.field_with(:name => "username").value = @config[:username]
-    login_form.field_with(:name => "password").value = @config[:password]
+    login_form.field_with(:name => "username").value = @username
+    login_form.field_with(:name => "password").value = @password
     result_page = login_form.submit
     if result_page.search("a.logout").any?
       puts "Logged in successfully."
+      instantiate_wikis
       return true
     else
       puts "Login failed."
@@ -71,76 +79,65 @@ class Redsync
   end
 
 
-  def downsync
-    @syncstat.refresh
-    puts "\nDownsync:"
-    statuses = {
-      Result::DOWNLOADED => 0,
-    }
-    @syncstat.remote_updated_page_names.each do |pagename|
-      statuses[download(pagename)] += 1
+  def instantiate_wikis
+    @wikis = @projects.inject({}) do |sum, project_identifier|
+      sum[project_identifier] = Wiki.new({
+        :url => @url + "/projects/" + project_identifier + "/wiki",
+        :cookies => @agent.cookie_jar.cookies(URI.parse(@url)),
+        :data_dir => File.join(@data_dir, project_identifier),
+        :extension => @extension
+      })
+      sum
     end
-    puts "Downloaded #{statuses[Result::DOWNLOADED]} pages."
   end
 
 
-  def download(pagename)
-    stat = @syncstat.for(pagename)
+  def status_check
+    @projects.each do |project_identifier|
+      wiki = @wikis[project_identifier]
+      wiki.load_pages_cache
+      wiki.scan
 
-    puts "--Download #{pagename}"
-    page = @agent.get(stat[:url] + "/edit")
-    File.open(stat[:local_file], "w+:UTF-8") { |f| f.write(page.search("textarea")[0].text) }
-    @syncstat.update(pagename, :downloaded_at => File.stat(stat[:local_file]).mtime.to_datetime)
-
-    return Result::DOWNLOADED
+      puts "#{wiki.pages_to_download.length} pages to download"
+      puts "#{wiki.pages_to_create.length} pages to create"
+      puts "#{wiki.pages_to_upload.length} pages to upload"
+    end
   end
 
 
-  def upsync
-    puts "\nUpsync:"
-    results = {
-      Result::UPLOADED => 0,
-      Result::CREATED => 0,
-      Result::ERROR_ON_UPLOAD => 0,
-      Result::ERROR_ON_CREATE => 0,
-    }
-    @syncstat.new_page_names.each do |pagename|
-      results[upload(pagename, true)] += 1
+  def sync_all
+    @projects.each do |project_identifier|
+      sync(project_identifier)
     end
-    @syncstat.local_updated_page_names.each do |pagename|
-      results[upload(pagename)] += 1
-    end
-    print "Created #{results[Result::CREATED]} pages."
-    print " (#{results[Result::ERROR_ON_CREATE]} errors)" if results[Result::ERROR_ON_CREATE] > 0
-    print "\n"
-    print "Uploaded #{results[Result::UPLOADED]} pages."
-    print " (#{results[Result::ERROR_ON_UPLOAD]} errors)" if results[Result::ERROR_ON_UPLOAD] > 0
-    print "\n"
   end
 
 
-  def upload(pagename, create = false)
-    now = DateTime.now
-    stat = @syncstat.for(pagename)
-    puts (create ? "--Create #{pagename}" : "--Upload #{pagename}")
+  def sync(project_identifier)
+    wiki = @wikis[project_identifier]
+    wiki.load_pages_cache
+    wiki.scan
+    wiki.downsync
+    wiki.upsync
+  end
 
-    page = @agent.get(@config[:wiki_base_url] + "/" + pagename + "/edit")
-    form = page.form_with(:id=>"wiki_form")
-    form.field_with(:name=>"content[text]").value = File.open(stat[:local_file], "r:UTF-8").read
-    result_page = form.submit
-    errors = result_page.search("#errorExplanation li").map{|li|li.text}
 
-    if errors.any?
-      print "--Error: #{pagename}: "
-      puts errors
-      return (create ? Result::ERROR_ON_CREATE : Result::ERROR_ON_UPLOAD)
-    else
-      now = DateTime.now
-      @syncstat.update(pagename, {
-        :downloaded_at => now,
-        :remote_updated_at => now
-      })
-      return (create ? Result::CREATED : Result::UPLOADED)
+  def interactive
+    wikis.each do |project_identifier, wiki|
+      wiki.load_pages_cache
+      wiki.scan
     end
+    ir b
   end
+
+
+  def to_s
+    str = "#<Redsync"
+    str << " url = \"#{@url}\"\n"
+    str << " username = \"#{@username}\"\n"
+    str << " projects = \"#{@projects}\"\n"
+    str << " data_dir = \"#{@data_dir}\"\n"
+    str << " extension = \"#{@extension}\"\n"
+    str << ">"
+  end
+
 end

data/lib/redsync/cli.rb

@@ -1,5 +1,6 @@
 require 'optparse'
 require 'yaml'
+require 'redsync'
 
 class Redsync
   class CLI
@@ -12,33 +13,40 @@ class Redsync
         redsync = Redsync.new(YAML.load_file(@options.delete(:config_file)).merge(@options))
         exit unless redsync.login
 
-        time do
-          redsync.downsync unless @options[:uponly]
-          redsync.upsync unless @options[:downonly]
+        case @options[:run_mode]
+        when :full_sync
+          time do
+            redsync.sync_all
+          end
+        when :interactive
+          redsync.interactive
+        when :status_check
+          redsync.status_check
         end
       end
 
 
       def parse_options
         @options = {
+          :run_mode => :full_sync,
           :config_file => "~/redsync.yml",
         }
 
         OptionParser.new do |opts|
           opts.banner = "Usage: redsync [options]"
-          opts.on("-v", "--[no-]verbose", "Output verbose logs") do |v|
+          opts.on("-v", "--verbose", "Output verbose logs") do |v|
             @options[:verbose] = v
           end
           opts.on("-c", "--config FILE", "Use specified config file instead of ~/redsync.yml") do |file|
             @options[:config_file] = file
           end
-          opts.on("-u", "--upsync-only", "Upsync only, don't downsync") do |v|
-            @options[:uponly] = v
+          opts.on("-s", "--status", "Status check. No uploads or downloads will happen") do |v|
+            @options[:run_mode] = :status_check
           end
-          opts.on("-d", "--downsync-only", "Downsync only, don't upsync") do |v|
-            @options[:downonly] = v
+          opts.on("-i", "--interactive", "Interactive mode (irb)") do |v|
+            @options[:run_mode] = :interactive
           end
-          opts.on("-D", "--debug", "Debug mode. Requires ruby-debug19") do |v|
+          opts.on("-D", "--debugger", "Debug mode. Requires ruby-debug19") do |v|
             @options[:debug] = v
           end
         end.parse!
@@ -46,6 +54,7 @@ class Redsync
         if @options[:debug]
           require 'ruby-debug'
           Debugger.settings[:autoeval] = true
+          Debugger.settings[:reload_source_on_change] = true
         end
 
         @options[:config_file] = File.expand_path(@options[:config_file])
@@ -74,7 +83,7 @@ class Redsync
           else
             c =~ /^y/i
           end
-        block.call if result && block
+        (result && block) ? block.call : result
       end
 
       

data/lib/redsync/sync_stat.rb

@@ -26,7 +26,7 @@ class Redsync
         links.each do |link|
           url = @config[:url] + link.attr("href")
           name = URI.decode(url.match(/^#{@config[:wiki_base_url]}\/(.*)$/)[1]).force_encoding("UTF-8")
-          local_file = File.join(@config[:data_dir], "#{name}.txt")
+          local_file = File.join(@config[:data_dir], "#{name}.#{@config[:extension]}")
 
           remote_updated_at = DateTime.parse(h3.text + "T00:00:00" + now.zone)
 
@@ -59,10 +59,10 @@ class Redsync
         next if File.directory?(fullpath)
         next if file =~ /^__redsync_/
         name = Iconv.iconv("UTF-8", "UTF-8-MAC", file) if RUBY_PLATFORM =~ /darwin/
-        name = name.first.match(/(.*)\.txt$/)[1]
+        name = name.first.match(/(.*)\.#{extension}$/)[1]
         next if @stat[name]
 
-        local_file = File.join(@config[:data_dir], "#{name}.txt")
+        local_file = File.join(@config[:data_dir], "#{name}.#{extension}")
         local_updated_at = File.stat(local_file).mtime.to_datetime
         update(name, {
           :name => name,

data/lib/redsync/wiki.rb

@@ -0,0 +1,168 @@
+require 'uri'
+require 'mechanize'
+require 'yaml'
+require 'iconv'
+
+class Redsync
+  class Wiki
+
+    attr_reader :url,
+                :data_dir,
+                :extension
+
+    # Valid option values:
+    #   :url => This wiki's base url. Required
+    #   :cookies => Mechanize::Cookie objects that's already logged in to Redmine. Required
+    #   :data_dir => Directory to read/write to. Required.
+    #   :extension => File extensions for page files. Defaults to "txt"
+    def initialize(options)
+      @url = options[:url].match(/(.*?)\/?$/)[1]
+      @data_dir = File.expand_path(options[:data_dir])
+      @extension = options[:extension]
+
+      @agent = Mechanize.new
+      options[:cookies].each do |cookie|
+        @agent.cookie_jar.add(URI.parse(@url), cookie)
+      end
+
+      @pages_cache = {}
+      @pages_cache_file = File.join(@data_dir, "__redsync_pages_cache.yml")
+
+      initialize_system_files
+    end
+
+
+    def cookies
+      @agent.cookie_jar.cookies(URI.parse(@url))
+    end
+
+    def initialize_system_files
+      unless File.exist? @data_dir
+        puts "Creating #{@data_dir}"
+        FileUtils.mkdir(@data_dir) 
+      end
+    end
+
+
+    def pages
+      @pages_cache
+    end
+
+
+    def downsync
+      queue = pages_to_download
+      queue.each_with_index do |page, i|
+        puts "--Download (#{i+1} of #{queue.count}) #{page.name}"
+        page.download
+      end
+      self.write_pages_cache
+    end
+
+
+    def upsync
+      queue = pages_to_create
+      queue.each_with_index do |page, i|
+        puts "--Create (#{i+1} of #{queue.count}) #{page.name}"
+        page.upload
+      end
+      self.write_pages_cache
+
+      queue = pages_to_upload
+      queue.each_with_index do |page, i|
+        puts "--Upload (#{i+1} of #{queue.count}) #{page.name}"
+        page.upload
+      end
+      self.write_pages_cache
+    end
+
+
+    def pages_to_download
+      list = []
+      list += @pages_cache.values.select { |page| page.exists_in == :remote_only }
+      list += @pages_cache.values.select { |page| page.exists_in == :both && !page.synced_at }
+      list += @pages_cache.values.select { |page| page.exists_in == :both && page.synced_at && (page.remote_updated_at > page.synced_at) }
+      list
+    end
+
+
+    def pages_to_create
+      list = []
+      list += @pages_cache.values.select { |page| page.exists_in == :local_only }
+      list
+    end
+
+
+    def pages_to_upload
+      list = []
+      list += @pages_cache.values.select { |page| page.exists_in == :both && (page.local_updated_at > page.synced_at) }
+      list
+    end
+
+
+    def scan
+      scan_remote
+      scan_local
+    end
+
+
+    def scan_remote
+      webpage = @agent.get(@url + "/date_index")
+      now = DateTime.now
+
+      # Get remote and local update times using remote list
+      webpage.search("#content h3").each do |h3|
+        links = h3.next_element.search("a")
+        links.each do |link|
+          url = URI.parse(@url).merge(link.attr("href")).to_s
+          wiki_page = WikiPage.new(self, url)
+          @pages_cache[wiki_page.name] = wiki_page unless @pages_cache[wiki_page.name]
+          @pages_cache[wiki_page.name].remote_updated_at = h3.text
+        end
+      end
+    end
+
+
+    def scan_local
+      Dir.entries(@data_dir).each do |file|
+        next if File.directory? file
+        next if file =~ /^__redsync_/
+        page_name = file.match(/([^\/\\]+?)\.#{@extension}$/)[1]
+        page_name = Iconv.iconv("UTF-8", "UTF-8-MAC", page_name).first if RUBY_PLATFORM =~ /darwin/
+        next if pages[page_name]
+        pp file
+        wiki_page = WikiPage.new(self, page_name)
+        pages[page_name] = wiki_page
+      end
+    end
+
+
+    def write_pages_cache
+      File.open(@pages_cache_file, "w+:UTF-8") do |f|
+        f.write(self.pages.values.map{ |page| page.to_hash}.to_yaml)
+      end
+    end
+
+
+    def load_pages_cache
+      return unless File.exist? @pages_cache_file
+      @pages_cache = {}
+      YAML.load_file(@pages_cache_file).each do |page_hash|
+        wiki_page = WikiPage.new(self, page_hash[:name])
+        wiki_page.remote_updated_at = page_hash[:remote_updated_at]
+        wiki_page.synced_at = page_hash[:synced_at]
+        @pages_cache[page_hash[:name]] = wiki_page
+      end
+    end
+
+
+    def to_s
+      str = "#<Redsync::Wiki"
+      str << " url = \"#{@url}\"\n"
+      str << " data_dir = \"#{@data_dir}\"\n"
+      str << " extension = \"#{@extension}\"\n"
+      str << " pages = #{@pages_cache.count}\n"
+      str << ">"
+    end
+
+  end
+end

data/lib/redsync/wiki_page.rb

@@ -0,0 +1,189 @@
+require 'fileutils'
+require 'date'
+require 'active_support/all'
+require 'redsync/wiki'
+
+class Redsync
+  class WikiPage
+    attr_reader   :name,
+                  :local_file,
+                  :url,
+                  :local_updated_at
+    attr_accessor :remote_updated_at,
+                  :synced_at
+
+
+    def initialize(wiki, name_or_url_or_fullpath)
+      @wiki = wiki
+
+      if name_or_url_or_fullpath =~ /^#{@wiki.data_dir}\/(.*)$/
+        @local_file = name_or_url_or_fullpath
+        @name = $1
+        @url = @wiki.url + "/" + @name
+      elsif name_or_url_or_fullpath =~ /^#{@wiki.url}\/(.*)$/
+        @url = name_or_url_or_fullpath
+        @name = URI.decode($1)
+        @local_file = File.join(@wiki.data_dir, "#{@name}.#{@wiki.extension}")
+      else
+        @name = name_or_url_or_fullpath
+        @local_file = File.join(@wiki.data_dir, "#{@name}.#{@wiki.extension}")
+        @url = @wiki.url + "/" + URI.encode(@name)
+      end
+
+      @agent = Mechanize.new
+      @wiki.cookies.each do |cookie|
+        @agent.cookie_jar.add(URI.parse(@url), cookie)
+      end
+    end
+
+
+    # Returns one of :remote_only, :local_only, :both or :nowhere
+    # (:nowhere should never happen...)
+    def exists_in
+      if self.local_exists? && self.remote_exists?
+        return :both
+      elsif self.local_exists?
+        return :local_only
+      elsif self.remote_exists?
+        return :remote_only
+      else
+        return :nowhere
+      end
+    end
+    
+
+    def local_exists?
+      File.exist? @local_file
+    end
+
+
+    def local_updated_at
+      if local_exists?
+        File.stat(@local_file).mtime.to_datetime
+      else
+        nil
+      end
+    end
+
+
+    def remote_exists?
+      !remote_updated_at.nil?
+    end
+
+
+    def remote_updated_at
+      at = @remote_updated_at
+      now = DateTime.now
+      if at && ([at.year, at.month, at.day, at.hour, at.minute, at.second] == [now.year, now.month, now.day, 0, 0, 0])
+        return @remote_updated_at = history[0][:timestamp]
+      else
+        return at
+      end
+    end
+
+
+    def remote_updated_at=(value)
+      if value
+        value = DateTime.parse(value.to_s)
+        value = DateTime.parse(value.to_s(:db) + DateTime.now.zone) if value.utc?
+        @remote_updated_at = value
+      end
+    end
+
+
+    def synced_at
+      @synced_at
+    end
+
+
+    def synced_at=(value)
+      if value
+        value = DateTime.parse(value.to_s)
+        value = DateTime.parse(value.to_s(:db) + DateTime.now.zone) if value.utc?
+        @synced_at = value
+      end
+    end
+
+
+    def history
+      puts "--Getting page history for #{@name}"
+      now = DateTime.now
+      history = []
+      page = @agent.get(@url + "/history")
+      page.search("table.wiki-page-versions tbody tr").each do |tr|
+        timestamp = DateTime.parse(tr.search("td")[3].text + now.zone) 
+        author_name = tr.search("td")[4].text.strip
+        history << {
+          :timestamp => timestamp,
+          :author_name => author_name
+        }
+      end
+      history
+    end
+
+    def read
+      page = @agent.get(@url + "/edit")
+      page.search("textarea")[0].text
+    end
+
+
+    def download_to(file)
+      File.open(file, "w+:UTF-8") { |f| f.write(self.read) }
+    end
+
+
+    def download
+      download_to(@local_file)
+      self.synced_at = self.local_updated_at
+    end
+
+
+    def write(text)
+      now = DateTime.now
+      page = @agent.get(@url + "/edit")
+      form = page.form_with(:id=>"wiki_form")
+      form.field_with(:name=>"content[text]").value = text
+      result_page = form.submit
+      errors = result_page.search("#errorExplanation li").map{ |li| li.text}
+    end
+
+
+    def write_from_file(file)
+      write(File.open(file, "r:UTF-8").read)
+    end
+
+    
+    def upload
+      write_from_file(@local_file)
+      self.synced_at = self.remote_updated_at = DateTime.now
+    end
+
+
+    def to_hash
+      {
+        :name => @name,
+        :url => @url,
+        :local_file => @local_file,
+        :remote_updated_at => @remote_updated_at,
+        :local_exists => local_exists?,
+        :synced_at => @synced_at,
+      }
+    end
+
+
+    def to_s
+      str = "#<Redsync::WikiPage"
+      str << " name = \"#{name}\"\n"
+      str << " url = \"#{url}\"\n"
+      str << " local_file = \"#{local_file}\"\n"
+      str << " remote_exists? = #{remote_exists?}\n"
+      str << " remote_updated_at = #{@remote_updated_at ? @remote_updated_at : "<never>"}\n"
+      str << " local_exists? = #{local_exists?}\n"
+      str << " local_updated_at = #{local_updated_at ? local_updated_at : "<never>"}\n"
+      str << " synced_at = #{@synced_at ? @synced_at : "<never>"}\n"
+      str << ">"
+    end
+
+    
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: redsync
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.2.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-12-07 00:00:00.000000000Z
+date: 2011-12-09 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
-  requirement: &2160416520 !ruby/object:Gem::Requirement
+  requirement: &2155972500 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: '2.0'
   type: :runtime
   prerelease: false
-  version_requirements: *2160416520
+  version_requirements: *2155972500
 - !ruby/object:Gem::Dependency
   name: activesupport
-  requirement: &2160416120 !ruby/object:Gem::Requirement
+  requirement: &2155972080 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,7 +32,18 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2160416120
+  version_requirements: *2155972080
+- !ruby/object:Gem::Dependency
+  name: ir_b
+  requirement: &2155971560 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *2155971560
 description: Sync Redmine's wiki pages to your local filesystem. Edit as you like,
   then upsync.
 email: merikonjatta@gmail.com
@@ -41,8 +52,11 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- lib/datetime_nil_compare.rb
 - lib/redsync/cli.rb
 - lib/redsync/sync_stat.rb
+- lib/redsync/wiki.rb
+- lib/redsync/wiki_page.rb
 - lib/redsync.rb
 - README.textile
 - config.yml.dist