massive_sitemap 2.0.0.rc3 → 2.0.0.rc4

data/CHANGELOG.md

@@ -7,6 +7,7 @@
   * updated Docu
   * switch to writer chain
   * add BigSitemap API
+  * FileManifest -> read all files into streams take that as reference
 
 ## v2.0.0 - 13-02-2012
   _inital release_
@@ -26,3 +27,4 @@
   * manifest handling:
   * moved Amazon S3 integration to [massive_sitemap-writer-s3](https://github.com/rngtng/massive_sitemap-writer-s3)
   * updated/fixed Ping
+  * move LockingFile into MassiveSitemap scope

data/VERSION

@@ -1 +1 @@
-2.0.0.rc3
+2.0.0.rc4

data/lib/massive_sitemap/builder/index.rb

@@ -11,10 +11,10 @@ module MassiveSitemap
       def initialize(writer, options = {}, &block)
         super(writer, options) do
           writer.each do |path, last_modified|
+            next if writer.current && path.include?(writer.current)
             add path, :last_modified => last_modified
           end
         end
-
       end
 
       def self.generate(writer, options = {}, &block)
@@ -22,7 +22,7 @@ module MassiveSitemap
       end
 
       def self.index_url(builder, writer)
-        ::File.join(builder.send(:url), writer.options[:filename])
+        ::File.join(builder.send(:url), writer.current)
       end
 
       def add_url!(location, attrs = {})

data/lib/massive_sitemap/builder/rotating.rb

@@ -7,17 +7,13 @@ module MassiveSitemap
       NUM_URLS = 1..50_000
 
       OPTS = Base::OPTS.merge(
-        :max_per_sitemap => NUM_URLS.max
+        :max_urls => NUM_URLS.max
       )
 
       def initialize(writer, options = {}, &block)
         @rotations = 0
         @urls      = 0
 
-        if options[:max_per_sitemap] && !NUM_URLS.member?(options[:max_per_sitemap])
-          raise ArgumentError, %Q(":max_per_sitemap" must be greater than #{NUM_URLS.min} and smaller than #{NUM_URLS.max})
-        end
-
         super
       end
 
@@ -34,7 +30,7 @@ module MassiveSitemap
       end
 
       def add_url!(location, attrs = {})
-        if @urls >= @options[:max_per_sitemap]
+        if @urls >= @options[:max_urls]
           close!
         end
         super

data/lib/massive_sitemap/lock.rb

@@ -0,0 +1,20 @@
+require 'fileutils'
+
+# Create Lock
+
+module MassiveSitemap
+  LOCK_FILE = 'generator.lock'
+
+  def lock!(&block)
+    if block
+      raise Errno::EACCES if ::File.exists?(LOCK_FILE)
+      ::File.open(LOCK_FILE, 'w', ::File::EXCL)
+      begin
+        block.call
+      ensure
+        FileUtils.rm(LOCK_FILE) #unlock!
+      end
+    end
+  end
+  module_function :lock!
+end

data/lib/massive_sitemap/writer/base.rb

@@ -11,22 +11,6 @@ module MassiveSitemap
         @stream  = nil
       end
 
-      # Interface
-      def open_stream
-        @string ||= StringIO.new
-      end
-
-      def close_stream(stream)
-      end
-
-      def init?
-        true
-      end
-
-      def streams
-        []
-      end
-
       # API
       def init!(options = {})
         close!
@@ -54,6 +38,35 @@ module MassiveSitemap
       def each(&block)
         streams.each(&block)
       end
+
+      def current
+        stream
+      end
+
+      # def flush!
+      #  @streams = []
+      # end
+
+      # Interface
+      protected
+      def open_stream
+        @string ||= StringIO.new
+      end
+
+      def close_stream(stream)
+      end
+
+      def init?
+        true
+      end
+
+      def streams
+        @streams ||= []
+      end
+
+      def stream
+        nil
+      end
     end
 
   end

data/lib/massive_sitemap/writer/file.rb

@@ -13,9 +13,9 @@ module MassiveSitemap
         :root            => '.',
         :force_overwrite => false,
         :filename        => "sitemap.xml",
-        :index_filename  => "sitemap_index.xml",
       )
 
+      protected
       def open_stream
         ::File.dirname(tmp_filename).tap do |dir|
           FileUtils.mkdir_p(dir) unless ::File.exists?(dir)
@@ -39,11 +39,14 @@ module MassiveSitemap
 
       def streams
         files.map do |path|
-          next if path.include?(options[:index_filename])
           [::File.basename(path), ::File.stat(path).mtime]
         end.compact
       end
 
+      def stream
+        options[:filename]
+      end
+
       private
       def filename
         ::File.join options[:root], options[:filename]

data/lib/massive_sitemap/writer/gzip_file.rb

@@ -9,6 +9,7 @@ module MassiveSitemap
     class GzipFile < File
       OPTS = File::OPTS
 
+      protected
       def open_stream
         ::Zlib::GzipWriter.new(super)
       end

data/lib/massive_sitemap/writer/string.rb

@@ -7,11 +7,7 @@ module MassiveSitemap
   module Writer
 
     class String < Base
-
-      def open_stream
-        @string ||= StringIO.new
-      end
-
+      # accessors to conent, mainly used in tests
       def to_s
         @string.string rescue ""
       end
@@ -23,6 +19,11 @@ module MassiveSitemap
       def include?(other_string)
         to_s.include?(other_string)
       end
+
+      protected
+      def open_stream
+        @string ||= StringIO.new
+      end
     end
 
   end

data/lib/massive_sitemap.rb

@@ -2,6 +2,7 @@ require 'massive_sitemap/writer/file'
 require 'massive_sitemap/writer/gzip_file'
 require 'massive_sitemap/builder/rotating'
 require 'massive_sitemap/builder/index'
+require 'massive_sitemap/lock'
 require 'massive_sitemap/ping'
 
 # Page at -> <url>
@@ -32,20 +33,26 @@ module MassiveSitemap
   }
 
   def generate(options = {}, &block)
-    @options = DEFAULTS.merge options
+    lock! do
+      @options = DEFAULTS.merge options
 
-    unless @options[:url]
-      raise ArgumentError, 'you must specify ":url" string'
-    end
-    @options[:index_url] ||= @options[:url]
+      unless @options[:url]
+        raise ArgumentError, 'you must specify ":url" string'
+      end
+      @options[:index_url] ||= @options[:url]
+
+      if @options[:max_urls] && !Builder::Rotating::NUM_URLS.member?(@options[:max_urls])
+        raise ArgumentError, %Q(":max_urls" must be greater than #{NUM_URLS.min} and smaller than #{NUM_URLS.max})
+      end
 
-    @options[:writer] = Writer::GzipFile if @options[:gzip]
+      @options[:writer] = Writer::GzipFile if @options[:gzip]
 
-    @writer = @options[:writer].new @options
-    Builder::Rotating.generate(@writer, @options, &block)
+      @writer = @options[:writer].new @options
+      Builder::Rotating.generate(@writer, @options, &block)
 
-    @writer.init!(:filename => @options[:index_filename], :force_overwrite => true)
-    Builder::Index.generate(@writer, @options.merge(:url => @options[:index_url]))
+      @writer.init!(:filename => @options[:index_filename], :force_overwrite => true)
+      Builder::Index.generate(@writer, @options.merge(:url => @options[:index_url]))
+    end
   end
   module_function :generate
 

data/spec/builder/index_spec.rb

@@ -23,8 +23,16 @@ describe MassiveSitemap::Builder::Index do
     writer.should include("<loc>http://test.de/test</loc>")
   end
 
-  it 'returns index_url' do
-    writer = MassiveSitemap::Writer::File.new(:filename => "sitemap_index.xml")
-    MassiveSitemap::Builder::Index.generate(writer, :url => "test.de").should == "http://test.de/sitemap_index.xml"
+  context "with file writer" do
+    let(:index_filename) { "sitemap_index.xml" }
+    let(:writer) { MassiveSitemap::Writer::File.new(:filename => index_filename) }
+
+    after do
+      FileUtils.rm(index_filename)
+    end
+
+    it 'returns index_url' do
+      MassiveSitemap::Builder::Index.generate(writer, :url => "test.de").should == "http://test.de/sitemap_index.xml"
+    end
   end
 end

data/spec/builder/rotating_spec.rb

@@ -8,12 +8,6 @@ describe MassiveSitemap::Builder::Rotating do
   let(:writer) { MassiveSitemap::Writer::String.new }
   let(:builder) { MassiveSitemap::Builder::Rotating.new(writer) }
 
-  it 'raises error when max_per_sitemap > MAX_URLS' do
-    expect do
-      MassiveSitemap::Builder::Rotating.new(writer, :max_per_sitemap => MassiveSitemap::Builder::Rotating::NUM_URLS.max + 1)
-    end.to raise_error(ArgumentError)
-  end
-
   it 'generates one url' do
     MassiveSitemap::Builder::Rotating.new(writer) do
       add_url! 'test'
@@ -22,7 +16,7 @@ describe MassiveSitemap::Builder::Rotating do
   end
 
   it 'generates two url' do
-    MassiveSitemap::Builder::Rotating.new(writer, :max_per_sitemap => 1) do
+    MassiveSitemap::Builder::Rotating.new(writer, :max_urls => 1) do
       add_url! 'test'
       add_url! 'test2'
     end
@@ -42,7 +36,7 @@ describe MassiveSitemap::Builder::Rotating do
     it 'generates two url' do
       expect do
         expect do
-          MassiveSitemap::Builder::Rotating.new(writer, :max_per_sitemap => 1) do
+          MassiveSitemap::Builder::Rotating.new(writer, :max_urls => 1) do
             add 'test'
             add 'test2'
           end
@@ -54,7 +48,7 @@ describe MassiveSitemap::Builder::Rotating do
       File.open(filename, 'w') {}
       expect do
         expect do
-          MassiveSitemap::Builder::Rotating.new(writer, :max_per_sitemap => 1) do
+          MassiveSitemap::Builder::Rotating.new(writer, :max_urls => 1) do
             begin
               add 'test'
             rescue MassiveSitemap::Writer::File::FileExistsException => e

data/spec/lock_spec.rb

@@ -0,0 +1,46 @@
+require "massive_sitemap/lock"
+
+describe MassiveSitemap do
+  describe "lock!" do
+    let(:lock_file) { MassiveSitemap::LOCK_FILE }
+
+    after do
+      FileUtils.rm(lock_file) rescue nil
+    end
+
+    it 'does nothing without block' do
+      MassiveSitemap.lock!
+      ::File.exists?(lock_file).should be_false
+    end
+
+    it 'creates lockfile' do
+      File.exists?(lock_file).should be_false
+      MassiveSitemap.lock! do
+        ::File.exists?(lock_file).should be_true
+      end
+    end
+
+    it 'deletes lockfile' do
+      MassiveSitemap.lock! {}
+      ::File.exists?(lock_file).should be_false
+    end
+
+    it 'deletes lockfile in case of error' do
+      expect do
+        MassiveSitemap.lock! do
+          raise ArgumentError
+        end
+      end.to raise_error
+      ::File.exists?(lock_file).should be_false
+    end
+
+    it 'fails if lockfile exists' do
+      ::File.open(lock_file, 'w',) {}
+      expect do
+        MassiveSitemap.lock! do
+          puts "Hi"
+        end
+      end.to raise_error
+    end
+  end
+end

data/spec/massive_sitemap_spec.rb

@@ -29,6 +29,12 @@ describe MassiveSitemap do
         end.to raise_error(ArgumentError)
       end
 
+      it 'raises error when max_urls > MAX_URLS' do
+        expect do
+          MassiveSitemap.generate(:max_urls => MassiveSitemap::Builder::Rotating::NUM_URLS.max + 1)
+        end.to raise_error(ArgumentError)
+      end
+
       it "does not create empty sitemap file" do
         expect do
           MassiveSitemap.generate(:url => 'test.de/')

data/spec/writer/file_spec.rb

@@ -47,13 +47,13 @@ describe MassiveSitemap::Writer::File do
     end
   end
 
-  it 'create file' do
+  it 'creates file' do
     expect do
       writer.close!
     end.to change { File.exists?(filename) }.to(true)
   end
 
-  it 'create second file on rotation' do
+  it 'creates second file on rotation' do
     expect do
       expect do
         writer.close!
@@ -63,19 +63,19 @@ describe MassiveSitemap::Writer::File do
     end.to change { File.exists?(filename2) }.to(true)
   end
 
-  it 'write into file' do
+  it 'writes into file' do
     writer.print 'test'
     writer.close!
     `cat '#{filename}'`.should == "test"
   end
 
-  it 'init new file closes current' do
+  it 'inits new file closes current' do
     writer.print 'test'
     writer.init!(:filename => filename2)
     `cat '#{filename}'`.should == "test"
   end
 
-  it 'write into second file' do
+  it 'writes into second file' do
     writer.print 'test'
     writer.init!(:filename => filename2)
     writer.print 'test2'
@@ -83,15 +83,11 @@ describe MassiveSitemap::Writer::File do
     `cat '#{filename2}'`.should == "test2"
   end
 
-  context "opening write file" do
+  context "file exists" do
     before do
       File.open(filename, 'w') {}
     end
 
-    after do
-      FileUtils.rm(filename) rescue nil
-    end
-
     it 'raises when file exits' do
       writer = MassiveSitemap::Writer::File.new
       expect do
@@ -99,10 +95,11 @@ describe MassiveSitemap::Writer::File do
       end.to raise_error(MassiveSitemap::Writer::File::FileExistsException)
     end
 
-    it 'raises when file exits' do
+    it "dosn't raise when overwrite set" do
       writer = MassiveSitemap::Writer::File.new(:force_overwrite => true)
       expect do
         writer.init!
+        writer.close!
       end.to_not raise_error(MassiveSitemap::Writer::File::FileExistsException)
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: massive_sitemap
 version: !ruby/object:Gem::Version
-  version: 2.0.0.rc3
+  version: 2.0.0.rc4
   prerelease: 6
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-02-12 00:00:00.000000000Z
+date: 2012-02-13 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70334535721800 !ruby/object:Gem::Requirement
+  requirement: &70276344811920 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70334535721800
+  version_requirements: *70276344811920
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70334535721340 !ruby/object:Gem::Requirement
+  requirement: &70276340400560 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,7 +32,7 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70334535721340
+  version_requirements: *70276340400560
 description: MassiveSitemap - build huge sitemaps painfree. Differential updates keeps
   generation time short and reduces load on DB. It's heavealy inspired by BigSitemaps
   and offers compatiable API
@@ -55,22 +55,22 @@ files:
 - lib/massive_sitemap/builder/base.rb
 - lib/massive_sitemap/builder/index.rb
 - lib/massive_sitemap/builder/rotating.rb
+- lib/massive_sitemap/lock.rb
 - lib/massive_sitemap/ping.rb
 - lib/massive_sitemap/writer/base.rb
 - lib/massive_sitemap/writer/file.rb
 - lib/massive_sitemap/writer/gzip_file.rb
-- lib/massive_sitemap/writer/locking_file.rb
 - lib/massive_sitemap/writer/string.rb
 - massive_sitemap.gemspec
 - spec/builder/base_spec.rb
 - spec/builder/index_spec.rb
 - spec/builder/rotating_spec.rb
+- spec/lock_spec.rb
 - spec/massive_sitemap_spec.rb
 - spec/ping_spec.rb
 - spec/spec_helper.rb
 - spec/writer/file_spec.rb
 - spec/writer/gzip_file_spec.rb
-- spec/writer/locking_file_spec.rb
 homepage: http://github.com/rngtng/massive_sitemap
 licenses: []
 post_install_message: 
@@ -99,9 +99,9 @@ test_files:
 - spec/builder/base_spec.rb
 - spec/builder/index_spec.rb
 - spec/builder/rotating_spec.rb
+- spec/lock_spec.rb
 - spec/massive_sitemap_spec.rb
 - spec/ping_spec.rb
 - spec/spec_helper.rb
 - spec/writer/file_spec.rb
 - spec/writer/gzip_file_spec.rb
-- spec/writer/locking_file_spec.rb

data/lib/massive_sitemap/writer/locking_file.rb

@@ -1,33 +0,0 @@
-require 'zlib'
-
-require "massive_sitemap/writer/file"
-# Create Lock before writing to file
-
-module MassiveSitemap
-  module Writer
-
-    class LockingFile < File
-      OPTS = File::OPTS
-
-      LOCK_FILE = 'generator.lock'
-
-      def open_stream
-        ::File.open(LOCK_FILE, 'w', ::File::EXCL) #lock!
-        super
-      end
-
-      def close_stream(stream)
-        super
-        FileUtils.rm(LOCK_FILE) #unlock!
-      end
-
-      def init?
-        if ::File.exists?(LOCK_FILE)
-          raise Errno::EACCES
-        end
-        super
-      end
-    end
-
-  end
-end

data/spec/writer/locking_file_spec.rb

@@ -1,34 +0,0 @@
-require "massive_sitemap/writer/locking_file"
-
-describe MassiveSitemap::Writer::LockingFile do
-  let(:filename) { 'sitemap.xml' }
-  let(:tmp_filename) { "#{filename}.tmp" }
-  let(:lock_file) {  MassiveSitemap::Writer::LockingFile::LOCK_FILE }
-  let(:writer) { MassiveSitemap::Writer::LockingFile.new.tap { |w| w.init! } }
-
-  after do
-    FileUtils.rm(filename) rescue nil
-    FileUtils.rm(tmp_filename) rescue nil
-    FileUtils.rm(lock_file) rescue nil
-  end
-
-  it 'creates lockfile' do
-    expect do
-      writer
-    end.to change { File.exists?(lock_file) }.to(true)
-  end
-
-  it 'deletes lockfile' do
-    writer
-    expect do
-      writer.close!
-    end.to change { File.exists?(lock_file) }.to(false)
-  end
-
-  it 'fails if lockfile exists' do
-    File.open(lock_file, 'w') {}
-    expect do
-      writer
-    end.to raise_error
-  end
-end