massive_sitemap 2.0.0.rc5 → 2.0.0.rc6

data/CHANGELOG.md

@@ -29,4 +29,4 @@
   * updated/fixed Ping
   * move LockingFile into MassiveSitemap scope
   * FileManifest -> read all files into streams take that as reference
-
+  * add test for stream_id delete

data/VERSION

@@ -1 +1 @@
-2.0.0.rc5
+2.0.0.rc6

data/lib/massive_sitemap/writer/file.rb

@@ -17,7 +17,7 @@ module MassiveSitemap
 
       def initialize(options = {})
         super
-        @files = Dir[::File.join(@options[:root], "*#{::File.extname(filename)}")]
+        init_stream_ids
       end
 
       protected
@@ -35,12 +35,12 @@ module MassiveSitemap
         # Move from tmp_file into acutal file
         ::File.delete(filename) if ::File.exists?(filename)
         ::File.rename(tmp_filename, filename)
-        @files << filename
+        add_stream_id(filename)
         rotate
       end
 
       def init?
-        if !@options[:force_overwrite] && @files.include?(filename)
+        if !@options[:force_overwrite] && find_stream_id(filename)
           error_message = "Can not create file: #{filename} exits"
           rotate #push next possible filename
           raise FileExistsException, error_message
@@ -48,12 +48,58 @@ module MassiveSitemap
         true
       end
 
-      def stream_ids
-        @files.map do |path|
-          [::File.basename(path), ::File.stat(path).mtime]
+      # Keep state of Files
+      def init_stream_ids
+        @stream_ids = {}
+        load_stream_ids
+
+        if @options[:partial_update]
+          delete_stream_ids upper_stream_ids(@stream_ids.keys)
+        end
+
+        if (keep_file_for = @options[:keep_file_for].to_i) > 0
+          delete_stream_ids chaos_monkey_stream_ids(@stream_ids.keys.sort, keep_file_for)
+        end
+      end
+
+      def load_stream_ids
+        Dir[::File.join(@options[:root], "*#{::File.extname(filename)}")].each do |path|
+          add_stream_id(path, ::File.stat(path).mtime)
+        end
+      end
+
+      def upper_stream_ids(stream_id_keys)
+        {}.tap do |cluster|
+          stream_id_keys.each do |path|
+            filename, rotation,  ext = split_filename(path)
+            _,        rotation2, _   = split_filename(cluster[filename])
+            if rotation.to_i >= rotation2.to_i
+              cluster[filename] = path
+            end
+          end
+        end.values
+      end
+
+      def chaos_monkey_stream_ids(stream_id_keys, days)
+        return [] if days < 1
+        offset = Time.now.to_i / (24 * 60 * 60)
+        (0...stream_id_keys.size).step(days).map do |index|
+          stream_id_keys.at((offset % days) + index)
         end.compact
       end
 
+      def delete_stream_ids(to_delete)
+        @stream_ids.delete_if { |key, value| to_delete.include?(key) }
+      end
+
+      def find_stream_id(path)
+        @stream_ids.keys.include?(::File.basename(path))
+      end
+
+      def add_stream_id(path, last_modified = Time.now)
+        @stream_ids[::File.basename(path)] = last_modified
+      end
+
       def stream_id
         @stream_id && ::File.basename(@stream_id)
       end
@@ -79,6 +125,7 @@ module MassiveSitemap
       def split_filename(filename)
         filename.to_s.scan(/^([^.]*?)(?:-([0-9]+))?(\..+)?$/).flatten
       end
+
     end
 
   end

data/spec/lock_spec.rb

@@ -37,9 +37,7 @@ describe MassiveSitemap do
     it 'fails if lockfile exists' do
       ::File.open(lock_file, 'w') {}
       expect do
-        MassiveSitemap.lock! do
-          puts "Hi"
-        end
+        MassiveSitemap.lock! {}
       end.to raise_error
     end
   end

data/spec/writer/file_spec.rb

@@ -1,4 +1,5 @@
 require "spec_helper"
+require "time"
 
 require "massive_sitemap/writer/file"
 
@@ -168,4 +169,49 @@ describe MassiveSitemap::Writer::File do
       end
     end
   end
+
+  describe "upper_stream_ids" do
+    let(:writer) {  MassiveSitemap::Writer::File.new }
+
+    it { writer.send(:upper_stream_ids, %w(sitemap-1.xml)).should == %w(sitemap-1.xml) }
+    it { writer.send(:upper_stream_ids, %w(sitemap-2.xml sitemap-1.xml)).should == %w(sitemap-2.xml) }
+    it { writer.send(:upper_stream_ids, %w(sitemap.xml sitemap_user-1.xml)).should == %w(sitemap.xml sitemap_user-1.xml) }
+  end
+
+  describe "chaos_monkey_stream_ids" do
+    let(:writer) {  MassiveSitemap::Writer::File.new }
+
+    context "one file" do
+      it { writer.send(:chaos_monkey_stream_ids, %w(sitemap-1.xml), 0).should == [] }
+      it { writer.send(:chaos_monkey_stream_ids, %w(sitemap-1.xml), 1).should == %w(sitemap-1.xml) }
+
+      it "keeps file for 2 days" do
+        Time.stub!(:now).and_return(Time.parse("1-1-2012"))
+        writer.send(:chaos_monkey_stream_ids, %w(sitemap-1.xml), 2).should == []
+      end
+
+      it "deletes file on snd day" do
+        Time.stub!(:now).and_return(Time.parse("2-1-2012"))
+        writer.send(:chaos_monkey_stream_ids, %w(sitemap-1.xml), 2).should == %w(sitemap-1.xml)
+      end
+    end
+
+    context "many files" do
+      it "keeps file for 2 days" do
+        Time.stub!(:now).and_return(Time.parse("1-1-2012"))
+        writer.send(:chaos_monkey_stream_ids, %w(sitemap-1.xml sitemap-2.xml sitemap-3.xml), 2).should == %w(sitemap-2.xml)
+      end
+
+      it "deletes file on 2nd day" do
+        Time.stub!(:now).and_return(Time.parse("2-1-2012"))
+        writer.send(:chaos_monkey_stream_ids, %w(sitemap-1.xml sitemap-2.xml sitemap-3.xml), 2).should == %w(sitemap-1.xml sitemap-3.xml)
+      end
+
+      it "deletes file on 3rd day" do
+        Time.stub!(:now).and_return(Time.parse("3-1-2012"))
+        writer.send(:chaos_monkey_stream_ids, %w(sitemap-1.xml sitemap-2.xml sitemap-3.xml), 2).should == %w(sitemap-2.xml)
+      end
+    end
+  end
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: massive_sitemap
 version: !ruby/object:Gem::Version
-  version: 2.0.0.rc5
+  version: 2.0.0.rc6
   prerelease: 6
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ date: 2012-02-15 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70096501425420 !ruby/object:Gem::Requirement
+  requirement: &70204488347100 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70096501425420
+  version_requirements: *70204488347100
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70096501424960 !ruby/object:Gem::Requirement
+  requirement: &70204488346640 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,7 +32,7 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70096501424960
+  version_requirements: *70204488346640
 description: MassiveSitemap - build huge sitemaps painfree. Differential updates keeps
   generation time short and reduces load on DB. It's heavealy inspired by BigSitemaps
   and offers compatiable API