filey-diff 0.0.1

data/.gitignore

@@ -0,0 +1,2 @@
+Gemfile.lock
+pkg

data/.travis.yml

@@ -0,0 +1,5 @@
+language: ruby
+rvm:
+- 1.8.7
+- 1.9.2
+- 1.9.3

data/Gemfile

@@ -0,0 +1,2 @@
+source :rubygems
+gemspec

data/README.md

@@ -0,0 +1,47 @@
+# Filey diff
+
+[![Build Status](https://secure.travis-ci.org/laurilehmijoki/filey-diff.png)]
+(http://travis-ci.org/laurilehmijoki/filey-diff)
+
+A Ruby library for comparing file-like objects from various data sources.
+
+## Central concepts
+
+### Filey
+
+A file-like object. Can be, for example, a file system file or an AWS S3
+object.
+
+### Data source
+
+Provides Fileys.
+
+The current built-in data sources support Amazon Web Services S3 and file
+system.
+
+## Operations
+
+### List outdated files
+
+Given two data sources A and B, list the changed files that A has but B doesn't.
+
+### List missing files
+
+Given two data sources A and B, list the files that A has but B doesn't.
+
+### List changed files
+
+Given two data sources A and B, list the files on A that have a different MD5
+hash than the corresponding file on B.
+
+## Example use cases
+
+Arnie has a blog on AWS S3. He has just finished a new post and wants to upload
+only the new post into S3. With the help of Filey diff Arnie can write a Ruby
+program that uploads only the new post and nothing else.
+
+## License
+
+Copyright (C) 2012 Lauri Lehmijoki
+
+Distributed under the Apache-2.0 license http://www.apache.org/licenses/LICENSE-2.0.html

data/Rakefile

@@ -0,0 +1,10 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+desc "Build the project"
+task :default => 'test'
+
+desc "Run tests"
+task :test do
+    sh "bundle exec rspec"
+end

data/filey-diff.gemspec

@@ -0,0 +1,26 @@
+Gem::Specification.new do |s|
+  s.name              = 'filey-diff'
+  s.version           = '0.0.1'
+
+  s.summary     = "Compare two data sources that contain file-like objects"
+  s.description =
+    """
+    Find missing or outdated files.
+    For example, compare your local file system to an AWS S3 bucket.
+    """
+
+  s.authors  = ["Lauri Lehmijoki"]
+  s.email    = 'lauri.lehmijoki@iki.fi'
+  s.homepage = 'http://github.com/laurilehmijoki/filey-diff'
+
+  s.require_paths = %w[lib]
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+
+  if RUBY_VERSION < "1.9"
+    s.add_dependency 'require_relative', "~> 1.0.3"
+  end
+
+  s.add_development_dependency 'rake', "~> 0.9"
+  s.add_development_dependency 'rspec', "~> 2.11"
+end

data/lib/filey-diff/comparison.rb

@@ -0,0 +1,35 @@
+module Filey
+  class Comparison
+    def self.list_outdated(data_source_a, data_source_b)
+      select_in_outer_array(data_source_b, data_source_a) { |b_item, a_item|
+        b_item.full_path == a_item.full_path and
+          b_item.last_modified < a_item.last_modified
+      }
+    end
+
+    def self.list_changed(data_source_a, data_source_b)
+      select_in_outer_array(data_source_b, data_source_a) { |b_item, a_item|
+        b_item.full_path == a_item.full_path and
+          b_item.md5 != a_item.md5
+      }
+    end
+
+    def self.list_missing(data_source_a, data_source_b)
+      intersection = select_in_outer_array(data_source_a, data_source_b) do
+        |a_item, b_item|
+        b_item.full_path == a_item.full_path
+      end
+      data_source_a.get_fileys - intersection
+    end
+
+    private
+
+    def self.select_in_outer_array(outer, inner)
+      outer.get_fileys.select { |outer_item|
+        inner.get_fileys.select { |inner_item|
+          yield outer_item, inner_item
+        }.length > 0
+      }
+    end
+  end
+end

data/lib/filey-diff/data-sources/aws_sdk_s3.rb

@@ -0,0 +1,28 @@
+module Filey
+  module DataSources
+    class AwsSdkS3 < DataSource
+      def initialize(s3_bucket)
+        @s3_bucket = s3_bucket
+      end
+
+      private
+
+      def do_internal_load
+        @s3_bucket.objects.map { |s3_object|
+          if (s3_object.key.include?'/')
+            path = s3_object.key.scan(/(.*\/).*/).first.first
+            name = s3_object.key.scan(/.*\/(.*)/).first.first
+          else
+            path = ''
+            name = s3_object.key
+          end
+          normalised_path = "./#{path}"
+          Filey.new(normalised_path,
+                    name,
+                    s3_object.last_modified,
+                    s3_object.etag.gsub(/"/, ''))
+        }
+      end
+    end
+  end
+end

data/lib/filey-diff/data-sources/data_source.rb

@@ -0,0 +1,13 @@
+module Filey
+  module DataSources
+    class DataSource
+      def get_fileys
+        if @cached
+          @cached
+        else
+          @cached = do_internal_load
+        end
+      end
+    end
+  end
+end

data/lib/filey-diff/data-sources/file_system.rb

@@ -0,0 +1,25 @@
+require 'digest/md5'
+
+module Filey
+  module DataSources
+    class FileSystem < DataSource
+      def initialize(root_directory)
+        @root_directory = root_directory
+      end
+
+      private
+
+      def do_internal_load
+        Dir.glob(@root_directory + '/**/*').select { |file|
+          File.file?(file)
+        }.map { |file|
+          path = file.scan(/(.*\/).*/).first.first.sub(@root_directory, '')
+          name = file.scan(/.*\/(.*)/).first.first
+          normalised_path = ".#{path}"
+          md5 = Digest::MD5.hexdigest(File.read(file))
+          Filey.new(normalised_path, name, File.mtime(file), md5)
+        }
+      end
+    end
+  end
+end

data/lib/filey-diff/filey.rb

@@ -0,0 +1,36 @@
+module Filey
+  class Filey
+    attr_reader :path, :name, :last_modified, :md5
+
+    def initialize(path, name, last_modified, md5)
+      raise InvalidPathError unless path.match(/^\..*\/$/)
+      raise InvalidTimeError unless last_modified.instance_of?(Time)
+      raise InvalidNameError if     name.match(/\//)
+      raise InvalidMd5Error  unless md5.length == 32
+      @path = path
+      @name = name
+      @last_modified = last_modified
+      @md5 = md5
+    end
+
+    def full_path
+      @path + @name
+    end
+
+    def <=> (another)
+      full_path <=> another.full_path
+    end
+
+    class InvalidTimeError < Exception
+    end
+
+    class InvalidNameError < Exception
+    end
+
+    class InvalidPathError < Exception
+    end
+
+    class InvalidMd5Error < Exception
+    end
+  end
+end

data/lib/filey-diff.rb

@@ -0,0 +1,7 @@
+require 'rubygems'
+require 'require_relative' if RUBY_VERSION < "1.9"
+require_relative 'filey-diff/data-sources/data_source'
+require_relative 'filey-diff/data-sources/aws_sdk_s3'
+require_relative 'filey-diff/data-sources/file_system'
+require_relative 'filey-diff/filey'
+require_relative 'filey-diff/comparison'

data/spec/comparison_spec.rb

@@ -0,0 +1,77 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe Filey::Comparison do
+  before {
+    @scifi = Filey::Filey.new('./', 'scifi.txt', Time.now,
+                              '9cdfb439c7876e703e307864c9167a15')
+    @scifi_changed = Filey::Filey.new('./', 'scifi.txt', Time.now,
+                              '9cdfb439c7876e703e307864c9167DDD')
+    @deep_space = Filey::Filey.new('./', 'abandoned.txt', Time.now,
+                                   '9cdfb439c7876e703e307864c9167a15')
+    @outdated_file_object = Filey::Filey.new('./', 'foo.txt', Time.now - 10,
+                                             '9cdfb439c7876e703e307864c9167a15')
+    @latest_file_object = Filey::Filey.new('./', 'foo.txt', Time.now,
+                                           '9cdfb439c7876e703e307864c9167a15')
+  }
+
+  context 'finding outdated files' do
+    before {
+      data_source_a = DataSource.new([ @scifi, @latest_file_object ])
+      data_source_b = DataSource.new([ @outdated_file_object, @deep_space ])
+      @outdated_file_objects = Filey::Comparison.list_outdated(data_source_a, data_source_b)
+    }
+
+    it 'lists the outdated files when comparing two data sources' do
+      @outdated_file_objects.length.should be(1)
+    end
+
+    it 'lists the outdated files when comparing two data sources' do
+      @outdated_file_objects.should include(@outdated_file_object)
+    end
+  end
+
+  context 'finding missing files' do
+    before {
+      data_source_a = DataSource.new([ @deep_space ])
+      data_source_b = DataSource.new([ @scifi ])
+      @missing_file_objects = Filey::Comparison.list_missing(data_source_a, data_source_b)
+    }
+
+    it 'lists the missing files when comparing two data sources' do
+      @missing_file_objects.should include(@deep_space)
+    end
+
+    it 'lists the missing files when comparing two data sources' do
+      @missing_file_objects.length.should be(1)
+    end
+  end
+
+  context 'finding changed files' do
+    before {
+      data_source_a = DataSource.new([ @scifi ])
+      data_source_b = DataSource.new([ @scifi_changed ])
+      @changed_files = Filey::Comparison.list_changed(data_source_a, data_source_b)
+    }
+
+    it 'compares filey md5 hashes' do
+      @changed_files.should include(@scifi_changed)
+    end
+
+    it 'compares filey md5 hashes' do
+      @changed_files.length.should be(1)
+    end
+
+    context 'same md5 hashes' do
+      before {
+        data_source_a = DataSource.new([ @scifi ])
+        data_source_b = DataSource.new([ @scifi ])
+        @changed_files = Filey::Comparison.list_changed(data_source_a, data_source_b)
+      }
+
+      it 'notices when the md5 hashes are same' do
+        @changed_files.length.should be(0)
+      end
+    end
+  end
+
+end

data/spec/data_sources_spec.rb

@@ -0,0 +1,65 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+shared_examples "a data source" do |source|
+  let(:data_source) { described_class.new(source) }
+
+  it 'normalises the objects into Fileys' do
+    filey = data_source.get_fileys.sort[0]
+    filey.path.should eq('./cameron/80s/')
+    filey.name.should eq('aliens.txt')
+  end
+
+  it 'provides an md5 hash of the filey content' do
+    filey = data_source.get_fileys.sort[0]
+    filey.md5.should eq(Digest::MD5.hexdigest('Hudson'))
+  end
+
+  it 'normalises the objects into Fileys' do
+    filey = data_source.get_fileys.sort[1]
+    filey.path.should eq('./cameron/90s/')
+    filey.name.should eq('t2.txt')
+  end
+
+  it 'normalises the objects into Fileys' do
+    filey = data_source.get_fileys.sort[2]
+    filey.path.should eq('./')
+    filey.name.should eq('movies.txt')
+  end
+
+  it 'normalises the objects into Fileys' do
+    data_source.get_fileys.each { |file_object|
+      file_object.should be_an_instance_of(Filey::Filey)
+    }
+  end
+end
+
+objects = [
+  { :path => 'cameron/80s/aliens.txt', :mtime => Time.now,
+    :content => 'Hudson' },
+  { :path => 'cameron/90s/t2.txt', :mtime => Time.now,
+    :content => 't1000' },
+  { :path => 'movies.txt', :mtime => Time.now,
+    :content => 'foo' }
+]
+
+describe Filey::DataSources::AwsSdkS3 do
+  s3_bucket = S3Bucket.new(
+    objects.map { |object|
+      S3Object.new(object[:path], object[:mtime], object[:content])
+    }
+  )
+  it_should_behave_like "a data source", s3_bucket
+end
+
+describe Filey::DataSources::FileSystem do
+  require 'tmpdir'
+  @directory = Dir.mktmpdir
+  objects.each { |object|
+    fs_path = "#{@directory}/#{object[:path]}"
+    FileUtils.mkdir_p(fs_path.scan(/(.*\/)/).first.first)
+    File.open(fs_path, 'w') do |file|
+      file.write object[:content]
+    end
+  }
+  it_should_behave_like "a data source", @directory
+end

data/spec/filey_spec.rb

@@ -0,0 +1,81 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe Filey::Filey do
+  context 'path validation' do
+    it 'requires path to start with a dot and end with a slash' do
+      expect {
+        Filey::Filey.new('', 'aliens.txt', Time.now,
+                      '9cdfb439c7876e703e307864c9167a15')
+      }.to raise_error(Filey::Filey::InvalidPathError)
+    end
+
+    it 'requires path to start with a dot and end with a slash' do
+      expect {
+        Filey::Filey.new('.', 'aliens.txt', Time.now,
+                      '9cdfb439c7876e703e307864c9167a15')
+      }.to raise_error(Filey::Filey::InvalidPathError)
+    end
+
+   it 'requires path to start with a dot and end with a slash' do
+      expect {
+        Filey::Filey.new('/', 'aliens.txt', Time.now,
+                      '9cdfb439c7876e703e307864c9167a15')
+      }.to raise_error(Filey::Filey::InvalidPathError)
+    end
+
+    it 'requires path to start with a dot and end with a slash' do
+      Filey::Filey.new('./ripley/', 'aliens.txt', Time.now,
+                      '9cdfb439c7876e703e307864c9167a15')
+    end
+
+    it 'requires path to start with a dot and end with a slash' do
+      Filey::Filey.new('./', 'aliens.txt', Time.now,
+                      '9cdfb439c7876e703e307864c9167a15')
+    end
+  end
+
+  context 'time validation' do
+    it 'requires the last_modified property to be an instance of Time' do
+      expect {
+        Filey::Filey.new('./', 'aliens.txt', '',
+                      '9cdfb439c7876e703e307864c9167a15')
+      }.to raise_error(Filey::Filey::InvalidTimeError)
+    end
+
+    it 'requires the last_modified property to be an instance of Time' do
+      Filey::Filey.new('./', 'aliens.txt', Time.now,
+                      '9cdfb439c7876e703e307864c9167a15')
+    end
+  end
+
+  context 'name validation' do
+    it 'requires the name not to contain slashes' do
+      expect {
+        Filey::Filey.new('./', '/aliens.txt', Time.now,
+                      '9cdfb439c7876e703e307864c9167a15')
+      }.to raise_error(Filey::Filey::InvalidNameError)
+    end
+
+    it 'requires the name not to contain slashes' do
+      Filey::Filey.new('./', 'aliens.txt', Time.now,
+                      '9cdfb439c7876e703e307864c9167a15')
+    end
+  end
+
+  context 'path with name' do
+    it 'can concatenate the path and name' do
+      Filey::Filey.new('./ripley/', 'aliens.txt', Time.now,
+                      '9cdfb439c7876e703e307864c9167a15').
+        full_path.should eq('./ripley/aliens.txt')
+    end
+  end
+
+  context 'md5 validation' do
+    it 'raises an error if the md5 hash is not valid' do
+      expect {
+        Filey::Filey.new('./ripley/', 'aliens.txt', Time.now,
+                       'i-am-not-a-valid-md5-hash')
+      }.to raise_error(Filey::Filey::InvalidMd5Error)
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,35 @@
+require 'rspec'
+require 'digest/md5'
+require File.dirname(__FILE__) + '/../lib/filey-diff'
+
+class S3Object
+  attr_reader :key, :last_modified
+
+  def initialize(key, last_modified, content)
+    @key = key
+    @last_modified = last_modified
+    @content = content
+  end
+
+  def etag
+    Digest::MD5.hexdigest(@content)
+  end
+end
+
+class S3Bucket
+  attr_reader :objects
+
+  def initialize(s3_objects)
+    @objects = s3_objects
+  end
+end
+
+class DataSource
+  def initialize(file_objects)
+    @file_objects = file_objects
+  end
+
+  def get_fileys
+    @file_objects
+  end
+end

metadata

@@ -0,0 +1,103 @@
+--- !ruby/object:Gem::Specification
+name: filey-diff
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Lauri Lehmijoki
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-10-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.11'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.11'
+description: ! "\n    Find missing or outdated files.\n    For example, compare your
+  local file system to an AWS S3 bucket.\n    "
+email: lauri.lehmijoki@iki.fi
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .travis.yml
+- Gemfile
+- README.md
+- Rakefile
+- filey-diff.gemspec
+- lib/filey-diff.rb
+- lib/filey-diff/comparison.rb
+- lib/filey-diff/data-sources/aws_sdk_s3.rb
+- lib/filey-diff/data-sources/data_source.rb
+- lib/filey-diff/data-sources/file_system.rb
+- lib/filey-diff/filey.rb
+- spec/comparison_spec.rb
+- spec/data_sources_spec.rb
+- spec/filey_spec.rb
+- spec/spec_helper.rb
+homepage: http://github.com/laurilehmijoki/filey-diff
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 4129900562195660113
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 4129900562195660113
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Compare two data sources that contain file-like objects
+test_files:
+- spec/comparison_spec.rb
+- spec/data_sources_spec.rb
+- spec/filey_spec.rb
+- spec/spec_helper.rb