hasta 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: adeafa9f4dae6acb9dd59dc39432d4e4b3d86b79
-  data.tar.gz: 3f15a082f3c72d9a75df32e728cff7cc0ff88787
+  metadata.gz: 181a02eb6d37332af6efde332af20305f692f3e0
+  data.tar.gz: b484f8ad5a8884b7f13e03caee7277f6b00d521b
 SHA512:
-  metadata.gz: 0e0dc797ebb1acab0781324a47ed6184f51797a1b3893da9ac069024dc9cfadd56f6dcb5c1338855e59c1c0a82745c601102fd62e4de58a198ed278332ac5f24
-  data.tar.gz: 84a5056c2c069512a99eb8b4d9b58c9dcb0827732ffe555f7708705da0579706be7659abbfc6f4b97e4e458b02e01d2a25c38489c240e0a166d1e2b316d87132
+  metadata.gz: a2206da5b8b905a66037ac9c23e7faab4c4a3254ef127988620bbe36eb6a04273677d8cf42785dacb889c0b7b35c9636549462a80ca6849f2b576839141b9756
+  data.tar.gz: c7d96b9a9d4383e25156cad5ef58250b9713881eb70251b83347724264f08ef7adbe8ecdbc261f17c776bb12d6106466beb167a1284736fd8a0147a79d2bef25

data/.travis.yml

@@ -0,0 +1,6 @@
+language: ruby
+rvm:
+    - 1.8.7
+    - 1.9.3
+    - 2.0.0
+script: bundle exec rake

data/README.md

@@ -1,4 +1,5 @@
 # Hasta
+[![travis-ci](https://travis-ci.org/swipely/hasta.png?branch=master)](https://travis-ci.org/swipely/hasta)
 
 <b>HA</b>doop <b>S</b>treaming <b>T</b>est h<b>A</b>rness for Amazon EMR
 
@@ -85,15 +86,17 @@ To control which credentials Hasta is using when communicating with S3, update y
 ## Data Filtering
 
 Hasta automatically filters input data to minimize execution time.
-By default, Hasta looks for a file in the current directory named `filter_config.txt` for the filtering configuration.
+By default, Hasta looks for a file in the current directory named `filter_config.yml` for the filtering configuration.
 You can change the filter configuration file by setting the `HASTA_DATA_FILTER_FILE` environment variable.
 If you want to disable data filtering, you can set the `HASTA_DATA_FILTERING` environment variable to `OFF`.
 
 ### Filter Configuration
 
-The filter configuration file contains regular expressions, one per line.
-Any line of input data that matches at least one of the regular expressions is included in the test input.
-Lines that do no match any of the regular expressions are excluded from the test input.
+The filter configuration file is a YAML file containing a Hash that maps S3 URIs (as Strings) to Arrays of regular expressions (also as Strings)
+Any line of input data that comes from an S3 URI whose prefix matches one of the S3 URIs in the filter configuration that matches at least one of the regular expressions is included in the test input.
+Any line of input data that comes from an S3 URI whose prefix matches one of the S3 URIs in the filter configuration that does not match any of the regular expressions is excluded from the test input.
+Input data that does not come from an S3 URI whose prefix matches on of the S3 URIs in the filter configuration is not filtered.
+If an input S3 URI matches multiple S3 URIs in the filter configuration, the most specific match is the one that is chosen for filtering purposes.
 
 ### Caching
 

data/lib/hasta/configuration.rb

@@ -14,7 +14,7 @@ module Hasta
   # Global configuration settings
   class Configuration
     attr_accessor :project_root
-    attr_writer :local_storage_root, :cache_storage_root, :project_steps, :logger, :filter
+    attr_writer :local_storage_root, :cache_storage_root, :project_steps, :logger, :filters
 
     def local_storage_root
       @local_storage_root ||= '~/fog'
@@ -43,13 +43,13 @@ module Hasta
       )
     end
 
-    def filter
-      unless @filter || ENV['HASTA_DATA_FILTERING'] == 'OFF'
-        filter_file = ENV['HASTA_DATA_FILTER_FILE'] || 'filter_config.txt'
-        @filter ||= Filter.from_file(filter_file)
+    def filters
+      unless @filters || ENV['HASTA_DATA_FILTERING'] == 'OFF'
+        filter_file = ENV['HASTA_DATA_FILTER_FILE'] || 'filter_config.yml'
+        @filters ||= Filters.from_file(filter_file)
       end
 
-      @filter
+      @filters
     end
 
     private
@@ -76,9 +76,9 @@ module Hasta
     end
 
     def resolver
-      if filter
+      if filters
         ResolveCachedS3File.new(
-          S3FileCache.new(fog_cache_storage), ResolveFilteredS3File.new(filter)
+          S3FileCache.new(fog_cache_storage), ResolveFilteredS3File.new(filters)
         )
       else
         Hasta::Storage::ResolveS3File

data/lib/hasta/filters.rb

@@ -0,0 +1,34 @@
+# Copyright Swipely, Inc.  All rights reserved.
+
+require 'yaml'
+
+require 'hasta/s3_uri'
+
+module Hasta
+  # Defines filters for different S3 path prefixes
+  class Filters
+    def self.from_file(file)
+      Hasta.logger.debug "Loading data filter file: #{File.expand_path(file)}"
+      new(YAML.load_file(file))
+    rescue => ex
+      raise ConfigurationError.new,
+        "Failed to load filter configuration file: #{file} - #{ex.message}"
+    end
+
+    def initialize(filters)
+      @filters = filters.map { |s3_uri, regexes|
+        [S3URI.parse(s3_uri), Filter.new(*regexes.map { |regex| Regexp.new(regex) }) ]
+      }.sort_by { |s3_uri, regexes| s3_uri.depth }.reverse
+    end
+
+    def for_s3_uri(target_s3_uri)
+      if match = filters.find { |s3_uri, filter| target_s3_uri.start_with?(s3_uri) }
+        match[1]
+      end
+    end
+
+    private
+
+    attr_reader :filters
+  end
+end

data/lib/hasta/resolve_filtered_s3_file.rb

@@ -1,22 +1,27 @@
 # Copyright Swipely, Inc.  All rights reserved.
 
+require 'hasta/filters'
 require 'hasta/filtered_s3_file'
 
 module Hasta
   # Creates a Hasta filtered S3 file instance given a Fog file
   class ResolveFilteredS3File
-    def initialize(filter, child_resolver = Hasta::Storage::ResolveS3File)
-      @filter = filter
+    def initialize(filters, child_resolver = Hasta::Storage::ResolveS3File)
+      @filters = filters
       @child_resolver = child_resolver
     end
 
     def resolve(fog_file)
-      FilteredS3File.new(child_resolver.resolve(fog_file), filter)
+      s3_file = child_resolver.resolve(fog_file)
+      if filter = filters.for_s3_uri(s3_file.s3_uri)
+        FilteredS3File.new(s3_file, filter)
+      else
+        s3_file
+      end
     end
 
-
     private
 
-    attr_reader :filter, :child_resolver
+    attr_reader :filters, :child_resolver
   end
 end

data/lib/hasta/s3_uri.rb

@@ -20,7 +20,7 @@ module Hasta
     end
 
     def directory?
-      path.end_with?('/')
+      path.nil? || path.end_with?('/')
     end
 
     def file?
@@ -35,6 +35,24 @@ module Hasta
       end
     end
 
+    def depth
+      slashes = (path && path.chars.count { |ch| ch == '/' }) || 0
+      if path.nil?
+        1
+      elsif directory?
+        1 + slashes
+      else
+        2 + slashes
+      end
+    end
+
+    def start_with?(s3_uri)
+      return true if self == s3_uri
+      return false if s3_uri.file?
+
+      (bucket == s3_uri.bucket) && (s3_uri.path.nil? || path.start_with?(s3_uri.path))
+    end
+
     def parent
       if path.nil?
         nil

data/lib/hasta/version.rb

@@ -1,3 +1,3 @@
 module Hasta
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

data/spec/fixtures/hasta/filter_config.yml

@@ -0,0 +1,3 @@
+---
+s3://my-bucket:
+- \AOnly Allow This Line\z

data/spec/hasta/configuration_spec.rb

@@ -5,25 +5,19 @@ require 'spec_helper'
 require 'hasta/in_memory_data_source'
 
 describe Hasta::Configuration do
-  describe '#filter' do
+  describe '#filters' do
     subject { described_class.new }
 
     context 'given a custom path is specified' do
       before do
-        ENV['HASTA_DATA_FILTER_FILE'] = 'spec/fixtures/hasta/filter_config.txt'
+        ENV['HASTA_DATA_FILTER_FILE'] = 'spec/fixtures/hasta/filter_config.yml'
       end
 
       after do
         ENV.delete('HASTA_DATA_FILTER_FILE')
       end
 
-      let(:allowed_line) { 'Only Allow This Line' }
-      let(:disallowed_line) { 'Do Not Allow Any Other Lines' }
-      let(:filter_proc) { subject.filter.to_proc }
-
-      it { expect(subject.filter).to_not be_nil }
-      it { expect(filter_proc.call(allowed_line)).to be_true }
-      it { expect(filter_proc.call(disallowed_line)).to_not be_true }
+      it { expect(subject.filters).to_not be_nil }
 
       context 'given filtering is disabled' do
         before do
@@ -41,7 +35,7 @@ describe Hasta::Configuration do
         let(:local_storage_root) { Dir.mktmpdir('config_test_local_dir') }
         let(:local_s3_uri) { Hasta::S3URI.new('my-bucket', 'path/to/my/file.txt') }
 
-        it { expect(subject.filter).to be_nil }
+        it { expect(subject.filters).to be_nil }
         it { expect(subject.combined_storage.files_for(local_s3_uri)).to_not be_empty }
       end
     end

data/spec/hasta/emr_job_definition_spec.rb

@@ -6,13 +6,13 @@ describe Hasta::EmrJobDefinition do
   shared_context 'with a filter defined' do
     before do
       Hasta.configure do |config|
-        config.filter = Hasta::Filter.new(/.*/)
+        config.filters = Hasta::Filters.new({ 's3://my-bucket' => ['.*'] })
       end
     end
 
     after do
       Hasta.configure do |config|
-        config.filter = nil
+        config.filters = nil
       end
     end
   end

data/spec/hasta/filters_spec.rb

@@ -0,0 +1,55 @@
+# Copyright Swipely, Inc.  All rights reserved.
+
+require 'spec_helper'
+
+require 'hasta/filters'
+
+describe Hasta::Filters do
+  describe '.from_file' do
+    let(:non_existent_file) { 'spec/fixtures/hasta/non_existent_file.dll' }
+
+    context 'given an non-existent file' do
+      it 'raises a ConfigurationError' do
+        expect {
+          described_class.from_file(non_existent_file)
+        }.to raise_error(Hasta::ConfigurationError)
+      end
+    end
+  end
+
+  describe '#for_s3_uri' do
+    subject { described_class.new(filters) }
+
+    let(:filters) {
+      {
+        's3://my-bucket/path1/' => ['.*'],
+        's3://my-bucket/path/to/file.txt' => ['[a-z]{2}', '\A_.*'],
+        's3://my-bucket/path1/path2/' => ['z.+', 'a.+', 'x.?'],
+      }
+    }
+
+    context 'given no match' do
+      let(:s3_uri) { Hasta::S3URI.new('other-bucket', 'path/to/file.txt') }
+
+      it 'returns nil' do
+        expect(subject.for_s3_uri(s3_uri)).to be_nil
+      end
+    end
+
+    context 'given a single match' do
+      let(:s3_uri) { Hasta::S3URI.new('my-bucket', 'path/to/file.txt') }
+
+      it 'selects the only match' do
+        expect(subject.for_s3_uri(s3_uri).to_s).to eq('#<Hasta::Filter:[/[a-z]{2}/, /\\A_.*/]>')
+      end
+    end
+
+    context 'given multiple matches' do
+      let(:s3_uri) { Hasta::S3URI.new('my-bucket', 'path1/path2/file3.txt') }
+
+      it 'selects the most specific match' do
+        expect(subject.for_s3_uri(s3_uri).to_s).to eq('#<Hasta::Filter:[/a.+/, /x.?/, /z.+/]>')
+      end
+    end
+  end
+end

data/spec/hasta/resolve_filtered_s3_file_spec.rb

@@ -0,0 +1,30 @@
+# Copyright Swipely, Inc.  All rights reserved.
+
+require 'spec_helper'
+
+describe Hasta::ResolveFilteredS3File do
+  describe '#resolve' do
+    subject { described_class.new(filters) }
+
+    let(:filters) { Hasta::Filters.new('s3://my-bucket/path/to/my/files/' => ['\A\d{1,3}.*']) }
+    let(:fog_file) {
+      double('Fog::File',
+        :directory => double('Fog::Directory', :key => bucket_name),
+        :key => path
+      )
+    }
+    let(:bucket_name) { 'my-bucket' }
+
+    context 'given a filtered file' do
+      let(:path) { 'path/to/my/files/1.txt' }
+
+      it { expect(subject.resolve(fog_file)).to be_kind_of(Hasta::FilteredS3File) }
+    end
+
+    context 'given a non-filtered file' do
+      let(:path) { 'path/to/your/files/1.txt' }
+
+      it { expect(subject.resolve(fog_file)).to be_kind_of(Hasta::S3File) }
+    end
+  end
+end

data/spec/hasta/resolved_cached_s3_file_spec.rb

@@ -9,8 +9,8 @@ describe Hasta::ResolveCachedS3File do
     subject { described_class.new(file_cache, child_resolver) }
 
     let(:file_cache) { double(Hasta::S3FileCache) }
-    let(:child_resolver) { Hasta::ResolveFilteredS3File.new(filter) }
-    let(:filter) { Hasta::Filter.new(/.*/) }
+    let(:child_resolver) { Hasta::ResolveFilteredS3File.new(filters) }
+    let(:filters) { Hasta::Filters.new({ "s3://#{bucket_name}" => ['.*'] }) }
 
     let(:fog_file) {
       double('Fog::File',
@@ -24,9 +24,10 @@ describe Hasta::ResolveCachedS3File do
     let(:path) { 'path/to/my/file.txt' }
     let(:body) { "Parts\n" }
     let(:s3_uri) { Hasta::S3URI.new(bucket_name, path) }
+    let(:exp_filter) { filters.for_s3_uri(s3_uri) }
 
     let(:exp_fingerprint) {
-      Digest::MD5.hexdigest("#{Digest::MD5.hexdigest(body)}_#{filter.to_s}")
+      Digest::MD5.hexdigest("#{Digest::MD5.hexdigest(body)}_#{exp_filter.to_s}")
     }
 
     let(:result) { subject.resolve(fog_file) }

data/spec/hasta/s3_uri_spec.rb

@@ -14,6 +14,7 @@ describe Hasta::S3URI do
 
       it { expect(subject.bucket).to eq(bucket) }
       it { expect(subject.path).to be_nil }
+      it { expect(subject.depth).to eq(1) }
     end
 
     context 'given a bucket and a file path' do
@@ -23,6 +24,7 @@ describe Hasta::S3URI do
 
       it { expect(subject.bucket).to eq(bucket) }
       it { expect(subject.path).to eq(path) }
+      it { expect(subject.depth).to eq(4) }
     end
 
     context 'given a bucket and a directory path' do
@@ -32,6 +34,7 @@ describe Hasta::S3URI do
 
       it { expect(subject.bucket).to eq(bucket) }
       it { expect(subject.path).to eq(path) }
+      it { expect(subject.depth).to eq(4) }
     end
 
     context 'given an s3n URI with a bucket and a directory path' do
@@ -41,6 +44,7 @@ describe Hasta::S3URI do
 
       it { expect(subject.bucket).to eq(bucket) }
       it { expect(subject.path).to eq(path) }
+      it { expect(subject.depth).to eq(4) }
     end
 
     context 'given an invalid uri' do
@@ -66,6 +70,12 @@ describe Hasta::S3URI do
 
       it { expect(subject).to_not be_file }
     end
+
+    context 'given a bucket only path' do
+      let(:path) { nil }
+
+      it { expect(subject).to_not be_file }
+    end
   end
 
   describe '#basename' do
@@ -148,4 +158,24 @@ describe Hasta::S3URI do
       it { expect(subject.parent).to eq(described_class.new(bucket, 'path/to/my/')) }
     end
   end
+
+  describe '#start_with?' do
+    subject { described_class.new(bucket, path) }
+
+    let(:bucket) { 'my-bucket' }
+    let(:path) { 'path/to/my/favorite/file.txt' }
+
+    let(:other_bucket_uri) { described_class.new('some-other-bucket', path) }
+    let(:other_path_uri) { described_class.new(bucket, 'path/to/your/favorite/file.txt') }
+    let(:similar_prefix_uri) { described_class.new(bucket, 'path/to/my/fav') }
+    let(:bucket_only_uri) { described_class.new(bucket, nil) }
+
+    it { expect(subject.start_with?(subject)).to be_true }
+    it { expect(subject.start_with?(subject.parent)).to be_true }
+    it { expect(subject.start_with?(subject.parent.parent)).to be_true }
+    it { expect(subject.start_with?(other_bucket_uri)).to be_false }
+    it { expect(subject.start_with?(other_path_uri)).to be_false }
+    it { expect(subject.start_with?(similar_prefix_uri)).to be_false }
+    it { expect(subject.start_with?(bucket_only_uri)).to be_true }
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: hasta
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - danhodge
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-09 00:00:00.000000000 Z
+date: 2014-04-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fog
@@ -118,6 +118,7 @@ files:
 - .cane
 - .gitignore
 - .rspec
+- .travis.yml
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -134,6 +135,7 @@ files:
 - lib/hasta/execution_context.rb
 - lib/hasta/filter.rb
 - lib/hasta/filtered_s3_file.rb
+- lib/hasta/filters.rb
 - lib/hasta/identity_mapper.rb
 - lib/hasta/identity_reducer.rb
 - lib/hasta/in_memory_data_sink.rb
@@ -157,7 +159,7 @@ files:
 - lib/hasta/tasks.rb
 - lib/hasta/tasks/runner.rb
 - lib/hasta/version.rb
-- spec/fixtures/hasta/filter_config.txt
+- spec/fixtures/hasta/filter_config.yml
 - spec/fixtures/hasta/json/emr_node.json
 - spec/fixtures/hasta/json/pipeline_definition.json
 - spec/fixtures/hasta/lib/failing_mapper.rb
@@ -175,6 +177,7 @@ files:
 - spec/hasta/execution_context_spec.rb
 - spec/hasta/filter_spec.rb
 - spec/hasta/filtered_s3_file_spec.rb
+- spec/hasta/filters_spec.rb
 - spec/hasta/identity_mapper_spec.rb
 - spec/hasta/identity_reducer_spec.rb
 - spec/hasta/interpolate_string_spec.rb
@@ -182,6 +185,7 @@ files:
 - spec/hasta/local_storage_spec.rb
 - spec/hasta/mapper_spec.rb
 - spec/hasta/reducer_spec.rb
+- spec/hasta/resolve_filtered_s3_file_spec.rb
 - spec/hasta/resolved_cached_s3_file_spec.rb
 - spec/hasta/s3_data_source_spec.rb
 - spec/hasta/s3_file_cache_spec.rb
@@ -217,7 +221,7 @@ signing_key:
 specification_version: 4
 summary: HAdoop Streaming Test hArness
 test_files:
-- spec/fixtures/hasta/filter_config.txt
+- spec/fixtures/hasta/filter_config.yml
 - spec/fixtures/hasta/json/emr_node.json
 - spec/fixtures/hasta/json/pipeline_definition.json
 - spec/fixtures/hasta/lib/failing_mapper.rb
@@ -235,6 +239,7 @@ test_files:
 - spec/hasta/execution_context_spec.rb
 - spec/hasta/filter_spec.rb
 - spec/hasta/filtered_s3_file_spec.rb
+- spec/hasta/filters_spec.rb
 - spec/hasta/identity_mapper_spec.rb
 - spec/hasta/identity_reducer_spec.rb
 - spec/hasta/interpolate_string_spec.rb
@@ -242,6 +247,7 @@ test_files:
 - spec/hasta/local_storage_spec.rb
 - spec/hasta/mapper_spec.rb
 - spec/hasta/reducer_spec.rb
+- spec/hasta/resolve_filtered_s3_file_spec.rb
 - spec/hasta/resolved_cached_s3_file_spec.rb
 - spec/hasta/s3_data_source_spec.rb
 - spec/hasta/s3_file_cache_spec.rb

data/spec/fixtures/hasta/filter_config.txt

@@ -1 +0,0 @@
-\AOnly Allow This Line\z