imw 0.2.18 → 0.3.0

data/lib/imw/serializer.rb

@@ -0,0 +1,7 @@
+module IMW
+  module Serializer
+
+    autoload JsonSerializer, 'imw/serializer/json_serializer'
+
+  end
+end

data/lib/imw/serializer/json_serializer.rb

@@ -0,0 +1,17 @@
+module IMW
+  module Serializer
+    class JsonSerializer
+
+      def initialize file_url, mode
+        @file_url = file_url
+        @mode     = mode
+        File.open(file_url, mode)
+      end
+
+      def write line
+        File.write(line)
+      end
+
+    end
+  end
+end

data/lib/imw/uri.rb

@@ -0,0 +1,41 @@
+module IMW
+
+  class Uri
+
+    attr_reader :scheme, :format
+
+    @@schemes = {
+      %r{^hdfs:} => 'Hdfs',
+      %r{^s3:}   => 'S3',
+    }
+
+    @@formats = {
+      %r{.csv$}   => 'Csv',
+      %r{.tsv$}   => 'Tsv',
+      %r{.json$}  => 'Json',
+      %r{.ya?ml$} => 'Yaml',
+    }
+
+    def initialize uri
+      @scheme = lookup_scheme(uri)
+      @format = lookup_format(uri)
+    end
+
+    def lookup_scheme uri
+      @@schemes.keys.each do |key|
+        next unless uri =~ key
+        return @@schemes[key]
+      end
+      'Local'
+    end
+
+    def lookup_format uri
+      @@formats.keys.each do |key|
+        next unless uri =~ key
+        return @@formats[key]
+      end
+      raise InvalidFormatError.new("#{File.extname(uri)} is not currently supported")
+    end
+
+  end
+end

data/spec/resource_spec.rb

@@ -0,0 +1,78 @@
+require 'rubygems'
+require 'rspec'
+require 'imw' ; include IMW
+
+describe "IMW::Resource" do
+
+  context "The Resource class" do
+
+    it "should respond to the method open" do
+      IMW::Resource.should respond_to(:open)
+    end
+
+    context "Resource.open" do
+      before :each do
+        @uri = "test.csv"
+      end
+
+      it "should return an IMW::Resource object" do
+        Resource.open(@uri).should be_instance_of(IMW::Resource)
+      end
+
+      it "should return the value of the block if given a block" do
+        Resource.open(@uri) { |obj| nil }.should be_nil
+      end
+
+      it "should accept a block and yield an IMW::Resource object" do
+        Resource.open(@uri) do |obj|
+          obj.should be_instance_of(IMW::Resource)
+        end
+      end
+
+    end
+
+    it "should respond to the method exists?" do
+      Resource.should respond_to(:exists?)
+    end
+
+    context "Resource.exists?" do
+      before :each do
+        @file = "test"
+      end
+
+      it "should return either true or false" do
+        Resource.exists?(@file).should == !!Resource.exists?(@should)
+      end
+
+    end
+  end
+
+  it "should read a Resource and return a string" do
+    Resource
+  end
+
+  context "A Resource instance" do
+    before :each do
+      @uri = "test.csv"
+      @resource = Resource.new(@uri)
+    end
+
+    it "should accept a Resource access mode when instantiated" do
+      lambda { Resource.new(@uri, 'w') }.should_not raise_error(Exception)
+    end
+
+    it "should raise an error if given an invalid Resource mode" do
+      lambda { Resource.new(@uri, 'f') }.should raise_error(IMW::Error::FileModeError)
+    end
+
+    it "should return the uri as a IMW::Uri object" do
+      @resource.uri.should be_instance_of(IMW::Uri)
+    end
+
+    it "should respond to the method close" do
+      @resource.should respond_to(:close)
+    end
+
+  end
+
+end

data/spec/uri_spec.rb

@@ -0,0 +1,55 @@
+require 'rubygems'
+require 'rspec'
+require 'imw' ; include IMW
+
+describe "IMW::Uri" do
+  before :each do
+    @uri = Uri.new("test.csv")
+  end
+
+  it "should respond to scheme" do
+    @uri.should respond_to(:scheme)
+  end
+
+  context "A Uri.scheme" do
+    before :each do
+      @local = "/path/to/file.csv"
+      @s3    = "s3://s3-bucket/path/to/file.csv"
+      @hdfs  = "hdfs://namenodehost/path/to/file.csv"
+    end
+
+    it "should understand the local file scheme" do
+      Uri.new(@local).scheme.should == 'Local'
+    end
+
+    it "should understand the s3 file scheme" do
+      Uri.new(@s3).scheme.should == 'S3'
+    end
+
+    it "should understand the hadoop file system" do
+      Uri.new(@hdfs).scheme.should == 'Hdfs'
+    end
+
+  end
+
+  it "should respond to format" do
+    @uri.should respond_to(:format)
+  end
+
+  context "A Uri.format" do
+    before :each do
+      @csv     = "foo.csv"
+      @tsv     = "foo.tsv"
+      @json    = "foo.json"
+      @yaml    = "foo.yml"
+      @invalid = "foor.bar"
+    end
+
+    it "should raise an error when given an invalid format" do
+      lambda { Uri.new(@invalid) }.should raise_error(IMW::Error::InvalidFormatError)
+    end
+
+  end
+
+end
+

metadata

@@ -1,28 +1,27 @@
 --- !ruby/object:Gem::Specification 
 name: imw
 version: !ruby/object:Gem::Version 
-  hash: 51
+  hash: 19
   prerelease: false
   segments: 
   - 0
-  - 2
-  - 18
-  version: 0.2.18
+  - 3
+  - 0
+  version: 0.3.0
 platform: ruby
 authors: 
+- Travis Dempsey
 - Dhruv Bansal
-- Philip (flip) Kromer
+- mrflip
 autorequire: 
 bindir: bin
 cert_chain: []
 
-date: 2011-02-16 00:00:00 -06:00
+date: 2011-07-15 00:00:00 -05:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
-  prerelease: false
-  name: activesupport
-  version_requirements: &id001 !ruby/object:Gem::Requirement 
+  requirement: &id001 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -31,12 +30,12 @@ dependencies:
         segments: 
         - 0
         version: "0"
-  requirement: *id001
   type: :runtime
-- !ruby/object:Gem::Dependency 
+  name: nokogiri
   prerelease: false
-  name: addressable
-  version_requirements: &id002 !ruby/object:Gem::Requirement 
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  requirement: &id002 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -45,26 +44,44 @@ dependencies:
         segments: 
         - 0
         version: "0"
-  requirement: *id002
-  type: :runtime
-- !ruby/object:Gem::Dependency 
+  type: :development
+  name: shoulda
   prerelease: false
-  name: uuidtools
-  version_requirements: &id003 !ruby/object:Gem::Requirement 
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
-    - - ">="
+    - - ~>
       - !ruby/object:Gem::Version 
-        hash: 3
+        hash: 23
         segments: 
+        - 1
         - 0
-        version: "0"
-  requirement: *id003
-  type: :runtime
+        - 0
+        version: 1.0.0
+  type: :development
+  name: bundler
+  prerelease: false
+  version_requirements: *id003
 - !ruby/object:Gem::Dependency 
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 7
+        segments: 
+        - 1
+        - 5
+        - 2
+        version: 1.5.2
+  type: :development
+  name: jeweler
   prerelease: false
-  name: rake
-  version_requirements: &id004 !ruby/object:Gem::Requirement 
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  requirement: &id005 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -73,175 +90,50 @@ dependencies:
         segments: 
         - 0
         version: "0"
-  requirement: *id004
-  type: :runtime
-description: The Infinite Monkeywrench (IMW) is a Ruby frameworks to simplify the tasks of acquiring, extracting, transforming, loading, and packaging data. It minimizes programmer time by encapsulating common data workflows and patterns and creating interfaces to many other useful Ruby libraries.
-email: coders@infochimps.org
-executables: 
-- imw
-- tsv_to_json.rb
+  type: :development
+  name: rcov
+  prerelease: false
+  version_requirements: *id005
+description: Infinite Monkey Wrench - A framework to make collecting and parsing data fun again.
+email: travis@infochimps.com
+executables: []
+
 extensions: []
 
 extra_rdoc_files: 
-- LICENSE
-- README.rdoc
+- LICENSE.txt
+- README.textile
 files: 
 - Gemfile
 - Gemfile.lock
-- LICENSE
-- README.rdoc
+- LICENSE.txt
+- README.textile
 - Rakefile
 - VERSION
-- bin/imw
-- bin/tsv_to_json.rb
-- etc/imwrc.rb
-- examples/dataset.rb
-- examples/metadata.yml
+- examples/foo.rb
+- examples/html_selector.rb
+- examples/nes_game_list.csv
+- examples/nes_gamespot.csv
+- examples/nes_nintendo.csv
+- examples/nes_unlicensed.csv
+- examples/nes_wikipedia.csv
+- examples/nibbler_test.rb
+- examples/script.rb
 - lib/imw.rb
-- lib/imw/archives.rb
-- lib/imw/archives/rar.rb
-- lib/imw/archives/tar.rb
-- lib/imw/archives/tarbz2.rb
-- lib/imw/archives/targz.rb
-- lib/imw/archives/zip.rb
-- lib/imw/boot.rb
-- lib/imw/compressed_files.rb
-- lib/imw/compressed_files/bz2.rb
-- lib/imw/compressed_files/compressible.rb
-- lib/imw/compressed_files/gz.rb
-- lib/imw/dataset.rb
-- lib/imw/dataset/paths.rb
-- lib/imw/dataset/workflow.rb
-- lib/imw/formats.rb
-- lib/imw/formats/delimited.rb
-- lib/imw/formats/excel.rb
-- lib/imw/formats/json.rb
-- lib/imw/formats/pdf.rb
-- lib/imw/formats/sgml.rb
-- lib/imw/formats/yaml.rb
-- lib/imw/metadata.rb
-- lib/imw/metadata/contains_metadata.rb
-- lib/imw/metadata/dsl.rb
-- lib/imw/metadata/field.rb
-- lib/imw/metadata/has_metadata.rb
-- lib/imw/metadata/has_summary.rb
-- lib/imw/metadata/schema.rb
-- lib/imw/parsers.rb
-- lib/imw/parsers/flat.rb
-- lib/imw/parsers/html_parser.rb
-- lib/imw/parsers/html_parser/matchers.rb
-- lib/imw/parsers/line_parser.rb
-- lib/imw/parsers/regexp_parser.rb
-- lib/imw/repository.rb
+- lib/imw/error.rb
+- lib/imw/recordizer.rb
+- lib/imw/recordizer/html_selector_recordizer.rb
+- lib/imw/recordizer/string_slice_recordizer.rb
 - lib/imw/resource.rb
-- lib/imw/runner.rb
-- lib/imw/schemes.rb
-- lib/imw/schemes/ftp.rb
-- lib/imw/schemes/hdfs.rb
-- lib/imw/schemes/http.rb
-- lib/imw/schemes/local.rb
-- lib/imw/schemes/remote.rb
-- lib/imw/schemes/s3.rb
-- lib/imw/schemes/sql.rb
-- lib/imw/tools.rb
-- lib/imw/tools/aggregator.rb
-- lib/imw/tools/archiver.rb
-- lib/imw/tools/downloader.rb
-- lib/imw/tools/extension_analyzer.rb
-- lib/imw/tools/summarizer.rb
-- lib/imw/tools/transferer.rb
-- lib/imw/utils.rb
-- lib/imw/utils/dynamically_extendable.rb
-- lib/imw/utils/error.rb
-- lib/imw/utils/extensions/hpricot.rb
-- lib/imw/utils/has_uri.rb
-- lib/imw/utils/log.rb
-- lib/imw/utils/misc.rb
-- lib/imw/utils/paths.rb
-- lib/imw/utils/uri.rb
-- lib/imw/utils/uuid.rb
-- lib/imw/utils/validate.rb
-- lib/imw/utils/version.rb
-- spec/data/formats/delimited/sample.csv
-- spec/data/formats/delimited/sample.tsv
-- spec/data/formats/delimited/with_schema/ace-hardware-locations.tsv
-- spec/data/formats/delimited/with_schema/all-countries-ip-address-to-geolocation-data.tsv
-- spec/data/formats/delimited/with_schema/complete-list-of-starbucks-locations.tsv
-- spec/data/formats/delimited/with_schema/myspace-user-activity-stream-cumulative-word-count-from-from-dec.tsv
-- spec/data/formats/delimited/with_schema/myspace-user-activity-stream-myspace-application-adds-by-zip-cod.tsv
-- spec/data/formats/delimited/with_schema/myspace-user-activity-stream-myspace-application-counts.tsv
-- spec/data/formats/delimited/with_schema/myspace-user-activity-stream-user-count-by-latlong.tsv
-- spec/data/formats/delimited/with_schema/myspace-user-activity-stream-user-count-by-zip-code.tsv
-- spec/data/formats/delimited/with_schema/myspace-user-activity-stream-word-count-by-day-from-december-200.tsv
-- spec/data/formats/delimited/without_schema/ace-hardware-locations.tsv
-- spec/data/formats/delimited/without_schema/all-countries-ip-address-to-geolocation-data.tsv
-- spec/data/formats/delimited/without_schema/complete-list-of-starbucks-locations.tsv
-- spec/data/formats/delimited/without_schema/myspace-user-activity-stream-cumulative-word-count-from-from-dec.tsv
-- spec/data/formats/delimited/without_schema/myspace-user-activity-stream-myspace-application-adds-by-zip-cod.tsv
-- spec/data/formats/delimited/without_schema/myspace-user-activity-stream-myspace-application-counts.tsv
-- spec/data/formats/delimited/without_schema/myspace-user-activity-stream-user-count-by-latlong.tsv
-- spec/data/formats/delimited/without_schema/myspace-user-activity-stream-user-count-by-zip-code.tsv
-- spec/data/formats/delimited/without_schema/myspace-user-activity-stream-word-count-by-day-from-december-200.tsv
-- spec/data/formats/excel/sample.xls
-- spec/data/formats/json/sample.json
-- spec/data/formats/none/sample
-- spec/data/formats/sgml/sample.xml
-- spec/data/formats/text/sample.txt
-- spec/data/formats/yaml/sample.yaml
-- spec/data/schema-tabular.yaml
-- spec/imw/archives/rar_spec.rb
-- spec/imw/archives/tar_spec.rb
-- spec/imw/archives/tarbz2_spec.rb
-- spec/imw/archives/targz_spec.rb
-- spec/imw/archives/zip_spec.rb
-- spec/imw/archives_spec.rb
-- spec/imw/compressed_files/bz2_spec.rb
-- spec/imw/compressed_files/compressible_spec.rb
-- spec/imw/compressed_files/gz_spec.rb
-- spec/imw/compressed_files_spec.rb
-- spec/imw/dataset/paths_spec.rb
-- spec/imw/dataset/workflow_spec.rb
-- spec/imw/formats/delimited_spec.rb
-- spec/imw/formats/excel_spec.rb
-- spec/imw/formats/json_spec.rb
-- spec/imw/formats/sgml_spec.rb
-- spec/imw/formats/yaml_spec.rb
-- spec/imw/metadata/contains_metadata_spec.rb
-- spec/imw/metadata/field_spec.rb
-- spec/imw/metadata/has_metadata_spec.rb
-- spec/imw/metadata/has_summary_spec.rb
-- spec/imw/metadata/schema_spec.rb
-- spec/imw/metadata_spec.rb
-- spec/imw/parsers/line_parser_spec.rb
-- spec/imw/parsers/regexp_parser_spec.rb
-- spec/imw/resource_spec.rb
-- spec/imw/schemes/hdfs_spec.rb
-- spec/imw/schemes/http_spec.rb
-- spec/imw/schemes/local_spec.rb
-- spec/imw/schemes/remote_spec.rb
-- spec/imw/schemes/s3_spec.rb
-- spec/imw/schemes/sql_spec.rb
-- spec/imw/tools/aggregator_spec.rb
-- spec/imw/tools/archiver_spec.rb
-- spec/imw/tools/extension_analyzer_spec.rb
-- spec/imw/tools/summarizer_spec.rb
-- spec/imw/tools/transferer_spec.rb
-- spec/imw/utils/dynamically_extendable_spec.rb
-- spec/imw/utils/has_uri_spec.rb
-- spec/imw/utils/paths_spec.rb
-- spec/imw/utils/shared_paths_spec.rb
-- spec/imw_spec.rb
-- spec/rcov.opts
-- spec/spec_helper.rb
-- spec/support/custom_matchers.rb
-- spec/support/file_contents_matcher.rb
-- spec/support/paths_matcher.rb
-- spec/support/random.rb
-- spec/support/without_regard_to_order_matcher.rb
+- lib/imw/serializer.rb
+- lib/imw/serializer/json_serializer.rb
+- lib/imw/uri.rb
+- spec/resource_spec.rb
+- spec/uri_spec.rb
 has_rdoc: true
 homepage: http://github.com/infochimps/imw
-licenses: []
-
+licenses: 
+- MIT
 post_install_message: 
 rdoc_options: []
 
@@ -271,54 +163,11 @@ rubyforge_project:
 rubygems_version: 1.3.7
 signing_key: 
 specification_version: 3
-summary: The Infinite Monkeywrench (IMW) makes acquiring, extracting, transforming, loading, and packaging data easy.
+summary: Infinite Monkey Wrench - A framework to make collecting and parsing data fun again.
 test_files: 
-- examples/dataset.rb
-- spec/imw/archives/rar_spec.rb
-- spec/imw/archives/tar_spec.rb
-- spec/imw/archives/tarbz2_spec.rb
-- spec/imw/archives/targz_spec.rb
-- spec/imw/archives/zip_spec.rb
-- spec/imw/archives_spec.rb
-- spec/imw/compressed_files/bz2_spec.rb
-- spec/imw/compressed_files/compressible_spec.rb
-- spec/imw/compressed_files/gz_spec.rb
-- spec/imw/compressed_files_spec.rb
-- spec/imw/dataset/paths_spec.rb
-- spec/imw/dataset/workflow_spec.rb
-- spec/imw/formats/delimited_spec.rb
-- spec/imw/formats/excel_spec.rb
-- spec/imw/formats/json_spec.rb
-- spec/imw/formats/sgml_spec.rb
-- spec/imw/formats/yaml_spec.rb
-- spec/imw/metadata/contains_metadata_spec.rb
-- spec/imw/metadata/field_spec.rb
-- spec/imw/metadata/has_metadata_spec.rb
-- spec/imw/metadata/has_summary_spec.rb
-- spec/imw/metadata/schema_spec.rb
-- spec/imw/metadata_spec.rb
-- spec/imw/parsers/line_parser_spec.rb
-- spec/imw/parsers/regexp_parser_spec.rb
-- spec/imw/resource_spec.rb
-- spec/imw/schemes/hdfs_spec.rb
-- spec/imw/schemes/http_spec.rb
-- spec/imw/schemes/local_spec.rb
-- spec/imw/schemes/remote_spec.rb
-- spec/imw/schemes/s3_spec.rb
-- spec/imw/schemes/sql_spec.rb
-- spec/imw/tools/aggregator_spec.rb
-- spec/imw/tools/archiver_spec.rb
-- spec/imw/tools/extension_analyzer_spec.rb
-- spec/imw/tools/summarizer_spec.rb
-- spec/imw/tools/transferer_spec.rb
-- spec/imw/utils/dynamically_extendable_spec.rb
-- spec/imw/utils/has_uri_spec.rb
-- spec/imw/utils/paths_spec.rb
-- spec/imw/utils/shared_paths_spec.rb
-- spec/imw_spec.rb
-- spec/spec_helper.rb
-- spec/support/custom_matchers.rb
-- spec/support/file_contents_matcher.rb
-- spec/support/paths_matcher.rb
-- spec/support/random.rb
-- spec/support/without_regard_to_order_matcher.rb
+- examples/foo.rb
+- examples/html_selector.rb
+- examples/nibbler_test.rb
+- examples/script.rb
+- spec/resource_spec.rb
+- spec/uri_spec.rb