gorillib 0.4.2 → 0.5.0

data/Rakefile

@@ -42,6 +42,7 @@ Jeweler::Tasks.new do |gem|
     reject{|f| File.directory?(f) }.
     reject{|f| ignores.any?{|i| File.fnmatch(i, f) || File.fnmatch(i+'/**/*', f) || File.fnmatch(i+'/*', f) } }
   gem.test_files = gem.files.grep(/^spec\//)
+  gem.extra_rdoc_files = [gem.files.grep(/^notes\//), gem.files.grep(/\.md$/)].flatten.uniq
   gem.require_paths = ['lib']
 end
 Jeweler::RubygemsDotOrgTasks.new

data/VERSION

@@ -1 +1 @@
-0.4.2
+0.5.0

data/gorillib.gemspec

@@ -5,16 +5,26 @@
 
 Gem::Specification.new do |s|
   s.name = "gorillib"
-  s.version = "0.4.2"
+  s.version = "0.5.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Infochimps"]
-  s.date = "2012-09-06"
+  s.date = "2012-10-22"
   s.description = "Gorillib: infochimps lightweight subset of ruby convenience methods"
   s.email = "coders@infochimps.org"
   s.extra_rdoc_files = [
+    "CHANGELOG.md",
     "LICENSE.md",
-    "README.md"
+    "README.md",
+    "TODO.md",
+    "notes/HOWTO.md",
+    "notes/bucket.md",
+    "notes/builder.md",
+    "notes/collection.md",
+    "notes/factories.md",
+    "notes/model-overlay.md",
+    "notes/model.md",
+    "notes/structured-data-classes.md"
   ]
   s.files = [
     ".gitignore",
@@ -37,6 +47,7 @@ Gem::Specification.new do |s|
     "lib/gorillib/array/compact_blank.rb",
     "lib/gorillib/array/deep_compact.rb",
     "lib/gorillib/array/extract_options.rb",
+    "lib/gorillib/array/hashify.rb",
     "lib/gorillib/array/simple_statistics.rb",
     "lib/gorillib/array/wrap.rb",
     "lib/gorillib/base.rb",
@@ -99,12 +110,16 @@ Gem::Specification.new do |s|
     "lib/gorillib/model/schema_magic.rb",
     "lib/gorillib/model/serialization.rb",
     "lib/gorillib/model/serialization/csv.rb",
+    "lib/gorillib/model/serialization/json.rb",
+    "lib/gorillib/model/serialization/lines.rb",
+    "lib/gorillib/model/serialization/tsv.rb",
     "lib/gorillib/model/validate.rb",
     "lib/gorillib/numeric/clamp.rb",
     "lib/gorillib/object/blank.rb",
     "lib/gorillib/object/try.rb",
     "lib/gorillib/object/try_dup.rb",
     "lib/gorillib/pathname.rb",
+    "lib/gorillib/pathname/utils.rb",
     "lib/gorillib/serialization/to_wire.rb",
     "lib/gorillib/some.rb",
     "lib/gorillib/string/constantize.rb",
@@ -115,16 +130,26 @@ Gem::Specification.new do |s|
     "lib/gorillib/string/truncate.rb",
     "lib/gorillib/type/boolean.rb",
     "lib/gorillib/type/extended.rb",
+    "lib/gorillib/type/ip_address.rb",
     "lib/gorillib/type/url.rb",
     "lib/gorillib/utils/capture_output.rb",
     "lib/gorillib/utils/console.rb",
     "lib/gorillib/utils/edge_cases.rb",
     "lib/gorillib/utils/nuke_constants.rb",
+    "notes/HOWTO.md",
+    "notes/bucket.md",
+    "notes/builder.md",
+    "notes/collection.md",
+    "notes/factories.md",
+    "notes/model-overlay.md",
+    "notes/model.md",
+    "notes/structured-data-classes.md",
     "spec/examples/builder/ironfan_spec.rb",
     "spec/extlib/hash_spec.rb",
     "spec/extlib/mash_spec.rb",
     "spec/gorillib/array/compact_blank_spec.rb",
     "spec/gorillib/array/extract_options_spec.rb",
+    "spec/gorillib/array/hashify_spec.rb",
     "spec/gorillib/array/simple_statistics_spec.rb",
     "spec/gorillib/builder_spec.rb",
     "spec/gorillib/collection_spec.rb",
@@ -133,6 +158,7 @@ Gem::Specification.new do |s|
     "spec/gorillib/datetime/to_flat_spec.rb",
     "spec/gorillib/enumerable/sum_spec.rb",
     "spec/gorillib/exception/raisers_spec.rb",
+    "spec/gorillib/factories_spec.rb",
     "spec/gorillib/hash/compact_spec.rb",
     "spec/gorillib/hash/deep_compact_spec.rb",
     "spec/gorillib/hash/deep_merge_spec.rb",
@@ -150,9 +176,9 @@ Gem::Specification.new do |s|
     "spec/gorillib/metaprogramming/delegation_spec.rb",
     "spec/gorillib/metaprogramming/singleton_class_spec.rb",
     "spec/gorillib/model/defaults_spec.rb",
-    "spec/gorillib/model/factories_spec.rb",
     "spec/gorillib/model/lint_spec.rb",
     "spec/gorillib/model/overlay_spec.rb",
+    "spec/gorillib/model/serialization/tsv_spec.rb",
     "spec/gorillib/model/serialization_spec.rb",
     "spec/gorillib/model_spec.rb",
     "spec/gorillib/numeric/clamp_spec.rb",
@@ -166,6 +192,7 @@ Gem::Specification.new do |s|
     "spec/gorillib/string/inflector_test_cases.rb",
     "spec/gorillib/string/truncate_spec.rb",
     "spec/gorillib/type/extended_spec.rb",
+    "spec/gorillib/type/ip_address_spec.rb",
     "spec/gorillib/utils/capture_output_spec.rb",
     "spec/spec_helper.rb",
     "spec/support/factory_test_helpers.rb",
@@ -186,7 +213,7 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
   s.rubygems_version = "1.8.24"
   s.summary = "include only what you need. No dependencies, no creep"
-  s.test_files = ["spec/extlib/mash_spec.rb", "spec/extlib/hash_spec.rb", "spec/support/gorillib_test_helpers.rb", "spec/support/hashlike_helper.rb", "spec/support/shared_examples/included_module.rb", "spec/support/hashlike_fuzzing_helper.rb", "spec/support/matchers/be_hash_eql.rb", "spec/support/matchers/be_array_eql.rb", "spec/support/matchers/enumerate_method.rb", "spec/support/matchers/evaluate_to_true.rb", "spec/support/model_test_helpers.rb", "spec/support/factory_test_helpers.rb", "spec/support/hashlike_via_delegation.rb", "spec/support/hashlike_struct_helper.rb", "spec/gorillib/model_spec.rb", "spec/gorillib/builder_spec.rb", "spec/gorillib/numeric/clamp_spec.rb", "spec/gorillib/string/human_spec.rb", "spec/gorillib/string/inflections_spec.rb", "spec/gorillib/string/inflector_test_cases.rb", "spec/gorillib/string/constantize_spec.rb", "spec/gorillib/string/truncate_spec.rb", "spec/gorillib/metaprogramming/class_attribute_spec.rb", "spec/gorillib/metaprogramming/singleton_class_spec.rb", "spec/gorillib/metaprogramming/delegation_spec.rb", "spec/gorillib/hashlike_spec.rb", "spec/gorillib/exception/raisers_spec.rb", "spec/gorillib/utils/capture_output_spec.rb", "spec/gorillib/collection_spec.rb", "spec/gorillib/type/extended_spec.rb", "spec/gorillib/hash/zip_spec.rb", "spec/gorillib/hash/reverse_merge_spec.rb", "spec/gorillib/hash/slice_spec.rb", "spec/gorillib/hash/keys_spec.rb", "spec/gorillib/hash/compact_spec.rb", "spec/gorillib/hash/deep_compact_spec.rb", "spec/gorillib/hash/deep_merge_spec.rb", "spec/gorillib/datetime/parse_spec.rb", "spec/gorillib/datetime/to_flat_spec.rb", "spec/gorillib/hashlike/deep_hash_spec.rb", "spec/gorillib/hashlike/hashlike_behavior_spec.rb", "spec/gorillib/hashlike/hashlike_via_accessors_spec.rb", "spec/gorillib/hashlike/behave_same_as_hash_spec.rb", "spec/gorillib/configurable_spec.rb", "spec/gorillib/model/lint_spec.rb", "spec/gorillib/model/defaults_spec.rb", "spec/gorillib/model/serialization_spec.rb", "spec/gorillib/model/overlay_spec.rb", "spec/gorillib/model/factories_spec.rb", "spec/gorillib/pathname_spec.rb", "spec/gorillib/enumerable/sum_spec.rb", "spec/gorillib/object/try_spec.rb", "spec/gorillib/object/blank_spec.rb", "spec/gorillib/object/try_dup_spec.rb", "spec/gorillib/array/compact_blank_spec.rb", "spec/gorillib/array/simple_statistics_spec.rb", "spec/gorillib/array/extract_options_spec.rb", "spec/gorillib/logger/log_spec.rb", "spec/spec_helper.rb", "spec/examples/builder/ironfan_spec.rb"]
+  s.test_files = ["spec/extlib/mash_spec.rb", "spec/extlib/hash_spec.rb", "spec/support/gorillib_test_helpers.rb", "spec/support/hashlike_helper.rb", "spec/support/shared_examples/included_module.rb", "spec/support/hashlike_fuzzing_helper.rb", "spec/support/matchers/be_hash_eql.rb", "spec/support/matchers/be_array_eql.rb", "spec/support/matchers/enumerate_method.rb", "spec/support/matchers/evaluate_to_true.rb", "spec/support/model_test_helpers.rb", "spec/support/factory_test_helpers.rb", "spec/support/hashlike_via_delegation.rb", "spec/support/hashlike_struct_helper.rb", "spec/gorillib/model_spec.rb", "spec/gorillib/builder_spec.rb", "spec/gorillib/numeric/clamp_spec.rb", "spec/gorillib/string/human_spec.rb", "spec/gorillib/string/inflections_spec.rb", "spec/gorillib/string/inflector_test_cases.rb", "spec/gorillib/string/constantize_spec.rb", "spec/gorillib/string/truncate_spec.rb", "spec/gorillib/metaprogramming/class_attribute_spec.rb", "spec/gorillib/metaprogramming/singleton_class_spec.rb", "spec/gorillib/metaprogramming/delegation_spec.rb", "spec/gorillib/hashlike_spec.rb", "spec/gorillib/exception/raisers_spec.rb", "spec/gorillib/utils/capture_output_spec.rb", "spec/gorillib/collection_spec.rb", "spec/gorillib/type/extended_spec.rb", "spec/gorillib/type/ip_address_spec.rb", "spec/gorillib/hash/zip_spec.rb", "spec/gorillib/hash/reverse_merge_spec.rb", "spec/gorillib/hash/slice_spec.rb", "spec/gorillib/hash/keys_spec.rb", "spec/gorillib/hash/compact_spec.rb", "spec/gorillib/hash/deep_compact_spec.rb", "spec/gorillib/hash/deep_merge_spec.rb", "spec/gorillib/datetime/parse_spec.rb", "spec/gorillib/datetime/to_flat_spec.rb", "spec/gorillib/hashlike/deep_hash_spec.rb", "spec/gorillib/hashlike/hashlike_behavior_spec.rb", "spec/gorillib/hashlike/hashlike_via_accessors_spec.rb", "spec/gorillib/hashlike/behave_same_as_hash_spec.rb", "spec/gorillib/configurable_spec.rb", "spec/gorillib/factories_spec.rb", "spec/gorillib/model/serialization/tsv_spec.rb", "spec/gorillib/model/lint_spec.rb", "spec/gorillib/model/defaults_spec.rb", "spec/gorillib/model/serialization_spec.rb", "spec/gorillib/model/overlay_spec.rb", "spec/gorillib/pathname_spec.rb", "spec/gorillib/enumerable/sum_spec.rb", "spec/gorillib/object/try_spec.rb", "spec/gorillib/object/blank_spec.rb", "spec/gorillib/object/try_dup_spec.rb", "spec/gorillib/array/compact_blank_spec.rb", "spec/gorillib/array/simple_statistics_spec.rb", "spec/gorillib/array/extract_options_spec.rb", "spec/gorillib/array/hashify_spec.rb", "spec/gorillib/logger/log_spec.rb", "spec/spec_helper.rb", "spec/examples/builder/ironfan_spec.rb"]
 
   if s.respond_to? :specification_version then
     s.specification_version = 3

data/lib/gorillib/array/hashify.rb

@@ -0,0 +1,11 @@
+class Array
+  #
+  # Gets value of block on each element;
+  # constructs a hash of element-value pairs
+  #
+  # @return [Hash] hash of key-value pairs
+  def hashify
+    raise ArgumentError, 'hashify requires a block' unless block_given?
+    Hash[ self.map{|el| [el, yield(el)] } ]
+  end
+end

data/lib/gorillib/base.rb

@@ -1,6 +1,7 @@
 require 'gorillib/object/blank'
 require 'gorillib/array/extract_options'
 require 'gorillib/array/compact_blank'
+require 'gorillib/array/hashify'
 require 'gorillib/hash/reverse_merge'
 require 'gorillib/hash/compact'
 require 'gorillib/hash/keys'

data/lib/gorillib/data_munging.rb

@@ -1,4 +1,6 @@
-require 'gorillib'
+require 'multi_json'
+
+require 'gorillib/base'
 require 'gorillib/array/wrap'
 require 'gorillib/type/extended'
 require 'gorillib/hash/slice'
@@ -6,3 +8,8 @@ require 'gorillib/pathname'
 require 'gorillib/logger/log'
 require 'gorillib/model'
 require 'gorillib/factories'
+require 'gorillib/model/serialization'
+require 'gorillib/model/serialization/csv'
+require 'gorillib/model/serialization/tsv'
+require 'gorillib/model/serialization/json'
+require 'gorillib/model/indexable'

data/lib/gorillib/exception/raisers.rb

@@ -25,7 +25,7 @@ Exception.class_eval do
   #   end
   #
   def polish(extra_info)
-    filename, _, method_name = self.class.caller_parts
+    filename, _, method_name = self.class.caller_parts(2)
     method_name.gsub!(/rescue in /, '')
     most_recent_line = backtrace.detect{|line|
       line.include?(filename) && line.include?(method_name) && line.end_with?("'") }
@@ -102,6 +102,11 @@ class ArgumentError
     raise self, message, *args
   end
 
+
+  def self.block_required!(block)
+    raise self.new("Block is required") unless block
+  end
+
   #
   # @param obj    [Object] Object to check
   # @param types  [Array[Symbol,Class,Module]] Types or methods to compare

data/lib/gorillib/factories.rb

@@ -32,19 +32,6 @@ module Gorillib
       klass.new(options)
     end
 
-    # Manufactures objects from their raw attributes hash
-    #
-    # A hash with a value for `:_type` is dispatched to the corresponding factory
-    # Everything else is returned directly
-    def self.make(obj)
-      if obj.respond_to?(:has_key?) && (obj.has_key?(:_type) || obj.has_key?('_type'))
-        factory = Gorillib::Factory(attrs[:_type])
-        factory.receive(obj)
-      else
-        obj
-      end
-    end
-
     def self.register_factory(factory, typenames)
       typenames.each{|typename| factories[typename] = factory }
     end
@@ -185,6 +172,14 @@ module Gorillib
       end
     end
 
+    # __________________________________________________________________________
+    #
+    #  Generic Factories
+    # __________________________________________________________________________
+
+    #
+    # Factory that accepts whatever given and uses it directly -- no nothin'
+    #
     class ::Whatever < BaseFactory
       def initialize(options={})
         options.slice!(:convert, :blankish)
@@ -200,6 +195,24 @@ module Gorillib
     end
     IdenticalFactory = ::Whatever unless defined?(IdenticalFactory)
 
+
+    # Manufactures objects from their raw attributes hash
+    #
+    # The hash must have a value for `:_type`, used to retrieve the actual factory
+    #
+    class ::GenericModel < BaseFactory
+      def blankish?(obj) obj.nil? ; end
+      def native?(obj)   false  ; end
+      def receive(attrs, &block)
+        Gorillib::Model::Validate.hashlike!(attrs){ "attributes for typed object" }
+        klass = Gorillib::Factory(attrs.fetch(:_type){ attrs.fetch("_type") })
+        #
+        klass.new(attrs, &block)
+      end
+      def self.receive(obj) allocate.receive(obj) end
+      register_factory!(GenericModel, :generic)
+    end
+
     # __________________________________________________________________________
     #
     #  Concrete Factories

data/lib/gorillib/model/base.rb

@@ -31,6 +31,7 @@ module Gorillib
     # @return [{Symbol => Object}] The Hash of all attributes
     def attributes
       self.class.field_names.inject(Hash.new) do |hsh, fn|
+        # hsh[fn] = attribute_set?(fn) ? read_attribute(fn) : nil
         hsh[fn] = read_attribute(fn)
         hsh
       end
@@ -199,6 +200,10 @@ module Gorillib
       str << '>'
     end
 
+    def to_s
+      inspect
+    end
+
     def inspect_compact
       str = "#<#{self.class.name.to_s}>"
     end
@@ -208,7 +213,6 @@ module Gorillib
     def to_inspectable
       compact_attributes
     end
-    private :to_inspectable
 
   protected
 
@@ -260,6 +264,7 @@ module Gorillib
       base.instance_eval do
         extend Gorillib::Model::NamedSchema
         extend Gorillib::Model::ClassMethods
+        self.meta_module
         @_own_fields ||= {}
       end
     end

data/lib/gorillib/model/schema_magic.rb

@@ -153,6 +153,7 @@ module Gorillib
 
       def inherited(base)
         base.instance_eval do
+          self.meta_module
           @_own_fields ||= {}
         end
         super

data/lib/gorillib/model/serialization/csv.rb

@@ -21,9 +21,11 @@ module Gorillib
         def each_in_csv(filename, options={})
           filename = Pathname.path_to(filename)
           options = csv_options.merge(options)
+          #
           pop_headers = options.delete(:pop_headers)
           num_fields  = options.delete(:num_fields){ (fields.length .. fields.length) }
           raise ArgumentError, "The :headers option to CSV changes its internal behavior; use 'pop_headers: true' to ignore the first line" if options[:headers]
+          #
           CSV.open(filename, options) do |csv_file|
             csv_file.shift if pop_headers
             csv_file.each do |tuple|

data/lib/gorillib/model/serialization/json.rb

@@ -0,0 +1,44 @@
+require_relative './lines'
+
+module Gorillib
+  module Model
+
+    module LoadFromJson
+      extend  Gorillib::Concern
+      include LoadLines
+
+      module ClassMethods
+
+        # Iterate a block over each line of a file having JSON records, one per
+        # line, in a big stack
+        #
+        # @yield an object instantiated from each line in the file.
+        def _each_from_json(filename, options={})
+          _each_raw_line(filename, options) do |line|
+            hsh = MultiJson.load(line)
+            yield receive(hsh)
+          end
+        end
+
+        # With a block, calls block on each object in turn (and returns nil)
+        #
+        # With no block, accumulates all the instances into the array it
+        # returns. As opposed to the with-a-block case, the memory footprint of
+        # this increases as the filesize does, so use caution with large files.
+        #
+        # @return with a block, returns nil; with no block, an array of this class' instances
+        def load_json(*args)
+          if block_given?
+            _each_from_json(*args, &Proc.new)
+          else
+            objs = []
+            _each_from_json(*args){|obj| objs << obj }
+            objs
+          end
+        end
+
+      end
+    end
+
+  end
+end

data/lib/gorillib/model/serialization/lines.rb

@@ -0,0 +1,30 @@
+module Gorillib
+  module Model
+
+    module LoadLines
+      extend Gorillib::Concern
+
+      module ClassMethods
+
+        # Iterate a block over each line of a file
+        # @yield each line in the file.
+        def _each_raw_line(filename, options={})
+          filename = Pathname.path_to(filename)
+          #
+          pop_headers = options.delete(:pop_headers)
+          #
+          File.open(filename) do |file|
+            file.readline if pop_headers
+            file.each do |line|
+              line.chomp! ; next if line.empty?
+              yield line
+            end
+            nil
+          end
+        end
+
+      end
+    end
+
+  end
+end

data/lib/gorillib/model/serialization/tsv.rb

@@ -0,0 +1,55 @@
+require_relative './lines'
+
+module Gorillib
+  module Model
+
+    module LoadFromTsv
+      extend  Gorillib::Concern
+      include LoadLines
+
+      included do |base|
+        # Options that will be passed to CSV. Be careful to modify with assignment (`+=`) and not in-place (`<<`)
+        base.class_attribute :tsv_options
+        base.tsv_options = Hash.new
+      end
+
+      module ClassMethods
+
+        # Iterate a block over each line of a TSV file
+        #
+        # @raise [Gorillib::Model::RawDataMismatchError] if a line has too many or too few fields
+        # @yield an object instantiated from each line in the file.
+        def _each_from_tsv(filename, options={})
+          options = tsv_options.merge(options)
+          num_fields  = options.delete(:num_fields){ (fields.length .. fields.length) }
+          max_fields  = num_fields.max # need to make sure "1\t2\t\t\t" becomes ["1","2","","",""]
+          #
+          _each_raw_line(filename, options) do |line|
+            tuple = line.split("\t", max_fields)
+            unless num_fields.include?(tuple.length) then raise Gorillib::Model::RawDataMismatchError, "yark, spurious fields: #{tuple.inspect}" ; end
+            yield from_tuple(*tuple)
+          end
+        end
+
+        # With a block, calls block on each object in turn (and returns nil)
+        #
+        # With no block, accumulates all the instances into the array it
+        # returns. As opposed to the with-a-block case, the memory footprint of
+        # this increases as the filesize does, so use caution with large files.
+        #
+        # @return with a block, returns nil; with no block, an array of this class' instances
+        def load_tsv(*args)
+          if block_given?
+            _each_from_tsv(*args, &Proc.new)
+          else
+            objs = []
+            _each_from_tsv(*args){|obj| objs << obj }
+            objs
+          end
+        end
+
+      end
+    end
+
+  end
+end

data/lib/gorillib/pathname/utils.rb

@@ -0,0 +1,34 @@
+class Pathname
+  # same as `#exist?`
+  def exists?(*args) exist?(*args) ; end
+
+  # @example It chains nicely:
+  #   # put each file in eg. dest/f/foo.json
+  #   Pathname.of(:dest, slug[0..0], "#{slug}.json").mkparent.open('w') do |file|
+  #     # ...
+  #   end
+  #
+  # @returns the path itself (not its parent)
+  def mkparent
+    dirname.mkpath
+    return self
+  end
+
+  #
+  # Executes the block (passing the opened file) if the file does not
+  # exist. Ignores the block otherwise. The block is required.
+  #
+  # @param options
+  # @option options[:force] Force creation of the file
+  #
+  # @returns the path itself (not the file)
+  def if_missing(options={}, &block)
+    ArgumentError.block_required!(block)
+    return self if exist? && (not options[:force])
+    #
+    mkparent
+    open((options[:mode] || 'w'), &block)
+    return self
+  end
+
+end