gorillib 0.4.2 → 0.5.0

data/lib/gorillib/type/extended.rb

@@ -2,6 +2,7 @@ require 'pathname'
 require 'date'
 require 'set'
 require 'gorillib/factories'
+require 'gorillib/type/ip_address'
 
 class ::Long      < ::Integer ; end
 class ::Double    < ::Float   ; end

data/lib/gorillib/type/ip_address.rb

@@ -0,0 +1,153 @@
+module IpAddresslike
+  ONES = 0xFFFFFFFF
+
+  # Masks off all but the `bitness` most-significant-bits
+  #
+  # @example /24 keeps only the first three quads
+  #   IpAddress.new('1.2.3.4').bitness_min(24) # '1.2.3.0'
+  #
+  def bitness_min(bitness)
+    raise ArgumentError, "IP addresses have only 32 bits (got #{bitness.inspect})" unless (0..32).include?(bitness)
+    lsbs = 32 - bitness
+    (packed >> lsbs) << lsbs
+  end
+
+  # Masks off all but the `bitness` most-significant-bits, filling with ones
+  #
+  # @example /24 fills the last quad
+  #   IpAddress.new('1.2.3.4').bitness_min(24) # '1.2.3.255'
+  #
+  def bitness_max(bitness)
+    raise ArgumentError, "IP addresses have only 32 bits (got #{bitness.inspect})" unless (0..32).include?(bitness)
+    packed | (ONES >> bitness)
+  end
+
+  def hex_str
+    packed.to_s(16)
+  end
+
+  def to_s
+    dotted
+  end
+
+end
+
+class ::IpAddress < ::String
+  include IpAddresslike
+
+  def dotted
+    self
+  end
+
+  def to_i
+    packed
+  end
+
+  # @returns [Integer] the 32-bit integer for this IP address
+  def packed
+    ip_a, ip_b, ip_c, ip_d = quads
+    ((ip_a << 24) + (ip_b << 16) + (ip_c << 8) + (ip_d))
+  end
+
+  def quads
+    self.split(".", 4).map{|qq| Integer(qq) }
+  end
+
+  # === class methods ===
+
+  def self.from_packed(pi)
+    str = [ (pi >> 24) & 0xFF, (pi >> 16) & 0xFF, (pi >>  8) & 0xFF, (pi) & 0xFF ].join(".")
+    new(str)
+  end
+
+  def self.from_dotted(str)
+    new(str)
+  end
+end
+
+# Stores an IP address in numeric form.
+#
+# IpNumeric instances are immutable, and memoize most of their methods.
+class ::IpNumeric
+  include IpAddresslike
+  include Comparable
+
+  def receive(val)
+    new(val)
+  end
+
+  def initialize(addr)
+    @packed = addr.to_int
+  end
+
+  def to_i       ; packed ; end
+  def to_int     ; packed ; end
+  def ==(other)  ; packed  == other.to_int ; end
+  def <=>(other) ; packed <=> other.to_int ; end
+  def +(int)     ; self.class.new(to_int + int) ; end
+
+
+  def packed ; @packed ; end
+
+  def dotted
+    @dotted ||= quads.join('.').freeze
+  end
+
+  def quads
+    @quads ||= [ (@packed >> 24) & 0xFF, (@packed >> 16) & 0xFF, (@packed >>  8) & 0xFF, (@packed) & 0xFF ].freeze
+  end
+
+  # === class methods ===
+
+  def self.from_packed(pi)
+    new(pi)
+  end
+
+  def self.from_dotted(dotted)
+    ip_a, ip_b, ip_c, ip_d = quads = dotted.split(".", 4).map(&:to_i)
+    obj = new((ip_a << 24) + (ip_b << 16) + (ip_c << 8) + (ip_d))
+    obj.instance_variable_set('@dotted', dotted.freeze)
+    obj.instance_variable_set('@quads',  quads.freeze)
+    obj
+  end
+end
+
+class ::IpRange < Range
+
+  def initialize(min_or_range, max=nil, exclusive=false)
+    if max.nil?
+      min       = min_or_range.min
+      max       = min_or_range.max
+      exclusive = min_or_range.exclude_end?
+    else
+      min       = min_or_range
+    end
+    raise ArgumentError, "Only inclusive #{self.class.name}s are implemented" if exclusive
+    super( IpNumeric.new(min), IpNumeric.new(max), false )
+  end
+
+  def bitness_blocks(bitness)
+    raise ArgumentError, "IP addresses have only 32 bits (got #{bitness.inspect})" unless (0..32).include?(bitness)
+    return [] if min.nil?
+    lsbs = 32 - bitness
+    middle_min = min.bitness_max(bitness) + 1
+    return [[min, max]] if max < middle_min
+    middle_max = max.bitness_min(bitness)
+    blks = []
+    stride = 1 << lsbs
+    #
+    blks << [min, IpNumeric.new(middle_min-1)]
+    (middle_min ... middle_max).step(stride){|beg| blks << [IpNumeric.new(beg), IpNumeric.new(beg+stride-1)] }
+    blks << [IpNumeric.new(middle_max), max]
+    blks
+  end
+
+  CIDR_RE = %r{\A(\d+\.\d+\.\d+\.\d+)/([0-3]\d)\z}
+
+  def self.from_cidr(cidr_str)
+    cidr_str =~ CIDR_RE or raise ArgumentError, "CIDR string should look like an ip address and bitness, eg 1.2.3.4/24 (got #{cidr_str})"
+    bitness    = $2.to_i
+    ip_address = IpNumeric.from_dotted($1)
+    new( ip_address.bitness_min(bitness), ip_address.bitness_max(bitness) )
+  end
+end

data/notes/HOWTO.md

@@ -0,0 +1,22 @@
+# Walkthrough
+
+**note (2012-06): you must use the dev branch, `version_1` of Gorillib** -- https://github.com/infochimps-labs/gorillib/tree/version_1/
+
+1. Here's a set of examples of Gorillib records in action.
+  * [examples of `Gorillib::Model`](https://github.com/infochimps-labs/gorillib/tree/version_1/examples/model)
+  * see example gorillib records in the /examples directories
+  
+__________________________________________________________________________
+
+2. Walkthrough dataflow for a new customer:
+  * models to represent records in a dataflow (gorillib/examples)
+  * describe the macro dataflow (wukong/examples)
+  * simulate the dataflow at commandline
+  * project the dataflow using Flume
+  * export the dataflow using graphviz
+
+https://github.com/infochimps-labs/wukong/wiki
+https://github.com/infochimps-labs/gorillib/wiki
+Plus source files (or you can instead see the Yard Docs)
+
+3. flume event model shim

data/notes/bucket.md

@@ -0,0 +1,155 @@
+# gorillib/bucket (WIP -- NOT READY YET) -- freeform storage with definable access
+
+* A `Go::Bucket` fulfills the contract of `Go::Model`
+
+## Arbitrary read/writes with `[]/[]=`, defined access with `.foo/.foo=`
+
+Overriding `method_missing` is uncouth. It screws up your stack traces, muddies your interface and leads to unassertive code:
+
+    ```ruby
+    Settings.defcon = 3
+    # ... elsewhere ...
+    if Settings.def_con.to_i <= 1
+      WOPR.launch_nukes!
+    end
+    ```
+
+Here, mispelling `def_con` as `defcon` (along with being wussy about the attribute's type and negligent about prescribing a default value) leads to the destruction of humanity.
+
+Nonetheless, for several reasons (prudent laziness, handling configuration of an external component, etc) it's important to handle arbitrary attributes uniformly.
+
+So the rule is that you can get or set anything you like using `[]` and `[]=` respectively, no need to define it first:
+
+    Settings[:whatever] = 3
+    Settings[:whatever]       #=> 3
+
+You don't get any magic, and the ugly accessor leaves you in no doubt that you're being lazy (ain't judging, just sayin').  When you define a setting you get accessors for that attribute and all associated magic:
+
+    Settings.option :defcon, Integer, :doc => 'Current NORAD defense condition', :default => 5, :validates => { :in => 1..5 }
+    Settings.defcon           #=> 5
+    Settings.defcon = 0
+    Settings.validate!        # raises a validation exception
+
+Defined fields' magic works whichever form of access you use -- here, type-converting the value on assignment:
+
+    Settings[:defcon]     = '5'   #=> 5
+    Settings.defcon       = '5'   #=> 5
+    Settings.receive_defcon('5')  #=> 5
+
+## Keys
+
+Keys be lower-cased identifiers: they should match `/\A([a-z][a-z0-9\_]*)\z/`.
+
+## Deep Hash
+
+     Hash.new {|h,k| h[k] = Hash.new(&h.default_proc)}
+
+### Dot addressing
+
+Can retrieve keys as 'x.y.z' meaning 'foo[x][y][z]'.
+
+* On a `get`, will  
+  - return the value, if `foo[:x][:y][:z]` exists
+  - return nil, if either `foo[:x]` or `foo[:x][:y]` is *unset*. It will not create `foo[:x]` or `foo[:x][:y]`.
+  - raise an error if either `foo[:x]` or `foo[:x][:y]` is set to a value that does not respond to `[]`
+
+* On a `set`, will  
+  - raise an error if either `foo[:x]` or `foo[:x][:y]` is set to a value that doesn't respond to `[]`
+  - set and return the value; any intermediate buckets (`foo[:x]` and `foo[:x][:y]`) will be created if they don't exist.
+
+
+__________________________________________________________________________
+
+
+## Examples
+
+### how hash-like is this?
+
+* obj.deep_get(:foo, :bar, :baz) #
+* obj.deep_set(:foo, :bar, :baz, val) # sets foo bar baz.
+  - where property has a type, it uses that type
+  - otherwise it uses Mash, and calls [] on it
+
+* TD votes NO on magic recursive hash behavior
+
+
+    {
+      :buck1  => { 
+        :buck2  =>  { :k3 => 33, :ehsh => {}, :arr => [11, 12, 13] },
+        :cars   => [{ :model => 'ford', :cylinders => 8 }, { :model => 'buick', :cylinders => 6 }],
+      }
+      :buck3 => {
+        :buck4  =>  { :k4 => 44 }
+        :k5     => nil,
+      }
+      :k6       => 69
+    }
+
+* `obj[:buck3]`               # b{ :buck4=>b{ :k4=>44 },:k5=>nil } -- it's a bucket
+* `obj[:buck5] = Hash.new`
+* `obj[:buck5].class`         # Bucket -- it's converted to bucket
+* `obj[:xxx]`                 # nil -- it's not there
+* `obj[:xxx][:yyy][:zzz]`     # fails -- it doesn't try to index into the nil `obj[:xxx]` cell.
+* `obj[:xxx]`                 # nil -- it didn't create the `obj[:xxx]` object when we tried to read on the previous line.
+
+    c.options_for 'wheels', 'interior.fabric', 'interior.carpeting'
+    
+    c[:wheels]                    # c{ }
+    c[:wheels][:whitewall]        # nil
+    c[:wheels][:whitewall] = true # true
+    
+    c.interior.fabric.color 
+    
+
+* `obj[:k6][:bomb]`           # raises ArgumentError; `obj[:k6]` is not a bucket.
+  
+* `obj[:'hello-there']        # undefined; `'hello-there'` is not a valid identifier.
+
+* `obj[:buck3][:buck4][:k5] = 55` 
+  `obj[:buck3]`                    # b{ :buck4=>b{ :k4=>44 },:k5 => 55}
+* `obj[:f][:g][:h] = 7`
+  `obj[:f]`                   # b{ :g => b{ :h => 7 } }
+
+* `obj[:foo][:bar][:baz] ||= 1` -- **hard** ?I think?
+
+* `obj[:foo]`       -- nil
+* `obj[:foo][:bar]` -- raise
+* `obj[:foo][:bar] = 3`  --
+  - now obj[:foo][:bar] is 3
+  - and obj[:foo] is a ?dsl_object?? but
+
+`obj[:foo][:bar]`
+
+Suppose `obj[:foo]` is set to a
+
+Seems clear these should do the right thing:
+
+* `obj.merge`
+* `obj.reverse_merge`
+* `obj.keys`
+* `obj.values`
+* ... and a few more
+
+Also:
+
+* `obj.to_a`?
+* `obj.each`?
+* other crazy Enumerable properties?
+
+### TODO
+
+* figure out the method structure for
+  - read/write/unset of attributes when Hash vs Accessors vs Instance Variables
+  - reader/writer: raw vs. hooks, dirty, etc.
+
+## Configliere Settings
+
+
+Configliere lets you define arbitrary attributes of a param, notably:
+
+     [:description]  Documentation for the param, used in the --help message
+     [:default]      Sets a default value (applied immediately)
+     [:env_var]      Environment variable to adopt (applied immediately, and after +:default+)
+     [:type]         Converts param's value to the given type, just before the finally block is called
+     [:finally]      Block of code to postprocess settings or handle complex configuration.
+     [:required]     Raises an error if, at the end of calling resolve!, the param's value is nil.

data/notes/builder.md

@@ -0,0 +1,170 @@
+# gorillib/builder -- construct elegant ruby Domain-Specific Languages (DSLs).
+
+`Gorillib::Builder` provides foundation models for elegant ruby DSLs (Domain-Specific Languages). A builder block's relaxed ruby syntax enables highly-readable specification of complex behavior:
+
+    ```ruby
+    garage do
+      car :ford_tudor do
+        manufacturer    'Ford Motor Company'
+        car_model       'Tudor'
+        year            1939
+        doors           2
+        style           :sedan
+        engine do
+          volume  350
+          cylinders     8
+        end
+      end
+
+      car :wildcat do
+        manufacturer    'Buick'
+        car_model       'Wildcat'
+        year            1968
+        doors           2
+        style           :convertible
+        engine do
+          volume  455
+          cylinders     8
+        end
+      end
+    end
+    ```
+
+### Defining a builder
+
+To make a class be a builder, simply `include Gorillib::Builder`:
+
+
+      ```ruby
+      class Garage
+        include Gorillib::Builder
+        collection :car
+      end
+
+      class Car
+        include Gorillib::Builder
+        field      :name,         String
+        field      :manufacturer, String
+        field      :car_model,    String
+        field      :year,         Integer
+        field      :doors,        Integer
+        field      :style,        Symbol, :validates => { :inclusion_in => [:sedan, :coupe, :convertible] }
+        member     :engine
+        belongs_to :garage
+      end
+
+      class Engine
+        include Gorillib::Builder
+        field      :volume, Integer
+        field      :cylinders,    Integer
+        belongs_to :car
+      end
+      ```
+
+### getset accessors
+
+Fields of a Builder class create a single "getset" accessor (and not the familiar 'foo/foo=' pair):
+
+* `car_model.foo`                -- returns value of foo
+* `car_model.foo(val)`           -- sets foo to `val`, which can be any value, even `nil`.
+
+### member fields
+
+A builder class can have `member` fields; the type of a member field should be a builder class itself
+
+With no arguments, the accessor returns the value of engine attribute, creating if necessary:
+
+      ```ruby
+      car.engine                    #=> #<Engine volume=~ cylinders=~>
+      ```
+
+If you pass in a hash of values, the engine is created if necessary and then asked to `receive!` the hash.
+
+      ```ruby
+      car.engine                    #=> #<Engine volume=~ cylinders=~>
+      car.engine(:cylinders => 8)   #=> #<Engine volume=~ cylinders=8>
+      car.engine.cylinders          #=> 8
+      ```
+
+If you provide a no-args block, it is `instance_eval`ed in the context of the member object:
+
+      ```ruby
+      car :ford_tudor do
+        engine(:cylinders => 8) do
+          self                      #=> #<Engine volume=~ cylinders=8>
+          volume  455
+          cylinders     self.cylinders - 2
+          self                      #=> #<Engine volume=455 cylinders=6>
+        end
+        engine                      #=> #<Engine volume=455 cylinders=6>
+      end
+      ```
+
+Some people disapprove of `instance_eval`, as they consider it unseemly to mess around with `self`. If you instead provide a one-arg block, the member object is passed in:
+
+      ```ruby
+      car :ford_tudor do |c|
+        c.engine(:cylinders => 8) do |eng|
+          self                      #=> #<Car ...>
+          eng.volume 455
+          eng.cylinders    eng.cylinders - 2
+          eng                       #=> #<Engine volume=455 cylinders=6>
+        end
+        c.engine                    #=> #<Engine volume=455 cylinders=6>
+      end
+      ```
+
+### collections
+
+A builder class can also have `collection` fields, to contain named builder objects.
+
+      ```ruby
+      garage do
+        car(:ford_tudor)            #=> #<Car name="ford tudor" ...>
+        cars                        #=> { :ford_tudor => #<Car ...>, :wildcat => #<Car ...> }
+      end
+      ```
+
+The collected items must respond to `id` (FIXME:). The singular accessor accepts arguments just like a `member` accessor:
+
+      ```ruby
+      garage do
+        car(:ford_tudor, :year => 1939)
+        #=> #<Car name=`<:ford_tudor year=1939 doors=~ ...>
+        car(:ford_tudor, :year => 1939) do
+          doors           2
+          style           :convertible
+        end
+        #=> #<Car name="ford tudor" year=1939 doors=2 ...>
+        car(:wildcat, :year => 1968) do |c|
+          c.doors         2
+          c.style         :convertible
+        end
+        #=> #<Car name= year=1939 doors=2 ...>
+
+
+
+
+### Model methods
+
+builders have the following:
+
+* `Model::Defaults`
+* `Model::Naming`
+* `Model::Conversion`
+
+
+
+### SubclassRegistry
+
+When a subclass happens, decorates class with `saucepot(...)`, equivalent to
+
+        update_or_create
+        registers
+
+* At class level, `registry_for(CookingUtensil)` gives
+  - `add_cooking_utensil()` and so forth for adding and getting
+  - Protected `cooking_utensils` hash class attribute
+* Enlisting a resource class (Kitchen.register(FryingPan) and gives you a magic `frying_pan` factory method with signature `frying_pan(name, *args, &block)`
+  - If resource does not exist, calls `FryingPan.new` with full set of params and block. Add it to `cooking_utensils` registry.
+  - If resource with that name exists, retrieve it. call `merge()` with the parameters, and `run_in_scope` with the block.