storable 0.8.6 → 0.9.pre.RC2

data/lib/proc_source.rb

@@ -1,113 +1,162 @@
-#--
+
+#
 # Based on:
-# http://github.com/imedo/background
-#++
+# https://github.com/imedo/background
+# https://github.com/imedo/background_lite
+# With improvements by:
+# https://github.com/notro/storable
+#
+
+# RubyToken was removed in Ruby 2.7
+if RUBY_VERSION < "2.7"
+  require 'irb/ruby-token'
+else
+  require './lib/core_ext.rb'
+end
 
-require 'stringio'
 require 'irb/ruby-lex'
-#SCRIPT_LINES__ = {} unless defined? SCRIPT_LINES__
+require 'pry'
+require 'stringio'
+
+SCRIPT_LINES__ = {} unless defined? SCRIPT_LINES__
+
 
 class ProcString < String
-  attr_accessor :file, :lines, :arity, :kind
+  # Filename where the proc is defined
+  attr_accessor :file
+
+  # Range of lines where the proc is defined. e.g. (12..16)
+  attr_accessor :lines
+  attr_accessor :arity, :kind
+
+  # Return a Proc object
+  # If #lines and #file is specified, these are tied to the proc.
   def to_proc(kind="proc")
-    result = eval("#{kind} #{self}")
+    if @file && @lines
+      raise "#lines must be a range" unless @lines.kind_of? Range
+      result = eval("#{kind} #{self}", binding, @file, @lines.min)
+    else
+      result = eval("#{kind} #{self}")
+    end
     result.source = self
     result
   end
+
+  # Return a lambda
   def to_lambda
-    to_proc "lamda"
+    to_proc "lambda"
   end
 end
 
 class RubyToken::Token
-  
-    # These EXPR_BEG tokens don't have associated end tags
-  FAKIES = [RubyToken::TkWHEN, RubyToken::TkELSIF, RubyToken::TkTHEN]
-  
+  # These EXPR_BEG tokens don't have associated end tags
+  FAKIES = [
+    RubyToken::TkWHEN,
+    RubyToken::TkELSIF,
+    RubyToken::TkELSE,
+    RubyToken::TkTHEN,
+  ]
+
+  def name
+    @name ||= nil
+  end
+
   def open_tag?
-    return false if @name.nil? || get_props.nil?
-    a = (get_props[1] == RubyToken::EXPR_BEG) &&
-          self.class.to_s !~ /_MOD/  && # ignore onliner if, unless, etc...
-          !FAKIES.member?(self.class)  
-    a 
+    return false if name.nil? || get_props.nil?
+    is_open = (
+      (get_props[1] == RubyToken::EXPR_BEG) &&
+      (self.class.to_s !~ /_MOD/)  &&  # ignore onliner if, unless, etc...
+      (!FAKIES.member?(self.class))
+    )
+    is_open
   end
-  
+
   def get_props
-    RubyToken::TkReading2Token[@name]
+    RubyToken::TkReading2Token[name]
   end
-  
+
 end
 
 # Based heavily on code from http://github.com/imedo/background
 # Big thanks to the imedo dev team!
 #
 module ProcSource
-  
-  def self.find(filename, start_line=0, block_only=true)
+  def self.find(filename, start_line=1, block_only=true)
     lines, lexer = nil, nil
     retried = 0
     loop do
       lines = get_lines(filename, start_line)
       return nil if lines.nil?
-      #p [start_line, lines[0]]
       if !line_has_open?(lines.join) && start_line >= 0
-        start_line -= 1 and retried +=1 and redo 
+        start_line -= 1 and retried +=1 and redo
       end
       lexer = RubyLex.new
       lexer.set_input(StringIO.new(lines.join))
       break
     end
+
     stoken, etoken, nesting = nil, nil, 0
-    while token = lexer.token
-      n = token.instance_variable_get(:@name)
-      
+
+    binding.pry
+    if RUBY_VERSION < "2.7"
+      tokens = lexer.instance_variable_get '@OP'
+    else
+      lexer.lex
+      tokens = lexer.instance_variable_get '@tokens'
+    end
+
+    # tokens.each
+
+    while (token = lexer.token) do
       if RubyToken::TkIDENTIFIER === token
-        #nothing
+        # nothing
       elsif token.open_tag? || RubyToken::TkfLBRACE === token
         nesting += 1
         stoken = token if nesting == 1
       elsif RubyToken::TkEND === token || RubyToken::TkRBRACE === token
         if nesting == 1
-          etoken = token 
+          etoken = token
           break
         end
         nesting -= 1
+      elsif RubyToken::TkLBRACE === token
+        nesting += 1
       elsif RubyToken::TkBITOR === token && stoken
-        #nothing
+        # nothing
       elsif RubyToken::TkNL === token && stoken && etoken
         break if nesting <= 0
       else
-        #p token
+        # nothing
       end
     end
-#     puts lines if etoken.nil?
+
     lines = lines[stoken.line_no-1 .. etoken.line_no-1]
-    
-    # Remove the crud before the block definition. 
+
+    # Remove the crud before the block definition.
     if block_only
       spaces = lines.last.match(/^\s+/)[0] rescue ''
       lines[0] = spaces << lines[0][stoken.char_no .. -1]
     end
     ps = ProcString.new lines.join
-    ps.file, ps.lines = filename, start_line .. start_line+etoken.line_no-1
-    
+    ps.file = filename
+    ps.lines = start_line .. start_line+etoken.line_no-1
     ps
   end
-  
+
   # A hack for Ruby 1.9, otherwise returns true.
   #
   # Ruby 1.9 returns an incorrect line number
   # when a block is specified with do/end. It
-  # happens b/c the line number returned by 
+  # happens b/c the line number returned by
   # Ruby 1.9 is based on the first line in the
   # block which contains a token (i.e. not a
-  # new line or comment etc...). 
+  # new line or comment etc...).
   #
-  # NOTE: This won't work in cases where the 
-  # incorrect line also contains a "do". 
+  # NOTE: This won't work in cases where the
+  # incorrect line also contains a "do".
   #
   def self.line_has_open?(str)
-    return true unless RUBY_VERSION >= '1.9'
+    return true unless RUBY_VERSION >= '1.9' && RUBY_VERSION < '2.0'
     lexer = RubyLex.new
     lexer.set_input(StringIO.new(str))
     success = false
@@ -117,12 +166,14 @@ module ProcSource
         break
       when RubyToken::TkDO
         success = true
+      when RubyToken::TkfLBRACE
+        success = true
       when RubyToken::TkCONSTANT
-        if token.instance_variable_get(:@name) == "Proc" &&
+        if token.name == "Proc" &&
            lexer.token.is_a?(RubyToken::TkDOT)
           method = lexer.token
           if method.is_a?(RubyToken::TkIDENTIFIER) &&
-             method.instance_variable_get(:@name) == "new"
+             method.name == "new"
             success = true
           end
         end
@@ -130,20 +181,26 @@ module ProcSource
     end
     success
   end
-  
-  
-  def self.get_lines(filename, start_line = 0)
+
+  def self.get_lines(filename, start_line = 1)
     case filename
       when nil
         nil
-      when "(irb)"         # special "(irb)" descriptor?
+
+      # We're in irb
+      when "(irb)"
         IRB.conf[:MAIN_CONTEXT].io.line(start_line .. -2)
-      when /^\(eval.+\)$/  # special "(eval...)" descriptor?
+
+      # Or an eval
+      when /^\(eval.+\)$/
         EVAL_LINES__[filename][start_line .. -2]
-      else                 # regular file
+
+      # Or most likely a .rb file
+      else
         # Ruby already parsed this file? (see disclaimer above)
         if defined?(SCRIPT_LINES__) && SCRIPT_LINES__[filename]
           SCRIPT_LINES__[filename][(start_line - 1) .. -1]
+
         # If the file exists we're going to try reading it in
         elsif File.exist?(filename)
           begin
@@ -156,24 +213,77 @@ module ProcSource
   end
 end
 
-class Proc #:nodoc:
-  attr_reader :file, :line
+class Proc  # :nodoc:
   attr_writer :source
-  
+  @@regexp = Regexp.new('^#<Proc:0x[0-9A-Fa-f]+@?\s*(.+):(\d+)(.+?)?>$')
+
   def source_descriptor
-    unless @file && @line
-      if md = /^#<Proc:0x[0-9A-Fa-f]+@(.+):(\d+)(.+?)?>$/.match(inspect)
-        @file, @line = md.captures
-      end
+    return [@file, @line] if @file && @line
+
+    source_location = nil
+    if RUBY_VERSION >= '2.7'
+      source_location = *self.source_location
+    else
+      inspection = inspect
+      md = @@regexp.match(inspection)
+      exmsg = 'Unable to parse proc inspect (%s)' % inspection
+      raise Exception, exmsg if md.nil?
+      source_location = md.captures
     end
-    [@file, @line.to_i]
+
+    file, line = *source_location
+    @file, @line = [file, line.to_i]
   end
-  
+
   def source
     @source ||= ProcSource.find(*self.source_descriptor)
   end
-  
-  # Create a Proc object from a string of Ruby code. 
+
+  def line
+    source_descriptor
+    @line
+  end
+
+  def file
+    source_descriptor
+    @file
+  end
+
+  # Dump to Marshal format.
+  #   p = Proc.new { false }
+  #   Marshal.dump p
+  def _dump(limit)
+    raise "can't dump proc, #source is nil" if source.nil?
+    str = Marshal.dump(source)
+    str
+  end
+
+  # Load from Marshal format.
+  #   p = Proc.new { false }
+  #   Marshal.load Marshal.dump p
+  def self._load(str)
+    @source = Marshal.load(str)
+    @source.to_proc
+  end
+
+  # Dump to JSON string
+  def to_json(*args)
+    raise "can't serialize proc, #source is nil" if source.nil?
+    {
+      'json_class' => self.class.name,
+      'data'       => [source.to_s, source.file, source.lines.min, source.lines.max]
+    }.to_json#(*args)
+  end
+
+  def self.json_create(o)
+    s, file, min, max = o['data']
+    ps = ProcString.new s
+    ps.file = file
+    ps.lines = (min..max)
+    ps.to_proc
+  end
+
+  # Create a Proc object from a string of Ruby code.
   # It's assumed the string contains do; end or { }.
   #
   #     Proc.from_string("do; 2+2; end")
@@ -181,7 +291,7 @@ class Proc #:nodoc:
   def self.from_string(str)
     eval "Proc.new #{str}"
   end
-  
+
 end
 
 if $0 == __FILE__
@@ -194,19 +304,18 @@ if $0 == __FILE__
   end
 
   a = Proc.new() { |a|
-    puts  "Hello Rudy2" 
+    puts  "Hello Rudy2"
   }
- 
+
   b = Proc.new() do |b|
     puts { "Hello Rudy3" } if true
   end
-  
+
   puts @blk.inspect, @blk.source
   puts [a.inspect, a.source]
   puts b.inspect, b.source
-  
+
   proc = @blk.source.to_proc
   proc.call(1)
 end
 
-

data/lib/storable.rb

@@ -35,14 +35,14 @@ class Storable
   require 'proc_source'  
   require 'storable/orderedhash' if USE_ORDERED_HASH
   unless defined?(SUPPORTED_FORMATS) # We can assume all are defined
-    VERSION = "0.8.5"
+    VERSION = "0.8.9"
     NICE_TIME_FORMAT  = "%Y-%m-%d@%H:%M:%S".freeze 
     SUPPORTED_FORMATS = [:tsv, :csv, :yaml, :json, :s, :string].freeze 
   end
   
   @debug = false
   class << self
-    attr_accessor :sensitive_fields, :field_names, :field_types, :debug
+    attr_accessor :sensitive_fields, :field_names, :field_types, :field_opts, :debug
   end
   
   # Passes along fields to inherited classes
@@ -66,37 +66,61 @@ class Storable
   # data is available by the standard accessors, class.product and class.product= etc...
   # The value of the field will be cast to the type (if provided) when read from a file. 
   # The value is not touched when the type is not provided. 
-  def self.field(args={}, &processor)
+  def self.field(*args, &processor)
     # TODO: Examine casting from: http://codeforpeople.com/lib/ruby/fattr/fattr-1.0.3/
-    args = {args => nil} unless args.kind_of?(Hash)
-
+    field_definitions = {}
+    if args.first.kind_of?(Hash)
+      args.first.each_pair do |fname,klass|
+        field_definitions[fname] = { :class => klass }
+      end
+    else
+      fname, opts = *args
+      if opts.nil?
+        field_definitions[fname] = {}
+      elsif Hash === opts
+        field_definitions[fname] = opts
+      else
+        raise ArgumentError, "Second argument must be a hash" 
+      end
+    end
+    
     self.field_names ||= []
     self.field_types ||= {}
-    args.each_pair do |m,t|
-      self.field_names << m
-      self.field_types[m] = t unless t.nil?
+    self.field_opts ||= {}
+    field_definitions.each_pair do |fname,opts|
+      self.field_names << fname
+      self.field_opts[fname] = opts
+      self.field_types[fname] = opts[:class] unless opts[:class].nil?
       
       # This processor automatically converts a Proc object
       # to a String of its source. 
-      processor = proc_processor if t == Proc && processor.nil?
+      processor = proc_processor if opts[:class] == Proc && processor.nil?
       
       unless processor.nil?
-        define_method("_storable_processor_#{m}", &processor)
+        define_method("_storable_processor_#{fname}", &processor)
       end
       
-      if method_defined?(m) # don't redefine the getter method
-        STDERR.puts "method exists: #{self}##{m}" if Storable.debug
+      if method_defined?(fname) # don't redefine the getter method
+        STDERR.puts "method exists: #{self}##{fname}" if Storable.debug
       else
-        define_method(m) do 
-          instance_variable_get("@#{m}") 
+        define_method(fname) do 
+          ret = instance_variable_get("@#{fname}")
+          if ret.nil? 
+            if opts[:default]
+              ret = opts[:default]
+            elsif opts[:meth]
+              ret = self.send(opts[:meth])
+            end
+          end
+          ret
         end
       end
       
-      if method_defined?("#{m}=") # don't redefine the setter methods
-        STDERR.puts "method exists: #{self}##{m}=" if Storable.debug
+      if method_defined?("#{fname}=") # don't redefine the setter methods
+        STDERR.puts "method exists: #{self}##{fname}=" if Storable.debug
       else
-        define_method("#{m}=") do |val| 
-          instance_variable_set("@#{m}",val)
+        define_method("#{fname}=") do |val| 
+          instance_variable_set("@#{fname}",val)
         end
       end
     end
@@ -145,15 +169,15 @@ class Storable
   
   # Returns an array of field names defined by self.field
   def field_names
-    self.class.field_names
+    self.class.field_names #|| self.class.ancestors.first.field_names
   end
   # Returns an array of field types defined by self.field. Fields that did 
   # not receive a type are set to nil.
   def field_types
-    self.class.field_types
+    self.class.field_types #|| self.class.ancestors.first.field_types
   end
   def sensitive_fields
-    self.class.sensitive_fields
+    self.class.sensitive_fields #|| self.class.ancestors.first.sensitive_fields
   end
   
   # Dump the object data to the given format. 
@@ -200,7 +224,7 @@ class Storable
   def init *args
     from_array *args
   end
-
+  
   def initialize *args
     init *args
   end
@@ -298,14 +322,16 @@ class Storable
   def to_hash
     preprocess if respond_to? :preprocess
     tmp = USE_ORDERED_HASH ? Storable::OrderedHash.new : {}
-    field_names.each do |fname|
-      next if sensitive? && self.class.sensitive_field?(fname)
-      v = self.send(fname)
-      v = process(fname, v) if has_processor?(fname)
-      if Array === v
-        v = v.collect { |v2| v2.kind_of?(Storable) ? v2.to_hash : v2 } 
+    if field_names
+      field_names.each do |fname|
+        next if sensitive? && self.class.sensitive_field?(fname)
+        v = self.send(fname)
+        v = process(fname, v) if has_processor?(fname)
+        if Array === v
+          v = v.collect { |v2| v2.kind_of?(Storable) ? v2.to_hash : v2 } 
+        end
+        tmp[fname] = v.kind_of?(Storable) ? v.to_hash : v
       end
-      tmp[fname] = v.kind_of?(Storable) ? v.to_hash : v
     end
     tmp
   end
@@ -329,9 +355,6 @@ class Storable
     hash = to_hash
     if YAJL_LOADED # set by Storable
       ret = Yajl::Encoder.encode(hash)
-      #raise "DELANO"
-      #ret.force_encoding("ISO-8859-1")
-      #p [:to, ret.encoding.name] if ret.respond_to?(:encoding)
       ret
     elsif JSON_LOADED
       JSON.generate(hash, *from, &blk)
@@ -508,7 +531,8 @@ class Storable
     end
     def proc_processor
       Proc.new do |val|
-        (Proc === val) ? val.source : val 
+        ret = (Proc === val) ? val.source : val 
+        ret
       end
     end
     # If the object already has a value for +@id+