nydp 0.5.0 → 0.6.0

data/lib/nydp/pair.rb

@@ -1,5 +1,5 @@
 class Nydp::Pair
-  NIL = Nydp::NIL
+  NIL = nil
   include Nydp::Helper, Enumerable
   extend Nydp::Helper
 
@@ -14,19 +14,95 @@ class Nydp::Pair
   def cadr       ; cdr.car                                    ; end
   def cdar       ; car.cdr                                    ; end
   def cddr       ; cdr.cdr                                    ; end
-  def car= thing ; @car = thing ; @_hash = nil                ; end
-  def cdr= thing ; @cdr = thing ; @_hash = nil                ; end
-  # def hash       ; @_hash ||= (car.hash + cdr.hash)           ; end
-  def hash       ; (car.hash + cdr.hash)                      ; end # can't cache hash of symbol, breaks when unmarshalling
+  def car= thing ; @car = thing                               ; end
+  def cdr= thing ; @cdr = thing                               ; end
   def eql? other ; self == other                              ; end
-  def copy       ; cons(car, cdr.copy)                        ; end
-  def +    other ; copy.append other                          ; end
+  def +    other ; copy_append other                          ; end
   def size       ; 1 + (cdr.is_a?(Nydp::Pair) ? cdr.size : 0) ; end
   def inspect    ; "(#{inspect_rest})"                        ; end
-  def &    other ; self.class.from_list((Set.new(self) & other).to_a)           ; end
-  def |    other ; self.class.from_list((Set.new(self) | other).to_a)           ; end
-  def -    other ; self.class.from_list((Set.new(self) - other).to_a)           ; end
-  def proper?    ; Nydp::NIL.is?(cdr) || (cdr.is_a?(Nydp::Pair) && cdr.proper?) ; end
+  def []       i ; nth i                                      ; end
+  def &    other ; self.class.from_list((Set.new(self) & Array(other)).to_a)    ; end
+  def |    other ; self.class.from_list((Set.new(self) | Array(other)).to_a)    ; end
+  def -    other ; self.class.from_list((Set.new(self) - Array(other)).to_a)    ; end
+  def proper?    ; (!cdr) || (cdr.is_a?(Nydp::Pair) && cdr.proper?)             ; end
+
+  # def method_missing m, *args
+  #   to_a.send m, *args
+  # end
+
+  # can't cache hash of symbol, breaks when unmarshalling
+  def hash
+    a    = 0
+    x    = self
+    while (x.is_a? Nydp::Pair)
+      a = a + x.car.hash
+      x = x.cdr
+    end
+
+    a + x.hash
+  end
+
+  def copy
+    a    = last = cons
+    x    = self
+    while (x.is_a? Nydp::Pair)
+      last = last.cdr = cons(x.car)
+      x    = x.cdr
+    end
+
+    last.cdr = x
+    a.cdr
+  end
+
+  def copy_append lastcdr
+    a    = last = cons
+    x    = self
+    while (x.is_a? Nydp::Pair)
+      last = last.cdr = cons(x.car)
+      x    = x.cdr
+    end
+
+    last.cdr = lastcdr
+    a.cdr
+  end
+
+  def map
+    a    = last = cons
+    x    = self
+    while (x.is_a? Nydp::Pair)
+      last = last.cdr = cons(yield x.car)
+      x    = x.cdr
+    end
+
+    last.cdr = yield(x) if x
+    a.cdr
+  end
+
+  def join str
+    to_a.join(str)
+  end
+
+  def compile_to_ruby indent, srcs, opts=nil
+    a,x = [], self
+    while x.is_a?(Nydp::Pair)
+      a << x.car.compile_to_ruby("", srcs)
+      x = x.cdr
+    end
+
+    if !x || (x.is_a?(Nydp::Literal) && !x.expression)
+      "#{indent}list(" + a.join(", ") + ")"
+    else
+      "#{indent}Nydp::Pair.from_list([" + a.join(", ") + "], #{x.compile_to_ruby "", srcs})"
+    end
+  end
+
+  def nth n
+    xs = self
+    while n > 0
+      xs, n = xs.cdr, n-1
+    end
+    xs.car
+  end
 
   def index_of x
     if x == car
@@ -51,24 +127,34 @@ class Nydp::Pair
 
   # returns Array of elements as they are
   def to_a list=[]
-    list << car
-    cdr.is_a?(Nydp::Pair) ? cdr.to_a(list) : list
+    x = self
+    while x.is_a?(Nydp::Pair)
+      list << x.car
+      x = x.cdr
+    end
+    list
   end
 
+  # this breaks everything even though it seems like the logical thing to do...
+  # def to_ary
+  #   to_a
+  # end
+
   def self.parse_list list
-    if sym? list.slice(-2), "."
+    if list.slice(-2) == :"."
       from_list(list[0...-2], list.slice(-1))
     else
       from_list list
     end
   end
 
-  def self.from_list list, last=Nydp::NIL, n=0
-    if n >= list.size
-      last
-    else
-      Nydp::Pair.new list[n], from_list(list, last, n+1)
+  def self.from_list list, last=nil, n=list.size-1
+    while (n >= 0)
+      last = cons(list[n], last)
+      n -= 1
     end
+
+    last
   end
 
   def == other
@@ -76,37 +162,48 @@ class Nydp::Pair
   end
 
   def each &block
-    yield car
-    cdr.each(&block) unless Nydp::NIL.is?(cdr)
+    xs = self
+    while xs
+      yield xs.car
+      xs = xs.cdr
+    end
   end
 
   def to_s
-    if car.is_a?(Nydp::Symbol) && car.is?(:quote)
-      if Nydp::NIL.is? cdr.cdr
+    if (car == :quote)
+      if !cdr.cdr
         "'#{cdr.car.to_s}"
       else
         "'#{cdr.to_s}"
       end
-    elsif car.is_a?(Nydp::Symbol) && car.is?(:"brace-list")
-      if Nydp::NIL.is? cdr
+    elsif (car == :"brace-list")
+      if !cdr
         "{}"
       else
         "{ #{cdr.to_s_rest} }"
       end
-    elsif car.is_a?(Nydp::Symbol) && car.is?(:quasiquote)
-      if Nydp::NIL.is? cdr.cdr
+    elsif (car == :"percent-syntax")
+      cdr.to_a.compact.join("%")
+    elsif (car == :"colon-syntax")
+      cdr.to_a.compact.join(":")
+    elsif (car == :"dot-syntax")
+      cdr.to_a.compact.join(".")
+    elsif (car == :"prefix-list")
+      "#{cdr.car.to_s}#{cdr.cdr.car.to_s}"
+    elsif (car == :quasiquote)
+      if !cdr.cdr
         "`#{cdr.car.to_s}"
       else
         "`#{cdr.to_s}"
       end
-    elsif car.is_a?(Nydp::Symbol) && car.is?(:unquote)
-      if Nydp::NIL.is? cdr.cdr
+    elsif (car == :unquote)
+      if !cdr.cdr
         ",#{cdr.car.to_s}"
       else
         ",#{cdr.to_s}"
       end
-    elsif car.is_a?(Nydp::Symbol) && car.is?(:"unquote-splicing")
-      if Nydp::NIL.is? cdr.cdr
+    elsif (car == :"unquote-splicing")
+      if !cdr.cdr
         ",@#{cdr.car.to_s}"
       else
         ",@#{cdr.to_s}"
@@ -116,42 +213,41 @@ class Nydp::Pair
     end
   end
 
+  def to_s_car
+    if (!car)
+      "nil"
+    elsif car.is_a?(String)
+      car.inspect
+    elsif car.is_a?(Symbol)
+      car._nydp_inspect
+    else
+      car.to_s
+    end
+  end
+
   def to_s_rest
     cdr_s = if cdr.is_a?(self.class)
               cdr.to_s_rest
-            elsif Nydp::NIL.is? cdr
-              nil
-            else
+            elsif cdr
               ". #{cdr.to_s}"
             end
 
-    [car.to_s, cdr_s].compact.join " "
+    [to_s_car, cdr_s].compact.join " "
   end
 
   def inspect_rest
-    res = [car.inspect]
+    res = [car._nydp_inspect]
     it = cdr
-    while it && it != Nydp::NIL
+    while it
       if it.is_a?(self.class)
-        res << it.car.inspect
+        res << it.car._nydp_inspect
         it = it.cdr
       else
         res << "."
-        res << it.inspect
+        res << it._nydp_inspect
         it = nil
       end
     end
     res.compact.join " "
   end
-
-  def append thing
-    if Nydp::NIL.is? self.cdr
-      self.cdr = thing
-    elsif pair? self.cdr
-      self.cdr.append thing
-    else
-      raise "can't append #{thing} to list #{self} : cdr is #{self.cdr.inspect}"
-    end
-    self
-  end
 end

data/lib/nydp/parser.rb

@@ -1,16 +1,14 @@
 module Nydp
   class Parser
-    attr_accessor :ns
-
-    def initialize ns
-      @ns = ns
-      # TODO pre-initialize all the hard-coded syms used here, eg
-      # @quote = sym(:quote)
-      # @quasiquote = sym(:quasiquote)
-    end
-
     def sym name
-      Nydp::Symbol.mk name.to_sym, ns
+      n = name.to_s.to_sym
+      if n == :nil
+        nil
+      elsif n == :t
+        true
+      else
+        n
+      end
     end
 
     def read_list token_stream, termination_token, list=[]
@@ -102,7 +100,7 @@ module Nydp
       when :string_open_delim
         string token_stream, token.last, close_delimiter_for(token.last)
       when :sym_open_delim
-        sym token_stream.next_string_fragment(token.last, /\|/, nil)
+        sym token_stream.next_string_fragment(token.last, /\|/, nil).string
       when :left_paren
         prefix_list token[1], read_list(token_stream, :right_paren)
       when :left_brace

data/lib/nydp/plugin.rb

@@ -1,40 +1,109 @@
-require 'nydp/image_store'
+require 'digest'
 
 module Nydp
   PLUGINS = []
-  @@image_store = Nydp::ImageStore.new
+  COMMENT_RX = /^.*##> /
+  GENERATED_CLASS_PREFIX_REGEXP = /#{Nydp::GENERATED_CLASS_PREFIX}/
 
+  module PluginHelper
+    def base_path ; "" ; end # override this to provide common prefix for plugin filenames
+
+    def file_readers filenames, bp=base_path
+      filenames.map { |f| Nydp::FileReader.new(f.gsub(bp, ""), f) }
+    end
+
+    def relative_path name
+      # File.join File.expand_path(File.dirname(__FILE__)), name
+      File.join File.expand_path(File.dirname(caller[0].split(/:\d+:/)[0])), name
+    end
+  end
+
+  def self.nydp_from_backtrace str
+    file, original_line, meth = str.split(/:/)
+    line = original_line.to_i - 2 # -1 to convert from 1-based index to zero-based index ; -1 to start looking backwards from previous line
+    filepath = File.expand_path file
+
+    if filepath =~ Nydp::GENERATED_CLASS_PREFIX_REGEXP
+      code = File.read filepath
+      lines = code.split /\n/
+
+      while line > 0 && !(lines[line] =~ COMMENT_RX)
+        line = line - 1
+      end
+
+      if (line >= 0 && (lines[line] =~ COMMENT_RX))
+        comment = lines[line].sub(COMMENT_RX, '').gsub(/\\n/, "\n")
+        return [(filepath.sub(base_gen_path + '/', '') + ":" + original_line), comment].join("\n")
+      else
+        return str
+      end
+    end
+
+    str
+  end
+
+  def self.enhance_backtrace bt
+    bt.map { |s| nydp_from_backtrace s }
+  end
+
+  def self.base_gen_path   ; File.expand_path("rubycode/")       ; end
   def self.plug_in  plugin ; PLUGINS << plugin                   ; end
   def self.load_rake_tasks ; PLUGINS.each &:load_rake_tasks      ; end
   def self.setup        ns ; PLUGINS.each { |plg| plg.setup ns } ; end
-  def self.loadfiles       ; PLUGINS.map(&:loadfiles).flatten    ; end
-  def self.testfiles       ; PLUGINS.map(&:testfiles).flatten    ; end
   def self.plugin_names    ; PLUGINS.map(&:name)                 ; end
-  def self.loadall ns, plugin, files
-    vm = VM.new(ns)
-    apply_function ns, :"script-run", :"plugin-start", plugin.name if plugin
+
+  def self.loadall ns, plugin, files, manifest
+    ns.apply :"script-run", :"plugin-start", plugin.name if plugin
     files.each { |f|
-      script_name = f.gsub plugin.base_path, ""
-      reader = Nydp::StreamReader.new(File.new(f))
-      Nydp::Runner.new(vm, ns, reader, nil, (script_name || f)).run
-      yield script_name if block_given?
+      Nydp::Runner.new(ns, f, nil, f.name, manifest).run
+      yield f.name if block_given?
     }
   ensure
-    apply_function ns, :"script-run", :"plugin-end", plugin.name if plugin
+    ns.apply :"script-run", :"plugin-end", plugin.name if plugin
   end
 
   def self.all_files ; PLUGINS.each_with_object([]) { |plg, list|  plg.loadfiles.each { |f| list << f } } ; end
-  def self.set_image_store store ; @@image_store = store ; end
-
-  def self.get_nydp ; @@image_store.get ; end
 
   def self.build_nydp &block
-    ns = Namespace.new
+    rc = base_gen_path
+    $LOAD_PATH.unshift rc unless $LOAD_PATH.include?(rc)
+
+    digest   = Digest::SHA256.hexdigest(all_files.map { |f| f.read }.join("\n"))
+    mname    = "Manifest_#{digest}"
+
+    ns = ::Nydp::Namespace.new
     setup(ns)
-    PLUGINS.each { |plg|
-      loadall ns, plg, plg.loadfiles, &block
-      loadall ns, plg, plg.testfiles, &block
+
+    digest = ""
+
+    PLUGINS.each { |plugin|
+      digest = install_plugin ns, plugin, digest, &block
     }
+
     ns
   end
+
+  def self.install_plugin ns, plugin, digest, &block
+    f0       = plugin.loadfiles.map { |f| f.read }.join("\n")
+    f1       = plugin.testfiles.map { |f| f.read }.join("\n")
+    digest   = Digest::SHA256.hexdigest([digest, f0, f1].join("\n"))
+    mname    = "Manifest_#{digest}"
+
+    if Nydp.logger
+      Nydp.logger.info "manifest name for plugin #{plugin.name.inspect} is #{mname.inspect}"
+    end
+
+    begin
+      require mname
+      const_get(mname).build ns
+
+    rescue LoadError => e
+      manifest = []
+      loadall ns, plugin, plugin.loadfiles, manifest, &block
+      loadall ns, plugin, plugin.testfiles, manifest, &block
+      Nydp::Evaluator.mk_manifest "Manifest_#{digest}", manifest
+    end
+
+    digest
+  end
 end

data/lib/nydp/runner.rb

@@ -1,19 +1,51 @@
+require 'digest'
 require 'readline'
 require 'nydp/readline_history'
 
 module Nydp
   class StringReader
-    def initialize string ; @string = string ; end
+    attr_accessor :name
+
+    def initialize name, string
+      @name, @string, @read = name, string, string
+    end
+
     def nextline
-      s = @string ; @string = nil ; s
+      s = @read ; @read = nil ; s
+    end
+
+    def read
+      @string
     end
   end
 
   class StreamReader
-    def initialize stream ; @stream = stream ; end
+    attr_accessor :name
+
+    def initialize name, stream
+      @name, @stream = name, stream
+    end
+
     def nextline
       @stream.readline unless @stream.eof?
     end
+
+    def read
+      ""
+    end
+  end
+
+  class FileReader < StreamReader
+    attr_accessor :filename
+
+    def initialize name, filename
+      super name, File.new(filename)
+      @filename = filename
+    end
+
+    def read
+      File.read filename
+    end
   end
 
   class ReadlineReader
@@ -37,52 +69,138 @@ module Nydp
   end
 
   class Evaluator
-    attr_accessor :vm, :ns, :name
+    attr_accessor :ns, :name
 
-    def initialize vm, ns, name
-      @name       = name
-      @vm         = vm
-      @ns         = ns
-      @precompile = Symbol.mk(:"pre-compile", ns)
-      @quote      = Symbol.mk(:quote, ns)
+    def initialize ns, name
+      @name = name
+      @ns   = ns
+      @rubydir = FileUtils.mkdir_p "rubycode"
     end
 
-    def compile_and_eval expr
+    def compile_expr expr
       begin
-        vm.thread_with_expr Pair.new(Compiler.compile(expr, Nydp::NIL), Nydp::NIL)
+        Compiler.compile(expr, nil, ns)
       rescue StandardError => e
-        raise Nydp::Error, "failed to eval #{expr.inspect}"
+        raise Nydp::Error, "failed to compile #{expr._nydp_inspect}"
       end
     end
 
-    def pre_compile expr
-      compile_and_eval(Pair.from_list [@precompile, Pair.from_list([@quote, expr])])
+    def self.mk_manifest name, class_list
+      class_expr = <<KLA
+#{class_list.map {|k| "require '#{k}'" }.join("\n")}
+
+class #{name}
+  def self.build ns
+    #{class_list.map {|k| "#{k}.new(ns).call" }.join("\n    ")}
+  end
+end
+KLA
+      File.open("rubycode/#{name}.rb", "w") { |f|
+        fullpath = File.expand_path("rubycode/#{name}.rb")
+        Nydp.logger.info "writing #{fullpath}" if Nydp.logger
+        f.write class_expr
+      }
+    end
+
+    class CompiledExpression
+      attr_accessor :ns
+      def initialize ns
+        @ns = ns
+      end
+
+      def list *things
+        Nydp::Pair.from_list things
+      end
+    end
+
+    def mk_ruby_source src, precompiled, compiled_expr, cname
+      srcs       = []
+      ruby_expr  = compiled_expr.compile_to_ruby "    ", srcs
+      six        = 0
+      srcs       = srcs.map { |s|
+        s = "  @@src_#{six} = #{s.inspect}"
+        six += 1
+        s
+      }
+      class_expr = "class #{cname} < Nydp::Runner::CompiledExpression
+
+#{srcs.join("\n")}
+
+  def src
+    #{src.inspect.inspect}
+  end
+
+  def precompiled
+    #{precompiled.inspect.inspect}
+  end
+
+  def call
+#{ruby_expr}
+  end
+end
+
+#{cname}
+"
+      File.open("rubycode/#{cname}.rb", "w") { |f| f.write class_expr }
+      class_expr
+    end
+
+    def mk_ruby_class src, precompiled, compiled_expr, cname
+      begin
+        require cname
+        self.class.const_get(cname)
+
+      rescue LoadError => e
+        # compiled_expr    = compile_expr precompiled # TODO : delete line in #evaluate and uncomment this one
+        fname = "rubycode/#{cname}.rb"
+        txt   = mk_ruby_source src, precompiled, compiled_expr, cname
+
+        eval(txt, nil, fname) || raise("failed to generate class #{cname} from src #{src}")
+      end
+    end
+
+    def eval_compiled compiled_expr, precompiled, src, manifest
+      return nil unless precompiled
+
+      digest  = Digest::SHA256.hexdigest(precompiled.inspect)
+      cname   = "#{GENERATED_CLASS_PREFIX}_#{digest.upcase}"
+      kla     = mk_ruby_class src, precompiled, compiled_expr, cname
+
+      manifest << cname
+
+      kla.new(ns).call
+
+    rescue Exception => e
+      raise Nydp::Error, "failed to eval #{compiled_expr._nydp_inspect} from src #{src._nydp_inspect}"
     end
 
-    def evaluate expr
-      compile_and_eval(pre_compile(expr))
+    def evaluate expr, manifest=[]
+      precompiled = ns.apply :"pre-compile-new-expression", expr
+      compiled    = compile_expr precompiled # TODO : we don't need this step if the class already exists! Do it later only when we need it
+      eval_compiled compiled, precompiled, expr, manifest
     end
   end
 
   class Runner < Evaluator
-    def initialize vm, ns, stream, printer=nil, name=nil
-      super vm, ns, name
+    def initialize ns, stream, printer=nil, name=nil, manifest=[]
+      super ns, name
       @printer    = printer
-      @parser     = Nydp.new_parser(ns)
+      @parser     = Nydp.new_parser
       @tokens     = Nydp.new_tokeniser stream
+      @manifest   = manifest
     end
 
     def print val
-      @printer.puts val.inspect if @printer
+      @printer.puts val._nydp_inspect if @printer
     end
 
     def run
       Nydp.apply_function ns, :"script-run", :"script-start", name
-      res = Nydp::NIL
+      res = nil
       begin
         while !@tokens.finished
           expr = @parser.expression(@tokens)
-          print(res = evaluate(expr)) unless expr.nil?
+          print(res = evaluate(expr, @manifest)) unless expr.nil?
         end
       ensure
         Nydp.apply_function ns, :"script-run", :"script-end", name