rubinius-compiler 1.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 4f7a4f90b65396c5b6a411d0dba6d2c27fac75c0
+  data.tar.gz: 31e27d1a8b667740b398c6456b37dd1e8e3b694f
+SHA512:
+  metadata.gz: e0278293e994de4801017219e759a19763e45454775231657a7df08ca28498fe56cece97f306f011138954041e590512fc8d85b18914eac9c79f8555748f7c9e
+  data.tar.gz: 0ed9518ee5010a0834630b6fb91b85b013515ce1eb2ded814962492aceb24533640aba5788f341b3f35b23eacaf538ab433d2c207dc6f17a55f84a4e6d93cbfc

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in rubinius-compiler.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,25 @@
+Copyright (c) 2013, Brian Shirai
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+3. Neither the name of the library nor the names of its contributors may be
+   used to endorse or promote products derived from this software without
+   specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/README.md

@@ -0,0 +1,29 @@
+# Rubinius::Compiler
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'rubinius-compiler'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install rubinius-compiler
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/lib/rubinius/compiler.rb

@@ -0,0 +1,11 @@
+require "rubinius/compiler/compiler"
+require "rubinius/compiler/stages"
+require "rubinius/compiler/locals"
+require "rubinius/compiler/generator_methods"
+require "rubinius/compiler/generator"
+require "rubinius/compiler/iseq"
+require "rubinius/compiler/opcodes"
+require "rubinius/compiler/compiled_file"
+require "rubinius/compiler/evaluator"
+require "rubinius/compiler/printers"
+require "rubinius/compiler/runtime"

data/lib/rubinius/compiler/compiled_file.rb

@@ -0,0 +1,338 @@
+# -*- encoding: us-ascii -*-
+
+module Rubinius::ToolSet.current::TS
+  ##
+  # A decode for the .rbc file format.
+
+  class CompiledFile
+    ##
+    # Create a CompiledFile with +magic+ bytes, +signature+, and +version+.
+    # The optional +stream+ is used to lazy load the body.
+
+    def initialize(magic, signature, version, stream=nil)
+      @magic = magic
+      @signature = signature
+      @version = version
+      @stream = stream
+      @data = nil
+    end
+
+    attr_reader :magic
+    attr_reader :signature
+    attr_reader :version
+    attr_reader :stream
+
+    ##
+    # From a stream-like object +stream+ load the data in and return a
+    # CompiledFile object.
+
+    def self.load(stream)
+      raise IOError, "attempted to load nil stream" unless stream
+
+      magic = stream.gets.strip
+      signature = Integer(stream.gets.strip)
+      version = Integer(stream.gets.strip)
+
+      return new(magic, signature, version, stream)
+    end
+
+    ##
+    # Writes the CompiledFile +code+ to +file+.
+    def self.dump(code, file, signature, version)
+      File.open(file, "wb") do |f|
+        new("!RBIX", signature, version).encode_to(f, code)
+      end
+    rescue SystemCallError
+      # just skip writing the compiled file if we don't have permissions
+    end
+
+    ##
+    # Encode the contets of this CompiledFile object to +stream+ with
+    # a body of +body+. Body use marshalled using CompiledFile::Marshal
+
+    def encode_to(stream, body)
+      stream.puts @magic
+      stream.puts @signature.to_s
+      stream.puts @version.to_s
+
+      mar = CompiledFile::Marshal.new
+      stream << mar.marshal(body)
+    end
+
+    ##
+    # Return the body object by unmarshaling the data
+
+    def body
+      return @data if @data
+
+      mar = CompiledFile::Marshal.new
+      @data = mar.unmarshal(stream)
+    end
+
+    ##
+    # A class used to convert an CompiledCode to and from
+    # a String.
+
+    class Marshal
+
+      ##
+      # Read all data from +stream+ and invoke unmarshal_data
+
+      def unmarshal(stream)
+        if stream.kind_of? String
+          str = stream
+        else
+          str = stream.read
+        end
+
+        @start = 0
+        @size = str.size
+        @data = str.data
+
+        unmarshal_data
+      end
+
+      ##
+      # Process a stream object +stream+ as as marshalled data and
+      # return an object representation of it.
+
+      def unmarshal_data
+        kind = next_type
+        case kind
+        when 116 # ?t
+          return true
+        when 102 # ?f
+          return false
+        when 110 # ?n
+          return nil
+        when 73  # ?I
+          return next_string.to_i(16)
+        when 100 # ?d
+          str = next_string.chop
+
+          # handle the special NaN, Infinity and -Infinity differently
+          if str[0] == ?\     # leading space
+            x = str.to_f
+            e = str[-5..-1].to_i
+
+            # This is necessary because (2**1024).to_f yields Infinity
+            if e == 1024
+              return x * 2 ** 512 * 2 ** 512
+            else
+              return x * 2 ** e
+            end
+          else
+            case str.downcase
+            when "infinity"
+              return 1.0 / 0.0
+            when "-infinity"
+              return -1.0 / 0.0
+            when "nan"
+              return 0.0 / 0.0
+            else
+              raise TypeError, "Invalid Float format: #{str}"
+            end
+          end
+        when 115 # ?s
+          enc = unmarshal_data
+          count = next_string.to_i
+          str = next_bytes count
+          str.force_encoding enc if enc and defined?(Encoding)
+          return str
+        when 120 # ?x
+          enc = unmarshal_data
+          count = next_string.to_i
+          str = next_bytes count
+          str.force_encoding enc if enc and defined?(Encoding)
+          return str.to_sym
+        when 99  # ?c
+          count = next_string.to_i
+          str = next_bytes count
+          return str.split("::").inject(Object) { |a,n| a.const_get(n) }
+        when 112 # ?p
+          count = next_string.to_i
+          obj = Rubinius::Tuple.new(count)
+          i = 0
+          while i < count
+            obj[i] = unmarshal_data
+            i += 1
+          end
+          return obj
+        when 105 # ?i
+          count = next_string.to_i
+          seq = Rubinius::InstructionSequence.new(count)
+          i = 0
+          while i < count
+            seq[i] = next_string.to_i
+            i += 1
+          end
+          return seq
+        when 69  # ?E
+          count = next_string.to_i
+          name = next_bytes count
+          return Encoding.find(name) if defined?(Encoding)
+        when 77  # ?M
+          version = next_string.to_i
+          if version != 1
+            raise "Unknown CompiledCode version #{version}"
+          end
+          code = Rubinius::CompiledCode.new
+          code.metadata      = unmarshal_data
+          code.primitive     = unmarshal_data
+          code.name          = unmarshal_data
+          code.iseq          = unmarshal_data
+          code.stack_size    = unmarshal_data
+          code.local_count   = unmarshal_data
+          code.required_args = unmarshal_data
+          code.post_args     = unmarshal_data
+          code.total_args    = unmarshal_data
+          code.splat         = unmarshal_data
+          code.literals      = unmarshal_data
+          code.lines         = unmarshal_data
+          code.file          = unmarshal_data
+          code.local_names   = unmarshal_data
+          return code
+        else
+          raise "Unknown type '#{kind.chr}'"
+        end
+      end
+
+      private :unmarshal_data
+
+      ##
+      # Returns the next character in _@data_ as a Fixnum.
+      #--
+      # The current format uses a one-character type indicator
+      # followed by a newline. If that format changes, this
+      # will break and we'll fix it.
+      #++
+      def next_type
+        chr = @data[@start]
+        @start += 2
+        chr
+      end
+
+      private :next_type
+
+      ##
+      # Returns the next string in _@data_ including the trailing
+      # "\n" character.
+      def next_string
+        count = @data.locate "\n", @start, @size
+        count = @size unless count
+        str = String.from_bytearray @data, @start, count - @start
+        @start = count
+        str
+      end
+
+      private :next_string
+
+      ##
+      # Returns the next _count_ bytes in _@data_, skipping the
+      # trailing "\n" character.
+      def next_bytes(count)
+        str = String.from_bytearray @data, @start, count
+        @start += count + 1
+        str
+      end
+
+      private :next_bytes
+
+      ##
+      # For object +val+, return a String represetation.
+
+      def marshal(val)
+        case val
+        when TrueClass
+          "t\n"
+        when FalseClass
+          "f\n"
+        when NilClass
+          "n\n"
+        when Fixnum, Bignum
+          "I\n#{val.to_s(16)}\n"
+        when String
+          if defined?(Encoding)
+            # We manually construct the Encoding data to avoid recursion
+            # marshaling an Encoding name as a String.
+            name = val.encoding.name
+            enc_name = "E\n#{name.bytesize}\n#{name}\n"
+          else
+            # The kernel code is all US-ASCII. When building melbourne for 1.8
+            # Ruby, we fake a bunch of encoding stuff so force US-ASCII here.
+            enc_name = "E\n8\nUS-ASCII\n"
+          end
+
+          "s\n#{enc_name}#{val.bytesize}\n#{val}\n"
+        when Symbol
+          s = val.to_s
+          if defined?(Encoding)
+            # We manually construct the Encoding data to avoid recursion
+            # marshaling an Encoding name as a String.
+            name = s.encoding.name
+            enc_name = "E\n#{name.bytesize}\n#{name}\n"
+          else
+            # The kernel code is all US-ASCII. When building melbourne for 1.8
+            # Ruby, we fake a bunch of encoding stuff so force US-ASCII here.
+            enc_name = "E\n8\nUS-ASCII\n"
+          end
+
+          "x\n#{enc_name}#{s.bytesize}\n#{s}\n"
+        when Rubinius::Tuple
+          str = "p\n#{val.size}\n"
+          val.each do |ele|
+            data = marshal(ele)
+            data.force_encoding(str.encoding) if defined?(Encoding)
+            str.append data
+          end
+          str
+        when Float
+          str = "d\n"
+          if val.infinite?
+            str.append "-" if val < 0.0
+            str.append "Infinity"
+          elsif val.nan?
+            str.append "NaN"
+          else
+            str.append " %+.54f %5d" % Math.frexp(val)
+          end
+          str.append "\n"
+        when Rubinius::InstructionSequence
+          str = "i\n#{val.size}\n"
+          val.opcodes.each do |op|
+            unless op.kind_of?(Fixnum)
+              raise TypeError, "InstructionSequence contains non Fixnum: #{op.inspect}"
+            end
+            str.append "#{op}\n"
+          end
+          str
+        when Rubinius::CompiledCode
+          str = "M\n1\n"
+          str.append marshal(val.metadata)
+          str.append marshal(val.primitive)
+          str.append marshal(val.name)
+          str.append marshal(val.iseq)
+          str.append marshal(val.stack_size)
+          str.append marshal(val.local_count)
+          str.append marshal(val.required_args)
+          str.append marshal(val.post_args)
+          str.append marshal(val.total_args)
+          str.append marshal(val.splat)
+          str.append marshal(val.literals)
+          str.append marshal(val.lines)
+          str.append marshal(val.file)
+          str.append marshal(val.local_names)
+          str
+        else
+          if val.respond_to? :rbx_marshal_constant
+            name = StringValue(val.rbx_marshal_constant)
+            "c\n#{name.size}\n#{name}\n"
+          else
+            raise ArgumentError, "Unknown type #{val.class}: #{val.inspect}"
+          end
+        end
+      end
+    end
+  end
+end
+

data/lib/rubinius/compiler/compiler.rb

@@ -0,0 +1,377 @@
+# -*- encoding: us-ascii -*-
+
+module Rubinius::ToolSet.current::TS
+
+  class CompileError < RuntimeError
+  end
+
+  class Compiler
+    attr_accessor :parser, :generator, :encoder, :packager, :writer
+
+    def self.compiler_error(msg, orig)
+      if defined?(RUBY_ENGINE) and RUBY_ENGINE == "rbx"
+        raise CompileError, msg, orig
+      else
+        orig.message.replace("#{orig.message} - #{msg}")
+        raise orig
+      end
+    end
+
+    if RBC_DB = Rubinius::Config['rbc.db']
+      def self.compiled_name(file)
+        full = "#{File.expand_path(file)}#{Rubinius::RUBY_LIB_VERSION}"
+        hash = Rubinius.invoke_primitive :sha1_hash, full
+        dir = hash[0,2]
+
+        "#{RBC_DB}/#{dir}/#{hash}"
+      end
+    else
+      def self.compiled_cache_writable?(db, dir)
+        File.owned?(db) or File.owned?(dir)
+      end
+
+      def self.compiled_name(file)
+        name = File.expand_path file
+
+        # If the file we are compiling is in a subdirectory of the current
+        # working directory when Rubinius is started, try to cache the
+        # compiled code in <cwd>/.rbx if:
+        #
+        #   1. <cwd>/.rbx exists and is owned by the process user
+        #       OR
+        #   2. if <cwd> is owned by the process user
+        #
+        # Otherwise, try to cache the compiled code in ~/.rbx if:
+        #
+        #   1. ~/.rbx exists and is owned by the process user
+        #       OR
+        #   2. ~/ is owned by the process user
+
+        unless @db
+          dir = Rubinius::OS_STARTUP_DIR
+          db = "#{dir}/.rbx"
+          unless name.prefix?(dir) and compiled_cache_writable?(db, dir)
+            # Yes, this retarded shit is necessary because people actually
+            # run under fucked environments with no HOME set.
+            return unless ENV["HOME"]
+
+            dir = File.expand_path "~/"
+            db = "#{dir}/.rbx"
+            return unless compiled_cache_writable?(db, dir)
+          end
+          @db = db
+        end
+
+        full = "#{name}#{Rubinius::RUBY_LIB_VERSION}"
+        hash = Rubinius.invoke_primitive :sha1_hash, full
+
+        "#{@db}/#{hash[0, 2]}/#{hash}"
+      end
+    end
+
+    def self.compile(file, output=nil, line=1, transforms=:default)
+      compiler = new :file, :compiled_file
+
+      parser = compiler.parser
+      parser.root AST::Script
+
+      if transforms.kind_of? Array
+        transforms.each { |t| parser.enable_category t }
+      else
+        parser.enable_category transforms
+      end
+
+      parser.input file, line
+
+      writer = compiler.writer
+      writer.version = Rubinius::RUBY_LIB_VERSION
+      writer.name = output ? output : compiled_name(file)
+
+      begin
+        compiler.run
+      rescue SyntaxError => e
+        raise e
+      rescue Exception => e
+        compiler_error "Error trying to compile #{file}", e
+      end
+
+    end
+
+    # Match old compiler's signature
+    def self.compile_file_old(file, flags=nil)
+      compile_file file, 1
+    end
+
+    def self.compile_file(file, line=1)
+      compiler = new :file, :compiled_code
+
+      parser = compiler.parser
+      parser.root AST::Script
+      parser.default_transforms
+      parser.input file, line
+
+      begin
+        compiler.run
+      rescue Exception => e
+        compiler_error "Error trying to compile #{file}", e
+      end
+    end
+
+    def self.compile_string(string, file="(eval)", line=1)
+      compiler = new :string, :compiled_code
+
+      parser = compiler.parser
+      parser.root AST::Script
+      parser.default_transforms
+      parser.input string, file, line
+
+      compiler.run
+    end
+
+    class LRUCache
+      class Entry
+        attr_reader :hits, :key
+        attr_accessor :value, :next_entry, :prev_entry
+
+        def initialize(key, value)
+          @key = key
+          @value = value
+          @hits = 0
+          @next_entry = nil
+          @prev_entry = nil
+        end
+
+        def insert_after(entry)
+          nxt = entry.next_entry
+
+          @prev_entry = entry
+          @next_entry = nxt
+
+          entry.next_entry = self
+          nxt.prev_entry = self if nxt
+        end
+
+        def insert_before(entry)
+          prev = entry.prev_entry
+
+          @prev_entry = prev
+          @next_entry = entry
+
+          entry.prev_entry = self
+          prev.next_entry = self if prev
+        end
+
+        def detach!
+          @next_entry.prev_entry = @prev_entry if @next_entry
+          @prev_entry.next_entry = @next_entry if @prev_entry
+
+          @next_entry = nil
+          @prev_entry = nil
+        end
+
+        def become_first!
+          @prev_entry = nil
+        end
+
+        def inc!
+          @hits += 1
+        end
+      end
+
+      def initialize(total)
+        @cache = {}
+        @total = total
+        @current = 0
+
+        @head = Entry.new(nil, nil)
+        @tail = Entry.new(nil, nil)
+
+        @tail.insert_after(@head)
+
+        @misses = 0
+      end
+
+      attr_reader :current, :misses
+
+      def clear!
+        Rubinius.synchronize(self) do
+          @cache = {}
+          @current = 0
+
+          @head = Entry.new(nil, nil, -1)
+          @tail = Entry.new(nil, nil, -2)
+
+          @tail.insert_after(@head)
+        end
+      end
+
+      def explain
+        entry = @head.next_entry
+        while entry != @tail
+          str, layout = entry.key
+          puts "hits: #{entry.hits}"
+          puts "layout: #{layout.inspect}"
+          puts "<STRING>"
+          puts str
+          puts "</STRING>"
+
+          entry = entry.next_entry
+        end
+      end
+
+      def retrieve(key)
+        Rubinius.synchronize(self) do
+          if entry = @cache[key]
+            entry.inc!
+
+            entry.detach!
+            entry.insert_before @tail
+
+            return entry.value
+          end
+
+          @misses += 1
+
+          nil
+        end
+      end
+
+      def set(key, value)
+        Rubinius.synchronize(self) do
+          if entry = @cache[key]
+            entry.value = value
+
+            entry.detach!
+            entry.insert_before @tail
+
+            return value
+          end
+
+          if @current == @total
+            entry = @head.next_entry
+
+            entry.detach!
+
+            @cache.delete entry.key
+          else
+            @current += 1
+          end
+
+          entry = Entry.new(key, value)
+
+          entry.insert_before @tail
+
+          @cache[key] = entry
+        end
+      end
+    end
+
+    total = Rubinius::Config['eval.cache']
+
+    case total
+    when Fixnum
+      if total == 0
+        @eval_cache = nil
+      else
+        @eval_cache = LRUCache.new(total)
+      end
+    when false
+      @eval_cache = nil
+    else
+      @eval_cache = LRUCache.new(50)
+    end
+
+    def self.eval_cache
+      @eval_cache
+    end
+
+    def self.compile_eval(string, variable_scope, file="(eval)", line=1)
+      if ec = @eval_cache
+        layout = variable_scope.local_layout
+        if code = ec.retrieve([string, layout, line])
+          return code
+        end
+      end
+
+      compiler = new :eval, :compiled_code
+
+      parser = compiler.parser
+      parser.root AST::EvalExpression
+      parser.default_transforms
+      parser.input string, file, line
+
+      compiler.generator.variable_scope = variable_scope
+
+      code = compiler.run
+
+      code.add_metadata :for_eval, true
+
+      if ec and parser.should_cache?
+        ec.set([string.dup, layout, line], code)
+      end
+
+      return code
+    end
+
+    def self.construct_block(string, binding, file="(eval)", line=1)
+      code = compile_eval string, binding.variables, file, line
+
+      code.scope = binding.constant_scope
+      code.name = binding.variables.method.name
+
+      # This has to be setup so __FILE__ works in eval.
+      script = Rubinius::CompiledCode::Script.new(code, file, true)
+      script.eval_source = string
+
+      code.scope.script = script
+
+      be = Rubinius::BlockEnvironment.new
+      be.under_context binding.variables, code
+
+      # Pass the BlockEnvironment this binding was created from
+      # down into the new BlockEnvironment we just created.
+      # This indicates the "declaration trace" to the stack trace
+      # mechanisms, which can be different from the "call trace"
+      # in the case of, say: eval("caller", a_proc_instance)
+      if binding.from_proc?
+        be.proc_environment = binding.proc_environment
+      end
+
+      return be
+    end
+
+    def self.compile_test_bytecode(string, transforms)
+      compiler = new :string, :bytecode
+
+      parser = compiler.parser
+      parser.root AST::Snippet
+      parser.input string
+      transforms.each { |x| parser.enable_transform x }
+
+      compiler.generator.processor Rubinius::TestGenerator
+
+      compiler.run
+    end
+
+    def self.compile_test_bytecode_19(string, transforms)
+      compiler = new :string, :bytecode
+
+      parser = compiler.parser
+      parser.root AST::Snippet
+      parser.input string
+      transforms.each { |x| parser.enable_transform x }
+
+      compiler.generator.processor TestGenerator
+
+      compiler.run
+    end
+
+    def initialize(from, to)
+      @start = Stages[from].new self, to
+    end
+
+    def run
+      @start.run
+    end
+  end
+end