packr 1.0.2 → 3.1.0

data/lib/packr/parser.rb

@@ -0,0 +1,21 @@
+class Packr
+  class Parser < RegexpGroup
+    
+    def put(expression, replacement)
+      expression = DICTIONARY.exec(expression) if expression.is_a?(String)
+      super(expression, replacement)
+    end
+    
+    # STRING1 requires backslashes to fix concat bug
+    DICTIONARY = RegexpGroup.new.
+      put(:OPERATOR,    /return|typeof|[\[(\^=,{}:;&|!*?]/.source).
+      put(:CONDITIONAL, /\/\*@\w*|\w*@\*\/|\/\/@\w*|@\w+/.source).
+      put(:COMMENT1,    /\/\/[^\n]*/.source).
+      put(:COMMENT2,    /\/\*[^*]*\*+([^\/][^*]*\*+)*\//.source).
+      put(:REGEXP,      /\/(\\[\/\\]|[^*\/])(\\.|[^\/\n\\])*\/[gim]*/.source).
+      put(:STRING1,     /\'(\\.|[^\'\\])*\'/.source).
+      put(:STRING2,     /"(\\.|[^"\\])*"/.source)
+    
+  end
+end
+

data/lib/packr/privates.rb

@@ -0,0 +1,19 @@
+class Packr
+  class Privates < Encoder
+    
+    IGNORE = {
+      :CONDITIONAL => Packr::IGNORE,
+      "(OPERATOR)(REGXEP)" => Packr::IGNORE
+    }
+    
+    PATTERN = /\b_[\da-zA-Z$][\w$]*\b/
+    
+    def initialize
+      super(PATTERN, lambda { |index|
+        "_" + Packr.encode62(index)
+      }, IGNORE)
+    end
+    
+  end
+end
+

data/lib/packr/regexp_group.rb

@@ -1,122 +1,122 @@
-class Packr
-  class RegexpGroup
-    
-    attr_accessor :values
-    
-    IGNORE = "\\0"
-    BACK_REF = /\\(\d+)/
-    ESCAPE_CHARS = /\\./
-    ESCAPE_BRACKETS = /\(\?[:=!]|\[[^\]]+\]/
-    BRACKETS = /\(/
-    KEYS = "~"
-    
-    def initialize(values, flags = nil)
-      @values = []
-      values.each { |key, value| @values << Item.new(key, value) }
-      if flags && flags.is_a(String)
-        @ignore_case = !!(flags =~ /i/)
-      end
-    end
-    
-    def union(*args)
-      values = {}
-      @values.each { |item| values[item.expression] = item.replacement }
-      args.each do |arg|
-        arg.values.each { |item| values[item.expression] = item.replacement }
-      end
-      self.class.new(values)
-    end
-    
-    def exec(string, &replacement)
-      string = string.to_s
-      regexp = value_of
-      
-      replacement ||= lambda do |match|
-        return "" if match.nil?
-        arguments = [match] + $~.captures + [$~.begin(0), string]
-        offset, result = 1, ""
-        @values.each do |item|
-          nxt = offset + item.length + 1
-          if arguments[offset] # do we have a result?
-            rep = item.replacement
-            if rep.is_a?(Proc)
-              args = arguments[offset...nxt]
-              index = arguments[-2]
-              result = rep.call *(args + [index, string])
-            else
-              result = rep.is_a?(Numeric) ? arguments[offset + rep] : rep.to_s
-            end
-          end
-          offset = nxt
-        end
-        result
-      end
-      
-      replacement.is_a?(Proc) ? string.gsub(regexp, &replacement) :
-          string.gsub(regexp, replacement.to_s)
-    end
-    
-    def test(string)
-      exec(string) != string
-    end
-    
-    def to_s
-      length = 0
-      "(" + @values.map { |item|
-        # Fix back references.
-        ref = item.to_s.gsub(BACK_REF) { |m| "\\" + (1 + $1.to_i + length).to_s }
-        length += item.length + 1
-        ref
-      }.join(")|(") + ")"
-    end
-    
-    def value_of(type = nil)
-      return self if type == Object
-      flag = @ignore_case ? Regexp::IGNORECASE : nil
-      Regexp.new(self.to_s, flag)
-    end
-    
-    class Item
-      attr_accessor :expression, :length, :replacement
-      
-      def initialize(expression, replacement)
-        @expression = expression.is_a?(Regexp) ? expression.source : expression.to_s
-        
-        if replacement.is_a?(Numeric)
-          replacement = "\\" + replacement.to_s
-        elsif replacement.nil?
-          replacement = ""
-        end
-        
-        # does the pattern use sub-expressions?
-        if replacement.is_a?(String) and replacement =~ /\\(\d+)/
-          # a simple lookup? (e.g. "\2")
-          if replacement.gsub(/\n/, " ") =~ /^\\\d+$/
-            # store the index (used for fast retrieval of matched strings)
-            replacement = replacement[1..-1].to_i
-          else # a complicated lookup (e.g. "Hello \2 \1")
-            # build a function to do the lookup
-            q = (replacement.gsub(/\\./, "") =~ /'/) ? '"' : "'"
-            replacement = replacement.gsub(/\r/, "\\r").gsub(/\\(\d+)/,
-                q + "+(args[\\1]||" + q+q + ")+" + q)
-            replacement_string = q + replacement.gsub(/(['"])\1\+(.*)\+\1\1$/, '\1') + q
-            replacement = lambda { |*args| eval(replacement_string) }
-          end
-        end
-        
-        @length = RegexpGroup.count(@expression)
-        @replacement = replacement
-      end
-      
-      def to_s
-        @expression
-      end
-    end
-    
-    def self.count(expression)
-      expression = expression.to_s.gsub(ESCAPE_CHARS, "").gsub(ESCAPE_BRACKETS, "")
-      expression.scan(BRACKETS).length
-    end
-    
-  end
-end
+class Packr
+  class RegexpGroup < Collection
+    
+    IGNORE          = "\\0"
+    BACK_REF        = /\\(\d+)/
+    ESCAPE_CHARS    = /\\./
+    ESCAPE_BRACKETS = /\(\?[:=!]|\[[^\]]+\]/
+    BRACKETS        = /\(/
+    LOOKUP          = /\\(\d+)/
+    LOOKUP_SIMPLE   = /^\\\d+$/
+    
+    def initialize(values = nil, ignore_case = false)
+      super(values)
+      @ignore_case = !!ignore_case
+    end
+    
+    def exec(string, override = nil)
+      string = string.to_s # type-safe
+      return string if @keys.empty?
+      override = 0 if override == IGNORE
+      string.gsub(Regexp.new(self.to_s, @ignore_case && Regexp::IGNORECASE)) do |match|
+        offset, i, result = 1, 0, match
+        arguments = [match] + $~.captures + [$~.begin(0), string]
+        # Loop through the items.
+        each do |item, key|
+          nxt = offset + item.length + 1
+          if arguments[offset] # do we have a result?
+            replacement = override.nil? ? item.replacement : override
+            case replacement
+            when Proc
+              result = replacement.call(*arguments[offset...nxt])
+            when Numeric
+              result = arguments[offset + replacement]
+            else
+              result = replacement
+            end
+          end
+          offset = nxt
+        end
+        result
+      end
+    end
+    
+    def insert_at(index, expression, replacement)
+      expression = expression.is_a?(Regexp) ? expression.source : expression.to_s
+      super(index, expression, replacement)
+    end
+    
+    def test(string)
+      exec(string) != string
+    end
+    
+    def to_s
+      offset = 1
+      "(" + map { |item, key|
+        # Fix back references.
+        expression = item.to_s.gsub(BACK_REF) { |m| "\\" + (offset + $1.to_i) }
+        offset += item.length + 1
+        expression
+      }.join(")|(") + ")"
+    end
+    
+    class Item
+      attr_accessor :expression, :length, :replacement
+      
+      def initialize(expression, replacement = nil)
+        @expression = expression
+        
+        if replacement.nil?
+          replacement = IGNORE
+        elsif replacement.respond_to?(:replacement)
+          replacement = replacement.replacement
+        elsif !replacement.is_a?(Proc)
+          replacement = replacement.to_s
+        end
+        
+        # does the pattern use sub-expressions?
+        if replacement.is_a?(String) and replacement =~ LOOKUP
+          # a simple lookup? (e.g. "\2")
+          if replacement.gsub(/\n/, " ") =~ LOOKUP_SIMPLE
+            # store the index (used for fast retrieval of matched strings)
+            replacement = replacement[1..-1].to_i
+          else # a complicated lookup (e.g. "Hello \2 \1")
+            # build a function to do the lookup
+            # Improved version by Alexei Gorkov:
+            q = '"'
+            replacement_string = replacement.
+                gsub(/\\/, "\\\\").
+                gsub(/"/, "\\x22").
+                gsub(/\n/, "\\n").
+                gsub(/\r/, "\\r").
+                gsub(/\\(\d+)/, q + "+(args[\\1]||" + q+q + ")+" + q).
+                gsub(/(['"])\1\+(.*)\+\1\1$/, '\1')
+            replacement = lambda { |*args| eval(q + replacement_string + q) }
+            
+            # My old crappy version:
+            # q = (replacement.gsub(/\\./, "") =~ /'/) ? '"' : "'"
+            # replacement = replacement.gsub(/\r/, "\\r").gsub(/\\(\d+)/,
+            #     q + "+(args[\\1]||" + q+q + ")+" + q)
+            # replacement_string = q + replacement.gsub(/(['"])\1\+(.*)\+\1\1$/, '\1') + q
+            # replacement = lambda { |*args| eval(replacement_string) }
+          end
+        end
+        
+        @length = RegexpGroup.count(@expression)
+        @replacement = replacement
+      end
+      
+      def to_s
+        @expression.respond_to?(:source) ? @expression.source : @expression.to_s
+      end
+    end
+    
+    def self.count(expression)
+      # Count the number of sub-expressions in a Regexp/RegexpGroup::Item.
+      expression = expression.to_s.gsub(ESCAPE_CHARS, "").gsub(ESCAPE_BRACKETS, "")
+      expression.scan(BRACKETS).length
+    end
+    
+  end
+end
+

data/lib/packr/shrinker.rb

@@ -0,0 +1,123 @@
+class Packr
+  class Shrinker
+    
+    ENCODED_DATA = /~\^(\d+)\^~/
+    PREFIX = '@'
+    SHRUNK = /\@\d+\b/
+    
+    def decode_data(script)
+      # put strings and regular expressions back
+      script.gsub(ENCODED_DATA) { |match| @strings[$1.to_i] }
+    end
+    
+    def encode_data(script)
+      # encode strings and regular expressions
+      @strings = [] # encoded strings and regular expressions
+      DATA.exec(script, lambda { |match, *args|
+        operator, regexp = args[0].to_s, args[1].to_s
+        replacement = "~^#{@strings.length}^~"
+        unless regexp.empty?
+          replacement = operator + replacement
+          match = regexp
+        end
+        @strings << match
+        replacement
+      })
+    end
+    
+    def shrink(script, protected_names = [])
+      script = encode_data(script)
+      protected_names ||= []
+      protected_names = protected_names.map { |s| s.to_s }
+      
+      # identify blocks, particularly identify function blocks (which define scope)
+      __block         = /((catch|do|if|while|with|function)\b[^~{};]*(\(\s*[^{};]*\s*\))\s*)?(\{[^{}]*\})/
+      __brackets      = /\{[^{}]*\}|\[[^\[\]]*\]|\([^\(\)]*\)|~[^~]+~/
+      __encoded_block = /~#?(\d+)~/
+      __identifier    = /[a-zA-Z_$][\w\$]*/
+      __scoped        = /~#(\d+)~/
+      __var           = /\bvar\b/
+      __vars          = /\bvar\s+[\w$]+[^;#]*|\bfunction\s+[\w$]+/
+      __var_tidy      = /\b(var|function)\b|\sin\s+[^;]+/
+      __var_equal     = /\s*=[^,;]*/
+      
+      blocks = [] # store program blocks (anything between braces {})
+      total = 0
+      # decoder for program blocks
+      decode_blocks = lambda do |script, encoded|
+        script = script.gsub(encoded) { |match| blocks[$1.to_i] } while script =~ encoded
+        script
+      end
+      
+      # encoder for program blocks
+      encode_blocks = lambda do |match|
+        prefix, block_type, args, block = $1 || "", $2, $3, $4
+        if block_type == 'function'
+          # decode the function block (THIS IS THE IMPORTANT BIT)
+          # We are retrieving all sub-blocks and will re-parse them in light
+          # of newly shrunk variables
+          block = args + decode_blocks.call(block, __scoped)
+          prefix = prefix.gsub(__brackets, "")
+          
+          # create the list of variable and argument names
+          args = args[1...-1]
+          
+          if args != '_no_shrink_'
+            vars = block.scan(__vars).join(";").gsub(__var, ";var")
+            vars = vars.gsub(__brackets, "") while vars =~ __brackets
+            vars = vars.gsub(__var_tidy, "").gsub(__var_equal, "")
+          end
+          block = decode_blocks.call(block, __encoded_block)
+          
+          # process each identifier
+          if args != '_no_shrink_'
+            count, short_id = 0, nil
+            ids = [args, vars].join(",").scan(__identifier)
+            processed = {}
+            ids.each do |id|
+              if !processed['#' + id] and !protected_names.include?(id)
+                processed['#' + id] = true
+                id = id.rescape
+                # encode variable names
+                count += 1 while block =~ Regexp.new("#{PREFIX}#{count}\\b")
+                reg = Regexp.new("([^\\w$.])#{id}([^\\w$:])")
+                block = block.gsub(reg, "\\1#{PREFIX}#{count}\\2") while block =~ reg
+                reg = Regexp.new("([^{,\\w$.])#{id}:")
+                block = block.gsub(reg, "\\1#{PREFIX}#{count}:")
+                count += 1
+              end
+            end
+            total = [total, count].max
+          end
+          replacement = "#{prefix}~#{blocks.length}~"
+          blocks << block
+        else
+          replacement = "~##{blocks.length}~"
+          blocks << (prefix + block)
+        end
+        replacement
+      end
+      
+      # encode blocks, as we encode we replace variable and argument names
+      script = script.gsub(__block, &encode_blocks) while script =~ __block
+      
+      # put the blocks back
+      script = decode_blocks.call(script, __encoded_block)
+      
+      short_id, count = nil, 0
+      shrunk = Encoder.new(SHRUNK, lambda { |object|
+        # find the next free short name
+        begin
+          short_id = Packr.encode52(count)
+          count += 1
+        end while script =~ Regexp.new("[^\\w$.]#{short_id}[^\\w$:]")
+        short_id
+      })
+      script = shrunk.encode(script)
+      
+      decode_data(script)
+    end
+    
+  end
+end
+

data/lib/packr/words.rb

@@ -1,84 +1,39 @@
-class Packr
-  class Words
-    
-    WORDS = /\w+/
-    attr_accessor :words
-    
-    def initialize(script)
-      script.to_s.scan(WORDS).each { |word| add(word) }
-      encode!
-    end
-    
-    def add(word)
-      @words ||= []
-      @words << (stored_word = Item.new(word)) unless stored_word = get(word)
-      word = stored_word
-      word.count = word.count + 1
-      word
-    end
-    
-    def get(word)
-      @words.find { |w| w.word == word.to_s }
-    end
-    
-    def has?(word)
-      !!(get word)
-    end
-    
-    def size
-      @words.size
-    end
-    
-    def to_s
-      @words.join("|")
-    end
-    
-  private
-    
-    def encode!
-      # sort by frequency
-      @words = @words.sort_by { |word| word.count }.reverse
-      
-      a = 62
-      e = lambda do |c|
-        (c < a ? '' : e.call((c.to_f / a).to_i) ) +
-            ((c = c % a) > 35 ? (c+29).chr : c.to_s(36))
-      end
-      
-      # a dictionary of base62 -> base10
-      encoded = (0...(@words.size)).map { |i| e.call(i) }
-      
-      index = 0
-      @words.each do |word|
-        if x = encoded.index(word.word)
-          word.index = x
-          def word.to_s; ""; end
-        else
-          index += 1 while has?(e.call(index))
-          word.index = index
-          index += 1
-        end
-        word.encoded = e.call(word.index)
-      end
-      
-      # sort by encoding
-      @words = @words.sort_by { |word| word.index }
-    end
-    
-    class Item
-      attr_accessor :word, :count, :encoded, :index
-      
-      def initialize(word)
-        @word = word
-        @count = 0
-        @encoded = ""
-        @index = -1
-      end
-      
-      def to_s
-        @word
-      end
-    end
-    
-  end
-end
+class Packr
+  class Words < Collection
+    
+    def add(word)
+      super unless has?(word)
+      word = get(word)
+      word.index = size if word.index.zero?
+      word.count = word.count + 1
+      word
+    end
+    
+    def sort!(&sorter)
+      return super if block_given?
+      super do |word1, word2|
+        # sort by frequency
+        count = word2.count - word1.count
+        index = word1.index - word2.index
+        count.nonzero? ? count : (index.nonzero? ? index : 0)
+      end
+    end
+    
+    class Item
+      attr_accessor :index, :count, :encoded, :replacement
+      
+      def initialize(word, item)
+        @word = word
+        @index = 0
+        @count = 0
+        @encoded = ""
+      end
+      
+      def to_s
+        @word
+      end
+    end
+    
+  end
+end
+