packr 1.0.2 → 3.1.0

data/History.txt

@@ -0,0 +1,16 @@
+=== 3.1.0 / 2009-02-22
+
+* Project is now a gem, not a Rails plugin
+* Improved local variable compression
+* Private field obfuscation
+* Changed variable protection, protected variables must be supplied
+  as options to Packr#pack.
+
+=== 1.0.2 / 2007-12-13
+
+* Added variable protection, protecting $super for use with Prototype
+
+=== 1.0.0 / 2007-12-04
+
+* Initial release, compatible with Packer 3.0
+

data/Manifest.txt

@@ -0,0 +1,19 @@
+History.txt
+Manifest.txt
+README.txt
+Rakefile
+bin/packr
+lib/packr.rb
+lib/string.rb
+lib/packr/map.rb
+lib/packr/collection.rb
+lib/packr/regexp_group.rb
+lib/packr/constants.rb
+lib/packr/encoder.rb
+lib/packr/minifier.rb
+lib/packr/parser.rb
+lib/packr/privates.rb
+lib/packr/shrinker.rb
+lib/packr/words.rb
+lib/packr/base62.rb
+test/test_packr.rb

data/README.txt

@@ -0,0 +1,112 @@
+= PackR
+
+* http://github.com/jcoglan/packr
+* http://dean.edwards.name/packer/
+* http://base2.googlecode.com
+
+== Description
+
+PackR is a Ruby version of Dean Edwards' JavaScript compressor.
+
+== Features
+
+* Whitespace and comment removal
+* Compression of local variable names
+* Compression and obfuscation of 'private' (_underscored) identifiers
+* Base-62 encoding
+
+== Synopsis
+
+To call from within a Ruby program:
+
+  require 'rubygems'
+  require 'packr'
+  
+  code = File.read('my_script.js')
+  compressed = Packr.pack(code)
+  File.open('my_script.min.js', 'wb') { |f| f.write(compressed) }
+
+This method takes a number of options to control compression, for example:
+
+  compressed = Packr.pack(code, :shrink_vars => true, :base62 => true)
+
+The full list of available options is:
+
+* <tt>:shrink_vars</tt> -- set to +true+ to compress local variable names
+* <tt>:private</tt> -- set to +true+ to obfuscate 'private' identifiers, i.e.
+  names beginning with a single underscore
+* <tt>:base62</tt> -- encode the program using base 62
+* <tt>:protect</tt> -- an array of variable names to protect from compression, e.g.
+
+  compressed = Packr.pack(code, :shrink_vars => true,
+                                :protect => %w[$super self])
+
+To call from the command line (use <tt>packr --help</tt> to see available options):
+
+  packr my_script.js > my_script.min.js
+  
+== Notes
+
+This program is not a JavaScript parser, and rewrites your files using regular
+expressions. Be sure to include semicolons and braces everywhere they are required
+so that your program will work correctly when packed down to a single line.
+
+By far the most efficient way to serve JavaScript over the web is to use PackR
+with the --shrink-vars flag, combined with gzip compression. If you don't have access
+to your server config to set up mod_deflate, you can generate gzip files using
+(on Unix-like systems):
+
+  packr -s my-file.js | gzip > my-file.js.gz
+  
+You can then get Apache to serve the files by putting this in your .htaccess file:
+
+  AddEncoding gzip .gz
+  RewriteCond %{HTTP:Accept-encoding} gzip
+  RewriteCond %{HTTP_USER_AGENT} !Safari
+  RewriteCond %{REQUEST_FILENAME}.gz -f
+  RewriteRule ^(.*)$ $1.gz [QSA,L]
+
+If you really cannot serve gzip files, use the --base62 option to further compress
+your code. This mode is at its best when compressing large files with many repeated
+tokens.
+
+The --private option can be used to stop other programs calling private methods
+in your code by renaming anything beginning with a single underscore. Beware that
+you should not use this if the generated file contains 'private' methods that need
+to be accessible by other files. Also know that all the files that access any
+particular private method must be compressed together so they all get the same
+rewritten name for the private method.
+
+== Requirements
+
+* Rubygems
+* Oyster (installed automatically)
+
+== Installation
+
+  sudo gem install packr -y
+
+== License
+
+(The MIT License)
+
+Copyright (c) 2004-2009 Dean Edwards, James Coglan
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,12 @@
+# -*- ruby -*-
+
+require 'rubygems'
+require 'hoe'
+require './lib/packr.rb'
+
+Hoe.new('packr', Packr::VERSION) do |p|
+  p.developer('James Coglan', 'jcoglan@googlemail.com')
+  p.extra_deps = %w(oyster)
+end
+
+# vim: syntax=Ruby

data/bin/packr

@@ -0,0 +1,91 @@
+require 'rubygems'
+require 'oyster'
+require 'packr'
+
+spec = Oyster.spec do
+  name    "packr -- JavaScript code compressor based on Dean Edwards' Packer"
+  
+  synopsis <<-EOS
+  packr [OPTIONS] INPUT_FILES > OUTPUT_FILE
+  cat INPUT_FILES | packr [OPTIONS] > OUTPUT_FILE
+  EOS
+  
+  description <<-EOS
+  PackR is a program for compressing JavaScript programs. It can remove whitespace
+  and comments, compress local variable names, compress/obfuscate private identifiers,
+  and encode the program in base-62.
+  
+  When invoked from the command line, it concatenates all the code in INPUT_FILES (or
+  from standard input) and compresses the code using the given options, printing the
+  result to standard output. You can pipe this output into another file to save it.
+  EOS
+  
+  flag    :'shrink-vars',   :default => true,
+  :desc => 'Shrink local variable names inside functions'
+  
+  flag    :private,         :default => false,
+  :desc => 'Obfuscate private identifiers, i.e. names beginning with a single underscore'
+  
+  flag    :base62,          :default => false,
+  :desc => 'Encode the program using base 62'
+  
+  array   :protect,         :default => [],
+  :desc => 'List of variable names to protect from compression when using --shrink-vars'
+  
+  notes <<-EOS
+  This program is not a JavaScript parser, and rewrites your files using regular
+  expressions. Be sure to include semicolons and braces everywhere they are required
+  so that your program will work correctly when packed down to a single line.
+  
+  By far the most efficient way to serve JavaScript over the web is to use PackR
+  with the --shrink-vars flag, combined with gzip compression. If you don't have access
+  to your server config to set up mod_deflate, you can generate gzip files using
+  (on Unix-like systems):
+  
+    packr -s my-file.js | gzip > my-file.js.gz
+    
+  You can then get Apache to serve the files by putting this in your .htaccess file:
+  
+    AddEncoding gzip .gz
+    RewriteCond %{HTTP:Accept-encoding} gzip
+    RewriteCond %{HTTP_USER_AGENT} !Safari
+    RewriteCond %{REQUEST_FILENAME}.gz -f
+    RewriteRule ^(.*)$ $1.gz [QSA,L]
+  
+  If you really cannot serve gzip files, use the --base62 option to further compress
+  your code. This mode is at its best when compressing large files with many repeated
+  tokens.
+  
+  The --private option can be used to stop other programs calling private methods
+  in your code by renaming anything beginning with a single underscore. Beware that
+  you should not use this if the generated file contains 'private' methods that need
+  to be accessible by other files. Also know that all the files that access any
+  particular private method must be compressed together so they all get the same
+  rewritten name for the private method.
+  EOS
+  
+  author <<-EOS
+  Original JavaScript version by Dean Edwards, Ruby port by James Coglan <jcoglan@googlemail.com>
+  EOS
+  
+  copyright <<-EOS
+  Copyright (c) 2004-2008 Dean Edwards, James Coglan. This program is free software,
+  distributed under the MIT license.
+  EOS
+end
+
+begin; opts = spec.parse
+rescue Oyster::HelpRendered; exit
+end
+
+inputs = opts[:unclaimed]
+code = inputs.empty? ?
+    $stdin.read :
+    inputs.map { |f| File.read(f) }.join("\n")
+
+$stdout.puts Packr.pack(code,
+  :shrink_vars => !!opts[:'shrink-vars'],
+  :protect     => opts[:protect],
+  :private     => !!opts[:private],
+  :base62      => !!opts[:base62])
+

data/lib/packr.rb

@@ -1,240 +1,68 @@
-# PackR -- a Ruby port of Packer by Dean Edwards
-# Packer version 3.0 (final) - copyright 2004-2007, Dean Edwards
-# http://www.opensource.org/licenses/mit-license
-
-require File.dirname(__FILE__) + '/string'
-require File.dirname(__FILE__) + '/packr/regexp_group'
-require File.dirname(__FILE__) + '/packr/words'
-
-class Packr
-  
-  PROTECTED_NAMES = %w($super)
-  
-  class << self
-    def protect_vars(*args)
-      @packr ||= self.new
-      @packr.protect_vars(*args)
-    end
-    
-    def minify(script)
-      @packr ||= self.new
-      @packr.minify(script)
-    end
-    
-    def pack(script, options = {})
-      @packr ||= self.new
-      @packr.pack(script, options)
-    end
-    
-    def pack_file(path, options = {})
-      @packr ||= self.new
-      @packr.pack_file(path, options)
-    end
-  end
-  
-  IGNORE = RegexpGroup::IGNORE
-  REMOVE = ""
-  SPACE = " "
-  WORDS = /\w+/
-  
-  CONTINUE = /\\\r?\n/
-  
-  ENCODE10 = "String"
-  ENCODE36 = "function(c){return c.toString(a)}"
-  ENCODE62 = "function(c){return(c<a?'':e(parseInt(c/a)))+((c=c%a)>35?String.fromCharCode(c+29):c.toString(36))}"
-  
-  UNPACK = lambda do |p,a,c,k,e,r|
-    "eval(function(p,a,c,k,e,r){e=#{e};if(!''.replace(/^/,String)){while(c--)r[#{r}]=k[c]" +
-        "||#{r};k=[function(e){return r[e]}];e=function(){return'\\\\w+'};c=1};while(c--)if(k[c])p=p." +
-        "replace(new RegExp('\\\\b'+e(c)+'\\\\b','g'),k[c]);return p}('#{p}',#{a},#{c},'#{k}'.split('|'),0,{}))"
-  end
-  
-  CLEAN = RegexpGroup.new(
-    "\\(\\s*;\\s*;\\s*\\)" => "(;;)", # for (;;) loops
-    "throw[^};]+[};]" => IGNORE, # a safari 1.3 bug
-    ";+\\s*([};])" => "\\1"
-  )
-  
-  DATA = RegexpGroup.new(
-    # strings
-    "STRING1" => IGNORE,
-    'STRING2' => IGNORE,
-    "CONDITIONAL" => IGNORE, # conditional comments
-    "(COMMENT1)\\n\\s*(REGEXP)?" => "\n\\3",
-    "(COMMENT2)\\s*(REGEXP)?" => " \\3",
-    "([\\[(\\^=,{}:;&|!*?])\\s*(REGEXP)" => "\\1\\2"
-  )
-  
-  JAVASCRIPT = RegexpGroup.new(
-    :COMMENT1 =>    /(\/\/|;;;)[^\n]*/.source,
-    :COMMENT2 =>    /\/\*[^*]*\*+([^\/][^*]*\*+)*\//.source,
-    :CONDITIONAL => /\/\*@|@\*\/|\/\/@[^\n]*\n/.source,
-    :REGEXP =>      /\/(\\[\/\\]|[^*\/])(\\.|[^\/\n\\])*\/[gim]*/.source,
-    :STRING1 =>     /'(\\.|[^'\\])*'/.source,
-    :STRING2 =>     /"(\\.|[^"\\])*"/.source
-  )
-  
-  WHITESPACE = RegexpGroup.new(
-    "(\\d)\\s+(\\.\\s*[a-z\\$_\\[(])" => "\\1 \\2", # http://dean.edwards.name/weblog/2007/04/packer3/#comment84066
-    "([+-])\\s+([+-])" => "\\1 \\2", # c = a++ +b;
-    "\\b\\s+\\$\\s+\\b" => " $ ", # var $ in
-    "\\$\\s+\\b" => "$ ", # object$ in
-    "\\b\\s+\\$" => " $", # return $object
-    "\\b\\s+\\b" => SPACE,
-    "\\s+" => REMOVE
-  )
-  
-  def initialize
-    @data = {}
-    DATA.values.each { |item| @data[JAVASCRIPT.exec(item.expression)] = item.replacement }
-    @data = RegexpGroup.new(@data)
-    @whitespace = @data.union(WHITESPACE)
-    @clean = @data.union(CLEAN)
-    @protected_names = PROTECTED_NAMES
-  end
-  
-  def protect_vars(*args)
-    args = args.map { |arg| arg.to_s.strip }.select { |arg| arg =~ /^[a-z\_\$][a-z0-9\_\$]*$/i }
-    @protected_names = (@protected_names + args).uniq
-  end
-  
-  def minify(script)
-    script = script.gsub(CONTINUE, "")
-    script = @data.exec(script)
-    script = @whitespace.exec(script)
-    script = @clean.exec(script)
-    script
-  end
-  
-  def pack(script, options = {})
-    script = minify(script + "\n")
-    script = shrink_variables(script) if options[:shrink_vars]
-    script = base62_encode(script) if options[:base62]
-    script
-  end
-  
-  def pack_file(path, options = {})
-    path = path.gsub(Regexp.new("^((#{RAILS_ROOT.gsub(/\./, "\\.")})?/)?"), RAILS_ROOT + '/')
-    script = File.read(path)
-    script = pack(script, options)
-    File.open(path, 'wb') { |f| f.write(script) }
-  end
-  
-private
-  
-  def base62_encode(script)
-    words = Words.new(script)
-    encode = lambda { |word| words.get(word).encoded }
-    
-    # build the packed script
-    
-    p = escape(script.gsub(Words::WORDS, &encode))
-    a = [[words.size, 2].max, 62].min
-    c = words.size
-    k = words.to_s
-    e = self.class.const_get("ENCODE#{a > 10 ? (a > 36 ? 62 : 36) : 10}")
-    r = a > 10 ? "e(c)" : "c"
-    
-    # the whole thing
-    UNPACK.call(p,a,c,k,e,r)
-  end
-  
-  def escape(script)
-    # single quotes wrap the final string so escape them
-    # also escape new lines required by conditional comments
-    script.gsub(/([\\'])/) { |match| "\\#{$1}" }.gsub(/[\r\n]+/, "\\n")
-  end
-  
-  def shrink_variables(script)
-    data = [] # encoded strings and regular expressions
-    regexp= /^[^'"]\//
-    store = lambda do |string|
-      replacement = "##{data.length}"
-      if string =~ regexp
-        replacement = string[0].chr + replacement
-        string = string[1..-1]
-      end
-      data << string
-      replacement
-    end
-    
-    # Base52 encoding (a-Z)
-    encode52 = lambda do |c|
-      (c < 52 ? '' : encode52.call((c.to_f / 52).to_i) ) +
-          ((c = c % 52) > 25 ? (c + 39).chr : (c + 97).chr)
-    end
-    
-    # identify blocks, particularly identify function blocks (which define scope)
-    __block = /(function\s*[\w$]*\s*\(\s*([^\)]*)\s*\)\s*)?(\{([^{}]*)\})/
-    __var = /var\s+/
-    __var_name = /var\s+[\w$]+/
-    __comma = /\s*,\s*/
-    blocks = [] # store program blocks (anything between braces {})
-    
-    # decoder for program blocks
-    encoded = /~(\d+)~/
-    decode = lambda do |script|
-      script = script.gsub(encoded) { |match| blocks[$1.to_i] } while script =~ encoded
-      script
-    end
-    
-    # encoder for program blocks
-    encode = lambda do |match|
-      block, func, args = match, $1, $2
-      if func # the block is a function block
-        
-        # decode the function block (THIS IS THE IMPORTANT BIT)
-        # We are retrieving all sub-blocks and will re-parse them in light
-        # of newly shrunk variables
-        block = decode.call(block)
-        
-        # create the list of variable and argument names
-        vars = block.scan(__var_name).join(",").gsub(__var, "")
-        ids = (args.split(__comma) + vars.split(__comma)).uniq
-        
-        #process each identifier
-        count = 0
-        ids.each do |id|
-          id = id.strip
-          if id and id.length > 1 and !@protected_names.include?(id) # > 1 char
-            id = id.rescape
-            # find the next free short name (check everything in the current scope)
-            short_id = encode52.call(count)
-            while block =~ Regexp.new("[^\\w$.]#{short_id}[^\\w$:]")
-              count += 1
-              short_id = encode52.call(count)
-            end
-            # replace the long name with the short name
-            reg = Regexp.new("([^\\w$.])#{id}([^\\w$:])")
-            block = block.gsub(reg, "\\1#{short_id}\\2") while block =~ reg
-            reg = Regexp.new("([^{,\\w$.])#{id}:")
-            block = block.gsub(reg, "\\1#{short_id}:")
-          end
-        end
-      end
-      replacement = "~#{blocks.length}~"
-      blocks << block
-      replacement
-    end
-    
-    # encode strings and regular expressions
-    script = @data.exec(script, &store)
-    
-    # remove closures (this is for base2 namespaces only)
-    script = script.gsub(/new function\(_\)\s*\{/, "{;#;")
-    
-    # encode blocks, as we encode we replace variable and argument names
-    script = script.gsub(__block, &encode) while script =~ __block
-    
-    # put the blocks back
-    script = decode.call(script)
-    
-    # put back the closure (for base2 namespaces only)
-    script = script.gsub(/\{;#;/, "new function(_){")
-    
-    # put strings and regular expressions back
-    script = script.gsub(/#(\d+)/) { |match| data[$1.to_i] }
-    
-    script
-  end
-end
+# PackR -- a Ruby port of Packer by Dean Edwards
+# Packer version 3.1 copyright 2004-2009, Dean Edwards
+# http://www.opensource.org/licenses/mit-license
+
+[ '/string',
+  '/packr/map',
+  '/packr/collection',
+  '/packr/regexp_group',
+  '/packr/constants',
+  '/packr/encoder',
+  '/packr/parser',
+  '/packr/minifier',
+  '/packr/privates',
+  '/packr/shrinker',
+  '/packr/words',
+  '/packr/base62'
+].each do |path|
+  require File.dirname(__FILE__) + path
+end
+
+class Packr
+  
+  VERSION = '3.1.0'
+  
+  DATA = Parser.new.
+    put("STRING1", IGNORE).
+    put('STRING2', IGNORE).
+    put("CONDITIONAL", IGNORE). # conditional comments
+    put("(OPERATOR)\\s*(REGEXP)", "\\1\\2")
+  
+  def self.encode62(c)
+    (c < 62 ? '' : encode62((c / 62.0).to_i)) +
+        ((c = c % 62) > 35 ? (c+29).chr : c.to_s(36))
+  end
+  
+  def self.encode52(c)
+    # Base52 encoding (a-Z)
+    encode = lambda do |d|
+      (d < 52 ? '' : encode.call((d / 52.0).to_i)) +
+          ((d = d % 52) > 25 ? (d + 39).chr : (d + 97).chr)
+    end
+    encoded = encode.call(c.to_i)
+    encoded = encoded[1..-1] + '0' if encoded =~ /^(do|if|in)$/
+    encoded
+  end
+  
+  def self.pack(script, options = {})
+    @packr ||= self.new
+    @packr.pack(script, options)
+  end
+  
+  def initialize
+    @minifier = Minifier.new
+    @shrinker = Shrinker.new
+    @privates = Privates.new
+    @base62   = Base62.new
+  end
+  
+  def pack(script, options = {})
+    script = @minifier.minify(script)
+    script = @shrinker.shrink(script, options[:protect]) if options[:shrink_vars]
+    script = @privates.encode(script) if options[:private]
+    script = @base62.encode(script) if options[:base62]
+    script
+  end
+  
+end
+