condenser 0.3 → 1.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1d734f957b2e58ee498581dc2cb8017804a94cd740cbb2e8dfd34b7ca0a857f8
-  data.tar.gz: 4a20d49078d81be94a432f5736184796728fdb346600b59b9088e3033ffcde90
+  metadata.gz: 88fa94af3fb5463ea3a65bfa7fe20ade29fcf8b7aaf6ce0d9755706f7ee4ca36
+  data.tar.gz: d4906ce2c57db79bf28745eecf49ec598d8a5813143d9f27571a0a6c629bb207
 SHA512:
-  metadata.gz: ee0b00db38f072943343a772443078ca079294b17bb351c8b4aef7522a1261c162f9ef5a1e096bcb13028f1468191584c9f5b4b6741fb5a0985246ed8fe0caeb
-  data.tar.gz: 463c7551669ebb224da52aa749db123175c320ce2e1265491d3cf39e80f473b525118efcc850574dfab66d98eaeaf36b22ead12c70b2142c60279985b1c5f4b6
+  metadata.gz: 8034556c85cb2e39767a74b5e1b3e01813690e5bde34a56185d947ea2b3e5958b16a24b79d7d74d57713499b1a1ef08d6a35293883e8b7a1c48fcc8c4cd30fa5
+  data.tar.gz: 50dada7f8d7e521eb13f915c30a9b48f581f7c95fd1f12bff3c41cde79a4c7abefce9c79c1f19babb1480551070bec9de5dbdcd83aefa8d4ad0180c130f2e857

data/lib/condenser.rb

@@ -15,10 +15,12 @@ class Condenser
   
   prepend Environment, Pipeline, Resolve
   
-  autoload :BabelProcessor,   'condenser/processors/rollup_processor'
-  autoload :RollupProcessor,  'condenser/processors/babel_processor'
+  autoload :BabelProcessor,   'condenser/processors/babel_processor'
+  autoload :RollupProcessor,  'condenser/processors/rollup_processor'
+  autoload :JSAnalyzer,       'condenser/processors/js_analyzer'
   autoload :NodeProcessor,    'condenser/processors/node_processor'
   autoload :UglifyMinifier,   'condenser/minifiers/uglify_minifier'
+  autoload :TerserMinifier,   'condenser/minifiers/terser_minifier'
   autoload :Erubi,            'condenser/templating_engine/erb'
   autoload :SassMinifier,     'condenser/minifiers/sass_minifier'
   autoload :SassTransformer,  'condenser/transformers/sass_transformer'
@@ -27,6 +29,7 @@ class Condenser
   autoload :JstTransformer,   'condenser/transformers/jst_transformer'
   autoload :FileWriter,       'condenser/writers/file_writer'
   autoload :ZlibWriter,       'condenser/writers/zlib_writer'
+  autoload :BrotliWriter,     'condenser/writers/brotli_writer'
   autoload :BuildCache,       'condenser/build_cache'
   
   def self.configure(&block)
@@ -35,11 +38,11 @@ class Condenser
   
   attr_accessor :logger, :digestor
   
-  def initialize(*path, logger: nil, digestor: nil, cache: nil, pipeline: nil, &block)
+  def initialize(*path, logger: nil, digestor: nil, cache: nil, pipeline: nil, npm_path: nil, &block)
     @logger = logger || Logger.new($stdout, level: :info)
     @path = []
-    @npm_path = nil
     append_path(path)
+    self.npm_path = npm_path
     @cache = cache || Cache::MemoryStore.new
     @build_cc = 0
     self.digestor = digestor || Digest::SHA256
@@ -48,14 +51,18 @@ class Condenser
       configure(&block)
     elsif pipeline != false
       self.configure do
+        # register_preprocessor 'application/javascript', Condenser::JSAnalyzer
         register_preprocessor 'application/javascript', Condenser::BabelProcessor
         register_exporter     'application/javascript', Condenser::RollupProcessor
         register_minifier     'application/javascript', Condenser::UglifyMinifier
-      
+        
         register_minifier  'text/css', Condenser::SassMinifier
       
         register_writer Condenser::FileWriter.new
         register_writer Condenser::ZlibWriter.new
+        if Gem::Specification::find_all_by_name('brotli').any?
+          register_writer Condenser::BrotliWriter.new
+        end
       end
     end
   end

data/lib/condenser/asset.rb

@@ -10,10 +10,10 @@ class Condenser
     include EncodingUtils
     
     attr_reader :environment, :filename, :content_types, :source_file, :source_path
-    attr_reader :linked_assets, :content_types_digest
+    attr_reader :linked_assets, :content_types_digest, :exports
     attr_writer :source, :sourcemap
 
-    attr_accessor :imports, :processed, :export
+    attr_accessor :imports, :processed
     
     def initialize(env, attributes={})
       @environment    = env
@@ -74,7 +74,7 @@ class Condenser
     
       d = []
       deps.each do |i|
-        i = [i, nil] if i.is_a?(String)
+        i = [i, @content_types] if i.is_a?(String)
         @environment.resolve(i[0], File.dirname(@source_file), accept: i[1]).each do |asset|
           d << asset
         end
@@ -90,7 +90,7 @@ class Condenser
       
       d = []
       deps.each do |i|
-        i = [i, nil] if i.is_a?(String)
+        i = [i, @content_types] if i.is_a?(String)
         @environment.resolve(i[0], File.dirname(@source_file), accept: i[1]).each do |asset|
           d << asset
         end
@@ -146,6 +146,7 @@ class Condenser
     def cache_key
       Digest::SHA1.base64digest(JSON.generate([
         Condenser::VERSION,
+        @environment.pipline_digest,
         @source_file,
         stat.ino,
         stat.mtime.to_f,

data/lib/condenser/environment.rb

@@ -11,7 +11,7 @@ class Condenser
     attr_reader :path, :npm_path
     attr_accessor :cache
     
-    def initialize(*args)
+    def initialize(*args, **kws, &block)
       @loaded_processors = Set.new
       @context_class = Class.new(Condenser::Context)
       super
@@ -42,16 +42,14 @@ class Condenser
     end
 
     def npm_path=(path)
-      path = File.expand_path(path)
-      raise ArgumentError, "Path \"#{path}\" does not exists" if !File.directory?(path)
-      @npm_path = path
+      if path.nil?
+        @npm_path = nil
+      else
+        path = File.expand_path(path)
+        raise ArgumentError, "Path \"#{path}\" does not exists" if !File.directory?(path)
+        @npm_path = path
+      end
     end
-    
-    # def npm_check
-    #   if app.root.join('package-lock.json') && npm ls != 0
-    #     npm i
-    #   end
-    # end
   
     def append_npm_path(*paths)
       paths.flatten.each do |path|

data/lib/condenser/helpers/parse_helpers.rb

@@ -0,0 +1,58 @@
+module Condenser::ParseHelpers
+
+  attr_accessor :matched
+
+  def eos?
+    @index >= @source.size
+  end
+
+  def scan_until(r)
+    index = @source.index(r, @index)
+    match = @source.match(r, @index)
+    if match
+      @matched = match.to_s
+      @old_index = @index
+      @index = index + @matched.size
+    else
+      @matched = nil
+    end
+    match
+  end
+
+  def pre_match
+    @source[@old_index...(@index-@matched.size)]
+  end
+
+  def rewind(by=1)
+    @index -= by
+  end
+
+  def forward(by=1)
+    @index += by
+  end
+
+  def seek(pos)
+    @old_index = nil
+    @matched = nil
+    @index = pos
+  end
+
+  def next_word
+    nw = @source.match(/\s*(\S+)/, @index)
+    nw.nil? ? nil : nw[1]
+  end
+
+  def current_line
+    start = (@source.rindex(/(\n|\z)/, @old_index) || 0) + 1
+    uptop = @source.index(/(\n|\z)/, @index) || (@old_index + @matched.length)
+    @source[start..uptop]
+  end
+
+  def cursor
+    start = (@source.rindex("\n", @old_index) || 0) + 1
+    uptop = @source.index("\n", @index) || (@old_index + @matched.length)
+    lineno = @source[0..start].count("\n") + 1
+    "#{lineno.to_s.rjust(4)}: " + @source[start..uptop] + "\n      #{'-'* (@old_index+1-start)}#{'^'*(@matched.length)}"
+  end
+
+end

data/lib/condenser/minifiers/terser_minifier.rb

@@ -0,0 +1,48 @@
+class Condenser::TerserMinifier < Condenser::NodeProcessor
+
+  def initialize(dir, options = {})
+    super(dir)
+    npm_install('terser')
+    
+    @options = options.merge({
+      warnings: true,
+      sourceMap: false,
+      keep_classnames: true
+    }).freeze
+  end
+
+  def call(environment, input)
+    opts = {
+      # 'moduleRoot' => nil,
+      # 'filename' => input[:filename],
+      # 'moduleId' => input[:filename].sub(/(\..+)+/, ''),
+      # 'filenameRelative' => input[:filename],#split_subpath(input[:load_path], input[:filename]),
+      # 'sourceFileName' => input[:filename],
+      # 'sourceMapTarget' => input[:filename]
+      # # 'inputSourceMap'
+    }.merge(@options)
+    
+    result = exec_runtime(<<-JS)
+      const Terser = require("#{npm_module_path('terser')}");
+      const source = #{JSON.generate(input[:filename] => input[:source])}
+      const options = #{JSON.generate(opts)};
+
+
+      var result = Terser.minify(source, options);
+      if (result.error !== undefined) {
+        console.log(JSON.stringify({'error': result.error.name + ": " + result.error.message}));
+        process.exit(1);
+      } else {
+        console.log(JSON.stringify(result));
+      }
+    JS
+
+    exec_runtime_error(result['error']) if result['error']
+    
+    result['warnings']&.each { |w| environment.logger.warn(w) }
+    
+    input[:source] = result['code']
+    input[:map] = result['map']
+  end
+
+end

data/lib/condenser/minifiers/uglify_minifier.rb

@@ -1,14 +1,12 @@
-# From npm install uglify-js
 class Condenser::UglifyMinifier < Condenser::NodeProcessor
 
   class Error < StandardError
   end
   
-  def self.call(environment, input)
-    new.call(environment, input)
-  end
-  
-  def initialize(options = {})
+  def initialize(dir, options = {})
+    super(dir)
+    npm_install('uglify-js')
+    
     @options = options.merge({
       warnings: true
     }).freeze
@@ -26,7 +24,7 @@ class Condenser::UglifyMinifier < Condenser::NodeProcessor
     }.merge(@options)
     
     result = exec_runtime(<<-JS)
-      const UglifyJS = require("#{File.expand_path('../node_modules', __FILE__)}/uglify-js");
+      const UglifyJS = require("#{npm_module_path('uglify-js')}");
       const source = #{JSON.generate(input[:filename] => input[:source])}
       const options = #{JSON.generate(opts)};
 

data/lib/condenser/pipeline.rb

@@ -12,7 +12,7 @@ class Condenser
       end
     end
     
-    def initialize(*args)
+    def initialize(*args, **kws, &block)
       CONFIG_VARS.each do |var_name|
         original_var = self.class.instance_variable_get("@#{var_name}")
         new_var = original_var.dup
@@ -28,6 +28,39 @@ class Condenser
       super
     end
     
+    def pipline_to_json(values)
+      if values.is_a?(Hash)
+        values.transform_values do |value|
+          pipline_to_json(value)
+        end
+      elsif values.is_a?(Array)
+        values.map do |value|
+          pipline_to_json(value)
+        end
+      elsif values.is_a?(Class) || values.is_a?(Module)
+        values.name
+      elsif values.nil? || values == true || values == false || values.is_a?(String) || values.is_a?(Symbol) || values.is_a?(Integer) || values.is_a?(Float)
+        values
+      else
+        { values.class.name => pipline_to_json(values.options) }
+      end
+    end
+    
+    def pipline_hash
+      JSON.generate(pipline_to_json([
+        @templates,
+        @preprocessors,
+        @transformers,
+        @postprocessors,
+        @minifiers,
+        @exporters
+      ]))
+    end
+    
+    def pipline_digest
+      @pipline_digest ||= Digest::MD5.hexdigest(pipline_hash)
+    end
+    
     def register_mime_type(mime_type, extensions: nil, extension: nil, charset: :default)
       extensions = Array(extensions || extension)
 
@@ -48,7 +81,7 @@ class Condenser
       if engine.nil?
         @preprocessors[mime_type].clear
       else
-        @preprocessors[mime_type]&.delete(engine)
+        @preprocessors[mime_type]&.reject! { |e| e == engine || e.is_a?(engine) }
       end
     end
 
@@ -89,9 +122,9 @@ class Condenser
       end
     end
         
-    def unregister_writer(mime_types, engine, added_mime_types=nil)
+    def unregister_writer(engine, for_mime_types: [])
       mime_types = @writers.keys if mime_types.nil?
-      Array(mime_types).each do |mime_type|
+      Array(for_mime_types || @writers.keys).each do |mime_type|
         @writers[mime_type].select! do |writer|
           if engine.nil? || engine == writer[0] || writer[0].is_a?(engine)
             if added_mime_types.nil? || added_mime_types == writer[1]

data/lib/condenser/processors/babel_processor.rb

@@ -2,34 +2,63 @@ require 'json'
 
 class Condenser::BabelProcessor < Condenser::NodeProcessor
   
-  # npm install @babel/core @babel/runtime-corejs3 @babel/plugin-transform-runtime @babel/preset-env rollup rollup-plugin-commonjs  rollup-plugin-node-resolve  @babel/plugin-proposal-class-properties babel-plugin-transform-class-extended-hook
+  attr_accessor :options
   
-  def self.call(environment, input)
-    new.call(environment, input)
-  end
-  
-  def initialize(options = {})
+  def initialize(dir = nil, options = {})
+    super(dir)
+    
+    options[:plugins] ||= [
+      ["@babel/plugin-transform-runtime", { corejs: 3, useESModules: true }]
+    ]
+    options[:presets] ||= [
+      ['@babel/preset-env', {
+        modules: false,
+        targets: { browsers: '> 1% and not dead' }
+      }]
+    ]
+
+    packages = options.slice(:plugins, :presets).values.reduce(&:+).map { |p| p.is_a?(Array) ? p[0] : p}
+    packages.unshift('@babel/core')
+    if packages.include?('@babel/plugin-transform-runtime')
+      runtime = options[:plugins].find { |i| i.is_a?(Array) ? i[0] == '@babel/plugin-transform-runtime' : i == '@babel/plugin-transform-runtime' }
+      packages << if runtime.is_a?(Array) && runtime[1][:corejs]
+        if runtime[1][:corejs].is_a?(Hash)
+          "@babel/runtime-corejs#{runtime[1][:corejs][:version]}"
+        else
+          "@babel/runtime-corejs#{runtime[1][:corejs]}"
+        end
+      else
+        '@babel/runtime'
+      end
+    end
+    
+    npm_install(*packages)
+    
+    options[:plugins].map! do |plugin|
+      if plugin.is_a?(Array)
+        plugin[0] = npm_module_path(plugin[0])
+        plugin
+      else
+        npm_module_path(plugin)
+      end
+    end
+    
+    options[:presets].map! do |plugin|
+      if plugin.is_a?(Array)
+        plugin[0] = npm_module_path(plugin[0])
+        plugin
+      else
+        npm_module_path(plugin)
+      end
+    end
+
     @options = {
-      ast: false,
-      compact: false,
-      plugins: [
-        ["#{node_modules_path}/babel-plugin-transform-class-extended-hook", {}],
-        ["#{node_modules_path}/@babel/plugin-proposal-class-properties", {}],
-        ["#{node_modules_path}/@babel/plugin-transform-runtime", {
-          corejs: 3,
-          useESModules: true
-        }],
-      ],
-      presets: [["#{File.expand_path('../node_modules', __FILE__)}/@babel/preset-env", {
-        "modules": false,
-        "targets": { "browsers": "> 1% and not dead" }
-        # "useBuiltIns": 'usage',
-        # corejs: { version: 3, proposals: true }
-      } ]],
-      sourceMap: false
+      ast:        false,
+      compact:    false,
+      sourceMap:  false
     }.merge(options)
   end
-
+  
   def call(environment, input)
     opts = {
       # 'moduleRoot' => nil,
@@ -43,7 +72,7 @@ class Condenser::BabelProcessor < Condenser::NodeProcessor
     }.merge(@options).select { |k,v| !v.nil? }
     
     result = exec_runtime(<<-JS)
-      const babel = require("#{File.expand_path('../node_modules', __FILE__)}/@babel/core");
+      const babel = require("#{File.join(npm_module_path('@babel/core'))}");
       const source = #{JSON.generate(input[:source])};
       const options = #{JSON.generate(opts).gsub(/"@?babel[\/-][^"]+"/) { |m| "require(#{m})"}};
       

data/lib/condenser/processors/js_analyzer.rb

@@ -0,0 +1,172 @@
+require 'condenser/helpers/parse_helpers'
+
+class Condenser::JSAnalyzer
+
+  include Condenser::ParseHelpers
+  
+  def self.setup(env)
+  end
+  
+  def self.call(environment, input)
+    new.call(environment, input)
+  end
+  
+  def call(environment, input)
+    seek(0)
+    @source = input[:source]
+    @stack =  []
+    
+    
+    input[:export_dependencies] = parse_imports
+
+    while !eos?
+      case @stack.last
+      when :tick_value
+        scan_until(/(\$\{|\`)/)
+        case matched
+        when '`'
+          @stack.pop
+        when '${'
+          @stack << :tick_statment
+        end
+      else
+        scan_until(/(\/\/|\/\*|\/|\(|\)|\{|\}|\"|\'|\`|export|\z)/)
+        case matched
+        when '//'
+          scan_until(/(\n|\z)/)
+        when '/*'
+          scan_until(/\*\//)
+        when '"'
+          double_quoted_value
+        when "'"
+          single_quoted_value
+        when '`'
+          @stack << :tick_value
+        when '/'
+          if match_index = @source.rindex(/(\w+|\))\s*\//, @index)
+            match = @source.match(/(\w+|\))\s*\//, match_index)
+            if match[0].length + match_index != @index
+              regex_value
+            end
+          else
+            regex_value
+          end
+        when '('
+          @stack.push :parenthesis
+        when ')'
+          raise 'error' if @stack.last != :parenthesis
+          @stack.pop
+        when '{'
+          @stack.push :brackets
+        when '}'
+          case @stack.last
+          when :brackets, :tick_statment
+            @stack.pop
+          else
+            raise 'error'
+          end
+        when 'export'
+          input[:exports] = true;
+          input[:default_export] = true if next_word == 'default'
+        else
+          @stack.pop
+        end
+      end
+    end
+  end
+  
+  def parse_imports
+    imports = []
+    
+    while @stack.first != :statment
+      case @stack.last
+      when nil
+        scan_until(/(\S+|\z)/)
+        case matched
+        when 'import'
+          @stack << :import
+        else
+          seek(@old_index)
+          @stack << :statment
+        end
+      when :import
+        scan_until(/[\"\'\`]/)
+        imports << case matched
+        when "\""
+          double_quoted_value
+        when "'"
+          single_quoted_value
+        when '`'
+          tick_quoted_value
+        end
+        scan_until(/(;|\n)/)
+        @stack.pop
+      end
+    end
+    
+    imports
+  end
+  
+  def double_quoted_value
+    ret_value = ""
+
+    while scan_until(/[\"\n]/)
+      if matched == "\n"
+        raise 'error'
+      elsif matched == "\""
+        if pre_match[-1] != "\\"
+          ret_value << pre_match
+          return ret_value
+        else
+          ret_value << pre_match << "\\\""
+        end
+
+        
+      else
+        ret_value << match
+      end
+    end
+  end
+  
+  def single_quoted_value
+    ret_value = ""
+
+    while scan_until(/[\'\n]/)
+      if matched == "\n"
+        raise 'error'
+      elsif matched == "\'" && pre_match[-1] != "\\"
+        ret_value << pre_match
+        return ret_value
+      else
+        ret_value << pre_match
+      end
+    end
+  end
+
+  def tick_quoted_value
+    ret_value = ""
+
+    while scan_until(/[\`]/)
+      if matched == "\`" && pre_match[-1] != "\\"
+        ret_value << pre_match
+        return ret_value
+      else
+        ret_value << pre_match
+      end
+    end
+  end
+  
+  def regex_value
+    ret_value = ""
+
+    while scan_until(/\//)
+      if matched == "/" && pre_match[-1] != "\\"
+        ret_value << pre_match
+        return ret_value
+      else
+        ret_value << pre_match
+      end
+    end
+  end
+
+end