condenser 0.0.2 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d37b98de2cd34f0a1c0a3a42e9ed3e5d77b51340bab616f5e4cf91c6cea92cc7
-  data.tar.gz: 427f873fae6ae0e19ab856af95d3134a09efadcfc18bfc16b9ba17a81a471de1
+  metadata.gz: 9be510da13d18c20e5422d21a17bbce10ac106411d8e0748a897bf485ad78cc1
+  data.tar.gz: 93c51021f99f6572f47992882906e9af0e913dd5023c3d1fb0406fa1eb074d79
 SHA512:
-  metadata.gz: 0b0df9e217f5d208922fd59ba4267ce3b83443ad992d71c2228d8df8bf11ec9ec16d82419f1fd8cf5bd1cafeb1343b240562c6585a5285abc91f955d6cfe67a5
-  data.tar.gz: '09044bad795ddf5e2f7e056eaa1ee970295a5634f74f1fa40169918afee269471f1988bc9be73963fae806908530ea479296971a83f1d2204b449791c3c2cb8e'
+  metadata.gz: f8747cb712a5debe29168b9b6a8a9f3a995e997676755d3ffb1a544d7a0a5563082cd857edccfafa45445ed36214fc1e34a9775a98e560518233c968600d2325
+  data.tar.gz: 58dcb5e01ca75410329a2941752031cdd5265af2cbc10ad8235a1a538cad1ea7ca00766dba3ecd15ff0a406aaddf032375069c3029732b8fa85d7900922d97fe

data/lib/condenser.rb

@@ -22,10 +22,15 @@ class Condenser
   attr_reader :logger
   attr_accessor :digestor
   
-  def initialize(root='.', digestor: nil)
+  def initialize(*path, digestor: nil, cache: nil)
+    super()
     @logger = Logger.new($stderr, level: :warn)
+    @path = []
+    @npm_path = nil
+    append_path(path)
+    append_path(EJS::ASSET_DIR)
+    @cache = cache || Cache::MemoryStore.new
     self.digestor = digestor || Digest::SHA256
-    super(root)
   end
 end
 
@@ -79,25 +84,38 @@ Condenser.configure do
   register_template  'application/erb', Condenser::Erubi
   
   # CSS
-  register_mime_type 'text/css', extension: '.css', charset: :css
-
+  require 'condenser/minifiers/sass_minifier'
+  register_mime_type      'text/css', extension: '.css', charset: :css
+  register_minifier       'text/css', Condenser::SassMinifier
+  
   # SASS
-  require 'condenser/processors/sass_processor'
+  require 'condenser/analyzers/sass_analyzer'
+  require 'condenser/transformers/sass_transformer'
   register_mime_type    'text/sass', extensions: %w(.sass .css.sass)
+  register_analyzer     'text/sass', Condenser::SassAnalyzer
   # register_transformer  'text/sass', 'text/css', SassProcessor
   
   # SCSS
   register_mime_type    'text/scss', extensions: %w(.scss .css.scss)
-  register_transformer  'text/scss', 'text/css', Condenser::ScssProcessor
+  register_analyzer     'text/scss', Condenser::ScssAnalyzer
+  register_transformer  'text/scss', 'text/css', Condenser::ScssTransformer
   
   # Javascript
+  require 'condenser/analyzers/javascript_analyzer'
   require 'condenser/processors/rollup_processor'
   require 'condenser/processors/babel_processor'
+  require 'condenser/minifiers/uglify_minifier'
   register_mime_type    'application/javascript', extension: '.js', charset: :unicode
+  register_analyzer     'application/javascript', Condenser::JavascriptAnalyzer
   register_preprocessor 'application/javascript', Condenser::BabelProcessor
   register_exporter     'application/javascript', Condenser::RollupProcessor
-  # register_minifier     'application/javascript', Uglifier
+  register_minifier     'application/javascript', Condenser::UglifyMinifier
 
+  # EJS
+  require 'condenser/transformers/ejs'
+  register_mime_type    'application/ejs', extensions: %w(.ejs .jst.ejs)
+  register_transformer  'application/ejs', 'application/javascript', Condenser::EjsTransformer
+  
   # Writers
   require 'condenser/writers/file_writer'
   require 'condenser/writers/zlib_writer'

data/lib/condenser/analyzers/javascript_analyzer.rb

@@ -0,0 +1,64 @@
+require 'json'
+require File.expand_path('../../processors/node_processor', __FILE__)
+
+class Condenser
+  class JavascriptAnalyzer < NodeProcessor
+    class Error < StandardError; end
+    
+    def self.call(environment, input)
+      new.call(environment, input)
+    end
+    
+    def initialize(options = {})
+      @options = options.merge({
+        ast: false,
+        sourceMap: false
+      }).freeze
+    end
+
+    def call(environment, input)
+      opts = {
+        'filename' => input[:filename],
+        'moduleId' => input[:filename].sub(/(\..+)+/, ''),
+        'filenameRelative' => input[:filename],
+      }.merge(@options)
+      
+      result = exec_runtime(<<-JS)
+        module.paths.push("#{File.expand_path('../../processors/node_modules', __FILE__)}")
+      
+        const babel = require('@babel/core');
+        const source = #{JSON.generate(input[:source])};
+        const options = #{JSON.generate(opts)};
+        let imports = [];
+        
+        options['plugins'] = [];
+        options['plugins'].push(function({ types: t }) {
+          return {
+            visitor: {
+              ImportDeclaration(path, state) {
+                imports.push(path.node.source.value);
+              }
+            }
+          };
+        });
+        
+        try {
+          const result = babel.transform(source, options);
+          console.log(JSON.stringify({imports: imports}));
+        } catch(e) {
+          console.log(JSON.stringify({'error': e.name + ": " + e.message}));
+          process.exit(1);
+        }
+      JS
+      
+      if result['error']
+        raise Error, result['error']
+      else
+        result['imports'].each do |i|
+          input[:dependencies] << i
+        end
+      end
+    end
+  
+  end
+end

data/lib/condenser/analyzers/sass_analyzer.rb

@@ -0,0 +1,64 @@
+require 'sass'
+
+class Condenser
+  class SassAnalyzer
+
+    # Internal: Defines default sass syntax to use. Exposed so the ScssProcessor
+    # may override it.
+    def self.syntax
+      :sass
+    end
+
+    # Public: Return singleton instance with default options.
+    #
+    # Returns SassProcessor object.
+    def self.instance
+      @instance ||= new
+    end
+
+    def self.call(environment, input)
+      instance.call(environment, input)
+    end
+
+    def initialize(options = {})
+      @sass_config = options[:sass_config] || {}
+    end
+
+    def call(environment, input)
+      engine_options = merge_options({
+        filename:     input[:filename],
+        syntax:       self.class.syntax,
+        cache_store:  SassTransformer::Cache.new(environment.cache),
+      })
+
+      engine = Sass::Engine.new(input[:source], engine_options)
+
+      engine.to_tree.grep(Sass::Tree::ImportNode) do |n|
+        next if n.imported_filename =~ /\Ahttps?:\/\//
+        environment.resolve(n.imported_filename, input[:source_file]).each do |a|
+          input[:dependencies] << a.filename
+        end
+      end
+    end
+
+    private
+
+    def merge_options(options)
+      defaults = @sass_config.dup
+
+      if load_paths = defaults.delete(:load_paths)
+        options[:load_paths] += load_paths
+      end
+
+      options.merge!(defaults)
+      options
+    end
+  end
+
+  class ScssAnalyzer < SassAnalyzer
+    def self.syntax
+      :scss
+    end
+  end
+  
+end

data/lib/condenser/asset.rb

@@ -26,7 +26,8 @@ class Condenser
       
       @linked_assets  = Set.new
       @dependencies   = Set.new
-      
+
+      @analyzed       = false
       @processed      = false
       @exported       = false
     end
@@ -48,20 +49,80 @@ class Condenser
       [dirname, basename].compact.join('/')
     end
     
-    def cache_key(data)
+    def analyze
+      analyze! if !@analyzed
     end
     
-    def process!(source=nil, source_digest=nil, content_digest=nil)
-      source ||= File.binread(@source_file)
-      source_digest ||= Digest::SHA1.base64digest(source)
+    def analyze!
+      @source = File.binread(@source_file)
+      @digest = @environment.digestor.digest(@source)
+      @digest_name = @environment.digestor.name.sub(/^.*::/, '').downcase
       content_digest ||= Digest::SHA1.base64digest(@content_types.join(':'))
 
-      result = @environment.cache.fetch("process/#{source_digest}/#{content_digest}") do
+      result = @environment.cache.fetch("analyze/#{[@digest].pack('m0')}:#{content_digest}") do
         dirname, basename, extensions, mime_types = @environment.decompose_path(@source_file)
         data = {
-          source: source,
+          source: @source,
+          source_file: @source_file,
+          filename: @filename.dup,
+          content_type: mime_types,
+          linked_assets: [],
+          dependencies: []
+        }
+
+        while @environment.templates.has_key?(data[:content_type].last)
+          templator = @environment.templates[data[:content_type].pop]
+          templator.call(@environment, data)
+          data[:filename] = data[:filename].gsub(/\.#{extensions.last}$/, '')
+        end
+      
+        case @environment.mime_types[data[:content_type].last][:charset]
+        when :unicode
+          detect_unicode(data[:source])
+        when :css
+          detect_css(data[:source])
+        when :html
+          detect_html(data[:source])
+        else
+          detect(data[:source]) if mime_types.last.start_with?('text/')
+        end
+        
+        @environment.analyzers_for(data[:content_type].last).each do |analyzer|
+          analyzer.call(@environment, data)
+        end
+        
+        data
+      end
+      
+      @analyzed = true
+      @linked_assets = result[:linked_assets]
+      @dependencies = result[:dependencies]
+    end
+
+    def cache_key(include_dependencies=true)
+      analyze
+      key = [[@digest].pack('m0'), @content_types.join(':')]
+
+      if include_dependencies
+        @dependencies.each do |i|
+          @environment.resolve(i, File.dirname(@source_file), accept: @content_types).each do |asset|
+            key << asset.cache_key
+          end
+        end
+      end
+
+      Digest::SHA1.base64digest(JSON.generate(key))
+    end
+    
+    def process!(source=nil, source_digest=nil, content_digest=nil)
+      analyze
+      content_digest ||= Digest::SHA1.base64digest(@content_types.join(':'))
+      
+      result = @environment.cache.fetch("process/#{cache_key}/#{content_digest}") do
+        dirname, basename, extensions, mime_types = @environment.decompose_path(@source_file)
+        data = {
+          source: @source,
           source_file: @source_file,
-          source_digest: source_digest,
         
           filename: @filename.dup,
           content_type: mime_types,
@@ -115,25 +176,17 @@ class Condenser
       @sourcemap = result[:map]
       @filename = result[:filename]
       @content_types = result[:content_type]
-      @linked_assets = result[:linked_assets]
       @digest = result[:digest]
       @digest_name = result[:digest_name]
-      @dependencies = result[:dependencies]
     end
     
     def export
-      source ||= File.binread(@source_file)
-      source_digest ||= Digest::SHA1.base64digest(source)
-      content_digest ||= Digest::SHA1.base64digest(@content_types.join(':'))
-      
-      process!(source, source_digest, content_digest) if !@processed
-      
-      result = @environment.cache.fetch("export/#{source_digest}/#{content_digest}") do
+      result = @environment.cache.fetch("export/#{cache_key}") do
+        process
         dirname, basename, extensions, mime_types = @environment.decompose_path(@filename)
         data = {
           source: @source,
           source_file: @source_file,
-          source_digest: source_digest,
         
           filename: @filename.dup,
           content_types: @content_types,
@@ -143,6 +196,7 @@ class Condenser
           dependencies: []
         }
         @environment.exporters[content_type]&.call(@environment, data)
+        @environment.minifiers[content_type]&.call(@environment, data)
         data[:digest] = @environment.digestor.digest(data[:source])
         data[:digest_name] = @environment.digestor.name.sub(/^.*::/, '').downcase
         data

data/lib/condenser/cache/file_store.rb

@@ -0,0 +1,181 @@
+# frozen_string_literal: true
+class Condenser::Cache
+  class FileStore < Condenser::CacheStore
+    GITKEEP_FILES = ['.gitkeep', '.keep'].freeze
+
+    # Public: Initialize the cache store.
+    #
+    # root     - A String path to a directory to persist cached values to.
+    # size     - A Integer of the maximum size the store will hold (in bytes).
+    #            (default: 25MB).
+    # logger   - The logger to which some info will be printed.
+    #            (default logger level is FATAL and won't output anything).
+    def initialize(root, size: 26_214_400, logger: nil)
+      @root     = root
+      @max_size = size
+      @gc_size  = size * 0.75
+      @logger   = logger
+    end
+
+    def get(key)
+      path = File.join(@root, "#{key}.cache")
+      
+      value = safe_open(path) do |f|
+        begin
+          unmarshaled_deflated(f.read, Zlib::MAX_WBITS)
+        rescue Exception => e
+          # @logger.error do
+            puts "#{self.class}[#{path}] could not be unmarshaled: #{e.class}: #{e.message}"
+          # end
+          nil
+        end
+      end
+
+      FileUtils.touch(path) if value
+      
+      value
+    end
+
+    def set(key, value)
+      path = File.join(@root, "#{key}.cache")
+      
+      # Ensure directory exists
+      FileUtils.mkdir_p File.dirname(path)
+      
+      # Check if cache exists before writing
+      exists = File.exist?(path)
+
+      # Serialize value
+      marshaled = Marshal.dump(value)
+
+      # Compress if larger than 4KB
+      if marshaled.bytesize > 4_096
+        deflater = Zlib::Deflate.new(
+          Zlib::BEST_COMPRESSION,
+          Zlib::MAX_WBITS,
+          Zlib::MAX_MEM_LEVEL,
+          Zlib::DEFAULT_STRATEGY
+        )
+        deflater << marshaled
+        raw = deflater.finish
+      else
+        raw = marshaled
+      end
+
+      # Write data
+      Condenser::Utils.atomic_write(path) do |f|
+        f.write(raw)
+        @size = size + f.size unless exists
+      end
+
+      # GC if necessary
+      gc! if size > @max_size
+
+      value
+    end
+    
+    # Public: Pretty inspect
+    #
+    # Returns String.
+    def inspect
+      "#<#{self.class}>"
+    end
+
+    # Public: Clear the cache
+    #
+    # adapted from ActiveSupport::Cache::FileStore#clear
+    #
+    # Deletes all items from the cache. In this case it deletes all the entries in the specified
+    # file store directory except for .keep or .gitkeep. Be careful which directory is specified
+    # as @root because everything in that directory will be deleted.
+    #
+    # Returns true
+    def clear(options=nil)
+      Dir.children(@root).each do |f|
+        next if GITKEEP_FILES.include?(f)
+        FileUtils.rm_r(File.join(@root, f))
+      end
+      true
+    end
+    
+    def size
+      @size ||= find_caches.inject(0) { |sum, (_, stat)| sum + stat.size }
+    end
+    
+    private
+
+    def safe_open(path, &block)
+      if File.exist?(path)
+        File.open(path, 'rb', &block)
+      end
+    rescue Errno::ENOENT
+    end
+    
+    # Returns an Array of [String filename, File::Stat] pairs sorted by
+    # mtime.
+    def find_caches
+      Dir.glob(File.join(@root, '**/*.cache')).reduce([]) { |stats, filename|
+        stat = safe_stat(filename)
+        # stat maybe nil if file was removed between the time we called
+        # dir.glob and the next stat
+        stats << [filename, stat] if stat
+        stats
+      }.sort_by { |_, stat| stat.mtime.to_i }
+    end
+    
+    def safe_stat(fn)
+      File.stat(fn)
+    rescue Errno::ENOENT
+      nil
+    end
+
+    def gc!
+      start_time = Time.now
+
+      caches = find_caches
+      size = caches.inject(0) { |sum, (_, stat)| sum + stat.size }
+
+      delete_caches, keep_caches = caches.partition { |filename, stat|
+        deleted = size > @gc_size
+        size -= stat.size
+        deleted
+      }
+
+      return if delete_caches.empty?
+
+      FileUtils.remove(delete_caches.map(&:first), force: true)
+      @size = keep_caches.inject(0) { |sum, (_, stat)| sum + stat.size }
+
+      @logger.warn do
+        secs = Time.now.to_f - start_time.to_f
+        "#{self.class}[#{@root}] garbage collected " +
+          "#{delete_caches.size} files (#{(secs * 1000).to_i}ms)"
+      end
+    end
+
+    # Internal: Unmarshal optionally deflated data.
+    #
+    # Checks leading marshal header to see if the bytes are uncompressed
+    # otherwise inflate the data an unmarshal.
+    #
+    # str - Marshaled String
+    # window_bits - Integer deflate window size. See ZLib::Inflate.new()
+    #
+    # Returns unmarshaled Object or raises an Exception.
+    def unmarshaled_deflated(str, window_bits = -Zlib::MAX_WBITS)
+      major, minor = str[0], str[1]
+      if major && major.ord == Marshal::MAJOR_VERSION &&
+          minor && minor.ord <= Marshal::MINOR_VERSION
+        marshaled = str
+      else
+        begin
+          marshaled = Zlib::Inflate.new(window_bits).inflate(str)
+        rescue Zlib::DataError
+          marshaled = str
+        end
+      end
+      Marshal.load(marshaled)
+    end
+    
+  end
+end