condenser 0.1 → 1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 78e391e2e34c47daeb8d0489f2ba3567d24ba9a2fecee0c8457c65276bbb6886
-  data.tar.gz: 354fd3b42f9eb4636a94babb725f87c0c4ea037c0943fa82380eb6f35a169338
+  metadata.gz: e2b1dab33dd825d2bf0902e871c68c0334d49ee7af48ee0875132f39adce20d2
+  data.tar.gz: a3c40f6f19722d0cea00e28592cd2049db5185057b299bf697fc169c9a19c89a
 SHA512:
-  metadata.gz: a0bf3d990b15d7b225b8d0af09da4359e95e4add21222bbace313ac8a7b6833a40ee17257498e5f2a65a8fa13f47a6467bb9eae910c6acdbd2ddf7ab64b45ac2
-  data.tar.gz: cff15370f99605a8f87c1e313afbb65efc169acf4df4877c44acbe6b0ea36a2a772ecb98641e5c093fb1d658d35b2d562968d890a5ee328d764d3e8229a6c9aa
+  metadata.gz: c1ca10d2f7dff116d1b4d4daeba50302509c7faffed458e2312d40c391b84a407d596487102a4e306abd9a96bd435c39d0fb23add68a3608aa64014b3fffe0d7
+  data.tar.gz: 3f530d21b0ec272d51aabd96f0218dfc1a02dbb8845dbefd3d92cf1517447bc436785a102415ad9fda696df26d9769f67485f8530445eb66b48dc3318ffd8e72

data/README.md

@@ -8,18 +8,21 @@ and SCSS.
 
 ## Installation
 
-Install Sprockets from RubyGems:
+In your project's `Gemfile` with Bundler:
 
-``` sh
-$ gem install condenser
+``` ruby
+gem 'condenser'
 ```
 
-Or include it in your project's `Gemfile` with Bundler:
+Or via Ruby Gems
 
-``` ruby
-gem 'condenser'
+``` sh
+$ gem install condenser
 ```
 
+If you are using Condenser with Rails, instead of the `condenser` gem use the
+`condenser-rails` gem.
+
 ## Guides
 
 For most people interested in Condenser, you will want to see the README below.
@@ -71,7 +74,8 @@ contains the directory `app/assets/javascripts`:
 
 ## File Order Processing
 
-By default files are processed in alphabetical order. This behavior can impact your asset compilation when one asset needs to be loaded before another.
+By default files are processed in alphabetical order. This behavior can impact
+your asset compilation when one asset needs to be loaded before another.
 
 For example if you have an `application.js` and it loads another directory
 
@@ -81,7 +85,8 @@ import initializers from 'config/initializers/*';
 initializers.forEach((i) => i());
 ```
 
-The files in that directory will be loaded in alphabetical order. If the directory looks like this:
+The files in that directory will be loaded in alphabetical order. If the directory
+looks like this:
 
 ```sh
 $ ls -1 config/initializers/
@@ -91,7 +96,10 @@ beta.js
 gamma.js
 ```
 
-Then `alpha.js` will be loaded before either of the other two. This can be a problem if `gamma.js` needs to be called before `alpha.js`. For files that are ordering dependent you can either require individual files manually:
+Then `alpha.js` will be loaded before either of the other two. This can be a
+problem if `gamma.js` needs to be called before `alpha.js`. For files that are
+order dependent you can either rename the files or require individual files
+manually:
 
 ```js
 import alpha from 'config/initializers/alpha';
@@ -103,13 +111,42 @@ alpha();
 beta();
 ```
 
+## Directives
+
+Sometimes you need to tell Condenser that a file depends on other files. For
+example a `js.erb` file maybe dynamically update depending on the contents of a
+folder as follows:
+
+```js
+console.log([<%= Dir.children("#{asset_path}/models").sort.map(&:inspect).join(', ') %>]);
+```
+
+To tell Condenser that this file depends on `models/*` simply add the depends_on
+directive:
+
+```js
+// depends_on models/*.js
+
+console.log([<%= Dir.children("#{asset_path}/models").sort.map(&:inspect).join(', ') %>]);
+```
+
+The directives must be the first portion of the file.
+
 ## Cache
 
-Compiling assets is slow. It requires a lot of disk use to pull assets off of hard drives, a lot of RAM to manipulate those files in memory, and a lot of CPU for compilation operations. Because of this Condenser has a cache to speed up asset compilation times. That's the good news. The bad news, is that Condenser has a cache and if you've found a bug it's likely going to involve the cache.
+Compiling assets is slow. It requires a lot of disk use to pull assets off of
+hard drives, a lot of RAM to manipulate those files in memory, and a lot of CPU
+for compilation operations. Because of this Condenser has a cache to speed up
+asset compilation times. That's the good news. The bad news, is that Condenser
+has a cache and if you've found a bug it's likely going to involve the cache.
 
-By default Condenser uses the file system to cache assets. It makes sense that Condenser does not want to generate assets that already exist on disk in `public/assets`, what might not be as intuitive is that Condenser needs to cache "partial" assets.
+By default Condenser uses the file system to cache assets. It makes sense that
+Condenser does not want to generate assets that already exist on disk in
+`public/assets`, what might not be as intuitive is that Condenser needs to cache
+"partial" assets.
 
-For example if you have an `application.js` and it is made up of `a.js`, `b.js`, all the way to `z.js`
+For example if you have an `application.js` and it is made up of `a.js`, `b.js`,
+all the way to `z.js`
 
 ```js
 import 'a';
@@ -118,19 +155,27 @@ import 'b';
 import 'z';
 ```
 
-The first time this file is compiled the `application.js` output will be written to disk, but also intermediary compiled files for `a.js` etc. will be written to the cache directory (usually `tmp/cache/assets`).
+The first time this file is compiled the `application.js` output will be written
+to disk, but also intermediary compiled files for `a.js` etc. will be written to
+the cache directory (usually `tmp/cache/assets`).
 
-So, if `b.js` changes it will get recompiled. However instead of having to recompile the other files from `a.js` to `z.js` since they did not change, we can use the prior intermediary files stored in the cached values . If these files were expensive to generate, then this "partial" asset cache strategy can save a lot of time.
+So, if `b.js` changes it will get recompiled. However instead of having to
+recompile the other files from `a.js` to `z.js` since they did not change,
+we can use the prior intermediary files stored in the cached values . If these
+files were expensive to generate, then this "partial" asset cache strategy can
+save a lot of time.
 
-Directives such as `import` in Javascript and `@import` in SCSS tell Condenser what assets need to be re-compiled when a file changes. Files are considered "fresh" based on their inode number, mtime, size and a combination of cache keys.
+Directives such as `import` in Javascript and `@import` in SCSS tell Condenser
+what assets need to be re-compiled when a file changes. Files are considered
+"fresh" based on their inode number, mtime, size and a combination of cache keys.
 
-In Rails you can force a "clean" install by clearing the `public/assets` and `tmp/cache/assets` directories.
+In Rails you can force a "clean" install by clearing the `public/assets` and
+`tmp/cache/assets` directories.
 
 ### Invoking Ruby with ERB
 
-Condenser provides an ERB engine for preprocessing assets using
-embedded Ruby code. Append `.erb` to a CSS or JavaScript asset's
-filename to enable the ERB engine.
+Condenser provides an ERB engine for preprocessing assets using embedded Ruby
+code. Append `.erb` to a CSS or JavaScript asset's filename to enable the ERB engine.
 
 For example if you have an `app/application/javascripts/app_name.js.erb`
 you could have this in the template
@@ -179,7 +224,9 @@ convert `foo.scss` to `foo.css`.
 
 ## Javascript, ES#, & ES Modules
 
-Condenser transforms Javascript for the browser by transpiling all the files `.js` through [babel](https://babeljs.io) and bundled together via [rollup.js](https://rollupjs.org/).
+Condenser transforms Javascript for the browser by transpiling all the files
+`.js` through [babel](https://babeljs.io) and bundled together via
+[rollup.js](https://rollupjs.org/).
 
 ```js
 // app/assets/javascript/application.js
@@ -189,7 +236,9 @@ var square = (n) => n * n
 console.log(square);
 ```
 
-Start a Rails server in development mode and visit `localhost:3000/assets/application.js`, and this asset will be transpiled to JavaScript:
+Start a Rails server in development mode and visit
+`localhost:3000/assets/application.js`, and this asset will be transpiled to
+JavaScript:
 
 ```js
 var square = function square(n) {

data/lib/condenser.rb

@@ -15,29 +15,39 @@ class Condenser
   
   prepend Environment, Pipeline, Resolve
   
-  autoload :BabelProcessor,   'condenser/processors/rollup_processor'
-  autoload :RollupProcessor,  'condenser/processors/babel_processor'
+  autoload :BabelProcessor,   'condenser/processors/babel_processor'
+  autoload :RollupProcessor,  'condenser/processors/rollup_processor'
+  autoload :JSAnalyzer,       'condenser/processors/js_analyzer'
+  autoload :PurgeCSSProcessor,'condenser/processors/purgecss_processor'
+  autoload :NodeProcessor,    'condenser/processors/node_processor'
   autoload :UglifyMinifier,   'condenser/minifiers/uglify_minifier'
+  autoload :TerserMinifier,   'condenser/minifiers/terser_minifier'
   autoload :Erubi,            'condenser/templating_engine/erb'
   autoload :SassMinifier,     'condenser/minifiers/sass_minifier'
   autoload :SassTransformer,  'condenser/transformers/sass_transformer'
   autoload :ScssTransformer,  'condenser/transformers/sass_transformer'
-  autoload :EjsTransformer,   'condenser/transformers/ejs'
+  autoload :EjsTemplare,      'condenser/templating_engine/ejs'
+  autoload :JstTransformer,   'condenser/transformers/jst_transformer'
   autoload :FileWriter,       'condenser/writers/file_writer'
   autoload :ZlibWriter,       'condenser/writers/zlib_writer'
+  autoload :BrotliWriter,     'condenser/writers/brotli_writer'
   autoload :BuildCache,       'condenser/build_cache'
   
   def self.configure(&block)
     instance_eval(&block)
   end
   
-  attr_accessor :logger, :digestor
+  attr_accessor :logger, :digestor, :base
   
-  def initialize(*path, logger: nil, digestor: nil, cache: nil, pipeline: nil, &block)
+  # base: If base is passed assets cache_keys will be realitve to this.
+  #       This allows deploy systems like Capistrano to take advantage of the
+  #       cache even though it precompiles assets in a different folder
+  def initialize(*path, logger: nil, digestor: nil, cache: nil, pipeline: nil, npm_path: nil, base: nil, &block)
     @logger = logger || Logger.new($stdout, level: :info)
     @path = []
-    @npm_path = nil
     append_path(path)
+    self.npm_path = npm_path
+    @base = base
     @cache = cache || Cache::MemoryStore.new
     @build_cc = 0
     self.digestor = digestor || Digest::SHA256
@@ -46,14 +56,18 @@ class Condenser
       configure(&block)
     elsif pipeline != false
       self.configure do
+        # register_preprocessor 'application/javascript', Condenser::JSAnalyzer
         register_preprocessor 'application/javascript', Condenser::BabelProcessor
         register_exporter     'application/javascript', Condenser::RollupProcessor
         register_minifier     'application/javascript', Condenser::UglifyMinifier
-      
+        
         register_minifier  'text/css', Condenser::SassMinifier
       
         register_writer Condenser::FileWriter.new
         register_writer Condenser::ZlibWriter.new
+        if Gem::Specification::find_all_by_name('brotli').any?
+          register_writer Condenser::BrotliWriter.new
+        end
       end
     end
   end
@@ -130,8 +144,12 @@ Condenser.configure do
   register_mime_type    'application/javascript', extension: '.js', charset: :unicode
   
   # EJS
-  register_mime_type    'application/ejs', extensions: %w(.ejs .jst.ejs), charset: :unicode
-  register_transformer  'application/ejs', 'application/javascript', Condenser::EjsTransformer
+  register_mime_type 'application/ejs', extensions: '.ejs', charset: :unicode
+  register_template  'application/ejs', Condenser::EjsTemplare
+  
+  # JST
+  register_mime_type    'application/jst', extensions: '.jst', charset: :unicode
+  register_transformer  'application/jst', 'application/javascript', Condenser::JstTransformer
   
   # Writers
   register_mime_type 'application/gzip',    extensions: %w(.gz .gzip)

data/lib/condenser/asset.rb

@@ -10,10 +10,10 @@ class Condenser
     include EncodingUtils
     
     attr_reader :environment, :filename, :content_types, :source_file, :source_path
-    attr_reader :linked_assets, :content_types_digest
+    attr_reader :linked_assets, :content_types_digest, :exports
     attr_writer :source, :sourcemap
 
-    attr_accessor :imports, :processed, :export
+    attr_accessor :imports, :processed
     
     def initialize(env, attributes={})
       @environment    = env
@@ -51,10 +51,6 @@ class Condenser
       [dirname, basename].compact.join('/')
     end
     
-    def stat
-      @stat ||= File.stat(@source_file)
-    end
-    
     def restat!
       @stat = nil
     end
@@ -71,10 +67,10 @@ class Condenser
         process
         @process_dependencies
       end
-      
+    
       d = []
       deps.each do |i|
-        i = [i, nil] if i.is_a?(String)
+        i = [i, @content_types] if i.is_a?(String)
         @environment.resolve(i[0], File.dirname(@source_file), accept: i[1]).each do |asset|
           d << asset
         end
@@ -90,7 +86,7 @@ class Condenser
       
       d = []
       deps.each do |i|
-        i = [i, nil] if i.is_a?(String)
+        i = [i, @content_types] if i.is_a?(String)
         @environment.resolve(i[0], File.dirname(@source_file), accept: i[1]).each do |asset|
           d << asset
         end
@@ -144,9 +140,11 @@ class Condenser
     end
     
     def cache_key
-      Digest::SHA1.base64digest(JSON.generate([
+      @cache_key ||= Digest::SHA1.base64digest(JSON.generate([
         Condenser::VERSION,
-        @source_file,
+        @environment.pipline_digest,
+        @environment.base ? @source_file.delete_prefix(@environment.base) : @source_file,
+        Digest::SHA256.file(@source_file).hexdigest,
         @content_types_digest
       ]))
     end
@@ -154,20 +152,26 @@ class Condenser
     def process_cache_version
       return @pcv if @pcv
 
-      f = [stat.ino, stat.mtime.to_f, stat.size]
+      f = []
       all_dependenies(process_dependencies, [], :process_dependencies) do |dep|
-        f << [dep.source_file, dep.stat.ino, dep.stat.mtime.to_f, dep.stat.size]
+        f << [
+          @environment.base ? dep.source_file.delete_prefix(@environment.base) : dep.source_file,
+          Digest::SHA256.file(dep.source_file).hexdigest
+        ]
       end
-
+      
       @pcv = Digest::SHA1.base64digest(JSON.generate(f))
     end
     
     def export_cache_version
       return @ecv if @ecv
 
-      f = [stat.ino, stat.mtime.to_f, stat.size]
+      f = []
       all_dependenies(export_dependencies, [], :export_dependencies) do |dep|
-        f << [dep.source_file, dep.stat.ino, dep.stat.mtime.to_f, dep.stat.size]
+        f << [
+          @environment.base ? dep.source_file.delete_prefix(@environment.base) : dep.source_file,
+          Digest::SHA256.file(dep.source_file).hexdigest
+        ]
       end
 
       @ecv = Digest::SHA1.base64digest(JSON.generate(f))
@@ -176,6 +180,7 @@ class Condenser
     def needs_reprocessing!
       @processed = false
       @pcv = nil
+      @cache_key = nil
       needs_reexporting!
     end
     
@@ -236,6 +241,7 @@ class Condenser
               data[:processors] << processor_klass.name
               @environment.load_processors(processor_klass)
 
+              @environment.logger.info { "Pre Processing #{self.filename} with #{processor.name}" }
               processor.call(@environment, data)
             end
           end
@@ -252,6 +258,17 @@ class Condenser
               data[:content_type] << to_mime_type
             end
           end
+          
+          if @environment.postprocessors.has_key?(data[:content_type].last)
+            @environment.postprocessors[data[:content_type].last].each do |processor|
+              processor_klass = (processor.is_a?(Class) ? processor : processor.class)
+              data[:processors] << processor_klass.name
+              @environment.load_processors(processor_klass)
+
+              @environment.logger.info { "Post Processing #{self.filename} with #{processor.name}" }
+              processor.call(@environment, data)
+            end
+          end
       
           if mime_types != @content_types
             raise ContentTypeMismatch, "mime type(s) \"#{@content_types.join(', ')}\" does not match requested mime type(s) \"#{data[:mime_types].join(', ')}\""
@@ -330,6 +347,15 @@ class Condenser
           data[:digest_name] = @environment.digestor.name.sub(/^.*::/, '').downcase
           data
         end
+
+        if @environment.build_cache.listening
+          # TODO we could skip file and all their depencies here if they are
+          # already in build_cache.@export_dependencies
+          all_export_dependencies.each do |sf|
+            @environment.build_cache.instance_variable_get(:@export_dependencies)[sf]&.add(self)
+          end
+        end
+
         Export.new(@environment, data)
       end
     end

data/lib/condenser/build_cache.rb

@@ -37,29 +37,47 @@ class Condenser
               globs << file.match(/([^\.]+)(\.|$)/).to_a[1]
               if path_match = @path.find { |p| file.start_with?(p) }
                 a = file.delete_prefix(path_match).match(/([^\.]+)(\.|$)/).to_a[1]
-                b = (File.dirname(a) + "/*")
+                b = File.join(File.dirname(a), "*")
               
                 globs << a << a.delete_prefix('/')
-                globs << a << b.delete_prefix('/')
+                globs << b << b.delete_prefix('/')
               end
             end
-            
+
+            others = []
             @map_cache&.delete_if do |k,v|
               if globs.any?{ |a| k.starts_with?(a) }
+                @export_dependencies[v.source_file]&.each do |a| 
+                  others << "/#{a.filename}".delete_suffix(File.extname(a.filename))
+                end
                 true
               else
                 false
               end
             end
-
+            @map_cache&.delete_if do |k,v|
+              others.any?{ |a| k.starts_with?(a) || k.starts_with?("/" + a) }
+            end
+            
+            others = []
             @lookup_cache.delete_if do |key, value|
               if globs.any?{ |a| key.starts_with?(a) }
+                value.each do |v|
+                  @export_dependencies[v.source_file]&.each do |a| 
+                    others << "/#{a.filename}".delete_suffix(File.extname(a.filename))
+                  end
+                end
                 value.each do |asset|
                   modified << asset.source_file
                 end
                 true
               end
             end
+            @lookup_cache&.delete_if do |k,v|
+              others.any?{ |a| k.starts_with?(a) || k.starts_with?("/" + a) }
+            end
+            
+
             
             removed.each do |file|
               @process_dependencies[file]&.delete_if do |asset|
@@ -127,7 +145,9 @@ class Condenser
       
       if value.nil?
         value = yield
-        self[key] = value
+        if (value.is_a?(Array) ? !value.empty? : value)
+          self[key] = value
+        end
       end
       
       value