condenser 0.0.12 → 1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 31107c23d48e2044816158df8cc737fdea156e2227763f4ad1409dcc36406388
-  data.tar.gz: 3936450c32a7340a50345cc389351c2894b3263ca7e1c8f54563336a0977a445
+  metadata.gz: 24f2e6a83798d231764a5940648a38a65f72c34e55e70e833292397bb98347a8
+  data.tar.gz: 384ceedcef19c3137ad969a255fe8519eb37ac20116c54879ccaa19a05a331c3
 SHA512:
-  metadata.gz: 0cf137d71f65b067b88e480756b74814e9e88df0342574fc7f466b87aa64515433a9c1dbf827c49aa1d816d20c0c640be6a97d55dba814aeeceade3ed200e37b
-  data.tar.gz: cf154ab552d503a93ce487f1c7f9561faa07391c7e29d7190a66ca7e3bd9c6a842dd574193720484a84bcfba6430be58665f141519c5a4b024621169c4c89fe7
+  metadata.gz: 906d4e278f46053c31bd55943fab2100468f604aa43472a98993ea71350384abde3b0181f08d8357c814f020c897a359fe496397012a402602f0cdc8777695ae
+  data.tar.gz: f96cfdc8570e74939b71a4257a3b8647495aac682f49cc5a60e39f653367fbf6e73fe77cd5965e1f92b4cab899cf3262be7533158cf4760d2e7e166a244fe0e7

data/README.md

@@ -8,18 +8,21 @@ and SCSS.
 
 ## Installation
 
-Install Sprockets from RubyGems:
+In your project's `Gemfile` with Bundler:
 
-``` sh
-$ gem install condenser
+``` ruby
+gem 'condenser'
 ```
 
-Or include it in your project's `Gemfile` with Bundler:
+Or via Ruby Gems
 
-``` ruby
-gem 'condenser'
+``` sh
+$ gem install condenser
 ```
 
+If you are using Condenser with Rails, instead of the `condenser` gem use the
+`condenser-rails` gem.
+
 ## Guides
 
 For most people interested in Condenser, you will want to see the README below.
@@ -71,7 +74,8 @@ contains the directory `app/assets/javascripts`:
 
 ## File Order Processing
 
-By default files are processed in alphabetical order. This behavior can impact your asset compilation when one asset needs to be loaded before another.
+By default files are processed in alphabetical order. This behavior can impact
+your asset compilation when one asset needs to be loaded before another.
 
 For example if you have an `application.js` and it loads another directory
 
@@ -81,7 +85,8 @@ import initializers from 'config/initializers/*';
 initializers.forEach((i) => i());
 ```
 
-The files in that directory will be loaded in alphabetical order. If the directory looks like this:
+The files in that directory will be loaded in alphabetical order. If the directory
+looks like this:
 
 ```sh
 $ ls -1 config/initializers/
@@ -91,7 +96,10 @@ beta.js
 gamma.js
 ```
 
-Then `alpha.js` will be loaded before either of the other two. This can be a problem if `gamma.js` needs to be called before `alpha.js`. For files that are ordering dependent you can either require individual files manually:
+Then `alpha.js` will be loaded before either of the other two. This can be a
+problem if `gamma.js` needs to be called before `alpha.js`. For files that are
+order dependent you can either rename the files or require individual files
+manually:
 
 ```js
 import alpha from 'config/initializers/alpha';
@@ -103,13 +111,42 @@ alpha();
 beta();
 ```
 
+## Directives
+
+Sometimes you need to tell Condenser that a file depends on other files. For
+example a `js.erb` file maybe dynamically update depending on the contents of a
+folder as follows:
+
+```js
+console.log([<%= Dir.children("#{asset_path}/models").sort.map(&:inspect).join(', ') %>]);
+```
+
+To tell Condenser that this file depends on `models/*` simply add the depends_on
+directive:
+
+```js
+// depends_on models/*.js
+
+console.log([<%= Dir.children("#{asset_path}/models").sort.map(&:inspect).join(', ') %>]);
+```
+
+The directives must be the first portion of the file.
+
 ## Cache
 
-Compiling assets is slow. It requires a lot of disk use to pull assets off of hard drives, a lot of RAM to manipulate those files in memory, and a lot of CPU for compilation operations. Because of this Condenser has a cache to speed up asset compilation times. That's the good news. The bad news, is that Condenser has a cache and if you've found a bug it's likely going to involve the cache.
+Compiling assets is slow. It requires a lot of disk use to pull assets off of
+hard drives, a lot of RAM to manipulate those files in memory, and a lot of CPU
+for compilation operations. Because of this Condenser has a cache to speed up
+asset compilation times. That's the good news. The bad news, is that Condenser
+has a cache and if you've found a bug it's likely going to involve the cache.
 
-By default Condenser uses the file system to cache assets. It makes sense that Condenser does not want to generate assets that already exist on disk in `public/assets`, what might not be as intuitive is that Condenser needs to cache "partial" assets.
+By default Condenser uses the file system to cache assets. It makes sense that
+Condenser does not want to generate assets that already exist on disk in
+`public/assets`, what might not be as intuitive is that Condenser needs to cache
+"partial" assets.
 
-For example if you have an `application.js` and it is made up of `a.js`, `b.js`, all the way to `z.js`
+For example if you have an `application.js` and it is made up of `a.js`, `b.js`,
+all the way to `z.js`
 
 ```js
 import 'a';
@@ -118,19 +155,27 @@ import 'b';
 import 'z';
 ```
 
-The first time this file is compiled the `application.js` output will be written to disk, but also intermediary compiled files for `a.js` etc. will be written to the cache directory (usually `tmp/cache/assets`).
+The first time this file is compiled the `application.js` output will be written
+to disk, but also intermediary compiled files for `a.js` etc. will be written to
+the cache directory (usually `tmp/cache/assets`).
 
-So, if `b.js` changes it will get recompiled. However instead of having to recompile the other files from `a.js` to `z.js` since they did not change, we can use the prior intermediary files stored in the cached values . If these files were expensive to generate, then this "partial" asset cache strategy can save a lot of time.
+So, if `b.js` changes it will get recompiled. However instead of having to
+recompile the other files from `a.js` to `z.js` since they did not change,
+we can use the prior intermediary files stored in the cached values . If these
+files were expensive to generate, then this "partial" asset cache strategy can
+save a lot of time.
 
-Directives such as `import` in Javascript and `@import` in SCSS tell Condenser what assets need to be re-compiled when a file changes. Files are considered "fresh" based on their inode number, mtime, size and a combination of cache keys.
+Directives such as `import` in Javascript and `@import` in SCSS tell Condenser
+what assets need to be re-compiled when a file changes. Files are considered
+"fresh" based on their inode number, mtime, size and a combination of cache keys.
 
-In Rails you can force a "clean" install by clearing the `public/assets` and `tmp/cache/assets` directories.
+In Rails you can force a "clean" install by clearing the `public/assets` and
+`tmp/cache/assets` directories.
 
 ### Invoking Ruby with ERB
 
-Condenser provides an ERB engine for preprocessing assets using
-embedded Ruby code. Append `.erb` to a CSS or JavaScript asset's
-filename to enable the ERB engine.
+Condenser provides an ERB engine for preprocessing assets using embedded Ruby
+code. Append `.erb` to a CSS or JavaScript asset's filename to enable the ERB engine.
 
 For example if you have an `app/application/javascripts/app_name.js.erb`
 you could have this in the template
@@ -179,7 +224,9 @@ convert `foo.scss` to `foo.css`.
 
 ## Javascript, ES#, & ES Modules
 
-Condenser transforms Javascript for the browser by transpiling all the files `.js` through [babel](https://babeljs.io) and bundled together via [rollup.js](https://rollupjs.org/).
+Condenser transforms Javascript for the browser by transpiling all the files
+`.js` through [babel](https://babeljs.io) and bundled together via
+[rollup.js](https://rollupjs.org/).
 
 ```js
 // app/assets/javascript/application.js
@@ -189,7 +236,9 @@ var square = (n) => n * n
 console.log(square);
 ```
 
-Start a Rails server in development mode and visit `localhost:3000/assets/application.js`, and this asset will be transpiled to JavaScript:
+Start a Rails server in development mode and visit
+`localhost:3000/assets/application.js`, and this asset will be transpiled to
+JavaScript:
 
 ```js
 var square = function square(n) {

data/lib/condenser.rb

@@ -15,29 +15,38 @@ class Condenser
   
   prepend Environment, Pipeline, Resolve
   
-  autoload :BabelProcessor,   'condenser/processors/rollup_processor'
-  autoload :RollupProcessor,  'condenser/processors/babel_processor'
+  autoload :BabelProcessor,   'condenser/processors/babel_processor'
+  autoload :RollupProcessor,  'condenser/processors/rollup_processor'
+  autoload :JSAnalyzer,       'condenser/processors/js_analyzer'
+  autoload :NodeProcessor,    'condenser/processors/node_processor'
   autoload :UglifyMinifier,   'condenser/minifiers/uglify_minifier'
+  autoload :TerserMinifier,   'condenser/minifiers/terser_minifier'
   autoload :Erubi,            'condenser/templating_engine/erb'
   autoload :SassMinifier,     'condenser/minifiers/sass_minifier'
   autoload :SassTransformer,  'condenser/transformers/sass_transformer'
   autoload :ScssTransformer,  'condenser/transformers/sass_transformer'
-  autoload :EjsTransformer,   'condenser/transformers/ejs'
+  autoload :EjsTemplare,      'condenser/templating_engine/ejs'
+  autoload :JstTransformer,   'condenser/transformers/jst_transformer'
   autoload :FileWriter,       'condenser/writers/file_writer'
   autoload :ZlibWriter,       'condenser/writers/zlib_writer'
+  autoload :BrotliWriter,     'condenser/writers/brotli_writer'
   autoload :BuildCache,       'condenser/build_cache'
   
   def self.configure(&block)
     instance_eval(&block)
   end
   
-  attr_accessor :logger, :digestor
+  attr_accessor :logger, :digestor, :base
   
-  def initialize(*path, logger: nil, digestor: nil, cache: nil, pipeline: nil, &block)
+  # base: If base is passed assets cache_keys will be realitve to this.
+  #       This allows deploy systems like Capistrano to take advantage of the
+  #       cache even though it precompiles assets in a different folder
+  def initialize(*path, logger: nil, digestor: nil, cache: nil, pipeline: nil, npm_path: nil, base: nil, &block)
     @logger = logger || Logger.new($stdout, level: :info)
     @path = []
-    @npm_path = nil
     append_path(path)
+    self.npm_path = npm_path
+    @base = base
     @cache = cache || Cache::MemoryStore.new
     @build_cc = 0
     self.digestor = digestor || Digest::SHA256
@@ -46,14 +55,18 @@ class Condenser
       configure(&block)
     elsif pipeline != false
       self.configure do
+        # register_preprocessor 'application/javascript', Condenser::JSAnalyzer
         register_preprocessor 'application/javascript', Condenser::BabelProcessor
         register_exporter     'application/javascript', Condenser::RollupProcessor
         register_minifier     'application/javascript', Condenser::UglifyMinifier
-      
+        
         register_minifier  'text/css', Condenser::SassMinifier
       
         register_writer Condenser::FileWriter.new
         register_writer Condenser::ZlibWriter.new
+        if Gem::Specification::find_all_by_name('brotli').any?
+          register_writer Condenser::BrotliWriter.new
+        end
       end
     end
   end
@@ -130,8 +143,12 @@ Condenser.configure do
   register_mime_type    'application/javascript', extension: '.js', charset: :unicode
   
   # EJS
-  register_mime_type    'application/ejs', extensions: %w(.ejs .jst.ejs), charset: :unicode
-  register_transformer  'application/ejs', 'application/javascript', Condenser::EjsTransformer
+  register_mime_type 'application/ejs', extensions: '.ejs', charset: :unicode
+  register_template  'application/ejs', Condenser::EjsTemplare
+  
+  # JST
+  register_mime_type    'application/jst', extensions: '.jst', charset: :unicode
+  register_transformer  'application/jst', 'application/javascript', Condenser::JstTransformer
   
   # Writers
   register_mime_type 'application/gzip',    extensions: %w(.gz .gzip)

data/lib/condenser/asset.rb

@@ -10,10 +10,10 @@ class Condenser
     include EncodingUtils
     
     attr_reader :environment, :filename, :content_types, :source_file, :source_path
-    attr_reader :linked_assets, :content_types_digest
+    attr_reader :linked_assets, :content_types_digest, :exports
     attr_writer :source, :sourcemap
 
-    attr_accessor :imports, :processed, :export
+    attr_accessor :imports, :processed
     
     def initialize(env, attributes={})
       @environment    = env
@@ -51,12 +51,8 @@ class Condenser
       [dirname, basename].compact.join('/')
     end
     
-    def stat
-      @stat ||= File.stat(@source_file)
-    end
-    
     def restat!
-      @stat = File.stat(@source_file)
+      @stat = nil
     end
     
     def inspect
@@ -71,10 +67,10 @@ class Condenser
         process
         @process_dependencies
       end
-      
+    
       d = []
       deps.each do |i|
-        i = [i, nil] if i.is_a?(String)
+        i = [i, @content_types] if i.is_a?(String)
         @environment.resolve(i[0], File.dirname(@source_file), accept: i[1]).each do |asset|
           d << asset
         end
@@ -90,7 +86,7 @@ class Condenser
       
       d = []
       deps.each do |i|
-        i = [i, nil] if i.is_a?(String)
+        i = [i, @content_types] if i.is_a?(String)
         @environment.resolve(i[0], File.dirname(@source_file), accept: i[1]).each do |asset|
           d << asset
         end
@@ -144,9 +140,11 @@ class Condenser
     end
     
     def cache_key
-      Digest::SHA1.base64digest(JSON.generate([
+      @cache_key ||= Digest::SHA1.base64digest(JSON.generate([
         Condenser::VERSION,
-        @source_file,
+        @environment.pipline_digest,
+        @environment.base ? @source_file.delete_prefix(@environment.base) : @source_file,
+        Digest::SHA256.file(@source_file).hexdigest,
         @content_types_digest
       ]))
     end
@@ -154,9 +152,12 @@ class Condenser
     def process_cache_version
       return @pcv if @pcv
 
-      f = [stat.ino, stat.mtime.to_f, stat.size]
+      f = []
       all_dependenies(process_dependencies, [], :process_dependencies) do |dep|
-        f << [dep.source_file, dep.stat.ino, dep.stat.mtime.to_f, dep.stat.size]
+        f << [
+          @environment.base ? dep.source_file.delete_prefix(@environment.base) : dep.source_file,
+          Digest::SHA256.file(dep.source_file).hexdigest
+        ]
       end
 
       @pcv = Digest::SHA1.base64digest(JSON.generate(f))
@@ -165,9 +166,12 @@ class Condenser
     def export_cache_version
       return @ecv if @ecv
 
-      f = [stat.ino, stat.mtime.to_f, stat.size]
+      f = []
       all_dependenies(export_dependencies, [], :export_dependencies) do |dep|
-        f << [dep.source_file, dep.stat.ino, dep.stat.mtime.to_f, dep.stat.size]
+        f << [
+          @environment.base ? dep.source_file.delete_prefix(@environment.base) : dep.source_file,
+          Digest::SHA256.file(dep.source_file).hexdigest
+        ]
       end
 
       @ecv = Digest::SHA1.base64digest(JSON.generate(f))
@@ -176,6 +180,7 @@ class Condenser
     def needs_reprocessing!
       @processed = false
       @pcv = nil
+      @cache_key = nil
       needs_reexporting!
     end
     

data/lib/condenser/build_cache.rb

@@ -1,9 +1,10 @@
 class Condenser
   class BuildCache
     
-    attr_reader :semaphore, :listening
+    attr_reader :semaphore, :listening, :logger
     
-    def initialize(path, listen: {})
+    def initialize(path, logger:, listen: {})
+      @logger = logger
       @path = path
       @map_cache = {}
       @lookup_cache = {}
@@ -22,43 +23,82 @@ class Condenser
         @semaphore = Mutex.new
         @listener = Listen.to(*path) do |modified, added, removed|
           @semaphore.synchronize do
-            added = added.reduce([]) do |rt, added_file|
-              rt << added_file.match(/([^\.]+)(\.|$)/).to_a[1]
-              if path_match = @path.find { |p| added_file.start_with?(p) }
-                a = added_file.delete_prefix(path_match).match(/([^\.]+)(\.|$)/).to_a[1]
-                b = (File.dirname(a) + "/*")
+            @logger.debug { "build cache semaphore locked by #{Thread.current.object_id}" }
+            @logger.debug do
+              (
+                removed.map { |f| "Asset removed: #{f}" } +
+                added.map { |f| "Asset created: #{f}" } +
+                modified.map { |f| "Asset updated: #{f}" }
+              ).join("\n")
+            end
+
+            globs = []
+            (added + removed).each do |file|
+              globs << file.match(/([^\.]+)(\.|$)/).to_a[1]
+              if path_match = @path.find { |p| file.start_with?(p) }
+                a = file.delete_prefix(path_match).match(/([^\.]+)(\.|$)/).to_a[1]
+                b = File.join(File.dirname(a), "*")
               
-                rt << a << a.delete_prefix('/')
-                rt << a << b.delete_prefix('/')
+                globs << a << a.delete_prefix('/')
+                globs << b << b.delete_prefix('/')
               end
             end
-          
-            removed.each do |file|
-              @map_cache&.delete_if do |k,v|
-                v.source_file == file
-              end
-            
-              @process_dependencies[file]&.delete_if do |asset|
-                asset.source_file == file
-              end
-            
-              @export_dependencies[file]&.delete_if do |asset|
-                asset.source_file == file
+
+            others = []
+            @map_cache&.delete_if do |k,v|
+              if globs.any?{ |a| k.starts_with?(a) }
+                @export_dependencies[v.source_file]&.each do |a| 
+                  others << "/#{a.filename}".delete_suffix(File.extname(a.filename))
+                end
+                true
+              else
+                false
               end
             end
-
+            @map_cache&.delete_if do |k,v|
+              others.any?{ |a| k.starts_with?(a) || k.starts_with?("/" + a) }
+            end
+            
+            others = []
             @lookup_cache.delete_if do |key, value|
-              if added.any?{ |a| key.starts_with?(a) }
+              if globs.any?{ |a| key.starts_with?(a) }
+                value.each do |v|
+                  @export_dependencies[v.source_file]&.each do |a| 
+                    others << "/#{a.filename}".delete_suffix(File.extname(a.filename))
+                  end
+                end
                 value.each do |asset|
                   modified << asset.source_file
                 end
                 true
               end
             end
-            @map_cache&.delete_if do |k,v|
-              added.any?{ |a| k.starts_with?(a) }
+            @lookup_cache&.delete_if do |k,v|
+              others.any?{ |a| k.starts_with?(a) || k.starts_with?("/" + a) }
             end
-          
+            
+
+            
+            removed.each do |file|
+              @process_dependencies[file]&.delete_if do |asset|
+                if asset.source_file == file
+                  true
+                else
+                  asset.needs_reprocessing!
+                  false
+                end
+              end
+            
+              @export_dependencies[file]&.delete_if do |asset|
+                if asset.source_file == file
+                  true
+                else
+                  asset.needs_reexporting!
+                  false
+                end
+              end
+            end
+            
             modified.each do |file|
               @process_dependencies[file]&.each do |asset|
                 asset.needs_reprocessing!
@@ -68,7 +108,8 @@ class Condenser
                 asset.needs_reexporting!
               end
             end
-          
+
+            @logger.debug { "build cache semaphore unlocked by #{Thread.current.object_id}" }
           end
         end
         @listener.start
@@ -104,7 +145,9 @@ class Condenser
       
       if value.nil?
         value = yield
-        self[key] = value
+        if (value.is_a?(Array) ? !value.empty? : value)
+          self[key] = value
+        end
       end
       
       value