condenser 1.0 → 1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 24f2e6a83798d231764a5940648a38a65f72c34e55e70e833292397bb98347a8
-  data.tar.gz: 384ceedcef19c3137ad969a255fe8519eb37ac20116c54879ccaa19a05a331c3
+  metadata.gz: 52e335c087aab2b9bb3ef4540e54bfc5b8de0a888cab8f095c2f1dd778458bd4
+  data.tar.gz: 566502264a1049f4ff5e33154caa56ef64ed74d14a8d72abd5e31628d59986c1
 SHA512:
-  metadata.gz: 906d4e278f46053c31bd55943fab2100468f604aa43472a98993ea71350384abde3b0181f08d8357c814f020c897a359fe496397012a402602f0cdc8777695ae
-  data.tar.gz: f96cfdc8570e74939b71a4257a3b8647495aac682f49cc5a60e39f653367fbf6e73fe77cd5965e1f92b4cab899cf3262be7533158cf4760d2e7e166a244fe0e7
+  metadata.gz: 5c0fdf66cb6b17ff6a519af5205061f37e3af2f12896e45c2798154c46ba2db3dea57e219595b33e5433a82db17751bc8444453c5bc0371fb27013e97157352d
+  data.tar.gz: 2486bd59e00c2d70ea683d2f69f1a71b0981b557144a08fd4965f9fe38a0c2da68d79e1c6fcba359718af3f9c26f4978619c3bb651f394cbb0db0172c450096f

data/lib/condenser/asset.rb

@@ -125,7 +125,7 @@ class Condenser
     
     def all_process_dependencies
       f = [@source_file]
-      all_dependenies(process_dependencies, [], :process_dependencies) do |dep|
+      all_dependenies(process_dependencies, Set.new, :process_dependencies) do |dep|
         f << dep.source_file
       end
       f
@@ -133,7 +133,7 @@ class Condenser
     
     def all_export_dependencies
       f = [@source_file]
-      all_dependenies(export_dependencies, [], :export_dependencies) do |dep|
+      all_dependenies(export_dependencies, Set.new, :export_dependencies) do |dep|
         f << dep.source_file
       end
       f
@@ -153,13 +153,13 @@ class Condenser
       return @pcv if @pcv
 
       f = []
-      all_dependenies(process_dependencies, [], :process_dependencies) do |dep|
+      all_dependenies(process_dependencies, Set.new, :process_dependencies) do |dep|
         f << [
           @environment.base ? dep.source_file.delete_prefix(@environment.base) : dep.source_file,
           Digest::SHA256.file(dep.source_file).hexdigest
         ]
       end
-
+      
       @pcv = Digest::SHA1.base64digest(JSON.generate(f))
     end
     
@@ -167,7 +167,7 @@ class Condenser
       return @ecv if @ecv
 
       f = []
-      all_dependenies(export_dependencies, [], :export_dependencies) do |dep|
+      all_dependenies(export_dependencies, Set.new, :export_dependencies) do |dep|
         f << [
           @environment.base ? dep.source_file.delete_prefix(@environment.base) : dep.source_file,
           Digest::SHA256.file(dep.source_file).hexdigest
@@ -206,9 +206,9 @@ class Condenser
             content_type: mime_types,
 
             map: nil,
-            linked_assets: [],
-            process_dependencies: [],
-            export_dependencies: [],
+            linked_assets: Set.new,
+            process_dependencies: Set.new,
+            export_dependencies: Set.new,
             
             processors: Set.new
           }
@@ -241,6 +241,7 @@ class Condenser
               data[:processors] << processor_klass.name
               @environment.load_processors(processor_klass)
 
+              @environment.logger.info { "Pre Processing #{self.filename} with #{processor.name}" }
               processor.call(@environment, data)
             end
           end
@@ -257,6 +258,17 @@ class Condenser
               data[:content_type] << to_mime_type
             end
           end
+          
+          if @environment.postprocessors.has_key?(data[:content_type].last)
+            @environment.postprocessors[data[:content_type].last].each do |processor|
+              processor_klass = (processor.is_a?(Class) ? processor : processor.class)
+              data[:processors] << processor_klass.name
+              @environment.load_processors(processor_klass)
+
+              @environment.logger.info { "Post Processing #{self.filename} with #{processor.name}" }
+              processor.call(@environment, data)
+            end
+          end
       
           if mime_types != @content_types
             raise ContentTypeMismatch, "mime type(s) \"#{@content_types.join(', ')}\" does not match requested mime type(s) \"#{data[:mime_types].join(', ')}\""
@@ -273,15 +285,14 @@ class Condenser
           @content_types = data[:content_type]
           @digest = data[:digest]
           @digest_name = data[:digest_name]
-          @linked_assets = data[:linked_assets]
-          @process_dependencies = data[:process_dependencies]
-          @export_dependencies = data[:export_dependencies]
+          @linked_assets = Set.new(data[:linked_assets])
+          @process_dependencies = Set.new(data[:process_dependencies])
+          @export_dependencies = Set.new(data[:export_dependencies])
           @default_export = data[:default_export]
           @exports = data[:exports]
           @processors = data[:processors]
           @processors_loaded = true
           @processed = true
-          
           data
         end
       end
@@ -292,9 +303,9 @@ class Condenser
       @content_types = result[:content_type]
       @digest = result[:digest]
       @digest_name = result[:digest_name]
-      @linked_assets = result[:linked_assets]
-      @process_dependencies = result[:process_dependencies]
-      @export_dependencies  = result[:export_dependencies]
+      @linked_assets = Set.new(result[:linked_assets])
+      @process_dependencies = Set.new(result[:process_dependencies])
+      @export_dependencies  = Set.new(result[:export_dependencies])
       @default_export = result[:default_export]
       @exports = result[:exports]
       @processors = result[:processors]
@@ -323,11 +334,15 @@ class Condenser
             export_dependencies: []
           }
         
-          if exporter = @environment.exporters[content_type]
-            exporter.call(@environment, data)
+          if @environment.exporters.has_key?(content_type)
+            @environment.exporters[content_type].each do |exporter|
+              @environment.logger.info { "Exporting #{self.filename} with #{exporter.name}" }
+              exporter.call(@environment, data)
+            end
           end
 
           if minifier = @environment.minifier_for(content_type)
+            @environment.logger.info { "Minifing #{self.filename} with #{minifier.name}" }
             minifier.call(@environment, data)
           end
         
@@ -335,6 +350,15 @@ class Condenser
           data[:digest_name] = @environment.digestor.name.sub(/^.*::/, '').downcase
           data
         end
+
+        if @environment.build_cache.listening
+          # TODO we could skip file and all their depencies here if they are
+          # already in build_cache.@export_dependencies
+          all_export_dependencies.each do |sf|
+            @environment.build_cache.instance_variable_get(:@export_dependencies)[sf]&.add(self)
+          end
+        end
+
         Export.new(@environment, data)
       end
     end

data/lib/condenser/build_cache.rb

@@ -33,7 +33,7 @@ class Condenser
             end
 
             globs = []
-            (added + removed).each do |file|
+            (added + removed + modified).each do |file|
               globs << file.match(/([^\.]+)(\.|$)/).to_a[1]
               if path_match = @path.find { |p| file.start_with?(p) }
                 a = file.delete_prefix(path_match).match(/([^\.]+)(\.|$)/).to_a[1]

data/lib/condenser/context.rb

@@ -35,7 +35,7 @@ class Condenser
       end
     end
 
-    attr_reader :environment, :filename
+    attr_reader :environment, :filename, :links, :dependencies
 
     def initialize(environment)
       @environment  = environment
@@ -110,29 +110,14 @@ class Condenser
     # `depend_on` allows you to state a dependency on a file without
     # including it.
     #
-    # This is used for caching purposes. Any changes made to
-    # the dependency file will invalidate the cache of the
-    # source file.
-    def depend_on(path)
-      if environment.absolute_path?(path) && environment.stat(path)
-        @dependencies << environment.build_file_digest_uri(path)
-      else
-        resolve(path)
-      end
-      nil
-    end
-
-    # `depend_on_asset` allows you to state an asset dependency
-    # without including it.
-    #
     # This is used for caching purposes. Any changes that would
     # invalidate the dependency asset will invalidate the source
-    # file. Unlike `depend_on`, this will recursively include
-    # the target asset's dependencies.
-    def depend_on_asset(path)
-      asset = environment.find!(path)
-      @dependencies << asset.source_file
-      asset
+    # file.
+    def depend_on(path)
+      d = environment.decompose_path(path)
+      @dependencies << [File.join(*d[0], d[1]), [d[3]]]
+
+      nil
     end
 
     # `depend_on_env` allows you to state a dependency on an environment
@@ -150,9 +135,8 @@ class Condenser
     #
     # Returns an Asset or nil.
     def link_asset(path)
-      asset = depend_on_asset(path)
-      @links << asset.path
-      asset
+      depend_on(path)
+      @links << path
     end
 
     # Returns a `data:` URI with the contents of the asset at the specified

data/lib/condenser/helpers/parse_helpers.rb

@@ -3,7 +3,7 @@ module Condenser::ParseHelpers
   attr_accessor :matched
 
   def eos?
-    @index >= @source.size
+    @index >= (@source.size - 1)
   end
 
   def scan_until(r)

data/lib/condenser/manifest.rb

@@ -71,7 +71,9 @@ class Condenser
 
       @data[asset.filename] = export.to_json
       outputs = export.write(@dir)
-      asset.linked_assets.each { |a| outputs += add_asset(a) }
+      asset.linked_assets.each do |la|
+        @environment.resolve(la).each { |a| outputs += add_asset(a) }
+      end
       outputs
     end
     

data/lib/condenser/minifiers/terser_minifier.rb

@@ -5,9 +5,8 @@ class Condenser::TerserMinifier < Condenser::NodeProcessor
     npm_install('terser')
     
     @options = options.merge({
-      warnings: true,
-      sourceMap: false,
-      keep_classnames: true
+      keep_classnames: true,
+      keep_fnames: true
     }).freeze
   end
 
@@ -28,13 +27,12 @@ class Condenser::TerserMinifier < Condenser::NodeProcessor
       const options = #{JSON.generate(opts)};
 
 
-      var result = Terser.minify(source, options);
-      if (result.error !== undefined) {
-        console.log(JSON.stringify({'error': result.error.name + ": " + result.error.message}));
-        process.exit(1);
-      } else {
+      Terser.minify(source, options).then((result) => {
         console.log(JSON.stringify(result));
-      }
+      }, (error) => {
+        console.log(JSON.stringify({'error': error.name + ": " + error.message}));
+        process.exit(1);
+      });
     JS
 
     exec_runtime_error(result['error']) if result['error']

data/lib/condenser/pipeline.rb

@@ -98,11 +98,16 @@ class Condenser
     end
     
     def register_exporter(mime_type, engine)
-      @exporters[mime_type] = engine
+      @exporters[mime_type] ||= []
+      @exporters[mime_type] << engine
     end
     
-    def unregister_exporter(mime_type, engine)
-      @exporters[mime_type] = nil
+    def unregister_exporter(mime_type, engine=nil)
+      if engine.nil?
+        @exporters[mime_type].clear
+      else
+        @exporters[mime_type]&.reject! { |e| e == engine || e.is_a?(engine) }
+      end
     end
     
     def register_minifier(mime_type, engine)

data/lib/condenser/processors/babel_processor.rb

@@ -4,7 +4,7 @@ class Condenser::BabelProcessor < Condenser::NodeProcessor
   
   attr_accessor :options
   
-  def initialize(dir = nil, options = {})
+  def initialize(dir = nil, **options)
     super(dir)
     
     options[:plugins] ||= [

data/lib/condenser/processors/css_media_combiner_processor.rb

@@ -0,0 +1,81 @@
+class Condenser::CSSMediaCombinerProcessor
+  
+  include Condenser::ParseHelpers
+  
+  def self.setup(env)
+  end
+  
+  def self.call(environment, input)
+    new.call(environment, input)
+  end
+
+  def reduce_media_query(queries)
+    output = ''
+    queries.each do |query, contents|
+      output << query if query
+      output << if contents.is_a?(Hash)
+        reduce_media_query(contents)
+      else
+        contents + '}'
+      end
+    end
+    output
+  end
+  
+  def call(environment, input)
+    seek(0)
+    @sourcefile = input[:source_file]
+    @source = input[:source]
+    @stack =  []
+    @selectors = []
+    @media_queries = {}
+
+    input[:source] = ''
+    while !eos?
+      output = if @selectors.empty?
+        input[:source]
+      else
+        (@selectors[0...-1].reduce(@media_queries) { |hash, selector| hash[selector] ||= {} }[@selectors.last] ||= '')
+      end
+      
+      case @stack.last
+      when :media_query
+        scan_until(/(@media[^\{]*{|\{|\})/)
+        case matched
+        when '{'
+          output << pre_match << matched
+          @stack << :statement
+        when '}'
+          output << pre_match
+          @stack.pop
+          @selectors.pop
+        else
+          output << pre_match
+          @selectors << matched.squish
+          @stack << :media_query
+        end
+      when :statement
+        scan_until(/(\{|\})/)
+        output << pre_match << matched
+        case matched
+        when '{'
+          @stack << :statement
+        when '}'
+          @stack.pop
+        end
+      else
+        case scan_until(/(@media[^\{]*{|\Z)/)
+        when ''
+          output << pre_match
+        else
+          output << pre_match
+          @selectors << matched.squish
+          @stack << :media_query
+        end
+      end
+    end
+
+    input[:source] << reduce_media_query(@media_queries)
+  end
+
+end

data/lib/condenser/processors/js_analyzer.rb

@@ -1,5 +1,3 @@
-require 'condenser/helpers/parse_helpers'
-
 class Condenser::JSAnalyzer
 
   include Condenser::ParseHelpers

data/lib/condenser/processors/node_processor.rb

@@ -8,6 +8,10 @@ class Condenser
     
     def self.setup(environment)
     end
+  
+    def name
+      self.class.name
+    end
 
     def self.call(environment, input)
       @instances ||= {}

data/lib/condenser/processors/purgecss_processor.rb

@@ -0,0 +1,72 @@
+require 'json'
+
+class Condenser::PurgeCSSProcessor < Condenser::NodeProcessor
+  
+  attr_accessor :options
+  
+  # Public: initialize with custom options.
+  #
+  # dir - String (path to node_modules directory)
+  # options - Hash
+  #   content - Array - html files to process
+  #       ex. [File.expand_path('./docs-src/**/*.erb'), File.expand_path('./docs-src/assets/javascripts/**/*.js')]
+  #
+  # Options are passed to PurgeCSS checkout [PurgeCSS Configurations](https://purgecss.com/configuration.html)
+  #
+  
+  def self.call(environment, input)
+    @instances ||= {}
+    @instances[environment] ||= new(environment.npm_path, {
+      content: [File.join(environment.base, '**/*.html'), File.join(environment.base, '**/*.js')]
+    })
+    @instances[environment].call(environment, input)
+  end
+  
+  
+  def initialize(dir = nil, options = {})
+    super(dir)
+    @options = options
+    npm_install('purgecss')
+  end
+  
+  def call(environment, input)
+    result = exec_runtime(<<-JS)
+      const { PurgeCSS } = require("#{File.join(npm_module_path('purgecss'))}")
+      const options = #{@options.to_json}
+      options.css = [{
+        raw: #{JSON.dump(input[:source])}
+      }]
+      if(options.safelist) {
+        options.safelist = options.safelist.map(s => {
+          if(s[0] == "/" && s[s.length - 1] == "/") {
+            return new RegExp(s.slice(1, -1))
+          }
+          return s
+        })
+      }
+      if(!options.defaultExtractor) {
+        options.defaultExtractor = content => content.match(/[\\w\\-\\/\\:]+(?<!:)/g) || []
+      }
+      const result = new PurgeCSS().purge(options)
+      try {
+        result.then(
+          r => console.log(JSON.stringify({
+            success: r[0]
+          })),
+          function(e) {console.log(JSON.stringify({'error': [e.name, e.message, e.stack]}))}
+        )
+      } catch(e) {
+        console.log(JSON.stringify({'error': [e.name, e.message, e.stack]}));
+      }
+    JS
+    if result['error']
+      if result['error'][0] == 'SyntaxError'
+        raise exec_syntax_error(result['error'][1], "/assets/#{input[:filename]}")
+      else
+        raise exec_runtime_error(result['error']["0"]["name"] + ": " + result['error']["0"]["reason"])
+      end
+    else
+      input[:source] = result["success"]["css"]
+    end
+  end
+end

data/lib/condenser/processors/rollup_processor.rb

@@ -7,7 +7,7 @@ class Condenser::RollupProcessor < Condenser::NodeProcessor
   
   def initialize(dir = nil, options = {})
     super(dir)
-    npm_install('rollup', 'rollup-plugin-commonjs', 'rollup-plugin-node-resolve')
+    npm_install('rollup', '@rollup/plugin-commonjs', '@rollup/plugin-node-resolve')
     
     @options = options.freeze
   end
@@ -50,7 +50,12 @@ class Condenser::RollupProcessor < Condenser::NodeProcessor
             return '';
           } catch(e) {
             if (e.name === "SyntaxError") {
-              if (e.message.startsWith('Unexpected token { in JSON at position ')) {
+              if (e.message.startsWith('Unexpected non-whitespace character after JSON at position ')) {
+                var pos = parseInt(e.message.slice(59));
+                emitMessages(buffer.slice(0,pos));
+                return emitMessages(buffer.slice(pos));
+              } else if (e.message.startsWith('Unexpected token { in JSON at position ')) {
+                // This can be removed, once dropping support for node <= v18
                 var pos = parseInt(e.message.slice(39));
                 emitMessages(buffer.slice(0,pos));
                 return emitMessages(buffer.slice(pos));
@@ -70,8 +75,8 @@ class Condenser::RollupProcessor < Condenser::NodeProcessor
         });
         
         const rollup = require("#{npm_module_path('rollup')}");
-        const commonjs = require("#{npm_module_path('rollup-plugin-commonjs')}");
-        const nodeResolve = require("#{npm_module_path('rollup-plugin-node-resolve')}");
+        const commonjs = require("#{npm_module_path('@rollup/plugin-commonjs')}");
+        const nodeResolve = require("#{npm_module_path('@rollup/plugin-node-resolve')}").nodeResolve;
         var rid = 0;
         var renderStack = {};
         var nodeResolver = null;
@@ -95,9 +100,8 @@ class Condenser::RollupProcessor < Condenser::NodeProcessor
             mainFields: ['module', 'main'],
             // modulesOnly: true,
             // preferBuiltins: false,
-            customResolveOptions: {
-              moduleDirectory: '#{npm_module_path}'
-            }
+            moduleDirectories: [],
+            modulePaths: ['#{npm_module_path}']
           });
         }
 
@@ -105,9 +109,9 @@ class Condenser::RollupProcessor < Condenser::NodeProcessor
         inputOptions.plugins = [];
         inputOptions.plugins.push({
           name: 'condenser',
-          resolveId: function (importee, importer) {
+          resolveId: function (importee, importer, options) {
             if (importee.startsWith('\\0') || (importer && importer.startsWith('\\0'))) {
-              return;
+              return null;
             }
 
             if (!(importer in renderStack)) {
@@ -115,24 +119,15 @@ class Condenser::RollupProcessor < Condenser::NodeProcessor
             }
 
             return request('resolve', [importee, importer]).then((value) => {
-              if (nodeResolver && (value === null || value === undefined)) {
-                return nodeResolver.resolveId.call(this, importee, importer).then((value) => {
-                  if (!(value === null || value === undefined) && !renderStack[importer].includes(value.id)) {
-                    renderStack[importer].push(value.id);
-                  }
-                  return value;
-                });
-              }
-
-              if (!(value === null || value === undefined) && !renderStack[importer].includes(value)) {
-                renderStack[importer].push(value);
-              }
-              return value;
+                if (!(value === null || value === undefined) && !renderStack[importer].includes(value)) {
+                  renderStack[importer].push(value);
+                }
+                return value;
             });
           },
           load: function(id) {
             if (id.startsWith('\\0')) {
-              return;
+              return null;
             }
 
             return request('load', [id]).then(function(value) {
@@ -140,17 +135,19 @@ class Condenser::RollupProcessor < Condenser::NodeProcessor
             });
           }
         });
+        
         inputOptions.plugins.push(nodeResolver);
         inputOptions.plugins.push(commonjs());
         
-        inputOptions.plugins.push({
-          name: 'nullHanlder',
-          resolveId: function (importee, importer) {
-            request('error', ["AssetNotFound", importee, importer, renderStack]).then(function(value) {
-              process.exit(1);
-            });
-          }
-        });
+        // inputOptions.plugins.push({
+        //   name: 'nullHanlder',
+        //   resolveId: function (importee, importer, options) {
+        //     request('log', [importee, importer, options])
+        //     // request('error', ["AssetNotFound", importee, importer, renderStack]).then(function(value) {
+        //     //   process.exit(1);
+        //     // });
+        //   }
+        // });
 
         const outputOptions = #{JSON.generate(output_options)};
 
@@ -204,10 +201,16 @@ class Condenser::RollupProcessor < Condenser::NodeProcessor
               @entry
             elsif importee.start_with?('@babel/runtime') || importee.start_with?('core-js-pure') || importee.start_with?('regenerator-runtime')
               x = File.join(npm_module_path, importee.gsub(/^\.\//, File.dirname(importer) + '/')).sub('/node_modules/regenerator-runtime', '/node_modules/regenerator-runtime/runtime.js')
-              x = "#{x}.js" if !x.end_with?('.js')
+              x = "#{x}.js" if !x.end_with?('.js', '.mjs')
+              File.file?(x) ? x : (x.delete_suffix('.js') + "/index.js")
+            elsif npm_module_path && importee&.start_with?(npm_module_path)
+              x = importee.end_with?('.js', '.mjs') ? importee : "#{importee}.js"
+              x = (x.delete_suffix('.js') + "/index.js") if !File.file?(x)
+              x
+            elsif importee.start_with?('.') && importer.start_with?(npm_module_path)
+              x = File.expand_path(importee, File.dirname(importer))
+              x = "#{x}.js" if !x.end_with?('.js', '.mjs')
               File.file?(x) ? x : (x.delete_suffix('.js') + "/index.js")
-            elsif npm_module_path && importer.start_with?(npm_module_path)
-              nil
             elsif importee.end_with?('*')
               File.join(File.dirname(importee), '*')
             else
@@ -217,8 +220,6 @@ class Condenser::RollupProcessor < Condenser::NodeProcessor
             importee = message['args'].first
             if importee == @entry
               { code: @input[:source], map: @input[:map] }
-            elsif importee.start_with?(npm_module_path)
-              { code: File.read(importee), map: nil }
             elsif importee.end_with?('*')
               importees = @environment.resolve(importee, importer ? File.dirname(@entry == importer ? @input[:source_file] : importer) : nil, accept: @input[:content_types].last)
               code = ""
@@ -263,7 +264,6 @@ class Condenser::RollupProcessor < Condenser::NodeProcessor
           # when 'get_cache'
           #   io.write(JSON.generate({rid: message['rid'], return: [(@environment.cache.get('rollup') || '{}')] }))
           end
-
           io.write(JSON.generate({rid: message['rid'], return: ret}))
         end
       end

data/lib/condenser/resolve.rb

@@ -38,9 +38,7 @@ class Condenser
           end
         
           paths.each do |path|
-            glob = path
-            glob = File.join(glob, dirname) if dirname
-            glob = File.join(glob, basename)
+            glob = File.join(*[path, dirname, basename].compact)
             glob << '.*' unless glob.end_with?('*')
           
             Dir.glob(glob).sort.each do |f|

data/lib/condenser/templating_engine/ejs.rb

@@ -1,4 +1,4 @@
-class Condenser::EjsTemplare
+class Condenser::EjsTemplate
 
   def self.setup(environment)
     require 'ejs' unless defined?(::EJS)