condenser 1.3 → 1.5

data/lib/condenser/processors/rollup_processor.rb

@@ -1,38 +1,99 @@
+# frozen_string_literal: true
+
 require 'json'
 require 'tmpdir'
 
-class Condenser::RollupProcessor < Condenser::NodeProcessor
+class Condenser::RollupProcessor
+  
+  @@setup = false
   
-  attr_accessor :options
+  def self.setup(environment)
+    install_npm_packages(environment.npm_path)
+  end
   
-  def initialize(dir = nil, options = {})
-    super(dir)
-    npm_install('rollup', '@rollup/plugin-commonjs', '@rollup/plugin-node-resolve')
+  def self.install_npm_packages(npm_path)
+    return if @@setup
     
-    @options = options.freeze
+    ::Condenser::NodeProcessor.new(npm_path).npm_install(
+      'rollup',
+      '@rollup/plugin-commonjs',
+      '@rollup/plugin-node-resolve',
+      '@rollup/plugin-replace'
+    )
+    @@setup = true
+  end
+  
+  def self.call(environment, input)
+    new(environment.npm_path).call(environment, input)
+  end
+
+  def name
+    self.class.name
+  end
+  
+  def options
+    {prefix: @prefix, dynamic_imports: @dynamic_imports}
+  end
+  
+  # @param prefix  [string] string to prefix to the url of dynamic imports
+  # @param imports [symbol] :inline will inline all the imports in the 
+  #                        output file if possible.
+  #                        false will keep all the imports as imports.
+  #                        :local will not inline URLS.
+  # @param dynamic_imports [symbol] Default is :local
+  #    :inline - Inline dynamic imports into the output file if possible.
+  #    :local  - Same as :inline except for URLs are kept as URLs
+  #    :keep   - Keep the dynamic imports but rewrite the import as a URL
+  def initialize(dir = nil, prefix: nil, dynamic_imports: :inline)
+    self.class.install_npm_packages(dir)
+    @npm_dir = dir
+    @prefix = prefix
+    @dynamic_imports = dynamic_imports
   end
 
   def call(environment, input)
-    @environment = environment
-    @input = input
+    Runner.new(@npm_dir, prefix: @prefix, dynamic_imports: @dynamic_imports).call(environment, input)
+  end
+  
+  
+  class Runner < Condenser::NodeProcessor
+    def initialize(dir = nil, prefix: nil, dynamic_imports: :inline)
+      super(dir)
+      @prefix = prefix
+      @dynamic_imports = dynamic_imports
+    end
+
+    def call(environment, input)
+      @environment = environment
+      @input = input
+    
+      @input_dir = File.realdirpath(Dir.mktmpdir)
+      @output_dir = File.realdirpath(Dir.mktmpdir)
+    
+      @entry = File.join(@input_dir, 'entry.js')
     
-    Dir.mktmpdir do |output_dir|
-      @entry = File.join(output_dir, 'entry.js')
       input_options = {
         input: @entry
       }
+    
       output_options = {
-        file: File.join(output_dir, 'result.js'),
-        format: 'iife',
-        # output: { sourcemap: true, format: 'iife' },
+        dir: @output_dir,
+        format: 'es',
         globals: [],
-        sourcemap: true
+        sourcemap: false,
+        inlineDynamicImports: true,
+        generatedCode: {
+          arrowFunctions: true,
+          constBindings: true,
+          objectShorthand: true,
+          preset: 'es2015',
+        }
       }
+
       if input[:source] =~ /export\s+{[^}]+};?\z/i
         output_options[:name] = File.basename(input[:filename], ".*").capitalize
-        # output_options[:output][:name] = File.basename(input[:filename], ".*").capitalize
       end
-      
+
       exec_runtime(<<-JS, @entry)
         const fs    = require('fs');
         const path  = require('path');
@@ -73,14 +134,19 @@ class Condenser::RollupProcessor < Condenser::NodeProcessor
           buffer += chunk;
           buffer = emitMessages(buffer);
         });
-        
+      
         const rollup = require("#{npm_module_path('rollup')}");
         const commonjs = require("#{npm_module_path('@rollup/plugin-commonjs')}");
         const nodeResolve = require("#{npm_module_path('@rollup/plugin-node-resolve')}").nodeResolve;
+        const asyncWalk = require("#{npm_module_path('estree-walker')}").asyncWalk;
+        const MagicString = require("#{npm_module_path('magic-string')}");
+        const replace = require("#{npm_module_path('@rollup/plugin-replace')}");
+      
         var rid = 0;
         var renderStack = {};
+        var dynamicImports = {};
         var nodeResolver = null;
-        
+      
         function request(method, args) {
           var trid = rid;
           rid += 1;
@@ -107,8 +173,58 @@ class Condenser::RollupProcessor < Condenser::NodeProcessor
 
         const inputOptions = #{JSON.generate(input_options)};
         inputOptions.plugins = [];
-        inputOptions.plugins.push({
+        
+        inputOptions.plugins.push(replace({
+            preventAssignment: true,
+            values: {
+              "process.env.NODE_ENV": JSON.stringify("production")
+            }
+        }), {
+          name: 'transform',
+          transform: async function(code, id) {
+            const magicString = new MagicString(code);
+            const ast = this.parse(code, { sourceType: 'module' });
+
+            await asyncWalk(ast, {
+              async enter(node) {
+                if(['ImportExpression'].includes(node.type) && node.source.type == 'Literal') {
+                  const asset = await request('resolveDynamicImport', [node.source.value, id]);
+
+                  if (asset.export) {
+                    dynamicImports[asset.source] = asset;
+                    magicString.update(node.source.start, node.source.end, `'${asset.source}'`);
+                  } else {
+                    dynamicImports[node.source.value] = asset;
+                  }
+                } else if (node.type == 'ImportDeclaration' ) {
+                  if (node.value == '@arcgis/lumina/controllers') {
+                    magicString.update(node.start, node.end, `'@arcgis/components-controllers'`);
+                  }
+                }
+              }
+            });
+
+            return {
+              code: magicString.toString(),
+              map: null
+            };
+          }
+        }, {
           name: 'condenser',
+          resolveDynamicImport: function (importee, importer, options) {
+            const asset = dynamicImports[importee];
+            request('resolve!!!!', [importee, dynamicImports]);
+
+            if (!asset) {
+              return;
+            }
+          
+            if (asset.export) {
+              return false;
+            }
+          
+            return asset.source;
+          },
           resolveId: function (importee, importer, options) {
             if (importee.startsWith('\\0') || (importer && importer.startsWith('\\0'))) {
               return null;
@@ -125,36 +241,32 @@ class Condenser::RollupProcessor < Condenser::NodeProcessor
                 return value;
             });
           },
+        
           load: function(id) {
             if (id.startsWith('\\0')) {
               return null;
             }
 
-            return request('load', [id]).then(function(value) {
-              return value;
-            });
+            return request('load', [id]);
           }
         });
-        
+      
         inputOptions.plugins.push(nodeResolver);
         inputOptions.plugins.push(commonjs());
-        
-        // inputOptions.plugins.push({
-        //   name: 'nullHanlder',
-        //   resolveId: function (importee, importer, options) {
-        //     request('log', [importee, importer, options])
-        //     // request('error', ["AssetNotFound", importee, importer, renderStack]).then(function(value) {
-        //     //   process.exit(1);
-        //     // });
-        //   }
-        // });
-
+      
         const outputOptions = #{JSON.generate(output_options)};
+        outputOptions.paths = (id) => {
+            const asset = dynamicImports[id];
+          
+            if (asset) {
+              return asset.path;
+            }
+        }
 
         async function build() {
           try {
             // inputOptions.cache = await JSON.parse(request('get_cache', []));
-            
+          
             const bundle = await rollup.rollup(inputOptions);
             await bundle.write(outputOptions);
             // await request('set_cache', [JSON.stringify(bundle)]);
@@ -167,134 +279,175 @@ class Condenser::RollupProcessor < Condenser::NodeProcessor
 
         build();
       JS
-      
-      input[:source] = File.read(File.join(output_dir, 'result.js'))
+
+      input[:source] = File.read(File.join(@output_dir, 'entry.js'))
       input[:source].delete_suffix!("//# sourceMappingURL=result.js.map\n")
+      input[:type] = 'module'
       # asset.map = File.read(File.join(output_dir, 'result.js.map'))
+      input
+    ensure
+      begin
+        [@input_dir, @output_dir].each { |dir| FileUtils.remove_entry(dir) }
+      rescue Errno::ENOENT
+      end
     end
-  end
   
-  def exec_runtime(script, input)
-    io = IO.popen([binary, '-e', script], 'r+')
-    buffer = ''
+    def exec_runtime(script, input)
+      io = IO.popen([binary, '--max_old_space_size=5120', '-e', script], 'r+')
+      buffer = String.new
     
-    begin
+      begin
       
-      while IO.select([io]) && io_read = io.read_nonblock(1_024)
-        buffer << io_read
-        messages = buffer.split("\n\n")
+        while IO.select([io]) && io_read = io.read_nonblock(1_024)
+          buffer << io_read
+          messages = buffer.split("\n\n")
         
-        buffer = if buffer.end_with?("\n\n")
-          ''
-        else
-          messages.pop
-        end
+          buffer = if buffer.end_with?("\n\n")
+            String.new
+          else
+            messages.pop
+          end
         
-        messages.each do |message|
-          message = JSON.parse(message)
-          
-          ret = case message['method']
-          when 'resolve'
-            importee, importer = message['args']
+          messages.each do |message|
+            message = JSON.parse(message)
+            # puts message.inspect
+            ret = case message['method']
+            when 'log'
+              puts message.inspect
+            when 'resolveDynamicImport'
+              importee, importer = message['args']
 
-            if importer.nil? && importee == @entry
-              @entry
-            elsif importee.start_with?('@babel/runtime') || importee.start_with?('core-js-pure') || importee.start_with?('regenerator-runtime')
-              x = File.join(npm_module_path, importee.gsub(/^\.\//, File.dirname(importer) + '/')).sub('/node_modules/regenerator-runtime', '/node_modules/regenerator-runtime/runtime.js')
-              x = "#{x}.js" if !x.end_with?('.js', '.mjs')
-              File.file?(x) ? x : (x.delete_suffix('.js') + "/index.js")
-            elsif npm_module_path && importee&.start_with?(npm_module_path)
-              x = importee.end_with?('.js', '.mjs') ? importee : "#{importee}.js"
-              x = (x.delete_suffix('.js') + "/index.js") if !File.file?(x)
-              x
-            elsif importee.start_with?('.') && importer.start_with?(npm_module_path)
-              x = File.expand_path(importee, File.dirname(importer))
-              x = "#{x}.js" if !x.end_with?('.js', '.mjs')
-              File.file?(x) ? x : (x.delete_suffix('.js') + "/index.js")
-            elsif importee.end_with?('*')
-              File.join(File.dirname(importee), '*')
-            else
-              @environment.find(importee, importer ? File.dirname(@entry == importer ? @input[:source_file] : importer) : nil, accept: @input[:content_types].last)&.source_file
-            end
-          when 'load'
-            importee = message['args'].first
-            if importee == @entry
-              { code: @input[:source], map: @input[:map] }
-            elsif importee.end_with?('*')
-              importees = @environment.resolve(importee, importer ? File.dirname(@entry == importer ? @input[:source_file] : importer) : nil, accept: @input[:content_types].last)
-              code = ""
-              code_imports = [];
-              importees.each_with_index.map do |f, i|
-                if f.has_default_export?
-                  code << "import _#{i} from '#{f.source_file}';\n"
-                  code_imports << "_#{i}"
-                elsif f.has_exports?
-                  code << "import * as _#{i} from '#{f.source_file}';\n"
-                  code_imports << "_#{i}"
+              asset = @environment.find(importee, importer ? (@entry == importer ? @input[:source_file] : importer) : nil, accept: @input[:content_types].last, npm: true)
+              asset ||= @environment.find(importee.delete_suffix('.js') + "/index.js", importer ? (@entry == importer ? @input[:source_file] : importer) : nil, accept: @input[:content_types].last, npm: true)
+              asset ||= @environment.find(importee.gsub(/\/[^\/]+$/, '')+ "/dist/index.js", importer ? (@entry == importer ? @input[:source_file] : importer) : nil, accept: @input[:content_types].last, npm: true)
+            
+              if asset.source_file == @input[:source_file]
+                {
+                  path: File.join("/", *[@prefix, asset.path].compact),
+                  # source: File.join(@input_dir, asset.filename),
+                  source: @entry,
+                  export: false,
+                  # filename: asset.filename
+                }
+              else
+                if @dynamic_imports != :inline
+                  {
+                    path: File.join("/", *[@prefix, asset.path].compact),
+                    source: File.join(@input_dir, asset.filename),
+                    export: true,
+                    filename: asset.filename
+                  }
                 else
-                  code << "import '#{f.source_file}';\n"
+                  {
+                    export: false,
+                    source: asset.source_file,
+                    filename: asset.filename
+                  }
                 end
               end
+            
+            when 'resolve'
+              importee, importer = message['args']
+
+              if importer.nil? && importee == @entry
+                @entry
+              elsif importee.start_with?('@babel/runtime') || importee.start_with?('core-js-pure') || importee.start_with?('regenerator-runtime')
+                x = File.join(npm_module_path, importee.gsub(/^\.\//, File.dirname(importer) + '/')).sub('/node_modules/regenerator-runtime', '/node_modules/regenerator-runtime/runtime.js')
+                x = "#{x}.js" if !x.end_with?('.js', '.mjs')
+                File.file?(x) ? x : (x.delete_suffix('.js') + "/index.js")
+              elsif npm_module_path && importee&.start_with?(npm_module_path)
+                x = importee.end_with?('.js', '.mjs') ? importee : "#{importee}.js"
+                x = (x.delete_suffix('.js') + "/index.js") if !File.file?(x)
+                x
+              elsif importee.start_with?('.') && importer.start_with?(npm_module_path)
+                x = File.expand_path(importee, File.dirname(importer))
+                x = "#{x}.js" if !x.end_with?('.js', '.mjs')
+                File.file?(x) ? x : (x.delete_suffix('.js') + "/index.js")
+              elsif importee.end_with?('*')
+                File.join(File.dirname(importee), '*')
+              else
+                asset = @environment.find(importee, importer ? (@entry == importer ? @input[:source_file] : importer) : nil, accept: @input[:content_types].last, npm: true)&.source_file
+              end
+            when 'load'
+              importee = message['args'].first
+              if importee == @entry
+                { code: @input[:source], map: @input[:map] }
+              elsif importee.end_with?('*')
+                importees = @environment.resolve(importee, importer ? File.dirname(@entry == importer ? @input[:source_file] : importer) : nil, accept: @input[:content_types].last, npm: true)
+                code = String.new
+                code_imports = [];
+                importees.each_with_index.map do |f, i|
+                  if f.has_default_export?
+                    code << "import _#{i} from '#{f.source_file}';\n"
+                    code_imports << "_#{i}"
+                  elsif f.has_exports?
+                    code << "import * as _#{i} from '#{f.source_file}';\n"
+                    code_imports << "_#{i}"
+                  else
+                    code << "import '#{f.source_file}';\n"
+                  end
+                end
               
-              code += "export default [#{code_imports.join(', ')}];"
+                code += "export default [#{code_imports.join(', ')}];"
 
-              { code: code, map: nil }
-            else
-              asset = @environment.find(importee, accept: @input[:content_types].last)
-              if asset
-                { code: asset.source, map: asset.sourcemap }
+                { code: code, map: nil }
               else
-                nil
+                asset = @environment.find(importee, accept: @input[:content_types].last)
+                if asset
+                  { code: asset.source, map: asset.sourcemap }
+                else
+                  nil
+                end
               end
-            end
-          when 'error'
-            io.write(JSON.generate({rid: message['rid'], return: nil}))
+            when 'error'
+              io.write(JSON.generate({rid: message['rid'], return: nil}))
             
-            case message['args'][0]
-            when 'AssetNotFound'
-              error_message = "Could not find import \"#{message['args'][1]}\" for \"#{message['args'][2]}\".\n\n"
-              error_message << build_tree(message['args'][3], input, message['args'][2])
-              raise exec_runtime_error(error_message)
-            else
-              raise exec_runtime_error(message['args'][0] + ': ' + message['args'][1])
+              case message['args'][0]
+              when 'AssetNotFound'
+                error_message = "Could not find import \"#{message['args'][1]}\" for \"#{message['args'][2]}\".\n\n"
+                error_message << build_tree(message['args'][3], input, message['args'][2])
+                raise exec_runtime_error(error_message)
+              else
+                raise exec_runtime_error(message['args'][0] + ': ' + message['args'][1])
+              end
+            # when 'set_cache'
+            #   @environment.cache.set('rollup', message['args'][0])
+            #   io.write(JSON.generate({rid: message['rid'], return: true}))
+            # when 'get_cache'
+            #   io.write(JSON.generate({rid: message['rid'], return: [(@environment.cache.get('rollup') || '{}')] }))
             end
-          # when 'set_cache'
-          #   @environment.cache.set('rollup', message['args'][0])
-          #   io.write(JSON.generate({rid: message['rid'], return: true}))
-          # when 'get_cache'
-          #   io.write(JSON.generate({rid: message['rid'], return: [(@environment.cache.get('rollup') || '{}')] }))
+            # puts JSON.generate({rid: message['rid'], return: ret})
+            io.write(JSON.generate({rid: message['rid'], return: ret}))
           end
-          io.write(JSON.generate({rid: message['rid'], return: ret}))
         end
+      rescue Errno::EPIPE, EOFError
       end
-    rescue Errno::EPIPE, EOFError
-    end
     
-    io.close
-    if $?.success?
-      true
-    else
-      raise exec_runtime_error(buffer)
+      io.close
+      if $?.success?
+        true
+      else
+        raise exec_runtime_error(buffer)
+      end
     end
-  end
 
-  def build_tree(renderStack, from, to, visited: nil)
-    visited ||= []
-    return if visited.include?(from)
-    visited << from
+    def build_tree(renderStack, from, to, visited: nil)
+      visited ||= []
+      return if visited.include?(from)
+      visited << from
     
-    if renderStack[from].nil? || renderStack[from].empty?
-      nil
-    elsif renderStack[from].include?(to)
-      from
-    else
-      renderStack[from].each do |dep|
-        if tree = build_tree(renderStack, dep, to, visited: visited)
-          return "#{from}\n└ #{tree.lines.each_with_index.map{|l, i| "#{i == 0 ? '' : '    '}#{l}"}.join("")}"
+      if renderStack[from].nil? || renderStack[from].empty?
+        nil
+      elsif renderStack[from].include?(to)
+        from
+      else
+        renderStack[from].each do |dep|
+          if tree = build_tree(renderStack, dep, to, visited: visited)
+            return "#{from}\n└ #{tree.lines.each_with_index.map{|l, i| "#{i == 0 ? '' : '    '}#{l}"}.join("")}"
+          end
         end
+        nil
       end
-      nil
     end
   end
-  
 end

data/lib/condenser/resolve.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class Condenser
   module Resolve
     
@@ -12,8 +14,14 @@ class Condenser
       @build_cache = BuildCache.new(path, logger: logger)
     end
     
-    def resolve(filename, base=nil, accept: nil)
-      filename = filename.delete_prefix("/") if path.none? { |p| filename.start_with?(p) }
+    def resolve(filename, base=nil, accept: nil, npm: false)
+      search_path = if npm
+        path + [File.join(npm_path, 'node_modules')]
+      else
+        path
+      end
+      
+      filename = filename.delete_prefix("/") if search_path.none? { |p| filename.start_with?(p) }
       dirname, basename, extensions, mime_types = decompose_path(filename, base)
       accept ||= mime_types.empty? ? ['*/*'] : mime_types
       accept = Array(accept)
@@ -26,7 +34,7 @@ class Condenser
           results = []
 
           paths = if dirname&.start_with?('/')
-            if pat = path.find { |pa| dirname.start_with?(pa) }
+            if pat = search_path.find { |pa| dirname.start_with?(pa) }
               dirname.delete_prefix!(pat)
               dirname.delete_prefix!('/')
               [pat]
@@ -34,7 +42,7 @@ class Condenser
               []
             end
           else
-            path
+            search_path
           end
         
           paths.each do |path|
@@ -48,7 +56,7 @@ class Condenser
               if (basename == '*' || basename == f_basename)
                 if accept == ['*/*'] || mime_type_match_accept?(f_mime_types, accept)
                   asset_dir = f_dirname.delete_prefix(path).delete_prefix('/')
-                  asset_basename = f_basename + f_extensions.join('')
+                  asset_basename = f_basename + f_extensions&.join('').to_s
                   asset_filename = asset_dir.empty? ? asset_basename : File.join(asset_dir, asset_basename)
                   results << build_cache.map("#{asset_filename}@#{f_mime_types.join('')}") do
                     Asset.new(self, {
@@ -105,9 +113,9 @@ class Condenser
       end
     end
 
-    def find(filename, base=nil, accept: nil)
+    def find(filename, base=nil, **kargs)
       build do
-        resolve(filename, base, accept: accept).first
+        resolve(filename, base, **kargs).first
       end
     end
     
@@ -148,11 +156,34 @@ class Condenser
       end
     end
     
+    def has_dir?(path)
+      path.count('/') > (path.start_with?('/') ? 1 : 0)
+    end
+    
+    def expand_path(path)
+      dir = if path.start_with?('/')
+        File.expand_path(path)
+      else
+        File.expand_path("/#{path}").delete_prefix('/')
+      end
+      dir.empty? ? nil : dir
+    end
+    
     def decompose_path(path, base=nil)
-      dirname = path.index('/') ? File.dirname(path) : nil
-      if base && path&.start_with?('.')
-        dirname = File.expand_path(dirname, base)
+      dirname = if base && path.start_with?('.')
+        if has_dir?(base)
+          if has_dir?(path)
+            expand_path(File.join(File.dirname(base), File.dirname(path)))
+          else
+            expand_path(File.dirname(base))
+          end
+        else
+          expand_path(File.dirname(path))
+        end
+      else
+        path.index('/') ? File.dirname(path) : nil
       end
+
       
       _, star, basename, extensions = path.match(/(([^\.\/]+)(\.[^\/]+)|\*|[^\/]+)$/).to_a
       if extensions == '.*'