decidim-core 0.24.3 → 0.25.0.rc4

data/lib/decidim/webpacker/webpacker.yml

@@ -0,0 +1,65 @@
+# File written by Decidim generator
+# Note: You must restart bin/webpack-dev-server for changes to take effect
+
+default: &default
+  source_path: app/packs
+  source_entry_path: entrypoints
+  public_output_path: decidim-packs
+  public_root_path: public
+  webpack_compile_output: true
+  cache_path: tmp/webpacker-cache
+  extract_css: true
+
+  additional_paths:
+    - node_modules
+    - app/packs
+
+  # Reload manifest.json on all requests so we reload latest compiled packs
+  cache_manifest: false
+
+development:
+  <<: *default
+  compile: true
+  # Compile test packs to decidim decidim-packs folder
+
+  # Reference: https://webpack.js.org/configuration/dev-server/
+  dev_server:
+    https: false
+    host: localhost
+    # Notice that we use a different port (to prevent blocking the default one) as
+    # there will be at least two webpack servers running
+    port: 3035
+    # Hot Module Replacement updates modules while the application is running without a full reload
+    hmr: false
+    client:
+      # Should we show a full-screen overlay in the browser when there are compiler errors or warnings?
+      overlay: true
+      # May also be a string
+      # webSocketURL:
+      #  hostname: "0.0.0.0"
+      #  pathname: "/ws"
+      #  port: 8080
+    # Should we use gzip compression?
+    compress: true
+    # Note that apps that do not check the host are vulnerable to DNS rebinding attacks
+    allowed_hosts: "all"
+    pretty: true
+    headers:
+      'Access-Control-Allow-Origin': '*'
+    static:
+      watch:
+        ignored: '**/node_modules/**'
+
+test:
+  <<: *default
+  compile: true
+  public_output_path: packs-test
+
+production:
+  <<: *default
+
+  # Production depends on precompilation of packs prior to booting for performance.
+  compile: false
+
+  # Cache manifest.json for performance
+  cache_manifest: true

data/lib/decidim/webpacker.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Decidim
+  module Webpacker
+    autoload :Configuration, "decidim/webpacker/configuration"
+    autoload :Runner, "decidim/webpacker/runner"
+
+    def self.configuration
+      @configuration ||= Configuration.new
+    end
+
+    def self.register_path(path, prepend: false)
+      if prepend
+        configuration.additional_paths.unshift(path)
+      else
+        configuration.additional_paths.push(path)
+      end
+    end
+
+    def self.register_entrypoints(entrypoints)
+      configuration.entrypoints.merge!(entrypoints.stringify_keys)
+    end
+
+    def self.register_stylesheet_import(import, type: :imports, group: :app)
+      type = type.to_s
+      key = group.to_s
+      configuration.stylesheet_imports[type] ||= {}
+      configuration.stylesheet_imports[type][key] ||= []
+      configuration.stylesheet_imports[type][key].push(import)
+    end
+  end
+end

data/lib/gem_overrides/rails/tasks/yarn.rake

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+# This is overridden because of a bug in Rails 6.0 which forces the yarn:install
+# task to the end of assets:precompile task which fails when bin/yarn has been
+# removed from Decidim applications. This should be fixed with Rails 6.1.
+#
+# See:
+# https://git.io/JEH9s (and the equivalent line in Rails 6.1)
+# https://github.com/rails/rails/commit/87e9ae053d661daa3b8549e1cc9ea5ecd3b8ad62

data/lib/gem_overrides/webpacker/runner.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+# This file overrides the webpacker/runner require in order to customize the
+# runner when it is started.
+
+require "#{Gem.loaded_specs["webpacker"].full_gem_path}/lib/webpacker/runner"
+require "decidim/webpacker"
+
+Webpacker::Runner.include(Decidim::Webpacker::Runner)

data/lib/tasks/decidim_active_storage_migration_tasks.rake

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+namespace :decidim do
+  namespace :active_storage_migrations do
+    desc "Migrates attachments from Carrierwave to ActiveStorage"
+    task migrate_from_carrierwave_to_active_storage: :environment do
+      # Setup a new logger, using the timestamp to identify each migration
+      logger = ActiveSupport::TaggedLogging.new(Logger.new("log/#{Time.current.strftime("%Y%m%d%H%M")}_activestorage_migration.log"))
+      routes_mappings = []
+
+      Decidim::CarrierWaveMigratorService::MIGRATION_ATTRIBUTES.each do |(klass, cw_attribute, cw_uploader, as_attribute)|
+        Decidim::CarrierWaveMigratorService.migrate_attachment!({
+                                                                  klass: klass, cw_attribute: cw_attribute,
+                                                                  cw_uploader: cw_uploader, as_attribute: as_attribute,
+                                                                  logger: logger,
+                                                                  routes_mappings: routes_mappings
+                                                                })
+      end
+      Decidim::CarrierWaveMigratorService.migrate_content_blocks_attachments!(logger: logger, routes_mappings: routes_mappings)
+
+      path = Rails.root.join("tmp/attachment_mappings.csv")
+      dirname = File.dirname(path)
+      FileUtils.mkdir_p(dirname) unless File.directory?(dirname)
+      File.open(path, "wb") do |file|
+        file.write(Decidim::Exporters::CSV.new(routes_mappings).export.read)
+      end
+    end
+
+    desc "Checks attachments migrated from Carrierwave to ActiveStorage"
+    task check_migration_from_carrierwave_to_active_storage: :environment do
+      # Setup a new logger, using the timestamp to identify each migration
+      logger = ActiveSupport::TaggedLogging.new(Logger.new("log/#{Time.current.strftime("%Y%m%d%H%M")}_activestorage_migration_check.log"))
+      Decidim::CarrierWaveMigratorService::MIGRATION_ATTRIBUTES.each do |(klass, cw_attribute, cw_uploader, as_attribute)|
+        Decidim::CarrierWaveMigratorService.check_migration({
+                                                              klass: klass, cw_attribute: cw_attribute,
+                                                              cw_uploader: cw_uploader, as_attribute: as_attribute,
+                                                              logger: logger
+                                                            })
+      end
+      Decidim::CarrierWaveMigratorService.check_content_blocks_attachments(logger: logger)
+    end
+  end
+end

data/lib/tasks/decidim_data_portability_tasks.rake

@@ -73,76 +73,22 @@ namespace :decidim do
   desc "Deletes all data portability files previous to `Decidim.data_portability_expiry_time` from now."
   task delete_data_portability_files: :environment do
     puts "DELETE DATA PORTABILITY FILES: -------------- START"
-    uploader = Decidim::DataPortabilityUploader.new
-    case uploader.provider
-    when "file" # file system
-      puts "Deleting files from filesystem..."
-      delete_data_portability_files_from_fs(uploader)
-    when "aws"
-      puts "Deleting files from aws..."
-      delete_data_portability_files_from_aws(uploader)
-    else
-      raise "Carrierwave fog_provider not supported: #{uploader.fog_provider}"
+    attachments = ActiveStorage::Attachment.joins(:blob).where(
+      name: "data_portability_file",
+      record_type: "Decidim::UserBaseEntity"
+    ).where(
+      "active_storage_blobs.created_at < ?", Decidim.data_portability_expiry_time.ago
+    )
+    attachments.each do |attachment|
+      delete_data_portability_file attachment
     end
     puts "DELETE DATA PORTABILITY FILES: --------------- END"
   end
 
-  def delete_data_portability_files_from_fs(uploader)
-    path = uploader.store_dir
-    Dir.glob(Rails.root.join(path, "*")).each do |filename|
-      next unless File.mtime(filename) < Decidim.data_portability_expiry_time.ago
-
-      puts "------"
-      puts "!! deleting: #{filename}"
-      File.delete(filename)
-      puts "ok----"
-    end
-  end
-
-  # Removes data portability files older than the configured expiry time.
-  #
-  # ==== Struct of AWS's File objects:
-  #
-  # <Fog::AWS::Storage::File
-  #   key="uploads/decidim/user/avatar/1056/IMG_20171104_210039_247.jpg",
-  #   cache_control=nil,
-  #   content_disposition=nil,
-  #   content_encoding=nil,
-  #   content_length=113294,
-  #   content_md5=nil,
-  #   content_type=nil,
-  #   etag="e5f3c6a85a18d2557ed11c034efb3007",
-  #   expires=nil,
-  #   last_modified=2018-12-20 15:29:59 UTC,
-  #   metadata={},
-  #   owner={:display_name=>nil, :id=>"7dad77eea8cfbc10f588314706a02175a97fad81abccfd5747e87a340cc26a34"},
-  #   storage_class="STANDARD",
-  #   encryption=nil,
-  #   encryption_key=nil,
-  #   version=nil,
-  #   kms_key_id=nil
-  # >
-  def delete_data_portability_files_from_aws(uploader)
-    files = get_data_portability_files_from_aws(uploader.store_path)
-    files.each do |file|
-      next unless file.last_modified < Decidim.data_portability_expiry_time.ago
-
-      puts "------"
-      puts "!! deleting: #{file.key}"
-      file.delete
-      puts "ok----"
-    end
-  end
-
-  # Retrieves the list of files in the data portability dir.
-  #
-  # This method has a high cost as it performs a couple of requests to aws.
-  #
-  # Return [Fog::AWS::Storage::File]
-  def get_data_portability_files_from_aws(data_portability_path)
-    fog_credentials = CarrierWave::Uploader::Base.fog_credentials
-    s3 = Fog::Storage.new(fog_credentials)
-    data_portability_dir = s3.directories.get(CarrierWave::Uploader::Base.fog_directory, prefix: data_portability_path)
-    data_portability_dir.files
+  def delete_data_portability_file(attachment)
+    puts "------"
+    puts "!! deleting: #{attachment.filename}"
+    attachment.purge
+    puts "ok----"
   end
 end

data/lib/tasks/decidim_tasks.rake

@@ -2,7 +2,7 @@
 
 namespace :decidim do
   desc "Install migrations from Decidim to the app."
-  task upgrade: [:choose_target_plugins, :"railties:install:migrations"]
+  task upgrade: [:choose_target_plugins, :"railties:install:migrations", :"decidim:webpacker:upgrade"]
 
   desc "Setup environment so that only decidim migrations are installed."
   task :choose_target_plugins do

data/lib/tasks/decidim_webpacker_tasks.rake

@@ -0,0 +1,194 @@
+# frozen_string_literal: true
+
+require "decidim/gem_manager"
+
+namespace :decidim do
+  namespace :webpacker do
+    desc "Installs Decidim webpacker files in Rails instance application"
+    task install: :environment do
+      raise "Decidim gem is not installed" if decidim_path.nil?
+
+      # Remove yarn.lock (because bin/rails webpacker:install has been executed)
+      remove_file_from_application "yarn.lock"
+      # Removing bin/yarn makes assets:precompile task to don't execute `yarn install`
+      remove_file_from_application "bin/yarn"
+      # Babel config
+      copy_file_to_application "babel.config.json"
+      # PostCSS configuration
+      copy_file_to_application "decidim-core/lib/decidim/webpacker/postcss.config.js", "postcss.config.js"
+      # Webpacker configuration
+      copy_file_to_application "decidim-core/lib/decidim/webpacker/webpacker.yml", "config/webpacker.yml"
+      # Webpack JS config files
+      copy_folder_to_application "decidim-core/lib/decidim/webpacker/webpack", "config"
+      # Modify the webpack binstubs
+      add_binstub_load_path "bin/webpack"
+      add_binstub_load_path "bin/webpack-dev-server"
+
+      # Install JS dependencies
+      install_decidim_npm
+
+      # Remove the webpacker dependencies as they come through Decidim dependencies.
+      # This ensures we can control their versions from Decidim dependencies to avoid version conflicts.
+      webpacker_packages = %w(
+        @rails/actioncable
+        @rails/activestorage
+        @rails/ujs
+        @rails/webpacker
+        turbolinks
+        webpack
+        webpack-cli
+        @webpack-cli/serve
+        webpack-dev-server
+      )
+      system! "npm uninstall #{webpacker_packages.join(" ")}"
+
+      # Add the Browserslist configuration to the project
+      add_decidim_browserslist_configuration
+    end
+
+    desc "Upgrade Decidim dependencies in Rails instance application"
+    task upgrade: :environment do
+      raise "Decidim gem is not installed" if decidim_path.nil?
+
+      # Update JS dependencies
+      install_decidim_npm
+    end
+
+    def install_decidim_npm
+      decidim_npm_packages.each do |type, packages|
+        system! "npm i --save-#{type} #{packages.join(" ")}"
+      end
+    end
+
+    def add_decidim_browserslist_configuration
+      package = JSON.parse(File.read(rails_app_path.join("package.json")))
+      return if package["browserslist"].is_a?(Array) && package["browserslist"].include?("extends @decidim/browserslist-config")
+
+      package["browserslist"] = ["extends @decidim/browserslist-config"]
+      File.write(rails_app_path.join("package.json"), JSON.pretty_generate(package))
+    end
+
+    def decidim_npm_packages
+      gem_path = unreleased_gem_path
+
+      if gem_path
+        package_spec = "./packages/%s"
+
+        # The packages folder needs to be copied to the application folder
+        # because the linked dependencies are not installed when packages
+        # are installed using file references outside the application root
+        # where the `package.json` is located at. For more information, see:
+        # https://github.com/npm/cli/issues/2339
+        FileUtils.rm_rf(rails_app_path.join("packages"))
+        FileUtils.cp_r(gem_path.join("packages"), rails_app_path)
+      else
+        package_spec = "@decidim/%s@~#{Decidim::GemManager.semver_friendly_version(decidim_gemspec.version.to_s)}"
+      end
+
+      local_npm_dependencies.transform_values { |names| names.map { |name| format(package_spec, name) } }
+    end
+
+    def unreleased_gem_path
+      if decidim_gemspec.source.is_a?(Bundler::Source::Rubygems)
+        return if released_version?
+
+        gem_path = Pathname(decidim_gemspec.full_gem_path)
+      else
+        gem_path = decidim_gemspec.source.path
+        gem_path = Pathname(ENV["BUNDLE_GEMFILE"]).dirname.join(gem_path) if gem_path.relative?
+      end
+
+      gem_path
+    end
+
+    def local_npm_dependencies
+      @local_npm_dependencies ||= begin
+        package_json = JSON.parse(File.read(decidim_path.join("package.json")))
+
+        {
+          prod: local_npm_dependencies_list(package_json["dependencies"]),
+          dev: local_npm_dependencies_list(package_json["devDependencies"])
+        }.freeze
+      end
+    end
+
+    def local_npm_dependencies_list(deps)
+      return [] unless deps
+
+      deps.values
+          .select { |ref| ref.starts_with?("file:packages/") }
+          .map { |ref| ref.delete_prefix("file:packages/") }
+    end
+
+    def decidim_path
+      @decidim_path ||= Pathname.new(decidim_gemspec.full_gem_path) if Gem.loaded_specs.has_key?("decidim")
+    end
+
+    def decidim_gemspec
+      @decidim_gemspec ||= Gem.loaded_specs["decidim"]
+    end
+
+    def released_version?
+      decidim_gemspec.version.segments.last != "dev"
+    end
+
+    def rails_app_path
+      @rails_app_path ||= Rails.root
+    end
+
+    def copy_file_to_application(origin_path, destination_path = origin_path)
+      FileUtils.cp(decidim_path.join(origin_path), rails_app_path.join(destination_path))
+    end
+
+    def copy_folder_to_application(origin_path, destination_path = origin_path)
+      FileUtils.cp_r(decidim_path.join(origin_path), rails_app_path.join(destination_path))
+    end
+
+    def remove_file_from_application(path)
+      FileUtils.rm(path, force: true)
+    end
+
+    def add_binstub_load_path(binstub_path)
+      file = rails_app_path.join(binstub_path)
+      lines = File.readlines(file)
+
+      # Skip if the load path is already added
+      return if lines.grep(
+        %r{^\$LOAD_PATH.unshift "#\{Gem.loaded_specs\["decidim-core"\].full_gem_path\}/lib/gem_overrides"$}
+      ).size.positive?
+
+      contents = ""
+      lines.each do |line|
+        contents += line
+        next unless line =~ %r{^require "bundler/setup"$}
+
+        contents += "\n"
+        contents += "# Add the Decidim override load path to override webpacker functionality\n"
+        contents += "$LOAD_PATH.unshift \"\#{Gem.loaded_specs[\"decidim-core\"].full_gem_path}/lib/gem_overrides\"\n"
+      end
+
+      File.write(file, contents)
+    end
+
+    def system!(command)
+      system("cd #{rails_app_path} && #{command}") || abort("\n== Command #{command} failed ==")
+    end
+  end
+end
+
+# Override the Webpacker instance for the rake tasks to correctly assign the
+# configuration file path. This is needed e.g. when `rails assets:precompile` is
+# being run. Otherwise webpacker might not recognize if the assets need to be
+# compiled again (i.e. if the asset hash has been changed).
+if (config_path = Decidim::Webpacker.configuration.configuration_file)
+  Webpacker.instance = Webpacker::Instance.new(
+    config_path: Pathname.new(config_path)
+  )
+end
+
+# Remove the yarn install prerequisity from assets:precompile
+Rake::Task["assets:precompile"].prerequisites.delete("webpacker:yarn_install")
+
+# Add gem overrides path to the beginning in order to override rake tasks
+# Needed because of a bug in Rails 6.0 (see the overridden task for details)
+$LOAD_PATH.unshift "#{Gem.loaded_specs["decidim-core"].full_gem_path}/lib/gem_overrides"