decidim-core 0.29.2 → 0.30.0.rc2

data/app/services/decidim/download_your_data_exporter.rb

@@ -1,41 +1,52 @@
 # frozen_string_literal: true
 
-require "decidim/seven_zip_wrapper"
-
 module Decidim
-  # Public: Generates a 7z(seven zip) file with data files ready to be persisted
+  # Public: Generates a zip archive with data files ready to be persisted
   # somewhere so users can download their data.
   #
-  # In fact, the 7z file wraps a ZIP file which finally contains the data files.
   class DownloadYourDataExporter
     DEFAULT_EXPORT_FORMAT = "CSV"
     ZIP_FILE_NAME = "download-your-data.zip"
 
+    include Decidim::TranslatableAttributes
+
     # Public: Initializes the class.
     #
     # user          - The user to export the data from.
-    # path          - The String path where to write the zip file.
-    # password      - The password to protect the zip file.
+    # name          - The name of the export in private area
     # export_format - The format of the data files inside the zip file. (CSV by default)
-    def initialize(user, path, password, export_format = DEFAULT_EXPORT_FORMAT)
+    def initialize(user, name, export_format = DEFAULT_EXPORT_FORMAT)
       @user = user
-      @path = File.expand_path path
       @export_format = export_format
-      @password = password
+      @name = name
     end
 
+    # i18n-tasks-use t("decidim.download_your_data.show.download_your_data")
     def export
-      tmpdir = Dir.mktmpdir("temporary-download-your-data-dir")
-      user_data, user_attachments = data_and_attachments_for_user
-      save_user_data(tmpdir, user_data)
-      save_user_attachments(tmpdir, user_attachments)
-
-      SevenZipWrapper.compress_and_encrypt(filename: @path, password: @password, input_directory: tmpdir)
+      user_export = user.private_exports.build
+      user_export.export_type = name
+      user_export.file.attach(io: data, filename: "#{name}.zip", content_type: "application/zip")
+      user_export.expires_at = Decidim.download_your_data_expiry_time.from_now
+      user_export.metadata = {}
+      user_export.save!
+      user_export
     end
 
     private
 
-    attr_reader :user, :export_format
+    attr_reader :user, :export_format, :name
+
+    def data
+      user_data, user_attachments = data_and_attachments_for_user
+      buffer = Zip::OutputStream.write_buffer do |out|
+        save_user_data(out, user_data)
+        save_user_attachments(out, user_attachments)
+        save_readme(out)
+      end
+
+      buffer.rewind
+      buffer
+    end
 
     def data_and_attachments_for_user
       export_data = []
@@ -43,7 +54,8 @@ module Decidim
 
       download_your_data_entities.each do |object|
         klass = Object.const_get(object)
-        export_data << [klass.model_name.name.parameterize.pluralize, Exporters.find_exporter(export_format).new(klass.user_collection(user), klass.export_serializer).export]
+        exporter = Exporters.find_exporter(export_format).new(klass.user_collection(user), klass.export_serializer)
+        export_data << [klass.model_name.name.parameterize.pluralize, exporter.export]
         attachments = klass.download_your_data_images(user)
         export_attachments << [klass.model_name.name.parameterize.pluralize, attachments.flatten] unless attachments.nil?
       end
@@ -55,30 +67,52 @@ module Decidim
       @download_your_data_entities ||= DownloadYourDataSerializers.data_entities
     end
 
-    def save_user_data(tmpdir, user_data)
+    def save_user_data(output, user_data)
       user_data.each do |entity, exporter_data|
         next if exporter_data.read == "\n"
 
-        file_name = File.join(tmpdir, "#{entity}-#{exporter_data.filename}")
-        File.write(file_name, exporter_data.read)
+        output.put_next_entry("#{entity}-#{exporter_data.filename}")
+        output.write exporter_data.read
       end
     end
 
-    def save_user_attachments(tmpdir, user_attachments)
+    def save_user_attachments(output, user_attachments)
       user_attachments.each do |entity, attachment_block|
         attachment_block.each do |attachment|
           next unless attachment.attached?
 
           blobs = attachment.is_a?(ActiveStorage::Attached::One) ? [attachment.blob] : attachment.blobs
           blobs.each do |blob|
-            Dir.mkdir(File.join(tmpdir, entity.parameterize))
-            file_name = File.join(tmpdir, entity.parameterize, blob.filename.to_s)
             blob.open do |blob_file|
-              File.write(file_name, blob_file.read.force_encoding("UTF-8"))
+              output.put_next_entry("#{entity.parameterize}/#{blob.filename}")
+              output.write blob_file.read.force_encoding("UTF-8")
             end
           end
         end
       end
     end
+
+    def save_readme(output)
+      output.put_next_entry("README.md")
+      output.write readme
+    end
+
+    def readme
+      readme_file = "# #{I18n.t("decidim.download_your_data.help.core.title", organization: translated_attribute(user.organization.name))}\n\n"
+      readme_file << "#{I18n.t("decidim.download_your_data.help.core.description", user_name: "#{user.name} (#{user.nickname})")}\n\n"
+      help_definition = DownloadYourDataSerializers.help_definitions_for(user)
+
+      help_definition.each do |entity, headers|
+        next if headers.empty?
+
+        readme_file << "\n\n## #{entity}\n\n"
+
+        headers.each do |header, help_value|
+          readme_file << "* #{header}: #{help_value}\n"
+        end
+      end
+
+      readme_file
+    end
   end
 end

data/app/services/decidim/onboarding_manager.rb

@@ -0,0 +1,272 @@
+# frozen_string_literal: true
+
+module Decidim
+  # This class is used to manage the onboarding funnel for users that
+  # intended to perform an action in the platform but it requires some
+  # verification steps to be completed.
+  #
+  # The flow is as follows:
+  #
+  # 1. A visitor (logged out) tries to perform an action that requires to be logged in
+  # 2. The intended action is stored in a cookie (onboarding key)
+  # 3. The visitor logs in and the onboarding cookie data is merged into the User#extended_data column.
+  # 4. After logging in, the user is redirected to the onboarding pending authorizations page
+  #    if there is any pending action stored in the User#extended_data column
+  # 5. The user completes the required authorizations
+  # 6. When the onboarding authorizations page detects that all the verification steps have been
+  #    completed, it will remove the pending action from the User#extended_data column and redirect
+  #    the user to the intended action.
+  #
+  class OnboardingManager
+    # The same key is set in onboarding_pending_action js file to manage the
+    # onboarding data in the cookie
+    DATA_KEY = "onboarding"
+
+    attr_reader :user
+
+    def initialize(user)
+      @user = user
+    end
+
+    delegate :ephemeral?, to: :user
+
+    # Checks if the onboarding data has an action and a model.
+    #
+    # Returns a boolean
+    def valid?
+      return if action.blank?
+
+      permissions_holder.present?
+    end
+
+    # Returns the action to be performed.
+    #
+    # Returns a string
+    def action
+      onboarding_action
+    end
+
+    # Returns the translation of the action to be performed if translation
+    # is found, otherwise the untranslated literal action key
+    #
+    # Returns a string
+    def action_text
+      @action_text ||= if component && I18n.exists?("#{component.manifest.name}.actions.#{action}", scope: "decidim.components")
+                         I18n.t("#{component.manifest.name}.actions.#{action}", scope: "decidim.components")
+                       elsif permissions_holder.respond_to?(:resource_manifest) &&
+                             I18n.exists?("#{permissions_holder.resource_manifest.name}.actions.#{action}", scope: "decidim.resources")
+                         I18n.t("#{permissions_holder.resource_manifest.name}.actions.#{action}", scope: "decidim.resources")
+                       else
+                         action
+                       end
+    end
+
+    # Checks if there is any pending action for the user.
+    #
+    # Returns a boolean
+    def pending_action?
+      onboarding_data.present?
+    end
+
+    # Checks if the user has completed all the required authorizations.
+    #
+    # active_authorization_methods - A list of the active authorization methods for the user.
+    #
+    # Returns a boolean
+    def finished_verifications?(authorization_methods = active_authorization_methods)
+      (authorization_handlers - authorization_methods).empty?
+    end
+
+    # Returns the model related to the action in the onboarding process.
+    #
+    # Returns an ActiveRecord model
+    def model
+      return if onboarding_model.blank?
+
+      @model ||= GlobalID::Locator.locate(onboarding_model)
+    end
+
+    # Returns the permissions_holder if present related to the action in the
+    # onboarding process.
+    #
+    # Returns an ActiveRecord model
+    def permissions_holder
+      return model if onboarding_permissions_holder.blank?
+
+      @permissions_holder ||= GlobalID::Locator.locate(onboarding_permissions_holder)
+    end
+
+    # Returns the model name related to the action in the onboarding process.
+    #
+    # Returns a string
+    def model_name
+      return unless valid?
+
+      @model_name ||= (model.presence || permissions_holder).class.model_name
+    end
+
+    # Returns the resource title associated to the action. If the model is defined
+    # its title is used, if not the permissions holder title
+    #
+    # Returns a translations Hash or a String
+    def model_title
+      return unless valid?
+
+      @model_title ||= begin
+        resource = model.presence || permissions_holder
+
+        method = [:title, :name].find { |m| resource.respond_to?(m) }
+
+        resource.send(method) if method
+      end
+    end
+
+    # Filters the given authorizations that are required for the onboarding process.
+    #
+    # authorizations - An array of Decidim::Authorization objects
+    #
+    # Returns an array of Decidim::Authorization objects
+    def filter_authorizations(authorizations)
+      filtered_authorizations = authorizations.select { |authorization| authorization_handlers.include?(authorization.name) }
+
+      # By calling authorize on each authorization the path generated for each
+      # one will include the specific options of the action if available
+      filtered_authorizations.each do |authorization|
+        next unless authorization.is_a? Decidim::Verifications::Adapter
+
+        authorization.authorize(nil, permissions.dig("authorization_handlers", authorization.name, "options") || {}, model, permissions_holder)
+      end
+      filtered_authorizations
+    end
+
+    # Returns a hash which can be passed to action_authorized_to helper method
+    # to determine the permissions status of the action
+    #
+    # Returns a Hash
+    def action_authorized_resources
+      return {} unless valid?
+
+      {
+        resource: model,
+        permissions_holder: onboarding_permissions_holder.presence && permissions_holder
+      }
+    end
+
+    # Returns the path to redirect after finishing the verification process. The path
+    # can be obtained from the user onboarding redirect_path data or if a resource is
+    # present using a ResourceLocatorPresenter
+    #
+    # Returns a String
+    def finished_redirect_path
+      @finished_redirect_path ||= onboarding_data["redirect_path"].presence || model_path
+    end
+
+    def root_path
+      component_path || Decidim::Core::Engine.routes.url_helpers.root_path
+    end
+
+    def component_path
+      return if component.blank?
+
+      EngineRouter.main_proxy(component).root_path
+    end
+
+    def expired?
+      return unless ephemeral?
+
+      session_duration > Decidim.config.expire_session_after.to_i
+    end
+
+    # Time in seconds since the last login or creation of the user
+    #
+    # Returns an Integer
+    def session_duration
+      Time.current.to_i - (user.last_sign_in_at || user.created_at).to_i
+    end
+
+    # This method is used to determine if an ephemeral user has an onboarding
+    # page to be redirected or only an authorization is required to complete the
+    # verification
+    #
+    # Returns a Boolean
+    def available_authorization_selection_page?
+      return true unless valid? && ephemeral?
+
+      authorization_handlers.count > 1
+    end
+
+    private
+
+    def active_authorization_methods
+      @active_authorization_methods ||= Verifications::Authorizations.new(organization: user.organization, user:).pluck(:name)
+    end
+
+    # Returns the permissions for the action and model in the onboarding process.
+    #
+    # Returns a hash
+    def permissions
+      @permissions ||= permissions_holder&.permissions&.fetch(action, nil) || component&.permissions&.fetch(action, nil)
+    end
+
+    # Returns the authorization handlers for the action and model in the onboarding process.
+    #
+    # Returns an array of strings
+    def authorization_handlers
+      return [] unless permissions
+
+      permissions["authorization_handlers"]&.keys&.map(&:to_s) || []
+    end
+
+    # Returns the model component or the permissions holder when it is a component.
+    #
+    # Returns a Decidim::Component
+    def component
+      return permissions_holder if permissions_holder.is_a?(Decidim::Component)
+      return if model.blank?
+      return unless model.respond_to?(:component)
+
+      model.component
+    end
+
+    # Returns the onboarding data from the user's extended_data related to the model in gid format.
+    #
+    # Returns a string
+    def onboarding_model
+      onboarding_data["model"]
+    end
+
+    # Returns the onboarding data from the user's extended_data related to the permissions_holder
+    # in gid format. This attribute is optional and may not be present
+    #
+    # Returns a string
+    def onboarding_permissions_holder
+      onboarding_data["permissions_holder"]
+    end
+
+    # Returns the onboarding data from the user's extended_data related to the action.
+    #
+    # Returns a string
+    def onboarding_action
+      onboarding_data["action"]
+    end
+
+    # Returns the onboarding data from the user's extended_data.
+    #
+    # Returns a hash
+    def onboarding_data
+      user.extended_data[DATA_KEY] || {}
+    end
+
+    # Returns the path associated to the model using ResourceLocatorPresenter. If model is not
+    # present returns the root path
+    #
+    # Returns a String
+    def model_path
+      return Decidim::Core::Engine.routes.url_helpers.root_path if model.blank?
+
+      ResourceLocatorPresenter.new(model).url
+    rescue NoMethodError
+      Decidim::Core::Engine.routes.url_helpers.root_path
+    end
+  end
+end

data/app/services/decidim/open_data_exporter.rb

@@ -9,48 +9,86 @@ module Decidim
   class OpenDataExporter
     FILE_NAME_PATTERN = "%{host}-open-data-%{entity}.csv"
 
-    attr_reader :organization, :path
+    attr_reader :organization, :path, :resource, :help_definition
+
+    include Decidim::TranslatableAttributes
 
     # Public: Initializes the class.
     #
     # organization - The Organization to export the data from.
     # path         - The String path where to write the zip file.
-    def initialize(organization, path)
+    # resource     - The String of the component or participatory space to export. If nil, it will export all.
+    def initialize(organization, path, resource = nil)
       @organization = organization
       @path = File.expand_path path
+      @resource = resource
+      @help_definition = {}
     end
 
     def export
       dirname = File.dirname(path)
       FileUtils.mkdir_p(dirname) unless File.directory?(dirname)
-      File.binwrite(path, data)
+      if resource.nil?
+        File.binwrite(path, data_for_all_resources)
+      else
+        File.write(path, data_for_resource(resource))
+      end
     end
 
     private
 
-    def data
+    def data_for_all_resources
       buffer = Zip::OutputStream.write_buffer do |out|
+        core_data_manifests.each do |manifest|
+          add_file_to_output(out, format(FILE_NAME_PATTERN, { host: organization.host, entity: manifest.name }), data_for_core(manifest).read)
+        end
         open_data_component_manifests.each do |manifest|
-          add_file_to_output(out, format(FILE_NAME_PATTERN, { host: organization.host, entity: manifest.name }), data_for_component(manifest))
+          add_file_to_output(out, format(FILE_NAME_PATTERN, { host: organization.host, entity: manifest.name }), data_for_component(manifest).read)
         end
         open_data_participatory_space_manifests.each do |manifest|
-          add_file_to_output(out, format(FILE_NAME_PATTERN, { host: organization.host, entity: manifest.name }), data_for_participatory_space(manifest))
+          add_file_to_output(out, format(FILE_NAME_PATTERN, { host: organization.host, entity: manifest.name }), data_for_participatory_space(manifest).read)
         end
+
+        add_file_to_output(out, "README.md", readme)
+        add_file_to_output(out, "LICENSE.md", license)
       end
 
       buffer.string
     end
 
+    def data_for_core(export_manifest)
+      collection = export_manifest.collection.call(organization)
+      exporter = Decidim::Exporters::CSV.new(collection, export_manifest.serializer)
+
+      get_help_definition(:core, exporter, export_manifest) unless collection.empty?
+
+      exporter.export
+    end
+
+    def data_for_resource(resource)
+      export_manifest = (core_data_manifests + open_data_component_manifests + open_data_participatory_space_manifests)
+                        .select { |manifest| manifest.name == resource.to_sym }.first
+
+      case export_manifest.respond_to?(:manifest) && export_manifest.manifest
+      when Decidim::ComponentManifest
+        data_for_component(export_manifest).read
+      when Decidim::ParticipatorySpaceManifest
+        data_for_participatory_space(export_manifest).read
+      else
+        data_for_core(export_manifest).read
+      end
+    end
+
     def data_for_component(export_manifest, col_sep = Decidim.default_csv_col_sep)
       headers = []
       collection = []
       ActiveRecord::Base.uncached do
-        components.where(manifest_name: export_manifest.manifest.name).find_each do |component|
+        components.where(manifest_name: export_manifest.manifest.name).unscope(:order).find_each do |component|
           export_manifest.collection.call(component).find_in_batches(batch_size: 100) do |batch|
-            exporter = Decidim::Exporters::CSV.new(batch, export_manifest.serializer)
+            serializer = export_manifest.open_data_serializer.nil? ? export_manifest.serializer : export_manifest.open_data_serializer
+            exporter = Decidim::Exporters::CSV.new(batch, serializer)
             headers.push(*exporter.headers)
             exported = exporter.export
-
             tmpdir = Dir::Tmpname.create(export_manifest.name.to_s) do
               # just get an empty file name
             end
@@ -59,6 +97,8 @@ module Decidim
             File.write(filename, exported.read)
 
             collection.push(filename)
+
+            get_help_definition(:components, exporter, export_manifest) unless collection.empty?
           end
         end
       end
@@ -76,16 +116,102 @@ module Decidim
     end
 
     def data_for_participatory_space(export_manifest)
-      collection = participatory_spaces.filter { |space| space.manifest.name == export_manifest.manifest.name }.flat_map do |participatory_space|
+      collection = participatory_spaces.flat_map do |participatory_space|
         export_manifest.collection.call(participatory_space)
       end
+      serializer = export_manifest.open_data_serializer.nil? ? export_manifest.serializer : export_manifest.open_data_serializer
+      exporter = Decidim::Exporters::CSV.new(collection, serializer)
+      get_help_definition(:spaces, exporter, export_manifest) unless collection.empty?
+
+      exporter.export
+    end
+
+    def get_help_definition(manifest_type, exporter, export_manifest)
+      help_definition[manifest_type] = {} if help_definition[manifest_type].nil?
+      help_definition[manifest_type][export_manifest.name] = {} if help_definition[manifest_type][export_manifest.name].blank?
+      exporter.headers_without_locales.each do |header|
+        help_definition[manifest_type][export_manifest.name][header] = I18n.t("decidim.open_data.help.#{export_manifest.name}.#{header}")
+      end
+    end
+
+    def readme
+      "# #{I18n.t("decidim.open_data.help.core.title", organization: translated_attribute(organization.name))}\n\n
+#{I18n.t("decidim.open_data.help.core.description")}\n\n
+#{core_readme}
+#{space_readme}
+#{component_readme}
+"
+    end
+
+    def core_readme
+      return unless help_definition.fetch(:core, false)
+
+      readme_file = "## #{I18n.t("decidim.open_data.help.core.main")}\n\n"
+      help_definition.fetch(:core, []).each do |element, headers|
+        readme_file << "### #{element}\n\n"
+
+        headers.each do |header, help_value|
+          readme_file << "* #{header}: #{help_value}\n"
+        end
+
+        readme_file << "\n\n"
+      end
+      readme_file
+    end
+
+    def space_readme
+      return unless help_definition.fetch(:spaces, false)
+
+      readme_file = "## #{I18n.t("decidim.open_data.help.core.spaces")}\n\n"
+
+      help_definition.fetch(:spaces, []).each do |space, headers|
+        readme_file << "### #{space}\n\n"
+
+        headers.each do |header, help_value|
+          readme_file << "* #{header}: #{help_value}\n"
+        end
+
+        readme_file << "\n\n"
+      end
+      readme_file
+    end
+
+    def component_readme
+      return unless help_definition.fetch(:components, false)
+
+      readme_file = "## #{I18n.t("decidim.open_data.help.core.components")}\n\n"
+
+      help_definition.fetch(:components, []).each do |component, headers|
+        readme_file << "### #{component}\n\n"
+
+        headers.each do |header, help_value|
+          readme_file << "* #{header}: #{help_value}\n"
+        end
+
+        readme_file << "\n\n"
+      end
 
-      Decidim::Exporters::CSV.new(collection, export_manifest.serializer).export
+      readme_file
     end
 
-    def add_file_to_output(output, file_name, data)
+    def license
+      link_database = "#{I18n.t("license_database_name", scope: "decidim.open_data.index.license")}: #{I18n.t("license_database_link", scope: "decidim.open_data.index.license")}"
+      link_contents = "#{I18n.t("license_contents_name", scope: "decidim.open_data.index.license")}: #{I18n.t("license_contents_link", scope: "decidim.open_data.index.license")}"
+
+      license_file = I18n.t("title", scope: "decidim.open_data.index.license")
+      license_file << "\n\n"
+      license_file << I18n.t("body_1_html", scope: "decidim.open_data.index.license", organization_name: translated_attribute(organization.name), link_database:, link_contents:)
+
+      license_file
+    end
+
+    def add_file_to_output(output, file_name, string)
       output.put_next_entry(file_name)
-      output.write data.read
+      output.write string
+    end
+
+    def core_data_manifests
+      @core_data_manifests ||= Decidim.open_data_manifests.select(&:include_in_open_data)
     end
 
     def open_data_component_manifests

data/app/services/decidim/traceability.rb

@@ -69,6 +69,19 @@ module Decidim
       end
     end
 
+    # Performs the given block and sets the author of the action without log.
+    #
+    # author - An object that implements `to_gid` or a String
+    #
+    # Returns whatever the given block returns.
+    def perform_action_without_log!(author)
+      PaperTrail.request(whodunnit: gid(author)) do
+        Decidim::ApplicationRecord.transaction do
+          block_given? ? yield : nil
+        end
+      end
+    end
+
     # Updates the `resource` with `update!` and sets the author of the version.
     #
     # resource - An ActiveRecord instance that implements `Decidim::Traceable`

data/app/uploaders/decidim/application_uploader.rb

@@ -60,7 +60,7 @@ module Decidim
     end
 
     def variant_path(key, options = {})
-      variant_url(key, **options.merge(only_path: true))
+      variant_url(key, **options, only_path: true)
     end
 
     def remote_url=(url)

data/app/validators/etiquette_validator.rb

@@ -6,9 +6,11 @@ class EtiquetteValidator < ActiveModel::EachValidator
   include ActionView::Helpers::SanitizeHelper
 
   def validate_each(record, attribute, value)
+    return unless Decidim.enable_etiquette_validator
     return if value.blank?
 
-    text_value = strip_tags(value)
+    # remove HTML tags, from WYSIWYG editor
+    text_value = clean_value(value)
 
     validate_caps(record, attribute, text_value)
     validate_marks(record, attribute, text_value)
@@ -17,8 +19,13 @@ class EtiquetteValidator < ActiveModel::EachValidator
 
   private
 
+  def clean_value(value)
+    ActionController::Base.helpers.strip_tags(value).to_s.strip
+  end
+
   def validate_caps(record, attribute, value)
-    return if value.scan(/[A-Z]/).length < value.length / 4
+    number_of_caps = value.scan(/[A-Z]/).length
+    return if number_of_caps.zero? || number_of_caps < value.length / 2 # 50%
 
     record.errors.add(attribute, options[:message] || :too_much_caps)
   end

data/app/validators/passthru_validator.rb

@@ -33,7 +33,7 @@ class PassthruValidator < ActiveModel::EachValidator
       value = dummy.public_send(dummy_attr)
     elsif dummy.respond_to? :file
       dummy.public_send("file=", value)
-      value = dummy.public_send(:file)
+      value = dummy.file
     end
 
     target_validators(attribute).each do |validator|