decidim-core 0.22.0 → 0.23.3

data/app/queries/decidim/similar_emendations.rb

@@ -30,8 +30,8 @@ module Decidim
                 .not_hidden
                 .where(
                   "GREATEST(#{title_similarity}, #{body_similarity}) >= ?",
-                  emendation.title,
-                  emendation.body,
+                  translated_attribute(emendation.title),
+                  translated_attribute(emendation.body),
                   amendable_module.similarity_threshold
                 )
                 .limit(amendable_module.similarity_limit)
@@ -46,11 +46,11 @@ module Decidim
     end
 
     def title_similarity
-      "similarity(title, ?)"
+      "similarity(title::text, ?)"
     end
 
     def body_similarity
-      "similarity(body, ?)"
+      "similarity(body::text, ?)"
     end
   end
 end

data/app/scrubbers/decidim/newsletter_scrubber.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Decidim
+  # Use this class as a scrubber to sanitize user input. The default
+  # scrubbed provided by Rails does not allow `iframe`s, and we're using
+  # them to embed videos, so we need to provide a whole new scrubber.
+  #
+  # Example:
+  #
+  #    sanitize(@page.body, scrubber: Decidim::UserInputScrubber.new)
+  #
+  # Lists of default tags and attributes are extracted from
+  # https://stackoverflow.com/a/35073814/2110884.
+  class NewsletterScrubber < Rails::Html::PermitScrubber
+    def initialize
+      super
+      self.tags = custom_allowed_tags
+      self.attributes = custom_allowed_attributes
+    end
+
+    private
+
+    def custom_allowed_attributes
+      Loofah::HTML5::SafeList::ALLOWED_ATTRIBUTES + %w(frameborder allowfullscreen) - %w(onerror)
+    end
+
+    def custom_allowed_tags
+      Loofah::HTML5::SafeList::ALLOWED_ELEMENTS_WITH_LIBXML2 + %w(iframe style)
+    end
+  end
+end

data/app/scrubbers/decidim/user_input_scrubber.rb

@@ -21,7 +21,7 @@ module Decidim
     private
 
     def custom_allowed_attributes
-      Loofah::HTML5::SafeList::ALLOWED_ATTRIBUTES + %w(frameborder allowfullscreen)
+      Loofah::HTML5::SafeList::ALLOWED_ATTRIBUTES + %w(frameborder allowfullscreen) - %w(onerror)
     end
 
     def custom_allowed_tags

data/app/serializers/decidim/importers/participatory_space_components_importer.rb

@@ -64,7 +64,7 @@ module Decidim
       end
 
       def override_step_settings_ids(attributes, step_settings)
-        return if step_settings.nil?
+        return unless @participatory_space.has_steps? && step_settings.present?
 
         @participatory_space.steps.each do |step|
           old_id = attributes["settings"]["steps"].keys.first

data/app/services/decidim/base_diff_renderer.rb

@@ -59,6 +59,22 @@ module Decidim
       diff
     end
 
+    def parse_user_group_changeset(attribute, values, type, diff)
+      return unless diff
+
+      old_user_group = Decidim::UserGroup.find_by(id: values[0])
+      new_user_group = Decidim::UserGroup.find_by(id: values[1])
+
+      diff.update(
+        attribute => {
+          type: type,
+          label: I18n.t(attribute, scope: i18n_scope),
+          old_value: old_user_group ? translated_attribute(old_user_group.name) : "",
+          new_value: new_user_group ? translated_attribute(new_user_group.name) : ""
+        }
+      )
+    end
+
     def parse_scope_changeset(attribute, values, type, diff)
       return unless diff
 
@@ -79,6 +95,7 @@ module Decidim
       return parse_i18n_changeset(attribute, values, type, diff) if [:i18n, :i18n_html].include?(type)
 
       return parse_scope_changeset(attribute, values, type, diff) if type == :scope
+      return parse_user_group_changeset(attribute, values, type, diff) if type == :user_group
 
       diff.update(
         attribute => {

data/app/services/decidim/open_data_exporter.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "zip"
+
 module Decidim
   # Public: It generates a ZIP file with Open Data CSV files ready
   # to be uploaded somewhere so users can download an organization

data/app/services/decidim/resource_search.rb

@@ -4,6 +4,8 @@ module Decidim
   # This is the base class to be used by other search services.
   # Searchlight documentation: https://github.com/nathanl/searchlight
   class ResourceSearch < Searchlight::Search
+    attr_reader :user, :organization, :component
+
     # Initialize the Searchlight::Search base class with the options provided.
     #
     # scope   - The scope used to create the base query
@@ -13,6 +15,31 @@ module Decidim
     def initialize(scope, options = {})
       super(options)
       @scope = scope
+      @user = options[:current_user] || options[:user]
+      @component = options[:component]
+      @organization = options[:organization] || component&.organization
+    end
+
+    # Public: Companion method to `search_search_text` which defines the
+    # attributes where we should search for text values in a model.
+    def self.text_search_fields(*fields)
+      @text_search_fields = fields if fields.any?
+      @text_search_fields
+    end
+
+    # Handle the search_text filter. We have to cast the JSONB columns
+    # into a `text` type so that we can search.
+    def search_search_text
+      return query unless self.class.text_search_fields.any?
+
+      fields = self.class.text_search_fields.dup
+
+      text_query = query.where(localized_search_text_in("#{query.model_name.plural}.#{fields.shift}"), text: "%#{search_text}%")
+
+      fields.each do |field|
+        text_query = text_query.or(query.where(localized_search_text_in("#{query.model_name.plural}.#{field}"), text: "%#{search_text}%"))
+      end
+      text_query
     end
 
     # Creates the SearchLight base query.
@@ -51,8 +78,65 @@ module Decidim
       query.includes(:scope).references(:decidim_scopes).where(conditions.join(" OR "), *clean_scope_ids.map(&:to_i))
     end
 
+    # Handle the origin filter.
+    def search_origin
+      renamed_origin = Array(origin).map do |search_value|
+        "#{search_value}_origin"
+      end
+      apply_scopes(%w(official_origin citizens_origin user_group_origin meeting_origin), renamed_origin)
+    end
+
+    # We overwrite the `results` method to ensure we only return unique
+    # results. We can't use `#uniq` because it returns an Array and we're
+    # adding scopes in the controller, and `#distinct` doesn't work here
+    # because in the later scopes we're ordering by `RANDOM()` in a DB level,
+    # and `SELECT DISTINCT` doesn't work with `RANDOM()` sorting, so we need
+    # to perform two queries.
+    #
+    # The correct behaviour is backed by tests.
+    def results
+      base_query.model.where(id: super.pluck(:id))
+    end
+
     private
 
+    # Private: To be used by classes that inherit from ResourceSearch.
+    #
+    # This method is useful when the values of the filters match the names of
+    # defined scopes in a model, it applies those scopes that are included in
+    # the search values.
+    #
+    # Example:
+    #   Consider you want to filter by state, and your model has an `open` and
+    #   a `closed` ActiveRecord scope.
+    #
+    #   def search_state
+    #     apply_scopes(%w(open closed), state)
+    #   end
+    #
+    #   In this scenario, the `state` variable has the input by the use, who
+    #   has selected which states they want to see. `states` here is an array
+    #   of strings.
+    #
+    # Returns an ActiveRecord::Relation.
+    def apply_scopes(scopes, search_values)
+      search_values = Array(search_values)
+
+      conditions = scopes.map do |scope|
+        search_values.member?(scope.to_s) ? query.try(scope) : nil
+      end.compact
+
+      return query unless conditions.any?
+
+      scoped_query = query.where(id: conditions.shift)
+
+      conditions.each do |condition|
+        scoped_query = scoped_query.or(query.where(id: condition))
+      end
+
+      scoped_query
+    end
+
     # Private: Creates an array of category ids.
     # It contains categories' subcategories ids as well.
     def all_category_ids
@@ -67,7 +151,7 @@ module Decidim
 
     # Private: Returns an array with checked category ids.
     def category_ids
-      [category_id].flatten
+      Array(category_id)
     end
 
     # Private: Returns an array with checked scope ids.
@@ -75,14 +159,19 @@ module Decidim
       if scope_id.is_a?(Hash)
         scope_id.values
       else
-        [scope_id].flatten
+        Array(scope_id)
       end
     end
 
-    # Private: Since component is not used by a search method we need
-    # to define the method manually.
-    def component
-      options[:component]
+    # Internal: builds the needed query to search for a text in the organization's
+    # available locales. Note that it is intended to be used as follows:
+    #
+    # Example:
+    #   Resource.where(localized_search_text_for(:title, text: "my_query"))
+    #
+    # The Hash with the `:text` key is required or it won't work.
+    def localized_search_text_in(field)
+      organization.available_locales.map { |l| "#{field} ->> '#{l}' ILIKE :text" }.join(" OR ")
     end
   end
 end

data/app/services/decidim/static_map_generator.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "httparty"
-
 module Decidim
   # This class generates a url to create a static map image for a geocoded resource
   class StaticMapGenerator
@@ -15,39 +13,25 @@ module Decidim
     end
 
     def data
-      return if Decidim.geocoder.nil? || @resource.blank?
+      return if @resource.blank? || map_utility.nil?
 
       Rails.cache.fetch(@resource.cache_key) do
-        request = HTTParty.get(uri, headers: { "Referer" => organization.host })
-        request.body
+        map_utility.image_data(
+          latitude: @resource.latitude,
+          longitude: @resource.longitude,
+          options: @options
+        )
       end
     end
 
     private
 
-    def uri
-      params = {
-        c: "#{@resource.latitude}, #{@resource.longitude}",
-        z: @options[:zoom],
-        w: @options[:width],
-        h: @options[:height],
-        f: "1"
-      }
-
-      if Decidim.geocoder[:here_api_key].present?
-        params[:apiKey] = Decidim.geocoder.fetch(:here_api_key)
-      else
-        params[:app_id] = Decidim.geocoder.fetch(:here_app_id)
-        params[:app_code] = Decidim.geocoder.fetch(:here_app_code)
-      end
-
-      URI.parse(Decidim.geocoder.fetch(:static_map_url)).tap do |uri|
-        uri.query = URI.encode_www_form params
-      end
-    end
-
     def organization
       @organization ||= @resource.component.organization
     end
+
+    def map_utility
+      @map_utility ||= Decidim::Map.static(organization: organization)
+    end
   end
 end

data/app/services/decidim/tokenizer.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require "openssl"
+
+module Decidim
+  # This class is used to generate secure tokens
+  class Tokenizer
+    #
+    # Initializes the Tokenizer.
+    #
+    # salt      - The salt fr the encryption (it should be at leas 30 chars long)
+    # length    - How long the key generated should be (in bytes)
+    #
+    def initialize(salt: nil, length: 32)
+      @salt = salt.presence || Tokenizer.random_salt
+      @length = length
+    end
+
+    def self.random_salt
+      SecureRandom.hex(32)
+    end
+
+    attr_reader :salt, :length
+
+    # returns a securely generated string of bytes
+    def digest(string)
+      OpenSSL::PKCS5.pbkdf2_hmac(string.to_s, salt, 20_000, length, "sha256")
+    end
+
+    def int_digest(string)
+      digest(string.to_s).bytes.inject { |a, b| (a << 8) + b }
+    end
+
+    def hex_digest(string)
+      digest(string.to_s).bytes.map { |c| c.ord.to_s(16) }.join
+    end
+  end
+end

data/app/services/decidim/traceability.rb

@@ -115,6 +115,9 @@ module Decidim
 
     def log(action, user, resource, extra_log_info = {})
       return unless user.is_a?(Decidim::User)
+      # If the record is not valid, it may not yet have an ID causing an
+      # exception when trying to save the log record.
+      return unless resource.valid?
 
       Decidim::ActionLogger.log(
         action,

data/app/uploaders/decidim/application_uploader.rb

@@ -1,10 +1,12 @@
 # frozen_string_literal: true
 
 module Decidim
-  # This class deals with uploading images to Decidim It is intended to just
+  # This class deals with uploading files to Decidim. It is intended to just
   # hold the uploads configuration, so you should inherit from this class and
   # then tweak any configuration you need.
   class ApplicationUploader < CarrierWave::Uploader::Base
+    process :validate_inside_organization
+
     # Override the directory where uploaded files will be stored.
     # This is a sensible default for uploaders that are meant to be mounted:
     def store_dir
@@ -14,5 +16,21 @@ module Decidim
 
       default_path
     end
+
+    def skip_ssrf_protection?(_uri)
+      true
+    end
+
+    protected
+
+    # Validates that the associated model is always within an organization in
+    # order to pass the organization specific settings for the file upload
+    # checks (e.g. file extension, mime type, etc.).
+    def validate_inside_organization
+      return if model.is_a?(Decidim::Organization)
+      return if model.respond_to?(:organization) && model.organization.is_a?(Decidim::Organization)
+
+      raise CarrierWave::IntegrityError, I18n.t("carrierwave.errors.not_inside_organization")
+    end
   end
 end

data/app/uploaders/decidim/attachment_uploader.rb

@@ -17,12 +17,28 @@ module Decidim
       process resize_to_limit: [nil, 1000]
     end
 
-    protected
-
     def extension_whitelist
-      %w(jpg jpeg gif png bmp pdf doc docx xls xlsx ppt ppx rtf txt odt ott odf otg ods ots)
+      case upload_context
+      when :admin
+        Decidim.organization_settings(model).upload_allowed_file_extensions_admin
+      else
+        Decidim.organization_settings(model).upload_allowed_file_extensions
+      end
     end
 
+    # CarrierWave automatically calls this method and validates the content
+    # type fo the temp file to match against any of these options.
+    def content_type_whitelist
+      case upload_context
+      when :admin
+        Decidim.organization_settings(model).upload_allowed_content_types_admin
+      else
+        Decidim.organization_settings(model).upload_allowed_content_types
+      end
+    end
+
+    protected
+
     # Strips out all embedded information from the image
     def strip
       return unless image?(self)
@@ -33,20 +49,10 @@ module Decidim
       end
     end
 
-    # CarrierWave automatically calls this method and validates the content
-    # type fo the temp file to match against any of these options.
-    def content_type_whitelist
-      [
-        %r{image\/},
-        %r{application\/vnd.oasis.opendocument},
-        %r{application\/vnd.ms-},
-        %r{application\/msword},
-        %r{application\/vnd.ms-word},
-        %r{application\/vnd.openxmlformats-officedocument},
-        %r{application\/vnd.oasis.opendocument},
-        %r{application\/pdf},
-        %r{application\/rtf}
-      ]
+    def upload_context
+      return :participant unless model.respond_to?(:context)
+
+      model.context
     end
 
     # Checks if the file is an image based on the content type. We need this so