dradis-csv 5.2.0 → 5.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8bad2356d2ad2625aa0180faf0ab79f8a8acf40ce59afcdcd7843e1df6eaf358
-  data.tar.gz: 00bf582bd667765e48311382f06b4f10587cfbe59424802dca5c96a1b9237b8d
+  metadata.gz: f66806e2230745ce4ab608c4930c575d92e35f577c7a86ff9f888327c313cda9
+  data.tar.gz: 2510f5a2d96208375f1a339976dc16c03490cdbabb297728095e178c635f2a20
 SHA512:
-  metadata.gz: 768891ef2bdf14268a94b4607300f8aca8ded2020335b51077efc74d0e5dad2aa36709968a2f7e09119519599c57e34610f4b4034ee0fa69a9462ad547fe9fab
-  data.tar.gz: 0021f7ee9aa94f498a2c469b4c11f3da43b85476c7bf4018e3970fa2252f93a6d02c5bc60163c81baaf802da67d39b3f3ed6bac69556587e834d19c326bfcd4e
+  metadata.gz: 5a6e489369e1d0a13bb3c60a02b02f1352680a0fe85c39448828891aac46b85a18227d97f62bc7486385dba3345bbb3027edf5fec786485bb54f2956ec5c720e
+  data.tar.gz: 90f1df72e003a07b155c3f445905332c63b902d49d753b0b167a33ad6158cfdc5a0408e22565ff36f42868a52f290d97aded1336d4354854d074fd294e45e645

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+v5.4.0 (September 2026)
+  - No changes
+
+v5.3.0 (August 2026)
+  - Add support for managing Mappings through the Mappings Manager
+  - Add support for saving column assignments from the upload mapper
+
 v5.2.0 (June 2026)
   - No changes
 

data/app/assets/javascripts/dradis/plugins/csv/upload.js

@@ -14,6 +14,11 @@ window.addEventListener('job-done', function () {
       Turbo.visit(redirectPath);
     }
   }
+
+  if ($('body.dradis-plugins-csv-upload.new').length) {
+    $('[data-behavior~=mapping-form] input[type="submit"]')
+      .val('Done!');
+  }
 });
 
 document.addEventListener('turbo:load', function() {
@@ -60,6 +65,16 @@ document.addEventListener('turbo:load', function() {
       _setDradisFieldSelect($(this));
     });
 
+    $('[data-behavior~=dradis-field-select]').on('change', function () {
+      console.log('change!');
+      _toggleCustomFieldInput($(this));
+    });
+
+    $('[data-behavior~=dradis-field-select]').not(':disabled').each(function () {
+      console.log('initializing!');
+      _toggleCustomFieldInput($(this));
+    });
+
     $('[data-behavior~=mapping-form]').submit(function () {
       var valid = _validateIdentifierSelected() && _validateNodeSelected();
 
@@ -87,13 +102,21 @@ document.addEventListener('turbo:load', function() {
         .attr('disabled', 'disabled')
         .addClass('d-none');
 
+      $row
+        .find('[data-behavior~=custom-destination-input]')
+        .prop('disabled', true)
+        .addClass('d-none')
+        .attr('required', false);
+
+      var $activeSelect;
+
       if ($select.val() == 'issue') {
-        $row
+        $activeSelect = $row
           .find('[data-behavior~=issue-field-select]')
           .removeAttr('disabled')
           .removeClass('d-none');
       } else if ($select.val() == 'evidence') {
-        $row
+        $activeSelect = $row
           .find('[data-behavior~=evidence-field-select]')
           .removeAttr('disabled')
           .removeClass('d-none');
@@ -103,6 +126,27 @@ document.addEventListener('turbo:load', function() {
           .attr('disabled', 'disabled')
           .removeClass('d-none');
       }
+
+      if ($activeSelect) {
+        _toggleCustomFieldInput($activeSelect);
+      }
+    }
+
+    function _toggleCustomFieldInput($select) {
+      var $customInput = $select.next('[data-behavior~=custom-destination-input]');
+
+      if (!$customInput.length) {
+        return;
+      }
+
+      var isCustom = $select
+        .find('option:selected')
+        .is('[data-behavior~=custom-destination-field]');
+
+      $customInput
+        .toggleClass('d-none', !isCustom)
+        .prop('disabled', !isCustom)
+        .attr('required', isCustom);
     }
 
     function _validateNodeSelected() {

data/app/controllers/dradis/plugins/csv/upload_controller.rb

@@ -3,21 +3,31 @@ module Dradis::Plugins::CSV
     include ProjectScoped
 
     before_action :load_attachment, only: [:new, :create]
-    before_action :load_rtp_fields, only: [:new]
+    before_action :load_rtp_fields, only: [:new, :create]
     before_action :load_csv_headers, only: [:new]
 
+    # Reached after the standard upload flow has already run the file
+    # through Importer#import. If a saved mapping matched, that import
+    # already happened (see importer.rb) and there's nothing left to map.
     def new
-      @default_columns = ['Column Header', 'Entity', 'Dradis Field']
+      if saved_mapping?
+        return redirect_to main_app.project_issues_path(current_project),
+          notice: 'CSV imported using its saved mapping.'
+      end
 
+      @default_columns = ['Column Header', 'Entity', 'Dradis Field']
       @log_uid = Log.new.uid
     end
 
     def create
+      save_mapping
+
       job_logger.write 'Enqueueing job to start in the background.'
 
       MappingImportJob.perform_later(
         default_user_id: current_user.id,
         file: @attachment.fullpath.to_s,
+        headers: csv_headers,
         mappings: mappings_params[:field_attributes].to_h,
         project_id: current_project.id,
         state: state,
@@ -31,6 +41,10 @@ module Dradis::Plugins::CSV
       @job_logger ||= Log.new(uid: params[:log_uid].to_i)
     end
 
+    def csv_headers
+      @csv_headers ||= ::CSV.open(@attachment.fullpath, &:readline)
+    end
+
     def load_attachment
       filename = CGI::escape params[:attachment]
       @attachment = Attachment.find(filename, conditions: { node_id: current_project.plugin_uploads_node.id })
@@ -62,7 +76,29 @@ module Dradis::Plugins::CSV
     end
 
     def mappings_params
-      params.require(:mappings).permit(field_attributes: [:field, :type])
+      params.require(:mappings).permit(field_attributes: [:field, :type, :custom_field])
+    end
+
+    def rtp_destination
+      rtp = current_project.report_template_properties
+      rtp && rtp.as_mapping_destination
+    end
+
+    # Persist the submitted column assignments so future uploads of this CSV
+    # format can reuse them. Mappings are scoped to a report template, so
+    # projects without one keep the upload-time mapper only.
+    def save_mapping
+      return unless rtp_destination
+
+      MappingForm.new(
+        destination: rtp_destination,
+        headers: csv_headers,
+        rtp_fields: @rtp_fields
+      ).save(column_mappings: mappings_params[:field_attributes].to_h)
+    end
+
+    def saved_mapping?
+      rtp_destination && Dradis::Plugins::CSV.mapping_exists?(headers: @headers, destination: rtp_destination)
     end
 
     def state

data/app/jobs/dradis/plugins/csv/mapping_import_job.rb

@@ -13,7 +13,7 @@ module Dradis::Plugins::CSV
     #   '2' => { 'type' => 'identifier' },
     #   '3' => { 'type' => 'evidence', 'field' => 'Port' }
     # }
-    def perform(default_user_id:, file:, mappings:, project_id:, state:, uid:)
+    def perform(default_user_id:, file:, headers:, mappings:, project_id:, state:, uid:)
       logger = Log.new(uid: uid)
       logger.write { "Job id is #{job_id}." }
 
@@ -25,7 +25,8 @@ module Dradis::Plugins::CSV
         state: state
       )
 
-      importer.import_csv(file: file, mappings: mappings)
+      logger.write { 'Worker process starting background task.' }
+      importer.import_rows(file: file, headers: headers, mappings: mappings)
 
       logger.write { 'Worker process completed.' }
     end

data/app/views/dradis/plugins/csv/upload/new.html.erb

@@ -20,6 +20,12 @@
     <div class="alert alert-danger d-none" data-behavior="node-type-validation-message">
       <p>A Node Label must be selected to import evidence records.</p>
     </div>
+
+    <% if @rtp_fields && (@rtp_fields[:issue].present? || @rtp_fields[:evidence].present?) %>
+      <div class="alert alert-info">
+        <p class="m-0">This mapping will be saved and applied automatically the next time you upload a CSV file with these same column headers.</p>
+      </div>
+    <% end %>
   </div>
 
   <%= form_with url: project_upload_index_path(current_project, format: :js), method: :post, local: false, data: { behavior: 'mapping-form' } do |f| %>
@@ -37,6 +43,7 @@
         </tr>
       </thead>
       <tbody>
+        <% custom_field_option = ['Custom Field', 'Custom Field', { data: { behavior: 'custom-destination-field' } }] %>
         <% @headers.each_with_index do |header, index| %>
           <tr class="issue-type">
             <td><%= header %></td>
@@ -48,11 +55,13 @@
             <td>
               <% if @rtp_fields %>
                 <div>
-                  <% issue_options = @rtp_fields[:issue].any? ? options_for_select(@rtp_fields[:issue]) : options_for_select([[header, header]], disabled: header, selected: header) %>
+                  <% issue_options = @rtp_fields[:issue].any? ? options_for_select(@rtp_fields[:issue] + [custom_field_option]) : options_for_select([[header, header]], disabled: header, selected: header) %>
                   <%= f.select "mappings[field_attributes][#{index}][field]", issue_options, {}, class: 'form-select w-75 field-select', data: { behavior: 'dradis-field-select issue-field-select', header: header, 'combobox-config': 'no-combobox' } %>
+                  <%= f.text_field "mappings[field_attributes][#{index}][custom_field]", disabled: true, class: 'form-control w-75 mt-1 custom-field-input d-none', style: 'min-width: 0;', placeholder: 'Enter field name', data: { behavior: 'custom-destination-input' } %>
 
-                  <% evidence_options = @rtp_fields[:evidence].any? ? options_for_select(@rtp_fields[:evidence]) : options_for_select([[header, header]], disabled: header, selected: header) %>
+                  <% evidence_options = @rtp_fields[:evidence].any? ? options_for_select(@rtp_fields[:evidence] + [custom_field_option]) : options_for_select([[header, header]], disabled: header, selected: header) %>
                   <%= f.select "mappings[field_attributes][#{index}][field]", evidence_options, {}, disabled: true, class: 'form-select w-75 field-select d-none', data: { behavior: 'dradis-field-select evidence-field-select', header: header, 'combobox-config': 'no-combobox' } %>
+                  <%= f.text_field "mappings[field_attributes][#{index}][custom_field]", disabled: true, class: 'form-control w-75 mt-1 custom-field-input d-none', style: 'min-width: 0;', placeholder: 'Enter field name', data: { behavior: 'custom-destination-input' } %>
 
                   <%= f.select "mappings[field_attributes][#{index}][field]", [['N/A', '']], {}, disabled: true, class: 'form-select w-75 field-select d-none', data: { behavior: 'dradis-field-select empty-field-select', header: header, 'combobox-config': 'no-combobox' } %>
                 </div>

data/lib/dradis/plugins/csv/field_processor.rb

@@ -0,0 +1,10 @@
+module Dradis::Plugins::CSV
+  class FieldProcessor < Dradis::Plugins::Upload::FieldProcessor
+    # data is a Hash of the current row, keyed by normalized header (see
+    # Importer#normalized_row), matching how MappingForm normalizes
+    # headers when it stores each field's source_field/content.
+    def value(args = {})
+      data[args[:field]]
+    end
+  end
+end

data/lib/dradis/plugins/csv/gem_version.rb

@@ -8,7 +8,7 @@ module Dradis
 
       module VERSION
         MAJOR = 5
-        MINOR = 2
+        MINOR = 4
         TINY = 0
         PRE = nil
 

data/lib/dradis/plugins/csv/importer.rb

@@ -1,29 +1,54 @@
 module Dradis::Plugins::CSV
   class Importer < Dradis::Plugins::Upload::Importer
+    # Sources are dynamic (one per mapped CSV format), so report the
+    # currently known sources grouped by entity for the Mappings Manager.
     def self.templates
-      {}
+      sources = Dradis::Plugins::CSV.mapping_sources.map(&:to_s)
+
+      {
+        evidence: sources.grep(/\Aevidence_/),
+        issue: sources.grep(/\Aissue_/)
+      }
     end
 
-    def import(params={})
-      logger.info { 'Uploading CSV file...' }
+    # Runs as part of the standard upload flow, before the user would reach
+    # the column mapper. If this project already has a saved mapping for the
+    # file's headers, import immediately using it, so a recognized format
+    # never needs re-mapping and gets the same treatment (Rules Engine
+    # included) as any other plugin's upload. Unrecognized formats no-op
+    # here; UploadController sends the user to the mapper instead.
+    def import(params = {})
+      return false if mapping_service.destination.blank?
+
+      headers = CSV.open(params[:file], &:readline)
 
-      logger.info { 'Done' }
+      unless Dradis::Plugins::CSV.mapping_exists?(headers: headers, destination: mapping_service.destination)
+        logger.info { 'No saved mapping found for this CSV format.' }
+        return false
+      end
+
+      import_rows(file: params[:file], headers: headers)
     end
 
-    def import_csv(params)
-      logger.info { 'Worker process starting background task.' }
+    # Entry point for both the column mapper form submission (see
+    # MappingImportJob) and the auto-recognized import above. mappings is the
+    # raw form submission and is only consulted as a fallback, for whichever
+    # of identifier/node/entity fields has no saved Mapping to read from
+    # instead (e.g. no RTP is set, so MappingForm never persisted one).
+    def import_rows(file:, headers:, mappings: nil)
+      filename = File.basename(file)
+      @issue_lookup = {}
 
-      mappings_groups = params[:mappings].group_by { |index, mapping| mapping['type'] }
+      @issue_source, @issue_fields, id_index = mapped_fields(entity: :issue, headers: headers)
+      @evidence_source, @evidence_fields, node_index = mapped_fields(entity: :evidence, headers: headers)
 
-      filename = File.basename(params[:file])
-      id_index = Integer(mappings_groups['identifier']&.first&.first, exception: false)
+      mappings_groups = (mappings || {}).group_by { |index, mapping| mapping['type'] }
+      id_index ||= Integer(mappings_groups['identifier']&.first&.first, exception: false)
+      @node_index = node_index || Integer(mappings_groups['node']&.first&.first, exception: false)
       @evidence_mappings = mappings_groups['evidence'] || []
-      @issue_lookup = {}
       @issue_mappings = mappings_groups['issue'] || []
-      @node_index = Integer(mappings_groups['node']&.first&.first, exception: false)
-
 
-      CSV.foreach(params[:file], headers: true).with_index do |row, index|
+      CSV.foreach(file, headers: true).with_index do |row, index|
         csv_id = row[id_index] || "#{filename}-#{index}"
         process_issue(csv_id: csv_id, row: row)
         process_node(csv_id: csv_id, row: row)
@@ -34,9 +59,39 @@ module Dradis::Plugins::CSV
 
     private
 
-    attr_accessor :evidence_mappings, :issue_lookup, :issue_mappings, :node_index
+    attr_accessor :evidence_fields, :evidence_mappings, :evidence_source,
+      :issue_fields, :issue_lookup, :issue_mappings, :issue_source, :node_index
+
+    # The saved Mapping's fields (with their content templates) for this
+    # entity, excluding the reserved identifier/node field (that drives
+    # csv_id/node_label directly in import_rows, not entity content) but
+    # returning its column index instead, so import_rows can read it
+    # straight off the Mapping rather than the column mapper form. Returns
+    # all nils when there's no RTP or no saved mapping, so build_text falls
+    # back to reading the CSV directly instead of applying a template, and
+    # import_rows falls back to the form for the reserved index too.
+    def mapped_fields(entity:, headers:)
+      return [nil, nil, nil] unless mapping_service.destination.present?
+
+      source = Dradis::Plugins::CSV.mapping_source(headers: headers, entity: entity)
+      mapping = Dradis::Plugins::CSV.get_mapping(source: source, destination: mapping_service.destination)
+
+      return [nil, nil, nil] unless mapping
+
+      reserved_field = entity == :issue ? Mapping::IDENTIFIER_FIELD : Mapping::NODE_LABEL_FIELD
+      reserved, fields = mapping.mapping_fields.partition { |field| field.destination_field == reserved_field }
+      reserved = reserved.first
+
+      reserved_index = reserved && headers.index do |header|
+        Dradis::Plugins::CSV.normalize_header(header) == reserved.source_field
+      end
+
+      [source, fields, reserved_index]
+    end
+
+    def build_text(mappings:, source:, fields:, row:)
+      return apply_template(source: source, fields: fields, row: row) if fields
 
-    def build_text(mappings:, row:)
       mappings.map do |index, mapping|
         next if project.report_template_properties && mapping['field'].blank?
 
@@ -46,17 +101,23 @@ module Dradis::Plugins::CSV
       end.compact.join("\n\n")
     end
 
+    def apply_template(source:, fields:, row:)
+      data = row.to_h.transform_keys { |header| Dradis::Plugins::CSV.normalize_header(header) }
+
+      mapping_service.apply_mapping(source: source, data: data, mapping_fields: fields)
+    end
+
     def process_evidence(csv_id:, node:, row:)
       logger.info{ "\t\t => Creating evidence: (node: #{node.label}, plugin_id: #{csv_id})" }
 
       issue = issue_lookup[csv_id]
-      evidence_content = build_text(mappings: @evidence_mappings, row: row)
+      evidence_content = build_text(mappings: evidence_mappings, source: evidence_source, fields: evidence_fields, row: row)
       content_service.create_evidence(issue: issue, node: node, content: evidence_content)
     end
 
     def process_issue(csv_id:, row:)
       logger.info { "\t => Creating new issue (plugin_id: #{csv_id})" }
-      issue_text = build_text(mappings: issue_mappings, row: row)
+      issue_text = build_text(mappings: issue_mappings, source: issue_source, fields: issue_fields, row: row)
       issue = content_service.create_issue(text: issue_text, id: csv_id)
 
       issue_lookup[csv_id] = issue

data/lib/dradis/plugins/csv/mapping.rb

@@ -0,0 +1,77 @@
+module Dradis::Plugins::CSV
+  # Unlike other integrations, CSV files don't have a fixed structure, so the
+  # list of sources can't be defined upfront. Instead, a new source is
+  # registered every time a user maps a new CSV format (i.e. a new set of
+  # column headers) through the column mapper (see MappingForm). The class
+  # methods below give Dradis::Plugins::Mappings::Base's default,
+  # constant-backed implementations (mapping_sources, source_fields,
+  # default_mapping) database-backed overrides instead, so there's no
+  # DEFAULT_MAPPING/SOURCE_FIELDS constant here for them to fall back to.
+  module Mapping
+    # Reserved destination fields that carry row metadata instead of entity
+    # content. They're stored alongside the content fields but are excluded
+    # from the entity text during import.
+    IDENTIFIER_FIELD = 'plugin_id'.freeze
+    NODE_LABEL_FIELD = 'node_label'.freeze
+  end
+
+  def self.default_mapping(_source)
+    {}
+  end
+
+  # Excludes the reserved identifier/node fields from the destination field
+  # list: they carry row metadata, not entity content (see Importer#mapped_fields,
+  # which excludes them the same way when building the import template).
+  def self.field_names(source:, destination: nil, field_type: 'destination')
+    super.reject do |field|
+      field_type == 'destination' &&
+        [Mapping::IDENTIFIER_FIELD, Mapping::NODE_LABEL_FIELD].include?(field)
+    end
+  end
+
+  # Whether a format matching these headers has already been mapped for this
+  # destination, i.e. whether either its issue or evidence Mapping (or both)
+  # was saved. Shared by Importer#import (to recognize a format on upload)
+  # and UploadController (to skip the mapper for a recognized format).
+  def self.mapping_exists?(headers:, destination:)
+    sources = %i[issue evidence].map { |entity| mapping_source(headers: headers, entity: entity) }
+
+    ::Mapping.exists?(component: component, source: sources, destination: destination)
+  end
+
+  def self.mapping_sources
+    ::Mapping.where(component: component).distinct.pluck(:source).map(&:to_sym)
+  end
+
+  # The source name for a CSV format: the entity (issue/evidence) the mapping
+  # populates, plus its normalized headers joined with '/', so the Mappings
+  # Manager shows something recognizable (e.g. issue_severity/title) instead
+  # of an opaque digest. Headers are sorted so reordering columns doesn't
+  # produce a new source; any '/' inside a header name is replaced so it
+  # can't be mistaken for the join separator. The entity prefix also lets
+  # Importer.templates group sources by entity with a simple regex.
+  def self.mapping_source(headers:, entity:)
+    normalized = headers.map { |header| normalize_header(header).gsub('/', '-') }.sort
+
+    "#{entity}_#{normalized.join('/')}"
+  end
+
+  def self.normalize_header(header)
+    header.to_s.delete(" \t\r\n")
+  end
+
+  # We're building a dynamic source here since we can't rely on a fixed set of
+  # sources unlike the other integrations.
+  def self.sample(source)
+    source_fields(source).index_with { |field| field }.to_json
+  end
+
+  def self.source_fields(source)
+    ::MappingField.
+      joins(:mapping).
+      where(mappings: { component: component, source: source.to_s }).
+      where.not(source_field: 'Custom Text').
+      distinct.
+      pluck(:source_field)
+  end
+end

data/lib/dradis/plugins/csv/mapping_form.rb

@@ -0,0 +1,99 @@
+module Dradis::Plugins::CSV
+  # Converts the column mapper's form structure into persisted Mapping
+  # records, one per entity (issue/evidence), so a CSV format only needs to
+  # be mapped once: #save is the forward direction (form -> records), run
+  # the first time a format is mapped. Later uploads of a recognized format
+  # read straight from those records instead (see Importer#mapped_fields),
+  # rather than reconstructing the form.
+  class MappingForm
+    # Mapper column types that produce fields for each entity: identifier
+    # columns are stored with the issue mapping, node columns with evidence.
+    ENTITY_TYPES = {
+      issue: %w[issue identifier],
+      evidence: %w[evidence node]
+    }.freeze
+
+    attr_reader :column_mappings, :destination, :headers, :rtp_fields
+
+    # rtp_fields is { issue: [...field names...], evidence: [...] }: the
+    # destination fields actually defined on the RTP, used to gate #save (see
+    # below).
+    def initialize(destination:, headers:, rtp_fields:)
+      @destination = destination
+      @headers = headers
+      @rtp_fields = rtp_fields
+    end
+
+    # Persists one Mapping per entity that has assigned columns, so a
+    # newly-mapped CSV format is recognized automatically on future uploads
+    # (see Importer#import). Only ever called for formats with no existing
+    # mapping: Importer#import and UploadController#new short-circuit before
+    # the mapper is reached once a format already has a saved mapping.
+    #
+    # column_mappings is the mapper form submission, keyed by column index:
+    #   {
+    #     '0' => { 'type' => 'node' },
+    #     '1' => { 'type' => 'issue', 'field' => 'Title' },
+    #     '2' => { 'type' => 'identifier' },
+    #     '3' => { 'type' => 'evidence', 'field' => 'Port' }
+    #   }
+    #
+    # An entity with no RTP fields defined has nowhere valid to point a
+    # mapping, so it's skipped entirely (including its identifier/node
+    # column, if any) rather than saving a mapping with a bogus destination.
+    def save(column_mappings:)
+      @column_mappings = column_mappings
+
+      return if destination.blank?
+
+      ::Mapping.transaction do
+        ENTITY_TYPES.keys.each do |entity|
+          next if rtp_fields[entity].blank?
+
+          fields = fields_for(entity)
+          next if fields.empty?
+
+          source = Dradis::Plugins::CSV.mapping_source(headers: headers, entity: entity)
+          mapping = ::Mapping.create!(component: component, source: source, destination: destination)
+          mapping.mapping_fields.create!(fields)
+        end
+      end
+    end
+
+    private
+
+    def component
+      Dradis::Plugins::CSV.component
+    end
+
+    def fields_for(entity)
+      types = ENTITY_TYPES.fetch(entity)
+
+      column_mappings.filter_map do |index, assignment|
+        next unless types.include?(assignment['type'])
+
+        header = Dradis::Plugins::CSV.normalize_header(headers[index.to_i])
+        next if header.blank?
+
+        destination_field =
+          case assignment['type']
+          when 'identifier'
+            Mapping::IDENTIFIER_FIELD
+          when 'node'
+            Mapping::NODE_LABEL_FIELD
+          else
+            field = assignment['field'] == 'Custom Field' ? assignment['custom_field'] : assignment['field']
+            next if field.blank?
+
+            field
+          end
+
+        {
+          source_field: header,
+          content: "{{ csv[#{header}] }}",
+          destination_field: destination_field
+        }
+      end
+    end
+  end
+end

data/lib/dradis/plugins/csv.rb

@@ -7,5 +7,8 @@ module Dradis
 end
 
 require 'dradis/plugins/csv/engine'
+require 'dradis/plugins/csv/field_processor'
 require 'dradis/plugins/csv/importer'
+require 'dradis/plugins/csv/mapping'
+require 'dradis/plugins/csv/mapping_form'
 require 'dradis/plugins/csv/version'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dradis-csv
 version: !ruby/object:Gem::Version
-  version: 5.2.0
+  version: 5.4.0
 platform: ruby
 authors:
 - Daniel Martin
@@ -56,12 +56,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".github/pull_request_template.md"
-- ".gitignore"
 - CHANGELOG.md
-- CHANGELOG.template
-- CONTRIBUTING.md
-- Gemfile
 - LICENSE
 - README.md
 - Rakefile
@@ -75,19 +70,15 @@ files:
 - app/views/dradis/plugins/csv/upload/new.html.erb
 - config/initializers/inflections.rb
 - config/routes.rb
-- dradis-csv.gemspec
 - lib/dradis-csv.rb
 - lib/dradis/plugins/csv.rb
 - lib/dradis/plugins/csv/engine.rb
+- lib/dradis/plugins/csv/field_processor.rb
 - lib/dradis/plugins/csv/gem_version.rb
 - lib/dradis/plugins/csv/importer.rb
+- lib/dradis/plugins/csv/mapping.rb
+- lib/dradis/plugins/csv/mapping_form.rb
 - lib/dradis/plugins/csv/version.rb
-- spec/features/upload_spec.rb
-- spec/fixtures/files/simple (copy).csv
-- spec/fixtures/files/simple.csv
-- spec/fixtures/files/simple_malformed.csv
-- spec/jobs/dradis/plugins/csv/mapping_import_job_spec.rb
-- spec/lib/dradis/plugins/csv/importer_spec.rb
 homepage: http://dradis.com
 licenses:
 - GPL-2
@@ -109,10 +100,4 @@ requirements: []
 rubygems_version: 3.6.9
 specification_version: 4
 summary: CSV add-on for the Dradis Framework.
-test_files:
-- spec/features/upload_spec.rb
-- spec/fixtures/files/simple (copy).csv
-- spec/fixtures/files/simple.csv
-- spec/fixtures/files/simple_malformed.csv
-- spec/jobs/dradis/plugins/csv/mapping_import_job_spec.rb
-- spec/lib/dradis/plugins/csv/importer_spec.rb
+test_files: []

data/.github/pull_request_template.md

@@ -1,45 +0,0 @@
-Please review [CONTRIBUTING.md](https://github.com/dradis/dradis-ce/blob/develop/CONTRIBUTING.md) and remove this line.
-
-### Summary
-
-Provide a general description of the code changes in your pull
-request... were there any bugs you had fixed? If so, mention them. If
-these bugs have open GitHub issues, be sure to tag them here as well,
-to keep the conversation linked together.
-
-
-### Testing Steps
-
-Provide steps to test functionality, described in detail for someone not familiar with this part of the application / code base
-
-
-### Other Information
-
-If there's anything else that's important and relevant to your pull
-request, mention that information here. This could include
-benchmarks, or other information.
-
-Thanks for contributing to Dradis!
-
-
-### Copyright assignment
-
-Collaboration is difficult with commercial closed source but we want
-to keep as much of the OSS ethos as possible available to users
-who want to fix it themselves.
-
-In order to unambiguously own and sell Dradis Framework commercial
-products, we must have the copyright associated with the entire
-codebase. Any code you create which is merged must be owned by us.
-That's not us trying to be a jerks, that's just the way it works.
-
-You can delete this section, but the following sentence needs to
-remain in the PR's description:
-
-> I assign all rights, including copyright, to any future Dradis
-> work by myself to Security Roots.
-
-### Check List
-
-- [ ] Added a CHANGELOG entry
-- [ ] Added specs

data/.gitignore

@@ -1,5 +0,0 @@
-.DS_Store
-
-pkg/
-
-Gemfile.lock

data/CHANGELOG.template

@@ -1,12 +0,0 @@
-[v#.#.#] ([month] [YYYY])
-  - [future tense verb] [feature]
-  - Upgraded gems:
-    - [gem]
-  - Bugs fixes:
-    - [future tense verb] [bug fix]
-    - Bug tracker items:
-      - [item]
-  - Security Fixes:
-    - High: (Authenticated|Unauthenticated) (admin|author|contributor) [vulnerability description]
-    - Medium: (Authenticated|Unauthenticated) (admin|author|contributor) [vulnerability description]
-    - Low: (Authenticated|Unauthenticated) (admin|author|contributor) [vulnerability description]

data/CONTRIBUTING.md

@@ -1,3 +0,0 @@
-# Plugin contribution guidelines
-
-See the Dradis Framework's [CONTRIBUTING.md](https://github.com/dradis/dradisframework/blob/master/CONTRIBUTING.md)

data/Gemfile

@@ -1,23 +0,0 @@
-source 'https://rubygems.org'
-
-# Declare your gem's dependencies in dradispro-duoweb.gemspec.
-# Bundler will treat runtime dependencies like base dependencies, and
-# development dependencies will be added by default to the :development group.
-gemspec
-
-# jquery-rails is used by the dummy application
-# gem "jquery-rails"
-
-# Declare any dependencies that are still in development here instead of in
-# your gemspec. These might include edge Rails or gems from your path or
-# Git. Remember to move these dependencies to your gemspec before releasing
-# your gem to rubygems.org.
-
-# To use debugger
-# gem 'debugger'
-
-if Dir.exist?('../dradis-plugins')
-  gem 'dradis-plugins', path: '../dradis-plugins'
-else
-  gem 'dradis-plugins', github: 'dradis/dradis-plugins'
-end

data/dradis-csv.gemspec

@@ -1,25 +0,0 @@
-$:.push File.expand_path('../lib', __FILE__)
-require 'dradis/plugins/csv/version'
-version = Dradis::Plugins::CSV::VERSION::STRING
-
-# Describe your gem and declare its dependencies:
-Gem::Specification.new do |spec|
-  spec.platform    = Gem::Platform::RUBY
-  spec.name        = 'dradis-csv'
-  spec.version     = version
-  spec.summary     = 'CSV add-on for the Dradis Framework.'
-  spec.description = 'This add-on allows you to upload and parse CSV output into Dradis.'
-
-  spec.license     = 'GPL-2'
-
-  spec.authors     = ['Daniel Martin']
-  spec.homepage    = 'http://dradis.com'
-
-  spec.files       = `git ls-files`.split("\n")
-  spec.executables = spec.files.grep(%r{^bin/}).map { |f| File.basename(f) }
-  spec.test_files  = spec.files.grep(%r{^(spec|features)/})
-
-  spec.add_dependency 'dradis-plugins', '>= 4.0'
-  spec.add_development_dependency 'bundler', '~> 2.0'
-  spec.add_development_dependency 'rake'
-end

data/spec/features/upload_spec.rb

@@ -1,315 +0,0 @@
-require 'rails_helper'
-
-# To run, execute from Dradis main app folder:
-#   bin/rspec [dradis-plugins path]/spec/features/upload_spec.rb
-
-describe 'upload feature', js: true do
-  before do
-    login_to_project_as_user
-    visit project_upload_path(@project)
-  end
-
-  context 'uploading a CSV file' do
-    let(:file_path) { File.expand_path('../fixtures/files/simple.csv', __dir__) }
-    before do
-      @headers = CSV.open(file_path, &:readline)
-
-      find('#state + .combobox').click
-      find('#state ~ .combobox-menu .combobox-option', text: 'Published').click
-
-      find('#uploader + .combobox').click
-      find('#uploader ~ .combobox-menu .combobox-option', text: 'Dradis::Plugins::CSV').click
-
-      attach_file 'file', file_path, visible: false, disabled: false
-
-      expect(page).to have_text('CSV Upload Mapping', wait: 30)
-    end
-
-    it 'redirects to the mapping page' do
-      expect(current_path).to eq(csv.new_project_upload_path(@project))
-    end
-
-    it 'lists the fields in the table' do
-      within('tbody') do
-        @headers.each do |header|
-          expect(page).to have_selector('td', text: header)
-        end
-      end
-    end
-
-    context 'mapping CSV columns' do
-      context 'when identifier not selected' do
-        it 'shows a validation message on the page' do
-          within all('tbody tr')[3] do
-            select 'Evidence Field'
-          end
-
-          click_button 'Import CSV'
-          expect(page).to have_text('An Issue ID must be selected.')
-        end
-      end
-
-      context 'when there are evidence type but no node type selected' do
-        it 'shows a validation message on the page' do
-          within all('tbody tr')[2] do
-            select 'Issue ID'
-          end
-
-          within all('tbody tr')[3] do
-            select 'Evidence Field'
-          end
-
-          click_button 'Import CSV'
-          expect(page).to have_text('A Node Label must be selected to import evidence records.')
-        end
-      end
-
-      context 'valid states' do
-        it 'imports the issues based on the selected state' do
-          select 'Issue ID', from: 'mappings[field_attributes][0][type]'
-          select 'Node', from: 'mappings[field_attributes][3][type]'
-          select 'Evidence Field', from: 'mappings[field_attributes][4][type]'
-          select 'Evidence Field', from: 'mappings[field_attributes][5][type]'
-
-          perform_enqueued_jobs do
-            click_button 'Import CSV'
-
-            find('#console .log', wait: 30, match: :first)
-
-            expect(page).to have_text('Worker process completed.')
-
-            expect(Issue.published.count).to eq(1)
-          end
-        end
-      end
-
-      context 'invalid states' do
-        it 'imports the issues as draft' do
-          select 'Issue ID', from: 'mappings[field_attributes][0][type]'
-          select 'Node', from: 'mappings[field_attributes][3][type]'
-          select 'Evidence Field', from: 'mappings[field_attributes][4][type]'
-          select 'Evidence Field', from: 'mappings[field_attributes][5][type]'
-
-          page.execute_script(<<~JS)
-            const select = document.querySelector('#state');
-            select.value = 'tampered_value';
-          JS
-
-          perform_enqueued_jobs do
-            click_button 'Import CSV'
-
-            find('#console .log', wait: 30, match: :first)
-
-            expect(page).to have_text('Worker process completed.')
-
-            expect(Issue.published.count).to eq(0)
-          end
-        end
-      end
-
-      context 'when project does not have RTP' do
-        it 'imports all columns as fields' do
-          select 'Issue ID', from: 'mappings[field_attributes][0][type]'
-          select 'Node', from: 'mappings[field_attributes][3][type]'
-          select 'Evidence Field', from: 'mappings[field_attributes][4][type]'
-          select 'Evidence Field', from: 'mappings[field_attributes][5][type]'
-
-          perform_enqueued_jobs do
-            click_button 'Import CSV'
-
-            find('#console .log', wait: 30, match: :first)
-
-            expect(page).to have_text('Worker process completed.')
-
-            issue = Issue.last
-            expect(issue.fields).to eq({ 'Description' => 'Test CSV', 'Title' => 'SQL Injection', 'VulnerabilityCategory' =>'High', 'plugin' => 'csv', 'plugin_id' => '1' })
-
-            node = issue.affected.first
-            expect(node.label).to eq('10.0.0.1')
-
-            evidence = node.evidence.first
-            expect(evidence.fields).to eq({ 'Label' => '10.0.0.1', 'Title' => 'SQL Injection', 'Location' => '10.0.0.1', 'Port' => '443' })
-          end
-        end
-      end
-
-      context 'when project have RTP' do
-        before do
-          rtp = create(:report_template_properties, evidence_fields: evidence_fields, issue_fields: issue_fields)
-          @project.update(report_template_properties: rtp)
-
-          page.refresh
-        end
-
-        context 'without fields' do
-          let (:evidence_fields) { [] }
-          let (:issue_fields) { [] }
-
-          it 'creates records with fields from the headers' do
-            select 'Issue ID', from: 'mappings[field_attributes][0][type]'
-            select 'Node', from: 'mappings[field_attributes][3][type]'
-            select 'Evidence Field', from: 'mappings[field_attributes][4][type]'
-            select 'Evidence Field', from: 'mappings[field_attributes][5][type]'
-
-            perform_enqueued_jobs do
-              click_button 'Import CSV'
-
-              find('#console .log', wait: 30, match: :first)
-
-              expect(page).to have_text('Worker process completed.')
-
-              issue = Issue.last
-              expect(issue.fields).to eq({ 'Description' => 'Test CSV', 'Title' => 'SQL Injection', 'Vulnerability Category' =>'High', 'plugin' => 'csv', 'plugin_id' => '1' })
-
-              node = issue.affected.first
-              expect(node.label).to eq('10.0.0.1')
-
-              evidence = node.evidence.first
-              expect(evidence.fields).to eq({ 'Label' => '10.0.0.1', 'Location' => '10.0.0.1', 'Title' => 'SQL Injection', 'Port' => '443' })
-            end
-          end
-        end
-
-        context 'with fields' do
-          let (:evidence_fields) {
-            [
-              { name: 'Location', type: :string, default: true },
-              { name: 'Port', type: :string, default: true}
-            ]
-          }
-
-          let (:issue_fields) {
-            [
-              { name: 'Title', type: :string, default: true },
-              { name: 'Description', type: :string, default: true},
-              { name: 'Severity', type: :string, default: true}
-            ]
-          }
-
-          it 'shows the available fields for the selected type' do
-            select 'Issue Field', from: 'mappings[field_attributes][1][type]'
-
-            issue_fields.each do |field|
-              expect(page).to have_selector('option', text: field[:name])
-            end
-
-            select 'Evidence Field', from: 'mappings[field_attributes][4][type]'
-
-            evidence_fields.each do |field|
-              expect(page).to have_selector('option', text: field[:name])
-            end
-          end
-
-          it 'can select which columns to import' do
-            select 'Issue ID', from: 'mappings[field_attributes][0][type]'
-
-            select 'Issue Field', from: 'mappings[field_attributes][1][type]'
-            select 'Title', from: 'mappings[field_attributes][1][field]'
-
-            select 'Issue Field', from: 'mappings[field_attributes][2][type]'
-            select 'Description', from: 'mappings[field_attributes][2][field]'
-
-            select 'Node', from: 'mappings[field_attributes][3][type]'
-
-            select 'Evidence Field', from: 'mappings[field_attributes][4][type]'
-            select 'Location', from: 'mappings[field_attributes][4][field]'
-
-            select 'Evidence Field', from: 'mappings[field_attributes][5][type]'
-            select 'Port', from: 'mappings[field_attributes][5][field]'
-
-            select 'Issue Field', from: 'mappings[field_attributes][6][type]'
-            select 'Severity', from: 'mappings[field_attributes][6][field]'
-
-            perform_enqueued_jobs do
-              click_button 'Import CSV'
-
-              find('#console .log', wait: 30, match: :first)
-
-              expect(page).to have_text('Worker process completed.')
-
-              issue = Issue.last
-              expect(issue.fields).to eq({ 'Description' => 'Test CSV', 'Title' => 'SQL Injection', 'Severity' => 'High', 'plugin' => 'csv', 'plugin_id' => '1' })
-
-              node = issue.affected.first
-              expect(node.label).to eq('10.0.0.1')
-
-              evidence = node.evidence.first
-              expect(evidence.fields).to eq({ 'Label' => '10.0.0.1', 'Location' => '10.0.0.1', 'Title' => 'SQL Injection', 'Port' => '443' })
-            end
-          end
-        end
-
-        context 'when no evidence fields' do
-          let (:evidence_fields) { [] }
-          let (:issue_fields) { [] }
-
-          it 'still creates evidence record' do
-            within all('tbody tr')[0] do
-              select 'Issue ID'
-            end
-
-            within all('tbody tr')[1] do
-              select 'Issue Field'
-            end
-
-            within all('tbody tr')[3] do
-              select 'Node'
-            end
-
-            within all('tbody tr')[5] do
-              select 'Issue Field'
-            end
-
-            perform_enqueued_jobs do
-              click_button 'Import CSV'
-
-              find('#console .log', wait: 30, match: :first)
-
-              expect(page).to have_text('Worker process completed.')
-
-              issue = Issue.last
-              expect(issue.fields).to include({ 'Title' => 'SQL Injection', 'plugin' => 'csv', 'plugin_id' => '1' })
-
-              node = issue.affected.first
-              expect(node.label).to eq('10.0.0.1')
-
-              evidence = node.evidence.first
-              expect(evidence.content).to eq('')
-            end
-          end
-        end
-      end
-    end
-  end
-
-  describe 'CSV file samples' do
-    before do
-      find('#uploader + .combobox').click
-      find('#uploader ~ .combobox-menu .combobox-option', text: 'Dradis::Plugins::CSV').click
-
-      attach_file 'file', file_path, visible: false, disabled: false
-    end
-
-    context 'uploading a malformed CSV file' do
-      let(:file_path) { File.expand_path('../fixtures/files/simple_malformed.csv', __dir__) }
-
-      it 'redirects to upload manager with error' do
-        find('.alert.alert-danger', wait: 30)
-
-        expect(page).to have_text('The uploaded file is not a valid CSV file')
-        expect(current_path).to eq(main_app.project_upload_manager_path(@project))
-      end
-    end
-
-    context 'uploading any file other than CSV' do
-      let(:file_path) { Rails.root.join('spec/fixtures/files/rails.png') }
-
-      it 'redirects to upload manager with error' do
-        find('.alert.alert-danger', wait: 30)
-
-        expect(page).to have_text('The uploaded file is not a CSV file.')
-        expect(current_path).to eq(main_app.project_upload_manager_path(@project))
-      end
-    end
-  end
-end

data/spec/fixtures/files/simple (copy).csv

@@ -1,2 +0,0 @@
-"Id","Title","Description","Host","Location","Port","Vulnerability Category"
-"1","SQL Injection","Test CSV","10.0.0.1","10.0.0.1","443","High"

data/spec/fixtures/files/simple.csv

@@ -1,2 +0,0 @@
-"Id","Title","Description","Host","Location","Port","Vulnerability Category"
-"1","SQL Injection","Test CSV","10.0.0.1","10.0.0.1","443","High"

data/spec/fixtures/files/simple_malformed.csv

@@ -1,2 +0,0 @@
-"Id";"Title";"Description";"Host";"Location";"Port"
-"1";"SQL Injection";"Test CSV";"10.0.0.1";"10.0.0.1";"443"

data/spec/jobs/dradis/plugins/csv/mapping_import_job_spec.rb

@@ -1,30 +0,0 @@
-require 'rails_helper'
-
-RSpec.describe Dradis::Plugins::CSV::MappingImportJob do
-  let(:file) { File.expand_path('../../../.../../../fixtures/files/simple.csv', __dir__) }
-
-  let(:perform_job) do
-    described_class.new.perform(
-      default_user_id: create(:user).id,
-      file: file,
-      mappings: {},
-      project_id: create(:project).id,
-      uid: 1
-    )
-  end
-
-  describe '#perform' do
-    it 'calls Importer#import_csv' do
-      dbl = double('Importer')
-      allow(Dradis::Plugins::CSV::Importer).to receive(:new).and_return(dbl)
-      expect(dbl).to receive(:import_csv).and_return(true)
-
-      perform_job
-    end
-
-    it 'writes a known final line in the log' do
-      perform_job
-      expect(Log.last.text).to eq 'Worker process completed.'
-    end
-  end
-end

data/spec/lib/dradis/plugins/csv/importer_spec.rb

@@ -1,140 +0,0 @@
-require 'rails_helper'
-
-RSpec.describe Dradis::Plugins::CSV::Importer do
-  let(:file) { File.expand_path('../../../.../../../fixtures/files/simple.csv', __dir__) }
-  let(:project) { create(:project) }
-
-  let(:instance) do
-    described_class.new(
-      default_user_id: create(:user).id,
-      logger: Log.new(uid: 1),
-      plugin: Dradis::Plugins::CSV,
-      project_id: project.id
-    )
-  end
-
-  let(:import_csv) do
-    instance.import_csv(file: file, mappings: mappings)
-  end
-
-  describe '#import_csv' do
-    context 'when project has RTP' do
-      let(:mappings) do
-        {
-          '0' => { 'type' => 'identifier' },
-          '1' => { 'type' => 'issue', 'field' => 'MyTitle' },
-          '3' => { 'type' => 'node', 'field' => '' },
-          '4' => { 'type' => 'evidence', 'field' => 'MyLocation' },
-          '5' => { 'type' => 'evidence', 'field' => '' }
-        }
-      end
-
-      before do
-        project.update(report_template_properties: create(:report_template_properties))
-      end
-
-      it 'uses the field as Dradis Field' do
-        import_csv
-
-        issue = Issue.first
-        expect(issue.fields).to eq({ 'MyTitle' => 'SQL Injection', 'plugin' => 'csv', 'plugin_id' => '1' })
-
-        node = issue.affected.first
-        expect(node.label).to eq('10.0.0.1')
-
-        evidence = node.evidence.first
-        expect(evidence.fields).to eq({ 'Label' => '10.0.0.1', 'Title' => '(No #[Title]# field)', 'MyLocation' => '10.0.0.1' })
-      end
-    end
-
-    context 'when project does not have RTP' do
-      let(:mappings) do
-        {
-          '0' => { 'type' => 'identifier' },
-          '1' => { 'type' => 'issue', 'field' => 'MyTitle' },
-          '3' => { 'type' => 'node', 'field' => '' },
-          '4' => { 'type' => 'evidence', 'field' => 'MyLocation' },
-          '5' => { 'type' => 'evidence', 'field' => '' },
-          '6' => { 'type' => 'issue', 'field' => '' }
-        }
-      end
-
-      it 'uses the column name as Dradis Field' do
-        import_csv
-
-        issue = Issue.first
-        expect(issue.fields).to eq({ 'Title' => 'SQL Injection', 'VulnerabilityCategory' => 'High', 'plugin' => 'csv', 'plugin_id' => '1' })
-
-        node = issue.affected.first
-        expect(node.label).to eq('10.0.0.1')
-
-        evidence = node.evidence.first
-        expect(evidence.fields).to eq({ 'Label' => '10.0.0.1', 'Location' => '10.0.0.1', 'Port' => '443', 'Title' => 'SQL Injection' })
-      end
-
-      it 'strips out whitespace from column header' do
-        import_csv
-
-        issue = Issue.first
-        expect(issue.fields.keys).to include('VulnerabilityCategory')
-      end
-    end
-
-    context 'when mapping does not have a node type' do
-      let(:mappings) do
-        {
-          '0' => { 'type' => 'identifier' },
-          '1' => { 'type' => 'issue' },
-          '4' => { 'type' => 'evidence' }
-        }
-      end
-
-      it 'does not create node and evidence' do
-        import_csv
-
-        issue = Issue.last
-        expect(issue.affected.length).to eq(0)
-        expect(issue.evidence.length).to eq(0)
-      end
-    end
-
-    context 'when no identifier is passed in' do
-      let(:mappings) do
-        {
-          '1' => { 'type' => 'issue' },
-          '4' => { 'type' => 'evidence' }
-        }
-      end
-
-      it 'uses filename and row index as csv_id' do
-        import_csv
-
-        issue = Issue.last
-        expect(issue.fields).to eq({ 'Title' => 'SQL Injection', 'plugin' => 'csv', 'plugin_id' => 'simple.csv-0' })
-      end
-    end
-
-    context 'when no evidence fields' do
-      let(:mappings) do
-        {
-          '0' => { 'type' => 'identifier' },
-          '1' => { 'type' => 'issue', 'field' => 'MyTitle' },
-          '3' => { 'type' => 'node', 'field' => '' }
-        }
-      end
-
-      it 'still creates evidence record' do
-        import_csv
-
-        issue = Issue.first
-        expect(issue.fields).to eq({ 'Title' => 'SQL Injection', 'plugin' => 'csv', 'plugin_id' => '1' })
-
-        node = issue.affected.first
-        expect(node.label).to eq('10.0.0.1')
-
-        evidence = node.evidence.first
-        expect(evidence.content).to eq('')
-      end
-    end
-  end
-end