etna 0.1.19 → 0.1.20

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4e4db6a6c798917fe8a1e96d7f02a2d604d259d3e2da08f083899655b2638b55
-  data.tar.gz: 989a69710cad57444da6ac526fc4728e706e0b4c891b720a8307b8207949453d
+  metadata.gz: 66704260fa8f715e91ccb8469029e93b2b0f4574af76f2700dd7001b0d7e3bbe
+  data.tar.gz: d16a63b2302f4933741eaade615cd001f615de2ba4bd67774b0c77785a920ace
 SHA512:
-  metadata.gz: bb98337dd93af44ae0bda211a413178f25ba6ad08b39c34dde855d6288ee291f26b8c629b67f24e2be56667405a93662308dca804fcf2b70aefbd41da3c8ea13
-  data.tar.gz: aed1a1f77965ef30eb67bffe021547fb92ad03b836dcd940bd98c55020309bee568981f058e5365580d29462f03dd41efb645c3b3f5575682d06e9af927e51a1
+  metadata.gz: 23158f8c57ebb498b2055354d05ccada416b8d74c4b5b9e894d261467ff28591b757bfef9e6ea3d37016c465a68bdb179a76621c23a861d4247f560a84925d58
+  data.tar.gz: 4fb492dae70d2ae361e6270b7d8178f8544ecbd388e09721a3550246ea17a2f99aea1ec466f783cdfc592d80e502a320ce17015d44757039266b4f4394e0df28

data/etna.completion

@@ -136,13 +136,87 @@ all_flag_completion_names="$all_flag_completion_names  "
 string_flag_completion_names="$string_flag_completion_names  "
 while [[ "$#" != "0" ]]; do
 if [[ "$#" == "1" ]];  then
-all_completion_names="help update_from_csv"
+all_completion_names="create_file_linking_csv help update_from_csv"
 all_completion_names="$all_completion_names $all_flag_completion_names"
 if [[ -z "$(echo $all_completion_names | xargs)" ]]; then
 return
 fi
 COMPREPLY=($(compgen -W "$all_completion_names" -- "$1"))
 return
+elif [[ "$1" == "create_file_linking_csv" ]]; then
+shift
+if [[ "$#" == "1" ]];  then
+all_completion_names="__project_name__"
+if [[ -z "$(echo $all_completion_names | xargs)" ]]; then
+return
+fi
+COMPREPLY=($(compgen -W "$all_completion_names" -- "$1"))
+return
+fi
+shift
+if [[ "$#" == "1" ]];  then
+all_completion_names="__bucket_name__"
+if [[ -z "$(echo $all_completion_names | xargs)" ]]; then
+return
+fi
+COMPREPLY=($(compgen -W "$all_completion_names" -- "$1"))
+return
+fi
+shift
+if [[ "$#" == "1" ]];  then
+all_completion_names="__attribute_name__"
+if [[ -z "$(echo $all_completion_names | xargs)" ]]; then
+return
+fi
+COMPREPLY=($(compgen -W "$all_completion_names" -- "$1"))
+return
+fi
+shift
+if [[ "$#" == "1" ]];  then
+all_completion_names="__extension__"
+if [[ -z "$(echo $all_completion_names | xargs)" ]]; then
+return
+fi
+COMPREPLY=($(compgen -W "$all_completion_names" -- "$1"))
+return
+fi
+shift
+all_flag_completion_names="$all_flag_completion_names --collection --file --regex --folder "
+string_flag_completion_names="$string_flag_completion_names --file --regex --folder "
+declare _completions_for_file="__file__"
+declare _completions_for_regex="__regex__"
+declare _completions_for_folder="__folder__"
+while [[ "$#" != "0" ]]; do
+if [[ "$#" == "1" ]];  then
+all_completion_names=""
+all_completion_names="$all_completion_names $all_flag_completion_names"
+if [[ -z "$(echo $all_completion_names | xargs)" ]]; then
+return
+fi
+COMPREPLY=($(compgen -W "$all_completion_names" -- "$1"))
+return
+elif [[ -z "$(echo $all_flag_completion_names | xargs)" ]]; then
+return
+elif [[ "$all_flag_completion_names" =~ $1\  ]]; then
+all_flag_completion_names="${all_flag_completion_names//$1\ /}"
+a=$1
+shift
+if [[ "$string_flag_completion_names" =~ $a\  ]]; then
+if [[ "$#" == "1" ]];  then
+a="${a//--/}"
+a="${a//-/_}"
+i="_completions_for_$a"
+all_completion_names="${!i}"
+COMPREPLY=($(compgen -W "$all_completion_names" -- "$1"))
+return
+fi
+shift
+fi
+else
+return
+fi
+done
+return
 elif [[ "$1" == "help" ]]; then
 shift
 all_flag_completion_names="$all_flag_completion_names  "
@@ -207,8 +281,9 @@ COMPREPLY=($(compgen -W "$all_completion_names" -- "$1"))
 return
 fi
 shift
-all_flag_completion_names="$all_flag_completion_names  "
-string_flag_completion_names="$string_flag_completion_names  "
+all_flag_completion_names="$all_flag_completion_names --json-values --hole-value "
+string_flag_completion_names="$string_flag_completion_names --hole-value "
+declare _completions_for_hole_value="__hole_value__"
 while [[ "$#" != "0" ]]; do
 if [[ "$#" == "1" ]];  then
 all_completion_names=""

data/lib/commands.rb

@@ -56,7 +56,7 @@ class EtnaApp
       def execute(host, ignore_ssl: false)
         polyphemus_client ||= Etna::Clients::Polyphemus.new(
             host: host,
-            token: token,
+            token: token(ignore_environment: true),
             ignore_ssl: ignore_ssl)
         workflow = Etna::Clients::Polyphemus::SetConfigurationWorkflow.new(
             polyphemus_client: polyphemus_client,
@@ -143,7 +143,7 @@ class EtnaApp
           tf = Tempfile.new
 
           begin
-            File.open(tf.path, 'wb') { |f| workflow.write_models_template_csv(f, project_name, target_model) }
+            File.open(tf.path, 'wb') { |f| workflow.write_models_template_csv(project_name, target_model, io: f) }
             FileUtils.cp(tf.path, file)
           ensure
             tf.close!
@@ -247,23 +247,58 @@ class EtnaApp
           include WithEtnaClients
           include WithLogger
 
+          boolean_flags << '--json-values'
+          string_flags << '--hole-value'
+
           def magma_crud
             @magma_crud ||= Etna::Clients::Magma::MagmaCrudWorkflow.new(
                 magma_client: magma_client,
                 project_name: @project_name)
           end
 
-          def execute(project_name, model_name, filepath)
+          def execute(project_name, model_name, filepath, hole_value: '_', json_values: false)
             @project_name = project_name
 
             update_attributes_workflow = Etna::Clients::Magma::UpdateAttributesFromCsvWorkflowSingleModel.new(
                 magma_crud: magma_crud,
                 project_name: project_name,
                 model_name: model_name,
-                filepath: filepath)
+                filepath: filepath,
+                hole_value: hole_value,
+                json_values: json_values)
             update_attributes_workflow.update_attributes
           end
         end
+
+        class CreateFileLinkingCsv < Etna::Command
+          include WithEtnaClients
+
+          string_flags << '--file'
+          string_flags << '--regex'
+          string_flags << '--folder'
+          boolean_flags << '--collection'
+
+          def execute(project_name, bucket_name, attribute_name, extension, collection: false, regex: "**/*/(?<identifier>.+)\\.#{extension}$", folder: "", file: "#{project_name}_#{attribute_name}.csv")
+            if folder.start_with?("/")
+              folder = folder.slice(1..-1)
+            end
+
+            regex = Regexp.new(regex)
+
+            workflow = Etna::Clients::Magma::SimpleFileLinkingWorkflow.new(
+                metis_client: metis_client,
+                project_name: project_name,
+                bucket_name: bucket_name,
+                folder: folder,
+                extension: extension,
+                attribute_name: attribute_name,
+                regex: regex,
+                file_collection: collection,
+            )
+
+            workflow.write_csv_io(filename: file)
+          end
+        end
       end
     end
   end

data/lib/etna.rb

@@ -16,6 +16,7 @@ require_relative './etna/client'
 require_relative './etna/symbolize_params'
 require_relative './etna/spec'
 require_relative './etna/clients'
+require_relative './etna/csvs'
 require_relative './etna/environment_scoped'
 
 class EtnaApp

data/lib/etna/clients/magma/formatting/models_csv.rb

@@ -1,342 +1,351 @@
 require 'ostruct'
+require_relative '../../../csvs'
 
 module Etna
   module Clients
     class Magma
-      class ModelsCsv
-        COPY_OPTIONS_SENTINEL = '$copy_from_original$'
-        COLUMNS = [
-            :comments,
-            :model_name, :identifier, :parent_model_name, :parent_link_type,
-            :attribute_name,
-            :new_attribute_name,
-            :attribute_type,
-            :link_model_name,
-            :description,
-            :display_name,
-            :format_hint,
-            :restricted,
-            :read_only,
-            :options,
-            :match,
-            :attribute_group,
-            :hidden,
-            :unique,
-            :matrix_constant,
-        ]
-
-        COLUMN_AS_BOOLEAN = -> (s) { ['true', 't', 'y', 'yes'].include?(s.downcase) }
-
-        COLUMNS_TO_ATTRIBUTES = {
-            attribute_name: :attribute_name,
-            attribute_type: [:attribute_type, -> (s) { AttributeType.new(s) }],
-            link_model_name: :link_model_name,
-            description: :desc,
-            display_name: :display_name,
-            format_hint: :format_hint,
-            restricted: [:restricted, COLUMN_AS_BOOLEAN],
-            read_only: [:read_only, COLUMN_AS_BOOLEAN],
-            options: [:validation, -> (s) { {"type" => "Array", "value" => s.split(',').map(&:strip)} }],
-            match: [:validation, -> (s) { {"type" => "Regexp", "value" => Regexp.new(s).source} }],
-            attribute_group: :attribute_group,
-            hidden: [:hidden, COLUMN_AS_BOOLEAN],
-            unique: [:unique, COLUMN_AS_BOOLEAN],
-        }
-
-        def self.apply_csv_row(changeset = ModelsChangeset.new, row = {}, &err_block)
-          changeset.tap do
-            models = changeset.models
+      module ModelsCsv
+        module Prettify
+          def prettify(name)
+            name.split('_').map(&:capitalize).join(' ')
+          end
+        end
 
-            if (matrix_constant = self.get_col_or_nil(row, :matrix_constant))
-              if matrix_constant.start_with?(COPY_OPTIONS_SENTINEL)
-                matrix_constant = matrix_constant.slice((COPY_OPTIONS_SENTINEL.length)..-1)
-                changeset.last_matrix_constant = matrix_constant
-              else
-                (changeset.matrix_constants[changeset.last_matrix_constant] ||= []) << matrix_constant
-              end
-            end
+        class Exporter < Etna::CsvExporter
+          include Prettify
+
+          def initialize
+            super([
+                :comments,
+                :model_name, :identifier, :parent_model_name, :parent_link_type,
+                :attribute_name,
+                :new_attribute_name,
+                :attribute_type,
+                :link_model_name,
+                :description,
+                :display_name,
+                :format_hint,
+                :restricted,
+                :read_only,
+                :options,
+                :match,
+                :attribute_group,
+                :hidden,
+                :unique,
+                :matrix_constant,
+            ])
+          end
 
-            template = if (model_name = self.get_col_or_nil(row, :model_name))
-              changeset.last_model_key = model_name
-              models.build_model(model_name).build_template.tap { |t| t.name = model_name }
-            else
-              last_model = changeset.last_model_key
-              if last_model.nil?
-                nil
-              else
-                models.model(last_model).build_template
-              end
+          def write_models(models, model_keys = models.model_keys.sort, output_io: nil, filename: nil)
+            with_row_writeable(filename: filename, output_io: output_io) do |row_writeable|
+              write_model_rows(models, model_keys, row_writeable)
             end
+          end
 
-            if (identifier = self.get_col_or_nil(row, :identifier))
-              if template.nil?
-                yield "Found identifier #{identifier} but no model_name had been given!"
-                next
-              end
-              template.identifier = identifier
-            end
+          def write_model_rows(models, model_keys, row_writeable)
+            ensure_parents(models, model_keys, row_writeable)
 
-            if (parent_model_name = self.get_col_or_nil(row, :parent_model_name))
-              self.process_parent_model_name(template: template, parent_model_name: parent_model_name, models: models, &err_block)
-            end
+            matrix_constants = {}
+            model_keys.each { |model_name| each_model_row(models, model_name, matrix_constants, row_writeable) }
 
-            if (parent_link_type = self.get_col_or_nil(row, :parent_link_type))
-              self.process_parent_link_type(template: template, parent_link_type: parent_link_type, models: models, &err_block)
+            matrix_constants.each do |digest, options|
+              row_writeable.write
+              row_writeable.write(matrix_constant: COPY_OPTIONS_SENTINEL + digest)
+              options.each do |option|
+                row_writeable.write(matrix_constant: option)
+              end
             end
+          end
 
-            if (attribute_name = self.get_col_or_nil(row, :attribute_name))
-              self.process_attribute(template: template, attribute_name: attribute_name, models: models, row: row, &err_block)
-              if (new_attribute_name = self.get_col_or_nil(row, :new_attribute_name))
-                self.process_new_attribute_name(template: template, changeset: changeset, new_attribute_name: new_attribute_name, attribute_name: attribute_name, &err_block)
+          def ensure_parents(models, model_keys, row_writeable)
+            q = model_keys.dup
+            seen = Set.new
+
+            until q.empty?
+              model_key = q.shift
+              next if model_key.nil?
+              next if seen.include?(model_key)
+              seen.add(model_key)
+
+              # For models that are only part of the trunk, but not of the tree of model_keys,
+              # we still need their basic information (identifier / parent) for validation and for
+              # potentially creating the required tree dependencies to connect it to a remote tree.
+              unless model_keys.include?(model_key)
+                each_model_trunk_row(models, model_key, row_writeable)
               end
+
+              q.push(*models.model(model_key).template.all_linked_model_names)
             end
           end
-        end
 
-        def self.process_new_attribute_name(template:, changeset:, new_attribute_name:, attribute_name:, &err_block)
-          renames = changeset.build_renames(template.name)
-          if renames.include?(attribute_name) && renames[attribute_name] != new_attribute_name
-            if block_given?
-              yield "Found multiple new_attribute_name values for #{template.name}'s #{attribute_name}': #{new_attribute_name} or #{renames[attribute_name]}?"
+          def each_model_trunk_row(models, model_name, row_writeable)
+            return unless (model = models.model(model_name))
+
+            # Empty link for better visual separation
+            row_writeable.write
+            row_writeable.write(model_name: model_name)
+
+            if !model.template.parent.nil?
+              parent_model = models.model(model.template.parent)
+              reciprocal = models.find_reciprocal(model: model, link_model: parent_model)
+
+              row_writeable.write(
+                  identifier: model.template.identifier,
+                  parent_model_name: model.template.parent,
+                  parent_link_type: reciprocal.attribute_type.to_s
+              )
+            else
+              row_writeable.write(
+                  identifier: model.template.identifier,
+              )
             end
           end
 
-          renames[attribute_name] = new_attribute_name
-        end
+          def each_model_row(models, model_name, matrix_constants, row_writeable)
+            return unless (model = models.model(model_name))
 
-        def self.process_parent_model_name(template:, parent_model_name:, models:, &err_block)
-          if template.nil?
-            yield "Found parent_model_name #{parent_model_name} but no model_name had been given!" if block_given?
-            return
+            each_model_trunk_row(models, model_name, row_writeable)
+            model.template.attributes.all.each do |attribute|
+              each_attribute_row(models, model, attribute, matrix_constants, row_writeable)
+            end
           end
 
-          if template.parent && !template.parent.empty? && template.parent != parent_model_name
-            yield "Model #{template.name} was provided multiple parent_model_names: #{template.parent} and #{parent_model_name}" if block_given?
-          end
+          def each_attribute_row(models, model, attribute, matrix_constants, row_writeable)
+            if attribute.attribute_type == Etna::Clients::Magma::AttributeType::IDENTIFIER
+              # Identifiers for models whose parent link type ends up being a table are non configurable, so we don't
+              # want to include them in the CSV.
+              if models.find_reciprocal(model: model, link_attribute_name: model.template.parent)&.attribute_type == Etna::Clients::Magma::AttributeType::TABLE
+                return
+              end
+            else
+              return unless Etna::Clients::Magma::AttributeValidator.valid_add_row_attribute_types.include?(attribute.attribute_type)
+            end
 
-          template.parent = parent_model_name
+            options = attribute.options&.join(', ')
+            if attribute.attribute_type == Etna::Clients::Magma::AttributeType::MATRIX
+              # Matrix attribute validations are massive, and functional.  For now, we don't support showing and editing
+              # them inline with this spreadsheet.  In the future, I think we should possibly introduce the concept of
+              # CONSTANTS or Matrix Types that are managed separately.
+              options = options || ''
+              digest = Digest::MD5.hexdigest(options)
+              matrix_constants[digest] ||= options.split(',').map(&:strip)
 
-          template.build_attributes.build_attribute(template.parent).tap do |parent_att|
-            parent_att.name = parent_att.attribute_name = parent_model_name
-            parent_att.attribute_type = AttributeType::PARENT
-            parent_att.link_model_name = parent_model_name
-            parent_att.desc = self.prettify(parent_model_name)
-            parent_att.display_name = self.prettify(parent_model_name)
-          end
-        end
+              options = COPY_OPTIONS_SENTINEL + digest
+            end
 
-        def self.process_parent_link_type(template:, parent_link_type:, models:, &err_block)
-          if template.nil?
-            yield "Found parent_link_type #{parent_link_type} but no model_name had been given!" if block_given?
-            return
+            row_writeable.write(
+                attribute_name: attribute.name,
+                attribute_type: attribute.attribute_type,
+                link_model_name: attribute.link_model_name,
+                reciprocal_link_type: models.find_reciprocal(model: model, attribute: attribute)&.attribute_type,
+                description: attribute.desc,
+                display_name: attribute.display_name,
+                match: attribute.match,
+                format_hint: attribute.format_hint,
+                restricted: attribute.restricted,
+                read_only: attribute.read_only,
+                options: options,
+                attribute_group: attribute.attribute_group,
+                hidden: attribute.hidden,
+                unique: attribute.unique,
+            )
           end
+        end
 
-          if template.parent.nil?
-            yield "Found parent_link_type #{parent_link_type} but no parent_model_name had been given!" if block_given?
-            return
-          end
+        class ModelsChangeset < Struct.new(:models, :renames, :matrix_constants, keyword_init: true)
+          def initialize(*args)
+            super
 
-          reciprocal = models.find_reciprocal(model_name: template.name, link_attribute_name: template.parent)
-          if reciprocal && reciprocal.attribute_type.to_s != parent_link_type
-            yield "Model #{template.name} was provided multiple parent_link_types: #{reciprocal.attribute_type} and #{parent_link_type}" if block_given?
+            self.models ||= Models.new
+            self.renames ||= {}
+            self.matrix_constants ||= {}
           end
 
-          if reciprocal && reciprocal.attribute_name != template.name
-            yield "Model #{template.name} is linked to #{template.parent}, but the reciprocal link is misnamed as '#{reciprocal.attribute_name}'." if block_given?
+          def build_renames(model_name)
+            renames[model_name] ||= {}
           end
+        end
 
-          models.build_model(template.parent).tap do |parent_model|
-            parent_model_attribute_by_model_name = parent_model.build_template.build_attributes.attribute(template.name)
-            if parent_model_attribute_by_model_name && !reciprocal
-              yield "Model #{template.parent} is linked as a parent to #{template.name}, but it already has an attribute named #{template.name} #{parent_model_attribute_by_model_name.raw}." if block_given?
-            end
+        COPY_OPTIONS_SENTINEL = '$copy_from_original$'
 
-            parent_model.build_template.build_attributes.build_attribute(template.name).tap do |attr|
-              attr.attribute_name = attr.name = template.name
-              attr.attribute_type = parent_link_type
-              attr.link_model_name = template.name
-              attr.desc = self.prettify(template.name)
-              attr.display_name = self.prettify(template.name)
-            end
+        class Importer < Etna::CsvImporter
+          include Prettify
+
+          # Columns of the row, _after format_row is called_, that should be applied to an attribute.
+          # This should line up with the attribute names _on the model itself_.
+          ATTRIBUTE_ROW_ENTRIES = [
+              :attribute_type,
+              :link_model_name, :desc,
+              :display_name, :format_hint,
+              :restricted, :read_only,
+              :validation, :attribute_group,
+              :hidden, :unique,
+          ]
+
+          def initialize
+            super(&method(:format_row))
           end
-        end
 
-        def self.process_attribute(template:, attribute_name:, models:, row:, &err_block)
-          if template.nil?
-            yield "Found attribute #{attribute_name} but no model_name had been given!" if block_given?
-            return
+          def format_row(row)
+            replace_row_column(row, :attribute_type) { |s| AttributeType.new(s) }
+            replace_row_column(row, :desc) { row.delete(:description) }
+            replace_row_column(row, :restricted, &COLUMN_AS_BOOLEAN)
+            replace_row_column(row, :read_only, &COLUMN_AS_BOOLEAN)
+            replace_row_column(row, :options) { |s| {"type" => "Array", "value" => s.split(',').map(&:strip)} }
+            replace_row_column(row, :match) { |s| {"type" => "Regexp", "value" => Regexp.new(s).source} }
+            replace_row_column(row, :validation) { row[:options] || row[:match] }
+            replace_row_column(row, :hidden, &COLUMN_AS_BOOLEAN)
+            replace_row_column(row, :unique, &COLUMN_AS_BOOLEAN)
           end
 
-          attributes = template.build_attributes
-          existing_attribute = attributes.attribute(attribute_name)
-          if existing_attribute
-            if existing_attribute.attribute_type != AttributeType::COLLECTION
-              yield "Attribute #{attribute_name} of model #{template.name} has duplicate definitions!" if block_given?
-            end
+          def process_row(row_processor, changeset)
+            models = changeset.models
 
-            attributes.raw.delete(attribute_name)
+            process_matrix_constants(changeset, row_processor)
+            process_model_config(models, row_processor)
+            process_parent_config(models, row_processor)
+            process_attribute_properties(changeset, row_processor)
           end
 
-          attributes.build_attribute(attribute_name).tap do |att|
-            COLUMNS_TO_ATTRIBUTES.each do |column, processor|
-              next unless (value = self.get_col_or_nil(row, column))
-              if processor.is_a?(Array)
-                processor, f = processor
-                value = f.call(value)
+          def prepare_changeset(filename: nil, input_io: nil, &validation_err_block)
+            ModelsChangeset.new.tap do |changeset|
+              context = {}
+              each_csv_row(filename: filename, input_io: input_io) do |row, lineno|
+                p = NestedRowProcessor.new(row, lineno, context)
+                process_row(p, changeset)
               end
 
-              if !att.send(processor).nil? && !att.send(processor).empty?
-                yield "Value for #{processor} on attribute #{attribute_name} has duplicate definitions!" if block_given?
-              end
-
-              att.send(:"#{processor}=", value)
-            end
-
-            if att.attribute_type == AttributeType::LINK && models.find_reciprocal(model_name: template.name, attribute: att).nil?
-              models.build_model(att.link_model_name).build_template.build_attributes.build_attribute(template.name).tap do |rec_att|
-                rec_att.attribute_name = template.name
-                rec_att.display_name = self.prettify(template.name)
-                rec_att.desc = self.prettify(template.name)
-                rec_att.attribute_type = AttributeType::COLLECTION
-                rec_att.link_model_name = template.name
+              # After configuring attributes, set defaults for certain attribute properties.
+              changeset.models.all.each do |model|
+                model.template.attributes.all.each do |attribute|
+                  attribute.set_field_defaults!
+                end
               end
             end
-
-            att.set_field_defaults!
+          rescue ImportError => e
+            validation_err_block.call(e.message)
           end
-        end
 
-        def self.prettify(name)
-          name.split('_').map(&:capitalize).join(' ')
-        end
 
-        def self.get_col_or_nil(row, col)
-          c = row[col]&.chomp
-          return nil if c&.empty?
-          c
-        end
-
-        def self.each_csv_row(models = Models.new, model_keys = models.model_keys.sort, &block)
-          yield COLUMNS.map(&:to_s)
-
-          self.ensure_parents(models, model_keys, &block)
-          matrix_constants = {}
-          model_keys.each { |model_name| self.each_model_row(models, model_name, matrix_constants, &block) }
+          private
 
-          matrix_constants.each do |digest, options|
-            yield row_from_columns
-            yield row_from_columns(matrix_constant: COPY_OPTIONS_SENTINEL + digest)
-            options.each do |option|
-              yield row_from_columns(matrix_constant: option)
+          def process_model_config(models, row_processor)
+            row_processor.process(:model_name) do |model_name|
+              models.build_model(model_name).build_template.tap { |t| t.name = model_name }
             end
-          end
-        end
-
-        def self.ensure_parents(models, model_keys, &block)
-          q = model_keys.dup
-          seen = Set.new
-
-          until q.empty?
-            model_key = q.shift
-            next if model_key.nil?
-            next if seen.include?(model_key)
-            seen.add(model_key)
-
-            # For models that are only part of the trunk, but not of the tree of model_keys,
-            # we still need their basic information (identifier / parent) for validation and for
-            # potentially creating the required tree dependencies to connect it to a remote tree.
-            unless model_keys.include?(model_key)
-              self.each_model_trunk_row(models, model_key, &block)
+            row_processor.process(:identifier, :model_name) do |identifier, template|
+              template.identifier = identifier
             end
-
-            q.push(*models.model(model_key).template.all_linked_model_names)
           end
-        end
 
-        def self.each_model_trunk_row(models, model_name, &block)
-          return unless (model = models.model(model_name))
+          def process_parent_config(models, row_processor)
+            row_processor.process(:parent_model_name, :model_name) do |parent_model_name, template|
+              if template.parent && !template.parent.empty? && template.parent != parent_model_name
+                raise ImportError.new("Model #{template.name} was provided multiple parent_model_names: #{template.parent} and #{parent_model_name}")
+              end
 
-          # Empty link for better visual separation
-          yield row_from_columns
-          yield row_from_columns(model_name: model_name)
+              template.parent = parent_model_name
 
-          unless model.template.parent.nil?
-            parent_model = models.model(model.template.parent)
-            reciprocal = models.find_reciprocal(model: model, link_model: parent_model)
+              template.build_attributes.build_attribute(template.parent).tap do |parent_att|
+                parent_att.name = parent_att.attribute_name = parent_model_name
+                parent_att.attribute_type = Etna::Clients::Magma::AttributeType::PARENT
+                parent_att.link_model_name = parent_model_name
+                parent_att.desc = prettify(parent_model_name)
+                parent_att.display_name = prettify(parent_model_name)
+              end
+            end
 
-            yield row_from_columns(
-                identifier: model.template.identifier,
-                parent_model_name: model.template.parent,
-                parent_link_type: reciprocal.attribute_type.to_s
-            )
-          else
-            yield row_from_columns(
-                identifier: model.template.identifier,
-            )
-          end
-        end
+            row_processor.process(:parent_link_type, :model_name, :parent_model_name) do |parent_link_type, template|
+              reciprocal = models.find_reciprocal(model_name: template.name, link_attribute_name: template.parent)
+              if reciprocal && reciprocal.attribute_type.to_s != parent_link_type
+                raise ImportError.new("Model #{template.name} was provided multiple parent_link_types: #{reciprocal.attribute_type} and #{parent_link_type}")
+              end
 
-        def self.each_model_row(models, model_name, matrix_constants, &block)
-          return unless (model = models.model(model_name))
+              if reciprocal && reciprocal.attribute_name != template.name
+                raise ImportError.new("Model #{template.name} is linked to #{template.parent}, but the reciprocal link is misnamed as '#{reciprocal.attribute_name}'.")
+              end
 
-          self.each_model_trunk_row(models, model_name, &block)
-          model.template.attributes.all.each do |attribute|
-            self.each_attribute_row(models, model, attribute, matrix_constants, &block)
+              models.build_model(template.parent).tap do |parent_model|
+                parent_model_attribute_by_model_name = parent_model.build_template.build_attributes.attribute(template.name)
+                if parent_model_attribute_by_model_name && !reciprocal
+                  raise ImportError.new("Model #{template.parent} is linked as a parent to #{template.name}, but it already has an attribute named #{template.name} #{parent_model_attribute_by_model_name.raw}.")
+                end
+
+                parent_model.build_template.build_attributes.build_attribute(template.name).tap do |attr|
+                  attr.attribute_name = attr.name = template.name
+                  attr.attribute_type = parent_link_type
+                  attr.link_model_name = template.name
+                  attr.desc = prettify(template.name)
+                  attr.display_name = prettify(template.name)
+                end
+              end
+            end
           end
-        end
 
-        def self.each_attribute_row(models = Models.new, model = Model.new, attribute = Attribute.new, matrix_constants = {}, &block)
-          if attribute.attribute_type == AttributeType::IDENTIFIER
-            # Identifiers for models whose parent link type ends up being a table are non configurable, so we don't
-            # want to include them in the CSV.
-            if models.find_reciprocal(model: model, link_attribute_name: model.template.parent)&.attribute_type == AttributeType::TABLE
-              return
+          def process_matrix_constants(changeset, row_processor)
+            row_processor.process(:matrix_constant) do |matrix_constant|
+              # Lines that start with the sentinel are a distinct 'group'
+              if matrix_constant.start_with?(COPY_OPTIONS_SENTINEL)
+                matrix_constant = matrix_constant.slice((COPY_OPTIONS_SENTINEL.length)..-1)
+                changeset.matrix_constants[matrix_constant] = []
+              else
+                # Until the sentinel is seen again, all further entries belong to the set.
+                changeset.matrix_constants[changeset.matrix_constants.keys.last] << matrix_constant
+              end
+
+              matrix_constant
             end
-          else
-            return unless AttributeValidator.valid_add_row_attribute_types.include?(attribute.attribute_type)
           end
 
-          options = attribute.options&.join(', ')
-          if attribute.attribute_type == AttributeType::MATRIX
-            # Matrix attribute validations are massive, and functional.  For now, we don't support showing and editing
-            # them inline with this spreadsheet.  In the future, I think we should possibly introduce the concept of
-            # CONSTANTS or Matrix Types that are managed separately.
-            options = options || ''
-            digest = Digest::MD5.hexdigest(options)
-            matrix_constants[digest] ||= COLUMNS_TO_ATTRIBUTES[:options][1].call(options)["value"]
+          def process_attribute_properties(changeset, row_processor)
+            models = changeset.models
 
-            options = COPY_OPTIONS_SENTINEL + digest
-          end
+            row_processor.process(:attribute_name, :model_name) do |attribute_name, template|
+              attributes = template.build_attributes
 
-          yield row_from_columns(
-              attribute_name: attribute.name,
-              attribute_type: attribute.attribute_type,
-              link_model_name: attribute.link_model_name,
-              reciprocal_link_type: models.find_reciprocal(model: model, attribute: attribute)&.attribute_type,
-              description: attribute.desc,
-              display_name: attribute.display_name,
-              match: attribute.match,
-              format_hint: attribute.format_hint,
-              restricted: attribute.restricted,
-              read_only: attribute.read_only,
-              options: options,
-              attribute_group: attribute.attribute_group,
-              hidden: attribute.hidden,
-              unique: attribute.unique,
-          )
-        end
+              existing_attribute = attributes.attribute(attribute_name)
+              if existing_attribute
+                if existing_attribute.attribute_type != Etna::Clients::Magma::AttributeType::COLLECTION
+                  raise ImportError.new("Attribute #{attribute_name} of model #{template.name} has duplicate definitions!")
+                end
 
-        def self.row_from_columns(**columns)
-          COLUMNS.map { |c| (columns[c] || '').to_s }
-        end
+                # Clear its definition; implicit attributes are overwritten by explicit definitions if they exist.
+                attributes.raw.delete(attribute_name)
+              end
 
-        class ModelsChangeset < Struct.new(:models, :renames, :matrix_constants, :last_matrix_constant, :last_model_key, keyword_init: true)
-          def initialize(*args)
-            super
+              attributes.build_attribute(attribute_name).tap { |att| att.name = att.attribute_name = attribute_name }
+            end
 
-            self.models ||= Models.new
-            self.renames ||= {}
-            self.matrix_constants ||= {}
+            row_processor.process(:new_attribute_name, :attribute_name, :model_name) do |new_attribute_name, att, template|
+              renames = changeset.build_renames(template.name)
+              if renames.include?(att.name) && renames[att.name] != new_attribute_name
+                raise ImportError.new("Found multiple new_attribute_name values for #{template.name}'s #{att.name}': #{new_attribute_name} or #{renames[att.name]}?")
+              end
+
+              renames[att.attribute_name] = new_attribute_name
+            end
+
+            ATTRIBUTE_ROW_ENTRIES.each do |prop_name|
+              row_processor.process(prop_name, :model_name, :attribute_name) do |value, template, att|
+                if att.raw.include?(prop_name.to_s)
+                  raise ImportError.new("Value for #{prop_name} on attribute #{att.attribute_name} has duplicate definitions!")
+                end
+
+                att.send(:"#{prop_name}=", value)
+
+                if att.attribute_type && att.link_model_name
+                  if att.attribute_type == Etna::Clients::Magma::AttributeType::LINK && models.find_reciprocal(model_name: template.name, attribute: att).nil?
+                    models.build_model(att.link_model_name).build_template.build_attributes.build_attribute(template.name).tap do |rec_att|
+                      rec_att.attribute_name = rec_att.name = template.name
+                      rec_att.display_name = prettify(template.name)
+                      rec_att.desc = prettify(template.name)
+                      rec_att.attribute_type = Etna::Clients::Magma::AttributeType::COLLECTION
+                      rec_att.link_model_name = template.name
+                    end
+                  end
+                end
+              end
+            end
           end
         end
       end