etna 0.1.19 → 0.1.20

data/lib/etna/clients/magma/models.rb

@@ -368,12 +368,56 @@ module Etna
           Attributes.new(raw['attributes'] ||= {})
         end
 
+        def dictionary
+          Dictionary.new(raw['dictionary'] ||= {})
+        end
+
+        def build_dictionary
+          Dictionary.new(raw['dictionary'] ||= {})
+        end
+
         def all_linked_model_names
           models = [ self.parent, ] + build_attributes.all.map { |v| v.link_model_name }
           models.select { |m| !m.nil? }.uniq
         end
       end
 
+      class Dictionary
+        attr_reader :raw
+
+        def initialize(raw = {})
+          @raw = raw
+        end
+
+        def dictionary_keys
+          raw.keys
+        end
+
+        def dictionary_model
+          raw['dictionary_model']
+        end
+
+        def dictionary_model=(val)
+          @raw['dictionary_model'] = val
+        end
+
+        def model_name
+          raw['model_name']
+        end
+
+        def model_name=(val)
+          @raw['model_name'] = val
+        end
+
+        def attributes
+          raw['attributes']
+        end
+
+        def attributes=(val)
+          @raw['attributes'] = val
+        end
+      end
+
       class Attributes
         attr_reader :raw
 
@@ -436,6 +480,12 @@ module Etna
           @raw['attribute_type'] && AttributeType.new(@raw['attribute_type'])
         end
 
+        def is_project_name_reference?(model_name)
+          return true if model_name == 'project' && attribute_type == AttributeType::IDENTIFIER
+          return true if link_model_name == 'project'
+          false
+        end
+
         def attribute_type=(val)
           val = val.to_s if val
           @raw['attribute_type'] = val
@@ -557,6 +607,7 @@ module Etna
         CHILD = AttributeType.new("child")
         COLLECTION = AttributeType.new("collection")
         FILE = AttributeType.new("file")
+        FILE_COLLECTION = AttributeType.new("file_collection")
         FLOAT = AttributeType.new("float")
         IDENTIFIER = AttributeType.new("identifier")
         IMAGE = AttributeType.new("image")

data/lib/etna/clients/magma/workflows/add_project_models_workflow.rb

@@ -28,27 +28,18 @@ module Etna
               next unless (validation = attribute&.validation)
               next unless (value = validation['value'])
 
-              if validation['type'] == 'Array' && value&.first.start_with?(ModelsCsv::COPY_OPTIONS_SENTINEL)
-                digest = value&.first.slice((ModelsCsv::COPY_OPTIONS_SENTINEL.length)..-1)
+              if validation['type'] == 'Array' && value&.first.start_with?(Etna::Clients::Magma::ModelsCsv::COPY_OPTIONS_SENTINEL)
+                digest = value&.first.slice((Etna::Clients::Magma::ModelsCsv::COPY_OPTIONS_SENTINEL.length)..-1)
                 attribute.validation = { 'type' => 'Array', 'value' => changeset.matrix_constants[digest] }
               end
             end
           end
         end
 
-        def prepare_changeset_from_csv(csv_io, &err_block)
-          line_no = 0
-          csv_lines = CSV.parse(csv_io, headers: true, header_converters: :symbol)
-          changeset = csv_lines.inject(ModelsCsv::ModelsChangeset.new) do |acc, n|
-            line_no += 1
-            ModelsCsv.apply_csv_row(acc, n) do |err|
-              err_block.call("Error detected on line #{line_no + 1}: #{err}")
-              return
-            end
-          end
-
+        def prepare_changeset_from_csv(filename: nil, io: nil, &err_block)
+          importer = ModelsCsv::Importer.new
+          changeset = importer.prepare_changeset(filename: filename, input_io: io, &err_block)
           self.class.validate_changeset(changeset, &err_block)
-
           changeset
         end
 
@@ -64,13 +55,11 @@ module Etna
           validator.errors.each(&err_block)
         end
 
-        def write_models_template_csv(io, project_name, target_model = 'project')
+        def write_models_template_csv(project_name, target_model = 'project', filename: nil, io: nil)
           models = magma_client.retrieve(RetrievalRequest.new(project_name: project_name, model_name: 'all')).models
           descendants = models.to_directed_graph.descendants(target_model)
-          csv = CSV.new(io)
-          ModelsCsv.each_csv_row(models, [target_model] + descendants.keys) do |row|
-            csv << row
-          end
+          exporter = ModelsCsv::Exporter.new
+          exporter.write_models(models, [target_model] + descendants.keys, filename: filename, output_io: io)
         end
       end
     end

data/lib/etna/clients/magma/workflows/create_project_workflow.rb

@@ -74,6 +74,20 @@ module Etna
           promote_to_administrator(user['email'])
           update_magma_client_token!
 
+          puts "Done with setting up the project in Janus!"
+        end
+
+        def setup_magma_project!
+          puts "Creating the project in Magma."
+          create_magma_project!
+          puts "Done! Adding your new project record."
+          create_magma_project_record!
+        end
+
+        def create!
+          setup_janus_project!
+          setup_magma_project!
+
           while true
             puts "Add more users? Y/n"
             break unless STDIN.gets.chomp == 'Y'
@@ -87,22 +101,14 @@ module Etna
             puts "Confirm? Y/n"
             break unless STDIN.gets.chomp == 'Y'
 
-            add_janus_user(email, name, role)
+            if role == 'administrator'
+              add_janus_user(email, name, 'editor')
+              promote_to_administrator(email)
+            else
+              add_janus_user(email, name, role)
+            end
           end
 
-          puts "Done with setting up the project in Janus!"
-        end
-
-        def setup_magma_project!
-          puts "Creating the project in Magma."
-          create_magma_project!
-          puts "Done! Adding your new project record."
-          create_magma_project_record!
-        end
-
-        def create!
-          setup_janus_project!
-          setup_magma_project!
           puts "All complete!"
           puts "You need to visit Janus to refresh your token."
           puts "You can now log into any app to manage your data."

data/lib/etna/clients/magma/workflows/crud_workflow.rb

@@ -47,7 +47,7 @@ module Etna
 
         # Todo: Introduce associative concatenation operations for response objects and return
         # one response that munges the batched responses together.
-        def update_records
+        def update_records(method: :update)
           @recorded_updates ||= UpdateRequest.new(project_name: project_name)
 
           request = UpdateRequest.new(project_name: project_name)
@@ -72,7 +72,7 @@ module Etna
           responses = []
           revisions.to_a.each_slice(batch_size) do |batch|
             request.revisions = batch.to_h
-            magma_client.update(request) unless read_only
+            magma_client.send(method, request) unless read_only
             responses << @recorded_updates.revisions.update(request.revisions)
           end
 

data/lib/etna/clients/magma/workflows/json_validators.rb

@@ -24,6 +24,10 @@ module Etna
           @errors << "Invalid empty #{key} for #{label}: \"#{raw[key]}\"." if raw.dig(key) && nil_or_empty?(raw[key])
         end
 
+        def check_key_empty(label, raw, key)
+          @errors << "Invalid key for #{label}: \"#{key}\"." if raw.dig(key)
+        end
+
         def check_in_set(label, raw, key, valid_values)
           @errors << "Invalid #{key} for #{label}: \"#{raw[key]}\".\nShould be one of #{valid_values}." if raw.dig(key) && !valid_values.include?(raw[key])
         end
@@ -139,11 +143,12 @@ module Etna
           model.template.attributes.attribute_keys.each do |attribute_name|
             attribute = model.template.attributes.attribute(attribute_name)
 
-            if attribute_name == model.template.identifier
+            reciprocal = @models.find_reciprocal(model: model, attribute: attribute)
+            if attribute_name == model.template.identifier && reciprocal&.attribute_type != AttributeType::TABLE
               attribute_types = [AttributeType::IDENTIFIER]
             elsif attribute_name == model.template.parent
               attribute_types = [AttributeType::PARENT]
-            elsif @models.find_reciprocal(model: model, attribute: attribute)&.attribute_type == AttributeType::PARENT
+            elsif reciprocal&.attribute_type == AttributeType::PARENT
               attribute_types = AttributeValidator.valid_parent_link_attribute_types
             else
               attribute_types = AttributeValidator.valid_add_row_attribute_types
@@ -184,7 +189,7 @@ module Etna
 
         def link_attributes
           @model.template.attributes.all.select do |attribute|
-            attribute.attribute_type == Etna::Clients::Magma::AttributeType::LINK
+            attribute.link_model_name
           end
         end
       end

data/lib/etna/clients/magma/workflows/model_synchronization_workflow.rb

@@ -85,7 +85,7 @@ module Etna
             if action.parent_link_type != Etna::Clients::Magma::AttributeType::TABLE
               template.build_attributes.build_attribute(template.identifier).tap do |attr|
                 attr.attribute_name = template.identifier
-                attr.attribute_type = Etna::Clients::Magma::AttributeType::STRING
+                attr.attribute_type = Etna::Clients::Magma::AttributeType::IDENTIFIER
               end
             end
           when RenameAttributeAction

data/lib/etna/clients/magma/workflows/update_attributes_from_csv_workflow.rb

@@ -5,7 +5,7 @@ require_relative './crud_workflow'
 module Etna
   module Clients
     class Magma
-      class UpdateAttributesFromCsvWorkflowBase < Struct.new(:magma_crud, :project_name, :filepath, :model_name, keyword_init: true)
+      class UpdateAttributesFromCsvWorkflowBase < Struct.new(:magma_crud, :project_name, :filepath, :model_name, :json_values, :hole_value, keyword_init: true)
         def initialize(opts)
           super(**{}.update(opts))
         end
@@ -37,7 +37,8 @@ module Etna
         end
 
         def update_attributes
-          magma_crud.update_records do |update_request|
+          method = json_values ? :update_json : :update
+          magma_crud.update_records(method: method) do |update_request|
             each_revision do |model_name, record_name, revision|
               update_request.update_revision(model_name, record_name, revision)
             end
@@ -53,7 +54,12 @@ module Etna
 
       class RowBase
         def stripped_value(attribute_value)
-          attribute_value ? attribute_value.strip : attribute_value
+          attribute_value = attribute_value&.strip
+
+          if attribute_value && @workflow.json_values && attribute_value != @workflow.hole_value
+            attribute_value = JSON.parse(attribute_value)
+          end
+          attribute_value
         end
 
         def nil_or_empty?(value)
@@ -74,6 +80,7 @@ module Etna
 
         class Row < RowBase
           attr_reader :model_name, :record_name
+
           def initialize(raw, workflow)
             # Assumes rows are in pairs, where
             #   [0] = model_name
@@ -112,9 +119,70 @@ module Etna
                   raise "Invalid attribute name: \"#{attribute_name}\"." if nil_or_empty?(attribute_name)
                   attribute_name.strip!
 
-                  raise "Invalid attribute #{attribute_name} for model #{model_name}." unless attribute = @workflow.find_attribute(model_name, attribute_name)
+                  unless (attribute = @workflow.find_attribute(model_name, attribute_name))
+                    raise "Invalid attribute #{attribute_name} for model #{model_name}."
+                  end
+
+                  stripped = stripped_value(@raw[index + 1])
+                  unless @workflow.hole_value.nil?
+                    next if stripped == @workflow.hole_value
+                  end
+
+                  if attribute.is_project_name_reference?(model_name)
+                    stripped&.downcase!
+                  end
+
+                  attributes[attribute_name] = stripped
+                end
+              end
+            end
+          end
+        end
+      end
+
+      class SimpleFileLinkingWorkflow < Struct.new(:metis_client, :project_name, :bucket_name, :folder, :extension, :attribute_name, :regex, :file_collection, keyword_init: true)
+        def write_csv_io(filename: nil, output_io: nil)
+          exporter = Etna::CsvExporter.new([:identifier, attribute_name.to_sym])
+          exporter.with_row_writeable(filename: filename, output_io: output_io) do |row_writeable|
+            find_matching_revisions.each do |identifier, value|
+              row_writeable << { identifier: identifier, attribute_name.to_sym => value.to_json }
+            end
+          end
+        end
+
+        def find_matching_revisions
+          {}.tap do |revisions|
+            metis_client.find(
+                Etna::Clients::Metis::FindRequest.new(
+                    project_name: project_name,
+                    bucket_name: bucket_name,
+                    params: [Etna::Clients::Metis::FindParam.new(
+                        attribute: 'name',
+                        predicate: 'glob',
+                        value: "#{folder}/**/*.#{extension}",
+                        type: 'file'
+                    )]
+                )).files.all.each do |file|
+              puts "Checking #{file.file_path}"
+              match = regex.match(file.file_path)
+              if match
+                match_map = match.names.zip(match.captures).to_h
+                if !match_map.include?('identifier')
+                  raise "Regex #{regex.source} does not include a ?<identifier> named matcher, please add one to regulate how identifiers are created."
+                end
 
-                  attributes[attribute_name] = stripped_value(@raw[index + 1])
+                puts "Found match"
+
+                revision = { 'path' => "metis://#{project_name}/#{bucket_name}/#{file.file_path}", 'original_filename' => "#{File.basename(file.file_path)}" }
+                if file_collection
+                  collection = revisions[match_map['identifier']] ||= []
+                  collection << revision
+                else
+                  record = revisions[match_map['identifier']] ||= {}
+                  unless record.empty?
+                    raise "Multiple files match #{match_map['identifier']}, found #{record['path']} and #{revision['path']}"
+                  end
+                  record.update(revision)
                 end
               end
             end
@@ -125,7 +193,7 @@ module Etna
       class UpdateAttributesFromCsvWorkflowSingleModel < UpdateAttributesFromCsvWorkflowBase
         def initialize(opts)
           super(**{}.update(opts))
-          raise "Single Model invokation must include keyword :model_name." if !opts[:model_name]
+          raise "Single Model invocation must include keyword :model_name." if !opts[:model_name]
           raise "Invalid model #{model_name} for project #{project_name}." unless model_exists?(model_name)
         end
 
@@ -139,6 +207,7 @@ module Etna
 
         class Row < RowBase
           attr_reader :record_name
+
           def initialize(raw, model_name, workflow)
             # Assumes CSV includes a column header to identify the attribute_name
             # Assumes index 0 is the record_name
@@ -160,7 +229,6 @@ module Etna
               row_hash = @raw.to_h
               row_keys = row_hash.keys
               row_keys[1..row_keys.length - 1].each do |attribute_name|
-
                 raise "Invalid attribute name: \"#{attribute_name}\"." if nil_or_empty?(attribute_name)
 
                 attribute_name_clean = attribute_name.strip

data/lib/etna/csvs.rb

@@ -0,0 +1,159 @@
+module Etna
+  class CsvImporter
+    def initialize(
+        strip: true,
+        filter_empties: true,
+        &row_formatter
+    )
+      # Removes any columns from a row that are empty strings.  Allows for some simpler 'empty' processing.  This occurs
+      # post stripping as well.
+      @filter_empties = filter_empties
+      @row_formatter = row_formatter
+    end
+
+    COLUMN_AS_BOOLEAN = -> (s) { ['true', 't', 'y', 'yes'].include?(s&.downcase) }
+
+    def each_csv_row(filename: nil, input_io: nil, &block)
+      if input_io.nil?
+        unless filename.nil?
+          File.open(filename, 'r') do |io|
+            return each_csv_row(input_io: io, &block)
+          end
+        end
+      end
+
+      lineno = 1
+      CSV.parse(input_io, headers: true, header_converters: :symbol) do |row|
+        lineno += 1
+        row = row.to_hash
+        row.keys.each { |k| row[k].strip! if row[k] =~ /^\s+$/ } if @strip
+        row.select! { |k, v| !v.empty? } if @filter_empties
+        @row_formatter.call(row) unless @row_formatter.nil?
+        yield row, lineno if block_given?
+      end
+    end
+
+    def replace_row_column(row, column, &block)
+      if !row[column].nil? || block.arity == 0
+        row[column] = yield row[column]
+      end
+    end
+
+    class ImportError < StandardError
+      attr_accessor :lineno
+
+      def initialize(msg, lineno = nil)
+        @lineno = lineno
+        super(msg)
+      end
+
+      def message
+        "line #{lineno}: #{super}"
+      end
+    end
+
+    class NestedRowProcessor
+      attr_reader :row, :lineno, :context, :errors
+
+      def initialize(row, lineno, context)
+        @row = row
+        @lineno = lineno
+        @context = context
+        @errors = []
+
+        # If a parent context changes, all child contexts are invalidated.  But since parent contexts are changed
+        # before the relationship of child contexts are declared, we have to track that so that when a child context
+        # dependency is declared we can clear it based on wether parents have changed.
+        @changed = {}
+      end
+
+      def process(column, *parents, &block)
+        if parents.any? { |p| @changed.include?(p) }
+          @changed[column] = true
+          @context[column] = nil
+        end
+
+        return self if (next_val = row[column]).nil?
+        @changed[column] = true
+
+        parent_values = parents.map do |p|
+          if @context[p].nil?
+            raise ImportError.new("Found a #{column} value, but no previous #{p} had been given!", @lineno)
+          end
+
+          @context[p]
+        end
+
+        begin
+          next_val = yield next_val, *parent_values, self if block_given?
+        rescue ImportError => e
+          e.lineno = @lineno
+          raise e
+        end
+
+        @context[column] = next_val
+        self
+      end
+    end
+  end
+
+  class CsvExporter
+    # column_headers should be an array of symbols, mapping the column heading names and ordering to export
+    # column_serializer is an optional block that takes column (string), column_value (string) and should
+    #   return a string representation of column_value to write to the csv.  By default, when nil, the exporter
+    #   will attempt to convert the value to a string via to_s or simply write an empty string for nil.
+
+    attr_reader :column_headers
+    def initialize(column_headers, &column_serializer)
+      @column_headers = column_headers
+      @column_serializer = column_serializer
+    end
+
+    def header_row
+      @column_headers.map(&:to_s)
+    end
+
+    def map_column_value(column, column_value)
+      @column_serializer&.call(column, column_value) || column_value&.to_s || ''
+    end
+
+    def row_from_columns(columns)
+      @column_headers.map { |c| self.map_column_value(c, columns[c] || '') }
+    end
+
+    def with_row_writeable(filename: nil, output_io: nil, &block)
+      if output_io.nil? && !filename.nil?
+        File.open(filename, 'w') do |io|
+          return with_row_writeable(output_io: io, &block)
+        end
+      end
+
+      writeable = self.class::RowWriteable.new(self, CSV.new(output_io))
+      yield writeable
+      writeable.ensure_headers
+      nil
+    end
+
+    class RowWriteable
+      def initialize(exporter, csv)
+        @exporter = exporter
+        @csv = csv
+        @written_headers = false
+      end
+
+      def <<(columns)
+        self.ensure_headers
+        @csv << @exporter.row_from_columns(columns)
+      end
+
+      def write(**columns)
+        self.<<(columns)
+      end
+
+      def ensure_headers
+        @csv << @exporter.header_row unless @written_headers
+        @written_headers = true
+      end
+    end
+  end
+end