etna 0.1.15 → 0.1.21

data/lib/etna/clients/magma/workflows.rb

@@ -0,0 +1,10 @@
+require_relative './workflows/crud_workflow'
+require_relative './workflows/file_linking_workflow'
+require_relative './workflows/ensure_containing_record_workflow'
+require_relative './workflows/model_synchronization_workflow'
+require_relative './workflows/file_attributes_blank_workflow'
+require_relative './workflows/record_synchronization_workflow'
+require_relative './workflows/update_attributes_from_csv_workflow'
+require_relative './workflows/create_project_workflow'
+require_relative './workflows/add_project_models_workflow'
+require_relative './workflows/attribute_actions_from_json_workflow'

data/lib/etna/clients/magma/workflows/add_project_models_workflow.rb

@@ -0,0 +1,67 @@
+require 'base64'
+require 'json'
+require 'ostruct'
+require_relative './model_synchronization_workflow'
+require_relative '../../janus/models'
+require_relative './json_validators'
+require_relative './json_converters'
+
+module Etna
+  module Clients
+    class Magma
+      class AddProjectModelsWorkflow < Struct.new(:magma_client, keyword_init: true)
+        def plan_synchronization(changeset, project, target_model = 'project')
+          apply_matrix_constants(changeset)
+          workflow = ModelSynchronizationWorkflow.new(target_client: magma_client,
+              source_models: changeset.models, renames: changeset.renames,
+              target_project: project, plan_only: true)
+          workflow.ensure_model_tree(target_model)
+          workflow
+        end
+
+        def apply_matrix_constants(changeset)
+          changeset.models.model_keys.each do |model_name|
+            model = changeset.models.model(model_name)
+            attributes = model.template.attributes
+            attributes.attribute_keys.each do |attribute_name|
+              attribute = attributes.attribute(attribute_name)
+              next unless (validation = attribute&.validation)
+              next unless (value = validation['value'])
+
+              if validation['type'] == 'Array' && value&.first.start_with?(Etna::Clients::Magma::ModelsCsv::COPY_OPTIONS_SENTINEL)
+                digest = value&.first.slice((Etna::Clients::Magma::ModelsCsv::COPY_OPTIONS_SENTINEL.length)..-1)
+                attribute.validation = { 'type' => 'Array', 'value' => changeset.matrix_constants[digest] }
+              end
+            end
+          end
+        end
+
+        def prepare_changeset_from_csv(filename: nil, io: nil, &err_block)
+          importer = ModelsCsv::Importer.new
+          changeset = importer.prepare_changeset(filename: filename, input_io: io, &err_block)
+          self.class.validate_changeset(changeset, &err_block)
+          changeset
+        end
+
+        def self.validate_changeset(changeset, &err_block)
+          changeset.models.model_keys.each do |model_name|
+            validator = AddModelValidator.new(changeset.models, model_name)
+            validator.validate
+            validator.errors.each(&err_block)
+          end
+
+          validator = RenamesValidator.new(changeset.models, changeset.renames)
+          validator.validate
+          validator.errors.each(&err_block)
+        end
+
+        def write_models_template_csv(project_name, target_model = 'project', filename: nil, io: nil)
+          models = magma_client.retrieve(RetrievalRequest.new(project_name: project_name, model_name: 'all')).models
+          descendants = models.to_directed_graph.descendants(target_model)
+          exporter = ModelsCsv::Exporter.new
+          exporter.write_models(models, [target_model] + descendants.keys, filename: filename, output_io: io)
+        end
+      end
+    end
+  end
+end

data/lib/etna/clients/magma/workflows/attribute_actions_from_json_workflow.rb

@@ -0,0 +1,62 @@
+# Updates, renames, or adds attributes to a model, from a JSON file.
+# This workflow:
+# 1) Validates the attributes JSON file.
+# 2) Validates that the model(s) and any link attribute models exist in Magma.
+# 3) Executes each of the updates. This will have to be direct Magma client calls,
+#     since the model_synch workflow only does Adds and does not currently
+#     handle other attribute actions.
+
+require 'json'
+require 'ostruct'
+require_relative './json_validators'
+require_relative './json_converters'
+
+module Etna
+  module Clients
+    class Magma
+      # Note!  These workflows are not perfectly atomic, nor perfectly synchronized due to nature of the backend.
+      # These primitives are best effort locally synchronized, but cannot defend the backend or simultaneous
+      # system updates.
+      class AttributeActionsFromJsonWorkflow < Struct.new(:magma_client, :project_name, :filepath, keyword_init: true)
+        attr_reader :actions
+        def initialize(**params)
+          super({}.update(params))
+
+          actions_json = JSON.parse(File.read(filepath))
+
+          converter = Etna::Clients::Magma::AttributeActionsConverter.new(actions_json)
+          @actions = converter.convert
+
+          validator = Etna::Clients::Magma::AttributeActionsValidator.new(
+            actions,
+            project_models)
+          validator.validate
+
+          raise "Attributes JSON has errors:\n  * #{format_errors(validator.errors)}" unless validator.valid?
+        end
+
+        def format_errors(errors)
+          errors.map { |e| e.gsub("\n", "\n\t") }.join("\n  * ")
+        end
+
+        def project_models
+          @project_models ||= magma_client.retrieve(Etna::Clients::Magma::RetrievalRequest.new(
+            project_name: project_name,
+            model_name: 'all')).models
+        end
+
+        def execute_actions
+          magma_client.update_model(Etna::Clients::Magma::UpdateModelRequest.new(
+            project_name: project_name,
+            actions: actions))
+        end
+
+        def run!
+          puts "Executing the attribute actions against Magma."
+          execute_actions
+          puts "All complete!"
+        end
+      end
+    end
+  end
+end

data/lib/etna/clients/magma/workflows/create_project_workflow.rb

@@ -0,0 +1,123 @@
+# Base workflow for setting up a project by a super user.
+# 1) Creates the project in janus
+# 2) Adds administrator(s) to Janus
+# 3) Refreshes the user's token with the new privileges.
+# 4) Creates the project in Magma.
+
+require 'base64'
+require 'json'
+require 'ostruct'
+require_relative './model_synchronization_workflow'
+require_relative '../../janus/models'
+require_relative './json_validators'
+require_relative './json_converters'
+
+module Etna
+  module Clients
+    class Magma
+      class CreateProjectWorkflow < Struct.new(:magma_client, :janus_client, :project_name, :project_name_full, keyword_init: true)
+        def create_janus_project!
+          janus_client.add_project(Etna::Clients::Janus::AddProjectRequest.new(
+            project_name: project_name,
+            project_name_full: project_name_full
+          ))
+        end
+
+        def add_janus_user(email, name, role)
+          janus_client.add_user(Etna::Clients::Janus::AddUserRequest.new(
+            project_name: project_name,
+            email: email,
+            name: name,
+            role: role,
+          ))
+        end
+
+        def promote_to_administrator(email)
+          janus_client.update_permission(Etna::Clients::Janus::UpdatePermissionRequest.new(
+            project_name: project_name,
+            # email: user['email'],
+            email: email,
+            role: 'administrator',
+          ))
+        end
+
+        def refreshed_token
+          janus_client.refresh_token(Etna::Clients::Janus::RefreshTokenRequest.new).token
+        end
+
+        def update_magma_client_token!
+          self.magma_client = Etna::Clients::Magma.new(
+            host: self.magma_client.host,
+            token: refreshed_token,
+            ignore_ssl: self.magma_client.ignore_ssl)
+        end
+
+        def create_magma_project!
+          magma_client.update_model(Etna::Clients::Magma::UpdateModelRequest.new(
+            project_name: project_name,
+            actions: [Etna::Clients::Magma::AddProjectAction.new]))
+        end
+
+        def create_magma_project_record!
+          magma_client.update(Etna::Clients::Magma::UpdateRequest.new(
+            project_name: project_name,
+            revisions: {
+                'project' => { project_name => { name: project_name } },
+            }))
+        end
+
+        def setup_janus_project!
+          puts "Creating Janus project."
+          create_janus_project!
+          puts "Done! Adding you as an administrator on the project."
+          add_janus_user(user['email'], "#{user['first']} #{user['last']}", 'editor')
+          promote_to_administrator(user['email'])
+          update_magma_client_token!
+
+          puts "Done with setting up the project in Janus!"
+        end
+
+        def setup_magma_project!
+          puts "Creating the project in Magma."
+          create_magma_project!
+          puts "Done! Adding your new project record."
+          create_magma_project_record!
+        end
+
+        def create!
+          setup_janus_project!
+          setup_magma_project!
+
+          while true
+            puts "Add more users? Y/n"
+            break unless STDIN.gets.chomp == 'Y'
+            puts "User name?"
+            name = STDIN.gets.chomp
+            puts "Email?"
+            email = STDIN.gets.chomp
+            puts "Role? (editor/viewer/administrator)"
+            role = STDIN.gets.chomp
+            puts "Adding #{name} (#{email}) as a #{role}."
+            puts "Confirm? Y/n"
+            break unless STDIN.gets.chomp == 'Y'
+
+            if role == 'administrator'
+              add_janus_user(email, name, 'editor')
+              promote_to_administrator(email)
+            else
+              add_janus_user(email, name, role)
+            end
+          end
+
+          puts "All complete!"
+          puts "You need to visit Janus to refresh your token."
+          puts "You can now log into any app to manage your data."
+        end
+
+        def user
+          @user ||= JSON.parse(Base64.urlsafe_decode64(magma_client.token.split('.')[1]))
+        end
+      end
+    end
+  end
+end

data/lib/etna/clients/magma/workflows/crud_workflow.rb

@@ -0,0 +1,85 @@
+require 'ostruct'
+
+module Etna
+  module Clients
+    class Magma
+      class MagmaCrudWorkflow < Struct.new(:magma_client, :project_name, :read_only, :logger, :batch_size, :cache_policy, keyword_init: true)
+        attr_reader :recorded_updates
+
+        def initialize(args)
+          super(**{logger: Etna::Logger.new('/dev/stdout', 999999, 1024 * 1024), batch_size: 30, cache_policy: :update_on_write}.update(args))
+        end
+
+        def lookup_record(model_name, record_id)
+          if (cached = (@cache ||= {}).dig(model_name, record_id))
+            return cached
+          end
+
+          result = magma_client.retrieve(RetrievalRequest.new(project_name: project_name, record_names: [record_id], model_name: model_name))\
+            .models.model(model_name).documents.document(record_id)
+
+          if cache_policy
+            ((@cache ||= {})[model_name] ||= {})[record_id = result]
+          end
+
+          result
+        end
+
+        def page_records(model_name, request = Etna::Clients::Magma::RetrievalRequest.new(
+                project_name: project_name,
+                model_name: model_name,
+                record_names: 'all',
+                page_size: 20,
+                page: 1,
+            ))
+
+          documents = Documents.new({})
+          last_page = nil
+          while last_page.nil? || last_page.raw.length >= 20
+            last_page = magma_client.retrieve(request).models.model(model_name).documents
+            documents += last_page unless block_given?
+            yield last_page if block_given?
+            request.page += 1
+          end
+
+          documents
+        end
+
+        # Todo: Introduce associative concatenation operations for response objects and return
+        # one response that munges the batched responses together.
+        def update_records(method: :update)
+          @recorded_updates ||= UpdateRequest.new(project_name: project_name)
+
+          request = UpdateRequest.new(project_name: project_name)
+          yield request
+
+          case cache_policy
+          when :update_on_write
+            @cache ||= {}
+            request.revisions.each do |model_name, revisions|
+              model_revisions = @cache[model_name] ||= {}
+
+              revisions.each do |record_name, revision|
+                if model_revisions.include? record_name
+                  model_revisions[record_name].update(revision)
+                end
+              end
+            end
+          end
+
+          revisions = request.revisions
+
+          responses = []
+          revisions.to_a.each_slice(batch_size) do |batch|
+            request.revisions = batch.to_h
+            magma_client.send(method, request) unless read_only
+            responses << @recorded_updates.revisions.update(request.revisions)
+          end
+
+          responses
+        end
+      end
+    end
+  end
+end
+

data/lib/etna/clients/magma/workflows/ensure_containing_record_workflow.rb

@@ -0,0 +1,44 @@
+require 'ostruct'
+require_relative './crud_workflow'
+
+module Etna
+  module Clients
+    class Magma
+      class EnsureContainingRecordWorkflow < Struct.new(:magma_crud, :models,  keyword_init: true)
+        def self.from_remote_models(magma_crud:, **opts)
+          opts[:models] = magma_crud.magma_client.retrieve(RetrievalRequest.new(project_name: magma_crud.project_name, model_name: 'all')).models
+          opts[:magma_crud] = magma_crud
+          self.class.new(**opts)
+        end
+
+        def magma_client
+          magma_crud.magma_client
+        end
+
+        def ensure_record(model_name, record_identifiers)
+          raise "Could not find containing model #{model_name} defined." unless (model = models.model(model_name))
+
+          raise "Identifiers #{record_identifiers} do not contain #{model_name}" unless (id = record_identifiers[model_name])
+          record = magma_crud.lookup_record(model_name, id)
+
+          if record.nil?
+            attrs = { model.template.identifier => id }
+
+            parent_attr = model.template.attributes.all.select { |a| a.attribute_type == AttributeType::PARENT }.first
+            unless parent_attr.nil?
+              parent_attribute_name = parent_attr.attribute_name
+              parent_identifier = ensure_record(model.template.parent, record_identifiers)
+              attrs.update({parent_attribute_name => parent_identifier})
+            end
+
+            magma_crud.update_records do |update_request|
+              update_request.update_revision(model_name, id, attrs)
+            end
+          end
+
+          id
+        end
+      end
+    end
+  end
+end

data/lib/etna/clients/magma/workflows/file_attributes_blank_workflow.rb

@@ -0,0 +1,68 @@
+require 'ostruct'
+require_relative './crud_workflow'
+
+module Etna
+  module Clients
+    class Magma
+      class FileAttributesBlankWorkflow < Struct.new(:magma_crud, :model_names, :project_name, keyword_init: true)
+
+        def initialize(opts)
+          super(**{}.update(opts))
+        end
+
+        def magma_client
+          magma_crud.magma_client
+        end
+
+        def find_file_and_image_attributes
+          {}.tap do |all_matches|
+            model_names.each do |model_name|
+              attribute_names = models.model(model_name).template.attributes.all.select {|a|
+                a.attribute_type == Etna::Clients::Magma::AttributeType::FILE ||
+                a.attribute_type == Etna::Clients::Magma::AttributeType::IMAGE}.map { |a|
+                a.attribute_name}
+              all_matches[model_name] = attribute_names
+            end
+          end
+        end
+
+        def each_revision
+          find_file_and_image_attributes.each do |model_name, attribute_names|
+            # Query all records in this model
+            request = Etna::Clients::Magma::RetrievalRequest.new(project_name: project_name)
+            request.model_name = model_name
+            request.attribute_names = attribute_names
+            request.record_names = 'all'
+            documents = magma_client.retrieve(request).models.model(model_name).documents
+
+            documents.document_keys.each do |record_name|
+              document = documents.document(record_name)
+              attribute_names.each do |attribute_name|
+                yield [model_name, record_name, attribute_name] if !document[attribute_name]
+              end
+            end
+          end
+        end
+
+        def set_file_attrs_blank
+          magma_crud.update_records do |update_request|
+            each_revision do |model_name, record_name, attribute_name|
+              update_request.update_revision(model_name, record_name, revision_for(attribute_name))
+            end
+          end
+        end
+
+        def revision_for(attribute_name)
+          {attribute_name => {path: "::blank"}}
+        end
+
+        def models
+          @models ||= begin
+            magma_client.retrieve(RetrievalRequest.new(project_name: self.project_name, model_name: 'all')).models
+          end
+        end
+      end
+    end
+  end
+end
+