mxrb 0.1.1 → 0.1.3

data/lib/mxrb/dsl/builder.rb

@@ -6,6 +6,49 @@ require "base64"
 
 module Mxrb
   module Dsl
+    # Validates project-security references that cross DSL builders.
+    class SecurityValidator
+      def initialize(modules, security)
+        @modules = modules
+        @security = security
+      end
+
+      def validate!
+        return unless @security
+
+        errors = demo_users.flat_map { reference_errors(_1) }
+        duplicates = demo_users.group_by { _1.fetch(:name) }.select { |_name, users| users.size > 1 }
+        errors << "duplicate demo user name(s): #{duplicates.keys.join(', ')}" unless duplicates.empty?
+        return if errors.empty?
+
+        raise ValidationError, "security validation failed:\n- #{errors.join("\n- ")}"
+      end
+
+      private
+
+      def demo_users = Array(@security[:demo_users])
+
+      def reference_errors(user)
+        errors = []
+        missing_roles = user.fetch(:roles) - user_roles
+        errors << "demo user #{user[:name]} references missing user role(s): #{missing_roles.join(', ')}" unless
+          missing_roles.empty?
+        entity = user.fetch(:entity)
+        errors << "demo user #{user[:name]} references missing user entity #{entity}" unless
+          entity == "System.User" || entities.include?(entity)
+        errors
+      end
+
+      def user_roles = @security.fetch(:user_roles, []).map { _1.fetch(:name) }
+
+      def entities
+        @modules.flat_map do |mod|
+          definition = mod.to_h
+          definition.fetch(:entities, []).map { "#{definition.fetch(:name)}.#{_1.fetch(:name)}" }
+        end
+      end
+    end
+
     # Shared widget-building methods for PageBuilder and ContainerBuilder.
     # Includers must implement private `_widget_list` returning the target array.
     module WidgetDsl
@@ -361,9 +404,14 @@ module Mxrb
       end
 
       def validate!
+        validate_security!
         Architecture::Validator.new(graph).validate!
       end
 
+      def validate_security!
+        SecurityValidator.new(@modules, @security).validate!
+      end
+
       def definition
         {
           version: @mendix_version,
@@ -382,6 +430,8 @@ module Mxrb
     class SecurityBuilder
       def initialize
         @user_roles = []
+        @demo_users = []
+        @demo_users_declared = false
         @security_level = nil
         @admin_user_role = nil
         @demo_users_enabled = nil
@@ -414,6 +464,36 @@ module Mxrb
         @demo_users_enabled = enabled == true
       end
 
+      def demo_user(name, entity:, roles:, password:)
+        user_name = name.to_s
+        entity_name = entity.to_s
+        role_names = Array(roles).map(&:to_s).uniq
+        secret = password.to_s
+        raise ArgumentError, 'demo user name must not be empty' if user_name.empty?
+        raise ArgumentError, 'demo user name must not contain whitespace' unless user_name.match?(/\A[^\s]+\z/)
+        unless entity_name.match?(/\A[A-Za-z][A-Za-z0-9_]*\.[A-Za-z][A-Za-z0-9_]*\z/)
+          raise ArgumentError, 'demo user entity must be qualified as Module.Entity'
+        end
+        raise ArgumentError, 'demo user requires at least one role' if role_names.empty?
+        raise ArgumentError, 'demo user password must not be empty' if secret.empty?
+
+        @demo_users << {
+          name: user_name, entity: entity_name, roles: role_names, password: secret
+        }
+        @demo_users_declared = true
+        @demo_users_enabled = true
+      end
+
+      def evaluate(path)
+        instance_eval(File.read(path), path, 1)
+      end
+
+      def evaluate_dir(dir)
+        return unless File.directory?(dir)
+
+        Dir[File.join(dir, '*.rb')].sort.each { |path| evaluate(path) }
+      end
+
       def guest_access(enabled = true, role: nil)
         @guest_access_enabled = enabled == true
         @guest_user_role = role&.to_s
@@ -430,6 +510,7 @@ module Mxrb
       def to_h
         {
           user_roles: @user_roles,
+          demo_users: @demo_users_declared ? @demo_users : nil,
           security_level: @security_level,
           admin_user_role: @admin_user_role,
           demo_users_enabled: @demo_users_enabled,
@@ -791,15 +872,21 @@ module Mxrb
         @scheduled_events << sb.to_h
       end
 
-      def native_document(name, type:, deep_structure:, containment: 'Documents')
+      def native_document(name, type:, deep_structure:, containment: 'Documents',
+                          unit_id: nil, container_id: nil)
         raise ArgumentError, 'deep_structure requires a Hash' unless deep_structure.is_a?(Hash)
 
         @native_documents << {
           name: name.to_s, type: type.to_s, containment: containment.to_s,
+          unit_id: unit_id&.to_s, container_id: container_id&.to_s,
           doc: { '$Type' => type.to_s, 'Name' => name.to_s }.merge(deep_structure)
         }
       end
 
+      def bson_binary(base64, subtype: :generic)
+        BSON::Binary.new(Base64.strict_decode64(base64), subtype.to_sym)
+      end
+
       # Native responsive application shell. It deliberately uses only Mendix
       # core widgets, so generated projects do not depend on Atlas Core merely
       # to display their navigation.
@@ -989,6 +1076,7 @@ module Mxrb
         @generalization = nil
         @system_members = nil
         @indexes = nil
+        @oql_view = nil
       end
 
       ATTR_TYPES.each do |type|
@@ -1000,6 +1088,16 @@ module Mxrb
       def non_persistent!  = (@persistable = false)
       def documentation(d) = (@doc = d)
 
+      # Marks an entity as an OQL-backed view. Modern Mendix projects keep the
+      # query in a separate ViewEntitySourceDocument; older projects may keep
+      # it directly on the embedded entity.
+      def oql_view(source: nil, query: nil)
+        raise ArgumentError, 'oql_view requires source or query' if source.nil? && query.nil?
+
+        @oql_view = { source: source&.to_s, query: query&.to_s }.compact
+        @persistable = false
+      end
+
       def generalizes(entity)
         @generalization = entity.to_s
       end
@@ -1083,7 +1181,7 @@ module Mxrb
           name: @name, persistable: @persistable, documentation: @doc,
           attributes: @attributes, associations: @associations, lifecycle: @lifecycle,
           access_rules: @access_rules, generalization: @generalization,
-          system_members: @system_members, indexes: @indexes
+          system_members: @system_members, indexes: @indexes, oql_view: @oql_view
         }
       end
 

data/lib/mxrb/exporter.rb

@@ -9,6 +9,7 @@ module Mxrb
   # Exports an MPR into an editable, layered Ruby source tree.
   class Exporter
     LAYERS = %w[domain application presentation infrastructure].freeze
+    MODULE_PROGRESS_WEIGHT = 10
     MARKETPLACE_PROVENANCE = [
       ".mxrb/marketplace.lock.json",
       ".mxrb/marketplace",
@@ -63,22 +64,30 @@ module Mxrb
     end
 
     def export!(parallel: true)
-      FileUtils.mkdir_p(@output_dir)
-      Mxrb.open(@mpr_path) do |project|
-        @architecture = project.architecture_definition
-        export_app_structure
-        export_project_assets
-        export_native_units(project)
-        export_security(project)
-        export_architecture_contracts(project)
-        mods = project.modules
-        if parallel && mods.size > 1
-          threads = mods.map { |mod| Thread.new { export_module(mod) } }
-          threads.each(&:join)
-        else
-          mods.each { export_module(_1) }
+      Progress.with("Exporting #{File.basename(@mpr_path)}") do |progress|
+        FileUtils.mkdir_p(@output_dir)
+        Mxrb.open(@mpr_path) do |project|
+          @architecture = project.architecture_definition
+          units = project.all_units
+          modules = project.modules
+          assets = project_asset_files
+          progress.update(
+            current: 0,
+            total: 4 + assets.size + (units.size * 2) + (modules.size * MODULE_PROGRESS_WEIGHT),
+            detail: "preparing Ruby project"
+          )
+          export_app_structure
+          progress.advance(detail: "project structure")
+          export_project_assets(assets, progress)
+          export_native_units(project, units, progress)
+          export_security(project)
+          progress.advance(detail: "project security")
+          export_architecture_contracts(project)
+          progress.advance(detail: "navigation and design system")
+          export_modules(modules, progress, parallel:)
+          write_project(project)
+          progress.advance(detail: "project.rb")
         end
-        write_project(project)
       end
       @output_dir
     end
@@ -91,6 +100,8 @@ module Mxrb
 
       export_domain(root, mod)
       export_microflows(root, mod)
+      export_application_documents(root, mod)
+      export_infrastructure_documents(root, mod)
       export_pages(root, mod)
       export_menus(root, mod)
       export_nanoflows(root, mod)
@@ -106,25 +117,31 @@ module Mxrb
       ].each { write(File.join(@output_dir, _1, ".keep"), "") }
     end
 
-    def export_project_assets
+    def project_asset_files
       source_root = File.dirname(@mpr_path)
-      files = (Model::DesignSystem::ASSET_DIRECTORIES + MARKETPLACE_PROVENANCE).flat_map do |directory|
+      (Model::DesignSystem::ASSET_DIRECTORIES + MARKETPLACE_PROVENANCE).flat_map do |directory|
         root = File.join(source_root, directory)
         next [root] if File.file?(root) && !File.symlink?(root)
         next [] unless File.directory?(root) && !File.symlink?(root)
 
         Dir.glob(File.join(root, "**", "*"), File::FNM_DOTMATCH)
       end.select { File.file?(_1) && !File.symlink?(_1) }.sort
+    end
+
+    def export_project_assets(files, progress)
+      source_root = File.dirname(@mpr_path)
       entries = files.map do |source|
         relative = Pathname.new(source).relative_path_from(Pathname.new(source_root)).to_s
         target = File.join(@output_dir, relative)
         FileUtils.mkdir_p(File.dirname(target))
         FileUtils.cp(source, target)
-        {
+        result = {
           "path" => relative,
           "size" => File.size(source),
           "sha256" => Digest::SHA256.file(source).hexdigest
         }
+        progress.advance(detail: "asset #{relative}")
+        result
       end
       write(
         File.join(@output_dir, ".mxrb", "assets.json"),
@@ -132,8 +149,8 @@ module Mxrb
       )
     end
 
-    def export_native_units(project)
-      units = project.all_units.filter_map do |unit|
+    def export_native_units(project, project_units, progress)
+      units = project_units.filter_map do |unit|
         doc = project.parse_bson(unit)
         type = doc["$Type"]
         next if type.to_s.empty? || type == "Projects$Project"
@@ -147,16 +164,21 @@ module Mxrb
           "type" => type,
           "contents" => Base64.strict_encode64(IO::BsonCodec.serialize(doc))
         }
+      ensure
+        progress.advance(detail: "native baseline")
       end
       write(
         File.join(@output_dir, ".mxrb", "native_units.json"),
         JSON.pretty_generate("format_version" => project.format_version.to_s, "units" => units)
       )
-      source = project.all_units.filter_map do |unit|
+      source = project_units.filter_map do |unit|
         doc = project.parse_bson(unit)
         next if doc["$Type"].to_s.empty? || doc["$Type"] == "Projects$Project"
+        next if Model::Module::EDITABLE_DOCUMENT_TYPES.include?(doc["$Type"])
 
         native_unit_source(project, unit, doc)
+      ensure
+        progress.advance(detail: "editable native units")
       end
       write(
         File.join(@output_dir, ".mxrb", "native_units.rb"),
@@ -164,6 +186,18 @@ module Mxrb
       )
     end
 
+    def export_modules(modules, progress, parallel:)
+      operation = lambda do |mod|
+        export_module(mod)
+        progress.advance(MODULE_PROGRESS_WEIGHT, detail: "module #{mod.name}")
+      end
+      if parallel && modules.size > 1
+        modules.map { |mod| Thread.new { operation.call(mod) } }.each(&:join)
+      else
+        modules.each { operation.call(_1) }
+      end
+    end
+
     def native_unit_source(project, unit, doc)
       <<~RUBY
         native_unit #{ruby(unit.fetch("UnitID"))},
@@ -211,8 +245,10 @@ module Mxrb
 
     def export_module_scaffolding(root)
       %w[
-        domain/enumerations domain/rules domain/policies
-        application/ports/repositories application/queries application/validations application/jobs
+        domain/entities domain/dtos domain/oql_views domain/enumerations domain/constants
+        domain/rules domain/policies
+        application/ports/repositories application/queries application/queries/datasets
+        application/validations application/jobs application/jobs/scheduled_events
         presentation/features presentation/client_actions presentation/snippets presentation/view_models
         presentation/menus
         infrastructure/persistence/mendix infrastructure/persistence/external
@@ -238,26 +274,62 @@ module Mxrb
 
     def export_domain(root, mod)
       domain = File.join(root, "domain")
-      entities_dir = File.join(domain, "entities")
-      FileUtils.mkdir_p(entities_dir)
-
       associations = mod.associations.group_by(&:from_entity_id)
       architecture = architecture_module(mod.name)
-      entity_files = unique_entity_filenames(mod.entities)
+      paths = {}
+      grouped = mod.entities.group_by { entity_domain_route(_1) }
+      grouped.each do |route, entities|
+        unique_entity_filenames(entities).each do |id, filename|
+          paths[id] = File.join(route, filename)
+        end
+      end
+      domain_documents = mod.domain_documents
+      oql_documents = mod.oql_view_documents
+      attached_oql_documents = {}
       mod.entities.each do |entity|
         entity_metadata = architecture&.fetch(:entities, [])&.find { _1[:name] == entity.name }
+        source = entity_source(entity, mod, associations.fetch(entity.id, []), entity_metadata)
+        if oql_view_entity?(entity) && (document = oql_document_for(entity, oql_documents, mod.name))
+          source = "#{source.rstrip}\n\n#{native_document_declaration(document)}"
+          attached_oql_documents[document.fetch(:id)] = true
+        end
         write(
-          File.join(entities_dir, entity_files.fetch(entity.id)),
-          entity_source(entity, mod, associations.fetch(entity.id, []), entity_metadata)
+          File.join(domain, paths.fetch(entity.id)), source
         )
       end
-
-      loads = entity_files.values.sort.map do |filename|
-        %(evaluate File.join(__dir__, "entities", #{ruby(filename)}))
+      used = paths.values.to_h { [_1, true] }
+      domain_documents.reject { attached_oql_documents[_1.fetch(:id)] }.each do |document|
+        relative = unique_relative_path(document.fetch(:route), underscore(document.fetch(:name)), used)
+        write(File.join(domain, relative), mapping_document_source(document))
+        paths[document.fetch(:id)] = relative
+      end
+      loads = paths.values.sort.map do |relative|
+        segments = relative.split(File::SEPARATOR).map { ruby(_1) }.join(", ")
+        %(evaluate File.join(__dir__, #{segments}))
       end
       write(File.join(domain, "model.rb"), "#{loads.join("\n")}\n")
     end
 
+    def entity_domain_route(entity)
+      return 'oql_views' if oql_view_entity?(entity)
+      return 'dtos' unless entity.persistable
+
+      'entities'
+    end
+
+    def oql_view_entity?(entity)
+      entity.respond_to?(:oql_view?) && entity.oql_view?
+    end
+
+    def oql_document_for(entity, documents, module_name)
+      reference = entity.oql_source_document.to_s
+      return nil if reference.empty?
+
+      documents.find do |document|
+        [document.fetch(:name), "#{module_name}.#{document.fetch(:name)}"].include?(reference)
+      end
+    end
+
     def export_microflows(root, mod)
       files = unique_filenames(mod.microflows)
       by_layer = { "application" => [], "infrastructure" => [] }
@@ -296,6 +368,74 @@ module Mxrb
       write_path_aggregator(File.join(presentation, "presentation.rb"), paths)
     end
 
+    def export_infrastructure_documents(root, mod)
+      documents = mod.infrastructure_documents
+      return if documents.empty?
+
+      infrastructure = File.join(root, "infrastructure")
+      used = {}
+      paths = documents.map do |document|
+        base = underscore(document.fetch(:name))
+        relative = unique_relative_path(document.fetch(:route), base, used)
+        write(File.join(infrastructure, relative), mapping_document_source(document))
+        relative
+      end
+      append_to_aggregator(File.join(infrastructure, "infrastructure.rb"), paths)
+    end
+
+    def export_application_documents(root, mod)
+      documents = mod.application_documents
+      return if documents.empty?
+
+      application = File.join(root, 'application')
+      used = {}
+      paths = documents.map do |document|
+        relative = unique_relative_path(document.fetch(:route), underscore(document.fetch(:name)), used)
+        write(File.join(application, relative), mapping_document_source(document))
+        relative
+      end
+      append_to_aggregator(File.join(application, 'application.rb'), paths)
+    end
+
+    def unique_relative_path(directory, base, used)
+      candidate = File.join(directory, "#{base}.rb")
+      suffix = 2
+      while used[candidate]
+        candidate = File.join(directory, "#{base}_#{suffix}.rb")
+        suffix += 1
+      end
+      used[candidate] = true
+      candidate
+    end
+
+    def mapping_document_source(document)
+      <<~RUBY
+        # frozen_string_literal: true
+
+        #{native_document_declaration(document)}
+      RUBY
+    end
+
+    def native_document_declaration(document)
+      <<~RUBY.rstrip
+        native_document #{symbol(document.fetch(:name))},
+                        type: #{ruby(document.fetch(:type))},
+                        unit_id: #{ruby(document.fetch(:id))},
+                        container_id: #{ruby(document.fetch(:container_id))},
+                        containment: #{ruby(document.fetch(:containment))},
+                        deep_structure: #{native_ruby(document.fetch(:doc), 24)}
+      RUBY
+    end
+
+    def append_to_aggregator(path, relative_paths)
+      existing = File.exist?(path) ? File.read(path).lines : []
+      additions = relative_paths.sort.map do |relative|
+        segments = relative.split(File::SEPARATOR).map { ruby(_1) }.join(", ")
+        "evaluate File.join(__dir__, #{segments})\n"
+      end
+      write(path, (existing + additions).uniq.join)
+    end
+
     def export_menus(root, mod)
       menus = mod.menus
       return if menus.empty?
@@ -415,6 +555,12 @@ module Mxrb
       options = []
       options << "  admin_user_role #{ruby(doc["AdminUserRole"])}" unless doc["AdminUserRole"].to_s.empty?
       options << "  demo_users #{doc["EnableDemoUsers"] == true}"
+      demo_users = doc["DemoUsers"] || IO::BsonCodec.build_array([])
+      IO::BsonCodec.parse_array(demo_users).fetch(:items).each do |user|
+        assigned_roles = IO::BsonCodec.parse_array(user["UserRoles"]).fetch(:items)
+        options << "  demo_user #{ruby(user["UserName"])}, entity: #{ruby(user["Entity"])}, " \
+                   "roles: #{ruby(assigned_roles)}, password: #{ruby(user["Password"])}"
+      end
       guest = "  guest_access #{doc["EnableGuestAccess"] == true}"
       guest += ", role: #{ruby(doc["GuestUserRole"])}" unless doc["GuestUserRole"].to_s.empty?
       options << guest
@@ -580,6 +726,12 @@ module Mxrb
       end
       flags = []
       flags << "  non_persistent!" unless entity.persistable
+      if oql_view_entity?(entity)
+        options = []
+        options << "source: #{ruby(entity.oql_source_document)}" if entity.oql_source_document
+        options << "query: #{ruby(entity.oql_query)}" unless entity.oql_query.to_s.empty?
+        flags << "  oql_view #{options.join(', ')}" unless options.empty?
+      end
       flags << "  documentation #{ruby(entity.documentation)}" unless entity.documentation.to_s.empty?
       generalization = entity.respond_to?(:generalization_target) ? entity.generalization_target : nil
       flags << "  generalizes #{ruby(generalization)}" if generalization

data/lib/mxrb/initializer.rb

@@ -152,6 +152,7 @@ module Mxrb
         # frozen_string_literal: true
 
         require "dotenv/load"
+        Dotenv.load(File.join(__dir__, ".env"))
         require "mxrb"
 
         output = ENV.fetch("MXRB_OUTPUT_PATH", File.join(__dir__, "#{@mpr_name}"))

data/lib/mxrb/integrity/validator.rb

@@ -11,19 +11,26 @@ module Mxrb
     class Validator
       def initialize(path)
         @path = path
+        @progress = Progress::NullTask.instance
       end
 
       def validate
-        @errors = []
-        @warnings = []
-        @mpr = IO::MprFile.open(@path, readonly: true)
-        validate_tables
-        validate_units
-        validate_v2_files
-        Result.new(errors: @errors, warnings: @warnings)
+        Progress.with("Validating #{File.basename(@path)}") do |progress|
+          @progress = progress
+          @errors = []
+          @warnings = []
+          @mpr = IO::MprFile.open(@path, readonly: true)
+          validate_tables
+          progress.advance(detail: "MPR tables")
+          validate_units
+          validate_v2_files
+          progress.advance(detail: "v2 contents")
+          Result.new(errors: @errors, warnings: @warnings)
+        end
       rescue Error => e
         Result.new(errors: [e.message], warnings: @warnings || [])
       ensure
+        @progress = Progress::NullTask.instance
         @mpr&.close
       end
 
@@ -40,8 +47,10 @@ module Mxrb
 
       def validate_units
         @units = @mpr.all_units
+        @progress.update(total: @units.size + 3, detail: "#{@units.size} units")
         validate_root
         validate_unit_ids
+        @progress.advance(detail: "unit tree")
         validate_contents
       end
 
@@ -78,6 +87,8 @@ module Mxrb
           next unless doc
 
           validate_doc_identity(unit, doc)
+        ensure
+          @progress&.advance(detail: "unit #{unit['UnitID']}")
         end
       end
 

data/lib/mxrb/model/entity.rb

@@ -10,7 +10,7 @@ module Mxrb
       attr_accessor :id, :name, :qualified_name, :documentation,
                     :persistable, :location, :data_storage_guid,
                     :export_level, :generalization, :access_rules, :indexes,
-                    :system_members
+                    :system_members, :source, :oql_query, :native_type
 
       # Build from a BSON hash (embedded in DomainModel's "entities" array).
       def self.from_bson(doc, _domain_model_id, mpr)
@@ -19,6 +19,9 @@ module Mxrb
         e.name               = doc["name"]  || doc["Name"]
         e.qualified_name     = doc["$QualifiedName"] || doc["\$QualifiedName"]
         e.documentation      = doc["documentation"] || doc["Documentation"] || ""
+        e.native_type        = doc["$Type"]
+        e.source             = doc["source"] || doc["Source"]
+        e.oql_query          = doc["oqlQuery"] || doc["OqlQuery"] || doc["OQLQuery"]
         e.data_storage_guid  = doc["dataStorageGuid"] || doc["DataStorageGuid"]
         e.export_level       = doc["exportLevel"] || doc["ExportLevel"] || "Hidden"
         e.location           = parse_location(doc["location"] || doc["Location"])
@@ -58,6 +61,19 @@ module Mxrb
         @generalization['generalization'] || @generalization['Generalization']
       end
 
+      def oql_view?
+        source_type = @source.is_a?(Hash) ? @source.fetch('$Type', '') : ''
+        @native_type.to_s.match?(/ViewEntity/i) ||
+          source_type.match?(/OqlViewEntitySource/i) ||
+          !@oql_query.to_s.empty?
+      end
+
+      def oql_source_document
+        return unless @source.is_a?(Hash)
+
+        @source['SourceDocument'] || @source['sourceDocument']
+      end
+
       def to_bson
         {
           "$ID"             => @id,