mxrb 0.1.2 → 0.1.4

data/lib/mxrb/exporter.rb

@@ -8,7 +8,9 @@ require "digest"
 module Mxrb
   # Exports an MPR into an editable, layered Ruby source tree.
   class Exporter
+    MODES = %i[mendix ruby].freeze
     LAYERS = %w[domain application presentation infrastructure].freeze
+    MODULE_PROGRESS_WEIGHT = 10
     MARKETPLACE_PROVENANCE = [
       ".mxrb/marketplace.lock.json",
       ".mxrb/marketplace",
@@ -57,40 +59,64 @@ module Mxrb
       Microflows$RestCallAction
     ].freeze
 
-    def initialize(mpr_path, output_dir)
+    attr_reader :mode
+
+    def initialize(mpr_path, output_dir, mode: :mendix)
       @mpr_path = File.expand_path(mpr_path)
       @output_dir = File.expand_path(output_dir)
+      @mode = mode.to_sym
+      raise ArgumentError, "export mode must be mendix or ruby" unless MODES.include?(@mode)
     end
 
     def export!(parallel: true)
-      FileUtils.mkdir_p(@output_dir)
-      Mxrb.open(@mpr_path) do |project|
-        @architecture = project.architecture_definition
-        export_app_structure
-        export_project_assets
-        export_native_units(project)
-        export_security(project)
-        export_architecture_contracts(project)
-        mods = project.modules
-        if parallel && mods.size > 1
-          threads = mods.map { |mod| Thread.new { export_module(mod) } }
-          threads.each(&:join)
-        else
-          mods.each { export_module(_1) }
+      return export_ruby!(parallel:) if mode == :ruby
+
+      Progress.with("Exporting #{File.basename(@mpr_path)}") do |progress|
+        FileUtils.mkdir_p(@output_dir)
+        Mxrb.open(@mpr_path) do |project|
+          @architecture = project.architecture_definition
+          units = project.all_units
+          modules = project.modules
+          @inferred_public_artifacts = infer_public_artifacts(modules)
+          assets = project_asset_files
+          progress.update(
+            current: 0,
+            total: 4 + assets.size + (units.size * 2) + (modules.size * MODULE_PROGRESS_WEIGHT),
+            detail: "preparing Ruby project"
+          )
+          export_app_structure
+          progress.advance(detail: "project structure")
+          export_project_assets(assets, progress)
+          export_native_units(project, units, progress)
+          ruby_sources = export_ruby_app_sources(project)
+          export_security(project)
+          progress.advance(detail: "project security")
+          export_architecture_contracts(project)
+          progress.advance(detail: "navigation and design system")
+          export_modules(modules, progress, parallel:)
+          write_project(project, ruby_sources:)
+          progress.advance(detail: "project.rb")
         end
-        write_project(project)
       end
       @output_dir
     end
 
     private
 
+    def export_ruby!(parallel:)
+      sidecar = File.join(@output_dir, ".mxrb", "mendix")
+      self.class.new(@mpr_path, sidecar, mode: :mendix).export!(parallel:)
+      RubyApp::Exporter.new(@mpr_path, @output_dir, mendix_sidecar: sidecar).export!
+    end
+
     def export_module(mod)
       root = File.join(@output_dir, "modules", mod.name)
       LAYERS.each { FileUtils.mkdir_p(File.join(root, _1)) }
 
       export_domain(root, mod)
       export_microflows(root, mod)
+      export_application_documents(root, mod)
+      export_infrastructure_documents(root, mod)
       export_pages(root, mod)
       export_menus(root, mod)
       export_nanoflows(root, mod)
@@ -106,25 +132,31 @@ module Mxrb
       ].each { write(File.join(@output_dir, _1, ".keep"), "") }
     end
 
-    def export_project_assets
+    def project_asset_files
       source_root = File.dirname(@mpr_path)
-      files = (Model::DesignSystem::ASSET_DIRECTORIES + MARKETPLACE_PROVENANCE).flat_map do |directory|
+      (Model::DesignSystem::ASSET_DIRECTORIES + MARKETPLACE_PROVENANCE).flat_map do |directory|
         root = File.join(source_root, directory)
         next [root] if File.file?(root) && !File.symlink?(root)
         next [] unless File.directory?(root) && !File.symlink?(root)
 
         Dir.glob(File.join(root, "**", "*"), File::FNM_DOTMATCH)
       end.select { File.file?(_1) && !File.symlink?(_1) }.sort
+    end
+
+    def export_project_assets(files, progress)
+      source_root = File.dirname(@mpr_path)
       entries = files.map do |source|
         relative = Pathname.new(source).relative_path_from(Pathname.new(source_root)).to_s
         target = File.join(@output_dir, relative)
         FileUtils.mkdir_p(File.dirname(target))
         FileUtils.cp(source, target)
-        {
+        result = {
           "path" => relative,
           "size" => File.size(source),
           "sha256" => Digest::SHA256.file(source).hexdigest
         }
+        progress.advance(detail: "asset #{relative}")
+        result
       end
       write(
         File.join(@output_dir, ".mxrb", "assets.json"),
@@ -132,8 +164,8 @@ module Mxrb
       )
     end
 
-    def export_native_units(project)
-      units = project.all_units.filter_map do |unit|
+    def export_native_units(project, project_units, progress)
+      units = project_units.filter_map do |unit|
         doc = project.parse_bson(unit)
         type = doc["$Type"]
         next if type.to_s.empty? || type == "Projects$Project"
@@ -147,16 +179,21 @@ module Mxrb
           "type" => type,
           "contents" => Base64.strict_encode64(IO::BsonCodec.serialize(doc))
         }
+      ensure
+        progress.advance(detail: "native baseline")
       end
       write(
         File.join(@output_dir, ".mxrb", "native_units.json"),
         JSON.pretty_generate("format_version" => project.format_version.to_s, "units" => units)
       )
-      source = project.all_units.filter_map do |unit|
+      source = project_units.filter_map do |unit|
         doc = project.parse_bson(unit)
         next if doc["$Type"].to_s.empty? || doc["$Type"] == "Projects$Project"
+        next if Model::Module::EDITABLE_DOCUMENT_TYPES.include?(doc["$Type"])
 
         native_unit_source(project, unit, doc)
+      ensure
+        progress.advance(detail: "editable native units")
       end
       write(
         File.join(@output_dir, ".mxrb", "native_units.rb"),
@@ -164,6 +201,36 @@ module Mxrb
       )
     end
 
+    def export_ruby_app_sources(project)
+      files = project.mpr.ruby_app_sources
+      return false if files.empty?
+
+      payload = files.map do |file|
+        contents = file.fetch(:contents)
+        {
+          "path" => file.fetch(:path), "sha256" => file.fetch(:sha256),
+          "contents" => Base64.strict_encode64(contents), "mode" => file.fetch(:mode, 0o644)
+        }
+      end
+      write(
+        File.join(@output_dir, ".mxrb", "ruby_sources.json"),
+        JSON.pretty_generate("version" => 1, "files" => payload)
+      )
+      true
+    end
+
+    def export_modules(modules, progress, parallel:)
+      operation = lambda do |mod|
+        export_module(mod)
+        progress.advance(MODULE_PROGRESS_WEIGHT, detail: "module #{mod.name}")
+      end
+      if parallel && modules.size > 1
+        modules.map { |mod| Thread.new { operation.call(mod) } }.each(&:join)
+      else
+        modules.each { operation.call(_1) }
+      end
+    end
+
     def native_unit_source(project, unit, doc)
       <<~RUBY
         native_unit #{ruby(unit.fetch("UnitID"))},
@@ -211,8 +278,10 @@ module Mxrb
 
     def export_module_scaffolding(root)
       %w[
-        domain/enumerations domain/rules domain/policies
-        application/ports/repositories application/queries application/validations application/jobs
+        domain/entities domain/dtos domain/oql_views domain/enumerations domain/constants
+        domain/rules domain/policies
+        application/ports/repositories application/queries application/queries/datasets
+        application/validations application/jobs application/jobs/scheduled_events
         presentation/features presentation/client_actions presentation/snippets presentation/view_models
         presentation/menus
         infrastructure/persistence/mendix infrastructure/persistence/external
@@ -238,26 +307,62 @@ module Mxrb
 
     def export_domain(root, mod)
       domain = File.join(root, "domain")
-      entities_dir = File.join(domain, "entities")
-      FileUtils.mkdir_p(entities_dir)
-
       associations = mod.associations.group_by(&:from_entity_id)
       architecture = architecture_module(mod.name)
-      entity_files = unique_entity_filenames(mod.entities)
+      paths = {}
+      grouped = mod.entities.group_by { entity_domain_route(_1) }
+      grouped.each do |route, entities|
+        unique_entity_filenames(entities, route:).each do |id, filename|
+          paths[id] = File.join(route, filename)
+        end
+      end
+      domain_documents = mod.domain_documents
+      oql_documents = mod.oql_view_documents
+      attached_oql_documents = {}
       mod.entities.each do |entity|
         entity_metadata = architecture&.fetch(:entities, [])&.find { _1[:name] == entity.name }
+        source = entity_source(entity, mod, associations.fetch(entity.id, []), entity_metadata)
+        if oql_view_entity?(entity) && (document = oql_document_for(entity, oql_documents, mod.name))
+          source = "#{source.rstrip}\n\n#{native_document_declaration(document)}"
+          attached_oql_documents[document.fetch(:id)] = true
+        end
         write(
-          File.join(entities_dir, entity_files.fetch(entity.id)),
-          entity_source(entity, mod, associations.fetch(entity.id, []), entity_metadata)
+          File.join(domain, paths.fetch(entity.id)), source
         )
       end
-
-      loads = entity_files.values.sort.map do |filename|
-        %(evaluate File.join(__dir__, "entities", #{ruby(filename)}))
+      used = paths.values.to_h { [_1, true] }
+      domain_documents.reject { attached_oql_documents[_1.fetch(:id)] }.each do |document|
+        relative = unique_relative_path(document.fetch(:route), underscore(document.fetch(:name)), used)
+        write(File.join(domain, relative), mapping_document_source(document))
+        paths[document.fetch(:id)] = relative
+      end
+      loads = paths.values.sort.map do |relative|
+        segments = relative.split(File::SEPARATOR).map { ruby(_1) }.join(", ")
+        %(evaluate File.join(__dir__, #{segments}))
       end
       write(File.join(domain, "model.rb"), "#{loads.join("\n")}\n")
     end
 
+    def entity_domain_route(entity)
+      return 'oql_views' if oql_view_entity?(entity)
+      return 'dtos' unless entity.persistable
+
+      'entities'
+    end
+
+    def oql_view_entity?(entity)
+      entity.respond_to?(:oql_view?) && entity.oql_view?
+    end
+
+    def oql_document_for(entity, documents, module_name)
+      reference = entity.oql_source_document.to_s
+      return nil if reference.empty?
+
+      documents.find do |document|
+        [document.fetch(:name), "#{module_name}.#{document.fetch(:name)}"].include?(reference)
+      end
+    end
+
     def export_microflows(root, mod)
       files = unique_filenames(mod.microflows)
       by_layer = { "application" => [], "infrastructure" => [] }
@@ -265,7 +370,10 @@ module Mxrb
       mod.microflows.each do |flow|
         layer, category = flow_location(flow.name)
         relative = File.join(category, files.fetch(flow.id))
-        metadata = architecture&.fetch(:microflows, [])&.find { _1[:name] == flow.name }
+        metadata = flow_metadata(
+          mod.name, :microflow, flow.name,
+          architecture&.fetch(:microflows, [])&.find { _1[:name] == flow.name }
+        )
         write(File.join(root, layer, relative), microflow_source(flow, metadata))
         by_layer.fetch(layer) << relative
       end
@@ -296,6 +404,74 @@ module Mxrb
       write_path_aggregator(File.join(presentation, "presentation.rb"), paths)
     end
 
+    def export_infrastructure_documents(root, mod)
+      documents = mod.infrastructure_documents
+      return if documents.empty?
+
+      infrastructure = File.join(root, "infrastructure")
+      used = {}
+      paths = documents.map do |document|
+        base = underscore(document.fetch(:name))
+        relative = unique_relative_path(document.fetch(:route), base, used)
+        write(File.join(infrastructure, relative), mapping_document_source(document))
+        relative
+      end
+      append_to_aggregator(File.join(infrastructure, "infrastructure.rb"), paths)
+    end
+
+    def export_application_documents(root, mod)
+      documents = mod.application_documents
+      return if documents.empty?
+
+      application = File.join(root, 'application')
+      used = {}
+      paths = documents.map do |document|
+        relative = unique_relative_path(document.fetch(:route), underscore(document.fetch(:name)), used)
+        write(File.join(application, relative), mapping_document_source(document))
+        relative
+      end
+      append_to_aggregator(File.join(application, 'application.rb'), paths)
+    end
+
+    def unique_relative_path(directory, base, used)
+      candidate = File.join(directory, "#{base}.rb")
+      suffix = 2
+      while used[candidate]
+        candidate = File.join(directory, "#{base}_#{suffix}.rb")
+        suffix += 1
+      end
+      used[candidate] = true
+      candidate
+    end
+
+    def mapping_document_source(document)
+      <<~RUBY
+        # frozen_string_literal: true
+
+        #{native_document_declaration(document)}
+      RUBY
+    end
+
+    def native_document_declaration(document)
+      <<~RUBY.rstrip
+        native_document #{symbol(document.fetch(:name))},
+                        type: #{ruby(document.fetch(:type))},
+                        unit_id: #{ruby(document.fetch(:id))},
+                        container_id: #{ruby(document.fetch(:container_id))},
+                        containment: #{ruby(document.fetch(:containment))},
+                        deep_structure: #{native_ruby(document.fetch(:doc), 24)}
+      RUBY
+    end
+
+    def append_to_aggregator(path, relative_paths)
+      existing = File.exist?(path) ? File.read(path).lines : []
+      additions = relative_paths.sort.map do |relative|
+        segments = relative.split(File::SEPARATOR).map { ruby(_1) }.join(", ")
+        "evaluate File.join(__dir__, #{segments})\n"
+      end
+      write(path, (existing + additions).uniq.join)
+    end
+
     def export_menus(root, mod)
       menus = mod.menus
       return if menus.empty?
@@ -326,7 +502,10 @@ module Mxrb
       architecture = architecture_module(mod.name)
       flows.each do |flow|
         relative = File.join("client_actions", files.fetch(flow.id))
-        metadata = architecture&.fetch(:nanoflows, [])&.find { _1[:name] == flow.name }
+        metadata = flow_metadata(
+          mod.name, :nanoflow, flow.name,
+          architecture&.fetch(:nanoflows, [])&.find { _1[:name] == flow.name }
+        )
         write(File.join(presentation, relative), nanoflow_source(flow, metadata))
         paths << relative
       end
@@ -357,7 +536,7 @@ module Mxrb
       end
     end
 
-    def write_project(project)
+    def write_project(project, ruby_sources: false)
       module_loads = project.modules.map do |mod|
         %(  evaluate File.join(__dir__, "modules", #{ruby(mod.name)}, "module.rb"))
       end.join("\n")
@@ -373,6 +552,7 @@ module Mxrb
           mendix_version #{ruby(project.mendix_version || "10.18.0")}
           native_units File.join(__dir__, ".mxrb", "native_units.json")
           project_assets File.join(__dir__, ".mxrb", "assets.json"), root: __dir__
+      #{'    ruby_app_sources File.join(__dir__, ".mxrb", "ruby_sources.json")' if ruby_sources}
           evaluate File.join(__dir__, ".mxrb", "native_units.rb")
           evaluate File.join(__dir__, "app", "security", "security.rb")
           evaluate File.join(__dir__, "app", "navigation", "navigation.rb")
@@ -586,6 +766,12 @@ module Mxrb
       end
       flags = []
       flags << "  non_persistent!" unless entity.persistable
+      if oql_view_entity?(entity)
+        options = []
+        options << "source: #{ruby(entity.oql_source_document)}" if entity.oql_source_document
+        options << "query: #{ruby(entity.oql_query)}" unless entity.oql_query.to_s.empty?
+        flags << "  oql_view #{options.join(', ')}" unless options.empty?
+      end
       flags << "  documentation #{ruby(entity.documentation)}" unless entity.documentation.to_s.empty?
       generalization = entity.respond_to?(:generalization_target) ? entity.generalization_target : nil
       flags << "  generalizes #{ruby(generalization)}" if generalization
@@ -678,19 +864,21 @@ module Mxrb
       end
     end
 
-    def unique_entity_filenames(entities)
-      unique_filenames(entities)
+    def unique_entity_filenames(entities, route: nil)
+      suffix = route == "dtos" ? "_dto" : nil
+      unique_filenames(entities, suffix:)
     end
 
-    def unique_filenames(entities)
+    def unique_filenames(entities, suffix: nil)
       used = {}
       entities.to_h do |entity|
         base = underscore(entity.name)
+        base = "#{base}#{suffix}" if suffix && !base.end_with?(suffix)
         candidate = "#{base}.rb"
-        suffix = 2
+        sequence = 2
         while used[candidate]
-          candidate = "#{base}_#{suffix}.rb"
-          suffix += 1
+          candidate = "#{base}_#{sequence}.rb"
+          sequence += 1
         end
         used[candidate] = true
         [entity.id, candidate]
@@ -718,6 +906,8 @@ module Mxrb
       body << "  return_type #{symbol(flow.return_type)}" if flow.return_type.is_a?(String)
       body << "  documentation #{ruby(flow.documentation)}" unless flow.documentation.to_s.empty?
       body << "  allow_concurrent_execution #{flow.allow_concurrent_execution ? 'true' : 'false'}"
+      apply_entity_access = flow.respond_to?(:apply_entity_access) && flow.apply_entity_access
+      body << "  apply_entity_access #{apply_entity_access ? 'true' : 'false'}"
       body << "  mark_as_used #{flow.mark_as_used ? 'true' : 'false'}"
       body << "  excluded #{flow.excluded ? 'true' : 'false'}"
       body << "  allowed_roles #{flow.allowed_module_roles.map { symbol(_1) }.join(', ')}" unless flow.allowed_module_roles.empty?
@@ -737,10 +927,12 @@ module Mxrb
       elsif objects.any?
         body << "  # Native body baseline retained: this graph shape has no typed DSL mapping yet"
       end
+      declaration = [symbol(flow.name)]
+      declaration << "public: true" if metadata&.fetch(:public, false)
       <<~RUBY
         # frozen_string_literal: true
 
-        microflow #{symbol(flow.name)} do
+        microflow #{declaration.join(', ')} do
       #{body.join("\n")}
         end
       RUBY
@@ -781,7 +973,7 @@ module Mxrb
       body << "  allowed_roles #{page.allowed_module_roles.map { symbol(_1) }.join(', ')}" unless page.allowed_module_roles.empty?
       deep = page_deep_structure(page)
       if (source = metadata&.dig(:data_source) || page.data_source)
-        body << "  data_source #{source.fetch(:kind)}: #{symbol(source.fetch(:name))}"
+        body << "  data_source #{source.fetch(:kind)}: #{reference(source.fetch(:name))}"
       end
       body << "  deep_structure(#{native_ruby(deep, 2)})" if deep
       metadata&.fetch(:events, [])&.each do |event|
@@ -870,7 +1062,9 @@ module Mxrb
     def render_widget(widget, indent)
       pad       = " " * indent
       child_pad = " " * (indent + 2)
-      type      = widget.fetch(:type)
+      # Architecture metadata is serialized through JSON inside generated MPRs,
+      # so a second export can surface widget type values as strings.
+      type      = widget.fetch(:type).to_sym
       options   = widget.fetch(:options, {})
 
       # Snippet: snippet :name, from: "..."
@@ -1710,6 +1904,32 @@ module Mxrb
       @architecture&.fetch(:modules, [])&.find { _1[:name] == name }
     end
 
+    def infer_public_artifacts(modules)
+      modules.each_with_object({}) do |mod, public_artifacts|
+        mod.pages.each do |page|
+          source = page.data_source
+          next unless source
+
+          target_module, target_name = source.fetch(:name).to_s.split(".", 2)
+          next if target_name.to_s.empty? || target_module == mod.name
+
+          public_artifacts[[target_module, source.fetch(:kind).to_sym, target_name]] = true
+        end
+      end
+    end
+
+    def flow_metadata(module_name, kind, name, metadata)
+      result = metadata ? metadata.dup : {}
+      if @inferred_public_artifacts&.include?([module_name, kind, name])
+        result[:public] = true
+      end
+      result.empty? ? nil : result
+    end
+
+    def reference(value)
+      value.to_s.include?(".") ? ruby(value.to_s) : symbol(value)
+    end
+
     def symbol(value)
       value.to_s.to_sym.inspect
     end

data/lib/mxrb/http/server.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+require 'puma'
+require 'uri'
+
+module Mxrb
+  module Http
+    # Case-insensitive HTTP header collection used by the internal request API.
+    class Headers
+      def initialize(values = {})
+        @values = values.to_h { |name, value| [name.to_s.downcase, value.to_s] }
+      end
+
+      def [](name) = @values[name.to_s.downcase]
+    end
+
+    Request = Data.define(:path, :request_method, :body, :query, :headers) do
+      def [](name) = headers[name]
+    end
+
+    # Mutable response keeps the existing MXRB dispatchers independent of Rack.
+    class Response
+      attr_accessor :status, :body
+      attr_reader :headers
+
+      def initialize
+        @status = 200
+        @body = ''
+        @headers = {}
+      end
+
+      def [](name) = headers[name]
+
+      def []=(name, value)
+        headers[name] = value
+      end
+    end
+
+    # Small embedded-Puma adapter for MXRB's loopback-only HTTP services.
+    class Server
+      DEFAULT_MAX_THREADS = 5
+
+      attr_reader :host, :port, :puma
+
+      def initialize(host:, port:, logger: nil, max_threads: DEFAULT_MAX_THREADS, &dispatcher)
+        raise ArgumentError, 'HTTP dispatcher is required' unless dispatcher
+
+        @host = host.to_s
+        @port = Integer(port)
+        @dispatcher = dispatcher
+        @logger = logger
+        @max_threads = Integer(max_threads)
+      end
+
+      def start
+        @puma = Puma::Server.new(method(:call), nil, puma_options)
+        @puma.add_tcp_listener(host, port)
+        yield(@puma) if block_given?
+        @thread = @puma.run
+        @thread.join
+      end
+
+      def shutdown = @puma&.stop
+
+      def call(environment)
+        request = request_from(environment)
+        response = Response.new
+        @dispatcher.call(request, response)
+        [response.status, response.headers, [response.body.to_s]]
+      end
+
+      private
+
+      def puma_options
+        {
+          min_threads: 0,
+          max_threads: @max_threads,
+          environment: 'production',
+          log_writer: @logger || Puma::LogWriter.null,
+          queue_requests: true
+        }
+      end
+
+      def request_from(environment)
+        input = environment['rack.input']
+        body = input ? input.read.to_s : ''
+        input.rewind if input.respond_to?(:rewind)
+        Request.new(
+          path: "#{environment['SCRIPT_NAME']}#{environment.fetch('PATH_INFO', '/')}",
+          request_method: environment.fetch('REQUEST_METHOD', 'GET'),
+          body:,
+          query: URI.decode_www_form(environment['QUERY_STRING'].to_s).to_h,
+          headers: Headers.new(request_headers(environment))
+        )
+      end
+
+      def request_headers(environment)
+        environment.each_with_object({}) do |(name, value), headers|
+          next unless name.start_with?('HTTP_')
+
+          header = name.delete_prefix('HTTP_').split('_').map(&:capitalize).join('-')
+          headers[header] = value
+        end
+      end
+    end
+  end
+end