mxrb 0.1.1 → 0.1.3

data/lib/mxrb/model/module.rb

@@ -7,6 +7,40 @@ module Mxrb
     # $Type: Projects$Module
     # ContainmentName: "Modules"
     class Module < Unit
+      INFRASTRUCTURE_DOCUMENT_ROUTES = {
+        'ExportMappings$ExportMapping' => 'mappings/exports',
+        'ImportMappings$ImportMapping' => 'mappings/imports',
+        'JsonStructures$JsonStructure' => 'mappings/json_structures',
+        'MessageDefinitions$MessageDefinitionCollection' => 'mappings/message_definitions',
+        'MessageDefinitions$MessageDefinition2' => 'mappings/message_definitions',
+        'XmlSchemas$XmlSchema' => 'mappings/xml_schemas',
+        'Rest$PublishedRestService' => 'endpoints',
+        'WebServices$PublishedService' => 'endpoints',
+        'WebServices$PublishedWebService' => 'endpoints',
+        'ODataPublish$PublishedODataService' => 'endpoints',
+        'ODataPublish$PublishedODataService2' => 'endpoints',
+        'Rest$ConsumedRestService' => 'integrations',
+        'Rest$ConsumedODataService' => 'integrations',
+        'AppServices$ConsumedAppService' => 'integrations',
+        'ODataImport$ConsumedODataService' => 'integrations'
+      }.freeze
+      MAPPING_DOCUMENT_TYPES = INFRASTRUCTURE_DOCUMENT_ROUTES.keys.grep(
+        /Mappings|JsonStructures|MessageDefinitions|XmlSchemas/
+      ).freeze
+      APPLICATION_DOCUMENT_ROUTES = {
+        'DataSets$DataSet' => 'queries/datasets',
+        'ScheduledEvents$ScheduledEvent' => 'jobs/scheduled_events'
+      }.freeze
+      DOMAIN_DOCUMENT_ROUTES = {
+        'DomainModels$ViewEntitySourceDocument' => 'oql_views',
+        'Enumerations$Enumeration' => 'enumerations',
+        'Constants$Constant' => 'constants'
+      }.freeze
+      EDITABLE_DOCUMENT_TYPES = (
+        INFRASTRUCTURE_DOCUMENT_ROUTES.keys + APPLICATION_DOCUMENT_ROUTES.keys +
+        DOMAIN_DOCUMENT_ROUTES.keys
+      ).freeze
+
       attr_reader :name, :sort_index, :from_app_store,
                   :app_store_guid, :app_store_version, :export_level
 
@@ -84,6 +118,28 @@ module Mxrb
                               .map { unit_to_doc(_1) }
       end
 
+      def mapping_documents
+        infrastructure_documents.select { MAPPING_DOCUMENT_TYPES.include?(_1[:type]) }
+      end
+
+      def infrastructure_documents
+        @infrastructure_documents ||= routed_documents(INFRASTRUCTURE_DOCUMENT_ROUTES)
+      end
+
+      def application_documents
+        @application_documents ||= routed_documents(APPLICATION_DOCUMENT_ROUTES)
+      end
+
+      def domain_documents
+        @domain_documents ||= routed_documents(DOMAIN_DOCUMENT_ROUTES)
+      end
+
+      def oql_view_documents
+        @oql_view_documents ||= domain_documents.select do |document|
+          document[:type] == 'DomainModels$ViewEntitySourceDocument'
+        end
+      end
+
       def module_roles
         @module_roles ||= begin
           raw = @mpr.children_of(@id).find { _1["ContainmentName"] == "ModuleSecurity" }
@@ -109,6 +165,21 @@ module Mxrb
         @mpr.parse_contents(unit_hash[:raw])
       end
 
+      def routed_documents(routes)
+        document_units.filter_map do |unit|
+          route = routes[unit[:type]]
+          next unless route
+
+          raw = unit.fetch(:raw)
+          doc = @mpr.parse_contents(raw)
+          {
+            id: raw.fetch("UnitID"), container_id: raw.fetch("ContainerID"),
+            containment: raw.fetch("ContainmentName"), type: unit.fetch(:type),
+            name: doc["Name"] || doc["name"] || raw.fetch("UnitID"), doc:, route:
+          }
+        end
+      end
+
       # Documents live in ContainmentName = "Documents" recursively under this module.
       # We do a simple two-pass: direct Documents children + Documents inside Folders.
       def document_units

data/lib/mxrb/official_marketplace/module_package_importer.rb

@@ -122,6 +122,7 @@ module Mxrb
       def install(staged_files, protected_files = [])
         staged_files.each do |relative, source|
           install_file(relative, source) unless protected_files.include?(relative)
+          yield(relative) if block_given?
         end
       end
 
@@ -179,9 +180,12 @@ module Mxrb
       end
 
       def import!
-        validate_inputs!
-        Dir.mktmpdir('mxrb-module-import-') do |temporary|
-          Zip::File.open(@package_path) { import_archive(_1, temporary) }
+        Progress.with("Importing #{File.basename(@package_path)}") do |progress|
+          validate_inputs!
+          progress.update(detail: 'reading package')
+          Dir.mktmpdir('mxrb-module-import-') do |temporary|
+            Zip::File.open(@package_path) { import_archive(_1, temporary, progress) }
+          end
         end
       rescue Zip::Error, REXML::ParseException => e
         raise MarketplaceError, "invalid Mendix module package: #{e.message}"
@@ -189,12 +193,12 @@ module Mxrb
 
       private
 
-      def import_archive(archive, temporary)
+      def import_archive(archive, temporary, progress)
         reader = ModulePackageReader.new(archive)
         descriptor = reader.descriptor
         source_path = reader.extract_project(descriptor, temporary)
         staged_files = reader.stage_files(descriptor.files, temporary)
-        import_project(source_path, descriptor, staged_files, temporary)
+        import_project(source_path, descriptor, staged_files, temporary, progress)
       end
 
       def validate_inputs!
@@ -203,12 +207,13 @@ module Mxrb
         raise MarketplaceError, "target root not found: #{@target_root}" unless File.directory?(@target_root)
       end
 
-      def import_project(source_path, descriptor, staged_files, temporary) # rubocop:disable Metrics/MethodLength
+      def import_project(source_path, descriptor, staged_files, temporary, progress) # rubocop:disable Metrics/MethodLength
         source = IO::MprFile.open(source_path, readonly: true)
         target = IO::MprFile.open(@mpr_path)
         assets = ModulePackageAssets.new(@target_root, temporary)
         imported_ids = []
-        import_transaction(source, target, descriptor, staged_files, assets, imported_ids)
+        import_transaction(source, target, descriptor, staged_files, assets, imported_ids, progress)
+        progress.advance(detail: 'import transaction')
         import_result(source, target, descriptor, staged_files, imported_ids)
       rescue StandardError
         assets&.rollback
@@ -220,24 +225,30 @@ module Mxrb
       end
 
       # rubocop:disable Metrics/ParameterLists
-      def import_transaction(source, target, descriptor, staged_files, assets, imported_ids)
+      def import_transaction(source, target, descriptor, staged_files, assets, imported_ids, progress) # rubocop:disable Metrics/MethodLength
         validate_versions!(source, target, descriptor)
         module_unit, units = package_units(source, descriptor.name)
         validate_target!(target, descriptor.name, units)
+        progress.update(
+          current: 2, total: units.size + staged_files.size + 3,
+          detail: "#{units.size} model units"
+        )
         target.transaction do
-          imported_ids.concat(insert_units(source, target, module_unit, units))
-          assets.install(staged_files, @protected_files)
+          imported_ids.concat(insert_units(source, target, module_unit, units, progress))
+          assets.install(staged_files, @protected_files) do |relative|
+            progress.advance(detail: "asset #{relative}")
+          end
         end
       end
       # rubocop:enable Metrics/ParameterLists
 
-      def insert_units(source, target, module_unit, units)
+      def insert_units(source, target, module_unit, units, progress)
         units.map do |unit|
           container = unit == module_unit ? target.root_unit.fetch('UnitID') : unit.fetch('ContainerID')
           target.insert_unit(
             container_uuid: container, containment_name: unit.fetch('ContainmentName'),
             contents_doc: source.parse_contents(unit)
-          )
+          ).tap { progress.advance(detail: "unit #{unit.fetch('UnitID')}") }
         end
       end
 

data/lib/mxrb/official_marketplace/widget_package_installer.rb

@@ -200,17 +200,21 @@ module Mxrb
       end
 
       def install(archive, package)
-        validate_target!
-        inventory = WidgetPackageInventory.read(archive)
-        validate_identity!(inventory, package)
-        digest = Digest::SHA256.file(archive).hexdigest
-        destination = safe_path(File.join('widgets', inventory.project_filename))
-        cache = safe_path(cache_relative(inventory, package))
-        lock_path = safe_path(File.join('.mxrb', 'marketplace.lock.json'))
-        current = validate_boundaries!(package, destination, cache, lock_path)
-        install_transaction(
-          archive, package, inventory, digest, destination, cache, lock_path, current
-        )
+        Progress.with("Installing widget #{package.name}") do |progress|
+          validate_target!
+          progress.update(detail: 'reading widget package')
+          inventory = WidgetPackageInventory.read(archive)
+          validate_identity!(inventory, package)
+          digest = Digest::SHA256.file(archive).hexdigest
+          destination = safe_path(File.join('widgets', inventory.project_filename))
+          cache = safe_path(cache_relative(inventory, package))
+          lock_path = safe_path(File.join('.mxrb', 'marketplace.lock.json'))
+          current = validate_boundaries!(package, destination, cache, lock_path)
+          progress.update(detail: 'installing project assets')
+          install_transaction(
+            archive, package, inventory, digest, destination, cache, lock_path, current
+          )
+        end
       end
 
       private

data/lib/mxrb/official_marketplace.rb

@@ -223,14 +223,20 @@ module Mxrb
       end
 
       def json(url, authorization: default_authorization)
-        JSON.parse(get(url, accept: 'application/json', authorization:))
+        Progress.with('Loading Marketplace data') do |progress|
+          progress.update(detail: URI.parse(url).host)
+          JSON.parse(get(url, accept: 'application/json', authorization:))
+        end
       rescue JSON::ParserError => e
         raise MarketplaceError, "invalid JSON response: #{e.message}"
       end
 
       def download(url, destination, authorization: default_authorization)
-        File.binwrite(destination, get(url, accept: 'application/octet-stream', authorization:))
-        destination
+        Progress.with("Downloading #{File.basename(destination)}") do |progress|
+          progress.update(detail: URI.parse(url).host)
+          File.binwrite(destination, get(url, accept: 'application/octet-stream', authorization:))
+          destination
+        end
       end
 
       private

data/lib/mxrb/oql.rb

@@ -49,7 +49,7 @@ module Mxrb
 
       # Recursive BSON discovery is intentionally kept together so that its
       # source-type guard and ownership context cannot diverge.
-      # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+      # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength
       def discover(document, raw, module_name)
         found = []
         walk(document) do |node, path, ancestors|
@@ -59,6 +59,11 @@ module Mxrb
               node['Query'], raw, module_name, path + ['Query'], source_type, ancestors
             )
           end
+          if source_type == 'DomainModels$ViewEntitySourceDocument' && node['Oql'].is_a?(String)
+            found << build_query(
+              node['Oql'], raw, module_name, path + ['Oql'], source_type, ancestors
+            )
+          end
           node.each do |key, value|
             next unless key.to_s.match?(/\AOqlQuery\z/i) && value.is_a?(String)
 
@@ -67,7 +72,7 @@ module Mxrb
         end
         found.uniq { [_1.unit_id, _1.path] }
       end
-      # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
+      # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength
 
       # rubocop:disable Metrics/CyclomaticComplexity, Metrics/MethodLength
       def walk(node, path = [], ancestors = [], &block)
@@ -97,7 +102,8 @@ module Mxrb
         unit_type = @project.parse_bson(raw)['$Type'].to_s
         kind = if unit_type == 'DataSets$DataSet'
                  :dataset
-               elsif owner['$Type'].to_s.match?(/Entity/i)
+               elsif unit_type == 'DomainModels$ViewEntitySourceDocument' ||
+                     owner['$Type'].to_s.match?(/Entity/i)
                  :view_entity
                else
                  :oql

data/lib/mxrb/progress.rb

@@ -0,0 +1,242 @@
+# frozen_string_literal: true
+
+require 'io/console'
+require 'singleton'
+
+module Mxrb
+  # Terminal progress rendering shared by every long-running MXRB operation.
+  # It is enabled automatically only for an interactive terminal, writes to
+  # stderr, and therefore never contaminates command output or JSON on stdout.
+  module Progress
+    THREAD_KEY = :mxrb_progress_task
+    FALSE_VALUES = %w[0 false no off].freeze
+
+    # No-op object returned when progress rendering is disabled.
+    class NullTask
+      include Singleton
+
+      def start = self
+      def update(**) = self
+      def advance(*, **) = self
+      def add_total(*) = self
+      def finish(*) = self
+      def fail(*) = self
+      def enabled? = false
+    end
+
+    # Thread-safe terminal renderer for a single operation.
+    class Task # rubocop:disable Metrics/ClassLength
+      SPINNER = %w[| / - \\].freeze
+      BAR_WIDTH = 28
+      REFRESH_INTERVAL = 0.08
+
+      attr_reader :label, :total, :current
+
+      def initialize(label, total: nil, io: $stderr)
+        @label = label.to_s
+        @total = normalize_total(total)
+        @io = io
+        @current = 0
+        @detail = nil
+        @started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        @last_rendered_at = 0.0
+        @spinner_index = 0
+        @mutex = Mutex.new
+        @finished = false
+      end
+
+      def enabled? = true
+
+      def start
+        render(force: true)
+        start_spinner unless determinate?
+        self
+      end
+
+      def update(current: nil, total: nil, detail: nil, force: false)
+        became_determinate = false
+        @mutex.synchronize do
+          became_determinate = @total.nil? && !total.nil?
+          @total = normalize_total(total) unless total.nil?
+          @current = [[Integer(current), 0].max, @total || Float::INFINITY].min unless current.nil?
+          @detail = detail.to_s unless detail.nil?
+        end
+        stop_spinner if became_determinate
+        render(force:)
+        self
+      end
+
+      def advance(amount = 1, detail: nil, force: false)
+        @mutex.synchronize do
+          @current += amount
+          @current = [@current, @total].min if @total
+          @detail = detail.to_s unless detail.nil?
+        end
+        render(force:)
+        self
+      end
+
+      def add_total(amount)
+        @mutex.synchronize { @total = (@total || 0) + Integer(amount) }
+        render(force: true)
+        self
+      end
+
+      def finish(detail = nil)
+        stop_spinner
+        @mutex.synchronize do
+          return self if @finished
+
+          @detail = detail.to_s if detail
+          @current = @total if @total
+          @finished = true
+        end
+        render(force: true, final: true)
+        self
+      end
+
+      def fail(message = nil)
+        stop_spinner
+        @mutex.synchronize do
+          return self if @finished
+
+          @detail = message.to_s unless message.to_s.empty?
+          @finished = true
+        end
+        render(force: true, final: true, failed: true)
+        self
+      end
+
+      private
+
+      def normalize_total(value)
+        return nil if value.nil?
+
+        [Integer(value), 1].max
+      end
+
+      def determinate? = !@total.nil?
+
+      def start_spinner
+        return unless dynamic_terminal?
+
+        @spinner_thread = Thread.new do
+          loop do
+            sleep REFRESH_INTERVAL
+            break if @mutex.synchronize { @finished }
+
+            render(force: true)
+          end
+        end
+      end
+
+      def stop_spinner
+        thread = @spinner_thread
+        return unless thread
+
+        @mutex.synchronize { @finished = true }
+        thread.join
+        @spinner_thread = nil
+        @mutex.synchronize { @finished = false }
+      end
+
+      def render(force: false, final: false, failed: false) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+        line = nil
+        @mutex.synchronize do
+          now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          return if !force && !final && now - @last_rendered_at < REFRESH_INTERVAL
+
+          @last_rendered_at = now
+          line = rendered_line(now, failed:)
+        end
+        prefix = dynamic_terminal? ? "\r\e[2K" : ''
+        suffix = final || !dynamic_terminal? ? "\n" : ''
+        @io.write("#{prefix}#{truncate(line)}#{suffix}")
+        @io.flush if @io.respond_to?(:flush)
+      rescue IOError, Errno::EPIPE
+        nil
+      end
+
+      def rendered_line(now, failed:)
+        detail = @detail.to_s.empty? ? '' : " - #{@detail}"
+        elapsed = format('%.1fs', now - @started_at)
+        return "[mxrb] [FAILED] #{@label}#{detail} (#{elapsed})" if failed
+
+        determinate_line(detail, elapsed) || spinner_line(detail, elapsed)
+      end
+
+      def determinate_line(detail, elapsed)
+        return unless determinate?
+
+        ratio = [@current.fdiv(@total), 1.0].min
+        filled = (ratio * BAR_WIDTH).round
+        bar = '#' * filled + '-' * (BAR_WIDTH - filled)
+        "[mxrb] [#{bar}] #{format('%3d%%', ratio * 100)} #{@label}#{detail} (#{elapsed})"
+      end
+
+      def spinner_line(detail, elapsed)
+        frame = SPINNER[@spinner_index % SPINNER.length]
+        @spinner_index += 1
+        "[mxrb] [#{frame}] #{@label}#{detail} (#{elapsed})"
+      end
+
+      def dynamic_terminal?
+        @io.respond_to?(:tty?) && @io.tty?
+      end
+
+      def truncate(line)
+        width = @io.respond_to?(:winsize) ? @io.winsize.last : 100
+        width = 100 unless width.to_i.positive?
+        return line if line.length <= width
+
+        "#{line[0, width - 3]}..."
+      rescue IOError, Errno::ENOTTY
+        line
+      end
+    end # rubocop:enable Metrics/ClassLength
+
+    class << self
+      attr_writer :io
+
+      def configure(enabled: nil, io: nil)
+        @enabled = enabled unless enabled.nil?
+        @io = io if io
+      end
+
+      def reset!
+        @enabled = nil
+        @io = nil
+      end
+
+      def enabled?
+        return @enabled unless @enabled.nil?
+        return false if FALSE_VALUES.include?(ENV.fetch('MXRB_PROGRESS', '').downcase)
+
+        output.respond_to?(:tty?) && output.tty?
+      end
+
+      def with(label, total: nil) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+        active = Thread.current[THREAD_KEY]
+        return yield(active) if active
+        return yield(NullTask.instance) unless enabled?
+
+        task = Task.new(label, total:, io: output).start
+        Thread.current[THREAD_KEY] = task
+        result = yield(task)
+        task.finish
+        result
+      rescue StandardError => e
+        task&.fail(e.message)
+        raise
+      ensure
+        Thread.current[THREAD_KEY] = nil if defined?(task) && task
+      end
+
+      def current = Thread.current[THREAD_KEY] || NullTask.instance
+
+      private
+
+      def output = @io || $stderr
+    end
+  end
+end

data/lib/mxrb/runtime/clipboard.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require 'tempfile'
+
+module Mxrb
+  module Runtime
+    # Copies secrets through stdin so they never appear in the process list.
+    class Clipboard
+      COMMANDS = [
+        ['wl-copy'], ['pbcopy'], ['xclip', '-selection', 'clipboard'],
+        ['xsel', '--clipboard', '--input'], ['clip.exe']
+      ].freeze
+
+      def initialize(path: ENV.fetch('PATH', ''), runner: nil)
+        @path = path
+        @runner = runner || method(:capture)
+      end
+
+      def copy(text)
+        command = available_command
+        unless command
+          raise ToolchainError,
+                'no clipboard command found; install wl-copy, pbcopy, xclip, xsel, or clip.exe'
+        end
+
+        _stdout, stderr, status = @runner.call(command, text.to_s)
+        raise ToolchainError, "clipboard command failed: #{stderr}" unless status.success?
+
+        command.first
+      end
+
+      private
+
+      def available_command
+        COMMANDS.find { executable?(_1.first) }
+      end
+
+      def executable?(name)
+        @path.split(File::PATH_SEPARATOR).any? do |directory|
+          file = File.join(directory, name)
+          File.file?(file) && File.executable?(file)
+        end
+      end
+
+      def capture(command, input)
+        Tempfile.create('mxrb-clipboard') { capture_process(command, input, _1) }
+      end
+
+      def capture_process(command, input, stderr)
+        ::IO.pipe do |reader, writer|
+          pid = Process.spawn({ 'PATH' => @path }, *command, in: reader, out: File::NULL, err: stderr)
+          reader.close
+          writer.write(input)
+          writer.close
+          _child, status = Process.wait2(pid)
+          stderr.rewind
+          ['', stderr.read, status]
+        end
+      end
+    end
+  end
+end

data/lib/mxrb/runtime/database_workspace.rb

@@ -14,6 +14,7 @@ module Mxrb
       :host, :port, :database, :reader_user, :running, :model_fingerprint,
       :runtime_url
     )
+    AdminCredentials = Data.define(:username, :password, :state_dir, :path)
 
     # Materializes a private PostgreSQL database for one MPR. The Mendix Runtime
     # owns schema synchronization; analyst access uses a separate read-only role.
@@ -148,8 +149,35 @@ module Mxrb
         "postgresql://#{READER}:#{secret}@127.0.0.1:#{@port}/#{DATABASE}"
       end
 
+      def admin_credentials
+        unless File.file?(credentials_path)
+          raise ToolchainError,
+                "Runtime credentials not found; run `mxrb db up #{@project_path}` first"
+        end
+
+        values = JSON.parse(File.read(credentials_path))
+        password = values.fetch('admin_password')
+        AdminCredentials.new(admin_user_name, password, @state_dir, credentials_path)
+      rescue JSON::ParserError, KeyError => e
+        raise ToolchainError, "invalid Runtime credentials file #{credentials_path}: #{e.message}"
+      end
+
       private
 
+      def admin_user_name
+        mpr = IO::MprFile.open(@project_path, readonly: true)
+        begin
+          security = mpr.all_units.lazy.map { mpr.parse_contents(_1) }
+                        .find { _1['$Type'] == 'Security$ProjectSecurity' }
+          name = security.to_h.fetch('AdminUserName', '').to_s
+          raise ToolchainError, 'MPR has no configured administrator user' if name.empty?
+
+          name
+        ensure
+          mpr.close
+        end
+      end
+
       def bind_parameters(statement, params)
         values = params.to_h.transform_keys(&:to_s)
         return [statement, []] if values.empty?

data/lib/mxrb/scaffold/cli.rb

@@ -31,9 +31,12 @@ module Mxrb
       def generator
         chain = extract_option('--chain') if @command == 'page'
         template = extract_option('--template') if @command == 'page'
+        entity = extract_option('--entity') if @command == 'demo-user'
+        roles = extract_options('--role') if @command == 'demo-user'
         Generator.new(
           kind, scaffold_name, target: target, dry_run: @dry_run,
-                               page_chain: chain, page_template: template
+                               page_chain: chain, page_template: template,
+                               demo_entity: entity, demo_roles: roles
         )
       end
 
@@ -56,6 +59,12 @@ module Mxrb
       end
 
       def validate_action!
+        if @command == 'demo-user'
+          @argv.shift if @argv.first == 'new'
+          abort Help.text(@command) if @argv.empty?
+          return
+        end
+
         action = @argv.shift
         abort Help.text(@command) unless accepted_actions.include?(action)
       end
@@ -90,6 +99,16 @@ module Mxrb
         @argv.slice!(index, 2).last
       end
 
+      def extract_options(name)
+        values = []
+        while (index = @argv.index(name))
+          abort "#{name} requires a value" unless @argv[index + 1]
+
+          values << @argv.slice!(index, 2).last
+        end
+        values
+      end
+
       def kind = @command.tr('-', '_').to_sym
 
       def render(result) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength