markdown_exec 2.8.5 → 3.0.1

data/lib/hash_delegator.rb

@@ -22,8 +22,10 @@ require_relative 'array'
 require_relative 'array_util'
 require_relative 'block_types'
 require_relative 'cached_nested_file_reader'
+require_relative 'command_result'
 require_relative 'constants'
 require_relative 'directory_searcher'
+require_relative 'error_reporting'
 require_relative 'evaluate_shell_expressions'
 require_relative 'exceptions'
 require_relative 'fcb'
@@ -41,9 +43,11 @@ require_relative 'streams_out'
 require_relative 'string_util'
 require_relative 'table_extractor'
 require_relative 'text_analyzer'
+require_relative 'value_or_exception'
 
 $pd = false unless defined?($pd)
 $table_cell_truncate = true
+EXIT_STATUS_REQUIRED_EMPTY = 248
 
 module HashDelegatorSelf
   # Applies an ANSI color method to a string using a specified color key.
@@ -138,7 +142,8 @@ module HashDelegatorSelf
   def delete_consecutive_blank_lines!(blocks_menu)
     blocks_menu.process_and_conditionally_delete! do
       |prev_item, current_item, _next_item|
-      prev_item&.fetch(:chrome, nil) &&
+      !current_item.is_split? &&
+        prev_item&.fetch(:chrome, nil) &&
         !(prev_item && prev_item.oname.present?) &&
         current_item&.fetch(:chrome, nil) &&
         !(current_item && current_item.oname.present?)
@@ -395,9 +400,19 @@ module HashDelegatorSelf
   end
 
   def persist_fcb_self(all_fcbs, options)
-    fcb = MarkdownExec::FCB.new(options)
-    all_fcbs << fcb if all_fcbs
-    fcb
+    raise if all_fcbs.nil?
+
+    # if the id is present, update the existing fcb
+    if options[:id]
+      fcb = all_fcbs.find { |fcb| fcb.id == options[:id] }
+      if fcb
+        fcb.update(options)
+        return fcb
+      end
+    end
+    MarkdownExec::FCB.new(options).tap do |fcb|
+      all_fcbs << fcb
+    end
   end
 end
 
@@ -861,30 +876,105 @@ module MarkdownExec
         when :blocks
           result = SuccessResult.instance
           if @delegate_object[:bash]
-            # prepare block for menu, may fail and call HashDelegator.error_handler
-            result = fcb.for_menu!(
-              block_calls_scan: @delegate_object[:block_calls_scan],
-              block_name_match: @delegate_object[:block_name_match],
-              block_name_nick_match: @delegate_object[:block_name_nick_match],
-              id: "#{source_id}_bfnf_b_#{count}",
-              menu_format: @delegate_object[:menu_ux_row_format],
-              prompt: @delegate_object[:prompt_ux_enter_a_value],
-              table_center: @delegate_object[:table_center]
-            ) do |oname, color|
-              apply_block_type_color_option(oname, color)
+            begin
+              mf = MenuFilter.new(@delegate_object)
+              if fcb.body.count > 1 && mf.fcb_in_menu?(fcb) && fcb.is_split_displayed?(@delegate_object)
+                # make multiple FCB blocks, one for each line; only the first is active
+                id_prefix = "#{fcb.id}¤BlkFrmNstFls®block:#{count}©body:"
+                fcb0 = fcb
+                menu_lines = fcb.body
+                menu_lines.each.with_index do |menu_line, index|
+                  is_enabled_but_inactive = ((index + 1) % (@delegate_object[:select_page_height] / 2)).zero?
+                  if index.zero?
+                    # fcb.body = [menu_line]
+                    # fcb.center = center
+                    # fcb.collapse = collapse.nil? ? (line_obj[:collapse] == COLLAPSIBLE_TOKEN_COLLAPSE) : collapse
+                    # fcb.disabled = disabled ? TtyMenu::DISABLE : nil
+                    fcb.dname = fcb.indent + menu_line
+                    fcb.id = "#{id_prefix}#{index}"
+                    # fcb.indent = line_obj[:indent]
+                    fcb.is_split_first = true # the first block in a split
+                    fcb.is_split_rest = false
+                    # fcb.level = level
+                    # fcb.oname # computed
+                    # fcb.s0indent = indent
+                    fcb.s0printable = menu_line
+                    fcb.s1decorated = menu_line
+                    fcb.text = menu_line
+                    # fcb.token = line_obj[:collapse]
+                    # fcb.type = type
+                  else
+                    fcb = persist_fcb(
+                      body: fcb0.body,
+                      center: fcb0.center,
+                      chrome: true,
+                      collapse: false,
+                      disabled: is_enabled_but_inactive ? TtyMenu::ENABLE : TtyMenu::DISABLE,
+                      dname: fcb0.indent + menu_line,
+                      id: "#{id_prefix}#{index}",
+                      indent: fcb0.indent,
+                      is_enabled_but_inactive: is_enabled_but_inactive,
+                      is_split_first: false,
+                      is_split_rest: true, # subsequent blocks in a split
+                      level: fcb0.level,
+                      s0indent: fcb0.s0indent,
+                      s0printable: menu_line,
+                      s1decorated: menu_line,
+                      start_line: fcb0.start_line,
+                      text: menu_line,
+                      # token: ,
+                      type: fcb0.type
+                    )
+                  end
+
+                  result = fcb.for_menu!(
+                    block_calls_scan: @delegate_object[:block_calls_scan],
+                    block_name_match: @delegate_object[:block_name_match],
+                    block_name_nick_match: @delegate_object[:block_name_nick_match],
+                    id: fcb.id,
+                    menu_format: @delegate_object[:menu_ux_row_format],
+                    prompt: @delegate_object[:prompt_ux_enter_a_value],
+                    table_center: @delegate_object[:table_center]
+                  ) do |oname, color|
+                    apply_block_type_color_option(oname, color)
+                  end
+
+                  results[fcb.id] = result if result.failure?
+                  blocks << fcb unless result.failure?
+                end
+              else
+                # prepare block for menu, may fail and call HashDelegator.error_handler
+                result = fcb.for_menu!(
+                  block_calls_scan: @delegate_object[:block_calls_scan],
+                  block_name_match: @delegate_object[:block_name_match],
+                  block_name_nick_match: @delegate_object[:block_name_nick_match],
+                  id: fcb.id,
+                  menu_format: @delegate_object[:menu_ux_row_format],
+                  prompt: @delegate_object[:prompt_ux_enter_a_value],
+                  table_center: @delegate_object[:table_center]
+                ) do |oname, color|
+                  # decorate the displayed line
+                  apply_block_type_color_option(oname, color)
+                end
+                results[fcb.id] = result if result.failure?
+                blocks << fcb unless result.failure?
+              end
+            rescue StandardError
+              # ww $@, $!
+              HashDelegator.error_handler('blocks_from_nested_files',
+                                          { abort: true })
             end
-            results[fcb.id] = result if result.failure?
           else
             expand_references!(fcb, link_state)
+            blocks << fcb unless result.failure?
           end
-          blocks << fcb unless result.failure?
         when :filter # types accepted
           %i[blocks line]
         when :line
           unless @delegate_object[:no_chrome]
             # expand references only if block is recognized (not a comment)
             create_and_add_chrome_blocks(
-              blocks, fcb, id: "#{source_id}_bfnf_l_#{count}", init_ids: init_ids
+              blocks, fcb, id: "#{source_id}¤BlkFrmNstFls:#{count}®line", init_ids: init_ids
             ) do
               # expand references only if block is recognized (not a comment)
               expand_references!(fcb, link_state)
@@ -894,7 +984,7 @@ module MarkdownExec
       end
       OpenStruct.new(blocks: blocks, results: results)
     rescue StandardError
-      # ww $@, $!,
+      # ww $@, $!
       HashDelegator.error_handler('blocks_from_nested_files')
     end
 
@@ -907,13 +997,14 @@ module MarkdownExec
 
     def build_replacement_dictionary(
       commands, link_state,
-      initial_code_required: false, key_format:
+      initial_code_required: false,
+      occurrence_expressions: nil
     )
       evaluate_shell_expressions(
         (link_state&.inherited_lines_block || ''),
         commands,
         initial_code_required: initial_code_required,
-        key_format: key_format
+        occurrence_expressions: occurrence_expressions
       )
     end
 
@@ -987,21 +1078,6 @@ module MarkdownExec
       ]
     end
 
-    # Executes the allowed exec command and processes the output
-    # @param export [Object] The export configuration object
-    # @param inherited_code [Array] The inherited code lines
-    # @param code_lines [Array] The code lines to append to
-    # @param required [Hash] Required code information
-    # @return [String, Symbol] The command output or :ux_exec_prohibited if execution failed
-    def process_allowed_exec(export, inherited_code, code_lines, required)
-      output = export_exec_with_code(
-        export, inherited_code, code_lines, required
-      )
-      return :ux_exec_prohibited if output == :invalidated
-
-      output.split("\n")
-    end
-
     def code_from_automatic_ux_blocks(
       all_blocks,
       mdoc
@@ -1010,25 +1086,25 @@ module MarkdownExec
         return
       end
 
-      blocks = select_automatic_ux_blocks(all_blocks)
-      if blocks.empty?
-        blocks = select_automatic_ux_blocks(all_blocks)
-      end
+      blocks = select_automatic_ux_blocks(
+        all_blocks.reject(&:is_split_rest?)
+      )
       return if blocks.empty?
 
       @ux_most_recent_filename = @delegate_object[:filename]
 
       (blocks.each.with_object([]) do |block, merged_options|
-        code = code_from_ux_block_to_set_environment_variables(
+        command_result_w_e_t_nl = code_from_ux_block_to_set_environment_variables(
           block,
           mdoc,
           force: @delegate_object[:ux_auto_load_force_default],
-          only_default: true
+          only_default: true,
+          silent: true
         )
-        if code == :ux_exec_prohibited
+        if command_result_w_e_t_nl.failure?
           merged_options
         else
-          merged_options.push(code)
+          merged_options.push(command_result_w_e_t_nl.stdout)
         end
       end).to_a
     end
@@ -1036,7 +1112,8 @@ module MarkdownExec
     # parse YAML body defining the UX for a single variable
     # set ENV value for the variable and return code lines for the same
     def code_from_ux_block_to_set_environment_variables(
-      selected, mdoc, inherited_code: nil, force: true, only_default: false
+      selected, mdoc, inherited_code: nil, force: true, only_default: false,
+      silent:
     )
       exit_prompt = @delegate_object[:prompt_filespec_back]
 
@@ -1048,7 +1125,7 @@ module MarkdownExec
       )
 
       # process each ux block in sequence, setting ENV and collecting lines
-      code_lines = []
+      required_lines = []
       required[:blocks].each do |block|
         next unless block.type == BlockType::UX
 
@@ -1059,35 +1136,51 @@ module MarkdownExec
             prompt: @delegate_object[:prompt_ux_enter_a_value],
             validate: '^(?<name>[^ ].*)$'
           )
+          block.export = export
+          block.export_act = FCB.act_source(export)
+          block.export_init = FCB.init_source(export)
 
-          # preconditions are variable names that must be set before the UX block is executed.
+          # required are variable names that must be set before the UX block is executed.
           # if any precondition is not set, the sequence is aborted.
-          export.preconditions&.each do |precondition|
-            code_lines.push "[[ -z $#{precondition} ]] && exit 1"
+          required_variables = []
+          export.required&.each do |precondition|
+            required_variables.push "[[ -z $#{precondition} ]] && exit #{EXIT_STATUS_REQUIRED_EMPTY}"
           end
 
+          eval_code = join_array_of_arrays(
+            inherited_code, # inherited code
+            required_lines, # current block requirements
+            required_variables, # test conditions
+            required[:code] # current block code
+          )
           if only_default
-            value, exportable, transformable =
-              ux_block_export_automatic(
-                export, inherited_code, code_lines, required
-              )
+            command_result_w_e_t_nl =
+              ux_block_export_automatic(eval_code, export)
+            # do not display warnings on initializing call
+            return command_result_w_e_t_nl if command_result_w_e_t_nl.failure?
+
           else
-            value, exportable, transformable =
-              ux_block_export_activated(
-                export, inherited_code, code_lines, required, exit_prompt
-              )
+            command_result_w_e_t_nl =
+              ux_block_export_activated(eval_code, export, exit_prompt)
+            if command_result_w_e_t_nl.failure?
+              warn command_result_w_e_t_nl.warning if command_result_w_e_t_nl.warning&.present? && !silent
+              return command_result_w_e_t_nl
+            end
           end
-          return :ux_exec_prohibited if value == :ux_exec_prohibited
+          return command_result_w_e_t_nl if command_result_w_e_t_nl.failure?
 
-          if SelectResponse.continue?(value)
-            if transformable
-              value = transform_export_value(value, export)
+          required_lines.concat(command_result_w_e_t_nl.new_lines)
+          if SelectResponse.continue?(command_result_w_e_t_nl.stdout)
+            if command_result_w_e_t_nl.transformable
+              command_result_w_e_t_nl.stdout = transform_export_value(
+                command_result_w_e_t_nl.stdout, export
+              )
             end
 
-            if exportable
-              ENV[export.name] = value.to_s
-              code_lines.push code_line_safe_assign(export.name, value,
-                                                    force: force)
+            if command_result_w_e_t_nl.exportable
+              ENV[export.name] = command_result_w_e_t_nl.stdout.to_s
+              required_lines.push code_line_safe_assign(export.name, command_result_w_e_t_nl.stdout,
+                                                        force: force)
             end
           end
         else
@@ -1095,7 +1188,7 @@ module MarkdownExec
         end
       end
 
-      code_lines
+      CommandResult.new(stdout: required_lines)
     end
 
     # sets ENV
@@ -1331,7 +1424,9 @@ module MarkdownExec
     )
       # Initialize a counter for named group occurrences
       occurrence_count = Hash.new(0)
-      return occurrence_count if pattern.nil? || pattern == //
+      occurrence_expressions = {}
+      return [occurrence_count,
+              occurrence_expressions] if pattern.nil? || pattern == //
 
       blocks.each do |block|
         # Skip processing for shell-type blocks
@@ -1340,11 +1435,13 @@ module MarkdownExec
         # Scan each block name for matches of the pattern
         count_named_group_occurrences_block_body_fix_indent(block).scan(pattern) do |(_, _variable_name)|
           pattern.match($LAST_MATCH_INFO.to_s) # Reapply match for named groups
-          occurrence_count[$LAST_MATCH_INFO[group_name]] += 1
+          id = $LAST_MATCH_INFO[group_name]
+          occurrence_count[id] += 1
+          occurrence_expressions[id] = $LAST_MATCH_INFO['expression']
         end
       end
 
-      occurrence_count
+      [occurrence_count, occurrence_expressions]
     end
 
     def count_named_group_occurrences_block_body_fix_indent(block)
@@ -1459,17 +1556,17 @@ module MarkdownExec
           fcb.center = center
           fcb.chrome = true
           fcb.collapse = collapse.nil? ? (line_obj[:collapse] == COLLAPSIBLE_TOKEN_COLLAPSE) : collapse
-          fcb.token = line_obj[:collapse]
           fcb.disabled = disabled ? TtyMenu::DISABLE : nil
+          fcb.dname = line_obj[:indent] + decorated
           fcb.id = "#{id}.#{index}"
+          fcb.indent = line_obj[:indent]
           fcb.level = level
+          fcb.oname = line_obj[:text]
           fcb.s0indent = indent
           fcb.s0printable = line_obj[:text]
           fcb.s1decorated = decorated
-          fcb.dname = line_obj[:indent] + decorated
-          fcb.indent = line_obj[:indent]
-          fcb.oname = line_obj[:text]
           fcb.text = line_obj[:text]
+          fcb.token = line_obj[:collapse]
           fcb.type = type
           use_fcb = false # next line is new record
         else
@@ -1477,17 +1574,17 @@ module MarkdownExec
             center: center,
             chrome: true,
             collapse: collapse.nil? ? (line_obj[:collapse] == COLLAPSIBLE_TOKEN_COLLAPSE) : collapse,
-            token: line_obj[:collapse],
             disabled: disabled ? TtyMenu::DISABLE : nil,
+            dname: line_obj[:indent] + decorated,
             id: "#{id}.#{index}",
+            indent: line_obj[:indent],
             level: level,
+            oname: line_obj[:text],
             s0indent: indent,
             s0printable: line_obj[:text],
             s1decorated: decorated,
-            dname: line_obj[:indent] + decorated,
-            indent: line_obj[:indent],
-            oname: line_obj[:text],
             text: line_obj[:text],
+            token: line_obj[:collapse],
             type: type
           )
         end
@@ -1921,14 +2018,17 @@ module MarkdownExec
 
       elsif selected.type == BlockType::UX
         debounce_reset
+        command_result_w_e_t_nl = code_from_ux_block_to_set_environment_variables(
+          selected,
+          @dml_mdoc,
+          inherited_code: @dml_link_state.inherited_lines,
+          silent: true
+        )
+        ### TBD if command_result_w_e_t_nl.failure?
         next_state_append_code(
           selected,
           link_state,
-          code_from_ux_block_to_set_environment_variables(
-            selected,
-            @dml_mdoc,
-            inherited_code: @dml_link_state.inherited_lines
-          )
+          command_result_w_e_t_nl.failure? ? [] : command_result_w_e_t_nl.stdout
         )
 
       elsif selected.type == BlockType::VARS
@@ -1977,6 +2077,11 @@ module MarkdownExec
       @dml_block_state = find_block_state_by_name(block_name)
       dump_and_warn_block_state(name: block_name,
                                 selected: @dml_block_state.block)
+      if @dml_block_state.block.fetch(:is_enabled_but_inactive, false)
+        @dml_block_selection = BlockSelection.new(@dml_block_state.block.id)
+        return # do nothing
+      end
+
       next_block_state =
         execute_block_for_state_and_name(
           selected: @dml_block_state.block,
@@ -2150,7 +2255,7 @@ module MarkdownExec
         end
       elsif (selected_option = select_option_with_metadata(
         prompt_title,
-        [exit_prompt] + dirs.map do |file|
+        [exit_prompt] + dirs.map do |file| # tty_menu_items
           { name:
               format(
                 block_data['view'] || view,
@@ -2163,7 +2268,8 @@ module MarkdownExec
             oname: file }
         end,
         menu_options.merge(
-          cycle: true
+          cycle: true,
+          match_dml: false
         )
       ))
         if selected_option.dname != exit_prompt
@@ -2339,9 +2445,9 @@ module MarkdownExec
     def execute_inherited_save(
       code_lines: @dml_link_state.inherited_lines
     )
-      return unless (save_filespec = save_filespec_from_expression)
-
-      document_name_in_glob_as_file_name
+      return unless (save_filespec = save_filespec_from_expression(
+        document_name_in_glob_as_file_name
+      ))
 
       unless write_file_with_directory_creation(
         content: HashDelegator.join_code_lines(code_lines),
@@ -2397,27 +2503,6 @@ module MarkdownExec
       post_execution_process
     end
 
-    def execute_temporary_script(script_code, additional_code = [])
-      full_code = (additional_code || []) + [script_code]
-
-      Tempfile.create('script_exec') do |temp_file|
-        temp_file.write(HashDelegator.join_code_lines(full_code))
-        temp_file.flush
-        File.chmod(0o755, temp_file.path)
-
-        output = `#{temp_file.path}`
-
-        if $?.exitstatus != 0
-          return :invalidated
-        end
-
-        output
-      end
-    rescue StandardError => err
-      warn "Error executing script: #{err.message}"
-      nil
-    end
-
     def expand_blocks_with_replacements(
       menu_blocks, replacements, exclude_types: [BlockType::SHELL]
     )
@@ -2438,17 +2523,32 @@ module MarkdownExec
     def expand_references!(fcb, link_state)
       expand_variable_references!(
         blocks: [fcb],
+        echo_formatter: method(:format_echo_command),
+        group_name: :payload,
         initial_code_required: false,
-        key_format: @delegate_object[:variable_expression_format],
         link_state: link_state,
-        pattern: options_variable_expression_regexp
+        pattern: @delegate_object[:option_expansion_expression_regexp].present? &&
+          Regexp.new(@delegate_object[:option_expansion_expression_regexp])
       )
+
+      # variable expansions
       expand_variable_references!(
         blocks: [fcb],
-        echo_format: '%s',
-        group_name: :command,
+        echo_formatter: lambda do |variable|
+          %(echo "$#{variable}")
+        end,
+        group_name: @delegate_object[:variable_expansion_name_capture_group]&.to_sym,
+        initial_code_required: false,
+        link_state: link_state,
+        pattern: options_variable_expansion_regexp
+      )
+
+      # command substitutions
+      expand_variable_references!(
+        blocks: [fcb],
+        echo_formatter: lambda { |command| command },
+        group_name: @delegate_object[:command_substitution_name_capture_group]&.to_sym,
         initial_code_required: false,
-        key_format: @delegate_object[:command_substitution_format],
         link_state: link_state,
         pattern: options_command_substitution_regexp
       )
@@ -2456,23 +2556,25 @@ module MarkdownExec
 
     def expand_variable_references!(
       blocks:,
-      echo_format: 'echo "$%s"',
-      group_name: :variable,
+      echo_formatter:,
+      group_name:,
       initial_code_required: false,
-      key_format:,
       link_state:,
       pattern:
     )
-      variable_counts = count_named_group_occurrences(blocks, pattern,
-                                                      group_name: group_name)
+      variable_counts, occurrence_expressions = count_named_group_occurrences(
+        blocks, pattern, group_name: group_name
+      )
       return if variable_counts.nil? || variable_counts == {}
 
-      echo_commands = generate_echo_commands(variable_counts, echo_format)
+      echo_commands = generate_echo_commands(
+        variable_counts, formatter: echo_formatter
+      )
 
       replacements = build_replacement_dictionary(
         echo_commands, link_state,
         initial_code_required: initial_code_required,
-        key_format: key_format
+        occurrence_expressions: occurrence_expressions
       )
 
       return if replacements.nil?
@@ -2481,50 +2583,48 @@ module MarkdownExec
       expand_blocks_with_replacements(blocks, replacements)
     end
 
-    def export_echo_with_code_single(export_echo, inherited_code, code_lines,
-                                     required)
-      code = %(printf '%s' "#{export_echo}")
-      value = execute_temporary_script(
-        code,
-        (inherited_code || []) +
-         code_lines + required[:code]
-      )
-      if value == :invalidated
-        warn "A value must exist for: #{export.preconditions.join(', ')}"
-      end
-      value
-    end
-
-    def export_echo_with_code(export, inherited_code, code_lines, required,
-                              force:)
+    def export_echo_with_code(
+      bash_script_lines, export, force:, silent:
+    )
       exportable = true
+      command_result = nil
+      new_lines = []
       case export.echo
       when String, Integer, Float, TrueClass, FalseClass
-        value = export_echo_with_code_single(export.echo.to_s, inherited_code,
-                                             code_lines, required)
+        command_result = output_from_adhoc_bash_script_file(
+          join_array_of_arrays(
+            bash_script_lines,
+            %(printf '%s' "#{export.echo}")
+          )
+        )
+        if command_result.exit_status == EXIT_STATUS_REQUIRED_EMPTY
+          exportable = false
+          command_result.warning = warning_required_empty(export) unless silent
+        end
+
       when Hash
         # each item in the hash is a variable name and value
         export.echo.each do |name, expression|
-          value = export_echo_with_code_single(expression, inherited_code,
-                                               code_lines, required)
-          ENV[name] = value.to_s
-          code_lines.push code_line_safe_assign(name, value, force: force)
+          command_result = output_from_adhoc_bash_script_file(
+            join_array_of_arrays(
+              bash_script_lines,
+              %(printf '%s' "#{expression}")
+            )
+          )
+          if command_result.exit_status == EXIT_STATUS_REQUIRED_EMPTY
+            command_result.warning = warning_required_empty(export) unless silent
+          else
+            ENV[name] = command_result.stdout.to_s
+            new_lines << code_line_safe_assign(name, command_result.stdout,
+                                               force: force)
+          end
         end
+
+        # individual items have been exported, none remain
         exportable = false
       end
-      [value, exportable]
-    end
 
-    def export_exec_with_code(export, inherited_code, code_lines, required)
-      value = execute_temporary_script(
-        export.exec,
-        (inherited_code || []) +
-         code_lines + required[:code]
-      )
-      if value == :invalidated
-        warn "A value must exist for: #{export.preconditions.join(', ')}"
-      end
-      value
+      [command_result, exportable, new_lines]
     end
 
     # Retrieves a specific data symbol from the delegate object,
@@ -2585,6 +2685,13 @@ module MarkdownExec
       )
     end
 
+    def find_option_by_name(name)
+      name_sym = name.to_sym
+      @menu_from_yaml.find do |option|
+        option[:opt_name] == name_sym
+      end
+    end
+
     def format_and_execute_command(
       code_lines:,
       erls:,
@@ -2604,6 +2711,24 @@ module MarkdownExec
                              color_sym: :script_execution_frame_color)
     end
 
+    def format_echo_command(payload)
+      payload_match = payload.match(@delegate_object[:option_expansion_payload_regexp])
+      variable = payload_match[:option]
+      property = payload_match[:property]
+
+      echo_value = case property
+                   when 'default', 'description'
+                     item = find_option_by_name(variable)
+                     item ? item[property.to_sym] : ''
+                   when 'length'
+                     @delegate_object[variable.to_sym].to_s.length
+                   else
+                     @delegate_object[variable.to_sym]
+                   end
+
+      "echo #{Shellwords.escape(echo_value)}"
+    end
+
     # Format expression using environment variables and run state
     def format_expression(expr)
       data = link_load_format_data
@@ -2661,13 +2786,12 @@ module MarkdownExec
                              color_sym: :execution_report_preview_frame_color)
     end
 
-    def generate_echo_commands(variable_counts, echo_format)
+    def generate_echo_commands(variable_counts, formatter: nil)
       # commands to echo variables
       #
       commands = {}
       variable_counts.each_key do |variable|
-        command = format(echo_format, variable)
-        commands[variable] = command
+        commands[variable] = formatter.call(variable)
       end
       commands
     end
@@ -2816,17 +2940,20 @@ module MarkdownExec
 
       state = initial_state
       selected_types = yield :filter
+      index = 0
       cfile.readlines(
         @delegate_object[:filename],
         import_paths: options_import_paths
-      ).each_with_index do |nested_line, index|
-        next unless nested_line
+      ) do |nested_line|
+        next if nested_line.nil?
 
         update_line_and_block_state(
           nested_line, state, selected_types,
-          source_id: "#{@delegate_object[:filename]}_ibfnf_#{index}",
+          source_id: "ItrBlkFrmNstFls:#{index}¤#{nested_line.filename}:#{nested_line.index}",
           &block
         )
+
+        index += 1
       end
     end
 
@@ -2841,15 +2968,21 @@ module MarkdownExec
       else
         iter_blocks_from_nested_files do |btype, fcb|
           case btype
-          when :blocks
-            yield fcb
-          when :filter
-            %i[blocks]
+          when :blocks; yield fcb
+          when :filter; %i[blocks]
           end
         end
       end
     end
 
+    # join a list of arrays into a single array
+    # convert single items to arrays
+    def join_array_of_arrays(*args)
+      args.map do |item|
+        item.is_a?(Array) ? item : [item]
+      end.compact.flatten(1)
+    end
+
     def link_block_data_eval(link_state, code_lines, selected, link_block_data,
                              block_source:, shell:)
       all_code = HashDelegator.code_merge(link_state&.inherited_lines,
@@ -2975,7 +3108,8 @@ module MarkdownExec
 
       list = []
       iter_source_blocks(
-        @delegate_object[:list_blocks_type], source_id: source_id
+        @delegate_object[:list_blocks_type],
+        source_id: source_id
       ) do |block|
         list << (block_eval.present? ? eval(block_eval) : block.send(message))
       end
@@ -3239,14 +3373,33 @@ module MarkdownExec
         source_id: source_id
       )
 
-      ### compress empty lines
       HashDelegator.delete_consecutive_blank_lines!(menu_blocks)
-      HashDelegator.tables_into_columns!(menu_blocks, @delegate_object,
-                                         screen_width_for_table)
+      begin
+        HashDelegator.tables_into_columns!(menu_blocks, @delegate_object,
+                                           screen_width_for_table)
+      rescue NoMethodError
+        # an invalid table format
+      end
+      handle_consecutive_inactive_items!(menu_blocks)
 
       [all_blocks, menu_blocks, mdoc]
     end
 
+    def handle_consecutive_inactive_items!(menu_blocks)
+      consecutive_inactive_count = 0
+      menu_blocks.each do |fcb|
+        unless fcb.is_disabled?
+          consecutive_inactive_count = 0
+        else
+          consecutive_inactive_count += 1
+          if (consecutive_inactive_count % (@delegate_object[:select_page_height] / 3)).zero?
+            fcb.disabled = TtyMenu::ENABLE
+            fcb.is_enabled_but_inactive = true
+          end
+        end
+      end
+    end
+
     def menu_add_disabled_option(document_glob)
       raise unless document_glob.present?
       raise if @dml_menu_blocks.nil?
@@ -3386,9 +3539,9 @@ module MarkdownExec
       @delegate_object[:import_paths]&.split(':') || ''
     end
 
-    def options_variable_expression_regexp
-      @delegate_object[:variable_expression_regexp].present? &&
-        Regexp.new(@delegate_object[:variable_expression_regexp])
+    def options_variable_expansion_regexp
+      @delegate_object[:variable_expansion_regexp].present? &&
+        Regexp.new(@delegate_object[:variable_expansion_regexp])
     end
 
     def output_color_formatted(data_sym, color_sym)
@@ -3413,6 +3566,21 @@ module MarkdownExec
       }
     end
 
+    def output_from_adhoc_bash_script_file(bash_script_lines)
+      Tempfile.create('script_exec') do |temp_file|
+        temp_file.write(HashDelegator.join_code_lines(bash_script_lines))
+        temp_file.flush
+        File.chmod(0o755, temp_file.path)
+
+        output = `#{temp_file.path}`
+
+        CommandResult.new(stdout: output, exit_status: $?.exitstatus)
+      end
+    rescue StandardError => err
+      warn "Error executing script: #{err.message}"
+      nil
+    end
+
     def output_labeled_value(label, value, level)
       @fout.lout format_references_send_color(
         context: {
@@ -3430,9 +3598,7 @@ module MarkdownExec
     end
 
     def persist_fcb(options)
-      MarkdownExec::FCB.new(options).tap do |fcb|
-        @fcb_store << fcb
-      end
+      HashDelegator.persist_fcb_self(@fcb_store, options)
     end
 
     def pop_add_current_code_to_head_and_trigger_load(
@@ -4097,7 +4263,7 @@ module MarkdownExec
     # Presents a TTY prompt to select an option or exit,
     #  returns metadata including option and selected
     def select_option_with_metadata(
-      prompt_text, menu_items, opts = {}, menu_blocks: nil
+      prompt_text, tty_menu_items, opts = {}, menu_blocks: nil
     )
       @dml_menu_blocks = menu_blocks if menu_blocks
 
@@ -4120,10 +4286,10 @@ module MarkdownExec
           per_page: @delegate_object[:select_page_height]
         }.freeze
 
-        if menu_items.all? do |item|
+        if tty_menu_items.all? do |item|
           !item.is_a?(String) && item[:disabled]
         end
-          menu_items.each do |prompt_item|
+          tty_menu_items.each do |prompt_item|
             puts prompt_item[:dname]
           end
           return
@@ -4133,19 +4299,21 @@ module MarkdownExec
         # crashes if default is not an existing item
         #
         selection = @prompt.select(prompt_text,
-                                   menu_items,
+                                   tty_menu_items,
                                    opts.merge(props))
       rescue TTY::Prompt::ConfigurationError
         # prompt fails when collapsible block name has changed; clear default
         selection = @prompt.select(prompt_text,
-                                   menu_items,
+                                   tty_menu_items,
                                    opts.merge(props).merge(default: nil))
       rescue NoMethodError
         # no enabled options in page
         return
       end
 
-      selected = @dml_menu_blocks.find do |item|
+      menu_list = opts.fetch(:match_dml, true) ? @dml_menu_blocks : menu_items
+      menu_list ||= tty_menu_items
+      selected = menu_list.find do |item|
         if item.instance_of?(Hash)
           [item[:id], item[:name], item[:dname]].include?(selection)
         elsif item.instance_of?(MarkdownExec::FCB)
@@ -4155,7 +4323,15 @@ module MarkdownExec
         end
       end
 
+      # new FCB if selected is not an object
+      if selected.instance_of?(String)
+        selected = FCB.new(dname: selected)
+      elsif selected.instance_of?(Hash)
+        selected = FCB.new(selected)
+      end
+
       unless selected
+        report_and_reraise('menu item not found')
         HashDelegator.error_handler('select_option_with_metadata',
                                     error: 'menu item not found')
         exit 1
@@ -4407,164 +4583,193 @@ module MarkdownExec
       @delegate_object.merge!(options)
     end
 
-    def ux_block_export_activated(export, inherited_code, code_lines, required,
-                                  exit_prompt)
+    def ux_block_export_activated(
+      bash_script_lines, export, exit_prompt
+    )
       exportable = true
       transformable = true
-      [if export.allowed.present?
-         if export.allowed == :echo
-           output, exportable = export_echo_with_code(
-             export, inherited_code, code_lines, required, force: true
-           )
-           return :ux_exec_prohibited if output == :invalidated
-
-           menu_from_list_with_back(output.split("\n"))
-
-         elsif export.allowed == :exec
-           output = process_allowed_exec(export, inherited_code,
-                                         code_lines, required)
-           return :ux_exec_prohibited if output == :ux_exec_prohibited
-
-           menu_from_list_with_back(output)
-
-         else
-           menu_from_list_with_back(export.allowed)
-         end
-
-       # echo > exec
-       elsif export.echo
-         output, exportable = export_echo_with_code(
-           export, inherited_code, code_lines, required, force: true
-         )
-         return :ux_exec_prohibited if output == :invalidated
-
-         output
-
-       # exec > allowed
-       elsif export.exec
-         output = export_exec_with_code(
-           export, inherited_code, code_lines, required
-         )
-         return :ux_exec_prohibited if output == :invalidated
-
-         output
-
-       # allowed > prompt
-       elsif export.allowed && export.allowed.count.positive?
-         case (choice = prompt_select_from_list(
-           [exit_prompt] + export.allowed,
-           string: export.prompt,
-           color_sym: :prompt_color_after_script_execution
-         ))
-         when exit_prompt
-           exportable = false
-           transformable = false
-           nil
-         else
-           choice
-         end
-
-       # prompt > default
-       elsif export.prompt.present?
-         begin
-           loop do
-             print "#{export.prompt} [#{export.default}]: "
-             output = gets.chomp
-             output = export.default.to_s if output.empty?
-             caps = NamedCaptureExtractor.extract_named_groups(output,
-                                                               export.validate)
-             break if caps
-
-             # invalid input, retry
-           end
-         rescue Interrupt
-           exportable = false
-           transformable = false
-         end
-
-         output
-
-       # default
-       else
-         transformable = false
-         export.default
-       end, exportable, transformable]
-    end
-
-    def ux_block_export_automatic(export, inherited_code, code_lines, required)
-      transformable = true
-      exportable = true
+      new_lines = []
+      command_result = nil
+
+      case as = FCB.act_source(export)
+      when false, UxActSource::FALSE
+        raise 'Should not be reached.'
+
+      when ':allow', UxActSource::ALLOW
+        raise unless export.allow.present?
+
+        case export.allow
+        when :echo, ExportValueSource::ECHO
+          command_result, exportable, new_lines = export_echo_with_code(
+            bash_script_lines,
+            export,
+            force: true,
+            silent: false
+          )
+          if command_result.failure?
+            command_result
+          else
+            command_result = CommandResult.new(
+              stdout: menu_from_list_with_back(command_result.stdout.split("\n"))
+            )
+          end
 
-      if export.default == false
-        exportable = false
-        transformable = false
-        value = nil
-      else
-        if export.default.nil?
-          if export.echo.present?
-            export.default = :echo
-          elsif export.exec.present?
-            export.default = :exec
-          elsif export.allowed.present?
-            export.default = export.allowed.first
+        when ':exec', UxActSource::EXEC
+          command_result = output_from_adhoc_bash_script_file(
+            join_array_of_arrays(bash_script_lines, export.exec)
+          )
+
+          if command_result.exit_status == EXIT_STATUS_REQUIRED_EMPTY
+            command_result
+          else
+            command_result = CommandResult.new(
+              stdout: menu_from_list_with_back(
+                command_result.stdout.split("\n")
+              )
+            )
           end
+
+        else
+          command_result = CommandResult.new(
+            stdout: menu_from_list_with_back(export.allow)
+          )
         end
 
-        value = case export.default
-                when :allowed
-                  raise unless export.allowed.present?
+      when ':echo', UxActSource::ECHO
+        command_result, exportable, new_lines = export_echo_with_code(
+          bash_script_lines,
+          export,
+          force: true,
+          silent: false
+        )
 
-                  if export.allowed == :echo
-                    output, exportable = export_echo_with_code(
-                      export, inherited_code, code_lines, required, force: false
-                    )
-                    return :ux_exec_prohibited if output == :invalidated
+        command_result
 
-                    exportable && output.split("\n").first
+      when ':edit', UxActSource::EDIT
+        output = nil
+        begin
+          loop do
+            print "#{export.prompt} [#{export.default}]: "
+            output = gets.chomp
+            output = export.default.to_s if output.empty?
+            caps = NamedCaptureExtractor.extract_named_groups(output,
+                                                              export.validate)
+            break if caps
 
-                  elsif export.allowed == :exec
-                    output = process_allowed_exec(export, inherited_code,
-                                                  code_lines, required)
-                    return :ux_exec_prohibited if output == :ux_exec_prohibited
+            # invalid input, retry
+          end
+        rescue Interrupt
+          exportable = false
+          transformable = false
+        end
 
-                    output.first
+        command_result = CommandResult.new(stdout: output)
 
-                  else
-                    export.allowed.first
-                  end
+      when ':exec', UxActSource::EXEC
+        command_result = output_from_adhoc_bash_script_file(
+          join_array_of_arrays(bash_script_lines, export.exec)
+        )
 
-                # echo > default
-                when :echo
-                  raise unless export.echo.present?
+        command_result
 
-                  output, exportable = export_echo_with_code(
-                    export, inherited_code, code_lines, required, force: false
-                  )
-                  return :ux_exec_prohibited if output == :invalidated
+      else
+        transformable = false
+        command_result = CommandResult.new(stdout: export.default.to_s)
+      end
 
-                  output
+      # add message for required variables
+      if command_result.exit_status == EXIT_STATUS_REQUIRED_EMPTY
+        command_result.warning = warning_required_empty(export)
+        # warn command_result.warning
+      end
 
-                # exec > default
-                when :exec
-                  raise unless export.exec.present?
+      command_result.exportable = exportable
+      command_result.transformable = transformable
+      command_result.new_lines = new_lines
+      command_result
+    end
 
-                  output = export_exec_with_code(
-                    export, inherited_code, code_lines, required
-                  )
-                  return :ux_exec_prohibited if output == :invalidated
+    def ux_block_export_automatic(bash_script_lines, export)
+      transformable = true
+      exportable = true
+      new_lines = []
+      command_result = nil
+      silent = true
 
-                  output
+      case FCB.init_source(export)
+      when false, UxActSource::FALSE
+        exportable = false
+        transformable = false
+        command_result = CommandResult.new
+
+      when ':allow', UxActSource::ALLOW
+        raise unless export.allow.present?
+
+        case export.allow
+        when :echo, ExportValueSource::ECHO
+          command_result, exportable, new_lines = export_echo_with_code(
+            bash_script_lines,
+            export,
+            force: false,
+            silent: silent
+          )
+          unless command_result.failure?
+            command_result.stdout = (exportable && command_result.stdout.split("\n").first) || ''
+          end
 
-                # default
-                else
-                  transformable = false
-                  export.default.to_s
-                end
+        when :exec, ExportValueSource::EXEC
+          command_result = output_from_adhoc_bash_script_file(
+            join_array_of_arrays(bash_script_lines, export.exec)
+          )
+          unless command_result.failure?
+            command_result.stdout = command_result.stdout.split("\n").first
+          end
+
+        else
+          # must be a list
+          command_result = CommandResult.new(stdout: export.allow.first)
+        end
+
+      when ':default', UxActSource::DEFAULT
+        transformable = false
+        command_result = CommandResult.new(stdout: export.default.to_s)
+
+      when ':echo', UxActSource::ECHO
+        raise unless export.echo.present?
+
+        command_result, exportable, new_lines = export_echo_with_code(
+          bash_script_lines,
+          export,
+          force: false,
+          silent: silent
+        )
+
+      when ':exec', UxActSource::EXEC
+        raise unless export.exec.present?
+
+        command_result = output_from_adhoc_bash_script_file(
+          join_array_of_arrays(bash_script_lines, export.exec)
+        )
+
+      else
+        command_result = CommandResult.new(stdout: export.init.to_s)
+        # raise "Unknown FCB.init_source(export) #{FCB.init_source(export)}"
       end
 
-      [value,
-       exportable,
-       transformable]
+      # add message for required variables
+      if command_result.exit_status == EXIT_STATUS_REQUIRED_EMPTY
+        command_result.warning = warning_required_empty(export)
+        warn command_result.warning unless silent
+      end
+
+      command_result.exportable = exportable
+      command_result.transformable = transformable
+      command_result.new_lines = new_lines
+      command_result
+    end
+
+    def warning_required_empty(export)
+      "A value must exist for: #{export.required.join(', ')}"
     end
 
     def vux_await_user_selection(prior_answer: @dml_block_selection)
@@ -4791,7 +4996,8 @@ module MarkdownExec
     #
     # @return [Nil] Returns nil if no code block is selected
     #  or an error occurs.
-    def vux_main_loop
+    def vux_main_loop(menu_from_yaml: nil)
+      @menu_from_yaml = menu_from_yaml
       vux_init
       vux_load_code_files_into_state
       formatted_choice_ostructs = vux_formatted_names_for_state_chrome_blocks
@@ -4841,10 +5047,10 @@ module MarkdownExec
         case msg
         when :parse_document # once for each menu
           count = 0
-          vux_parse_document(source_id: "#{@delegate_object[:filename]}_vmlpd")
+          vux_parse_document(source_id: "#{@delegate_object[:filename]}¤VuxMainLoop®PrsDoc")
           vux_menu_append_history_files(
             formatted_choice_ostructs,
-            source_id: "#{@delegate_object[:filename]}_vmlhf"
+            source_id: "#{@delegate_object[:filename]}¤VuxMainLoop®HstFls"
           )
           vux_publish_document_file_name_for_external_automation
 
@@ -4856,7 +5062,7 @@ module MarkdownExec
           # yield :end_of_cli, @delegate_object
 
           if @delegate_object[:list_blocks]
-            list_blocks(source_id: "#{@delegate_object[:filename]}_vmleoc")
+            list_blocks(source_id: "#{@delegate_object[:filename]}¤VuxMainLoop®EndCLI")
             :exit
           end
 
@@ -5001,6 +5207,7 @@ module MarkdownExec
         mdoc_menu_and_blocks_from_nested_files(
           @dml_link_state, source_id: source_id
         )
+
       dump_delobj(@dml_blocks_in_file, @dml_menu_blocks, @dml_link_state)
     end
 
@@ -5081,8 +5288,8 @@ module MarkdownExec
         :prompt_color_after_script_execution
       )
 
-      menu_items = blocks_as_menu_items(menu_blocks)
-      if menu_items.empty?
+      tty_menu_items = blocks_as_menu_items(menu_blocks)
+      if tty_menu_items.empty?
         return SelectedBlockMenuState.new(nil, OpenStruct.new,
                                           MenuState::EXIT)
       end
@@ -5117,8 +5324,9 @@ module MarkdownExec
         { cycle: @delegate_object[:select_page_cycle],
           per_page: @delegate_object[:select_page_height] }
       )
-      selected_option = select_option_with_metadata(prompt_title, menu_items,
-                                                    selection_opts)
+      selected_option = select_option_with_metadata(
+        prompt_title, tty_menu_items, selection_opts
+      )
       determine_block_state(selected_option)
     end
 
@@ -5192,9 +5400,13 @@ module MarkdownExec
       save_expr = link_block_data.fetch(LinkKeys::SAVE, '')
       if save_expr.present?
         save_filespec = save_filespec_from_expression(save_expr)
-        File.write(save_filespec,
-                   HashDelegator.join_code_lines(link_state&.inherited_lines))
-        @delegate_object[:filename]
+        if save_filespec.present?
+          File.write(save_filespec,
+                     HashDelegator.join_code_lines(link_state&.inherited_lines))
+          @delegate_object[:filename]
+        else
+          link_block_data[LinkKeys::FILE] || @delegate_object[:filename]
+        end
       else
         link_block_data[LinkKeys::FILE] || @delegate_object[:filename]
       end
@@ -5202,6 +5414,8 @@ module MarkdownExec
   end
 
   class HashDelegator < HashDelegatorParent
+    include ::ErrorReporting
+
     # Cleans a value, handling both Hash and Struct types.
     # For Structs, the cleaned version is converted to a hash.
     def self.clean_value(value)
@@ -5414,7 +5628,7 @@ module MarkdownExec
           input: MarkdownExec::FCB.new(title: '',
                                        body: ['def add(x, y)',
                                               '  x + y', 'end']),
-          output: "def add(x, y)\n    x + y\n  end\n"
+          output: "def add(x, y)\n    x + y\n  end"
         },
         {
           input: MarkdownExec::FCB.new(title: 'foo', body: %w[bar baz]),