markdown_exec 3.2.0 → 3.4.0

data/examples/imports/organism_template.md

@@ -1,42 +1,99 @@
 # COMMON_NAME Classification
 
-**Common Name:** COMMON_NAME  
-**Species:** SPECIES  
-**Genus:** GENUS  
-**Family:** FAMILY  
-**Order:** ORDER  
-**Class:** CLASS  
-**Year Discovered:** YEAR_DISCOVERED
+**Common Name:** ${COMMON_NAME}
+**Species:** ${PLAIN_SPECIES}
+**Genus:** ${GENUS}
+**Family:** ${FAMILY}
+**Order:** ${ORDER}
+**Class:** ${CLASSIFICATION_TYPE}
+**Year Discovered:** ${YEAR_DISCOVERED}
+
+## Parameter Processing Examples
+
+__Raw Literal (=)__
+- Report title: `REPORT_TITLE`
+- Fixed classification: `CLASSIFICATION_TYPE`
+
+__Force-Quoted Literal (:q=)__
+- Quoted species name: `QUOTED_SPECIES`
+- Quoted common name: `QUOTED_NAME`
+
+__Variable Reference (:v=)__
+- Common name from variable: `BASE_NAME`
+- Discovery year from document: `DOC_YEAR`
+
+__Evaluated Expression (:e=)__
+- Generated filename: `GENERATED_FILE`
+- Formatted description: `DESCRIPTION`
+
+__Command Substitution (:c=)__
+- Current timestamp: `TIMESTAMP`
+- Latest report file: `LATEST_REPORT`
 
 ## Taxonomic Classification
 
 ```bash
-# Biological classification data
-export COMMON_NAME="COMMON_NAME"
-export SPECIES="SPECIES"
-export GENUS="GENUS"
-export FAMILY="FAMILY"
-export ORDER="ORDER"
-export CLASS="CLASS"
-export YEAR_DISCOVERED="YEAR_DISCOVERED"
-
-echo "Organism: COMMON_NAME"
-echo "Scientific name: SPECIES"
-echo "Discovered in: YEAR_DISCOVERED"
+# Biological classification data with different parameter types
+export common_name=COMMON_NAME
+export species="PLAIN_SPECIES"
+export genus="GENUS"
+export family="FAMILY"
+export order="ORDER"
+export class="CLASSIFICATION_TYPE"
+export year_discovered="YEAR_DISCOVERED"
+
+# Examples of parameter processing types
+echo "Report: REPORT_TITLE"
+echo "Quoted species: QUOTED_SPECIES"
+echo "Quoted name: QUOTED_NAME"
+echo "Base name from var: BASE_NAME"
+echo "Doc year from var: DOC_YEAR"
+echo "Generated file: GENERATED_FILE"
+echo "Description: DESCRIPTION"
+echo "Timestamp: TIMESTAMP"
+echo "Latest report: LATEST_REPORT"
+
+echo "Organism: $common_name"
+echo "Scientific name: $species"
+echo "Discovered in: $year_discovered"
+echo "Classification: $class"
 ```
 
 ## Classification Hierarchy
 
-```yaml
 organism:
-  common_name: COMMON_NAME
-  scientific_name: SPECIES
+  common_name: ${COMMON_NAME}
+  scientific_name: ${PLAIN_SPECIES}
   taxonomy:
-    genus: GENUS
-    family: FAMILY
-    order: ORDER
-    class: CLASS
-  discovery_year: YEAR_DISCOVERED
-```
+    genus: ${GENUS}
+    family: ${FAMILY}
+    order: ${ORDER}
+    class: ${CLASSIFICATION_TYPE}
+  discovery_year: ${YEAR_DISCOVERED}
+  
+metadata:
+  report_title: ${REPORT_TITLE}
+  classification_type: ${CLASSIFICATION_TYPE}
+  quoted_species: ${QUOTED_SPECIES}
+  quoted_name: ${QUOTED_NAME}
+  base_name: ${BASE_NAME}
+  doc_year: ${DOC_YEAR}
+  generated_file: ${GENERATED_FILE}
+  description: ${DESCRIPTION}
+  timestamp: ${TIMESTAMP}
+  latest_report: ${LATEST_REPORT}
+
+## Generated Documentation
+
+**Report:** ${REPORT_TITLE}  
+**Classification:** ${CLASSIFICATION_TYPE}  
+**Quoted Species:** ${QUOTED_SPECIES}  
+**Quoted Name:** ${QUOTED_NAME}  
+**Base Name:** ${BASE_NAME}  
+**Doc Year:** ${DOC_YEAR}  
+**Generated File:** ${GENERATED_FILE}  
+**Description:** ${DESCRIPTION}  
+**Timestamp:** ${TIMESTAMP}  
+**Latest Report:** ${LATEST_REPORT}  
 
-Biological organism template using raw replacement for taxonomic data. 
+Biological organism template demonstrating all five parameter processing modes: raw literal (=), force-quoted literal (:q=), variable reference (:v=), evaluated expression (:e=), and command substitution (:c=). 

data/lib/cached_nested_file_reader.rb

@@ -9,6 +9,7 @@ require 'fileutils'
 require_relative 'constants'
 require_relative 'exceptions'
 require_relative 'find_files'
+require_relative 'ww'
 
 ##
 # The CachedNestedFileReader class provides functionality to read file
@@ -22,9 +23,29 @@ require_relative 'find_files'
 class CachedNestedFileReader
   include Exceptions
 
-  def initialize(import_pattern: /^ *#import (.+)$/)
+  def initialize(
+    import_directive_line_pattern:,
+    import_directive_parameter_scan:,
+    import_parameter_variable_assignment:,
+    shell:,
+    shell_block_name:,
+    symbol_command_substitution:,
+    symbol_evaluated_expression:,
+    symbol_force_quoted_literal:,
+    symbol_raw_literal:,
+    symbol_variable_reference:
+  )
     @file_cache = {}
-    @import_pattern = import_pattern
+    @import_directive_line_pattern = import_directive_line_pattern
+    @import_directive_parameter_scan = import_directive_parameter_scan
+    @import_parameter_variable_assignment = import_parameter_variable_assignment
+    @shell = shell
+    @shell_block_name = shell_block_name
+    @symbol_command_substitution = symbol_command_substitution
+    @symbol_evaluated_expression = symbol_evaluated_expression
+    @symbol_force_quoted_literal = symbol_force_quoted_literal
+    @symbol_raw_literal = symbol_raw_literal
+    @symbol_variable_reference = symbol_variable_reference
   end
 
   def error_handler(name = '', opts = {})
@@ -45,25 +66,68 @@ class CachedNestedFileReader
   # return the processed lines
   def readlines(
     filename, depth = 0, context: '', import_paths: nil,
-    indention: '', substitutions: {}, use_template_delimiters: false, &block
+    indention: '', substitutions: {}, use_template_delimiters: false,
+    clear_cache: true,
+    read_cache: false,
+    &block
   )
+    # clear cache if requested
+    @file_cache.clear if clear_cache
+
     cache_key = build_cache_key(filename, substitutions)
     if @file_cache.key?(cache_key)
+      return ["# dup #{cache_key}"] unless read_cache
+
       @file_cache[cache_key].each(&block) if block
       return @file_cache[cache_key]
+
+      # do not return duplicates per filename and substitutions
+      # return an indicator that the file was already read
+
     end
     raise Errno::ENOENT, filename unless filename
 
     directory_path = File.dirname(filename)
     processed_lines = []
-    File.readlines(filename, chomp: true).each.with_index do |line, ind|
-      if Regexp.new(@import_pattern) =~ line
+    continued_line = nil # continued import directive
+    File.readlines(filename, chomp: true).each.with_index do |segment, ind|
+      wwt :readline, 'depth:', depth, 'filename:', filename, 'ind:', ind,
+          'segment:', segment
+
+      if continued_line || (Regexp.new(@import_directive_line_pattern) =~ segment)
+
+        line = (continued_line || '') + segment
+        # if segment ends in a continuation, prepend to next line
+        if line.end_with?('\\')
+          continued_line = line.chomp('\\')
+          next
+        end
+
+        continued_line = nil
+        Regexp.new(@import_directive_line_pattern) =~ line
         name_strip = $~[:name].strip
         params_string = $~[:params] || ''
         import_indention = indention + $~[:indention]
 
         # Parse parameters for text substitution
-        import_substitutions = parse_import_params(params_string)
+        import_substitutions, add_code = parse_import_params(params_string)
+        if add_code
+          # strings as NestedLines
+          add_lines = add_code.map.with_index do |line2, ind2|
+            nested_line = NestedLine.new(
+              line2,
+              depth + 1,
+              import_indention,
+              filename,
+              ind2
+            )
+            block&.call(nested_line)
+
+            nested_line
+          end
+          ww 'add_lines:', add_lines
+          processed_lines += add_lines
+        end
         merged_substitutions = substitutions.merge(import_substitutions)
 
         included_file_path =
@@ -78,15 +142,28 @@ class CachedNestedFileReader
 
         raise Errno::ENOENT, name_strip unless included_file_path
 
-        imported_lines = readlines(
-          included_file_path, depth + 1,
-          context: "#{filename}:#{ind + 1}",
-          import_paths: import_paths,
-          indention: import_indention,
-          substitutions: merged_substitutions,
-          use_template_delimiters: use_template_delimiters,
-          &block
-        )
+        # Create a cache key for the imported file that includes both filename and parameters
+        imported_cache_key = build_import_cache_key(included_file_path,
+                                                    name_strip, params_string, merged_substitutions)
+
+        # Check if we've already loaded this specific import
+        if @file_cache.key?(imported_cache_key)
+          imported_lines = @file_cache[imported_cache_key]
+        else
+          imported_lines = readlines(
+            included_file_path, depth + 1,
+            context: "#{filename}:#{ind + 1}",
+            import_paths: import_paths,
+            indention: import_indention,
+            substitutions: merged_substitutions,
+            use_template_delimiters: use_template_delimiters,
+            clear_cache: false,
+            &block
+          )
+
+          # Cache the imported lines with the specific import cache key
+          @file_cache[imported_cache_key] = imported_lines
+        end
 
         # Apply text substitutions to imported content
         processed_imported_lines = apply_substitutions(
@@ -96,7 +173,7 @@ class CachedNestedFileReader
         processed_lines += processed_imported_lines
       else
         # Apply substitutions to the current line
-        substituted_line = apply_line_substitutions(line, substitutions,
+        substituted_line = apply_line_substitutions(segment, substitutions,
                                                     use_template_delimiters)
         nested_line = NestedLine.new(substituted_line, depth, indention,
                                      filename, ind)
@@ -105,6 +182,7 @@ class CachedNestedFileReader
       end
     end
 
+    wwt :read_document_code, 'processed_lines:', processed_lines
     @file_cache[cache_key] = processed_lines
   rescue Errno::ENOENT => err
     warn_format('readlines', "#{err} @@ #{context}",
@@ -113,21 +191,91 @@ class CachedNestedFileReader
 
   private
 
+  def shell_code_block_for_assignment(key, expression)
+    ["```#{@shell} :#{@shell_block_name}",
+     format(@import_parameter_variable_assignment,
+            { key: key, value: expression }),
+     '```'].tap { wwr _1 }
+  end
+
   # Parse key=value parameters from the import line
   def parse_import_params(params_string)
     return {} if params_string.nil? || params_string.strip.empty?
 
+    add_code = []
     params = {}
-    # Match key=value pairs, handling quoted values
-    params_string.scan(
-      /([A-Za-z_]\w*)=(?:"([^"]*)"|'([^']*)'|(\S+))/
-    ) do |key, quoted_double, quoted_single, unquoted|
+
+    # First loop: store all scanned parameters in a temporary variable
+    scanned_params = []
+    params_string.scan(@import_directive_parameter_scan) do |key, op, quoted_double, quoted_single, unquoted|
       value = quoted_double || quoted_single || unquoted
-      # skip replacement of equal values
-      # otherwise, the text is not available for other substitutions
-      params[key] = value if key != value
+      scanned_params << [key, op, value]
     end
-    params
+    wwt 'scanned_params:', scanned_params
+
+    # Preceding loop: select items where op = '='
+    equal_op_params = scanned_params.select { |_key, op, _value| op == '=' }
+    wwt 'equal_op_params:', equal_op_params
+    # Process remaining parameters (non-equal operators)
+    scanned_params.each do |key, op, value|
+      wwt :import, 'key:', key, 'op:', op, 'value:', value
+
+      # adjust key for current stem value
+      varname = key
+      case op
+      when @symbol_command_substitution,
+           @symbol_evaluated_expression,
+           @symbol_force_quoted_literal,
+           @symbol_variable_reference
+        # perform all substitutions per equal_op_params
+        equal_op_params.each do |equal_key, _equal_op, equal_value|
+          varname = varname.gsub(equal_key, equal_value)
+        end
+        wwt :import_key, 'varname:', varname
+      end
+
+      case op
+      when @symbol_raw_literal
+        # skip replacement of equal values otherwise,
+        # the text is not available for other substitutions
+        next unless key != value
+
+        # replace the literal below
+      when @symbol_command_substitution
+        # add code to set the variable to the value of the parameter
+        add_code += shell_code_block_for_assignment(
+          varname, %($(#{value}))
+        )
+        # replace key with expansion of the added variable
+        value = "${#{varname}}"
+      when @symbol_evaluated_expression
+        # add code to set the variable to the value of the parameter
+        add_code += shell_code_block_for_assignment(
+          varname, %("#{value}")
+        )
+        # replace key with expansion of the added variable
+        value = "${#{varname}}"
+      when @symbol_force_quoted_literal
+        # add code to set the variable to the value of the parameter
+        add_code += shell_code_block_for_assignment(
+          varname, Shellwords.escape(value)
+        )
+        # replace key with expansion of the added variable
+        value = "${#{varname}}"
+      when @symbol_variable_reference
+        # variable exists
+        value = "${#{value}}"
+      else
+        wwe "Invalid op '#{op}'"
+      end
+
+      params[key] = value
+    end
+    [params, add_code].tap do
+      wwr 'params:', _1[0], 'add_code:', _1[1]
+    end
+  rescue StandardError
+    wwe $!
   end
 
   # Apply text substitutions to a collection of NestedLine objects
@@ -155,8 +303,10 @@ class CachedNestedFileReader
     if use_template_delimiters
       # Replace template-style placeholders: ${KEY} or {{KEY}}
       substitutions.each do |key, value|
-        substituted_line = substituted_line.gsub(/\$\{#{Regexp.escape(key)}\}/,
-                                                 value)
+        substituted_line = substituted_line.gsub(
+          /\$\{#{Regexp.escape(key)}\}/,
+          value
+        )
         substituted_line = substituted_line.gsub(
           /\{\{#{Regexp.escape(key)}\}\}/, value
         )
@@ -168,7 +318,6 @@ class CachedNestedFileReader
       # Replace each key with a unique temporary placeholder
       substitutions.each_with_index do |(key, value), index|
         temp_placeholder = "__MDE_TEMP_#{index}__"
-        # pattern = /\b#{Regexp.escape(key)}\b/
         pattern = Regexp.new(Regexp.escape(key))
         substituted_line = substituted_line.gsub(pattern, temp_placeholder)
         temp_placeholders[temp_placeholder] = value
@@ -190,6 +339,13 @@ class CachedNestedFileReader
     substitution_hash = substitutions.sort.to_h.hash
     "#{filename}##{substitution_hash}"
   end
+
+  # Build a cache key specifically for imported files
+  def build_import_cache_key(filename, name_strip, params_string, substitutions)
+    # Sort parameters for consistent key
+    sorted_params = substitutions.sort.to_h.hash
+    "#{filename}##{name_strip}##{params_string}##{sorted_params}"
+  end
 end
 
 return if $PROGRAM_NAME != __FILE__
@@ -207,7 +363,18 @@ class CachedNestedFileReaderTest < Minitest::Test
     @file1.write("Line1\nLine2\n @import #{@file2.path}\nLine3")
     @file1.rewind
     @reader = CachedNestedFileReader.new(
-      import_pattern: /^(?<indention> *)@import +(?<name>\S+)(?<params>(?: +[A-Za-z_]\w*=(?:"[^"]*"|'[^']*'|\S+))*) *$/
+      import_directive_line_pattern:
+        /^(?<indention> *)@import +(?<name>\S+)(?<params>(?: +[A-Za-z_]\w*=(?:"[^"]*"|'[^']*'|\S+))*) *$/,
+      import_directive_parameter_scan:
+        /([A-Za-z_]\w*)(:=|\?=|!=|=)(?:"([^"]*)"|'([^']*)'|(\S+))/,
+      import_parameter_variable_assignment: '%{key}=%{value}',
+      shell: 'bash',
+      shell_block_name: '(document_shell)',
+      symbol_command_substitution: ':c=',
+      symbol_evaluated_expression: ':e=',
+      symbol_raw_literal: '=',
+      symbol_force_quoted_literal: ':q=',
+      symbol_variable_reference: ':v='
     )
   end
 
@@ -278,9 +445,92 @@ class CachedNestedFileReaderTest < Minitest::Test
     @file2.reopen(@file2.path, 'w') { |f| f.write('ChangedLine') }
 
     # Second read (should read from cache, not the changed file)
-    result2 = @reader.readlines(@file2.path).map(&:to_s)
+    result2 = @reader.readlines(@file2.path, clear_cache: false,
+                                             read_cache: true).map(&:to_s)
 
     assert_equal result1, result2
     assert_equal %w[ImportedLine1 ImportedLine2], result2
   end
+
+  def test_import_caching_with_same_parameters
+    # Create a file that will be imported multiple times
+    shared_file = Tempfile.new('shared.txt')
+    shared_file.write("Shared content line 1\nShared content line 2")
+    shared_file.rewind
+
+    # Create a file that imports the same file multiple times with same parameters
+    importing_file = Tempfile.new('importing_multiple.txt')
+    importing_file.write("Start\n @import #{shared_file.path} PARAM=value\nMiddle\n @import #{shared_file.path} PARAM=value\nEnd")
+    importing_file.rewind
+
+    # Track how many times the shared file is actually read
+    read_count = 0
+    original_readlines = File.method(:readlines)
+    File.define_singleton_method(:readlines) do |filename, **opts|
+      if filename == shared_file.path
+        read_count += 1
+      end
+      original_readlines.call(filename, **opts)
+    end
+
+    result = @reader.readlines(importing_file.path).map(&:to_s)
+
+    # The shared file should only be read once, not twice
+    assert_equal 1, read_count,
+                 'Shared file should only be read once when imported with same parameters'
+
+    # Verify the content is correct
+    expected = ['Start', ' Shared content line 1', ' Shared content line 2',
+                'Middle', ' Shared content line 1', ' Shared content line 2', 'End']
+    assert_equal expected, result
+
+    # Restore original method
+    File.define_singleton_method(:readlines, original_readlines)
+
+    shared_file.close
+    shared_file.unlink
+    importing_file.close
+    importing_file.unlink
+  end
+
+  def test_import_caching_with_different_parameters
+    # Create a file that will be imported with different parameters
+    template_file = Tempfile.new('template.txt')
+    template_file.write('Hello NAME, your ID is ID')
+    template_file.rewind
+
+    # Create a file that imports the same file with different parameters
+    importing_file = Tempfile.new('importing_different.txt')
+    importing_file.write("Users:\n @import #{template_file.path} NAME=Alice ID=123\n @import #{template_file.path} NAME=Bob ID=456\nEnd")
+    importing_file.rewind
+
+    # Track how many times the template file is actually read
+    read_count = 0
+    original_readlines = File.method(:readlines)
+    File.define_singleton_method(:readlines) do |filename, **opts|
+      if filename == template_file.path
+        read_count += 1
+      end
+      original_readlines.call(filename, **opts)
+    end
+
+    result = @reader.readlines(importing_file.path).map(&:to_s)
+
+    # The template file should be read twice since parameters are different
+    assert_equal 2, read_count,
+                 'Template file should be read twice when imported with different parameters'
+
+    # Verify the content is correct
+    expected = ['Users:', ' Hello Alice, your 123 is 123',
+                ' Hello Bob, your 456 is 456', 'End']
+    assert_equal expected, result
+
+    # Restore original method
+    File.define_singleton_method(:readlines, original_readlines)
+
+    template_file.close
+    template_file.unlink
+    importing_file.close
+    importing_file.unlink
+  end
 end

data/lib/constants.rb

@@ -27,7 +27,14 @@ BLOCK_TYPE_COLOR_OPTIONS = {
   BlockType::OPTS => :menu_opts_color,
   BlockType::SAVE => :menu_save_color,
   BlockType::SHELL => :menu_bash_color,
-  BlockType::UX => :menu_ux_color,
+  BlockType::UX => {
+    :is_allow? => :menu_ux_color_allow,
+    :is_echo? => :menu_ux_color_echo,
+    :is_edit? => :menu_ux_color_edit,
+    :is_exec? => :menu_ux_color_exec,
+    :readonly => :menu_ux_color_readonly,
+    true => :menu_ux_color
+  },
   BlockType::VARS => :menu_vars_color
 }.freeze
 

data/lib/env_interface.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require_relative 'ww'
 
 # A class that provides an interface to ENV variables with customizable getters and setters
@@ -10,11 +11,15 @@ class EnvInterface
     # @param transform [Proc] Optional transformation to apply to the value
     # @return [Object] The environment variable value
     def get(key, default: nil, transform: nil)
-wwt :env, key, \
-      value = ENV[key]
-      return default if value.nil?
+      value = ENV.fetch(key, nil)
 
-      transform ? transform.call(value) : value
+      if value.nil?
+        default
+      else
+        transform ? transform.call(value) : value
+      end.tap do
+        wwt :env, key, _1
+      end
     end
 
     # Set an environment variable with optional transformation
@@ -23,9 +28,10 @@ wwt :env, key, \
     # @param transform [Proc] Optional transformation to apply before setting
     # @return [String] The set value
     def set(key, value, transform: nil)
-wwt :env, key, caller.deref[0..3], value
       transformed_value = transform ? transform.call(value) : value
-      ENV[key] = transformed_value.to_s
+      ENV[key] = transformed_value.to_s.tap do
+        wwt :env, key, _1
+      end
     end
 
     # Check if an environment variable exists
@@ -54,4 +60,4 @@ wwt :env, key, caller.deref[0..3], value
       ENV.clear
     end
   end
-end 
+end

data/lib/evaluate_shell_expressions.rb

@@ -25,6 +25,7 @@ def evaluate_shell_expressions(initial_code, expressions, shell: '/bin/bash',
     script << "\necho #{token}#{index}\n"
     script << expression << "\n"
   end
+  wwt :eval, 'script:', script
 
   # Execute
   stdout_str, _, status = Open3.capture3(shell, '-c', script)