poml 0.0.6 → 0.0.8

data/lib/poml/parser.rb

@@ -35,6 +35,9 @@ module Poml
       # Pre-process to handle JSON in attributes (convert \" to " inside attribute values)
       content = preprocess_json_attributes(content)
       
+      # Pre-process code blocks to escape XML special characters
+      content = preprocess_code_blocks(content)
+      
       # Remove XML comments but preserve surrounding whitespace
       content = content.gsub(/(\s*)<!--.*?-->(\s*)/m) do |match|
         before_space = $1
@@ -50,8 +53,9 @@ module Poml
       # Convert HTML-style void elements to XML self-closing format
       content = preprocess_void_elements(content)
       
-      # Apply template substitutions
-      content = @template_engine.substitute(content)
+      # Apply safe template substitutions - only for simple string values
+      # This avoids XML parsing issues with complex values in attributes
+      content = @template_engine.safe_substitute(content)
 
       # Check if content is wrapped in <poml> tags and extract syntax
       content = content.strip
@@ -120,8 +124,43 @@ module Poml
           
           # Handle for loop rendering
           if for_attribute
-            loop_elements = render_for_loop(child, attrs, for_attribute)
-            elements.concat(loop_elements)
+            # Convert for attribute to ForComponent structure
+            # Parse for attribute like "item in items"
+            if for_attribute =~ /^(\w+)\s+in\s+(.+)$/
+              loop_var = $1
+              list_expr = $2.strip
+              
+              # Create a for element that wraps the original element
+              # Remove the for attribute from the original element
+              loop_attrs = attrs.dup
+              loop_attrs.delete('for')
+              
+              # Create the wrapper element
+              original_element = Element.new(
+                tag_name: child.name.downcase.to_sym,
+                attributes: loop_attrs,
+                content: extract_text_content(child),
+                children: parse_element(child)
+              )
+              
+              # Create for component element
+              for_element = Element.new(
+                tag_name: :for,
+                attributes: { 'variable' => loop_var, 'items' => list_expr },
+                content: '',
+                children: [original_element]
+              )
+              
+              elements << for_element
+            else
+              # Invalid for syntax, process normally
+              elements << Element.new(
+                tag_name: child.name.downcase.to_sym,
+                attributes: attrs,
+                content: extract_text_content(child),
+                children: parse_element(child)
+              )
+            end
           else
             elements << Element.new(
               tag_name: child.name.downcase.to_sym,
@@ -174,65 +213,6 @@ module Poml
       !!value
     end
     
-    def render_for_loop(xml_element, attrs, for_attribute)
-      # Parse for attribute like "i in [1,2,3]" or "item in items"
-      if for_attribute =~ /^(\w+)\s+in\s+(.+)$/
-        loop_var = $1
-        list_expr = $2.strip
-        
-        # Evaluate the list expression
-        list = @template_engine.evaluate_attribute_expression(list_expr)
-        return [] unless list.is_a?(Array)
-        
-        # Create elements for each item in the list
-        elements = []
-        list.each_with_index do |item, index|
-          # Create loop context
-          old_loop_var = @context.variables[loop_var]
-          old_loop_context = @context.variables['loop']
-          
-          @context.variables[loop_var] = item
-          @context.variables['loop'] = {
-            'index' => index,
-            'length' => list.length,
-            'first' => index == 0,
-            'last' => index == list.length - 1
-          }
-          
-          # Remove for attribute and process element normally
-          loop_attrs = attrs.dup
-          loop_attrs.delete('for')
-          
-          element = Element.new(
-            tag_name: xml_element.name.downcase.to_sym,
-            attributes: loop_attrs,
-            content: extract_text_content(xml_element),
-            children: parse_element(xml_element)
-          )
-          
-          elements << element
-          
-          # Restore previous context
-          if old_loop_var
-            @context.variables[loop_var] = old_loop_var
-          else
-            @context.variables.delete(loop_var)
-          end
-          
-          if old_loop_context
-            @context.variables['loop'] = old_loop_context
-          else
-            @context.variables.delete('loop')
-          end
-        end
-        
-        elements
-      else
-        # Invalid for syntax, return empty
-        []
-      end
-    end
-    
     def preprocess_void_elements(content)
       # List of HTML void elements that should be self-closing in XML
       # Note: 'meta' is removed from this list because POML meta components can have content
@@ -264,5 +244,50 @@ module Poml
       
       content
     end
+
+    def preprocess_code_blocks(content)
+      # Escape XML special characters within <code> and <code-block> blocks to prevent XML parsing errors
+      # Use negative lookahead to avoid matching <code-block> when looking for <code>
+      content = content.gsub(/<code(?!-)[^>]*>(.*?)<\/code>/m) do |match|
+        full_match = match
+        opening_tag = full_match[/<code(?!-)[^>]*>/]
+        closing_tag = '</code>'
+        code_content = full_match.match(/<code(?!-)[^>]*>(.*?)<\/code>/m)[1]
+        
+        # Handle nil case
+        code_content ||= ''
+        
+        # Escape XML special characters in the code content
+        escaped_content = code_content.gsub('&', '&amp;')
+                                     .gsub('<', '&lt;')
+                                     .gsub('>', '&gt;')
+                                     .gsub('"', '&quot;')
+                                     .gsub("'", '&apos;')
+        
+        "#{opening_tag}#{escaped_content}#{closing_tag}"
+      end
+      
+      # Also handle <code-block> tags
+      content = content.gsub(/<code-block\b[^>]*>(.*?)<\/code-block>/m) do |match|
+        full_match = match
+        opening_tag = full_match[/<code-block\b[^>]*>/]
+        closing_tag = '</code-block>'
+        code_content = full_match.match(/<code-block\b[^>]*>(.*?)<\/code-block>/m)[1]
+        
+        # Handle nil case
+        code_content ||= ''
+        
+        # Escape XML special characters in the code content
+        escaped_content = code_content.gsub('&', '&amp;')
+                                     .gsub('<', '&lt;')
+                                     .gsub('>', '&gt;')
+                                     .gsub('"', '&quot;')
+                                     .gsub("'", '&apos;')
+        
+        "#{opening_tag}#{escaped_content}#{closing_tag}"
+      end
+      
+      content
+    end
   end
 end

data/lib/poml/renderer.rb

@@ -8,6 +8,9 @@ module Poml
     end
 
     def render(elements, format = 'dict')
+      # Set the output format in context so components can adjust behavior
+      @context.output_format = format
+      
       case format
       when 'raw'
         render_raw(elements)
@@ -15,6 +18,8 @@ module Poml
         render_dict(elements)
       when 'openai_chat'
         render_openai_chat(elements)
+      when 'openaiResponse'
+        render_openai_response(elements)
       when 'langchain'
         render_langchain(elements)
       when 'pydantic'
@@ -57,27 +62,125 @@ module Poml
       
       # Include additional metadata if present
       metadata['response_schema'] = @context.response_schema if @context.response_schema
-      metadata['tools'] = @context.tools if @context.tools && !@context.tools.empty?
       metadata['runtime_parameters'] = @context.runtime_parameters if @context.runtime_parameters && !@context.runtime_parameters.empty?
       
-      {
+      # Add schemas array for backward compatibility with tests
+      if @context.response_schema_with_metadata
+        metadata['schemas'] = [@context.response_schema_with_metadata]
+      end
+      
+      result = {
         'content' => content,
         'metadata' => metadata
       }
+      
+      # Include tools at top level (matching original implementation)
+      result['tools'] = @context.tools && !@context.tools.empty? ? @context.tools : []
+      
+      # Include output content if present
+      result['output'] = @context.output_content if @context.output_content
+      
+      result
     end
 
     def render_openai_chat(elements)
       # First render to collect structured messages
       content = render_raw(elements)
       
-      # Use structured messages if available
-      if @context.respond_to?(:chat_messages) && !@context.chat_messages.empty?
+      # Build messages array
+      messages = if @context.respond_to?(:chat_messages) && !@context.chat_messages.empty?
         @context.chat_messages
       elsif @context.chat
         parse_chat_messages(content)
       else
         [{ 'role' => 'user', 'content' => content }]
       end
+      
+      # openai_chat format always returns just the messages array
+      # Tools should be accessed via openaiResponse format instead
+      messages
+    end
+    
+    def convert_to_openai_schema(tool)
+      # If tool already has a schema field (from JSON format), use it
+      if tool['schema']
+        begin
+          return JSON.parse(tool['schema']) if tool['schema'].is_a?(String)
+          return tool['schema'] if tool['schema'].is_a?(Hash)
+        rescue JSON::ParserError
+          # Fall through to build from parameters
+        end
+      end
+      
+      # Build schema from flattened parameters (declarative format)
+      if tool['parameters']
+        if tool['parameters'].is_a?(Hash) && tool['parameters']['type'] == 'object'
+          # Already in JSON schema format
+          return tool['parameters']
+        else
+          # Convert flattened format to JSON schema
+          properties = {}
+          required = []
+          
+          tool['parameters'].each do |param_name, param_def|
+            properties[param_name] = {
+              'type' => param_def['type']
+            }
+            properties[param_name]['description'] = param_def['description'] if param_def['description']
+            required << param_name if param_def['required']
+          end
+          
+          return {
+            'type' => 'object',
+            'properties' => properties,
+            'required' => required
+          }
+        end
+      end
+      
+      # Fallback empty schema
+      { 'type' => 'object', 'properties' => {}, 'required' => [] }
+    end
+
+    def render_openai_response(elements)
+      # OpenAI Response format - standardized AI response structure
+      # Different from openai_chat which focuses on conversation messages
+      content = render_raw(elements)
+      
+      response = {
+        'content' => content.strip,
+        'type' => 'assistant'
+      }
+      
+      # Include chat messages if available (for compatibility)
+      if @context.chat_messages && !@context.chat_messages.empty?
+        response['messages'] = @context.chat_messages
+      end
+      
+      # Include metadata if available
+      metadata = {}
+      metadata['variables'] = @context.variables if @context.variables && !@context.variables.empty?
+      metadata['response_schema'] = @context.response_schema if @context.response_schema
+      metadata['runtime_parameters'] = @context.runtime_parameters if @context.runtime_parameters && !@context.runtime_parameters.empty?
+      
+      # Include custom metadata (title, description, etc.)
+      metadata.merge!(@context.custom_metadata) if @context.custom_metadata && !@context.custom_metadata.empty?
+      
+      # Include tools in metadata only if there are tools
+      metadata['tools'] = @context.tools if @context.tools && !@context.tools.empty?
+      
+      # Include schemas in metadata for backward compatibility
+      if @context.response_schema_with_metadata
+        metadata['schemas'] = [@context.response_schema_with_metadata]
+      end
+      
+      # Only include metadata if it has meaningful content
+      response['metadata'] = metadata unless metadata.empty?
+      
+      # Include tools at top level (matching existing structure)
+      response['tools'] = @context.tools && !@context.tools.empty? ? @context.tools : []
+      
+      response
     end
 
     def render_langchain(elements)
@@ -89,12 +192,91 @@ module Poml
     end
 
     def render_pydantic(elements)
-      # Simplified pydantic-like structure
-      {
-        'prompt' => render_raw(elements),
-        'variables' => @context.variables,
-        'chat_enabled' => @context.chat
+      # Enhanced pydantic format with Python interoperability features
+      raw_content = render_raw(elements)
+      
+      # Enhanced Pydantic-compatible structure
+      pydantic_output = {
+        'content' => raw_content,
+        'variables' => @context.variables || {},
+        'chat_enabled' => @context.chat,
+        'metadata' => {
+          'format' => 'pydantic',
+          'version' => '1.0',
+          'python_compatible' => true,
+          'strict_json_schema' => true
+        }
+      }
+      
+      # Add response schema if available (from output-schema components)
+      if @context.response_schema
+        pydantic_output['schemas'] = [make_schema_strict(@context.response_schema)]
+      else
+        pydantic_output['schemas'] = []
+      end
+      
+      # Add tools if available (from tool-definition components)
+      if @context.tools && !@context.tools.empty?
+        pydantic_output['tools'] = @context.tools.map { |tool| format_tool_for_pydantic(tool) }
+      else
+        pydantic_output['tools'] = []
+      end
+      
+      # Add custom metadata if available (from meta components)
+      if @context.custom_metadata && !@context.custom_metadata.empty?
+        pydantic_output['custom_metadata'] = @context.custom_metadata
+      else
+        pydantic_output['custom_metadata'] = {}
+      end
+      
+      pydantic_output
+    end
+    
+    private
+    
+    def make_schema_strict(schema)
+      # Convert schema to strict JSON schema format (Pydantic compatible)
+      return schema unless schema.is_a?(Hash)
+      
+      strict_schema = schema.dup
+      
+      if strict_schema['type'] == 'object'
+        strict_schema['additionalProperties'] = false
+        
+        # Make all properties required if not specified
+        if strict_schema['properties'] && !strict_schema['required']
+          strict_schema['required'] = strict_schema['properties'].keys
+        end
+        
+        # Recursively process nested objects
+        if strict_schema['properties']
+          strict_schema['properties'] = strict_schema['properties'].transform_values do |prop|
+            make_schema_strict(prop)
+          end
+        end
+      elsif strict_schema['type'] == 'array' && strict_schema['items']
+        strict_schema['items'] = make_schema_strict(strict_schema['items'])
+      end
+      
+      # Remove null defaults for strict compatibility
+      strict_schema.delete('default') if strict_schema['default'].nil?
+      
+      strict_schema
+    end
+    
+    def format_tool_for_pydantic(tool)
+      # Format tool definition for Pydantic compatibility
+      formatted_tool = {
+        'name' => tool['name'],
+        'description' => tool['description']
       }
+      
+      if tool['parameters']
+        # Convert parameters to Pydantic-compatible format
+        formatted_tool['parameters'] = make_schema_strict(tool['parameters'])
+      end
+      
+      formatted_tool
     end
 
     def parse_chat_messages(content)

data/lib/poml/template_engine.rb

@@ -15,8 +15,50 @@ module Poml
       end
     end
 
+    def safe_substitute(text)
+      return text unless text.is_a?(String)
+      
+      # Handle {{variable}} substitutions, but only substitute simple string values
+      # to avoid XML parsing issues with complex values in attributes
+      text.gsub(/\{\{(.+?)\}\}/) do |match|
+        expression = $1.strip
+        
+        # Don't substitute complex expressions (comparisons, ternary operators, etc.)
+        # These should be handled during rendering when variables are in proper scope
+        if expression.include?('==') || expression.include?('!=') || 
+           expression.include?('>=') || expression.include?('<=') || 
+           expression.include?('>') || expression.include?('<') ||
+           expression.include?('?') || expression.include?(':')
+          match # Keep {{...}} for complex expressions
+        else
+          # Check what type of value this expression would resolve to
+          # without actually evaluating it to string first
+          raw_value = evaluate_attribute_expression(expression)
+          
+          # Only substitute if:
+          # 1. The value exists (not nil)
+          # 2. The value is a simple type (string, number, boolean)
+          # Leave nil values and complex objects as template variables for component-level handling
+          case raw_value
+          when String, Numeric, TrueClass, FalseClass
+            raw_value.to_s
+          when NilClass
+            match # Keep {{...}} for nil values - they might be valid in child contexts
+          else
+            match # Return original {{...}} for complex values like arrays/hashes
+          end
+        end
+      end
+    end
+
     def evaluate_attribute_expression(expression)
       # Handle attribute expressions that might return non-string values
+      
+      # Handle string literals with quotes
+      if expression =~ /^"([^"]*)"$/ || expression =~ /^'([^']*)'$/
+        return $1  # Return the content without quotes
+      end
+      
       if expression =~ /^(\w+(?:\.\w+)*)\s*\+\s*(\d+)$/
         variable_path = $1
         increment = $2.to_i
@@ -51,6 +93,11 @@ module Poml
     def evaluate_expression(expression)
       # Handle dot notation and arithmetic expressions
       
+      # Handle string literals with quotes
+      if expression =~ /^"([^"]*)"$/ || expression =~ /^'([^']*)'$/
+        return $1  # Return the content without quotes
+      end
+      
       # Simple arithmetic operations like loop.index+1
       if expression =~ /^(\w+(?:\.\w+)*)\s*\+\s*(\d+)$/
         variable_path = $1
@@ -80,6 +127,22 @@ module Poml
     def evaluate_complex_expression(expression)
       # Handle more complex expressions like array literals, object access, etc.
       
+      # Handle logical OR operator (||) - return first truthy value
+      if expression =~ /^(.+?)\s*\|\|\s*(.+)$/
+        left_expression = $1.strip
+        right_expression = $2.strip
+        
+        left_value = evaluate_expression(left_expression)
+        
+        # Return left value if it's truthy (not nil, false, or empty string)
+        if left_value && left_value != "" && left_value != false
+          return left_value
+        else
+          # Evaluate and return the right expression
+          return evaluate_expression(right_expression)
+        end
+      end
+      
       # Try to parse as JSON first (for arrays and objects)
       begin
         require 'json'
@@ -112,32 +175,85 @@ module Poml
         
         condition_result = evaluate_condition(condition)
         if condition_result
-          evaluate_expression(true_value)
+          return evaluate_expression(true_value)
         else
-          evaluate_expression(false_value)
+          return evaluate_expression(false_value)
         end
       end
       
+      # Handle comparison operations as standalone expressions
+      if expression =~ /^(.+?)\s*(==|!=|>=|<=|>|<)\s*(.+)$/
+        condition_result = evaluate_condition(expression)
+        return condition_result
+      end
+      
       nil
     end
     
     def evaluate_condition(condition)
-      # Simple condition evaluation
-      case condition
-      when 'true'
-        true
-      when 'false'
-        false
-      when /^!(.+)$/
-        !evaluate_condition($1.strip)
+      # Handle comparison operations
+      if condition =~ /^(.+?)\s*(==|!=|>=|<=|>|<)\s*(.+)$/
+        left_operand = $1.strip
+        operator = $2.strip
+        right_operand = $3.strip
+        
+        # Evaluate operands
+        left_value = evaluate_operand(left_operand)
+        right_value = evaluate_operand(right_operand)
+        
+        # Perform comparison
+        case operator
+        when '=='
+          left_value == right_value
+        when '!='
+          left_value != right_value
+        when '>='
+          left_value >= right_value if left_value.respond_to?(:>=)
+        when '<='
+          left_value <= right_value if left_value.respond_to?(:<=)
+        when '>'
+          left_value > right_value if left_value.respond_to?(:>)
+        when '<'
+          left_value < right_value if left_value.respond_to?(:<)
+        else
+          false
+        end
+      else
+        # Simple condition evaluation
+        case condition
+        when 'true'
+          true
+        when 'false'
+          false
+        when /^!(.+)$/
+          !evaluate_condition($1.strip)
+        else
+          value = get_nested_variable(condition)
+          !!value
+        end
+      end
+    end
+    
+    def evaluate_operand(operand)
+      # Handle string literals with quotes
+      if operand =~ /^"([^"]*)"$/ || operand =~ /^'([^']*)'$/
+        return $1
+      elsif operand =~ /^-?\d+$/
+        return operand.to_i
+      elsif operand =~ /^-?\d*\.\d+$/
+        return operand.to_f
+      elsif operand == 'true'
+        return true
+      elsif operand == 'false'
+        return false
       else
-        value = get_nested_variable(condition)
-        !!value
+        # Variable lookup
+        get_nested_variable(operand)
       end
     end
     
     def get_nested_variable(path)
-      # Handle dot notation like "loop.index"
+      # Handle dot notation like "loop.index" or "items.length"
       parts = path.split('.')
       
       # Handle both Context objects and raw variable hashes
@@ -152,6 +268,15 @@ module Poml
           value = value[part]
         elsif value.is_a?(Hash) && value.key?(part.to_sym)
           value = value[part.to_sym]
+        elsif value.respond_to?(part) && (value.is_a?(Array) || value.is_a?(String))
+          # Handle method calls like .length, .size, .count on arrays/strings only
+          # Don't call methods on Hash objects to avoid conflicts with variable names
+          begin
+            value = value.send(part)
+          rescue
+            # If method call fails, return nil
+            return nil
+          end
         else
           return nil
         end

data/lib/poml/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Poml
-  VERSION = "0.0.6"
+  VERSION = "0.0.7"
 end

data/lib/poml.rb

@@ -21,14 +21,16 @@ module Poml
   # 
   # Parameters:
   # - markup: POML markup string or file path
-  # - format: 'raw', 'dict', 'openai_chat', 'langchain', 'pydantic' (default: 'dict')
+  # - format: 'raw', 'dict', 'openai_chat', 'openaiResponse', 'langchain', 'pydantic' (default: 'dict')
   # - context/variables: Hash of template variables
   # - stylesheet: CSS/styling rules
   # - chat: Enable chat mode (default: true)
   # - output_file: File path to write output to
   def self.process(markup:, format: 'dict', **options)
     # Handle file paths
+    source_file = nil
     content = if File.exist?(markup)
+      source_file = File.expand_path(markup)
       File.read(markup)
     else
       markup
@@ -51,6 +53,7 @@ module Poml
     context_options[:stylesheet] = options.delete(:stylesheet) if options.key?(:stylesheet)
     context_options[:chat] = options.delete(:chat) if options.key?(:chat)
     context_options[:syntax] = options.delete(:syntax) if options.key?(:syntax)
+    context_options[:source_path] = source_file if source_file
     
     result = render(content, format: format, **context_options)
     
@@ -78,6 +81,8 @@ module Poml
 
   # Convenience method for quick text rendering
   def self.to_text(content, **options)
+    # For text rendering, disable chat mode to allow chat components to render content
+    options[:chat] = false unless options.key?(:chat)
     render(content, format: 'raw', **options)
   end
 
@@ -86,8 +91,18 @@ module Poml
     render(content, format: 'openai_chat', **options)
   end
 
+  # Convenience method for OpenAI response format
+  def self.to_openai_response(content, **options)
+    render(content, format: 'openaiResponse', **options)
+  end
+
   # Convenience method for dict format
   def self.to_dict(content, **options)
     render(content, format: 'dict', **options)
   end
+
+  # Convenience method for Pydantic format with Python interoperability
+  def self.to_pydantic(content, **options)
+    render(content, format: 'pydantic', **options)
+  end
 end