liquid_xlsx 0.2.2 → 0.4.0

data/lib/liquid_xlsx/streaming_renderer.rb

@@ -0,0 +1,300 @@
+# frozen_string_literal: true
+
+module LiquidXlsx
+  # An independent AST walker: output rows and loop iterations are never retained.
+  class StreamingRenderer < Renderer
+    # MRI sizes its young heap relative to live input, including large Arrays.
+    # Occasional minor collections bound transient XML garbage without changing
+    # process-wide GC configuration or retaining output metadata.
+    GC_ROW_INTERVAL = 16_384
+
+    TEXT_ESCAPES = { "&" => "&amp;", "<" => "&lt;", ">" => "&gt;", "\r" => "&#13;" }.freeze
+    ATTRIBUTE_ESCAPES = TEXT_ESCAPES.merge('"' => "&quot;", "\n" => "&#10;", "\t" => "&#9;").freeze
+
+    def prepare
+      @rows = worksheet.parse_rows
+      @merges = worksheet.merge_cells
+      validate_cells
+      @ast = TemplateParser.new(@rows, worksheet.name).parse
+      @merge_state = RenderState.new(MergeCellsTransformer.new, @merges, FormulaTranslator.new)
+      prepare_nodes(@ast)
+      @merge_state.check_block_boundaries(worksheet.name)
+      # Only endpoints of header merges need mapping, bounded by template size.
+      @merge_rows = @merges.flat_map { |m| [m[:start_row], m[:end_row]] }.uniq.to_h { |r| [r, true] }
+      self
+    end
+
+    # True when the sheet iterates a collection; a one-shot source must feed only one sheet.
+    def loops?
+      @loops == true
+    end
+
+    def render(data, **_unused)
+      prepare unless @ast
+      file = Tempfile.new(["liquid_xlsx_rows", ".xml"])
+      file.binmode
+      unless @rows.any? { |row| row[:cells].any? { |cell| cell[:template] } }
+        part = FilePart.new(file, worksheet.to_xml, "")
+        return part
+      end
+
+      context = streaming_context(data)
+      @output = file
+      @row_buffer = +""
+      @row_number = 0
+      @max_col_index = 0
+      @max_col = "A"
+      walk(@ast, context)
+      file.flush
+      worksheet.update_merge_cells(@merge_state.build_merge_ranges)
+      prefix, suffix = worksheet.streaming_envelope(@row_number, @max_col)
+      part = FilePart.new(file, prefix, suffix)
+      part
+    ensure
+      file&.close! unless part
+      @output = nil
+      @row_buffer = nil
+    end
+
+    private
+
+    def validate_cells
+      first_loop = @rows.find do |row|
+        row[:cells].any? { |c| c[:template]&.strip&.match?(TemplateParser::FOR_START) }
+      end
+      @rows.each do |row|
+        row[:cells].each do |cell|
+          if cell[:raw_cell].at_xpath("xmlns:f")
+            unsupported("Formulas are not supported in streaming mode", row[:row_number], cell[:col])
+          end
+          # Inspect all visible runs as well, including tags in unused branches.
+          text = cell[:text].to_s + cell[:raw_cell].xpath("xmlns:is//xmlns:t").map(&:text).join
+          next unless text.match?(/\{%-?\s*(?:image_tag|sheet)\b/)
+
+          unsupported("image_tag and sheet tags are not supported in streaming mode", row[:row_number], cell[:col])
+        end
+      end
+      return unless first_loop
+
+      @merges.each do |merge|
+        next if merge[:end_row] < first_loop[:row_number]
+
+        ref = CellReference.new(merge[:ref].split(":").first)
+        unsupported("Merges at or below the first loop are not supported in streaming mode", ref.row, ref.col)
+      end
+    end
+
+    def unsupported(message, row, col)
+      raise StreamingUnsupportedError.new(message, sheet: worksheet.name, row: row, cell: "#{col}#{row}")
+    end
+
+    def prepare_nodes(nodes)
+      nodes.each do |node|
+        case node
+        when Nodes::RowNode
+          row = node.row_data
+          row[:stream_attrs] = (row[:row_attrs] || {}).filter_map do |key, value|
+            %( #{key}="#{escape_attribute(value)}") if value
+          end.join
+          row[:cells].each do |cell|
+            cell[:stream_index] = CellReference.col_to_index(cell[:col])
+            cell[:stream_style] = cell[:style] ? %( s="#{escape_attribute(cell[:style])}") : ""
+            text = cell[:template]
+            match = text&.strip&.match(SINGLE_VAR)
+            cell[:stream_lookup] = Liquid::Expression.parse(match[1]) if match
+            cell[:stream_string_fast] = text&.match?(SINGLE_VAR)
+            cell[:stream_static] = static_cell(cell) unless text
+          end
+        when Nodes::ForNode
+          @loops = true
+          prepare_nodes(node.body)
+          prepare_nodes(node.else_body || [])
+        when Nodes::IfNode
+          @merge_state.enter_block(:if, node.if_row, node.endif_row)
+          prepare_nodes(node.branches.flat_map { |b| b[:body] })
+        end
+      end
+    end
+
+    def streaming_context(data)
+      normalized = normalize_data(data)
+      # Normalize the root without walking or duplicating collection arrays.
+      normalized = normalized.to_h { |key, value| [key.to_s, streaming_value(value)] } if normalized.is_a?(Hash)
+      context = StreamingContext.new(normalized, {}, {}, true)
+      context.strict_variables = @options.fetch(:strict_variables, false)
+      context.strict_filters = @options.fetch(:strict_filters, false)
+      context.add_filters(@options[:filters]) if @options[:filters]
+      apply_resource_limits(context)
+      context
+    end
+
+    def streaming_value(value)
+      return value unless value.is_a?(Hash)
+      return value if value.each_key.all?(String)
+
+      value.to_h { |key, item| [key.to_s, streaming_value(item)] }
+    end
+
+    def streaming_item(item)
+      return item if item.is_a?(Hash) && item.each_key.all?(String)
+
+      stringify_value(item)
+    end
+
+    def walk(nodes, context)
+      nodes.each do |node|
+        case node
+        when Nodes::RowNode then write_row(node.row_data, context)
+        when Nodes::ForNode then walk_for(node, context)
+        when Nodes::IfNode
+          branch = node.branches.find do |candidate|
+            candidate[:condition].nil? || evaluate_condition(candidate[:condition], context, node.if_row)
+          end
+          walk(branch[:body], context) if branch
+        end
+      end
+    end
+
+    def walk_for(node, context)
+      collection = if context.strict_variables
+                     strict_collection_lookup(context, node)
+                   else
+                     safe_context_lookup(context, node.collection_name)
+                   end
+      collection = [] if collection.nil?
+      unless collection.respond_to?(:each)
+        raise RenderError.new("Cannot iterate over #{collection.class}: expected an object with #each",
+                              sheet: worksheet.name, row: node.for_row)
+      end
+      size = collection.size if collection.respond_to?(:size)
+      size = nil unless size.is_a?(Integer)
+      loop_info = {}
+      loop_info["length"] = size if size
+      index = 0
+      collection.each do |item|
+        loop_info["index"] = index + 1
+        loop_info["index0"] = index
+        loop_info["first"] = index.zero?
+        if size
+          loop_info["last"] = index == size - 1
+          loop_info["rindex"] = size - index
+          loop_info["rindex0"] = size - index - 1
+        end
+        context.stack do
+          context[node.variable_name] = streaming_item(item)
+          context["forloop"] = loop_info
+          walk(node.body, context)
+        end
+        index += 1
+      end
+      walk(node.else_body, context) if index.zero? && node.else_body
+    end
+
+    def write_row(row, context)
+      @row_number += 1
+      number = @row_number.to_s
+      xml = @row_buffer.clear
+      xml << '<row r="' << number << '"' << row[:stream_attrs] << ">"
+      row[:cells].each do |cell|
+        if cell[:template]
+          value = streaming_cell_value(cell, context, row[:row_number])
+          next if (value.nil? || value == "") && !cell[:style]
+
+          xml << '<c r="' << cell[:col] << number << '"' << cell[:stream_style]
+          append_value(xml, value)
+        else
+          content = cell[:stream_static]
+          next unless content
+
+          xml << '<c r="' << cell[:col] << number << '"' << content
+        end
+        xml << "</c>"
+        if cell[:stream_index] > @max_col_index
+          @max_col_index = cell[:stream_index]
+          @max_col = cell[:col]
+        end
+      end
+      xml << "</row>\n"
+      @output.write(xml)
+      GC.start(full_mark: false) if (@row_number % GC_ROW_INTERVAL).zero?
+      @merge_state.record_row(row[:row_number], @row_number) if @merge_rows[row[:row_number]]
+    end
+
+    def streaming_cell_value(cell, context, row)
+      if (expression = cell[:stream_lookup])
+        value = begin
+          context.evaluate(expression)
+        rescue Liquid::UndefinedVariable, Liquid::UndefinedDropMethod
+          nil
+        end
+        return value if value.is_a?(String) && cell[:stream_string_fast] && !@options[:liquid_resource_limits]
+        return value if numeric_for_excel?(value) || value == true || value == false
+        return date_time_to_serial(value) if date_time?(value)
+
+        return render_liquid_cell(cell, context, row)
+      end
+      render_cell_template(cell, context, nil, row)
+    end
+
+    # The compiled lookup already performed the original type-preserving probe.
+    # Go straight to Liquid on fallback so nil/Drop methods are read twice, as
+    # before, rather than probing a third time through render_cell_template.
+    def render_liquid_cell(cell, context, row)
+      ref = "#{cell[:col]}#{row}"
+      context.registers[:liquid_xlsx_source_row] = row
+      context.registers[:liquid_xlsx_source_cell] = ref
+      coerce_value(cached_parse(cell[:template]).render(context))
+    rescue Liquid::UndefinedVariable, Liquid::UndefinedDropMethod => e
+      raise MissingVariableError.new("Missing variable: #{e.message}", sheet: worksheet.name,
+                                                                       row: row, cell: ref, template: cell[:template])
+    rescue Liquid::Error => e
+      raise RenderError.new("Liquid render error: #{e.message}", sheet: worksheet.name,
+                                                                 row: row, cell: ref, template: cell[:template])
+    end
+
+    def static_cell(cell)
+      style = cell[:stream_style]
+      if cell[:type] == "s" && cell[:text]
+        %(#{style} t="s"><v>#{escape_text(cell[:value].to_s)}</v>)
+      elsif cell[:type] != "inlineStr" && cell[:value]
+        type = cell[:type] ? %( t="#{escape_attribute(cell[:type])}") : ""
+        %(#{style}#{type}><v>#{escape_text(cell[:value])}</v>)
+      elsif cell[:text].nil? || cell[:text].empty?
+        serialize_value("", style) if cell[:style]
+      else
+        serialize_value(cell[:text], style)
+      end
+    end
+
+    # Static cell content is compiled once; dynamic cells append directly into
+    # the reused row buffer, avoiding two XML-sized temporary strings per cell.
+    def serialize_value(value, style)
+      xml = style.dup
+      append_value(xml, value)
+      xml
+    end
+
+    def append_value(xml, value)
+      case value
+      when String
+        text = Worksheet.sanitize_text(value)
+        xml << ' t="inlineStr"><is><t'
+        xml << ' xml:space="preserve"' if text.match?(/\A\s|\s\z/) || text.include?("\n")
+        xml << ">" << escape_text(text) << "</t></is>"
+      when Numeric then xml << "><v>" << value.to_s << "</v>"
+      when true, false then xml << ' t="b"><v>' << (value ? "1" : "0") << "</v>"
+      else append_value(xml, value.to_s)
+      end
+    end
+
+    def escape_text(text)
+      return text unless text.match?(/[&<>\r]/)
+
+      text.gsub(/[&<>\r]/, TEXT_ESCAPES)
+    end
+
+    def escape_attribute(text)
+      text.to_s.gsub(/[&<>"\r\n\t]/, ATTRIBUTE_ESCAPES)
+    end
+  end
+end

data/lib/liquid_xlsx/tags/sheet_tag.rb

@@ -12,6 +12,10 @@ module LiquidXlsx
     # - +data+: Liquid expression — object to pass as local variable
     # - +as+: (optional) quoted string — local variable name, defaults to "item"
     class SheetTag < Liquid::Tag
+      # Exposed for Validator: it has to know which sheet the tag clones, to
+      # check that the sheet exists, without rendering the tag.
+      attr_reader :template_name, :as_name
+
       SYNTAX = /\A\s*(.+)\s*\z/
       # Known argument keys. Values are split on the positions of these keys
       # (not on whitespace) so that a name/data value may itself contain

data/lib/liquid_xlsx/template.rb

@@ -14,7 +14,12 @@ module LiquidXlsx
     # @param data [Hash, Liquid::Drop, #to_liquid] data to render (Hash/Drop,
     #   or any object whose `#to_liquid` returns one)
     # @param output_path [String]
-    def render_to_file(data, output_path)
+    def render_to_file(data, output_path, **options)
+      unless options.empty?
+        configured = self.class.new(@template_path, @options.merge(options))
+        return configured.render_to_file(data, output_path)
+      end
+
       package = render_package(data)
       package.write(output_path)
     rescue LiquidXlsx::Error
@@ -22,6 +27,8 @@ module LiquidXlsx
     rescue Zip::Error, Errno::ENOENT, Errno::EACCES, Errno::ENOTDIR, Errno::EISDIR => e
       raise OutputWriteError,
             "Failed to write output '#{output_path}': #{e.class}: #{e.message}"
+    ensure
+      package&.close_file_parts
     end
 
     # Render template and return binary content.
@@ -29,6 +36,8 @@ module LiquidXlsx
     #   or any object whose `#to_liquid` returns one)
     # @return [String] binary .xlsx content
     def render(data)
+      raise ArgumentError, "Streaming requires render_to_file; binary output is not supported" if @options[:streaming]
+
       package = render_package(data)
       package.to_binary
     rescue LiquidXlsx::Error
@@ -63,7 +72,10 @@ module LiquidXlsx
               "Failed to read template '#{@template_path}': #{e.class}: #{e.message}"
       end
 
+      workbook_rendered = true
       package
+    ensure
+      package&.close_file_parts unless workbook_rendered
     end
   end
 end

data/lib/liquid_xlsx/validation/block_boundary.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module LiquidXlsx
+  module Validation
+    # Whether a merged range crosses the boundary of a structural block.
+    #
+    # A merge that starts outside a `{% for %}` and ends inside it cannot be
+    # cloned per iteration, so the renderer refuses such templates. The rule
+    # lives here because two callers need it and they must agree: the renderer
+    # (which only sees blocks it actually executes) and the validator (which
+    # sees every block in the AST, executed or not).
+    module BlockBoundary
+      module_function
+
+      def crosses?(merge_start, merge_end, block_start, block_end)
+        # Merge entirely above or below the block
+        return false if merge_end < block_start || merge_start > block_end
+
+        # Merge strictly inside the block body (between the tag rows)
+        return false if merge_start > block_start && merge_end < block_end
+
+        # Merge entirely on a single structural tag row (it is consumed with it)
+        return false if merge_start == merge_end && (merge_start == block_start || merge_start == block_end)
+
+        # Any other overlap = crossing (including merges that swallow a tag row)
+        true
+      end
+    end
+  end
+end

data/lib/liquid_xlsx/validation/issue.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module LiquidXlsx
+  module Validation
+    # A single problem found in a template.
+    #
+    # `code` is a stable machine-readable symbol: host applications map it to
+    # their own localized text, so the English `message` here is a fallback for
+    # logs and specs, never the only thing a user can be shown.
+    Issue = Struct.new(:code, :severity, :sheet, :row, :cell, :message, :source,
+                       keyword_init: true) do
+      def error?
+        severity == :error
+      end
+
+      def warning?
+        severity == :warning
+      end
+
+      # Human-readable location: "Sheet1!B12", "Sheet1 row 12" or "Sheet1".
+      def location
+        return nil unless sheet
+
+        return "#{sheet}!#{cell}" if cell
+        return "#{sheet} row #{row}" if row
+
+        sheet
+      end
+
+      def to_h
+        { code: code, severity: severity, sheet: sheet, row: row, cell: cell,
+          message: message, source: source }
+      end
+    end
+  end
+end

data/lib/liquid_xlsx/validation/liquid_usage.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+module LiquidXlsx
+  module Validation
+    # Walks a parsed Liquid template and collects what it refers to: variable
+    # paths, filter names, and the names it binds itself.
+    #
+    # The point of doing this here rather than in the host application is that
+    # the host would have to parse every fragment a second time — and parse it
+    # with this gem's Liquid environment, or `{% sheet %}` and `{% image_tag %}`
+    # would come back as unknown tags. Handing back extracted usages keeps
+    # Liquid entirely on this side of the boundary.
+    class LiquidUsage
+      # Tags whose bodies Liquid exposes through an attachment rather than a
+      # nodelist ({% case %} branches).
+      def self.call(root)
+        collector = new
+        collector.walk(root)
+        collector
+      end
+
+      attr_reader :variables, :filters, :assigns, :locals
+
+      def initialize
+        @variables = []
+        @filters   = []
+        @assigns   = []
+        @locals    = []
+      end
+
+      def walk(node)
+        visit(node)
+        each_child(node) { |child| walk(child) }
+      end
+
+      private
+
+      def visit(node)
+        case node
+        when ::Liquid::Variable then visit_variable(node)
+        when ::Liquid::Assign   then visit_assign(node)
+        when ::Liquid::For      then visit_for(node)
+        when ::Liquid::TableRow then visit_for(node)
+        when ::Liquid::If       then visit_conditions(node.blocks)
+        when ::Liquid::Case     then visit_case(node)
+        when ::Liquid::Capture  then visit_capture(node)
+        end
+      end
+
+      def visit_variable(node)
+        collect_expression(node.name)
+        node.filters.each do |(name, *args)|
+          @filters << name.to_s
+          args.flatten.each { |arg| collect_expression(arg) }
+        end
+      end
+
+      def visit_assign(node)
+        @assigns << node.to.to_s
+        visit_variable(node.from) if node.from.is_a?(::Liquid::Variable)
+      end
+
+      # {% for item in list %} binds `item` (and `forloop`) for its body; the
+      # collection itself is an ordinary reference. `limit:`/`offset:` may be
+      # variables too.
+      def visit_for(node)
+        @locals << node.variable_name.to_s
+        @locals << "forloop"
+        collect_expression(node.collection_name)
+        collect_expression(node.limit)
+        collect_expression(node.from)
+      end
+
+      def visit_case(node)
+        collect_expression(node.left)
+        visit_conditions(node.blocks)
+      end
+
+      # Liquid does not expose the capture target, but a name captured and then
+      # used must not be reported as missing — read the ivar rather than let the
+      # host produce a false "unknown variable".
+      def visit_capture(node)
+        target = node.instance_variable_get(:@to)
+        @assigns << target.to_s if target
+      end
+
+      def visit_conditions(blocks)
+        Array(blocks).each do |condition|
+          next unless condition.respond_to?(:left)
+
+          collect_expression(condition.left)
+          collect_expression(condition.right)
+          visit_conditions([condition.child_condition]) if condition.child_condition
+        end
+      end
+
+      # Liquid's own children: BlockBody, nested tags, and the bodies {% case %}
+      # hangs off its conditions as attachments.
+      def each_child(node)
+        if node.respond_to?(:nodelist) && node.nodelist
+          node.nodelist.each { |child| yield child unless child.is_a?(::String) }
+        end
+
+        return unless node.is_a?(::Liquid::Case)
+
+        node.blocks.each do |condition|
+          attachment = condition.attachment
+          yield attachment if attachment
+        end
+      end
+
+      # A variable reference is a VariableLookup; literals and numbers are not
+      # references at all. Lookups that are themselves expressions (`items[i]`)
+      # end the path: what follows depends on a runtime value, so the caller is
+      # told the path is truncated instead of being handed a name to check.
+      def collect_expression(expression)
+        case expression
+        when ::Liquid::VariableLookup then collect_lookup(expression)
+        when ::Liquid::Variable       then visit_variable(expression)
+        when ::Liquid::RangeLookup
+          collect_expression(expression.instance_variable_get(:@start_obj))
+          collect_expression(expression.instance_variable_get(:@end_obj))
+        end
+      end
+
+      def collect_lookup(lookup)
+        root = lookup.name
+        # A lookup whose own name is an expression (`{{ [dynamic] }}`) has no
+        # static root to report.
+        return collect_expression(root) unless root.is_a?(::String)
+
+        segments  = []
+        truncated = false
+
+        Array(lookup.lookups).each do |segment|
+          unless segment.is_a?(::String)
+            collect_expression(segment)
+            truncated = true
+            break
+          end
+
+          segments << segment
+        end
+
+        @variables << VariableUse.new(root: root, segments: segments, truncated: truncated)
+      end
+    end
+  end
+end

data/lib/liquid_xlsx/validation/result.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module LiquidXlsx
+  module Validation
+    # Outcome of validating a template: what is wrong with it, and what it uses.
+    #
+    # Validation never raises for a broken template — a broken template is the
+    # expected input here, and a caller wants the whole list, not the first
+    # failure. Only programming errors (a missing file handed to the validator,
+    # for instance) still raise.
+    class Result
+      attr_reader :issues, :usages, :sheets
+
+      def initialize(issues: [], usages: [], sheets: [])
+        @issues = issues
+        @usages = usages
+        @sheets = sheets
+      end
+
+      def valid?
+        errors.empty?
+      end
+
+      def errors
+        issues.select(&:error?)
+      end
+
+      def warnings
+        issues.select(&:warning?)
+      end
+
+      def add(code:, severity: :error, sheet: nil, row: nil, cell: nil, message: nil, source: nil)
+        issues << Issue.new(code: code, severity: severity, sheet: sheet, row: row,
+                            cell: cell, message: message, source: source)
+        self
+      end
+
+      def add_usage(usage)
+        usages << usage
+        self
+      end
+
+      # Every variable reference in the workbook, deduplicated by path.
+      # Root names alone are rarely enough for a caller: `invoice.customer.name`
+      # and `invoice.customer.inn` are different questions to ask of a model.
+      def variables
+        usages.flat_map(&:variables).uniq(&:path)
+      end
+
+      # Root variable names the template expects to find in the data.
+      # Names bound by the template itself ({% for %}, {% assign %}) are not
+      # roots — they are excluded, otherwise every loop variable would look
+      # like a missing input.
+      def roots
+        bound = bound_names
+
+        variables.map(&:root).uniq.reject { |name| bound.include?(name) }
+      end
+
+      def filters
+        usages.flat_map(&:filters).uniq
+      end
+
+      # Names the template binds itself, in any sheet: loop variables, assigns,
+      # captures. Scope is deliberately ignored — a name bound anywhere counts
+      # as known everywhere. Tracking real scopes would let the validator report
+      # "unknown variable" for a name that is bound two rows above, and a false
+      # error costs more than a missed one here.
+      def bound_names
+        @bound_names ||= usages.flat_map { |usage| usage.assigns + usage.locals }.uniq
+      end
+
+      def to_h
+        { valid: valid?, issues: issues.map(&:to_h), sheets: sheets,
+          roots: roots, filters: filters,
+          variables: variables.map(&:path) }
+      end
+    end
+  end
+end