varar-core 0.5.0

data/lib/varar/core/plan.rb

@@ -0,0 +1,286 @@
+# frozen_string_literal: true
+
+require 'varar/core/span'
+require 'varar/core/ast'
+require 'varar/core/cell_diff'
+require 'varar/core/diagnostics'
+require 'varar/core/matcher'
+require 'varar/core/sentences'
+
+module Varar
+  module Core
+    DocString = Data.define(:content, :content_type, :span)
+
+    PlannedStep = Data.define(:text, :match_span, :param_spans, :step_def, :args, :formats, :data_table,
+                              :doc_string) do
+      def initialize(text:, match_span:, param_spans:, step_def:, args:, formats: [], data_table: nil,
+                     doc_string: nil)
+        super
+      end
+    end
+
+    HeaderBinding = Data.define(:match_span, :param_spans, :step_def)
+
+    PlannedExample = Data.define(:name, :scope_stack, :span, :steps, :header_binding, :row_checks,
+                                 :expected_outcome, :expected_error_message) do
+      def initialize(name:, scope_stack:, span:, steps:, header_binding: nil, row_checks: nil,
+                     expected_outcome: nil, expected_error_message: nil)
+        super
+      end
+    end
+
+    ExecutionPlan = Data.define(:var_doc, :examples, :diagnostics)
+
+    # Produce an ExecutionPlan from a VarDoc + Registry: match step expressions
+    # against every text block, attach trailing tables/fences, detect
+    # header-bound tables, and collect diagnostics. Port of plan.ts.
+    module Plan
+      BlockPlan = Data.define(:steps, :ambiguities)
+      Ambiguity = Data.define(:match_start, :match_end, :candidates)
+
+      module_function
+
+      def plan(var_doc, registry)
+        examples = []
+        diagnostics = []
+
+        var_doc.examples.each do |ex|
+          had_ambiguous = false
+          steps_by_block = {}
+
+          # Pass 1: plan each text-bearing block.
+          ex.body.each_with_index do |block, idx|
+            next unless %w[paragraph list_item blockquote].include?(block.kind)
+
+            result = plan_block(block.text, registry)
+
+            result.ambiguities.each do |collision|
+              span = lift_span(var_doc.source, block, collision.match_start, collision.match_end)
+              cp_start = Offsets.cp_index_for_utf16(block.text, collision.match_start)
+              cp_end = Offsets.cp_index_for_utf16(block.text, collision.match_end)
+              diagnostics << Diagnostics.ambiguous_match(
+                AmbiguousInput.new(
+                  text: block.text[cp_start...cp_end],
+                  span: span,
+                  candidates: collision.candidates.map do |c|
+                    Candidate.new(
+                      expression: c.expression,
+                      source_file: c.step_def.expression_source_file,
+                      source_line: c.step_def.expression_source_line
+                    )
+                  end
+                )
+              )
+              had_ambiguous = true
+            end
+
+            next unless !had_ambiguous && !result.steps.empty?
+
+            steps_by_block[idx] = result.steps.map do |hit|
+              PlannedStep.new(
+                text: Offsets.utf16_slice(block.text, hit.match_start, hit.match_end),
+                match_span: lift_span(var_doc.source, block, hit.match_start, hit.match_end),
+                param_spans: hit.param_spans.map { |p| lift_span(var_doc.source, block, p.start, p.end) },
+                step_def: hit.step_def,
+                args: hit.args,
+                formats: hit.formats
+              )
+            end
+          end
+
+          # Header-bound table detection.
+          bound = had_ambiguous ? nil : detect_header_bound(ex, steps_by_block, var_doc.source)
+          if bound
+            table, binding_step, header_spans = bound
+            header_binding = HeaderBinding.new(
+              match_span: binding_step.match_span,
+              param_spans: header_spans,
+              step_def: binding_step.step_def
+            )
+            table.rows.each do |row|
+              row_object = {}
+              table.header.cells.each_with_index do |cell_name, i|
+                row_object[cell_name] = i < row.cells.length ? row.cells[i] : ''
+              end
+              row_step = PlannedStep.new(
+                text: binding_step.text,
+                match_span: row.span,
+                param_spans: binding_step.param_spans,
+                step_def: binding_step.step_def,
+                args: binding_step.args + [row_object],
+                formats: binding_step.formats
+              )
+              row_checks = table.header.cells.each_with_index.map do |cell_name, i|
+                RowCheck.new(
+                  column: cell_name,
+                  value: i < row.cells.length ? row.cells[i] : '',
+                  span: i < row.cell_spans.length ? row.cell_spans[i] : row.span
+                )
+              end
+              examples << PlannedExample.new(
+                name: row.cells.join(' / '),
+                scope_stack: ex.scope_stack + [binding_step.text],
+                span: row.span,
+                steps: [row_step],
+                header_binding: header_binding,
+                row_checks: row_checks
+              )
+            end
+            next
+          end
+
+          # Error fence detection.
+          error_fence = ex.body.find { |b| b.kind == 'fence' && b.info == 'error' }
+
+          # Pass 2: attach trailing table / fence to the last step of a block.
+          attachments = {}
+          (1...ex.body.length).each do |idx|
+            here = ex.body[idx]
+            if here.kind == 'table' && steps_by_block.key?(idx - 1)
+              _prev_data, prev_doc = attachments[idx - 1] || [nil, nil]
+              attachments[idx - 1] = [here, prev_doc]
+            elsif here.kind == 'fence' && here.info != 'error' && steps_by_block.key?(idx - 1)
+              prev_data, = attachments[idx - 1] || [nil, nil]
+              attachments[idx - 1] = [
+                prev_data,
+                DocString.new(content: here.body, content_type: here.info, span: here.body_span)
+              ]
+            end
+          end
+
+          # Pass 3: rebuild the final step list, applying attachments.
+          final_steps = []
+          (0...ex.body.length).each do |idx|
+            block_steps = steps_by_block[idx] || []
+            attach = attachments[idx]
+            block_steps.each_with_index do |step, s_idx|
+              if s_idx == block_steps.length - 1 && attach
+                data_table, doc_string = attach
+                final_steps << PlannedStep.new(
+                  text: step.text, match_span: step.match_span, param_spans: step.param_spans,
+                  step_def: step.step_def, args: step.args, formats: step.formats,
+                  data_table: data_table, doc_string: doc_string
+                )
+              else
+                final_steps << step
+              end
+            end
+          end
+
+          runnable_steps = had_ambiguous ? [] : final_steps
+
+          diagnostics << Diagnostics.error_fence_without_step(error_fence.span) if error_fence && runnable_steps.empty?
+
+          next if final_steps.empty? && !had_ambiguous
+
+          expected_outcome = nil
+          expected_error_message = nil
+          if error_fence
+            expected_outcome = 'fail'
+            msg = error_fence.body.strip
+            expected_error_message = msg unless msg.empty?
+          end
+
+          examples << PlannedExample.new(
+            name: derive_example_name(ex.body),
+            scope_stack: ex.scope_stack,
+            span: ex.span,
+            steps: runnable_steps,
+            expected_outcome: expected_outcome,
+            expected_error_message: expected_error_message
+          )
+        end
+
+        ExecutionPlan.new(var_doc: var_doc, examples: examples, diagnostics: diagnostics)
+      end
+
+      def plan_block(text, registry)
+        all_steps = []
+        all_ambiguities = []
+
+        Sentences.split_sentences(text).each do |sentence|
+          hits = Matcher.find_hits(sentence.text, registry)
+          adjusted = hits.map do |h|
+            Hit.new(
+              expression: h.expression,
+              step_def: h.step_def,
+              match_start: h.match_start + sentence.start_offset,
+              match_end: h.match_end + sentence.start_offset,
+              args: h.args,
+              param_spans: h.param_spans.map do |p|
+                ParamSpan.new(start: p.start + sentence.start_offset, end: p.end + sentence.start_offset)
+              end,
+              formats: h.formats
+            )
+          end
+          resolved = Matcher.resolve_hits(adjusted)
+          if resolved.kind == 'ambiguous'
+            resolved.collisions.each do |c|
+              all_ambiguities << Ambiguity.new(match_start: c.match_start, match_end: c.match_end,
+                                               candidates: c.candidates)
+            end
+          elsif !resolved.steps.empty?
+            all_steps.concat(resolved.steps)
+          end
+        end
+
+        BlockPlan.new(steps: all_steps, ambiguities: all_ambiguities)
+      end
+
+      # Whole-word, case-sensitive start index of +word+ in +haystack+, or nil.
+      def word_offset(haystack, word)
+        m = /(?<![^\W_])#{Regexp.escape(word)}(?![^\W_])/.match(haystack)
+        m&.begin(0)
+      end
+
+      def detect_header_bound(ex, steps_by_block, source)
+        body = ex.body
+        (1...body.length).each do |idx|
+          here = body[idx]
+          next unless here.kind == 'table'
+
+          above = body[idx - 1]
+          next unless %w[paragraph list_item blockquote].include?(above.kind)
+
+          steps = steps_by_block[idx - 1]
+          next if steps.nil? || steps.empty?
+
+          header_cells = here.header.cells
+          offsets = header_cells.map { |cell| word_offset(above.text, cell) }
+          next if offsets.any?(&:nil?)
+
+          utf16_offsets = offsets.map { |o| Offsets.to_utf16_offset(above.text, o) }
+          header_spans = header_cells.each_index.map do |i|
+            lift_span(source, above, utf16_offsets[i], utf16_offsets[i] + Offsets.utf16_len(header_cells[i]))
+          end
+          return [here, steps.last, header_spans]
+        end
+        nil
+      end
+
+      def derive_example_name(body)
+        primary = body.find { |b| %w[paragraph list_item blockquote].include?(b.kind) }
+        return '' if primary.nil?
+
+        name = primary.text.gsub(/\s+/, ' ').strip
+        name.sub(/[.!?]$/, '')
+      end
+
+      def lift_segment_offset(segment_map, text_offset)
+        best = segment_map.first
+        segment_map.each { |entry| best = entry if entry.text_offset <= text_offset }
+        raise 'empty segment_map' if best.nil?
+
+        best.source_offset + (text_offset - best.text_offset)
+      end
+
+      def lift_span(source, block, block_start, block_end)
+        return block.span unless %w[paragraph list_item blockquote].include?(block.kind)
+
+        start_src = lift_segment_offset(block.segment_map, block_start)
+        end_src = lift_segment_offset(block.segment_map, block_end)
+        Offsets.span_from_offsets(source, start_src, end_src)
+      end
+    end
+  end
+end

data/lib/varar/core/registry.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require 'cucumber/cucumber_expressions/cucumber_expression'
+require 'cucumber/cucumber_expressions/parameter_type'
+require 'cucumber/cucumber_expressions/parameter_type_registry'
+
+module Varar
+  module Core
+    # One registered step definition. `compiled` is the CucumberExpression.
+    StepRegistration = Data.define(
+      :expression, :expression_source_file, :expression_source_line,
+      :handler, :compiled, :kind
+    )
+
+    # A registry of step definitions plus the shared cucumber ParameterTypeRegistry
+    # and per-type display formatters (kept beside it because ParameterType
+    # can't carry one). Port of registry.ts.
+    Registry = Data.define(:steps, :parameter_types, :formats)
+
+    module Registries
+      module_function
+
+      def create_registry
+        Registry.new(
+          steps: [],
+          parameter_types: Cucumber::CucumberExpressions::ParameterTypeRegistry.new,
+          formats: {}
+        )
+      end
+
+      # Compile +expression+ and append it. Returns a new Registry (the
+      # ParameterTypeRegistry is shared by reference but never mutated here).
+      # Raises on duplicate expressions, mirroring the TS message.
+      def add_step(registry, expression:, expression_source_file:, expression_source_line:, handler:, kind: nil)
+        duplicate = registry.steps.find { |s| s.expression == expression }
+        if duplicate
+          raise "duplicate step definition for \"#{expression}\" at " \
+                "#{duplicate.expression_source_file}:#{duplicate.expression_source_line} and " \
+                "#{expression_source_file}:#{expression_source_line}"
+        end
+
+        compiled = Cucumber::CucumberExpressions::CucumberExpression.new(expression, registry.parameter_types)
+        reg = StepRegistration.new(
+          expression: expression,
+          expression_source_file: expression_source_file,
+          expression_source_line: expression_source_line,
+          handler: handler,
+          compiled: compiled,
+          kind: kind
+        )
+        registry.with(steps: registry.steps + [reg])
+      end
+
+      # Register a custom parameter type with the shared ParameterTypeRegistry
+      # (mutated in place, same as TS, so previously compiled expressions gain
+      # the new type). Returns the same Registry unless a +format+ is given.
+      def define_parameter_type(registry, name:, regexp:, parse: nil, use_for_snippets: true,
+                                prefer_for_regexp_match: false, format: nil)
+        regexps = regexp.is_a?(Array) ? regexp : [regexp]
+        transformer = parse || ->(*groups) { groups[0] }
+        # `type` is return-type metadata only (used by snippet generation, never
+        # by matching, transformation, or any conformance artifact). The Ruby
+        # gem rejects a nil type (Python's accepts None), so pass Object.
+        pt = Cucumber::CucumberExpressions::ParameterType.new(
+          name, regexps, Object, transformer, use_for_snippets, prefer_for_regexp_match
+        )
+        registry.parameter_types.define_parameter_type(pt)
+        return registry if format.nil?
+
+        registry.with(formats: registry.formats.merge(name => format))
+      end
+    end
+  end
+end

data/lib/varar/core/scanner.rb

@@ -0,0 +1,346 @@
+# frozen_string_literal: true
+
+require 'varar/core/span'
+require 'varar/core/ast'
+require 'varar/core/table_cells'
+
+module Varar
+  module Core
+    # Markdown block scanner. Port of scanner.ts. All offsets count UTF-16
+    # code units; split_lines advances by utf16_len(line) + 1 per newline, and
+    # code-point indices from String#index are converted to UTF-16 before use.
+    module Scanner
+      RawLine = Data.define(:text, :start_offset, :end_offset)
+
+      # Regexes — verbatim ports of the TS constants (`#` escaped as `\#` so
+      # Ruby does not read `#{...}` as interpolation).
+      THEMATIC_RE = /^\s*([-*_])(\s*\1){2,}\s*$/
+      UL_RE = /^(\s*)([-*+])\s+(.*)$/
+      OL_RE = /^(\s*)(\d+)([.)])\s+(.*)$/
+      BQ_RE = /^>\s?(.*)$/
+      FENCE_RE = /^(`{3,})\s*(\S*)\s*$/
+      ROW_RE = /^\|(.+)\|\s*$/
+      DELIM_RE = /^\|\s*:?-+:?\s*(\|\s*:?-+:?\s*)*\|\s*$/
+      HEADING_RE = /^(\#{1,6})\s+(.*?)(?:\s+\#+)?\s*$/
+      PARA_HEADING_RE = /^\#{1,6}\s+/
+
+      module_function
+
+      # Scan +source+ into an immutable Array of Block nodes.
+      def scan(source, plugins = [])
+        blocks = []
+        lines = split_lines(source)
+
+        i = 0
+        while i < lines.length
+          line = lines[i]
+          if line.text.strip.empty?
+            i += 1
+            next
+          end
+
+          matched = run_plugins(source, lines, i, plugins)
+          if matched
+            blocks << matched[0]
+            i = matched[1]
+            next
+          end
+
+          fence_result = try_fence(source, lines, i)
+          if fence_result
+            blocks << fence_result[0]
+            i = fence_result[1]
+            next
+          end
+
+          table_result = try_table(source, lines, i)
+          if table_result
+            blocks << table_result[0]
+            i = table_result[1]
+            next
+          end
+
+          thematic = try_thematic(source, line)
+          if thematic
+            blocks << thematic
+            i += 1
+            next
+          end
+
+          bq_result = try_blockquote(source, lines, i)
+          if bq_result
+            blocks << bq_result[0]
+            i = bq_result[1]
+            next
+          end
+
+          heading = try_heading(source, line)
+          if heading
+            blocks << heading
+            i += 1
+            next
+          end
+
+          list_item = try_list_item(source, line)
+          if list_item
+            blocks << list_item
+            i += 1
+            next
+          end
+
+          paragraph, next_i = consume_paragraph(source, lines, i, plugins)
+          blocks << paragraph
+          i = next_i
+        end
+
+        blocks
+      end
+
+      def run_plugins(source, lines, start_idx, plugins)
+        plugins.each do |p|
+          r = p.try_scan(source: source, lines: lines, start_idx: start_idx)
+          return r if r
+        end
+        nil
+      end
+
+      # Split +source+ into RawLines with UTF-16 start/end offsets.
+      def split_lines(source)
+        out = []
+        start_u16 = 0
+        current_u16 = 0
+        start_cp = 0
+
+        source.each_char.with_index do |ch, cp_i|
+          if ch == "\n"
+            out << RawLine.new(text: source[start_cp...cp_i], start_offset: start_u16, end_offset: current_u16)
+            start_u16 = current_u16 + 1 # '\n' is BMP → 1 UTF-16 unit
+            start_cp = cp_i + 1
+          end
+          current_u16 += ch.ord > 0xFFFF ? 2 : 1
+        end
+
+        out << RawLine.new(text: source[start_cp..] || '', start_offset: start_u16, end_offset: current_u16)
+        out
+      end
+
+      def try_thematic(source, line)
+        return nil unless THEMATIC_RE.match?(line.text)
+
+        ThematicBreak.new(span: Offsets.span_from_offsets(source, line.start_offset, line.end_offset))
+      end
+
+      def try_heading(source, line)
+        m = HEADING_RE.match(line.text)
+        return nil unless m
+
+        Heading.new(
+          level: m[1].length,
+          text: (m[2] || '').strip,
+          span: Offsets.span_from_offsets(source, line.start_offset, line.end_offset)
+        )
+      end
+
+      def try_list_item(source, line)
+        if (ul = UL_RE.match(line.text))
+          text = ul[3] || ''
+          marker_start = line.start_offset + Offsets.utf16_len(ul[1] || '')
+          marker_end = marker_start + Offsets.utf16_len(ul[2] || '')
+          cp_idx = line.text.index(text)
+          text_start = line.start_offset + Offsets.to_utf16_offset(line.text, cp_idx)
+          return ListItem.new(
+            text: text,
+            span: Offsets.span_from_offsets(source, line.start_offset, line.end_offset),
+            segment_map: [SegmentOffset.new(text_offset: 0, source_offset: text_start)],
+            ordered: false,
+            marker_span: Offsets.span_from_offsets(source, marker_start, marker_end)
+          )
+        end
+
+        if (ol = OL_RE.match(line.text))
+          text = ol[4] || ''
+          marker_start = line.start_offset + Offsets.utf16_len(ol[1] || '')
+          marker_end = marker_start + Offsets.utf16_len(ol[2] || '') + Offsets.utf16_len(ol[3] || '')
+          cp_idx = line.text.index(text)
+          text_start = line.start_offset + Offsets.to_utf16_offset(line.text, cp_idx)
+          return ListItem.new(
+            text: text,
+            span: Offsets.span_from_offsets(source, line.start_offset, line.end_offset),
+            segment_map: [SegmentOffset.new(text_offset: 0, source_offset: text_start)],
+            ordered: true,
+            marker_span: Offsets.span_from_offsets(source, marker_start, marker_end)
+          )
+        end
+
+        nil
+      end
+
+      def try_blockquote(source, lines, start_idx)
+        return nil if start_idx >= lines.length
+
+        first = lines[start_idx]
+        m = BQ_RE.match(first.text)
+        return nil unless m
+
+        first_segment = m[1] || ''
+        cp_idx = first.text.index(first_segment)
+        segments = [first_segment]
+        segment_map = [
+          SegmentOffset.new(
+            text_offset: 0,
+            source_offset: first.start_offset + Offsets.to_utf16_offset(first.text, cp_idx)
+          )
+        ]
+        joined_text_offset = Offsets.utf16_len(first_segment)
+
+        i = start_idx + 1
+        end_offset = first.end_offset
+        while i < lines.length
+          ln = lines[i]
+          next_m = BQ_RE.match(ln.text)
+          break unless next_m
+
+          segment = next_m[1] || ''
+          cp_idx2 = ln.text.index(segment)
+          joined_text_offset += 1 # newline separator
+          segment_map << SegmentOffset.new(
+            text_offset: joined_text_offset,
+            source_offset: ln.start_offset + Offsets.to_utf16_offset(ln.text, cp_idx2)
+          )
+          segments << segment
+          joined_text_offset += Offsets.utf16_len(segment)
+          end_offset = ln.end_offset
+          i += 1
+        end
+
+        [
+          Blockquote.new(
+            text: segments.join("\n"),
+            span: Offsets.span_from_offsets(source, first.start_offset, end_offset),
+            segment_map: segment_map
+          ),
+          i
+        ]
+      end
+
+      def consume_paragraph(source, lines, start_idx, plugins)
+        raise 'invariant: start_idx out of range' if start_idx >= lines.length
+
+        first = lines[start_idx]
+        end_idx = start_idx
+        while end_idx + 1 < lines.length
+          candidate_idx = end_idx + 1
+          candidate = lines[candidate_idx]
+          break if candidate.text.strip.empty?
+          break if PARA_HEADING_RE.match?(candidate.text)
+          break if UL_RE.match?(candidate.text)
+          break if OL_RE.match?(candidate.text)
+          break if BQ_RE.match?(candidate.text)
+          break if FENCE_RE.match?(candidate.text)
+          break if ROW_RE.match?(candidate.text)
+          break if THEMATIC_RE.match?(candidate.text)
+          break if run_plugins(source, lines, candidate_idx, plugins)
+
+          end_idx += 1
+        end
+
+        last = lines[end_idx]
+        start_offset = first.start_offset
+        end_offset = last.end_offset
+        [
+          Paragraph.new(
+            text: Offsets.utf16_slice(source, start_offset, end_offset),
+            span: Offsets.span_from_offsets(source, start_offset, end_offset),
+            segment_map: [SegmentOffset.new(text_offset: 0, source_offset: start_offset)]
+          ),
+          end_idx + 1
+        ]
+      end
+
+      def try_fence(source, lines, start_idx)
+        return nil if start_idx >= lines.length
+
+        start = lines[start_idx]
+        open_m = FENCE_RE.match(start.text)
+        return nil unless open_m
+
+        fence_marker = open_m[1] || ''
+        info = (open_m[2] || '').strip
+
+        i = start_idx + 1
+        body_start = nil
+        body_end = nil
+        end_offset = start.end_offset
+
+        while i < lines.length
+          ln = lines[i]
+          close_m = FENCE_RE.match(ln.text)
+          if close_m && (close_m[1] || '').length >= fence_marker.length
+            end_offset = ln.end_offset
+            break
+          end
+          body_start = ln.start_offset if body_start.nil?
+          body_end = ln.end_offset + 1 # +1 to include the '\n' after this line
+          i += 1
+        end
+
+        body = body_start.nil? || body_end.nil? ? '' : Offsets.utf16_slice(source, body_start, body_end)
+
+        fallback = start.end_offset
+        body_span = Offsets.span_from_offsets(source, body_start || fallback, body_end || fallback)
+        [
+          Fence.new(
+            info: info,
+            body: body,
+            body_span: body_span,
+            span: Offsets.span_from_offsets(source, start.start_offset, end_offset)
+          ),
+          i + 1
+        ]
+      end
+
+      def try_table(source, lines, start_idx)
+        return nil if start_idx + 1 >= lines.length
+
+        header_line = lines[start_idx]
+        delim_line = lines[start_idx + 1]
+        return nil unless ROW_RE.match?(header_line.text)
+        return nil unless DELIM_RE.match?(delim_line.text)
+
+        header_cells, header_cell_spans = TableCells.parse_row_cells(header_line.text, header_line.start_offset,
+                                                                     source)
+        header = Row.new(
+          cells: header_cells,
+          cell_spans: header_cell_spans,
+          span: Offsets.span_from_offsets(source, header_line.start_offset, header_line.end_offset)
+        )
+
+        rows = []
+        i = start_idx + 2
+        while i < lines.length
+          ln = lines[i]
+          break unless ROW_RE.match?(ln.text)
+
+          cells, cell_spans = TableCells.parse_row_cells(ln.text, ln.start_offset, source)
+          rows << Row.new(
+            cells: cells,
+            cell_spans: cell_spans,
+            span: Offsets.span_from_offsets(source, ln.start_offset, ln.end_offset)
+          )
+          i += 1
+        end
+
+        last_row = rows.last
+        end_offset = last_row ? last_row.span.end_offset : delim_line.end_offset
+        [
+          Table.new(
+            span: Offsets.span_from_offsets(source, header_line.start_offset, end_offset),
+            header: header,
+            rows: rows
+          ),
+          i
+        ]
+      end
+    end
+  end
+end