trevl 0.1.0

data/lib/trevl/processor.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+module Trevl
+  class Processor
+    attr_reader :trevl_content, :processed_content, :parse_errors
+
+    def initialize(trevl_content)
+      @trevl_content = trevl_content
+      @parse_errors = []
+      @processed_content = process_content
+    end
+
+    def self.from_hash(trevl_hash)
+      new(trevl_hash.to_yaml)
+    end
+
+    def components
+      @processed_content["components"]
+    end
+
+    def component_yaml_content
+      comps = @processed_content["components"]
+      stringified = if comps.is_a?(Array)
+        comps.map { |c| c.is_a?(Hash) ? CoreExt::Hash.deep_stringify_keys(c) : c }
+      else
+        comps
+      end
+      YAML.dump(stringified, canonical: false)
+    end
+
+    def full_yaml_content
+      YAML.dump(CoreExt::Hash.deep_stringify_keys(@processed_content), canonical: false)
+    end
+
+    def valid?
+      @parse_errors.empty?
+    end
+
+    private
+
+    def process_content
+      yaml_content = begin
+        YAML.safe_load(trevl_content, permitted_classes: [Hash, Symbol], aliases: true)
+      rescue Psych::SyntaxError => e
+        @parse_errors << "Error parsing TREVL YAML: #{e.message}"
+        raise ParseError, "Error parsing TREVL YAML: #{e.message}"
+      end
+
+      unless yaml_content.is_a?(Hash) || yaml_content.is_a?(Array)
+        @parse_errors << "TREVL content must be a Hash or Array"
+        raise ParseError, "TREVL content must be a Hash or Array"
+      end
+
+      # Support both top-level hash with "components" key and bare array of components
+      if yaml_content.is_a?(Array)
+        yaml_content = {"components" => yaml_content}
+      end
+
+      if yaml_content["components"].is_a?(Array)
+        yaml_content["components"] = apply_templates(yaml_content["components"])
+      end
+
+      yaml_content
+    rescue ParseError
+      raise
+    rescue => e
+      @parse_errors << "Error processing TREVL content: #{e.message}"
+      raise ParseError, "Error processing TREVL content: #{e.message}"
+    end
+
+    def apply_templates(components)
+      components.map { |component| apply_template_to_component(component) }
+    end
+
+    def apply_template_to_component(component)
+      return component unless component.is_a?(Hash) && component["template"]
+
+      template_name = component["template"]
+      template_hash = Trevl.template_store.find(template_name)
+      return component unless template_hash
+
+      deep_merge_with_overwrite(template_hash, component.except("template"))
+    end
+
+    def deep_merge_with_overwrite(template_hash, component_hash)
+      return component_hash unless template_hash.is_a?(Hash)
+
+      result = {}
+
+      component_hash.each do |key, component_val|
+        result[key] = if template_hash.key?(key) && template_hash[key].is_a?(Hash) && component_val.is_a?(Hash)
+          deep_merge_with_overwrite(template_hash[key], component_val)
+        else
+          component_val
+        end
+      end
+
+      template_hash.each do |key, template_val|
+        next if component_hash.key?(key)
+        result[key] = template_val
+      end
+
+      result
+    end
+  end
+end

data/lib/trevl/renderer/data_fetcher.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Trevl
+  class Renderer
+    # Normalizes API responses and resolves API parameters.
+    #
+    #   fetcher = DataFetcher.new
+    #   fetcher.coerce_payload([{x: 1}])
+    #   # => {"data" => [{"x" => 1}], "meta" => {}}
+    #
+    class DataFetcher
+      def coerce_payload(result)
+        case result
+        when Array
+          {"data" => result, "meta" => {}}
+        when Hash
+          data = result["data"] || result[:data]
+          meta = result["meta"] || result[:meta] || {}
+          meta = {} unless meta.is_a?(Hash)
+          if data.is_a?(Array)
+            {"data" => data, "meta" => CoreExt::Hash.deep_stringify_keys(meta)}
+          else
+            {"data" => [], "meta" => CoreExt::Hash.deep_stringify_keys(meta)}
+          end
+        else
+          {"data" => [], "meta" => {}}
+        end
+      end
+
+      def merge_api_params(component, query_params)
+        api_params = component["api_parameters"] || component["api-parameters"] || {}
+        api_params = CoreExt::Hash.deep_stringify_keys(api_params) if api_params.is_a?(Hash)
+        api_params = resolve_placeholders(api_params, query_params)
+
+        overrides = query_params["param_override"] || query_params[:param_override]
+        if overrides.is_a?(Hash)
+          overrides = CoreExt::Hash.deep_stringify_keys(overrides)
+          api_params = CoreExt::Hash.deep_merge(api_params, overrides)
+        end
+
+        api_params
+      end
+
+      private
+
+      def resolve_placeholders(params, query_params)
+        params.each_with_object({}) do |(key, value), result|
+          if value.is_a?(String) && value.start_with?("$")
+            param_key = value.delete_prefix("$")
+            raw = query_params[param_key] || query_params[param_key.to_sym]
+            resolved = raw.is_a?(Array) ? raw.first : raw
+            result[key] = resolved unless resolved.nil?
+          elsif value.is_a?(Hash)
+            resolved = resolve_placeholders(value, query_params)
+            result[key] = resolved unless resolved.empty?
+          else
+            result[key] = value
+          end
+        end
+      end
+    end
+  end
+end

data/lib/trevl/renderer/ref_parser.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Trevl
+  class Renderer
+    # Parses $-prefixed variable references in TREVL YAML.
+    #
+    #   parser = RefParser.new
+    #   parser.parse("salary.data.q50")
+    #   # => {resource: nil, endpoint: "salary", scope: :data, keys: ["q50"]}
+    #
+    class RefParser
+      def parse(ref_without_dollar)
+        parts = ref_without_dollar.to_s.split(".")
+        resource = nil
+
+        if parts.length >= 4 && %w[data meta].include?(parts[2])
+          resource = parts.shift
+        end
+
+        endpoint = parts[0].to_s
+        return {resource: resource, endpoint: endpoint, scope: :data, keys: [], legacy: true} if parts.length < 2
+
+        case parts[1]
+        when "data"
+          {resource: resource, endpoint: endpoint, scope: :data, keys: parts[2..] || [], legacy: false}
+        when "meta"
+          {resource: resource, endpoint: endpoint, scope: :meta, keys: parts[2..] || [], legacy: false}
+        else
+          {resource: resource, endpoint: endpoint, scope: :data, keys: [parts.last], legacy: true}
+        end
+      end
+
+      def ref_key(parsed)
+        [parsed[:resource], parsed[:endpoint]].compact.join("/")
+      end
+
+      def ref_key_for(ref_without_dollar)
+        ref_key(parse(ref_without_dollar))
+      end
+
+      def extract_references(obj)
+        refs = []
+        case obj
+        when Hash then obj.each_value { |v| refs.concat(extract_references(v)) }
+        when Array then obj.each { |v| refs.concat(extract_references(v)) }
+        when String then refs << obj.delete_prefix("$") if obj.start_with?("$")
+        end
+        refs.uniq
+      end
+
+      def dig(obj, keys)
+        return obj if keys.nil? || keys.empty?
+        keys.reduce(obj) do |acc, key|
+          return nil if acc.nil?
+          next nil unless acc.is_a?(Hash)
+          acc[key.to_s] || acc[key.to_sym]
+        end
+      end
+    end
+  end
+end

data/lib/trevl/renderer/transform.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+require "execjs"
+
+module Trevl
+  class Renderer
+    # Applies computed fields and postprocess transformations to data rows.
+    #
+    #   transform = Transform.new(ref_parser)
+    #   transform.apply_computed_fields(rows, payloads, component)
+    #
+    class Transform
+      def initialize(ref_parser)
+        @refs = ref_parser
+      end
+
+      # Applies computed fields (per-row JavaScript expressions) to all rows.
+      def apply_computed_fields(rows_by_ref_key, payload_by_ref_key, component)
+        computeds = collect_computed_fields(component)
+        return rows_by_ref_key unless computeds.any?
+
+        rows_by_ref_key.each do |rk, rows|
+          rows.each do |row|
+            computeds.each do |computed|
+              arg_names, arg_values = resolve_arguments(computed, row, rk, payload_by_ref_key)
+              js = "(function(#{arg_names.join(", ")}) { return #{computed["code"]} })(#{arg_values.map(&:to_json).join(", ")})"
+              row[computed["name"]] = ExecJS.eval(js)
+            end
+          end
+        end
+
+        rows_by_ref_key
+      end
+
+      # Applies postprocess JavaScript to the full dataset.
+      def apply_postprocess(rows_by_ref_key, component, warnings)
+        code = component["postprocess"]
+        return rows_by_ref_key unless code.is_a?(String) && !code.strip.empty?
+
+        rows_by_ref_key.each do |rk, rows|
+          js = "(function() { var $result = #{rows.to_json}; #{code.strip.chomp(";")}; return $result; })()"
+          begin
+            result = ExecJS.eval(js)
+            if result.is_a?(Array)
+              rows_by_ref_key[rk] = result
+            else
+              warnings << "Postprocess for '#{component["id"]}' returned #{result.class} instead of Array"
+              Trevl.logger.warn("[Trevl::Renderer] Postprocess returned non-Array for '#{component["id"]}'")
+            end
+          rescue ExecJS::Error => e
+            raise RenderError, "Postprocess JavaScript error for '#{component["id"]}': #{e.message}"
+          end
+        end
+
+        rows_by_ref_key
+      end
+
+      private
+
+      def collect_computed_fields(component)
+        computeds = Array(component["computed"])
+
+        if component["queries"].is_a?(Array)
+          existing_names = computeds.map { |c| c["name"] }.compact
+          component["queries"].each do |q|
+            Array(q["computed"]).each do |c|
+              computeds << c unless existing_names.include?(c["name"])
+            end
+          end
+        end
+
+        computeds
+      end
+
+      def resolve_arguments(computed, row, rk, payload_by_ref_key)
+        args = computed["arguments"] || {}
+        arg_names = []
+        arg_values = []
+
+        args.each do |name, ref|
+          arg_names << name
+          if ref.is_a?(String) && ref.start_with?("$")
+            p = @refs.parse(ref.delete_prefix("$"))
+            p_key = @refs.ref_key(p)
+            ep_meta = (payload_by_ref_key[p_key] || {})["meta"] || {}
+            val = if p[:scope] == :meta
+              @refs.dig(ep_meta, p[:keys])
+            elsif p_key == rk
+              @refs.dig(row, p[:keys])
+            else
+              first = payload_by_ref_key[p_key]&.dig("data")&.first
+              @refs.dig(first, p[:keys])
+            end
+            arg_values << val
+          else
+            arg_values << ref
+          end
+        end
+
+        [arg_names, arg_values]
+      end
+    end
+  end
+end

data/lib/trevl/renderer.rb

@@ -0,0 +1,345 @@
+# frozen_string_literal: true
+
+require_relative "renderer/ref_parser"
+require_relative "renderer/data_fetcher"
+require_relative "renderer/transform"
+
+module Trevl
+  class Renderer
+    attr_reader :component, :query_params, :data_sources, :inline_source, :raw_data, :warnings
+
+    def initialize(component_hash, query_params = {}, data_sources: {}, data: nil)
+      @component = component_hash
+      @query_params = query_params || {}
+      @data_sources = normalize_data_sources(data_sources)
+      @inline_source = data ? DataSource::Static.new(data: data) : nil
+      @raw_data = {}
+      @warnings = []
+      @refs = RefParser.new
+      @fetcher = DataFetcher.new
+      @transform = Transform.new(@refs)
+    end
+
+    def render
+      case component["type"]
+      when "text" then render_text
+      when "filter" then has_api? ? render_filter : nil
+      when "table" then has_api? ? render_table : nil
+      when "score" then has_api? ? render_score : nil
+      when "chart" then has_api? ? render_chart : nil
+      end
+    end
+
+    # Public class methods kept for backward compatibility and testing
+    def self.parse_trevl_ref(ref) = RefParser.new.parse(ref)
+    def self.coerce_fetch_payload(result) = DataFetcher.new.coerce_payload(result)
+    def self.deep_dig_indifferent(obj, keys) = RefParser.new.dig(obj, keys)
+
+    private
+
+    def has_api?
+      api_name = component["api"]
+      return true if inline_source
+      !api_name.nil? && !api_name.to_s.strip.empty?
+    end
+
+    # Resolution order per component: explicitly injected data_sources win,
+    # then inline data (when given, it answers any api name and components
+    # without one), then the global registry. Injected and inline sources
+    # never touch shared state (thread-safe by construction).
+    def data_source
+      @data_source ||= data_sources[normalize_source_name(component["api"])] ||
+        inline_source ||
+        DataSource.for(component["api"])
+    end
+
+    def normalize_data_sources(sources)
+      (sources || {}).each_with_object({}) do |(name, source), normalized|
+        normalized[normalize_source_name(name)] = source.is_a?(Class) ? source.new : source
+      end
+    end
+
+    def normalize_source_name(name)
+      name.to_s.strip.downcase
+    end
+
+    def request_params
+      @request_params ||= @fetcher.merge_api_params(component, query_params)
+    end
+
+    # --- Data pipeline ---
+
+    def fetch_and_prepare(references_source)
+      references = @refs.extract_references(references_source)
+      parsed_refs = references.map { |ref| @refs.parse(ref) }
+      ref_keys = parsed_refs.map { |p| @refs.ref_key(p) }.uniq
+
+      payload_by_ref_key = {}
+      ref_keys.each do |key|
+        parsed = parsed_refs.find { |p| @refs.ref_key(p) == key }
+        payload_by_ref_key[key] = @fetcher.coerce_payload(
+          data_source.fetch(parsed[:endpoint], request_params, resource: parsed[:resource])
+        )
+        warn_no_data(key) if payload_by_ref_key[key]["data"].empty?
+      end
+
+      @raw_data = payload_by_ref_key.transform_values { |p| CoreExt::Hash.deep_dup(p) }
+      payload_by_ref_key
+    end
+
+    def transform_rows(payload_by_ref_key)
+      rows_by_ref_key = payload_by_ref_key.transform_values { |p| p["data"] }
+      counts_before = rows_by_ref_key.transform_values(&:length)
+
+      rows_by_ref_key = @transform.apply_computed_fields(rows_by_ref_key, payload_by_ref_key, component)
+      rows_by_ref_key = @transform.apply_postprocess(rows_by_ref_key, component, @warnings)
+
+      rows_by_ref_key.each do |key, rows|
+        warn_postprocess_emptied(key, counts_before[key]) if rows.empty? && counts_before[key].to_i > 0
+      end
+
+      rows_by_ref_key
+    end
+
+    # --- Component renderers ---
+
+    def render_chart
+      payload_by_ref_key = fetch_and_prepare(component["highchartsData"])
+      rows_by_ref_key = transform_rows(payload_by_ref_key)
+      highcharts = build_highcharts(rows_by_ref_key, payload_by_ref_key)
+
+      build_result("chart", "highchartsData" => highcharts, "display" => component["display"])
+    end
+
+    def render_score
+      display = component["display"] || {}
+      payload_by_ref_key = fetch_and_prepare(display)
+      rows_by_ref_key = payload_by_ref_key.transform_values { |p| p["data"] }
+      rows_by_ref_key = @transform.apply_computed_fields(rows_by_ref_key, payload_by_ref_key, component)
+      resolved_display = resolve_refs(display, payload_by_ref_key, rows_by_ref_key)
+
+      build_result("score", "display" => resolved_display)
+    end
+
+    def render_table
+      return nil unless component["tableData"]
+
+      table_def = component["tableData"]
+      column_defs = Array(table_def["columns"])
+
+      payload_by_ref_key = fetch_and_prepare(column_defs)
+      rows_by_ref_key = transform_rows(payload_by_ref_key)
+
+      primary_rk = payload_by_ref_key.keys.first
+      rows = primary_rk ? (rows_by_ref_key[primary_rk] || []) : []
+
+      output_rows = rows.map do |row|
+        column_defs.each_with_object({}) do |col_def, output_row|
+          identifier = col_def["identifier"]
+          next if identifier.nil? || identifier.empty?
+          output_row[identifier] = resolve_cell_value(col_def["value"], row, payload_by_ref_key)
+        end
+      end
+
+      {"id" => component["id"], "type" => "table",
+       "tableData" => {"headers" => table_def["headers"], "columns" => output_rows}.compact}
+    end
+
+    def render_filter
+      filter_defs = Array(component["filters"])
+      return nil if filter_defs.empty?
+
+      rows_cache = {}
+
+      resolved_filters = filter_defs.map do |filter_def|
+        options_template = filter_def["options"] || {}
+        references = @refs.extract_references(options_template)
+        parsed = references.map { |ref| @refs.parse(ref) }.first
+        rk = parsed ? @refs.ref_key(parsed) : nil
+
+        payload = if parsed
+          rows_cache[rk] ||= @fetcher.coerce_payload(
+            data_source.fetch(parsed[:endpoint], request_params, resource: parsed[:resource])
+          )
+        else
+          {"data" => [], "meta" => {}}
+        end
+
+        options = payload["data"].map do |row|
+          options_template.transform_values { |tmpl| resolve_filter_value(tmpl, row) }
+        end
+
+        filter_def.merge("options" => options).except("queries")
+      end
+
+      {"id" => component["id"], "type" => "filter", "filters" => resolved_filters}
+    end
+
+    def render_text
+      {"id" => component["id"], "type" => "text", "text" => component["text"],
+       "subheader" => component["subheader"], "body" => component["body"]}.compact
+    end
+
+    # --- Highcharts builder ---
+
+    def build_highcharts(rows_by_ref_key, payload_by_ref_key)
+      template = CoreExt::Hash.deep_dup(component["highchartsData"])
+      series_templates = template.delete("series") || []
+      all_categories = nil
+
+      built_series = series_templates.map do |s_tmpl|
+        data_template = s_tmpl.delete("data") || {}
+        rk = detect_ref_key(data_template, rows_by_ref_key)
+        rows = rk ? (rows_by_ref_key[rk] || []) : []
+        meta = rk ? ((payload_by_ref_key[rk] || {})["meta"] || {}) : {}
+
+        x_field_keys = extract_category_keys(data_template["x"])
+        data_points = build_data_points(data_template, rows, meta, rk)
+        all_categories = rows.map { |row| @refs.dig(row, x_field_keys) } if x_field_keys
+
+        resolve_series_fields(s_tmpl, rows, meta: meta).merge("data" => data_points)
+      end
+
+      template["xAxis"] = inject_categories(template["xAxis"], all_categories) if all_categories
+      resolve_refs(template, payload_by_ref_key).merge("series" => built_series)
+    end
+
+    def build_data_points(data_template, rows, meta, current_ref_key)
+      rows.each_with_index.map do |row, idx|
+        data_template.each_with_object({}) do |(key, val), point|
+          point[key] = resolve_data_value(key, val, row, idx, meta: meta, current_ref_key: current_ref_key)
+        end
+      end
+    end
+
+    # --- Reference resolution ---
+
+    def resolve_refs(obj, payload_by_ref_key, rows_override = nil)
+      case obj
+      when Hash then obj.transform_values { |v| resolve_refs(v, payload_by_ref_key, rows_override) }
+      when Array then obj.map { |v| resolve_refs(v, payload_by_ref_key, rows_override) }
+      when String
+        return obj unless obj.start_with?("$")
+        parsed = @refs.parse(obj.delete_prefix("$"))
+        rk = @refs.ref_key(parsed)
+        payload = payload_by_ref_key[rk] || {}
+        if parsed[:scope] == :meta
+          @refs.dig(payload["meta"] || {}, parsed[:keys]) || obj
+        else
+          rows = rows_override&.dig(rk) || payload["data"] || []
+          @refs.dig(rows.first, parsed[:keys]) || obj
+        end
+      else obj
+      end
+    end
+
+    def resolve_data_value(key, val, row, index, meta:, current_ref_key:)
+      if val.is_a?(Hash)
+        return val.transform_values { |v| resolve_data_value(nil, v, row, index, meta: meta, current_ref_key: current_ref_key) }
+      end
+      return val unless val.is_a?(String) && val.start_with?("$")
+
+      parsed = @refs.parse(val.delete_prefix("$"))
+      rk = @refs.ref_key(parsed)
+
+      if rk != current_ref_key
+        return index if key == "x"
+        return row[parsed[:endpoint]] || row[parsed[:endpoint].to_sym] if parsed[:legacy] && parsed[:keys].empty?
+        return nil
+      end
+
+      return index if key == "x"
+      return @refs.dig(meta, parsed[:keys]) if parsed[:scope] == :meta
+      @refs.dig(row, parsed[:keys])
+    end
+
+    def resolve_series_fields(s_tmpl, rows, meta:)
+      s_tmpl.each_with_object({}) do |(key, val), result|
+        if val.is_a?(String) && val.start_with?("$")
+          parsed = @refs.parse(val.delete_prefix("$"))
+          row_keys = (parsed[:legacy] && parsed[:keys].empty?) ? [parsed[:endpoint]] : parsed[:keys]
+          resolved = if parsed[:scope] == :meta
+            @refs.dig(meta, parsed[:keys])
+          else
+            @refs.dig(rows.first, row_keys)
+          end
+          result[key] = resolved.nil? ? val : resolved
+        else
+          result[key] = val
+        end
+      end
+    end
+
+    def resolve_cell_value(value_ref, row, payload_by_ref_key)
+      if value_ref.is_a?(String) && value_ref.start_with?("$")
+        parsed = @refs.parse(value_ref.delete_prefix("$"))
+        rk = @refs.ref_key(parsed)
+        ep_meta = (payload_by_ref_key[rk] || {})["meta"] || {}
+        row_keys = (parsed[:legacy] && parsed[:keys].empty?) ? [parsed[:endpoint]] : parsed[:keys]
+        (parsed[:scope] == :meta) ? @refs.dig(ep_meta, parsed[:keys]) : @refs.dig(row, row_keys)
+      else
+        value_ref
+      end
+    end
+
+    def resolve_filter_value(tmpl, row)
+      return tmpl unless tmpl.is_a?(String)
+
+      tmpl.gsub(/\$([a-zA-Z0-9_-]+)\.data\.([a-zA-Z0-9_.]+)/) do
+        path = Regexp.last_match(2).split(".")
+        @refs.dig(row, path) || Regexp.last_match(0)
+      end.gsub(/\$([a-zA-Z0-9_-]+)\.([a-zA-Z0-9_]+)/) do
+        field = Regexp.last_match(2)
+        next Regexp.last_match(0) if %w[data meta].include?(field)
+        (row[field] || row[field.to_sym]) || Regexp.last_match(0)
+      end
+    end
+
+    # --- Helpers ---
+
+    def detect_ref_key(data_template, rows_by_ref_key = {})
+      first_candidate = nil
+      data_template.each_value do |val|
+        next unless val.is_a?(String) && val.start_with?("$")
+        rk = @refs.ref_key_for(val.delete_prefix("$"))
+        return rk if rows_by_ref_key.key?(rk)
+        first_candidate ||= rk
+      end
+      first_candidate
+    end
+
+    def extract_category_keys(x_ref)
+      return nil unless x_ref.is_a?(String) && x_ref.start_with?("$")
+      p = @refs.parse(x_ref.delete_prefix("$"))
+      return nil if p[:scope] == :meta
+      return [p[:endpoint]] if p[:legacy] && p[:keys].empty?
+      p[:keys].empty? ? nil : p[:keys]
+    end
+
+    def inject_categories(x_axis, categories)
+      case x_axis
+      when Array then x_axis.map { |axis| axis.merge("categories" => categories) }
+      when Hash then [x_axis.merge("categories" => categories)]
+      else [{"categories" => categories}]
+      end
+    end
+
+    def build_result(type, fields = {})
+      result = {"id" => component["id"], "type" => type}.merge(fields).compact
+      result["warnings"] = @warnings if @warnings.any?
+      result
+    end
+
+    def warn_no_data(rk)
+      msg = "No data returned for '#{rk}'"
+      @warnings << msg
+      Trevl.logger.warn("[Trevl::Renderer] #{msg} (component: #{component["id"]})")
+    end
+
+    def warn_postprocess_emptied(rk, count_before)
+      msg = "Postprocess removed all #{count_before} rows for '#{rk}'"
+      @warnings << msg
+      Trevl.logger.warn("[Trevl::Renderer] #{msg} (component: #{component["id"]})")
+    end
+  end
+end