lex-transformer 0.1.4 → 0.3.0

data/lib/legion/extensions/transformer/client.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+require_relative 'engines/registry'
+require_relative 'helpers/schema_validator'
+require_relative 'definitions'
+
+module Legion
+  module Extensions
+    module Transformer
+      class Client
+        def transform(payload:, transformation: nil, engine: nil, schema: nil, engine_options: {}, name: nil)
+          return transform_by_name(name: name, payload: payload, engine_options: engine_options) if transformation.nil? && name
+
+          eng = resolve_engine(engine, transformation)
+          rendered = eng.render(transformation, payload, **engine_options)
+          rendered = parse_rendered(rendered)
+
+          return rendered if rendered.is_a?(Hash) && rendered[:success] == false
+
+          if schema
+            validation = Helpers::SchemaValidator.validate(schema: schema, data: rendered)
+            return { success: false, status: 'transformer.validation_failed', errors: validation[:errors] } unless validation[:valid]
+          end
+
+          { success: true, result: rendered }
+        end
+
+        def transform_chain(steps:, payload:)
+          result = payload.dup
+          steps.each do |step|
+            eng = resolve_engine(step[:engine], step[:transformation])
+            step_opts = step[:engine_options] || {}
+            rendered = eng.render(step[:transformation], result, **step_opts)
+            rendered = parse_rendered(rendered)
+
+            return rendered if rendered.is_a?(Hash) && rendered[:success] == false
+
+            if step[:schema]
+              validation = Helpers::SchemaValidator.validate(schema: step[:schema], data: rendered)
+              return { success: false, status: 'transformer.validation_failed', errors: validation[:errors] } unless validation[:valid]
+            end
+
+            if rendered.is_a?(Hash)
+              result = result.merge({ args: rendered }.merge(rendered))
+            else
+              result[:args] = rendered
+            end
+          end
+          { success: true, result: result }
+        end
+
+        private
+
+        def transform_by_name(name:, payload:, engine_options: {})
+          definition = Definitions.fetch(name)
+          return { success: false, error: 'definition_not_found' } unless definition
+
+          if definition[:conditions] && conditioner_available?
+            cond_result = evaluate_conditions(definition[:conditions], payload)
+            return { success: false, reason: 'conditions_not_met' } unless cond_result
+          end
+
+          merged_opts = Definitions.merge_options(definition, **engine_options)
+
+          transform(
+            transformation: definition[:transformation],
+            payload:        payload,
+            engine:         definition[:engine],
+            schema:         definition[:schema],
+            engine_options: merged_opts
+          )
+        end
+
+        def conditioner_available?
+          defined?(Legion::Extensions::Conditioner::Client)
+        end
+
+        def evaluate_conditions(conditions, payload)
+          client = Legion::Extensions::Conditioner::Client.new
+          result = client.evaluate(conditions: conditions, values: payload)
+          result[:passed]
+        rescue StandardError
+          true
+        end
+
+        def resolve_engine(engine_name, transformation)
+          if engine_name
+            Engines::Registry.fetch(engine_name)
+          else
+            Engines::Registry.detect(transformation)
+          end
+        end
+
+        def parse_rendered(rendered)
+          return rendered unless rendered.is_a?(String)
+
+          Legion::JSON.load(rendered)
+        rescue StandardError
+          rendered
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/transformer/definitions.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Transformer
+      class Definitions
+        class << self
+          def fetch(name)
+            defns = load_definitions
+            return nil unless defns&.key?(name)
+
+            symbolize_definition(defns[name])
+          end
+
+          def names
+            defns = load_definitions
+            return [] unless defns
+
+            defns.keys
+          end
+
+          def merge_options(definition, **overrides)
+            base = definition[:engine_options] || {}
+            base.merge(overrides)
+          end
+
+          private
+
+          def load_definitions
+            return nil unless defined?(Legion::Settings)
+
+            Legion::Settings.dig('lex-transformer', 'definitions')
+          rescue StandardError
+            nil
+          end
+
+          def symbolize_definition(raw)
+            defn = {}
+            defn[:transformation] = raw['transformation'] || raw[:transformation]
+            defn[:engine] = (raw['engine'] || raw[:engine])&.to_sym
+            defn[:engine_options] = symbolize_hash(raw['engine_options'] || raw[:engine_options] || {})
+            defn[:schema] = raw['schema'] || raw[:schema]
+            defn[:conditions] = raw['conditions'] || raw[:conditions]
+            defn
+          end
+
+          def symbolize_hash(hash)
+            return {} unless hash.is_a?(Hash)
+
+            hash.transform_keys(&:to_sym)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/transformer/engines/base.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Transformer
+      module Engines
+        class Base
+          def render(template, payload, **_opts)
+            raise NotImplementedError, "#{self.class}#render must be implemented"
+          end
+
+          def name
+            raise NotImplementedError, "#{self.class}#name must be implemented"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/transformer/engines/erb.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require 'tilt'
+require_relative 'base'
+
+module Legion
+  module Extensions
+    module Transformer
+      module Engines
+        class Erb < Base
+          def name
+            :erb
+          end
+
+          def render(template, payload, **_opts)
+            tilt_template = Tilt['erb'].new { template }
+            variables = build_variables(template, payload)
+            tilt_template.render(Object.new, variables)
+          end
+
+          private
+
+          def build_variables(template, payload)
+            variables = { **payload }
+            variables[:crypt] = Legion::Crypt if defined?(Legion::Crypt) && template.include?('crypt')
+            variables[:settings] = Legion::Settings if defined?(Legion::Settings) && template.include?('settings')
+            variables[:cache] = Legion::Cache if defined?(Legion::Cache) && template.include?('cache')
+            if payload.key?(:task_id) && template.include?('task') && defined?(Legion::Data::Model::Task)
+              variables[:task] = Legion::Data::Model::Task[payload[:task_id]]
+            end
+            variables
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/transformer/engines/jsonpath.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require_relative 'base'
+
+module Legion
+  module Extensions
+    module Transformer
+      module Engines
+        class Jsonpath < Base
+          def name
+            :jsonpath
+          end
+
+          def render(expression, payload, **_opts)
+            extract(expression, payload)
+          end
+
+          private
+
+          def extract(path, data)
+            path = path.delete_prefix('$.') if path.start_with?('$.')
+            segments = path.split('.')
+            result = data
+            segments.each do |segment|
+              case result
+              when Hash
+                key = result.key?(segment.to_sym) ? segment.to_sym : segment
+                result = result[key]
+              when Array
+                result = result[segment.to_i]
+              else
+                return nil
+              end
+            end
+            result
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/transformer/engines/liquid.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require_relative 'base'
+
+module Legion
+  module Extensions
+    module Transformer
+      module Engines
+        class Liquid < Base
+          def name
+            :liquid
+          end
+
+          def render(template, payload, **_opts)
+            require 'liquid'
+            liquid_template = ::Liquid::Template.parse(template)
+            liquid_template.render(stringify_keys(payload))
+          end
+
+          private
+
+          def stringify_keys(hash)
+            hash.each_with_object({}) do |(k, v), result|
+              result[k.to_s] = v.is_a?(Hash) ? stringify_keys(v) : v
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/transformer/engines/llm.rb

@@ -0,0 +1,153 @@
+# frozen_string_literal: true
+
+require 'timeout'
+require_relative 'base'
+
+module Legion
+  module Extensions
+    module Transformer
+      module Engines
+        class Llm < Base
+          AUTH_ERRORS = /auth|credentials|forbidden|api.key|unauthorized/i
+          RETRY_SLEEP = 1
+
+          def name
+            :llm
+          end
+
+          def render(prompt, payload, **opts)
+            raise 'Legion::LLM is not available or not started' unless llm_available?
+
+            @last_raw = nil
+            resolved = resolve_options(opts)
+            max_retries = resolved.delete(:max_retries)
+            attempts = 0
+
+            loop do
+              result = attempt_render(prompt, payload, resolved, attempts)
+              return result unless result == :retry
+
+              attempts += 1
+              return build_failure if attempts > max_retries
+
+              sleep(RETRY_SLEEP)
+            end
+          end
+
+          private
+
+          def llm_available?
+            defined?(Legion::LLM) && Legion::LLM.respond_to?(:started?) && Legion::LLM.started?
+          end
+
+          def resolve_options(opts)
+            defaults = settings_defaults
+            resolved = defaults.merge(opts)
+            resolved[:max_retries] = (resolved[:max_retries] || 1).to_i
+            resolved
+          end
+
+          def settings_defaults
+            return {} unless defined?(Legion::Settings)
+
+            settings = begin
+              Legion::Settings.dig('lex-transformer', 'llm')
+            rescue StandardError
+              nil
+            end
+            return {} unless settings.is_a?(Hash)
+
+            settings.transform_keys(&:to_sym)
+          end
+
+          def attempt_render(prompt, payload, opts, attempt)
+            chat = call_llm(prompt, payload, opts, attempt)
+            content = extract_response(chat)
+            validate_json(content)
+            content
+          rescue Timeout::Error, IOError, Errno::ECONNREFUSED, Errno::ECONNRESET
+            :retry
+          rescue ::JSON::ParserError
+            @last_raw = content
+            :retry
+          rescue RuntimeError => e
+            raise if auth_error?(e)
+
+            { success: false, error: e.class.to_s, message: e.message }
+          rescue StandardError => e
+            { success: false, error: e.class.to_s, message: e.message }
+          end
+
+          def call_llm(prompt, payload, opts, attempt)
+            context = Legion::JSON.dump(payload)
+
+            return call_structured(prompt, context, opts) if opts[:structured] && opts[:schema] && structured_available?
+
+            full_prompt = build_prompt(prompt, context, attempt)
+            full_prompt = inject_schema_into_prompt(full_prompt, opts[:schema]) if opts[:structured] && opts[:schema]
+
+            llm_opts = build_llm_opts(opts)
+            Legion::LLM.chat(message: full_prompt, **llm_opts)
+          end
+
+          def structured_available?
+            Legion::LLM.respond_to?(:structured)
+          end
+
+          def call_structured(prompt, context, opts)
+            llm_opts = build_llm_opts(opts)
+            Legion::LLM.structured(
+              message: "#{prompt}\n\nPayload:\n```json\n#{context}\n```",
+              schema:  opts[:schema],
+              **llm_opts
+            )
+          end
+
+          def inject_schema_into_prompt(prompt, schema)
+            schema_json = Legion::JSON.dump(schema)
+            "#{prompt}\n\nYour response MUST conform to this JSON schema:\n```json\n#{schema_json}\n```"
+          end
+
+          def build_prompt(prompt, context, attempt)
+            base = "#{prompt}\n\nPayload:\n```json\n#{context}\n```\n\nRespond with valid JSON only. No explanation, no markdown fences."
+            return base unless attempt.positive?
+
+            "#{base}\n\nIMPORTANT: Your previous response was not valid JSON. Return ONLY a valid JSON object or array, nothing else."
+          end
+
+          def extract_response(chat)
+            content = chat.respond_to?(:content) ? chat.content : chat.to_s
+            content = content.strip
+            content = content.sub(/\A```(?:json)?\n?/, '').sub(/\n?```\z/, '') if content.start_with?('```')
+            content
+          end
+
+          def validate_json(content)
+            ::JSON.parse(content)
+          end
+
+          def auth_error?(error)
+            AUTH_ERRORS.match?(error.message)
+          end
+
+          def build_llm_opts(opts)
+            llm_opts = {}
+            llm_opts[:model] = opts[:model] if opts[:model]
+            llm_opts[:provider] = opts[:provider] if opts[:provider]
+            llm_opts[:temperature] = opts[:temperature] if opts[:temperature]
+            llm_opts[:system_prompt] = opts[:system_prompt] if opts[:system_prompt]
+            llm_opts
+          end
+
+          def build_failure
+            if @last_raw
+              { success: false, error: 'invalid_json', message: 'max retries exhausted', raw: @last_raw }
+            else
+              { success: false, error: 'timeout', message: 'max retries exhausted' }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/transformer/engines/registry.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require_relative 'erb'
+require_relative 'static'
+require_relative 'liquid'
+require_relative 'jsonpath'
+require_relative 'llm'
+
+module Legion
+  module Extensions
+    module Transformer
+      module Engines
+        class Registry
+          ENGINES = {}.freeze
+
+          class << self
+            def register(engine_class)
+              @engines ||= {}
+              instance = engine_class.new
+              @engines[instance.name] = instance
+              @engines[instance.name.to_s] = instance
+            end
+
+            def fetch(name)
+              @engines ||= {}
+              @engines[name.to_sym] || @engines[name.to_s] || raise(ArgumentError, "Unknown engine: #{name}")
+            end
+
+            def detect(template)
+              if template.is_a?(String) && (template.include?('<%') || template.include?('%>'))
+                fetch(:erb)
+              else
+                fetch(:static)
+              end
+            end
+
+            def reset!
+              @engines = {}
+            end
+          end
+
+          register(Erb)
+          register(Static)
+          register(Liquid)
+          register(Jsonpath)
+          register(Llm)
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/transformer/engines/static.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require_relative 'base'
+
+module Legion
+  module Extensions
+    module Transformer
+      module Engines
+        class Static < Base
+          def name
+            :static
+          end
+
+          def render(template, _payload, **_opts)
+            template
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/transformer/helpers/schema_validator.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Transformer
+      module Helpers
+        class SchemaValidator
+          class << self
+            def validate(schema:, data:)
+              return { valid: true } if schema.nil? || schema.empty?
+
+              errors = []
+              data ||= {}
+
+              check_required_keys(schema[:required_keys], data, errors)
+              check_types(schema[:types], data, errors)
+
+              errors.empty? ? { valid: true } : { valid: false, errors: errors }
+            end
+
+            private
+
+            def check_required_keys(required_keys, data, errors)
+              return if required_keys.nil? || required_keys.empty?
+
+              required_keys.each do |key|
+                errors << "missing required key: #{key}" unless data.key?(key)
+              end
+            end
+
+            def check_types(types, data, errors)
+              return if types.nil? || types.empty?
+
+              types.each do |key, expected_type|
+                next unless data.key?(key)
+
+                actual = data[key]
+                next if actual.is_a?(expected_type)
+
+                errors << "#{key} expected #{expected_type}, got #{actual.class}"
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end