lex-transformer 0.1.4 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 24dbbb177015cc67b411b4992d4b220082ae85fc3740a233439ba983ab496736
-  data.tar.gz: 5b836b34ed00f6937773638ea9dc9056e224f64b785ceb55b03bb0381c3ab899
+  metadata.gz: d7f4f19cb531c6f7cb50302bb3f2aab973b55976bb05d1b870ac2ad4f7bdb9a0
+  data.tar.gz: a445f592b23aae48a5510a7c6f17ee3cf0ea0a29dd3c9d67bc405b5b208a0c72
 SHA512:
-  metadata.gz: b0b9b63fa282d53dff4b096a585788cec55440b178eeaa6b2b2b93ebd5caa7323b671f135e0a85e8f6bf3a970ce651fd1054f5b3305c7ff707afdf500911e524
-  data.tar.gz: fa7c078497b01e4a89ebfa10401e38f79a8914e2778e5737ede513e7e17571cd6b481ebf02b61b478bf753db4d2edc2c4da1acbc515c343db8ab33e0faf9690c
+  metadata.gz: 21096cee7c407f8df2aa5307e3fe71e35008103a798bc7f9c9cd8dfd1724728af7d65fbeeada8b5df657c412bdcfd44c60fd100efb2a3f63815c6615e63f9917
+  data.tar.gz: 4e6348dc31ae9217b34040010ecf8b613b8b0e97ae1995854a3f30c821bce8a5b6d190bb283ad9a06e305fe128b68bd8a3533772451d608c55912659c1970ec9

data/.github/workflows/ci.yml

@@ -0,0 +1,16 @@
+name: CI
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  ci:
+    uses: LegionIO/.github/.github/workflows/ci.yml@main
+
+  release:
+    needs: ci
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    uses: LegionIO/.github/.github/workflows/release.yml@main
+    secrets:
+      rubygems-api-key: ${{ secrets.RUBYGEMS_API_KEY }}

data/.gitignore

@@ -9,3 +9,5 @@
 
 # rspec failure tracking
 .rspec_status
+
+Gemfile.lock

data/.rubocop.yml

@@ -1,33 +1,49 @@
+AllCops:
+  TargetRubyVersion: 3.4
+  NewCops: enable
+  SuggestExtensions: false
+
 Layout/LineLength:
-  Max: 120
+  Max: 160
+Layout/SpaceAroundEqualsInParameterDefault:
+  EnforcedStyle: space
+Layout/HashAlignment:
+  EnforcedHashRocketStyle: table
+  EnforcedColonStyle: table
+
 Metrics/MethodLength:
   Max: 50
 Metrics/ClassLength:
   Max: 1500
+Metrics/ModuleLength:
+  Max: 1500
 Metrics/BlockLength:
-  Max: 50
-Metrics/PerceivedComplexity:
-  Max: 20
-Metrics/CyclomaticComplexity:
-  Max: 117
+  Max: 40
+  Exclude:
+    - 'spec/**/*'
 Metrics/AbcSize:
-  Max: 52
-Layout/SpaceAroundEqualsInParameterDefault:
-  EnforcedStyle: space
-Style/SymbolArray:
-  Enabled: true
-Layout/HashAlignment:
-  EnforcedHashRocketStyle: table
-  EnforcedColonStyle: table
+  Max: 60
+Metrics/CyclomaticComplexity:
+  Max: 15
+Metrics/PerceivedComplexity:
+  Max: 17
+Metrics/ParameterLists:
+  Enabled: false
+
 Style/Documentation:
   Enabled: false
-AllCops:
-  TargetRubyVersion: 2.5
-  NewCops: enable
-  SuggestExtensions: false
+Style/SymbolArray:
+  Enabled: true
 Style/FrozenStringLiteralComment:
-  Enabled: false
+  Enabled: true
+  EnforcedStyle: always
+
 Naming/FileName:
   Enabled: false
-Style/ClassAndModuleChildren:
-  Enabled: false
+Naming/PredicateMethod:
+  Enabled: false
+Naming/PredicatePrefix:
+  Enabled: false
+
+Gemspec/DevelopmentDependencies:
+  Enabled: false

data/CHANGELOG.md

@@ -0,0 +1,35 @@
+# Changelog
+
+## [0.2.1] - 2026-03-17
+
+### Added
+- LLM transform engine (`engines/llm.rb`) — provider-agnostic natural language transformation via `Legion::LLM.chat`
+- Registered as `:llm` in engine registry (explicit only, no auto-detection)
+
+## [0.2.0] - 2026-03-17
+
+### Added
+- Pluggable engine system with abstract `Engines::Base` and `Engines::Registry`
+- 4 template engines: ERB (extracted from runner), Static (JSON passthrough), Liquid, JSONPath (dot-notation extraction)
+- `Engines::Registry.detect` auto-selects ERB or static engine based on template content
+- `render_transformation` accepts optional `engine:` keyword to force a specific engine
+- `Helpers::SchemaValidator` validates transformed output against declared schemas (required keys, type checks)
+- `transform` accepts optional `schema:` parameter for post-render validation
+- `transform_chain` method for sequential multi-step pipelines with per-step engine selection and schema validation
+- Standalone `Client` class for framework-independent usage with `transform` and `transform_chain` methods
+- SimpleCov coverage reporting
+- Modern packaging: grouped test dependencies, `require_relative` in gemspec, `rubocop-rspec`
+
+### Fixed
+- `dispatch_multiplied` payload mutation bug: `new_payload = payload` replaced with `new_payload = payload.dup`
+- Spec filename typo: `tranformer_spec.rb` renamed to `transformer_spec.rb`
+
+### Changed
+- ERB rendering logic extracted from runner into `Engines::Erb` class
+- `build_template_variables` moved to `Engines::Erb` as private `build_variables`
+- Runner no longer directly requires `tilt` (delegated to `Engines::Erb`)
+
+## [0.1.4] - 2026-03-13
+
+### Added
+- Initial release

data/CLAUDE.md

@@ -0,0 +1,79 @@
+# lex-transformer: Payload Transformation Engine for LegionIO
+
+**Repository Level 3 Documentation**
+- **Parent**: `/Users/miverso2/rubymine/legion/extensions-core/CLAUDE.md`
+- **Grandparent**: `/Users/miverso2/rubymine/legion/CLAUDE.md`
+
+## Purpose
+
+Legion Extension that transforms task payloads between services in a relationship chain. Uses ERB template-based transformation (via the `tilt` gem) to map data from one task's output into the format expected by the next task's input. Supports single-hash output (1:1 dispatch) and array output (fan-out/multiply). Requires `legion-data` (`data_required? true`).
+
+**GitHub**: https://github.com/LegionIO/lex-transformer
+**License**: MIT
+**Version**: 0.2.1
+
+## Architecture
+
+```
+Legion::Extensions::Transformer
+├── Actors/
+│   └── Transform          # Subscription actor consuming transform requests
+├── Runners/
+│   └── Transform          # Executes template-based payload transformation
+│       ├── transform             # Entry point: render + dispatch
+│       ├── render_transformation # ERB rendering via tilt, or plain JSON if no ERB tags
+│       ├── build_template_variables # Inject crypt/settings/cache/task into ERB scope
+│       ├── dispatch_transformed  # Route Hash (single) or Array (fan-out) results
+│       ├── dispatch_multiplied   # Fan-out: create a new task per array element
+│       └── send_task             # Publish transformed payload to next runner
+└── Transport/
+    ├── Exchanges/Task     # Publishes to the task exchange
+    ├── Queues/Transform   # Subscribes to transformation queue
+    └── Messages/Message   # Transform request message format
+```
+
+## Key Files
+
+| Path | Purpose |
+|------|---------|
+| `lib/legion/extensions/transformer.rb` | Entry point (`data_required? true`) |
+| `lib/legion/extensions/transformer/runners/transform.rb` | Core transformation logic |
+| `lib/legion/extensions/transformer/actors/transform.rb` | AMQP subscription actor |
+| `lib/legion/extensions/transformer/transport.rb` | Transport setup |
+
+## Template Variables Available in ERB
+
+| Variable | Available when |
+|----------|---------------|
+| All payload keys | Always (splatted into template scope) |
+| `crypt` | Template string contains `'crypt'` |
+| `settings` | Template string contains `'settings'` |
+| `cache` | Template string contains `'cache'` |
+| `task` | Template string contains `'task'` and payload has `task_id` |
+
+## Dispatch Behavior
+
+- **Hash result**: Updates task to `transformer.succeeded`, dispatches single task, updates to `task.queued`
+- **Array result**: Fan-out via `dispatch_multiplied` - creates a new task record per element, dispatches each, marks original as `task.multiplied`
+- **Plain JSON** (no ERB tags): Parsed directly without template rendering
+
+## Dependencies
+
+| Gem | Purpose |
+|-----|---------|
+| `tilt` (>= 2.3) | Template engine abstraction for ERB rendering |
+| `legion-data` | Required - task record creation for fan-out |
+
+## Testing
+
+```bash
+bundle install
+bundle exec rspec
+bundle exec rubocop
+```
+
+Spec files: `spec/legion/extensions/tranformer_spec.rb` (note: typo in filename), `spec/legion/extensions/transform_runner_spec.rb`
+
+---
+
+**Maintained By**: Matthew Iverson (@Esity)

data/Dockerfile

@@ -6,4 +6,4 @@ RUN apk update && apk add build-base tzdata gcc git
 
 COPY . ./
 RUN gem install lex-transformer legion-data --no-document --no-prerelease
-CMD ruby --jit $(which legionio)
+CMD ruby --yjit $(which legionio)

data/Gemfile

@@ -1,3 +1,16 @@
+# frozen_string_literal: true
+
 source 'https://rubygems.org'
 
 gemspec
+
+group :test do
+  gem 'base64'
+  gem 'liquid', '~> 5.0'
+  gem 'rake'
+  gem 'rspec', '~> 3.13'
+  gem 'rspec_junit_formatter'
+  gem 'rubocop', '~> 1.75'
+  gem 'rubocop-rspec'
+  gem 'simplecov'
+end

data/README.md

@@ -1,38 +1,140 @@
-# Legion::Extensions::Transformer
+# lex-transformer
 
-Used to transform data to match the action for a relationship
+Payload transformation engine for [LegionIO](https://github.com/LegionIO/LegionIO). Transforms task payloads between services in a relationship chain using pluggable template engines. Maps data from one task's output into the format expected by the next task's input.
 
 ## Installation
 
-Add this line to your application's Gemfile:
+```bash
+gem install lex-transformer
+```
+
+Or add to your Gemfile:
 
 ```ruby
 gem 'lex-transformer'
 ```
 
-And then execute:
+## Standalone Client
+
+Use the transformer without the full LegionIO framework:
+
+```ruby
+require 'legion/extensions/transformer/client'
+
+client = Legion::Extensions::Transformer::Client.new
+
+# Single transform
+result = client.transform(
+  transformation: '{"greeting":"hello <%= name %>"}',
+  payload: { name: 'world' }
+)
+result[:success] # => true
+result[:result]  # => { greeting: "hello world" }
+
+# With explicit engine
+result = client.transform(
+  transformation: '{"greeting":"hello {{ name }}"}',
+  payload: { name: 'world' },
+  engine: :liquid
+)
+
+# With schema validation
+result = client.transform(
+  transformation: '{"name":"test"}',
+  payload: {},
+  schema: { required_keys: [:name, :email] }
+)
+result[:success] # => false
+result[:status]  # => "transformer.validation_failed"
+result[:errors]  # => ["missing required key: email"]
+```
+
+### Transform Chains
+
+Pipe data through sequential transformation steps:
+
+```ruby
+result = client.transform_chain(
+  steps: [
+    { transformation: '{"user":"<%= login %>"}', schema: { required_keys: [:user] } },
+    { transformation: '{"message":"Welcome, <%= user %>!"}' }
+  ],
+  payload: { login: 'alice' }
+)
+result[:success]        # => true
+result[:result][:args]  # => { message: "Welcome, alice!" }
+```
+
+Each step's output merges into the running payload, so subsequent steps can reference keys from earlier steps.
+
+## Engines
+
+| Engine | Name | Detection | Description |
+|--------|------|-----------|-------------|
+| ERB | `:erb` | `<%` or `%>` in template | Full ERB template rendering via `tilt` |
+| Static | `:static` | Default (no ERB markers) | Plain JSON passthrough |
+| Liquid | `:liquid` | Explicit only | Liquid template rendering (`{{ var }}`) |
+| JSONPath | `:jsonpath` | Explicit only | Dot-notation value extraction from payload |
+| LLM | `:llm` | Explicit only | Natural language transformation via Legion::LLM |
 
-    $ bundle
+Auto-detection selects ERB when the template contains ERB markers, otherwise falls back to Static. Use the `engine:` parameter to force a specific engine.
 
-Or install it yourself as:
+### LLM Engine
 
-    $ gem install lex-transformer
+The LLM engine uses natural language instructions as the "template":
 
-## Adding to Legion
-You can manually install with a `gem install lex-transformer` command or by adding it into your settings with something like this
-```json
-{
-  "extensions": {
-    "transformer": {
-      "enabled": true, "workers": 1
-    }
-  }
+```ruby
+client.transform(
+  transformation: "Summarize this webhook payload into a Slack notification with the PR title and author",
+  payload: { action: "opened", pull_request: { title: "Fix auth", user: { login: "alice" } } },
+  engine: :llm
+)
+```
+
+Requires `legion-llm` to be started. Provider-agnostic — uses whatever LLM provider is configured (Ollama, Bedrock, Anthropic, OpenAI, Gemini).
+
+## Schema Validation
+
+Validate transformed output against a declared schema:
+
+```ruby
+schema = {
+  required_keys: [:name, :email],                    # keys that must be present
+  types: { name: String, email: String, age: Integer } # optional type checks
 }
 ```
 
-## Usage
-*to be added*
+When validation fails, the transform returns `{ success: false, status: 'transformer.validation_failed', errors: [...] }`.
+
+## Runners
+
+### Transform
+
+#### `transform(transformation:, engine: nil, schema: nil, **payload)`
+
+Renders the transformation template against the payload, optionally validates the result, then dispatches:
+
+- Hash result: routes to next task (`task.queued`)
+- Array result: fan-out via `dispatch_multiplied` (one task per element, marked `task.multiplied`)
+- Schema failure: `transformer.validation_failed`, no dispatch
+
+#### `transform_chain(steps:, **payload)`
+
+Sequential pipeline. Each step has `:transformation`, optional `:engine`, optional `:schema`. Output of step N feeds into step N+1. Stops on first schema failure.
+
+## Transport
+
+- **Exchange**: `task` (inherits from `Legion::Transport::Exchanges::Task`)
+- **Queue**: `task.transform`
+- **Routing keys**: `task.subtask`, `task.subtask.transform`
+
+## Requirements
+
+- Ruby >= 3.4
+- [LegionIO](https://github.com/LegionIO/LegionIO) framework (for AMQP actor mode)
+- `tilt` >= 2.3
+- Standalone Client works without the framework
 
 ## License
 
-The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+MIT

data/Rakefile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'bundler/gem_tasks'
 require 'rspec/core/rake_task'
 

data/docker_deploy.rb

@@ -1,4 +1,5 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 
 name = 'transformer'
 require "./lib/legion/extensions/#{name}/version"

data/lex-transformer.gemspec

@@ -1,6 +1,6 @@
-lib = File.expand_path('lib', __dir__)
-$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require 'legion/extensions/transformer/version'
+# frozen_string_literal: true
+
+require_relative 'lib/legion/extensions/transformer/version'
 
 Gem::Specification.new do |spec|
   spec.name          = 'lex-transformer'
@@ -8,27 +8,23 @@ Gem::Specification.new do |spec|
   spec.authors       = ['Esity']
   spec.email         = ['matthewdiverson@gmail.com']
 
-  spec.summary       = 'LEX-Transformer is used to transform payloads for the output'
-  spec.description   = 'Runs transformer statements against tasks in a relationship'
-  spec.homepage      = 'https://bitbucket.org/legion-io/lex-transformer'
+  spec.summary       = 'Payload transformation engine for LegionIO task chains'
+  spec.description   = 'Template-based payload transformation between tasks with multiple engines, schema validation, and composition'
+  spec.homepage      = 'https://github.com/LegionIO/lex-transformer'
   spec.license       = 'MIT'
-  spec.required_ruby_version = Gem::Requirement.new('>= 2.5.0')
+  spec.required_ruby_version = '>= 3.4'
 
   spec.metadata['homepage_uri'] = spec.homepage
-  spec.metadata['source_code_uri'] = 'https://bitbucket.org/legion-io/lex-transformer'
-  spec.metadata['documentation_uri'] = 'https://legionio.atlassian.net/wiki/spaces/LEX/pages/612270222'
-  spec.metadata['changelog_uri'] = 'https://legionio.atlassian.net/wiki/spaces/LEX/pages/614858842'
-  spec.metadata['bug_tracker_uri'] = 'https://bitbucket.org/legion-io/lex-transformer/issues'
+  spec.metadata['source_code_uri'] = 'https://github.com/LegionIO/lex-transformer'
+  spec.metadata['documentation_uri'] = 'https://github.com/LegionIO/lex-transformer'
+  spec.metadata['changelog_uri'] = 'https://github.com/LegionIO/lex-transformer/blob/main/CHANGELOG.md'
+  spec.metadata['bug_tracker_uri'] = 'https://github.com/LegionIO/lex-transformer/issues'
+  spec.metadata['rubygems_mfa_required'] = 'true'
 
   spec.files = Dir.chdir(File.expand_path(__dir__)) do
     `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   end
   spec.require_paths = ['lib']
 
-  spec.add_dependency 'tilt'
-
-  spec.add_development_dependency 'bundler', '>= 2'
-  spec.add_development_dependency 'rake'
-  spec.add_development_dependency 'rspec'
-  spec.add_development_dependency 'rubocop'
+  spec.add_dependency 'tilt', '>= 2.3'
 end

data/lib/legion/extensions/transformer/actors/transform.rb

@@ -1,20 +1,26 @@
-module Legion::Extensions::Transformer
-  module Actor
-    class Transform < Legion::Extensions::Actors::Subscription
-      def runner_function
-        'transform'
-      end
+# frozen_string_literal: true
 
-      def check_subtask?
-        false
-      end
+module Legion
+  module Extensions
+    module Transformer
+      module Actor
+        class Transform < Legion::Extensions::Actors::Subscription
+          def runner_function
+            'transform'
+          end
 
-      def generate_task?
-        false
-      end
+          def check_subtask?
+            false
+          end
+
+          def generate_task?
+            false
+          end
 
-      def use_runner?
-        false
+          def use_runner?
+            false
+          end
+        end
       end
     end
   end

data/lib/legion/extensions/transformer/client.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require_relative 'engines/registry'
+require_relative 'helpers/schema_validator'
+
+module Legion
+  module Extensions
+    module Transformer
+      class Client
+        def transform(transformation:, payload:, engine: nil, schema: nil)
+          eng = resolve_engine(engine, transformation)
+          rendered = eng.render(transformation, payload)
+          rendered = parse_rendered(rendered)
+
+          if schema
+            validation = Helpers::SchemaValidator.validate(schema: schema, data: rendered)
+            return { success: false, status: 'transformer.validation_failed', errors: validation[:errors] } unless validation[:valid]
+          end
+
+          { success: true, result: rendered }
+        end
+
+        def transform_chain(steps:, payload:)
+          result = payload.dup
+          steps.each do |step|
+            eng = resolve_engine(step[:engine], step[:transformation])
+            rendered = eng.render(step[:transformation], result)
+            rendered = parse_rendered(rendered)
+
+            if step[:schema]
+              validation = Helpers::SchemaValidator.validate(schema: step[:schema], data: rendered)
+              return { success: false, status: 'transformer.validation_failed', errors: validation[:errors] } unless validation[:valid]
+            end
+
+            if rendered.is_a?(Hash)
+              result = result.merge({ args: rendered }.merge(rendered))
+            else
+              result[:args] = rendered
+            end
+          end
+          { success: true, result: result }
+        end
+
+        private
+
+        def resolve_engine(engine_name, transformation)
+          if engine_name
+            Engines::Registry.fetch(engine_name)
+          else
+            Engines::Registry.detect(transformation)
+          end
+        end
+
+        def parse_rendered(rendered)
+          return rendered unless rendered.is_a?(String)
+
+          Legion::JSON.load(rendered)
+        rescue StandardError
+          rendered
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/transformer/engines/base.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Transformer
+      module Engines
+        class Base
+          def render(template, payload)
+            raise NotImplementedError, "#{self.class}#render must be implemented"
+          end
+
+          def name
+            raise NotImplementedError, "#{self.class}#name must be implemented"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/transformer/engines/erb.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require 'tilt'
+require_relative 'base'
+
+module Legion
+  module Extensions
+    module Transformer
+      module Engines
+        class Erb < Base
+          def name
+            :erb
+          end
+
+          def render(template, payload)
+            tilt_template = Tilt['erb'].new { template }
+            variables = build_variables(template, payload)
+            tilt_template.render(Object.new, variables)
+          end
+
+          private
+
+          def build_variables(template, payload)
+            variables = { **payload }
+            variables[:crypt] = Legion::Crypt if defined?(Legion::Crypt) && template.include?('crypt')
+            variables[:settings] = Legion::Settings if defined?(Legion::Settings) && template.include?('settings')
+            variables[:cache] = Legion::Cache if defined?(Legion::Cache) && template.include?('cache')
+            if payload.key?(:task_id) && template.include?('task') && defined?(Legion::Data::Model::Task)
+              variables[:task] = Legion::Data::Model::Task[payload[:task_id]]
+            end
+            variables
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/transformer/engines/jsonpath.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require_relative 'base'
+
+module Legion
+  module Extensions
+    module Transformer
+      module Engines
+        class Jsonpath < Base
+          def name
+            :jsonpath
+          end
+
+          def render(expression, payload)
+            extract(expression, payload)
+          end
+
+          private
+
+          def extract(path, data)
+            path = path.delete_prefix('$.') if path.start_with?('$.')
+            segments = path.split('.')
+            result = data
+            segments.each do |segment|
+              case result
+              when Hash
+                key = result.key?(segment.to_sym) ? segment.to_sym : segment
+                result = result[key]
+              when Array
+                result = result[segment.to_i]
+              else
+                return nil
+              end
+            end
+            result
+          end
+        end
+      end
+    end
+  end
+end