lex-transformer 0.1.4 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 24dbbb177015cc67b411b4992d4b220082ae85fc3740a233439ba983ab496736
-  data.tar.gz: 5b836b34ed00f6937773638ea9dc9056e224f64b785ceb55b03bb0381c3ab899
+  metadata.gz: 111582b1dfcbe44b2cf91d3b316bd6394eb09e4d750a7b19be8844e91257e2c5
+  data.tar.gz: 6efebcb5ea769c9426e17e6be93aa1cc0ca1bf5093e02d9f5ea0fe5254809760
 SHA512:
-  metadata.gz: b0b9b63fa282d53dff4b096a585788cec55440b178eeaa6b2b2b93ebd5caa7323b671f135e0a85e8f6bf3a970ce651fd1054f5b3305c7ff707afdf500911e524
-  data.tar.gz: fa7c078497b01e4a89ebfa10401e38f79a8914e2778e5737ede513e7e17571cd6b481ebf02b61b478bf753db4d2edc2c4da1acbc515c343db8ab33e0faf9690c
+  metadata.gz: 8af3daa3467d9e37e571319c876a17ff772e3558c0510fcca75a633e970ed32b0497c1e31f63faa047e0eb2f6e8488d8a55820d0d5e46394450957228f4ec3af
+  data.tar.gz: a8dc1caf0ce16261c8eecf6d6ed37b2af4ff1cf4736465047a67cfd96eeb431a1454d4a31f2c34861219fea481b8d79b42d13b860a3935d240f81b89669c870d

data/.github/workflows/ci.yml

@@ -0,0 +1,16 @@
+name: CI
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  ci:
+    uses: LegionIO/.github/.github/workflows/ci.yml@main
+
+  release:
+    needs: ci
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    uses: LegionIO/.github/.github/workflows/release.yml@main
+    secrets:
+      rubygems-api-key: ${{ secrets.RUBYGEMS_API_KEY }}

data/.gitignore

@@ -9,3 +9,6 @@
 
 # rspec failure tracking
 .rspec_status
+
+Gemfile.lock
+Gemfile.lock

data/.rubocop.yml

@@ -1,33 +1,49 @@
+AllCops:
+  TargetRubyVersion: 3.4
+  NewCops: enable
+  SuggestExtensions: false
+
 Layout/LineLength:
-  Max: 120
+  Max: 160
+Layout/SpaceAroundEqualsInParameterDefault:
+  EnforcedStyle: space
+Layout/HashAlignment:
+  EnforcedHashRocketStyle: table
+  EnforcedColonStyle: table
+
 Metrics/MethodLength:
   Max: 50
 Metrics/ClassLength:
   Max: 1500
+Metrics/ModuleLength:
+  Max: 1500
 Metrics/BlockLength:
-  Max: 50
-Metrics/PerceivedComplexity:
-  Max: 20
-Metrics/CyclomaticComplexity:
-  Max: 117
+  Max: 40
+  Exclude:
+    - 'spec/**/*'
 Metrics/AbcSize:
-  Max: 52
-Layout/SpaceAroundEqualsInParameterDefault:
-  EnforcedStyle: space
-Style/SymbolArray:
-  Enabled: true
-Layout/HashAlignment:
-  EnforcedHashRocketStyle: table
-  EnforcedColonStyle: table
+  Max: 60
+Metrics/CyclomaticComplexity:
+  Max: 15
+Metrics/PerceivedComplexity:
+  Max: 17
+Metrics/ParameterLists:
+  Enabled: false
+
 Style/Documentation:
   Enabled: false
-AllCops:
-  TargetRubyVersion: 2.5
-  NewCops: enable
-  SuggestExtensions: false
+Style/SymbolArray:
+  Enabled: true
 Style/FrozenStringLiteralComment:
-  Enabled: false
+  Enabled: true
+  EnforcedStyle: always
+
 Naming/FileName:
   Enabled: false
-Style/ClassAndModuleChildren:
-  Enabled: false
+Naming/PredicateMethod:
+  Enabled: false
+Naming/PredicatePrefix:
+  Enabled: false
+
+Gemspec/DevelopmentDependencies:
+  Enabled: false

data/CHANGELOG.md

@@ -0,0 +1,54 @@
+# Changelog
+
+## [0.3.0] - 2026-03-19
+
+### Added
+- LLM engine error handling with categorized retry (timeout/network retry, auth errors raise, provider errors return failure)
+- LLM engine model/provider/temperature/system_prompt kwargs via `engine_options`
+- LLM engine structured output support (`structured: true` + `schema:`)
+- LLM engine JSON response validation with correction prompt on retry
+- Settings-based LLM defaults (`lex-transformer.llm.*`)
+- Named transform definitions via Settings (`lex-transformer.definitions.*`)
+- `Definitions` class for Settings-based definition lookup
+- `name:` parameter on `Client#transform` for named definition execution
+- `engine_options:` parameter on `Client#transform` and `Client#transform_chain`
+- Conditioner integration for named definitions with conditions
+- LLM failure hash passthrough (no dispatch on engine failure)
+- Integration specs for LLM engine with client, chain, and named definitions
+
+### Changed
+- All engine `render` signatures accept `**opts` (backward-compatible, non-LLM engines ignore)
+
+## [0.2.1] - 2026-03-17
+
+### Added
+- LLM transform engine (`engines/llm.rb`) — provider-agnostic natural language transformation via `Legion::LLM.chat`
+- Registered as `:llm` in engine registry (explicit only, no auto-detection)
+
+## [0.2.0] - 2026-03-17
+
+### Added
+- Pluggable engine system with abstract `Engines::Base` and `Engines::Registry`
+- 4 template engines: ERB (extracted from runner), Static (JSON passthrough), Liquid, JSONPath (dot-notation extraction)
+- `Engines::Registry.detect` auto-selects ERB or static engine based on template content
+- `render_transformation` accepts optional `engine:` keyword to force a specific engine
+- `Helpers::SchemaValidator` validates transformed output against declared schemas (required keys, type checks)
+- `transform` accepts optional `schema:` parameter for post-render validation
+- `transform_chain` method for sequential multi-step pipelines with per-step engine selection and schema validation
+- Standalone `Client` class for framework-independent usage with `transform` and `transform_chain` methods
+- SimpleCov coverage reporting
+- Modern packaging: grouped test dependencies, `require_relative` in gemspec, `rubocop-rspec`
+
+### Fixed
+- `dispatch_multiplied` payload mutation bug: `new_payload = payload` replaced with `new_payload = payload.dup`
+- Spec filename typo: `tranformer_spec.rb` renamed to `transformer_spec.rb`
+
+### Changed
+- ERB rendering logic extracted from runner into `Engines::Erb` class
+- `build_template_variables` moved to `Engines::Erb` as private `build_variables`
+- Runner no longer directly requires `tilt` (delegated to `Engines::Erb`)
+
+## [0.1.4] - 2026-03-13
+
+### Added
+- Initial release

data/CLAUDE.md

@@ -0,0 +1,135 @@
+# lex-transformer: Payload Transformation Engine for LegionIO
+
+**Repository Level 3 Documentation**
+- **Parent**: `/Users/miverso2/rubymine/legion/extensions-core/CLAUDE.md`
+- **Grandparent**: `/Users/miverso2/rubymine/legion/CLAUDE.md`
+
+## Purpose
+
+Legion Extension that transforms task payloads between services in a relationship chain. Uses pluggable template engines (ERB, Static, Liquid, JSONPath, LLM) to map data from one task's output into the format expected by the next task's input. Supports single-hash output (1:1 dispatch), array output (fan-out/multiply), schema validation, and sequential transform chains. Requires `legion-data` (`data_required? true`).
+
+**GitHub**: https://github.com/LegionIO/lex-transformer
+**License**: MIT
+**Version**: 0.2.1
+
+## Architecture
+
+```
+Legion::Extensions::Transformer
+├── Actors/
+│   └── Transform          # Subscription actor consuming transform requests
+├── Engines/
+│   ├── Base               # Abstract engine interface
+│   ├── Registry           # Maps engine name symbols to engine classes
+│   ├── Erb                # ERB rendering via tilt
+│   ├── Static             # JSON passthrough (no templating)
+│   ├── Liquid             # Liquid template rendering
+│   ├── Jsonpath           # Dot-notation value extraction from payload
+│   └── Llm                # Natural language transformation via Legion::LLM
+├── Helpers/
+│   └── SchemaValidator    # Validates transform output against required_keys/types schema
+├── Runners/
+│   └── Transform          # Executes template-based payload transformation
+│       ├── transform             # Entry point: render + dispatch or validate
+│       ├── transform_chain       # Sequential pipeline: N steps, output feeds next
+│       ├── render_transformation # Engine dispatch (ERB/Static/Liquid/JSONPath/LLM)
+│       ├── build_template_variables # Inject crypt/settings/cache/task into scope
+│       ├── dispatch_transformed  # Route Hash (single) or Array (fan-out) results
+│       ├── dispatch_multiplied   # Fan-out: create a new task per array element
+│       └── send_task             # Publish transformed payload to next runner
+├── Transport/
+│   ├── Exchanges/Task     # Publishes to the task exchange
+│   ├── Queues/Transform   # Subscribes to transformation queue
+│   └── Messages/Message   # Transform request message format
+└── Client                 # Standalone client: transform + transform_chain (uses Engines::Registry directly)
+```
+
+## Key Files
+
+| Path | Purpose |
+|------|---------|
+| `lib/legion/extensions/transformer.rb` | Entry point (`data_required? true`) |
+| `lib/legion/extensions/transformer/runners/transform.rb` | Core transformation logic |
+| `lib/legion/extensions/transformer/actors/transform.rb` | AMQP subscription actor |
+| `lib/legion/extensions/transformer/client.rb` | Standalone client (transform, transform_chain) |
+| `lib/legion/extensions/transformer/engines/registry.rb` | Engine name -> class lookup + auto-detection |
+| `lib/legion/extensions/transformer/helpers/schema_validator.rb` | Output schema validation |
+| `lib/legion/extensions/transformer/transport.rb` | Transport setup |
+
+## Template Engines
+
+| Engine | Name | Detection | Description |
+|--------|------|-----------|-------------|
+| ERB | `:erb` | `<%` or `%>` in template | Full ERB template rendering via `tilt` |
+| Static | `:static` | Default (no ERB markers) | Plain JSON passthrough |
+| Liquid | `:liquid` | Explicit only | Liquid template rendering (`{{ var }}`) |
+| JSONPath | `:jsonpath` | Explicit only | Dot-notation value extraction from payload |
+| LLM | `:llm` | Explicit only | Natural language transformation via `Legion::LLM` |
+
+Auto-detection: ERB when template contains `<%` or `%>`, otherwise Static. Pass `engine:` to force a specific engine.
+
+The LLM engine requires `legion-llm` to be started; it is provider-agnostic (Ollama, Bedrock, Anthropic, OpenAI, Gemini).
+
+## Template Variables Available in ERB
+
+| Variable | Available when |
+|----------|---------------|
+| All payload keys | Always (splatted into template scope) |
+| `crypt` | Template string contains `'crypt'` |
+| `settings` | Template string contains `'settings'` |
+| `cache` | Template string contains `'cache'` |
+| `task` | Template string contains `'task'` and payload has `task_id` |
+
+## Schema Validation
+
+```ruby
+schema = {
+  required_keys: [:name, :email],
+  types: { name: String, email: String, age: Integer }
+}
+```
+
+On failure: `{ success: false, status: 'transformer.validation_failed', errors: [...] }` — no dispatch.
+
+## Dispatch Behavior
+
+- **Hash result**: Updates task to `transformer.succeeded`, dispatches single task, updates to `task.queued`
+- **Array result**: Fan-out via `dispatch_multiplied` — creates a new task record per element, dispatches each, marks original as `task.multiplied`
+- **Schema failure**: `transformer.validation_failed`, no dispatch
+
+## Transform Chains
+
+`transform_chain(steps:, **payload)` runs steps sequentially. Each step specifies `:transformation`, optional `:engine`, and optional `:schema`. Output of step N is merged into the running payload for step N+1. Stops on first schema failure.
+
+## Standalone Client
+
+`Legion::Extensions::Transformer::Client` includes the Transform runner:
+
+```ruby
+require 'legion/extensions/transformer/client'
+client = Legion::Extensions::Transformer::Client.new
+result = client.transform(transformation: '{"x":"<%= y %>"}', payload: { y: 'hello' })
+result[:success] # => true
+result[:result]  # => { x: "hello" }
+```
+
+## Dependencies
+
+| Gem | Purpose |
+|-----|---------|
+| `tilt` (>= 2.3) | Template engine abstraction for ERB rendering |
+| `legion-data` | Required — task record creation for fan-out |
+
+## Testing
+
+```bash
+bundle install
+bundle exec rspec     # 86 examples, 0 failures
+bundle exec rubocop   # 0 offenses
+```
+
+Spec files: `spec/legion/extensions/tranformer_spec.rb` (note: typo in filename is intentional in repo), `spec/legion/extensions/transform_runner_spec.rb`
+
+---
+
+**Maintained By**: Matthew Iverson (@Esity)

data/Dockerfile

@@ -6,4 +6,4 @@ RUN apk update && apk add build-base tzdata gcc git
 
 COPY . ./
 RUN gem install lex-transformer legion-data --no-document --no-prerelease
-CMD ruby --jit $(which legionio)
+CMD ruby --yjit $(which legionio)

data/Gemfile

@@ -1,3 +1,16 @@
+# frozen_string_literal: true
+
 source 'https://rubygems.org'
 
 gemspec
+
+group :test do
+  gem 'base64'
+  gem 'liquid', '~> 5.0'
+  gem 'rake'
+  gem 'rspec', '~> 3.13'
+  gem 'rspec_junit_formatter'
+  gem 'rubocop', '~> 1.75'
+  gem 'rubocop-rspec'
+  gem 'simplecov'
+end

data/README.md

@@ -1,38 +1,140 @@
-# Legion::Extensions::Transformer
+# lex-transformer
 
-Used to transform data to match the action for a relationship
+Payload transformation engine for [LegionIO](https://github.com/LegionIO/LegionIO). Transforms task payloads between services in a relationship chain using pluggable template engines. Maps data from one task's output into the format expected by the next task's input.
 
 ## Installation
 
-Add this line to your application's Gemfile:
+```bash
+gem install lex-transformer
+```
+
+Or add to your Gemfile:
 
 ```ruby
 gem 'lex-transformer'
 ```
 
-And then execute:
+## Standalone Client
+
+Use the transformer without the full LegionIO framework:
+
+```ruby
+require 'legion/extensions/transformer/client'
+
+client = Legion::Extensions::Transformer::Client.new
+
+# Single transform
+result = client.transform(
+  transformation: '{"greeting":"hello <%= name %>"}',
+  payload: { name: 'world' }
+)
+result[:success] # => true
+result[:result]  # => { greeting: "hello world" }
+
+# With explicit engine
+result = client.transform(
+  transformation: '{"greeting":"hello {{ name }}"}',
+  payload: { name: 'world' },
+  engine: :liquid
+)
+
+# With schema validation
+result = client.transform(
+  transformation: '{"name":"test"}',
+  payload: {},
+  schema: { required_keys: [:name, :email] }
+)
+result[:success] # => false
+result[:status]  # => "transformer.validation_failed"
+result[:errors]  # => ["missing required key: email"]
+```
+
+### Transform Chains
+
+Pipe data through sequential transformation steps:
+
+```ruby
+result = client.transform_chain(
+  steps: [
+    { transformation: '{"user":"<%= login %>"}', schema: { required_keys: [:user] } },
+    { transformation: '{"message":"Welcome, <%= user %>!"}' }
+  ],
+  payload: { login: 'alice' }
+)
+result[:success]        # => true
+result[:result][:args]  # => { message: "Welcome, alice!" }
+```
+
+Each step's output merges into the running payload, so subsequent steps can reference keys from earlier steps.
+
+## Engines
+
+| Engine | Name | Detection | Description |
+|--------|------|-----------|-------------|
+| ERB | `:erb` | `<%` or `%>` in template | Full ERB template rendering via `tilt` |
+| Static | `:static` | Default (no ERB markers) | Plain JSON passthrough |
+| Liquid | `:liquid` | Explicit only | Liquid template rendering (`{{ var }}`) |
+| JSONPath | `:jsonpath` | Explicit only | Dot-notation value extraction from payload |
+| LLM | `:llm` | Explicit only | Natural language transformation via Legion::LLM |
 
-    $ bundle
+Auto-detection selects ERB when the template contains ERB markers, otherwise falls back to Static. Use the `engine:` parameter to force a specific engine.
 
-Or install it yourself as:
+### LLM Engine
 
-    $ gem install lex-transformer
+The LLM engine uses natural language instructions as the "template":
 
-## Adding to Legion
-You can manually install with a `gem install lex-transformer` command or by adding it into your settings with something like this
-```json
-{
-  "extensions": {
-    "transformer": {
-      "enabled": true, "workers": 1
-    }
-  }
+```ruby
+client.transform(
+  transformation: "Summarize this webhook payload into a Slack notification with the PR title and author",
+  payload: { action: "opened", pull_request: { title: "Fix auth", user: { login: "alice" } } },
+  engine: :llm
+)
+```
+
+Requires `legion-llm` to be started. Provider-agnostic — uses whatever LLM provider is configured (Ollama, Bedrock, Anthropic, OpenAI, Gemini).
+
+## Schema Validation
+
+Validate transformed output against a declared schema:
+
+```ruby
+schema = {
+  required_keys: [:name, :email],                    # keys that must be present
+  types: { name: String, email: String, age: Integer } # optional type checks
 }
 ```
 
-## Usage
-*to be added*
+When validation fails, the transform returns `{ success: false, status: 'transformer.validation_failed', errors: [...] }`.
+
+## Runners
+
+### Transform
+
+#### `transform(transformation:, engine: nil, schema: nil, **payload)`
+
+Renders the transformation template against the payload, optionally validates the result, then dispatches:
+
+- Hash result: routes to next task (`task.queued`)
+- Array result: fan-out via `dispatch_multiplied` (one task per element, marked `task.multiplied`)
+- Schema failure: `transformer.validation_failed`, no dispatch
+
+#### `transform_chain(steps:, **payload)`
+
+Sequential pipeline. Each step has `:transformation`, optional `:engine`, optional `:schema`. Output of step N feeds into step N+1. Stops on first schema failure.
+
+## Transport
+
+- **Exchange**: `task` (inherits from `Legion::Transport::Exchanges::Task`)
+- **Queue**: `task.transform`
+- **Routing keys**: `task.subtask`, `task.subtask.transform`
+
+## Requirements
+
+- Ruby >= 3.4
+- [LegionIO](https://github.com/LegionIO/LegionIO) framework (for AMQP actor mode)
+- `tilt` >= 2.3
+- Standalone Client works without the framework
 
 ## License
 
-The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+MIT

data/Rakefile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'bundler/gem_tasks'
 require 'rspec/core/rake_task'
 

data/docker_deploy.rb

@@ -1,4 +1,5 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 
 name = 'transformer'
 require "./lib/legion/extensions/#{name}/version"

data/lex-transformer.gemspec

@@ -1,6 +1,6 @@
-lib = File.expand_path('lib', __dir__)
-$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require 'legion/extensions/transformer/version'
+# frozen_string_literal: true
+
+require_relative 'lib/legion/extensions/transformer/version'
 
 Gem::Specification.new do |spec|
   spec.name          = 'lex-transformer'
@@ -8,27 +8,23 @@ Gem::Specification.new do |spec|
   spec.authors       = ['Esity']
   spec.email         = ['matthewdiverson@gmail.com']
 
-  spec.summary       = 'LEX-Transformer is used to transform payloads for the output'
-  spec.description   = 'Runs transformer statements against tasks in a relationship'
-  spec.homepage      = 'https://bitbucket.org/legion-io/lex-transformer'
+  spec.summary       = 'Payload transformation engine for LegionIO task chains'
+  spec.description   = 'Template-based payload transformation between tasks with multiple engines, schema validation, and composition'
+  spec.homepage      = 'https://github.com/LegionIO/lex-transformer'
   spec.license       = 'MIT'
-  spec.required_ruby_version = Gem::Requirement.new('>= 2.5.0')
+  spec.required_ruby_version = '>= 3.4'
 
   spec.metadata['homepage_uri'] = spec.homepage
-  spec.metadata['source_code_uri'] = 'https://bitbucket.org/legion-io/lex-transformer'
-  spec.metadata['documentation_uri'] = 'https://legionio.atlassian.net/wiki/spaces/LEX/pages/612270222'
-  spec.metadata['changelog_uri'] = 'https://legionio.atlassian.net/wiki/spaces/LEX/pages/614858842'
-  spec.metadata['bug_tracker_uri'] = 'https://bitbucket.org/legion-io/lex-transformer/issues'
+  spec.metadata['source_code_uri'] = 'https://github.com/LegionIO/lex-transformer'
+  spec.metadata['documentation_uri'] = 'https://github.com/LegionIO/lex-transformer'
+  spec.metadata['changelog_uri'] = 'https://github.com/LegionIO/lex-transformer/blob/main/CHANGELOG.md'
+  spec.metadata['bug_tracker_uri'] = 'https://github.com/LegionIO/lex-transformer/issues'
+  spec.metadata['rubygems_mfa_required'] = 'true'
 
   spec.files = Dir.chdir(File.expand_path(__dir__)) do
     `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   end
   spec.require_paths = ['lib']
 
-  spec.add_dependency 'tilt'
-
-  spec.add_development_dependency 'bundler', '>= 2'
-  spec.add_development_dependency 'rake'
-  spec.add_development_dependency 'rspec'
-  spec.add_development_dependency 'rubocop'
+  spec.add_dependency 'tilt', '>= 2.3'
 end

data/lib/legion/extensions/transformer/actors/transform.rb

@@ -1,20 +1,26 @@
-module Legion::Extensions::Transformer
-  module Actor
-    class Transform < Legion::Extensions::Actors::Subscription
-      def runner_function
-        'transform'
-      end
+# frozen_string_literal: true
 
-      def check_subtask?
-        false
-      end
+module Legion
+  module Extensions
+    module Transformer
+      module Actor
+        class Transform < Legion::Extensions::Actors::Subscription
+          def runner_function
+            'transform'
+          end
 
-      def generate_task?
-        false
-      end
+          def check_subtask?
+            false
+          end
+
+          def generate_task?
+            false
+          end
 
-      def use_runner?
-        false
+          def use_runner?
+            false
+          end
+        end
       end
     end
   end