lex-transformer 0.1.4 → 0.3.0

data/lib/legion/extensions/transformer/runners/transform.rb

@@ -1,66 +1,93 @@
-require 'tilt'
+# frozen_string_literal: true
 
-module Legion::Extensions::Transformer
-  module Runners
-    module Transform
-      def transform(transformation:, **payload)
-        if transformation.include?('<%') || transformation.include?('%>')
-          template = Tilt['erb'].new { transformation }
-          variables = { **payload }
-          variables[:crypt] = Legion::Crypt if transformation.include? 'crypt'
-          variables[:settings] = Legion::Settings if transformation.include? 'settings'
-          variables[:cache] = Legion::Cache if transformation.include? 'cache'
-          if payload.key?(:task_id) && transformation.include?('task')
-            variables[:task] = Legion::Data::Model::Task[payload[:task_id]]
+require_relative '../engines/registry'
+require_relative '../helpers/schema_validator'
+
+module Legion
+  module Extensions
+    module Transformer
+      module Runners
+        module Transform
+          def transform(transformation:, engine: nil, schema: nil, engine_options: {}, **payload)
+            payload[:args] = render_transformation(transformation, payload, engine: engine, engine_options: engine_options)
+
+            return payload[:args] if payload[:args].is_a?(Hash) && payload[:args][:success] == false
+
+            if schema
+              validation = Helpers::SchemaValidator.validate(schema: schema, data: payload[:args])
+              return { success: false, status: 'transformer.validation_failed', errors: validation[:errors] } unless validation[:valid]
+            end
+            dispatch_transformed(payload)
+            generate_task_log(task_id: payload[:task_id], function: 'transform', values: payload) if payload[:debug] && payload.key?(:task_id)
+            { success: true, **payload }
           end
 
-          payload[:args] = from_json(template.render(self, variables))
-        else
-          payload[:args] = from_json(transformation)
-        end
+          def transform_chain(steps:, **payload)
+            result = payload
+            steps.each do |step|
+              step_opts = step[:engine_options] || {}
+              rendered = render_transformation(step[:transformation], result, engine: step[:engine], engine_options: step_opts)
+              rendered = from_json(rendered) if rendered.is_a?(String)
+
+              return rendered if rendered.is_a?(Hash) && rendered[:success] == false
+
+              if step[:schema]
+                validation = Helpers::SchemaValidator.validate(schema: step[:schema], data: rendered)
+                return { success: false, status: 'transformer.validation_failed', errors: validation[:errors] } unless validation[:valid]
+              end
+              result = result.merge({ args: rendered }.merge(rendered))
+            end
+            { success: true, **result }
+          end
+
+          def render_transformation(transformation, payload, engine: nil, engine_options: {})
+            eng = if engine
+                    Engines::Registry.fetch(engine)
+                  else
+                    Engines::Registry.detect(transformation)
+                  end
+
+            rendered = eng.render(transformation, payload, **engine_options)
+            return rendered if rendered.is_a?(Hash) && rendered[:success] == false
 
-        case payload[:args]
-        when Hash
-          unless payload[:task_id].nil?
-            task_update(payload[:task_id], 'transformer.succeeded', function_args: payload[:args])
+            rendered.is_a?(String) ? from_json(rendered) : rendered
           end
-          send_task(**payload)
-          task_update(payload[:task_id], 'task.queued', use_database: false) unless payload[:task_id].nil?
-        when Array
-          payload[:args].each do |thing|
-            new_payload = payload
-            task = Legion::Runner::Status.generate_task_id(function_args: thing,
-                                                           status:        'task.queued',
-                                                           args:          thing,
-                                                           **new_payload)
-            new_payload[:task_id] = task[:task_id]
-            new_payload[:args] = thing
-            send_task(**new_payload)
+
+          def dispatch_transformed(payload)
+            case payload[:args]
+            when Hash
+              task_update(payload[:task_id], 'transformer.succeeded', function_args: payload[:args]) unless payload[:task_id].nil?
+              send_task(**payload)
+              task_update(payload[:task_id], 'task.queued', use_database: false) unless payload[:task_id].nil?
+            when Array
+              dispatch_multiplied(payload)
+            end
           end
-          unless payload[:task_id].nil?
-            task_update(payload[:task_id],
-                        'task.multiplied',
-                        function_args: payload[:args])
+
+          def dispatch_multiplied(payload)
+            payload[:args].each do |thing|
+              new_payload = payload.dup
+              task = Legion::Runner::Status.generate_task_id(function_args: thing, status: 'task.queued', args: thing, **new_payload)
+              new_payload[:task_id] = task[:task_id]
+              new_payload[:args] = thing
+              send_task(**new_payload)
+            end
+            task_update(payload[:task_id], 'task.multiplied', function_args: payload[:args]) unless payload[:task_id].nil?
           end
-        end
 
-        if payload[:debug] && payload.key?(:task_id)
-          generate_task_log(task_id: payload[:task_id], function: 'transform', values: payload)
-        end
-        { success: true, **payload }
-      end
+          def send_task(**opts)
+            payload = {}
+            %i[task_id relationship_id trigger_function_id runner_class function_id function chain_id debug args].each do |thing|
+              payload[thing] = opts[thing] if opts.key? thing
+            end
 
-      def send_task(**opts)
-        payload = {}
-        %i[task_id relationship_id trigger_function_id runner_class function_id function chain_id debug args].each do |thing| # rubocop:disable Layout/LineLength
-          payload[thing] = opts[thing] if opts.key? thing
-        end
+            Legion::Extensions::Transformer::Transport::Messages::Message.new(**payload).publish
+          end
 
-        Legion::Extensions::Transformer::Transport::Messages::Message.new(**payload).publish
+          include Legion::Extensions::Helpers::Lex
+          extend Legion::Extensions::Helpers::Task
+        end
       end
-
-      include Legion::Extensions::Helpers::Lex
-      extend Legion::Extensions::Helpers::Task
     end
   end
 end

data/lib/legion/extensions/transformer/transport/exchanges/task.rb

@@ -1,7 +1,13 @@
-module Legion::Extensions::Transformer
-  module Transport
-    module Exchanges
-      class Transformer < Legion::Transport::Exchanges::Task
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Transformer
+      module Transport
+        module Exchanges
+          class Transformer < Legion::Transport::Exchanges::Task
+          end
+        end
       end
     end
   end

data/lib/legion/extensions/transformer/transport/messages/message.rb

@@ -1,24 +1,34 @@
-module Legion::Extensions::Transformer::Transport::Messages
-  class Message < Legion::Transport::Message
-    def type
-      'task'
-    end
+# frozen_string_literal: true
 
-    def message
-      @options[:function] = function.values[:name]
-      @options
-    end
+module Legion
+  module Extensions
+    module Transformer
+      module Transport
+        module Messages
+          class Message < Legion::Transport::Message
+            def type
+              'task'
+            end
 
-    def routing_key
-      "#{function.runner.extension.values[:name]}.#{function.runner.values[:name]}.#{function.values[:name]}"
-    end
+            def message
+              @options[:function] = function.values[:name]
+              @options
+            end
 
-    def exchange
-      Legion::Transport::Exchange.new(function.runner.extension.values[:exchange])
-    end
+            def routing_key
+              "#{function.runner.extension.values[:name]}.#{function.runner.values[:name]}.#{function.values[:name]}"
+            end
+
+            def exchange
+              Legion::Transport::Exchange.new(function.runner.extension.values[:exchange])
+            end
 
-    def function
-      @function ||= Legion::Data::Model::Function[@options[:function_id]]
+            def function
+              @function ||= Legion::Data::Model::Function[@options[:function_id]]
+            end
+          end
+        end
+      end
     end
   end
 end

data/lib/legion/extensions/transformer/transport/queues/transform.rb

@@ -1,9 +1,15 @@
-module Legion::Extensions::Transformer
-  module Transport
-    module Queues
-      class Transform < Legion::Transport::Queue
-        def queue_name
-          'task.transform'
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Transformer
+      module Transport
+        module Queues
+          class Transform < Legion::Transport::Queue
+            def queue_name
+              'task.transform'
+            end
+          end
         end
       end
     end

data/lib/legion/extensions/transformer/transport.rb

@@ -1,17 +1,23 @@
-module Legion::Extensions::Transformer
-  module Transport
-    extend Legion::Extensions::Transport
+# frozen_string_literal: true
 
-    def self.additional_e_to_q
-      [
-        {
-          to:          Legion::Extensions::Transformer::Transport::Queues::Transform,
-          routing_key: 'task.conditioner.succeeded'
-        }, {
-          to:          Legion::Extensions::Transformer::Transport::Queues::Transform,
-          routing_key: 'task.subtask.transform'
-        }
-      ]
+module Legion
+  module Extensions
+    module Transformer
+      module Transport
+        extend Legion::Extensions::Transport
+
+        def self.additional_e_to_q
+          [
+            {
+              to:          Legion::Extensions::Transformer::Transport::Queues::Transform,
+              routing_key: 'task.conditioner.succeeded'
+            }, {
+              to:          Legion::Extensions::Transformer::Transport::Queues::Transform,
+              routing_key: 'task.subtask.transform'
+            }
+          ]
+        end
+      end
     end
   end
 end

data/lib/legion/extensions/transformer/version.rb

@@ -1,7 +1,9 @@
+# frozen_string_literal: true
+
 module Legion
   module Extensions
     module Transformer
-      VERSION = '0.1.4'.freeze
+      VERSION = '0.3.0'
     end
   end
 end

data/lib/legion/extensions/transformer.rb

@@ -1,4 +1,7 @@
+# frozen_string_literal: true
+
 require 'legion/extensions/transformer/version'
+require_relative 'transformer/client'
 
 module Legion
   module Extensions

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: lex-transformer
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.3.0
 platform: ruby
 authors:
 - Esity
-autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-03-03 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: tilt
@@ -16,108 +15,63 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '2'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '2'
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rspec
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rubocop
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-description: Runs transformer statements against tasks in a relationship
+        version: '2.3'
+description: Template-based payload transformation between tasks with multiple engines,
+  schema validation, and composition
 email:
 - matthewdiverson@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/ci.yml"
 - ".gitignore"
 - ".rspec"
 - ".rubocop.yml"
-- CODE_OF_CONDUCT.md
+- CHANGELOG.md
+- CLAUDE.md
 - Dockerfile
 - Gemfile
-- Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile
-- bitbucket-pipelines.yml
 - docker_deploy.rb
 - lex-transformer.gemspec
 - lib/legion/extensions/transformer.rb
 - lib/legion/extensions/transformer/actors/transform.rb
+- lib/legion/extensions/transformer/client.rb
+- lib/legion/extensions/transformer/definitions.rb
+- lib/legion/extensions/transformer/engines/base.rb
+- lib/legion/extensions/transformer/engines/erb.rb
+- lib/legion/extensions/transformer/engines/jsonpath.rb
+- lib/legion/extensions/transformer/engines/liquid.rb
+- lib/legion/extensions/transformer/engines/llm.rb
+- lib/legion/extensions/transformer/engines/registry.rb
+- lib/legion/extensions/transformer/engines/static.rb
+- lib/legion/extensions/transformer/helpers/schema_validator.rb
 - lib/legion/extensions/transformer/runners/transform.rb
 - lib/legion/extensions/transformer/transport.rb
 - lib/legion/extensions/transformer/transport/exchanges/task.rb
 - lib/legion/extensions/transformer/transport/messages/message.rb
 - lib/legion/extensions/transformer/transport/queues/transform.rb
 - lib/legion/extensions/transformer/version.rb
-homepage: https://bitbucket.org/legion-io/lex-transformer
+homepage: https://github.com/LegionIO/lex-transformer
 licenses:
 - MIT
 metadata:
-  homepage_uri: https://bitbucket.org/legion-io/lex-transformer
-  source_code_uri: https://bitbucket.org/legion-io/lex-transformer
-  documentation_uri: https://legionio.atlassian.net/wiki/spaces/LEX/pages/612270222
-  changelog_uri: https://legionio.atlassian.net/wiki/spaces/LEX/pages/614858842
-  bug_tracker_uri: https://bitbucket.org/legion-io/lex-transformer/issues
-post_install_message: 
+  homepage_uri: https://github.com/LegionIO/lex-transformer
+  source_code_uri: https://github.com/LegionIO/lex-transformer
+  documentation_uri: https://github.com/LegionIO/lex-transformer
+  changelog_uri: https://github.com/LegionIO/lex-transformer/blob/main/CHANGELOG.md
+  bug_tracker_uri: https://github.com/LegionIO/lex-transformer/issues
+  rubygems_mfa_required: 'true'
 rdoc_options: []
 require_paths:
 - lib
@@ -125,15 +79,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.5.0
+      version: '3.4'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
-signing_key: 
+rubygems_version: 3.6.9
 specification_version: 4
-summary: LEX-Transformer is used to transform payloads for the output
+summary: Payload transformation engine for LegionIO task chains
 test_files: []

data/CODE_OF_CONDUCT.md

@@ -1,74 +0,0 @@
-# Contributor Covenant Code of Conduct
-
-## Our Pledge
-
-In the interest of fostering an open and welcoming environment, we as
-contributors and maintainers pledge to making participation in our project and
-our community a harassment-free experience for everyone, regardless of age, body
-size, disability, ethnicity, gender identity and expression, level of experience,
-nationality, personal appearance, race, religion, or sexual identity and
-orientation.
-
-## Our Standards
-
-Examples of behavior that contributes to creating a positive environment
-include:
-
-* Using welcoming and inclusive language
-* Being respectful of differing viewpoints and experiences
-* Gracefully accepting constructive criticism
-* Focusing on what is best for the community
-* Showing empathy towards other community members
-
-Examples of unacceptable behavior by participants include:
-
-* The use of sexualized language or imagery and unwelcome sexual attention or
-advances
-* Trolling, insulting/derogatory comments, and personal or political attacks
-* Public or private harassment
-* Publishing others' private information, such as a physical or electronic
-  address, without explicit permission
-* Other conduct which could reasonably be considered inappropriate in a
-  professional setting
-
-## Our Responsibilities
-
-Project maintainers are responsible for clarifying the standards of acceptable
-behavior and are expected to take appropriate and fair corrective action in
-response to any instances of unacceptable behavior.
-
-Project maintainers have the right and responsibility to remove, edit, or
-reject comments, commits, code, wiki edits, issues, and other contributions
-that are not aligned to this Code of Conduct, or to ban temporarily or
-permanently any contributor for other behaviors that they deem inappropriate,
-threatening, offensive, or harmful.
-
-## Scope
-
-This Code of Conduct applies both within project spaces and in public spaces
-when an individual is representing the project or its community. Examples of
-representing a project or community include using an official project e-mail
-address, posting via an official social media account, or acting as an appointed
-representative at an online or offline event. Representation of a project may be
-further defined and clarified by project maintainers.
-
-## Enforcement
-
-Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported by contacting the project team at matthewdiverson@gmail.com. All
-complaints will be reviewed and investigated and will result in a response that
-is deemed necessary and appropriate to the circumstances. The project team is
-obligated to maintain confidentiality with regard to the reporter of an incident.
-Further details of specific enforcement policies may be posted separately.
-
-Project maintainers who do not follow or enforce the Code of Conduct in good
-faith may face temporary or permanent repercussions as determined by other
-members of the project's leadership.
-
-## Attribution
-
-This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
-available at [http://contributor-covenant.org/version/1/4][version]
-
-[homepage]: http://contributor-covenant.org
-[version]: http://contributor-covenant.org/version/1/4/

data/Gemfile.lock

@@ -1,58 +0,0 @@
-PATH
-  remote: .
-  specs:
-    lex-transformer (0.1.4)
-      tilt
-
-GEM
-  remote: https://rubygems.org/
-  specs:
-    ast (2.4.2)
-    diff-lcs (1.4.4)
-    parallel (1.20.1)
-    parser (3.0.0.0)
-      ast (~> 2.4.1)
-    rainbow (3.0.0)
-    rake (13.0.3)
-    regexp_parser (2.1.1)
-    rexml (3.2.4)
-    rspec (3.10.0)
-      rspec-core (~> 3.10.0)
-      rspec-expectations (~> 3.10.0)
-      rspec-mocks (~> 3.10.0)
-    rspec-core (3.10.1)
-      rspec-support (~> 3.10.0)
-    rspec-expectations (3.10.1)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-mocks (3.10.2)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-support (3.10.2)
-    rubocop (1.11.0)
-      parallel (~> 1.10)
-      parser (>= 3.0.0.0)
-      rainbow (>= 2.2.2, < 4.0)
-      regexp_parser (>= 1.8, < 3.0)
-      rexml
-      rubocop-ast (>= 1.2.0, < 2.0)
-      ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 1.4.0, < 3.0)
-    rubocop-ast (1.4.1)
-      parser (>= 2.7.1.5)
-    ruby-progressbar (1.11.0)
-    tilt (2.0.10)
-    unicode-display_width (2.0.0)
-
-PLATFORMS
-  ruby
-
-DEPENDENCIES
-  bundler (>= 2)
-  lex-transformer!
-  rake
-  rspec
-  rubocop
-
-BUNDLED WITH
-   2.1.4

data/bitbucket-pipelines.yml

@@ -1,19 +0,0 @@
-image: ruby:2.7
-
-pipelines:
-  tags:
-    "v*":
-      - step:
-          name: Push to RubyGems
-          deployment: RubyGems
-          script:
-            - gem install gem-release
-            - (umask  077 ; echo $gem_creds | base64 --decode > ~/.gem/credentials)
-            - gem release
-          artifacts:
-            - pkg/**
-      - step:
-          name: Push to Docker
-          deployment: Docker
-          script:
-            - './docker_deploy.rb'