legionio 0.3.1 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 054fe7a9c7707f232d0e93719567254ff8b1ca190f0248abc3b8ba1bcba45a84
-  data.tar.gz: 658bd9bb7e565268e15fc768f7b7691eef6c77738ccd4fba1d87b6bf44e9fbb6
+  metadata.gz: 7990b0c192a08c1fd8569cc2a08718c695814136f39c8df08e2b7b4753bfd216
+  data.tar.gz: 3f2bd0e6d8b7108d5ad17a069927bd7d6e5b4e1ba2dc6746ffac3582f11b9cdc
 SHA512:
-  metadata.gz: 98a94510b0408b5f857adaaaa41efe74de4dcc0c509c21d5630001d9aa4a5af16cb2d7ecf5ff0174d310e6f9e159c81948c2e87c44067100d95e106db8b7e390
-  data.tar.gz: 1c21c38abdbfc70e384a09604f88a712e5a1d240409443f7f50a8df75ed4f528f856c1a23b038d89d0f7e7c167d3b645df0ef07378d77ce6b311bbb34d27bd9c
+  metadata.gz: '05488db705432fec76ee9c07cc5655871e37abfbe02db5319fd2fb9ca60ad637c4d16eb44572f4f8cbac38b54ffbee75c49d3ccfadd7b4c9357996779a21f4fa'
+  data.tar.gz: 80fc575ff7e64dc60c60510aebb158c0e0cae2608158f1722e15e64718da43111efc760e4dc8891fc014c42dede039b66bcb99e3ef8ff0b0a80c735d79172f5c

data/.circleci/config.yml

@@ -5,16 +5,15 @@ jobs:
       - image: circleci/ruby:2.5-node
     steps:
       - checkout
-      - run: gem install rubocop
+      - run: gem install rubocop rubocop-rspec rubocop-rake
       - run:
           name: Run Rubocop
           command: rubocop
       - store_test_results:
           path: test_results
-
-  "ruby-two-seven":
+  "ruby-three":
     docker:
-      - image: circleci/ruby:2.7-node
+      - image: circleci/ruby:3-node
       - image: mysql:5.7
         environment:
           MYSQL_DATABASE: 'legion'
@@ -24,6 +23,9 @@ jobs:
       - image: rabbitmq:3.7
     steps:
       - checkout
+      - run:
+          name: bundle update
+          command: gem update bundler
       - run:
           name: Bundle Install
           command: bundle install
@@ -33,9 +35,9 @@ jobs:
           when: always
       - store_test_results:
           path: test-results
-  "ruby-two-five":
+  "ruby-two-seven":
     docker:
-      - image: circleci/ruby:2.5-node
+      - image: circleci/ruby:2.7-node
       - image: mysql:5.7
         environment:
           MYSQL_DATABASE: 'legion'
@@ -45,6 +47,9 @@ jobs:
       - image: rabbitmq:3.7
     steps:
       - checkout
+      - run:
+          name: bundle update
+          command: gem update bundler
       - run:
           name: Bundle Install
           command: bundle install
@@ -54,9 +59,9 @@ jobs:
           when: always
       - store_test_results:
           path: test-results
-  "ruby-two-six":
+  "ruby-two-five":
     docker:
-      - image: circleci/ruby:2.6-node
+      - image: circleci/ruby:2.5-node
       - image: mysql:5.7
         environment:
           MYSQL_DATABASE: 'legion'
@@ -66,6 +71,9 @@ jobs:
       - image: rabbitmq:3.7
     steps:
       - checkout
+      - run:
+          name: bundle update
+          command: gem update bundler
       - run:
           name: Bundle Install
           command: bundle install
@@ -75,9 +83,9 @@ jobs:
           when: always
       - store_test_results:
           path: test-results
-  "jruby-nine-two":
+  "ruby-two-six":
     docker:
-      - image: circleci/jruby:9.2-jra
+      - image: circleci/ruby:2.6-node
       - image: mysql:5.7
         environment:
           MYSQL_DATABASE: 'legion'
@@ -87,6 +95,9 @@ jobs:
       - image: rabbitmq:3.7
     steps:
       - checkout
+      - run:
+          name: bundle update
+          command: gem update bundler
       - run:
           name: Bundle Install
           command: bundle install
@@ -117,3 +128,9 @@ workflows:
           filters:
             branches:
               only: /\bdevelop\b|\bmaster\b/
+      - ruby-three:
+          requires:
+            - ruby-two-five
+          filters:
+            branches:
+              only: /\bdevelop\b|\bmaster\b/

data/.gitignore

@@ -12,3 +12,4 @@
 *.key
 # rspec failure tracking
 .rspec_status
+legionio.key

data/.rubocop.yml

@@ -1,3 +1,8 @@
+require:
+  - rubocop-rspec
+
+RSpec/FilePath:
+  Enabled: false
 Layout/LineLength:
   Max: 160
   IgnoredPatterns:
@@ -19,7 +24,7 @@ Metrics:
     - .trace
 Metrics/MethodLength:
   Max: 50
-  ExcludedMethods:
+  IgnoredMethods:
     - .unknown
     - .fatal
     - .error
@@ -33,7 +38,7 @@ Metrics/ModuleLength:
   Max: 1500
 Metrics/BlockLength:
   Max: 40
-  ExcludedMethods:
+  IgnoredMethods:
     - .unknown
     - .fatal
     - .error

data/CHANGELOG.md

@@ -1,5 +1,26 @@
 # Legion Changelog
 
+## v0.4.0
+* Moving all legion-cli code to inside the legion gem instead of a seperate gem
+* Removing bundled Legion::Data gem from LegionIO(must be installed with gem install legion-data)
+* Removing JRuby support
+* Removing the following gems from gemspec, hashdiff, legion-data, mysql2, sequel, bunny, bundler, codecov
+* Skipping auto install if using bundler. It only works with the binary
+* Adding new docker_deploy.rb script to automatically push a new docker tag with a deployment
+* Adding multiple version dependencies to previously open ended gems
+* Updating Dockerfile to have `--no-document --no-prerelease` flags
+* Updating README
+* Enabling rubocop-rspec
+* Adding support to specify a LEX version to automatically install
+* Updating `legion-transport` gem min version to 1.1.8
+
+## v0.3.6
+* Updating executables to use Legion::CLI
+* Updating bitbucket-pipelines
+* Changing base legion command to legionio and updating Dockerfile
+* Blocking truffleruby from using Legion:Data as it's slow for unknown reasons
+* Changing task update reverting to RMQ message to level debug
+
 ## v0.3.0
 * Updating dependencies based on if we are using `JRUBY` or not
 * Cleaning up some files and rubocop errors

data/Dockerfile

@@ -0,0 +1,9 @@
+FROM ruby:2.7-alpine
+LABEL maintainer="Matthew Iverson <matthewdiverson@gmail.com>"
+
+RUN mkdir /etc/legionio
+RUN apk update && apk add build-base postgresql-dev mysql-client mariadb-dev tzdata gcc git
+
+COPY . ./
+RUN gem install legionio legion-data tzinfo-data tzinfo --no-document --no-prerelease
+CMD legionio

data/Gemfile

@@ -1,14 +1,3 @@
 source 'https://rubygems.org'
 
 gemspec
-
-gem 'lex-conditioner'
-gem 'lex-esphome'
-gem 'lex-health'
-gem 'lex-http'
-gem 'lex-influxdb'
-gem 'lex-lex'
-gem 'lex-log'
-gem 'lex-scheduler', path: '../../lex/lex-scheduler'
-gem 'lex-tasker'
-gem 'lex-transformer'

data/README.md

@@ -1,46 +1,162 @@
-# Legion
+# LegionIO
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library
- into a gem. Put your Ruby code in the file `lib/legion`. To experiment with that code, run `bin/console` for 
- an interactive prompt.
+LegionIO is a framework for automated and connecting things. You can see all the docs inside confluence  
+https://legionio.atlassian.net/wiki/spaces/LEGION/overview
+https://legionio.atlassian.net/wiki/spaces/LEX/pages/7864551/Extensions
 
-
-## Installation
-
-Add this line to your application's Gemfile:
-
-```ruby
-gem 'legion'
+### What does it do?
+LegionIO is an async job engine designed for scheduling tasks and creating relationships between things that wouldn't 
+otherwise be connected. Relationships do not have to be a single path. Both of these would work
+* `foo → bar → cat → dog`
 ```
+a → b → c
+    b → e → z
+        e → g
+```
+In the second scenario, when a runs, it causes b to run which then causes both c and e to run in parallel
+
+It supports both conditions and transformation. The idea of a transformation is you can't connect two indepedent services 
+and expect them to know how to talk to each other. 
+
+### Running
+Run `gem install legionio` to install legion. If you want to use database features, you will need to 
+run `gem install legion-data` also. 
+
+After installing gem you can use the commands `legionio` to start legion, `legion` to access things
+and `lex_gen` to generate a new legion extension
+
+### Example Legion Extensions(LEX)
+* [lex-http](https://bitbucket.org/legion-io/lex-http/src/master/) - Gives legion the ability to make http requests, [docs](https://legionio.atlassian.net/wiki/spaces/LEX/pages/12910593/Lex+Http)
+* [lex-influxdb](https://bitbucket.org/legion-io/lex-influxdb/src/master/) - Write, read, and manage influxdb nodes, [docs](https://legionio.atlassian.net/wiki/spaces/LEX/pages/614891774/Lex+Influxdb)
+* [lex-log](https://bitbucket.org/legion-io/lex-log/src/master/) - Send log items to either stdout or a file with lex-log, [docs](https://legionio.atlassian.net/wiki/spaces/LEX/pages/614858995/Lex+Log)
+* [lex-memcache](https://bitbucket.org/legion-io/lex-memcached/src/master/) - run memcached commands like set, add, append, delete, flush, reset_stats against memcached servers, [docs](https://legionio.atlassian.net/wiki/spaces/LEX/pages/614858753/Lex+Memcached)
+* [lex-pihole](https://bitbucket.org/legion-io/lex-pihole/src/master/) - Allows Legion to interact with [Pi-Hole](https://pi-hole.net/). Can do things like get status, add/remove domains from the list, etc 
+* [lex-ping](https://bitbucket.org/legion-io/lex-ping/src/master/) - You can ping things?, [docs](https://legionio.atlassian.net/wiki/spaces/LEX/pages/631373895/Lex+Ping)
+* [lex-pushover](https://bitbucket.org/legion-io/lex-pushover/src/master/) - Connects Legion to [Pushover](https://pushover.net/), [docs]()
+* [lex-redis](https://bitbucket.org/legion-io/lex-redis/src/master/) - similiar to lex-memcached but for redis
+* [lex-sleepiq](https://bitbucket.org/legion-io/lex-sleepiq/src/master/) - Control your SleepIQ bed with Legion!
+* [lex-ssh](https://bitbucket.org/legion-io/lex-ssh/src/master/) - Send commands to a server via SSH in an async fashion, [docs](https://legionio.atlassian.net/wiki/spaces/LEX/pages/614891551/Lex+SSH)
+
+Bitbucket repos for extensions that are active or being worked on
+[lex list](https://bitbucket.org/legion-io/workspace/projects/LEX)
+A nice list in the wiki to view all the extensions, their docs and status
+[Legion Extensions](https://legionio.atlassian.net/wiki/spaces/LEX/pages/7864551/Extensions)
+
+### Scheduling Tasks
+1) Ensure you have the Legion::Data gem installed and configured
+2) Make sure to have `lex-scheduler` extension installed so that it generates the schedules table in the database
+3) From there you can add a function to be run at a given cron syntax or interval
+4) Setting the interval column will make the job run X seconds after the last time it is completed and will ignore the cron colum
+5) Setting the cron column will ensure the job runs at the given times regardless of when it was run last, only works if interval is null
+6) Cron supports both `*/5 * * * *` style and verbose like `every minute` and `every day at noon`
+
+
+### Creating Relationships
+To be populated
+
+### Conditions
+You can create complex conditional statements to ensure that when a triggers b, b only runs if certain conditions 
+are met. Example conditional statement
+```json
+{
+  "all": [{
+    "fact": "pet.type",
+	"value": "dog",
+	"operator": "equal"
+  },{
+	"fact":"pet.hungry",
+	"operator":"is_true"
+  }]
+}
 
-And then execute:
-
-    $ bundle
-
-Or install it yourself as:
-
-    $ gem install legion
-
-## Usage
-
-TODO: Write usage instructions here
-
-## Development
-
-After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests.
- You can also run `bin/console` for an interactive prompt that will allow you to experiment.
-
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version,
- update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git 
- tag for the version, push git commits and tags, and push the `.gem` file 
- to [rubygems.org](https://rubygems.org).
-
-## Contributing
-
-Bug reports and pull requests are welcome on GitHub at https://bitbucket.org/whonodes/legion/issues.
- This project is intended to be a safe, welcoming space for collaboration, and contributors are 
- expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
-
-## License
-
-The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+```
+You can nest conditions in an unlimited fashion to create and/or scenarios to meet your needs
+```json
+{
+  "all": [
+	"any":[
+	  {"fact":"pet.type", "value":"dog","operator":"equal"},
+	  {"fact":"pet.type", "value":"cat","operator":"equal"}
+	],
+	{
+	  "fact": "pet.hungry",
+	  "operator": "is_true"
+	},{
+	  "fact":"pet.overweight",
+	  "operator":"is_false"
+	}]
+}
+```
+*Conditions are supported by the `lex-conditioner` extension and are not required to be run inside the legion framework*  
+You can read the docs with more examples in the [wiki](https://legionio.atlassian.net/wiki/spaces/LEX/pages/614957181/Lex+Conditioner)
+
+
+### Transformations
+Transformations are a critical piece of interconnecting two independent items. Without it, service B doesn't know what
+to do with the result from service A
+`lex-conditioner` uses a combination of the [tilt](https://rubygems.org/gems/tilt) gem and erb style syntax.
+##### Examples
+Creating a new pagerduty incident 
+```json
+{"message":"New PagerDuty incident assigned to <%= assignee %> with a priority of <%= severity %>","from":"PagerDuty"}
+```
+Example transformation to make the `lex-log` extension output a message
+```json
+{"message":"transform2","level":"fatal"}
+```
+You can also call Legion services to get the data you need, example sending a pushover message
+```json
+{"message":"This is my pushover body", "title": "this is my title", "token":"<%= Legion::Settings['lex']['pushover']['token'] %>" }
+```
+Or if you wanted to make a real time call via `Legion::Crypt` to get a [Hashicorp Vault](https://www.vaultproject.io/) value
+```json
+{"message":"this is another body", "title":"vault token example", "token":"<%= Legion::Crypt.read('pushover/token') %> "}
+```
+*Transformations are supported by the `lex-transformation` extension and are not "technically" required to be run inside the legion framework*  
+You can read the docs with more examples in the [wiki](https://legionio.atlassian.net/wiki/spaces/LEX/pages/612270222/Lex+Transformer)
+
+## FAQ
+### Does it scale?
+Yes. Actually quite well. The framework uses RabbitMQ to ensure jobs are scheduled and run in a FIFO order. As you add
+more works, it just subscribes to the queues the workers can support and does more work. It is really geared towards a
+docker/K8 type of environment however it can be run locally, on a VM, etc.   
+
+As of right now, it has been tested to around 100 workers running in docker without any performance issues. You will 
+likely see performance issues on the DB or RabbitMQ side before Legion has issues. 
+
+Another benefit is that you can run multiple LEXs in one worker or you could have dedicated workers that only run a single LEX.  
+In example if you have to make a ton of ssh connections via `lex-ssh`, maybe you want to run 10 pods with no other extensions in them
+but then run a pod with `lex-pagerduty`, `lex-log` and `lex-http` to send out notifications after each ssh task is completed
+
+### High Availability
+Because you can run this thing with multiple processes and it will distribute the work, it is naturally HA oriented. 
+if a worker goes down for some reason, another one should pick it up(assuming another work has that LEX enabled). There
+are no hidden features, pay walls, etc to get HA. Just run more instances of LegionIO
+
+### Price and License
+LegionIO is completely free. It was build using free time. There are no features held back, no private repos.
+Everything is under an MIT license to keep it as open as possible. With that, the devs can't always help with support,
+well because it's free.
+
+### Who is it geared for?
+Anyone? Everyone? It could be used in a homelab to automate updating VMs. It could be used by someone to take ESPHome
+sensor data and pipe it to influxdb. At least that is what @Esity does. It could also be used by a company or enterprise looking
+to replace other tools.
+
+### But it is written in ruby
+Yep. 
+
+### Similiar projects
+There are multiple projects that are similiar. Some things like IFTTT are great(but is it?) but then again, cost money.
+* [Node-Red](https://nodered.org/) - No HA but has some good features and a great drag and drop interface
+* [n8n.io](https://n8n.io/) - Working on HA but [not there yet](https://github.com/n8n-io/n8n/pull/1294)
+* [StackStorm](https://stackstorm.com/) - Written in Python, has potential but I feel they are removing features to convince you to pay for it
+* [Jenkins](https://www.jenkins.io/) - It's jenkins. I don't need to say anything else
+* [Huginn]() - Another IFTTT style app written in ruby. Not sure on this one but it doesn't have HA from what I can tell [github issue](https://github.com/huginn/huginn/issues/2198)
+
+### Other fun facts
+* Supports Hashicorp vault for storing secrets/settings/etc
+* Can enable global message encryption so that all messages going through RMQ are encrypted with aes-256-cbc
+* Each worker generates a private/public key that can be used for internode communication, it also will generate a cluster secret
+for all nodes to have so they can share data accross the entire cluster. The cluster secret by default is stored only in memory and
+and is generated when the first worker starts

data/Rakefile

@@ -29,10 +29,4 @@ namespace :deploy do
     puts Gem.methods(false)
     puts Legion::VERSION
   end
-
-  task :staging do
-  end
-
-  task :production do
-  end
 end

data/bitbucket-pipelines.yml

@@ -1,55 +1,20 @@
-image: ruby:2.5.0
+image: ruby:2.7
 
 pipelines:
   tags:
-    '0.1.0':
-    - step:
-        caches:
-        - bundler
-        deployment: rubygems
-        script:
-          - mkdir -p ~/.gem/credentials
-          - (umask  077 ; echo $gem_creds | base64 --decode > ~/.gem/credentials)
-          - gem build legion.gemspec
-          - gem push legionio-*.gem
-  branches:
-    master:
-    - step:
-        caches:
-        - bundler
-        script:
-        - gem install bundle rubocop
-        - bundle install
-        - rubocop
-        - rake
-        services:
-        - mysql
-        - broker
-  default:
-    - step:
-        caches:
-          - bundler
-        script:
-          - gem install bundle rubocop
-          - bundle install
-          - rubocop
-          - rake
-        services:
-          - mysql
-          - broker
-definitions:
-  caches:
-    bundler: vendor/bundle
-  services:
-    mysql:
-      image: mysql:5.7
-      environment:
-        MYSQL_DATABASE: 'legion'
-        MYSQL_ROOT_PASSWORD: 'legion'
-        MYSQL_USER: 'legion'
-        MYSQL_PASSWORD: 'legion'
-    broker:
-      image: rabbitmq:3
-      environment:
-        RABBITMQ_DEFAULT_USER: guest
-        RABBITMQ_DEFAULT_PASS: guest
+    "v*":
+      - step:
+          name: Push to RubyGems
+          deployment: RubyGems
+          script:
+            - gem install gem-release
+            - (umask  077 ; echo $gem_creds | base64 --decode > ~/.gem/credentials)
+            - gem release
+            - sleep(2)
+          artifacts:
+            - pkg/**
+      - step:
+          name: Push to Docker
+          deployment: Docker
+          script:
+            - './docker_deploy.rb'