legionio 0.3.5 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 975a4241d50fa9fb365969e764bd3c1efd21f4633a08219562091be186c2dbf6
-  data.tar.gz: 1abdd099d6ca58f013d9c04e4b8e662aa9d58d73c83580308029326190ab9acd
+  metadata.gz: 7990b0c192a08c1fd8569cc2a08718c695814136f39c8df08e2b7b4753bfd216
+  data.tar.gz: 3f2bd0e6d8b7108d5ad17a069927bd7d6e5b4e1ba2dc6746ffac3582f11b9cdc
 SHA512:
-  metadata.gz: 8103b343813c674ead056c47d778023011c62b74b2878a15ed9e58b1c74cbf93f7565b4fe89dfab4ae27c4dcf35ab99a4e7ff6f628bfbf8a6902669cdb2ea026
-  data.tar.gz: 2c8be72aebd8831bb88881a4de35fca123d67d6247181811a85dd9340c7cdf6743f9e7cb10f528307d250b3ed312193036f486f5905147b7eda11643c61935be
+  metadata.gz: '05488db705432fec76ee9c07cc5655871e37abfbe02db5319fd2fb9ca60ad637c4d16eb44572f4f8cbac38b54ffbee75c49d3ccfadd7b4c9357996779a21f4fa'
+  data.tar.gz: 80fc575ff7e64dc60c60510aebb158c0e0cae2608158f1722e15e64718da43111efc760e4dc8891fc014c42dede039b66bcb99e3ef8ff0b0a80c735d79172f5c

data/.circleci/config.yml

@@ -0,0 +1,136 @@
+version: 2.1 # use CircleCI 2.0
+jobs:
+  "rubocop":
+    docker:
+      - image: circleci/ruby:2.5-node
+    steps:
+      - checkout
+      - run: gem install rubocop rubocop-rspec rubocop-rake
+      - run:
+          name: Run Rubocop
+          command: rubocop
+      - store_test_results:
+          path: test_results
+  "ruby-three":
+    docker:
+      - image: circleci/ruby:3-node
+      - image: mysql:5.7
+        environment:
+          MYSQL_DATABASE: 'legion'
+          MYSQL_ROOT_PASSWORD: 'legion'
+          MYSQL_USER: 'legion'
+          MYSQL_PASSWORD: 'legion'
+      - image: rabbitmq:3.7
+    steps:
+      - checkout
+      - run:
+          name: bundle update
+          command: gem update bundler
+      - run:
+          name: Bundle Install
+          command: bundle install
+      - run:
+          name: Run RSpec
+          command: bundle exec rspec --format progress --format RspecJunitFormatter -o test-results/rspec/results.xml
+          when: always
+      - store_test_results:
+          path: test-results
+  "ruby-two-seven":
+    docker:
+      - image: circleci/ruby:2.7-node
+      - image: mysql:5.7
+        environment:
+          MYSQL_DATABASE: 'legion'
+          MYSQL_ROOT_PASSWORD: 'legion'
+          MYSQL_USER: 'legion'
+          MYSQL_PASSWORD: 'legion'
+      - image: rabbitmq:3.7
+    steps:
+      - checkout
+      - run:
+          name: bundle update
+          command: gem update bundler
+      - run:
+          name: Bundle Install
+          command: bundle install
+      - run:
+          name: Run RSpec
+          command: bundle exec rspec --format progress --format RspecJunitFormatter -o test-results/rspec/results.xml
+          when: always
+      - store_test_results:
+          path: test-results
+  "ruby-two-five":
+    docker:
+      - image: circleci/ruby:2.5-node
+      - image: mysql:5.7
+        environment:
+          MYSQL_DATABASE: 'legion'
+          MYSQL_ROOT_PASSWORD: 'legion'
+          MYSQL_USER: 'legion'
+          MYSQL_PASSWORD: 'legion'
+      - image: rabbitmq:3.7
+    steps:
+      - checkout
+      - run:
+          name: bundle update
+          command: gem update bundler
+      - run:
+          name: Bundle Install
+          command: bundle install
+      - run:
+          name: Run RSpec
+          command: bundle exec rspec --format progress --format RspecJunitFormatter -o test-results/rspec/results.xml
+          when: always
+      - store_test_results:
+          path: test-results
+  "ruby-two-six":
+    docker:
+      - image: circleci/ruby:2.6-node
+      - image: mysql:5.7
+        environment:
+          MYSQL_DATABASE: 'legion'
+          MYSQL_ROOT_PASSWORD: 'legion'
+          MYSQL_USER: 'legion'
+          MYSQL_PASSWORD: 'legion'
+      - image: rabbitmq:3.7
+    steps:
+      - checkout
+      - run:
+          name: bundle update
+          command: gem update bundler
+      - run:
+          name: Bundle Install
+          command: bundle install
+      - run:
+          name: Run RSpec
+          command: bundle exec rspec --format progress --format RspecJunitFormatter -o test-results/rspec/results.xml
+          when: always
+      - store_test_results:
+          path: test-results
+
+workflows:
+  version: 2
+  cop_rake_deploy:
+    jobs:
+      - rubocop
+      - ruby-two-seven:
+          requires:
+            - ruby-two-five
+          filters:
+            branches:
+              only: /\bdevelop\b|\bmaster\b/
+      - ruby-two-five:
+          requires:
+            - rubocop
+      - ruby-two-six:
+          requires:
+            - ruby-two-five
+          filters:
+            branches:
+              only: /\bdevelop\b|\bmaster\b/
+      - ruby-three:
+          requires:
+            - ruby-two-five
+          filters:
+            branches:
+              only: /\bdevelop\b|\bmaster\b/

data/.gitignore

@@ -0,0 +1,15 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+/legion/.idea/
+/.idea/
+*.key
+# rspec failure tracking
+.rspec_status
+legionio.key

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.rubocop.yml

@@ -0,0 +1,94 @@
+require:
+  - rubocop-rspec
+
+RSpec/FilePath:
+  Enabled: false
+Layout/LineLength:
+  Max: 160
+  IgnoredPatterns:
+    - .unknown
+    - .fatal
+    - .error
+    - .warn
+    - .info
+    - .debug
+    - .trace
+Metrics:
+  IgnorePatterns:
+    - .unknown
+    - .fatal
+    - .error
+    - .warn
+    - .info
+    - .debug
+    - .trace
+Metrics/MethodLength:
+  Max: 50
+  IgnoredMethods:
+    - .unknown
+    - .fatal
+    - .error
+    - .warn
+    - .info
+    - .debug
+    - .trace
+Metrics/ClassLength:
+  Max: 1500
+Metrics/ModuleLength:
+  Max: 1500
+Metrics/BlockLength:
+  Max: 40
+  IgnoredMethods:
+    - .unknown
+    - .fatal
+    - .error
+    - .warn
+    - .info
+    - .debug
+    - .trace
+Metrics/AbcSize:
+  Max: 50
+  IgnoredMethods:
+    - .unknown
+    - .fatal
+    - .error
+    - .warn
+    - .info
+    - .debug
+    - .trace
+Metrics/CyclomaticComplexity:
+  Max: 15
+  IgnoredMethods:
+    - .unknown
+    - .fatal
+    - .error
+    - .warn
+    - .info
+    - .debug
+    - .trace
+Metrics/PerceivedComplexity:
+  Max: 17
+  IgnoredMethods:
+    - .unknown
+    - .fatal
+    - .error
+    - .warn
+    - .info
+    - .debug
+    - .trace
+Layout/SpaceAroundEqualsInParameterDefault:
+  EnforcedStyle: space
+Style/SymbolArray:
+  Enabled: true
+Layout/HashAlignment:
+  EnforcedHashRocketStyle: table
+  EnforcedColonStyle: table
+Style/Documentation:
+  Enabled: false
+AllCops:
+  TargetRubyVersion: 2.5
+  NewCops: enable
+Style/FrozenStringLiteralComment:
+  Enabled: false
+Naming/FileName:
+  Enabled: false

data/CHANGELOG.md

@@ -0,0 +1,37 @@
+# Legion Changelog
+
+## v0.4.0
+* Moving all legion-cli code to inside the legion gem instead of a seperate gem
+* Removing bundled Legion::Data gem from LegionIO(must be installed with gem install legion-data)
+* Removing JRuby support
+* Removing the following gems from gemspec, hashdiff, legion-data, mysql2, sequel, bunny, bundler, codecov
+* Skipping auto install if using bundler. It only works with the binary
+* Adding new docker_deploy.rb script to automatically push a new docker tag with a deployment
+* Adding multiple version dependencies to previously open ended gems
+* Updating Dockerfile to have `--no-document --no-prerelease` flags
+* Updating README
+* Enabling rubocop-rspec
+* Adding support to specify a LEX version to automatically install
+* Updating `legion-transport` gem min version to 1.1.8
+
+## v0.3.6
+* Updating executables to use Legion::CLI
+* Updating bitbucket-pipelines
+* Changing base legion command to legionio and updating Dockerfile
+* Blocking truffleruby from using Legion:Data as it's slow for unknown reasons
+* Changing task update reverting to RMQ message to level debug
+
+## v0.3.0
+* Updating dependencies based on if we are using `JRUBY` or not
+* Cleaning up some files and rubocop errors
+* Adding new settings option for a lex definition with functionality to add it to the override settings stack
+* Fixing supervision filename
+* Cleaning up `Legion::Service`
+* Cleaning up rogue comments
+* Updating exe/legion to match bin/legion
+
+## v0.1.1
+Testing
+
+## v0.1.0
+Initial Release

data/Dockerfile

@@ -0,0 +1,9 @@
+FROM ruby:2.7-alpine
+LABEL maintainer="Matthew Iverson <matthewdiverson@gmail.com>"
+
+RUN mkdir /etc/legionio
+RUN apk update && apk add build-base postgresql-dev mysql-client mariadb-dev tzdata gcc git
+
+COPY . ./
+RUN gem install legionio legion-data tzinfo-data tzinfo --no-document --no-prerelease
+CMD legionio

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 BuildTools
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,162 @@
+# LegionIO
+
+LegionIO is a framework for automated and connecting things. You can see all the docs inside confluence  
+https://legionio.atlassian.net/wiki/spaces/LEGION/overview
+https://legionio.atlassian.net/wiki/spaces/LEX/pages/7864551/Extensions
+
+### What does it do?
+LegionIO is an async job engine designed for scheduling tasks and creating relationships between things that wouldn't 
+otherwise be connected. Relationships do not have to be a single path. Both of these would work
+* `foo → bar → cat → dog`
+```
+a → b → c
+    b → e → z
+        e → g
+```
+In the second scenario, when a runs, it causes b to run which then causes both c and e to run in parallel
+
+It supports both conditions and transformation. The idea of a transformation is you can't connect two indepedent services 
+and expect them to know how to talk to each other. 
+
+### Running
+Run `gem install legionio` to install legion. If you want to use database features, you will need to 
+run `gem install legion-data` also. 
+
+After installing gem you can use the commands `legionio` to start legion, `legion` to access things
+and `lex_gen` to generate a new legion extension
+
+### Example Legion Extensions(LEX)
+* [lex-http](https://bitbucket.org/legion-io/lex-http/src/master/) - Gives legion the ability to make http requests, [docs](https://legionio.atlassian.net/wiki/spaces/LEX/pages/12910593/Lex+Http)
+* [lex-influxdb](https://bitbucket.org/legion-io/lex-influxdb/src/master/) - Write, read, and manage influxdb nodes, [docs](https://legionio.atlassian.net/wiki/spaces/LEX/pages/614891774/Lex+Influxdb)
+* [lex-log](https://bitbucket.org/legion-io/lex-log/src/master/) - Send log items to either stdout or a file with lex-log, [docs](https://legionio.atlassian.net/wiki/spaces/LEX/pages/614858995/Lex+Log)
+* [lex-memcache](https://bitbucket.org/legion-io/lex-memcached/src/master/) - run memcached commands like set, add, append, delete, flush, reset_stats against memcached servers, [docs](https://legionio.atlassian.net/wiki/spaces/LEX/pages/614858753/Lex+Memcached)
+* [lex-pihole](https://bitbucket.org/legion-io/lex-pihole/src/master/) - Allows Legion to interact with [Pi-Hole](https://pi-hole.net/). Can do things like get status, add/remove domains from the list, etc 
+* [lex-ping](https://bitbucket.org/legion-io/lex-ping/src/master/) - You can ping things?, [docs](https://legionio.atlassian.net/wiki/spaces/LEX/pages/631373895/Lex+Ping)
+* [lex-pushover](https://bitbucket.org/legion-io/lex-pushover/src/master/) - Connects Legion to [Pushover](https://pushover.net/), [docs]()
+* [lex-redis](https://bitbucket.org/legion-io/lex-redis/src/master/) - similiar to lex-memcached but for redis
+* [lex-sleepiq](https://bitbucket.org/legion-io/lex-sleepiq/src/master/) - Control your SleepIQ bed with Legion!
+* [lex-ssh](https://bitbucket.org/legion-io/lex-ssh/src/master/) - Send commands to a server via SSH in an async fashion, [docs](https://legionio.atlassian.net/wiki/spaces/LEX/pages/614891551/Lex+SSH)
+
+Bitbucket repos for extensions that are active or being worked on
+[lex list](https://bitbucket.org/legion-io/workspace/projects/LEX)
+A nice list in the wiki to view all the extensions, their docs and status
+[Legion Extensions](https://legionio.atlassian.net/wiki/spaces/LEX/pages/7864551/Extensions)
+
+### Scheduling Tasks
+1) Ensure you have the Legion::Data gem installed and configured
+2) Make sure to have `lex-scheduler` extension installed so that it generates the schedules table in the database
+3) From there you can add a function to be run at a given cron syntax or interval
+4) Setting the interval column will make the job run X seconds after the last time it is completed and will ignore the cron colum
+5) Setting the cron column will ensure the job runs at the given times regardless of when it was run last, only works if interval is null
+6) Cron supports both `*/5 * * * *` style and verbose like `every minute` and `every day at noon`
+
+
+### Creating Relationships
+To be populated
+
+### Conditions
+You can create complex conditional statements to ensure that when a triggers b, b only runs if certain conditions 
+are met. Example conditional statement
+```json
+{
+  "all": [{
+    "fact": "pet.type",
+	"value": "dog",
+	"operator": "equal"
+  },{
+	"fact":"pet.hungry",
+	"operator":"is_true"
+  }]
+}
+
+```
+You can nest conditions in an unlimited fashion to create and/or scenarios to meet your needs
+```json
+{
+  "all": [
+	"any":[
+	  {"fact":"pet.type", "value":"dog","operator":"equal"},
+	  {"fact":"pet.type", "value":"cat","operator":"equal"}
+	],
+	{
+	  "fact": "pet.hungry",
+	  "operator": "is_true"
+	},{
+	  "fact":"pet.overweight",
+	  "operator":"is_false"
+	}]
+}
+```
+*Conditions are supported by the `lex-conditioner` extension and are not required to be run inside the legion framework*  
+You can read the docs with more examples in the [wiki](https://legionio.atlassian.net/wiki/spaces/LEX/pages/614957181/Lex+Conditioner)
+
+
+### Transformations
+Transformations are a critical piece of interconnecting two independent items. Without it, service B doesn't know what
+to do with the result from service A
+`lex-conditioner` uses a combination of the [tilt](https://rubygems.org/gems/tilt) gem and erb style syntax.
+##### Examples
+Creating a new pagerduty incident 
+```json
+{"message":"New PagerDuty incident assigned to <%= assignee %> with a priority of <%= severity %>","from":"PagerDuty"}
+```
+Example transformation to make the `lex-log` extension output a message
+```json
+{"message":"transform2","level":"fatal"}
+```
+You can also call Legion services to get the data you need, example sending a pushover message
+```json
+{"message":"This is my pushover body", "title": "this is my title", "token":"<%= Legion::Settings['lex']['pushover']['token'] %>" }
+```
+Or if you wanted to make a real time call via `Legion::Crypt` to get a [Hashicorp Vault](https://www.vaultproject.io/) value
+```json
+{"message":"this is another body", "title":"vault token example", "token":"<%= Legion::Crypt.read('pushover/token') %> "}
+```
+*Transformations are supported by the `lex-transformation` extension and are not "technically" required to be run inside the legion framework*  
+You can read the docs with more examples in the [wiki](https://legionio.atlassian.net/wiki/spaces/LEX/pages/612270222/Lex+Transformer)
+
+## FAQ
+### Does it scale?
+Yes. Actually quite well. The framework uses RabbitMQ to ensure jobs are scheduled and run in a FIFO order. As you add
+more works, it just subscribes to the queues the workers can support and does more work. It is really geared towards a
+docker/K8 type of environment however it can be run locally, on a VM, etc.   
+
+As of right now, it has been tested to around 100 workers running in docker without any performance issues. You will 
+likely see performance issues on the DB or RabbitMQ side before Legion has issues. 
+
+Another benefit is that you can run multiple LEXs in one worker or you could have dedicated workers that only run a single LEX.  
+In example if you have to make a ton of ssh connections via `lex-ssh`, maybe you want to run 10 pods with no other extensions in them
+but then run a pod with `lex-pagerduty`, `lex-log` and `lex-http` to send out notifications after each ssh task is completed
+
+### High Availability
+Because you can run this thing with multiple processes and it will distribute the work, it is naturally HA oriented. 
+if a worker goes down for some reason, another one should pick it up(assuming another work has that LEX enabled). There
+are no hidden features, pay walls, etc to get HA. Just run more instances of LegionIO
+
+### Price and License
+LegionIO is completely free. It was build using free time. There are no features held back, no private repos.
+Everything is under an MIT license to keep it as open as possible. With that, the devs can't always help with support,
+well because it's free.
+
+### Who is it geared for?
+Anyone? Everyone? It could be used in a homelab to automate updating VMs. It could be used by someone to take ESPHome
+sensor data and pipe it to influxdb. At least that is what @Esity does. It could also be used by a company or enterprise looking
+to replace other tools.
+
+### But it is written in ruby
+Yep. 
+
+### Similiar projects
+There are multiple projects that are similiar. Some things like IFTTT are great(but is it?) but then again, cost money.
+* [Node-Red](https://nodered.org/) - No HA but has some good features and a great drag and drop interface
+* [n8n.io](https://n8n.io/) - Working on HA but [not there yet](https://github.com/n8n-io/n8n/pull/1294)
+* [StackStorm](https://stackstorm.com/) - Written in Python, has potential but I feel they are removing features to convince you to pay for it
+* [Jenkins](https://www.jenkins.io/) - It's jenkins. I don't need to say anything else
+* [Huginn]() - Another IFTTT style app written in ruby. Not sure on this one but it doesn't have HA from what I can tell [github issue](https://github.com/huginn/huginn/issues/2198)
+
+### Other fun facts
+* Supports Hashicorp vault for storing secrets/settings/etc
+* Can enable global message encryption so that all messages going through RMQ are encrypted with aes-256-cbc
+* Each worker generates a private/public key that can be used for internode communication, it also will generate a cluster secret
+for all nodes to have so they can share data accross the entire cluster. The cluster secret by default is stored only in memory and
+and is generated when the first worker starts