archfiend 0.1.0

data/Gemfile

@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+
+git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in archfiend.gemspec
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,122 @@
+PATH
+  remote: .
+  specs:
+    archfiend (0.1.0)
+      activesupport
+      oj
+      pry
+      thor
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activemodel (5.2.1)
+      activesupport (= 5.2.1)
+    activerecord (5.2.1)
+      activemodel (= 5.2.1)
+      activesupport (= 5.2.1)
+      arel (>= 9.0)
+    activesupport (5.2.1)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 0.7, < 2)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+    arel (9.0.0)
+    ast (2.4.0)
+    coderay (1.1.2)
+    concurrent-ruby (1.0.5)
+    config (1.7.0)
+      activesupport (>= 3.0)
+      deep_merge (~> 1.2.1)
+      dry-validation (>= 0.10.4)
+    deep_merge (1.2.1)
+    diff-lcs (1.3)
+    dry-configurable (0.7.0)
+      concurrent-ruby (~> 1.0)
+    dry-container (0.6.0)
+      concurrent-ruby (~> 1.0)
+      dry-configurable (~> 0.1, >= 0.1.3)
+    dry-core (0.4.7)
+      concurrent-ruby (~> 1.0)
+    dry-equalizer (0.2.1)
+    dry-inflector (0.1.2)
+    dry-logic (0.4.2)
+      dry-container (~> 0.2, >= 0.2.6)
+      dry-core (~> 0.2)
+      dry-equalizer (~> 0.2)
+    dry-types (0.13.2)
+      concurrent-ruby (~> 1.0)
+      dry-container (~> 0.3)
+      dry-core (~> 0.4, >= 0.4.4)
+      dry-equalizer (~> 0.2)
+      dry-inflector (~> 0.1, >= 0.1.2)
+      dry-logic (~> 0.4, >= 0.4.2)
+    dry-validation (0.12.2)
+      concurrent-ruby (~> 1.0)
+      dry-configurable (~> 0.1, >= 0.1.3)
+      dry-core (~> 0.2, >= 0.2.1)
+      dry-equalizer (~> 0.2)
+      dry-logic (~> 0.4, >= 0.4.0)
+      dry-types (~> 0.13.1)
+    i18n (1.1.0)
+      concurrent-ruby (~> 1.0)
+    jaro_winkler (1.5.1)
+    method_source (0.9.0)
+    minitest (5.11.3)
+    oj (3.6.7)
+    parallel (1.12.1)
+    parser (2.5.1.2)
+      ast (~> 2.4.0)
+    pg (0.21.0)
+    powerpack (0.1.2)
+    pry (0.11.3)
+      coderay (~> 1.1.0)
+      method_source (~> 0.9.0)
+    rainbow (3.0.0)
+    rake (10.5.0)
+    rspec (3.8.0)
+      rspec-core (~> 3.8.0)
+      rspec-expectations (~> 3.8.0)
+      rspec-mocks (~> 3.8.0)
+    rspec-core (3.8.0)
+      rspec-support (~> 3.8.0)
+    rspec-expectations (3.8.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-mocks (3.8.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-support (3.8.0)
+    rubocop (0.58.2)
+      jaro_winkler (~> 1.5.1)
+      parallel (~> 1.10)
+      parser (>= 2.5, != 2.5.1.1)
+      powerpack (~> 0.1)
+      rainbow (>= 2.2.2, < 4.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (~> 1.0, >= 1.0.1)
+    rubocop-rspec (1.29.1)
+      rubocop (>= 0.58.0)
+    ruby-progressbar (1.10.0)
+    thor (0.20.0)
+    thread_safe (0.3.6)
+    tzinfo (1.2.5)
+      thread_safe (~> 0.1)
+    unicode-display_width (1.4.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activerecord
+  archfiend!
+  bundler (~> 1.15)
+  config
+  pg (~> 0.21)
+  rake (~> 10.0)
+  rspec (~> 3.0)
+  rubocop
+  rubocop-rspec
+
+BUNDLED WITH
+   1.16.4

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2018 Toptal, Maciek Dubiński
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,114 @@
+# Archfiend
+
+Archfiend (/ˈɑrtʃˈfind/) - a basic daemon generator. It features boilerplate for common development tools (ActiveRecord, RSpec, FactoryBot, RuboCop),
+handles daemonizing, configuration, logging and more. It provides a way to structure backend applications in an organized manner.
+
+[![Build Status](https://travis-ci.com/toptal/archfiend.svg?branch=master)](https://travis-ci.com/toptal/archfiend)
+
+## Contents
+* [Getting started](#getting-started)
+* [Primitives](#primitives)
+* [Featured gems](#featured-gems)
+* [Changelog](#changelog)
+* [Contributing](#contributing)
+* [License](#license)
+* [Code of Conduct](#code-of-conduct)
+
+## Getting started
+
+Archfiend provides the `archfiend` executable, which features a daemon generator.
+
+```bash
+  $ bundle exec archfiend new my_new_daemon # Creates the new daemon
+  $ cd my_new_daemon
+  $ cp config/database.yml{.example,}       # Currently, it ships with ActiveRecord, database configuration is required
+  $ bin/start                               # Start without daemonization
+  2018-10-09T13:59:55.874Z 88087 TID-ovtqe0m53 INFO: Starting up
+  2018-10-09T13:59:56.301Z 88087 TID-ovtqu0g3f INFO: Starting thread BarThreadLoop
+  2018-10-09T13:59:56.303Z 88431 TID-ovtqe0m53 INFO: Starting subprocess FooSubprocessLoop
+```
+
+## Primitives
+
+Daemons generated with Archfiend are structured around `ThreadLoops` and `SubprocessLoops`.
+They provide entry points for any user code. There is no limit on the number of each, you should
+use your judgement on how many your target environment can realistically handle.
+
+### ThreadLoops
+
+When started, Archfiend instantiates each [ThreadLoop](lib/archfiend/thread_loop.rb) subclass in a separate thread and calls its `#run` method.
+By default, `#run` wraps looped execution of `#iterate` with `StandardError` handling.
+You are supposed to either provide the `#iterate` method or shadow the `#run` method to fully take over
+the control of the thread.
+Bear in mind that any unhandled exception will terminate the whole process. If your code deals with
+exceptions that are not subclasses of `StandardError`, make sure you handle it on your own.
+
+With MRI, parallel threads execution happens only when one of threads waits for IO. This should be
+fairly often in network communicating daemons.
+
+### SubprocessLoops
+
+When started, archfiend instantiates each [SubprocessLoop](lib/archfiend/subprocess_loop.rb) subclass in a separate fork and calls its `#run` method.
+By default, `#run` wraps looped execution of `#iterate` with `StandardError` handling.
+You are supposed to either provide the `#iterate` method or shadow the `#run` method to fully take over
+the control of the process.
+Subprocesses are killed when the main daemon process exits. If a subprocess gets terminated, it does not kill the main
+process. The main process doesn't restart its subprocesses by default.
+
+## Featured gems
+
+### ActiveRecord
+
+ActiveRecord is available and configured using the `config/database.yml`, you can create regular AR model files
+inside `app/models`.
+When creating multi-threaded daemons, make sure connection pool is handled correctly (that connections are
+released to the pool after a thread finishes) and that the pool has enough open connections (default is 5).
+
+Migrations are provided by the [ActiveRecord::Migrations](https://github.com/ioquatix/activerecord-migrations) gem, you can use following rake tasks
+```bash
+rake db:migrate                       # Migrate the database (options: VERSION=x, VERBOSE=false, SCOPE=blog)
+rake db:rollback                      # Rolls the schema back to the previous version (specify steps w/ STEP=n)
+rake db:migrations:new[name,options]  # Creates a new migration file with the specified name
+```
+
+Please see `bundle exec rake -T` for more options.
+
+### Clockwork
+
+Archfiend provides the [Clockwork](https://github.com/Rykian/clockwork) integration, so daemons can schedule recurring tasks.
+Uncomment the contents of the `clockwork/clockwork.rb` file to enable it. If the daemon doesn't use clockwork, feel free
+to remove the file, the directory and the Gemfile entry.
+
+### Settings
+
+The [config](https://github.com/railsconfig/config) gem is used to handle multi-environment settings.
+
+### rbtrace
+
+For debugging, daemons generated with Archfiend include [rbtrace](http://github.com/tmm1/rbtrace).
+By default, it is enabled in the development environment.
+
+### RSpec / RuboCop
+
+Newly generated daemons come with integrations for RSpec and RuboCop.
+
+## Changelog
+
+See [CHANGES.md](CHANGES.md) for the list of changes.
+
+## Roadmap
+
+See [ROADMAP.md](ROADMAP.md) for the uncommitted list of ideas for future.
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/toptal/archfiend.
+This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+
+## Code of Conduct
+
+Everyone interacting in the Archfiend project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/toptal/archfiend/blob/master/CODE_OF_CONDUCT.md).

data/ROADMAP.md

@@ -0,0 +1,11 @@
+# Archfiend Roadmap
+
+The general goal of this project is to have a low-dependency, solid, pluggable Ruby daemon generator.
+
+Ideas include
+* exhaustive documentation for running (`bin/start`, `bin/console`) and development of generated daemons
+* method to opt-out from any integrated gem
+* removal of `daemons` dependency
+* removal of `config` dependency
+* removal of `clockwork` dependency
+* stand-alone generator option, to generate daemons with no external run time dependencies (with Archfiend base classes baked in)

data/Rakefile

@@ -0,0 +1,6 @@
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/archfiend.gemspec

@@ -0,0 +1,35 @@
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'archfiend/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'archfiend'
+  spec.version       = Archfiend::VERSION
+  spec.authors       = ['Maciek Dubiński']
+  spec.email         = ['maciek@dubinski.net']
+
+  spec.summary       = 'A tool to simplify creation and development of Ruby daemons'
+  spec.description   = 'A tool to simplify creation and development of Ruby daemons.'
+  spec.homepage      = 'https://github.com/toptal/archfiend'
+  spec.license       = 'MIT'
+
+  spec.files = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'activesupport' # CLI
+  spec.add_development_dependency 'activerecord'
+  spec.add_development_dependency 'bundler', '~> 1.15'
+  spec.add_development_dependency 'config'
+  spec.add_dependency 'oj'
+  spec.add_development_dependency 'pg', '~> 0.21'
+  spec.add_dependency 'pry'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'rubocop'
+  spec.add_development_dependency 'rubocop-rspec'
+  spec.add_dependency 'thor' # CLI
+end

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'archfiend'
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require 'irb'
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/exe/archfiend

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require 'archfiend'
+
+Archfiend::CLI.start(ARGV)

data/lib/archfiend.rb

@@ -0,0 +1,18 @@
+require 'archfiend/version'
+require 'archfiend/application'
+require 'archfiend/logging'
+require 'archfiend/logging/base_formatter'
+require 'archfiend/logging/default_formatter'
+require 'archfiend/logging/json_formatter'
+require 'archfiend/logging/multi_logger'
+require 'archfiend/shared_loop/runnable'
+require 'archfiend/thread_loop'
+require 'archfiend/subprocess_loop'
+require 'archfiend/generators/daemon'
+require 'archfiend/generators/options'
+require 'archfiend/generators/extensions'
+require 'archfiend/generators/utils'
+require 'archfiend/cli'
+
+module Archfiend
+end

data/lib/archfiend/application.rb

@@ -0,0 +1,132 @@
+module Archfiend
+  class Application
+    extend Forwardable
+
+    def_delegators :utils, :logger, :env, :name
+
+    # The main application entry point, it starts up all the subthreads, all the subprocesses,
+    # registers the exit handler and then it blocks the execution.
+    def run
+      logger.info 'Starting up'
+
+      setup
+
+      ThreadLoop.start_all(self)
+      SubprocessLoop.start_all(self)
+
+      run_clockwork
+
+      # Setup after subprocesses are created, so the handler is not copied to them
+      setup_cleanup
+      loop { sleep 1 }
+    end
+
+    def setup
+      return if @already_setup
+
+      setup_timezone
+      setup_settings
+      setup_logger
+      setup_activerecord
+      setup_debug
+
+      run_initializers
+      require_app_classes
+
+      @already_setup = true
+    end
+
+    private
+
+    def run_initializers
+      Dir[utils.root.join('config', 'initializers', '**', '*.rb')].sort.each do |ruby_file|
+        load ruby_file
+      end
+    end
+
+    def require_app_classes
+      rb_pattern = utils.root.join('app', '**', '*.rb')
+      Dir[rb_pattern].each { |file_name| require file_name }
+    end
+
+    def setup_activerecord
+      # TODO: ensure connection timezone is correct
+
+      yaml_text = ERB.new(IO.read(utils.root.join('config', 'database.yml'))).result
+      ::ActiveRecord::Base.configurations = YAML.load(yaml_text) # rubocop:disable Security/YAMLLoad
+      ::ActiveRecord::Base.establish_connection(env.to_sym)
+      ::ActiveRecord::Base.logger = utils.logger if %w[development test].include?(env)
+    end
+
+    def setup_timezone
+      Time.zone_default = ActiveSupport::TimeZone['UTC']
+    end
+
+    def setup_settings
+      ::Config.load_and_set_settings(::Config.setting_files(utils.root.join('config'), env))
+    end
+
+    # Starts a new thread that runs tasks defined in the Clockwork module.
+    def run_clockwork
+      return unless self.class.const_defined?('Clockwork')
+
+      th = Thread.new do
+        ::Clockwork.run
+      end
+      th[:name] = 'Clockwork'
+    end
+
+    POSSIBLE_LOGGER_LEVELS = %i[debug info warn error fatal unknown].freeze
+    def setup_logger
+      logger_level = Settings.logger&.level
+      fail "Please set logger.level setting (#{POSSIBLE_LOGGER_LEVELS.inspect})" unless POSSIBLE_LOGGER_LEVELS.include?(logger_level)
+      utils.logger.level = logger_level
+      utils.logger.progname = name
+    end
+
+    # Registers actions to be performed when the ruby vm exits
+    def setup_cleanup
+      at_exit do
+        utils.logger.info 'Exiting'
+        SubprocessLoop.kill_all
+      end
+    end
+
+    def setup_debug
+      return unless Settings.debug&.rbtrace
+
+      require 'rbtrace'
+    rescue LoadError => e
+      puts 'Settings.debug.rbtrace is true but cannot load the "rbtrace" gem, make sure it is present in Gemfile'
+      puts "(#{e})"
+      exit 1
+    end
+  end
+
+  module Utilities
+    def logger
+      @logger ||= Archfiend::Logging.create(env, root.join('log'))
+    end
+
+    def app
+      @app ||= const_get('Application').new
+    end
+
+    def env
+      ENV['APP_ENV'] || ENV['RAILS_ENV'] || 'development'
+    end
+
+    # Returns all dependency groups for loading based on:
+    # * The App environment;
+    # * The environment variable APP_GROUPS;
+    # @return [Array<string>] All gem groups that needs to be included for current env
+    def groups(*groups)
+      groups.unshift(:default, env.to_sym)
+      env_groups = ENV['APP_GROUPS'] || ENV['RAILS_GROUPS']
+      groups.concat env_groups.to_s.split(',')
+      groups.compact!
+      groups.uniq!
+      groups
+    end
+  end
+end