desmond 0.5.4

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: cc12e128251786d7865236f627e20573ad622606
+  data.tar.gz: b65335f02f5d8332fab20b96cf93f0574444525e
+SHA512:
+  metadata.gz: 4c45fc27e36157d1c9ab3b26a143276f78fcae7e20611aa40da0cc43bf186faf8a9d1b20a63a2bdeba28b583c06fcb48c07be0140b576526dc49b969b32c409b
+  data.tar.gz: b8a68ef6e23f7b3294e1a8b7a20649a1d2d5f81008b69e39e0efcc371a99e029baf5de664a36b894a77b7940d646f64422ca1b5ce84377f3411bcfe8da4d309c

data/bin/desmond

@@ -0,0 +1,28 @@
+#!/usr/bin/env ruby
+
+require 'rake'
+require 'daemons'
+require_relative '../lib/desmond'
+
+#
+# script to run the desmond workers in background processes
+# easily manageable using start/stop/restart commands
+#
+
+# set_daemon really early, so it true as early as possible
+DesmondConfig.send :set_daemon
+current_dir = Dir.pwd
+options = { backtrace: true, log_dir: File.join(current_dir, 'log/') }
+options = options.merge(dir_mode: :normal, dir: ARGV[1]) if ARGV.size > 1
+Daemons.run_proc("desmond_for_#{DesmondConfig.app_id}", options) do
+  begin
+    Dir.chdir(current_dir)
+    Rake.application.init
+    Rake.application.load_rakefile
+    Rake::Task['desmond:run'].invoke
+  rescue Exception => e
+    DesmondConfig.logger.error "Fatal error in desmond process: #{e.message}"
+    DesmondConfig.logger.error e.backtrace.join("\n\t")
+    raise
+  end
+end

data/lib/desmond.rb

@@ -0,0 +1,185 @@
+require 'que'
+require 'active_record'
+require 'yaml'
+require 'aws-sdk'
+require 'active_support/hash_with_indifferent_access'
+
+require_relative 'desmond/monkey_patches'
+require_relative 'desmond/utils/pg_util'
+require_relative 'desmond/utils/s3_util'
+require_relative 'desmond/utils/log_censor'
+
+require_relative 'desmond/execution_error'
+
+require_relative 'desmond/streams/base'
+require_relative 'desmond/streams/csv'
+require_relative 'desmond/streams/database'
+require_relative 'desmond/streams/s3'
+
+require_relative 'desmond/job_run_finders'
+require_relative 'desmond/jobs/base'
+require_relative 'desmond/jobs/base_no_job_id'
+require_relative 'desmond/jobs/pg_export'
+require_relative 'desmond/jobs/export'
+require_relative 'desmond/jobs/import'
+require_relative 'desmond/jobs/unload'
+require_relative 'desmond/jobs/sequential'
+
+require_relative 'desmond/models/job_run'
+
+##
+# manages the gem's configuration
+#
+class DesmondConfig
+  @is_daemon = false
+  @logger = Logger.new(STDOUT)
+  @exception_notifier = []
+  class << self
+    attr_accessor :logger
+    attr_reader :is_daemon
+  end
+
+  ##
+  # determins the environment we are running in:
+  # - development
+  # - staging
+  # - production
+  #
+  def self.environment
+    (ENV['RACK_ENV'] || 'development').to_sym
+  end
+  ##
+  # set where the desmond configuration file is located
+  # and reload the configration
+  #
+  def self.config_file=(file)
+    @config = load_config_file(file)
+  end
+  ##
+  # retrieve the desmond configuration
+  #
+  def self.config
+    self.config_file = 'config/desmond.yml' if @config.nil?
+    @config
+  end
+
+  ##
+  # retrieves the app_id from the config, defaults to 'desmond'
+  #
+  def self.app_id
+    config['app_id'] || 'desmond'
+  end
+
+  ##
+  # adds the given argument to the list of block to be called when
+  # an uncaught exception occurs during job execution
+  #
+  # blocks will be called with the arguments (exception, job_class, job_run)
+  #
+  def self.add_exception_notifier(&block)
+    @exception_notifier << block
+  end
+  def self.clear_exception_notifier
+    @exception_notifier = []
+  end
+  def self.register_with_exception_notifier(options={})
+    options.each do |notifier_name, options|
+      ExceptionNotifier.register_exception_notifier(notifier_name, options)
+    end
+    DesmondConfig.add_exception_notifier do |exception, job_class, job_run|
+      ExceptionNotifier.notify_exception(exception, :data => { :class => job_class, run: job_run })
+    end
+  end
+
+  ##
+  # change 'app_id' to +value+
+  # only use this in the 'test' environment otherwise the change
+  # will not be shared with the worker processes, use the desmond.yml
+  # configuration file instead
+  #
+  def self.app_id=(value)
+    fail 'Do not use this DesmondConfig.app_id= outside of "test"' if self.environment != :test
+    config['app_id'] = value
+  end
+  ##
+  # retrieve the location of the ActiveRecord database configuration file.
+  # should really only be used when developing desmond, otherwise the using app
+  # should establish the ActiveRecord connections
+  #
+  def self.database
+    load_config_file(config['database'] || 'config/database.yml')
+  end
+  ##
+  # determines whether the tasks are allowed to use the configured
+  # connection instead of using the users credentials
+  #
+  def self.system_connection_allowed?
+    config['system_connection_allowed'] || false
+  end
+  ##
+  # change 'system_connection_allowed' to +value+
+  # only use this in the 'test' environment otherwise the change
+  # will not be shared with the worker processes, use the desmond.yml
+  # configuration file instead
+  #
+  def self.system_connection_allowed=(value)
+    fail 'Do not use this DesmondConfig.system_connection_allowed= outside of "test"' if self.environment != :test
+    config['system_connection_allowed'] = value
+  end
+
+  def self.load_config_file(file)
+    return {} unless File.exist?(file)
+    ActiveSupport::HashWithIndifferentAccess.new(YAML.load_file(file))
+  end
+  private_class_method :load_config_file
+
+  def self.set_daemon
+    @is_daemon = true
+  end
+  private_class_method :set_daemon
+
+  def self.exception_notification(exception, job_class, job_run)
+    @exception_notifier.each do |thing|
+      thing.call(exception, job_class, job_run) rescue nil
+    end
+  end
+  private_class_method :exception_notification
+end
+
+# configure ActiveRecord, but the app using us should really do this,
+# mostly included for development purposes on desmond itself
+if ActiveRecord::Base.configurations.empty?
+  ActiveRecord::Base.configurations = DesmondConfig.database
+  ActiveRecord::Base.establish_connection DesmondConfig.environment
+end
+
+# configure que
+Que.connection = ActiveRecord
+ActiveRecord::Base.schema_format = :sql # otherwise the que_jobs table gets missed
+Que.mode = :sync if DesmondConfig.environment == :test
+Que.mode = :off if DesmondConfig.environment != :test
+
+# configure log censoring, so that password and AWS secret keys don't end up in the logs
+CENSORED_KEYS = %w(password secret_access_key)
+Que.log_formatter = proc do |data|
+  tmp = ActiveSupport::HashWithIndifferentAccess.new(data)
+  if tmp.include?(:job) && !tmp[:job].nil?
+    tmp[:job][:args] = tmp[:job][:args].map do |arg|
+      censor_hash_keys(arg, CENSORED_KEYS) if arg.is_a?(Hash)
+    end
+  end
+  tmp.to_json
+end
+
+#
+# overwriting the get logger method of Que, to always return
+# Desmond's configured logger. Que tends to overwrite the logger
+# in background situations, which we'll ignore by this.
+#
+module Que
+  class << self
+    def logger
+      DesmondConfig.logger
+    end
+  end
+end

data/lib/desmond/capistrano.rb

@@ -0,0 +1,42 @@
+namespace :desmond do
+  def pid_dir
+    fetch(:pid_dir, "#{fetch(:release_path)}/tmp/pids")
+  end
+
+  def cmd(cmd)
+    "cd #{current_path};#{env} #{workers} #{que_command} #{cmd} #{pid_dir}"
+  end
+
+  desc 'Stop the desmond process'
+  task :stop do
+    on roles(:app) do
+      within release_path do
+        with rack_env: fetch(:rack_env), que_worker_count: fetch(:que_num_workers, 1) do
+          execute :bundle, :exec, :desmond, 'stop', pid_dir
+        end
+      end
+    end
+  end
+
+  desc 'Start the desmond process'
+  task :start do
+    on roles(:app) do
+      within release_path do
+        with rack_env: fetch(:rack_env), que_worker_count: fetch(:que_num_workers, 1) do
+          execute :bundle, :exec, :desmond, 'start', pid_dir
+        end
+      end
+    end
+  end
+
+  desc 'Restart the desmond process'
+  task :restart do
+    on roles(:app) do
+      within release_path do
+        with rack_env: fetch(:rack_env), que_worker_count: fetch(:que_num_workers, 1) do
+          execute :bundle, :exec, :desmond, 'restart', pid_dir
+        end
+      end
+    end
+  end
+end

data/lib/desmond/rake.rb

@@ -0,0 +1,84 @@
+require_relative '../desmond'
+require 'fileutils'
+
+#
+# returns migration files in the given directory +dirname+
+#
+def migrations(dirname)
+  Dir.foreach(dirname).select do |entry|
+    !entry.start_with?('.')
+  end
+end
+
+#
+# returns the next migration number to use from the given directory +dirname+
+#
+def next_migration_number(dirname)
+  current_migration_number_str = migrations(dirname).map do |file|
+    File.basename(file).split('_').first
+  end.max || '0000'
+  current_migration_number = current_migration_number_str.to_i
+
+  num_digits = current_migration_number_str.size
+  if num_digits > 5
+    # timestamp format
+    [Time.now.utc.strftime('%Y%m%d%H%M%S'), format('%.14d', current_migration_number + 1)].max
+  else
+    # counter format
+    format("%.#{num_digits}d", current_migration_number + 1)
+  end
+end
+
+#
+# copy migration +source_file+ to migrations folder +dest_folder+
+# if it doesn't exist ther yet
+#
+def copy_migration(source_file, dest_folder)
+  # check if migration already exists
+  migration_exists = migrations(dest_folder).map do |file|
+    File.basename(file).split('_')[1..-1].join('_').eql?(File.basename(source_file))
+  end.any?
+  return if migration_exists
+
+  # copy migration file
+  migration_number = next_migration_number(dest_folder)
+  base_migration_name = File.basename source_file
+  migration_name = "#{migration_number}_#{base_migration_name}"
+  dest_file = File.join dest_folder, migration_name
+  FileUtils.cp(source_file, dest_file)
+end
+
+# hook into db:migrate to create our migrations and migrate Que
+namespace :db do
+  task :migrate => :desmond_migrate
+
+  task :desmond_migrate do
+    Rake::Task['desmond:migrate'].invoke
+  end
+end
+
+namespace :desmond do
+  desc 'Setup database for desmond manually'
+  task :migrate do
+    Que.migrate! version: 3
+
+    migration = File.join File.dirname(__FILE__), 'migrations', 'add_desmond_job_runs.rb'
+    copy_migration(migration, 'db/migrate')
+  end
+
+  desc 'Start daemon for desmond'
+  task :run do
+    require 'que/rake_tasks' # so that que's tasks don't get directly included in using apps
+    ActiveRecord::Base.logger = DesmondConfig.logger
+    DesmondConfig.send :set_daemon
+    Rake::Task['que:work'].invoke
+  end
+
+  desc 'Clear job runs and queues'
+  task :clear do
+    require 'que/rake_tasks' # so that que's tasks don't get directly included in using apps
+    ActiveRecord::Base.logger = DesmondConfig.logger
+    Rake::Task['que:clear'].invoke
+    ActiveRecord::Base.connection.execute('TRUNCATE desmond_job_runs')
+  end
+end

metadata

@@ -0,0 +1,226 @@
+--- !ruby/object:Gem::Specification
+name: desmond
+version: !ruby/object:Gem::Version
+  version: 0.5.4
+platform: ruby
+authors:
+- Tobias Thiel
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-03-31 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '6'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '6'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.17'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.17'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: que
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: daemons
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sinatra-activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Background tasks & Exporting and importing data out of Amazon AWS RedShift
+email: tobi@605.tv
+executables:
+- desmond
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/desmond
+- lib/desmond.rb
+- lib/desmond/capistrano.rb
+- lib/desmond/rake.rb
+homepage: http://605.tv
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.12
+signing_key: 
+specification_version: 4
+summary: Background tasks & CSV Export/Import
+test_files: []