writefully 0.3.6

data/lib/writefully/process.rb

@@ -0,0 +1,84 @@
+require 'listen'
+require 'logger'
+
+require 'celluloid'
+
+require 'active_record'
+require 'writefully'
+require 'redis'
+require 'redis-namespace'
+
+%w(tag post site tagging authorship).each do |model|
+  require File.dirname(__FILE__) + "/../../app/models/writefully/#{model}"
+end
+
+Writefully::Source.to_load.each do |model|
+  require File.join(Writefully.options[:app_directory], 'app', 'models', model)
+end
+
+require 'github_api'
+
+require 'writefully/tools'
+require 'writefully/workers'
+require 'writefully/news_agency'
+
+module Writefully
+  Process = Struct.new(:config) do
+
+    def listen
+      log_start
+      connect_to_database!
+      start_news_agency!
+      start_dispatcher!
+      boot_listener!
+    end
+
+    # connect to db
+    def connect_to_database!
+      ActiveRecord::Base.establish_connection(Writefully.db_config)
+    end
+
+    # Dispatcher monitors job queue and throws job at workers
+    def start_dispatcher!
+      Tools::Dispatcher.supervise_as :dispatch
+    end
+
+    # Supervises the actors that manage all the work with converting content
+    # and sorting it into its place
+    def start_news_agency!
+      NewsAgency.run!
+    end
+
+    def log_start
+      Writefully.logger.info("This is doctor Frasier Crane. I'm listening...")
+    end
+
+    def process_message
+      Proc.new do |modified, added, removed|
+        queue_jobs(Indices.build_from(modified), :write)
+        queue_jobs(Indices.build_from(added),    :write)
+        queue_jobs(Indices.build_from(removed),  :remove)
+      end
+    end
+
+    # this listener listens to the specified content folder 
+    # queues the changes detected into the job queue
+    def boot_listener!
+      listener = Listen.to config[:content], wait_for_delay: 2, &process_message
+      listener.start
+      while listener.listen?
+        sleep 0.5
+      end
+    end
+
+    JOBS = { 
+      write:  -> (index) { Writefully.add_job :journalists, index.merge({task: :publish}) },
+      remove: -> (index) { Wrotefully.add_job :journalists, index.merge({task: :remove})  }
+    }
+
+    def queue_jobs indices, action
+      indices.uniq.each { |index| JOBS[action].call(index) if Source.valid_resources.include?(index[:resource]) }
+    end
+
+  end
+end

data/lib/writefully/source.rb

@@ -0,0 +1,52 @@
+require 'base64'
+
+module Writefully
+  module Source
+
+    class << self
+      def content_path
+        Writefully.options[:content]
+      end
+
+      def models_path
+        File.join(Writefully.options[:app_directory], 'app', 'models')
+      end
+
+      def sample_content file
+        open(File.dirname(__FILE__) + "/../sample/#{file}").read
+      end
+
+      def sample_content_paths 
+        valid_resources.map do |resource|
+          ["#{resource}/1-change-me/README.md", 
+           "#{resource}/1-change-me/meta.yml", 
+           "#{resource}/1-change-me/assets/writefully.png"]
+        end.flatten
+      end
+
+      def sample_content_properties file
+        { 
+          path: file,
+          content: sample_content(file),
+          message: "added sample #{file}"
+        }
+      end
+
+      def valid_resources
+        skim_for(::Regexp.new('Writefully::Post')).map { |r| r.pluralize }
+      end
+
+      def to_load
+        skim_for ::Regexp.new('Writefully')
+      end
+
+      def skim_for matcher
+        Dir.chdir(models_path) do 
+          Dir.glob('*').select do |file|
+            open(File.join(models_path, file)).read.strip.match(matcher) if File.file?(file)
+          end.collect { |file| file.split('.')[0] }
+        end
+      end
+    end
+  end
+end

data/lib/writefully/storage.rb

@@ -0,0 +1,33 @@
+module Writefully
+  module Storage
+    class << self 
+      def directory
+        @directory ||= connection.directories.get(Writefully.options[:storage_folder])
+      end
+
+      def store_file(path, body)
+        directory.files.create({
+          key: path,
+          body: body,
+          public: true
+        })
+      end
+
+      def endpoint
+        provider_endpoints[Writefully.options[:storage_provider].downcase.to_sym]
+      end
+
+      def provider_endpoints
+        { aws: "https://#{Writefully.options[:storage_folder]}.s3.amazonaws.com"}
+      end
+
+      def connection
+        @connection ||= Fog::Storage.new({
+          provider:              Writefully.options[:storage_provider],
+          aws_access_key_id:     Writefully.options[:storage_key],
+          aws_secret_access_key: Writefully.options[:storage_secret]
+        })
+      end
+    end
+  end
+end

data/lib/writefully/taxon.rb

@@ -0,0 +1,28 @@
+module Writefully
+  Taxon = Struct.new(:incoming, :existing, :type) do 
+    def non_existing
+      get_difference.map { |token| Tag.new(build_attributes(token)) }
+    end
+
+    def selector
+      @selector ||= type.underscore.to_sym
+    end
+
+    def get_difference
+      (parameterized(incoming) - parameterized(existing)).map { |t| t.titleize }
+    end
+
+    def type_attribute 
+      selector == :'writefully/tag' ? { type: nil } : { type: selector.to_s.classify }
+    end
+
+    def build_attributes token
+      type_attribute
+        .merge({ name: token, slug: token.parameterize })
+    end
+
+    def parameterized items
+      items.map { |t| t.parameterize }
+    end
+  end
+end

data/lib/writefully/tools/dispatcher.rb

@@ -0,0 +1,59 @@
+module Writefully
+  module Tools
+    class Dispatcher
+      include Celluloid
+
+      attr_reader :job
+
+      def initialize
+        every 1.second do 
+          async.heartbeat
+        end
+      end
+
+      def get_job_data
+        Writefully.redis.with { |c| c.spop 'jobs' }
+      end
+
+      def heartbeat
+        @job      = Marshal.load(get_job_data)
+        run_job if is_job?
+      end
+
+      def run_job
+        if is_retry? and retry_valid? then retry_job
+        elsif is_job? and job_valid?  then dispatch
+        else mark_as_failed end
+      end
+
+      def dispatch
+        Celluloid::Actor[job[:worker]].perform(job[:message])
+      end
+
+      def retry_job
+        after((job[:tries] * job[:tries]).seconds) { dispatch(job) }
+      end
+
+      def mark_as_failed
+        Writefully.redis.with { |c| c.sadd 'failed', Marshal.dump(job) }
+      end
+
+      def is_job?
+        job.has_key?(:worker) and job.has_key?(:message)
+      end
+
+      def is_retry?
+        is_job? and job.has_key?(:tries)
+      end
+
+      def job_valid?
+        job.keys.count == 2
+      end
+
+      def retry_valid?
+        job[:tries] <= 5
+      end
+
+    end
+  end
+end

data/lib/writefully/tools/hammer.rb

@@ -0,0 +1,41 @@
+module Writefully
+  module Tools
+    class Hammer
+      include Celluloid
+
+      attr_reader :api, :message
+
+      def initialize message
+        @message = message
+        @api = Github.new oauth_token: message[:auth_token]
+      end
+
+      def hook_config
+        { name: 'web',
+          events: ["push", "member"],
+          active: true,
+          config: { 
+            url: "#{message[:domain]}/writefully/hook", 
+            content_type: 'json',
+            secret: Writefully.options[:hook_secret]
+          } 
+        }
+      end
+
+      def forge
+        Writefully.logger.info "Forging #{message[:site_slug]}"
+        api.repos.create name: message[:site_slug], auto_init: true
+      rescue Exception => e
+        raise e
+      end
+
+      def add_hook_for repo_name
+        Writefully.logger.info "Adding hook for #{message[:site_slug]}"
+        api.repos.hooks.create message[:user_name], repo_name, hook_config
+      rescue Exception => e
+        raise e
+      end
+
+    end
+  end
+end

data/lib/writefully/tools/initializer.rb

@@ -0,0 +1,38 @@
+module Writefully
+  module Tools
+    class Initializer
+      include Celluloid
+
+      INITIALIZE_SCRIPT = File.dirname(__FILE__) + "/../../../scripts/initialize.sh"
+
+      attr_reader :message, :api
+
+      def initialize message
+        @message = message
+        @api = Github.new oauth_token: message[:auth_token]
+      end
+
+      def add_sample_content
+        Writefully.logger.info "Adding Sample content #{message[:site_slug]}"
+        Source.sample_content_paths.map do |path|
+          api.repos.contents.create  message[:user_name], 
+                                     message[:site_slug], 
+                                     path, sample_content_for(path.split('/').last)     
+        end
+      end
+
+      def build_content_folder
+        Writefully.logger.info "Creating content folder #{message[:site_slug]}"
+        system(content_folder_setup_command)
+      end
+
+      def content_folder_setup_command
+        ['bash', INITIALIZE_SCRIPT, Writefully.options[:content], message[:site_slug], message[:ssh_url]].join(' ')
+      end
+
+      def sample_content_for file_name
+        Source.sample_content_properties(file_name)
+      end
+    end
+  end
+end

data/lib/writefully/tools/pencil.rb

@@ -0,0 +1,71 @@
+module Writefully
+  module Tools
+    class Pencil
+      include Celluloid
+
+      attr_reader :resource, :content, :asset, :index, :site_id, :asset
+
+      class ContentModelNotFound < StandardError; end
+      class SomeAssetsNotUploaded < StandardError; end
+
+      def initialize(index)
+        @site_id  = Site.where(slug: index[:site]).first.id
+        @index    = index
+        @content  = Content.new(index)
+        @asset    = Asset.new(index)
+      end
+
+      def perform
+        assets_uploaded = upload_assets.map(&:value).compact
+        written_to_db   = future.write if can_update_db?(assets_uploaded)
+        terminate if written_to_db.value
+      end
+
+      def computed_attributes
+        content.meta.merge({ 
+          "content" => asset.convert_for(content.body),
+          "details" => asset.convert_for(content.details)
+        })
+      end
+
+      def write
+        compute_type.by_site(site_id).where(slug: content.slug)
+                      .first_or_initialize
+                        .update_attributes(computed_attributes)
+      ensure 
+        ::ActiveRecord::Base.clear_active_connections! if defined?(::ActiveRecord)
+      end
+      
+      def upload_assets
+        asset.names.map do |name|
+          Celluloid::Actor[:pigeons].future.upload(asset.endpoint, asset.path, name)
+        end
+      end
+
+      def can_update_db? uploaded
+        if asset.names.count == uploaded.count
+          true
+        else 
+          raise SomeAssetsNotUploaded, "Some assets was not uploaded"
+        end
+      end
+
+    private
+
+      def compute_type
+        index[:resource].classify.constantize
+      rescue NameError
+        fallback_type
+      end
+
+      def fallback_type
+        if index[:resource] == "posts"
+          "Writefully::Post".constantize
+        else 
+          raise ContentModelNotFound, "Model #{index[:resource].classify} was not found"
+        end
+      end
+      
+    end
+  end
+end

data/lib/writefully/tools/pigeon.rb

@@ -0,0 +1,19 @@
+module Writefully
+  module Tools
+    class Pigeon
+      include Celluloid
+
+      def upload endpoint, path, name
+        file = File.open(File.join(path, name))
+        Writefully::Storage.store_file(File.join(endpoint, name), file)
+        true
+      rescue Exception => e
+        nil
+      end
+
+      def remove endpoint, path, name
+        
+      end
+    end
+  end
+end

data/lib/writefully/tools/synchronizer.rb

@@ -0,0 +1,24 @@
+module Writefully
+  module Tools
+    class Synchronizer
+      include Celluloid
+
+      SYNC_SCRIPT = File.dirname(__FILE__) + "/../../../scripts/sync.sh"
+
+      attr_reader :message
+
+      def initialize message
+        @message = message
+      end
+
+      def sync
+        Writefully.logger.info "Synchronizing #{message[:site_slug]}"
+        system(sync_command)
+      end
+
+      def sync_command
+        ['bash', SYNC_SCRIPT, Writefully.options[:content], message[:site_slug]].join(' ')
+      end
+    end
+  end
+end

data/lib/writefully/tools.rb

@@ -0,0 +1,13 @@
+module Writefully
+  module Tools
+
+  end
+end
+
+require 'writefully/tools/eraser'
+require 'writefully/tools/pencil'
+require 'writefully/tools/pigeon'
+require 'writefully/tools/hammer'
+require 'writefully/tools/dispatcher'
+require 'writefully/tools/initializer'
+require 'writefully/tools/synchronizer'

data/lib/writefully/version.rb

@@ -0,0 +1,3 @@
+module Writefully
+  VERSION = "0.3.6"
+end

data/lib/writefully/workers/base_worker.rb

@@ -0,0 +1,25 @@
+module Writefully
+  module Workers
+    class BaseWorker
+      include Celluloid
+
+      trap_exit :actor_died
+
+      attr_reader :message
+
+      def perform(message)
+        @message = message
+        self.__send__ message[:task]
+      end
+
+      def close_db_connection!
+        ::ActiveRecord::Base.clear_active_connections! if defined?(::ActiveRecord)
+      end
+
+      def actor_died(actor, reason)
+        Writefully.logger.error "An error occured #{reason.message}"
+        on_death(actor, reason) if self.respond_to?(:on_death)
+      end
+    end
+  end
+end

data/lib/writefully/workers/handyman.rb

@@ -0,0 +1,44 @@
+module Writefully
+  module Workers
+    class Handyman < BaseWorker
+      include Helpers::Handyman::Builder
+
+      attr_reader :site
+
+      def build
+        @site      = Site.where(id: message[:site_id]).first
+        @hammer    = Tools::Hammer.new_link(message.merge({ domain: site.domain }))
+        # create the repository
+        repo = @hammer.future.forge
+
+        # add sample content
+        @initializer = Tools::Initializer.new_link(message.merge({ ssh_url: repo.value.ssh_url }))
+        initialize_sample_content
+
+        # add web hook
+        hook = @hammer.future.add_hook_for(repo.value.name)
+        complete_site_setup(repo.value, hook.value)
+      ensure
+        @hammer.terminate
+        @initializer.terminate
+        close_db_connection!
+      end
+
+      def synchronize
+        @synchronizer = Tools::Synchronizer.new_link(message)
+        synced = @synchronizer.future.sync
+        Writefully.logger.info "Synchronized #{message[:site_slug]}" if synced.value
+      ensure
+        @synchronizer.terminate
+      end
+
+      def on_death actor, reason
+        if site
+          Writefully.redis.with { |c| s.sadd "site:#{site.id}:errors", reason.message }
+          site.update_attributes(processing: false, healty: false)
+        end
+      end
+
+    end
+  end
+end

data/lib/writefully/workers/helpers/handyman/builder.rb

@@ -0,0 +1,17 @@
+module Writefully::Workers::Helpers::Handyman
+  module Builder
+    extend ActiveSupport::Concern
+
+    def complete_site_setup repo, hook
+      site_repository = { name: repo.name, id: repo.id, hook_id: hook.id }
+      site.update_attributes(repository: site_repository, processing: false, healthy: true)
+    end
+
+    def initialize_sample_content
+      added_sample_content = @initializer.add_sample_content
+      created_directory    = @initializer.build_content_folder
+
+      [added_sample_content, created_directory]
+    end
+  end
+end

data/lib/writefully/workers/helpers.rb

@@ -0,0 +1,8 @@
+require 'active_support/concern'
+
+module Writefully::Workers
+  module Helpers
+  end
+end
+
+require 'writefully/workers/helpers/handyman/builder'

data/lib/writefully/workers/journalist.rb

@@ -0,0 +1,23 @@
+module Writefully
+  module Workers
+    class Journalist < BaseWorker
+      def publish
+        Writefully.logger.info "Publishing #{message[:resource]} #{message[:slug]}"
+        pencil = Tools::Pencil.new_link(message)
+        pencil.perform
+      end
+
+      def remove
+        Writefully.logger.info "Removing #{message[:resource]} #{message[:slug]}"
+      end
+
+      def message_with_tries
+        message.merge({ tries: (message[:tries] || 1) + 1 })
+      end
+
+      def on_death actor, reason
+        Writefully.add_job :journalists, message_with_tries
+      end
+    end
+  end
+end

data/lib/writefully/workers.rb

@@ -0,0 +1,9 @@
+module Writefully
+  module Workers
+
+  end
+end
+require 'writefully/workers/helpers'
+require 'writefully/workers/base_worker'
+require 'writefully/workers/journalist'
+require 'writefully/workers/handyman'

data/lib/writefully/workers_helpers.rb

@@ -0,0 +1,7 @@
+
+module Writefully
+  module Helpers
+
+  end
+end
+