writefully 0.3.6 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/app/controllers/writefully/hooks_controller.rb +5 -0
- data/app/models/writefully/authorship.rb +2 -0
- data/app/models/writefully/post.rb +2 -0
- data/app/models/writefully/site.rb +2 -0
- data/app/models/writefully/tag.rb +2 -0
- data/app/models/writefully/tagging.rb +2 -0
- data/bin/writefully +1 -1
- data/lib/sample/README.md +1 -1
- data/lib/sample/meta.yml +0 -1
- data/lib/writefully/cli.rb +8 -6
- data/lib/writefully/loader.rb +72 -0
- data/lib/writefully/process.rb +22 -13
- data/lib/writefully/tools/dispatcher.rb +7 -11
- data/lib/writefully/tools/pigeon.rb +1 -2
- data/lib/writefully/tools/retryer.rb +30 -0
- data/lib/writefully/tools.rb +1 -0
- data/lib/writefully/version.rb +1 -1
- data/lib/writefully/workers/journalist.rb +1 -1
- data/lib/writefully.rb +4 -64
- metadata +31 -2
- data/lib/tasks/writefully_tasks.rake +0 -9
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 06d7aa28aba44269ffe7397686a8afdf2a239856
|
4
|
+
data.tar.gz: 06e9d0183726001182722aaabcdabd2b1fcffffe
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 77342eedab7649f552741419a9b5bb550fa053e0f9dac8a65bdfa7df75dbcdb3d351b6c5e37661077f0c06980c370e91acf52d26742348d9d04650450be98b05
|
7
|
+
data.tar.gz: 708d5924614dc7b80f71c994b21b65d5c7ec778ef94465a6e57e37feea17261b1eca5bb9fa61cc419f774e0d27f15d0a7288145e202a4bd6c0eb04c47493d8a8
|
@@ -11,12 +11,16 @@ module Writefully
|
|
11
11
|
|
12
12
|
def create
|
13
13
|
self.__send__ request.headers["X-Github-Event"].to_sym
|
14
|
+
end
|
15
|
+
|
16
|
+
def ping
|
14
17
|
head :ok
|
15
18
|
end
|
16
19
|
|
17
20
|
def push
|
18
21
|
Writefully.add_job :handyman, { task: :synchronize,
|
19
22
|
site_slug: body.repository.name } if branch_match?
|
23
|
+
head :ok
|
20
24
|
end
|
21
25
|
|
22
26
|
def member
|
@@ -24,6 +28,7 @@ module Writefully
|
|
24
28
|
unless authorship
|
25
29
|
Authorship.create_from_data(body.member)
|
26
30
|
end
|
31
|
+
head :ok
|
27
32
|
end
|
28
33
|
|
29
34
|
protected
|
data/bin/writefully
CHANGED
data/lib/sample/README.md
CHANGED
data/lib/sample/meta.yml
CHANGED
data/lib/writefully/cli.rb
CHANGED
@@ -1,28 +1,30 @@
|
|
1
1
|
require 'thor'
|
2
2
|
require 'writefully/process'
|
3
|
-
|
3
|
+
require 'pry'
|
4
4
|
module Writefully
|
5
5
|
class CLI < Thor
|
6
6
|
desc "start", "Start listening to the content directory"
|
7
7
|
|
8
8
|
method_options %w( daemonize -d ) => :boolean
|
9
|
-
method_options %w( config -c ) => :string
|
10
9
|
|
11
|
-
def start
|
12
|
-
config = Writefully.config_from(
|
10
|
+
def start(file)
|
11
|
+
config = Writefully.config_from(file)
|
13
12
|
|
14
13
|
if options.daemonize?
|
15
14
|
Process.daemon(true, true)
|
16
15
|
pid = waitpid(spawn(listen(config)))
|
17
16
|
write pid, config[:pidfile]
|
18
17
|
else
|
18
|
+
Signal.trap("INT") { $stdout.puts "Writefully exiting..."; exit }
|
19
19
|
listen(config)
|
20
20
|
end
|
21
21
|
end
|
22
22
|
|
23
23
|
desc "stop", "Stop listening for content directory changes"
|
24
|
-
def stop(
|
25
|
-
|
24
|
+
def stop(file)
|
25
|
+
config = Writefully.config_from(file)
|
26
|
+
|
27
|
+
pid = open(config[:pidfile]).read.strip.to_i
|
26
28
|
Process.kill("HUP", pid)
|
27
29
|
true
|
28
30
|
rescue Errno::ENOENT
|
@@ -0,0 +1,72 @@
|
|
1
|
+
require 'writefully/storage'
|
2
|
+
require 'erb'
|
3
|
+
|
4
|
+
module Writefully
|
5
|
+
SCOPES = %w(repo public_repo user write:repo_hook)
|
6
|
+
|
7
|
+
class << self
|
8
|
+
|
9
|
+
def options=(config)
|
10
|
+
@_options = config
|
11
|
+
end
|
12
|
+
|
13
|
+
def options
|
14
|
+
@_options ||= config_from(config_yml)
|
15
|
+
end
|
16
|
+
|
17
|
+
def redis
|
18
|
+
r = Redis.new(host: 'localhost', port: 6379)
|
19
|
+
@_redis ||= ConnectionPool.new(size: 5, timeout: 5) do
|
20
|
+
Redis::Namespace.new(:"#{env}:writefully", redis: r)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def add_job worker, message
|
25
|
+
Writefully.redis.with { |c| c.sadd "jobs", convert_job(worker, message) }
|
26
|
+
end
|
27
|
+
|
28
|
+
def convert_job worker, message
|
29
|
+
Marshal.dump({worker: worker, message: message})
|
30
|
+
end
|
31
|
+
|
32
|
+
def env
|
33
|
+
ENV["RACK_ENV"] || ENV["RAILS_ENV"] || 'development'
|
34
|
+
end
|
35
|
+
|
36
|
+
def github_app
|
37
|
+
@_github_app ||= Github.new(client_id: options[:github_client],
|
38
|
+
client_secret: options[:github_secret])
|
39
|
+
end
|
40
|
+
|
41
|
+
def logger
|
42
|
+
@logger ||= Logger.new(log_location)
|
43
|
+
end
|
44
|
+
|
45
|
+
def log_location
|
46
|
+
env == 'development' ? STDOUT : Writefully.options[:logfile]
|
47
|
+
end
|
48
|
+
|
49
|
+
def db_config
|
50
|
+
YAML::load(ERB.new(IO.read(File.join(options[:app_directory], 'config', 'database.yml'))).result)[env]
|
51
|
+
end
|
52
|
+
|
53
|
+
def config_from(path = nil)
|
54
|
+
YAML::load(ERB.new(IO.read(path)).result)[env]
|
55
|
+
rescue Errno::ENOENT
|
56
|
+
$stdout.puts "config/writefully.yml does not exist"
|
57
|
+
end
|
58
|
+
|
59
|
+
|
60
|
+
def config_yml
|
61
|
+
Rails.root.join('config', 'writefully.yml') if defined?(Rails)
|
62
|
+
end
|
63
|
+
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
require 'writefully/taxon'
|
68
|
+
require 'writefully/asset'
|
69
|
+
require 'writefully/source'
|
70
|
+
require 'writefully/postable'
|
71
|
+
require 'writefully/indices'
|
72
|
+
require 'writefully/content'
|
data/lib/writefully/process.rb
CHANGED
@@ -1,38 +1,46 @@
|
|
1
|
+
require 'fog'
|
1
2
|
require 'listen'
|
2
3
|
require 'logger'
|
3
|
-
|
4
4
|
require 'celluloid'
|
5
|
-
|
5
|
+
require 'github_api'
|
6
6
|
require 'active_record'
|
7
|
-
require '
|
7
|
+
require 'friendly_id'
|
8
|
+
require 'activerecord-import'
|
9
|
+
require 'connection_pool'
|
8
10
|
require 'redis'
|
9
11
|
require 'redis-namespace'
|
12
|
+
require 'writefully/loader'
|
13
|
+
require 'writefully/tools'
|
14
|
+
require 'writefully/workers'
|
15
|
+
require 'writefully/news_agency'
|
10
16
|
|
11
17
|
%w(tag post site tagging authorship).each do |model|
|
12
18
|
require File.dirname(__FILE__) + "/../../app/models/writefully/#{model}"
|
13
19
|
end
|
14
20
|
|
15
|
-
Writefully::Source.to_load.each do |model|
|
16
|
-
require File.join(Writefully.options[:app_directory], 'app', 'models', model)
|
17
|
-
end
|
18
|
-
|
19
|
-
require 'github_api'
|
20
|
-
|
21
|
-
require 'writefully/tools'
|
22
|
-
require 'writefully/workers'
|
23
|
-
require 'writefully/news_agency'
|
24
|
-
|
25
21
|
module Writefully
|
26
22
|
Process = Struct.new(:config) do
|
27
23
|
|
28
24
|
def listen
|
25
|
+
set_options
|
29
26
|
log_start
|
27
|
+
load_models
|
30
28
|
connect_to_database!
|
31
29
|
start_news_agency!
|
32
30
|
start_dispatcher!
|
33
31
|
boot_listener!
|
34
32
|
end
|
35
33
|
|
34
|
+
def set_options
|
35
|
+
Writefully.options = config
|
36
|
+
end
|
37
|
+
|
38
|
+
def load_models
|
39
|
+
Writefully::Source.to_load.each do |model|
|
40
|
+
require File.join(config[:app_directory], 'app', 'models', model)
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
36
44
|
# connect to db
|
37
45
|
def connect_to_database!
|
38
46
|
ActiveRecord::Base.establish_connection(Writefully.db_config)
|
@@ -41,6 +49,7 @@ module Writefully
|
|
41
49
|
# Dispatcher monitors job queue and throws job at workers
|
42
50
|
def start_dispatcher!
|
43
51
|
Tools::Dispatcher.supervise_as :dispatch
|
52
|
+
Tools::Retryer.supervise_as :retryer
|
44
53
|
end
|
45
54
|
|
46
55
|
# Supervises the actors that manage all the work with converting content
|
@@ -21,9 +21,9 @@ module Writefully
|
|
21
21
|
end
|
22
22
|
|
23
23
|
def run_job
|
24
|
-
if
|
25
|
-
elsif
|
26
|
-
|
24
|
+
if retry_valid? then retry_job
|
25
|
+
elsif job_valid? then dispatch
|
26
|
+
end
|
27
27
|
end
|
28
28
|
|
29
29
|
def dispatch
|
@@ -31,11 +31,7 @@ module Writefully
|
|
31
31
|
end
|
32
32
|
|
33
33
|
def retry_job
|
34
|
-
|
35
|
-
end
|
36
|
-
|
37
|
-
def mark_as_failed
|
38
|
-
Writefully.redis.with { |c| c.sadd 'failed', Marshal.dump(job) }
|
34
|
+
Celluloid::Actor[:retryer].retry(job)
|
39
35
|
end
|
40
36
|
|
41
37
|
def is_job?
|
@@ -43,15 +39,15 @@ module Writefully
|
|
43
39
|
end
|
44
40
|
|
45
41
|
def is_retry?
|
46
|
-
is_job? and job.has_key?(:tries)
|
42
|
+
is_job? and job[:message].has_key?(:tries) and job[:message].has_key?(:run)
|
47
43
|
end
|
48
44
|
|
49
45
|
def job_valid?
|
50
|
-
job.keys.count == 2
|
46
|
+
is_job? and job.keys.count == 2
|
51
47
|
end
|
52
48
|
|
53
49
|
def retry_valid?
|
54
|
-
job[:
|
50
|
+
is_retry? and (job[:message][:run] == false)
|
55
51
|
end
|
56
52
|
|
57
53
|
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
module Writefully
|
2
|
+
module Tools
|
3
|
+
class Retryer
|
4
|
+
include Celluloid
|
5
|
+
|
6
|
+
attr_accessor :job
|
7
|
+
|
8
|
+
def retry(job)
|
9
|
+
@job = job
|
10
|
+
if job[:message][:tries] <= 5
|
11
|
+
after(delay) { queue_retry }
|
12
|
+
else
|
13
|
+
mark_as_failed
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
def queue_retry
|
18
|
+
Writefully.add_job job[:worker], job[:message].merge({ run: true })
|
19
|
+
end
|
20
|
+
|
21
|
+
def delay
|
22
|
+
(job[:message][:tries] * job[:message][:tries]).seconds
|
23
|
+
end
|
24
|
+
|
25
|
+
def mark_as_failed
|
26
|
+
Writefully.redis.with { |c| c.sadd 'failed', Marshal.dump(job) }
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
data/lib/writefully/tools.rb
CHANGED
@@ -9,5 +9,6 @@ require 'writefully/tools/pencil'
|
|
9
9
|
require 'writefully/tools/pigeon'
|
10
10
|
require 'writefully/tools/hammer'
|
11
11
|
require 'writefully/tools/dispatcher'
|
12
|
+
require 'writefully/tools/retryer'
|
12
13
|
require 'writefully/tools/initializer'
|
13
14
|
require 'writefully/tools/synchronizer'
|
data/lib/writefully/version.rb
CHANGED
data/lib/writefully.rb
CHANGED
@@ -1,77 +1,17 @@
|
|
1
1
|
require 'yaml'
|
2
2
|
require 'hashie'
|
3
|
+
require 'celluloid'
|
3
4
|
require 'github_api'
|
5
|
+
require 'active_record'
|
4
6
|
require 'activerecord-import'
|
5
7
|
|
6
8
|
require 'connection_pool'
|
7
9
|
require 'redis'
|
8
10
|
require 'redis-namespace'
|
9
11
|
|
10
|
-
require 'writefully/engine'
|
11
|
-
require 'writefully/storage'
|
12
|
-
|
13
12
|
module Writefully
|
14
|
-
SCOPES = %w(repo public_repo user write:repo_hook)
|
15
|
-
|
16
|
-
class << self
|
17
|
-
|
18
|
-
def options
|
19
|
-
@_options ||= config_from(config_yml)
|
20
|
-
end
|
21
|
-
|
22
|
-
def redis
|
23
|
-
r = Redis.new(host: 'localhost', port: 6379)
|
24
|
-
@_redis ||= ConnectionPool.new(size: 5, timeout: 5) do
|
25
|
-
Redis::Namespace.new(:"#{env}:writefully", redis: r)
|
26
|
-
end
|
27
|
-
end
|
28
|
-
|
29
|
-
def add_job worker, message
|
30
|
-
Writefully.redis.with { |c| c.sadd "jobs", convert_job(worker, message) }
|
31
|
-
end
|
32
|
-
|
33
|
-
def convert_job worker, message
|
34
|
-
Marshal.dump({worker: worker, message: message})
|
35
|
-
end
|
36
|
-
|
37
|
-
def env
|
38
|
-
ENV["RACK_ENV"] || ENV["RAILS_ENV"] || 'development'
|
39
|
-
end
|
40
13
|
|
41
|
-
def github_app
|
42
|
-
@_github_app ||= Github.new(client_id: options[:github_client],
|
43
|
-
client_secret: options[:github_secret])
|
44
|
-
end
|
45
|
-
|
46
|
-
def logger
|
47
|
-
@logger ||= Logger.new(log_location)
|
48
|
-
end
|
49
|
-
|
50
|
-
def log_location
|
51
|
-
env == 'development' ? STDOUT : Writefully.options[:logfile]
|
52
|
-
end
|
53
|
-
|
54
|
-
def db_config
|
55
|
-
YAML::load(ERB.new(IO.read(File.join(options[:app_directory], 'config', 'database.yml'))).result)[env]
|
56
|
-
end
|
57
|
-
|
58
|
-
def config_from(path = nil)
|
59
|
-
YAML::load(ERB.new(IO.read(path)).result)[env]
|
60
|
-
rescue Errno::ENOENT
|
61
|
-
$stdout.puts "config/writefully.yml does not exist"
|
62
|
-
end
|
63
|
-
|
64
|
-
|
65
|
-
def config_yml
|
66
|
-
Rails.root.join('config', 'writefully.yml') if defined?(Rails)
|
67
|
-
end
|
68
|
-
|
69
|
-
end
|
70
14
|
end
|
71
15
|
|
72
|
-
require 'writefully/
|
73
|
-
require 'writefully/
|
74
|
-
require 'writefully/source'
|
75
|
-
require 'writefully/postable'
|
76
|
-
require 'writefully/indices'
|
77
|
-
require 'writefully/content'
|
16
|
+
require 'writefully/engine'
|
17
|
+
require 'writefully/loader'
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: writefully
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Zack Siri
|
@@ -80,6 +80,34 @@ dependencies:
|
|
80
80
|
- - '>='
|
81
81
|
- !ruby/object:Gem::Version
|
82
82
|
version: '0'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: fog
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - '>='
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0'
|
90
|
+
type: :runtime
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - '>='
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: unf
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - '>='
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
type: :runtime
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - '>='
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0'
|
83
111
|
- !ruby/object:Gem::Dependency
|
84
112
|
name: listen
|
85
113
|
requirement: !ruby/object:Gem::Requirement
|
@@ -392,12 +420,12 @@ files:
|
|
392
420
|
- lib/sample/meta.yml
|
393
421
|
- lib/sample/README.md
|
394
422
|
- lib/sample/writefully.png
|
395
|
-
- lib/tasks/writefully_tasks.rake
|
396
423
|
- lib/writefully/asset.rb
|
397
424
|
- lib/writefully/cli.rb
|
398
425
|
- lib/writefully/content.rb
|
399
426
|
- lib/writefully/engine.rb
|
400
427
|
- lib/writefully/indices.rb
|
428
|
+
- lib/writefully/loader.rb
|
401
429
|
- lib/writefully/news_agency.rb
|
402
430
|
- lib/writefully/postable.rb
|
403
431
|
- lib/writefully/process.rb
|
@@ -410,6 +438,7 @@ files:
|
|
410
438
|
- lib/writefully/tools/initializer.rb
|
411
439
|
- lib/writefully/tools/pencil.rb
|
412
440
|
- lib/writefully/tools/pigeon.rb
|
441
|
+
- lib/writefully/tools/retryer.rb
|
413
442
|
- lib/writefully/tools/synchronizer.rb
|
414
443
|
- lib/writefully/tools.rb
|
415
444
|
- lib/writefully/version.rb
|