pipes 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.md +11 -1
- data/lib/pipes.rb +4 -3
- data/lib/pipes/server.rb +50 -0
- data/lib/pipes/server/views/pipes.erb +88 -0
- data/lib/pipes/store.rb +18 -12
- data/lib/pipes/version.rb +1 -1
- data/pipes.gemspec +1 -1
- metadata +41 -15
data/README.md
CHANGED
|
@@ -101,7 +101,7 @@ end
|
|
|
101
101
|
|
|
102
102
|
There's more advanced ways of defining stages, more on that later.
|
|
103
103
|
|
|
104
|
-
Stages are
|
|
104
|
+
Stages are ordered lexically. That is, the order in which you define your stages in the config determines the order they will be run.
|
|
105
105
|
|
|
106
106
|
The name of the stage is arbitrary. Above, we have `content_writers`, `publishers` and `notifiers`, though there's no significant meaning. The name of the stage can be later extracted and presented to the user or referenced as a symbol.
|
|
107
107
|
|
|
@@ -443,6 +443,12 @@ When Pipes enqueues `Writers::HTMLWriter` and all its dependencies, it will chec
|
|
|
443
443
|
|
|
444
444
|
This option helps prevent adding redundant jobs to the queue. See the section *Queueing Up Additional Jobs*.
|
|
445
445
|
|
|
446
|
+
## Resque Tab
|
|
447
|
+
|
|
448
|
+
Pipes provides a Resque interface tab to help visualize and manage your queues.
|
|
449
|
+
|
|
450
|
+
|
|
451
|
+
|
|
446
452
|
## Working With Resque Priorities
|
|
447
453
|
|
|
448
454
|
Pipes is designed to work on top of Resque's already-existing queueing system. That is, the queue priorities Resque provides will continue to be honored.
|
|
@@ -621,6 +627,10 @@ Pipes.enqueue([Writers::HTMLWriter], {follow_links: true}, {})
|
|
|
621
627
|
Pipes.enqueue([Writers::HTMLWriter], {follow_links: true}, {resolve: true})
|
|
622
628
|
```
|
|
623
629
|
|
|
630
|
+
---
|
|
631
|
+
|
|
632
|
+
Until Resque 2.0 is released, Pipes could be cloggering your `before_fork` hook. Resque only allows one `before_fork` hook and Pipes [defines one](https://github.com/mikepack/pipes/blob/master/lib/pipes/resque_hooks.rb). In Resque 2.0, you can define multiple hooks. If you don't define your own `before_fork` hook, you'll be fine.
|
|
633
|
+
|
|
624
634
|
## Future Improvements
|
|
625
635
|
|
|
626
636
|
- Better atomicity
|
data/lib/pipes.rb
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
module Pipes
|
|
2
2
|
# Default options
|
|
3
|
-
@redis
|
|
4
|
-
@resolve
|
|
3
|
+
@redis = $redis
|
|
4
|
+
@resolve = true
|
|
5
5
|
|
|
6
6
|
class << self
|
|
7
7
|
attr_reader :redis
|
|
@@ -56,4 +56,5 @@ require 'pipes/utils'
|
|
|
56
56
|
require 'pipes/stage_parser'
|
|
57
57
|
require 'pipes/store'
|
|
58
58
|
require 'pipes/runner'
|
|
59
|
-
require 'pipes/resque_hooks'
|
|
59
|
+
require 'pipes/resque_hooks'
|
|
60
|
+
require 'pipes/server'
|
data/lib/pipes/server.rb
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
require 'resque/server'
|
|
2
|
+
|
|
3
|
+
module Pipes
|
|
4
|
+
module Server
|
|
5
|
+
def self.included(base)
|
|
6
|
+
base.class_eval do
|
|
7
|
+
helpers do
|
|
8
|
+
def local_template(path)
|
|
9
|
+
File.read(File.join(File.dirname(__FILE__), "server/views/#{path}"))
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
def titleize(stage)
|
|
13
|
+
stage.to_s.split('_').each(&:capitalize!).join(' ')
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
get "/pipes" do
|
|
18
|
+
erb local_template('pipes.erb')
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
get "/pipes.poll" do
|
|
22
|
+
@polling = true
|
|
23
|
+
erb local_template('pipes.erb'), {:layout => false}
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
post "/pipes/force_next" do
|
|
27
|
+
Pipes::Store.remaining_jobs.clear
|
|
28
|
+
Pipes::Store.next_stage
|
|
29
|
+
redirect url_path("pipes")
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
post "/pipes/clear" do
|
|
33
|
+
Pipes::Store.clear(params[:stage_name].to_sym)
|
|
34
|
+
redirect url_path("pipes")
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
post "/pipes/clear_all" do
|
|
38
|
+
Pipes::Store.clear_all
|
|
39
|
+
redirect url_path("pipes")
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
Resque::Server.tabs << 'Pipes'
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
Resque::Server.class_eval do
|
|
49
|
+
include Pipes::Server
|
|
50
|
+
end
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
<div class="pipes">
|
|
2
|
+
<h1>Your Stages
|
|
3
|
+
<form method="POST" action="<%= url_path('pipes/clear_all') %>">
|
|
4
|
+
<input type='submit' name='' value='Clear All' onclick='return confirm("Are you absolutely sure? This cannot be undone.");' />
|
|
5
|
+
</form>
|
|
6
|
+
</h1>
|
|
7
|
+
|
|
8
|
+
Below is the current status of your configured stages.
|
|
9
|
+
|
|
10
|
+
<div class="section">
|
|
11
|
+
<h2>Jobs Remaining Before Next Stage
|
|
12
|
+
<form method="POST" action="<%= url_path('pipes/force_next') %>">
|
|
13
|
+
<input type='submit' name='' value='Force Next Stage' onclick='return confirm("Are you absolutely sure? This cannot be undone.");' />
|
|
14
|
+
</form>
|
|
15
|
+
</h2>
|
|
16
|
+
<%= Pipes::Store.remaining_jobs %> remaining job(s)
|
|
17
|
+
</div>
|
|
18
|
+
|
|
19
|
+
<% Pipes::Store.stages.each do |stage| %>
|
|
20
|
+
<div class="section">
|
|
21
|
+
<h2><%= titleize(stage) %>
|
|
22
|
+
<form method="POST" action="<%= url_path('pipes/clear') %>">
|
|
23
|
+
<input name="stage_name" type="hidden" value="<%= stage %>" />
|
|
24
|
+
<input type='submit' name='' value='Clear Stage' onclick='return confirm("Are you absolutely sure? This cannot be undone.");' />
|
|
25
|
+
</form>
|
|
26
|
+
</h2>
|
|
27
|
+
<div class="stage">
|
|
28
|
+
<% Pipes::Store.pending_jobs(stage).each do |job| %>
|
|
29
|
+
<div class="job">
|
|
30
|
+
<span class="class"><%= job[:class] %></span><br />
|
|
31
|
+
<span class="args">Args: <%= job[:args].join(', ') %></span>
|
|
32
|
+
</div>
|
|
33
|
+
<% end %>
|
|
34
|
+
<div style="clear: both;"></div>
|
|
35
|
+
</div>
|
|
36
|
+
</div>
|
|
37
|
+
<% end %>
|
|
38
|
+
|
|
39
|
+
<hr />
|
|
40
|
+
<%= partial :overview %>
|
|
41
|
+
</pipes>
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
<% ################## STYLES ################## %>
|
|
45
|
+
|
|
46
|
+
<style>
|
|
47
|
+
.section {
|
|
48
|
+
margin-top: 30px;
|
|
49
|
+
}
|
|
50
|
+
.pipes form {
|
|
51
|
+
display: inline-block !important;
|
|
52
|
+
float: none !important;
|
|
53
|
+
margin: 0 0 0 5px !important;
|
|
54
|
+
}
|
|
55
|
+
.stage {
|
|
56
|
+
border: 1px solid #DDD;
|
|
57
|
+
padding-bottom: 20px;
|
|
58
|
+
-webkit-border-radius: 5px;
|
|
59
|
+
-moz-border-radius: 5px;
|
|
60
|
+
border-radius: 5px;
|
|
61
|
+
background: #EEE;
|
|
62
|
+
margin-bottom: 30px;
|
|
63
|
+
-webkit-box-shadow: 0px 0px 15px rgba(50, 50, 50, 0.3);
|
|
64
|
+
-moz-box-shadow: 0px 0px 15px rgba(50, 50, 50, 0.3);
|
|
65
|
+
box-shadow: 0px 0px 15px rgba(50, 50, 50, 0.3);
|
|
66
|
+
}
|
|
67
|
+
.stage .job {
|
|
68
|
+
float: left;
|
|
69
|
+
background: #FFF;
|
|
70
|
+
width: 190px;
|
|
71
|
+
margin: 20px 0 0 20px;
|
|
72
|
+
border: 1px solid #78c3c8;
|
|
73
|
+
overflow: hidden;
|
|
74
|
+
padding: 10px;
|
|
75
|
+
-webkit-border-radius: 5px;
|
|
76
|
+
-moz-border-radius: 5px;
|
|
77
|
+
border-radius: 5px;
|
|
78
|
+
}
|
|
79
|
+
.stage .job:hover {
|
|
80
|
+
overflow: visible;
|
|
81
|
+
}
|
|
82
|
+
.stage .job .class {
|
|
83
|
+
font-weight: bold;
|
|
84
|
+
}
|
|
85
|
+
.stage .job .args {
|
|
86
|
+
color: #444;
|
|
87
|
+
}
|
|
88
|
+
</style>
|
data/lib/pipes/store.rb
CHANGED
|
@@ -79,6 +79,24 @@ module Pipes
|
|
|
79
79
|
remaining_jobs.clear
|
|
80
80
|
end
|
|
81
81
|
|
|
82
|
+
# Stages specified in the configuration.
|
|
83
|
+
#
|
|
84
|
+
def self.stages
|
|
85
|
+
StageParser.new.stage_names
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
# Jobs left in a given stage.
|
|
89
|
+
#
|
|
90
|
+
def self.pending_jobs(stage)
|
|
91
|
+
Redis::List.new(stage_key(stage), marshal: true)
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
# Jobs remaining before the next stage will be evaluated
|
|
95
|
+
#
|
|
96
|
+
def self.remaining_jobs
|
|
97
|
+
@remaining_jobs ||= Redis::Counter.new(@redis_remaining_key)
|
|
98
|
+
end
|
|
99
|
+
|
|
82
100
|
private
|
|
83
101
|
|
|
84
102
|
def self.valid_for_queue?(stage, pending, job, options)
|
|
@@ -92,22 +110,10 @@ module Pipes
|
|
|
92
110
|
!pending.include?(job)
|
|
93
111
|
end
|
|
94
112
|
|
|
95
|
-
def self.stages
|
|
96
|
-
StageParser.new.stage_names
|
|
97
|
-
end
|
|
98
|
-
|
|
99
113
|
def self.stage_key(name)
|
|
100
114
|
"#{@redis_stages_key}:#{name}"
|
|
101
115
|
end
|
|
102
116
|
|
|
103
|
-
def self.pending_jobs(stage)
|
|
104
|
-
Redis::List.new(stage_key(stage), marshal: true)
|
|
105
|
-
end
|
|
106
|
-
|
|
107
|
-
def self.remaining_jobs
|
|
108
|
-
@remaining_jobs ||= Redis::Counter.new(@redis_remaining_key)
|
|
109
|
-
end
|
|
110
|
-
|
|
111
117
|
def self.namespace
|
|
112
118
|
"#{Pipes.namespace + ':' if Pipes.namespace}#{@namespace}"
|
|
113
119
|
end
|
data/lib/pipes/version.rb
CHANGED
data/pipes.gemspec
CHANGED
|
@@ -16,7 +16,7 @@ Gem::Specification.new do |gem|
|
|
|
16
16
|
gem.test_files = gem.files.grep(%r{^(spec)/})
|
|
17
17
|
gem.require_paths = ["lib"]
|
|
18
18
|
|
|
19
|
-
gem.add_dependency 'resque', '~> 1.
|
|
19
|
+
gem.add_dependency 'resque', '~> 1.23.0'
|
|
20
20
|
gem.add_dependency 'redis-objects', '~> 0.5.3'
|
|
21
21
|
gem.add_dependency 'abyss', '~> 0.4.0'
|
|
22
22
|
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: pipes
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.3.0
|
|
5
5
|
prerelease:
|
|
6
6
|
platform: ruby
|
|
7
7
|
authors:
|
|
@@ -9,22 +9,27 @@ authors:
|
|
|
9
9
|
autorequire:
|
|
10
10
|
bindir: bin
|
|
11
11
|
cert_chain: []
|
|
12
|
-
date: 2012-
|
|
12
|
+
date: 2012-11-27 00:00:00.000000000 Z
|
|
13
13
|
dependencies:
|
|
14
14
|
- !ruby/object:Gem::Dependency
|
|
15
15
|
name: resque
|
|
16
|
-
requirement:
|
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
|
17
17
|
none: false
|
|
18
18
|
requirements:
|
|
19
19
|
- - ~>
|
|
20
20
|
- !ruby/object:Gem::Version
|
|
21
|
-
version: 1.
|
|
21
|
+
version: 1.23.0
|
|
22
22
|
type: :runtime
|
|
23
23
|
prerelease: false
|
|
24
|
-
version_requirements:
|
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
25
|
+
none: false
|
|
26
|
+
requirements:
|
|
27
|
+
- - ~>
|
|
28
|
+
- !ruby/object:Gem::Version
|
|
29
|
+
version: 1.23.0
|
|
25
30
|
- !ruby/object:Gem::Dependency
|
|
26
31
|
name: redis-objects
|
|
27
|
-
requirement:
|
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
|
28
33
|
none: false
|
|
29
34
|
requirements:
|
|
30
35
|
- - ~>
|
|
@@ -32,10 +37,15 @@ dependencies:
|
|
|
32
37
|
version: 0.5.3
|
|
33
38
|
type: :runtime
|
|
34
39
|
prerelease: false
|
|
35
|
-
version_requirements:
|
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
41
|
+
none: false
|
|
42
|
+
requirements:
|
|
43
|
+
- - ~>
|
|
44
|
+
- !ruby/object:Gem::Version
|
|
45
|
+
version: 0.5.3
|
|
36
46
|
- !ruby/object:Gem::Dependency
|
|
37
47
|
name: abyss
|
|
38
|
-
requirement:
|
|
48
|
+
requirement: !ruby/object:Gem::Requirement
|
|
39
49
|
none: false
|
|
40
50
|
requirements:
|
|
41
51
|
- - ~>
|
|
@@ -43,10 +53,15 @@ dependencies:
|
|
|
43
53
|
version: 0.4.0
|
|
44
54
|
type: :runtime
|
|
45
55
|
prerelease: false
|
|
46
|
-
version_requirements:
|
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
57
|
+
none: false
|
|
58
|
+
requirements:
|
|
59
|
+
- - ~>
|
|
60
|
+
- !ruby/object:Gem::Version
|
|
61
|
+
version: 0.4.0
|
|
47
62
|
- !ruby/object:Gem::Dependency
|
|
48
63
|
name: rspec
|
|
49
|
-
requirement:
|
|
64
|
+
requirement: !ruby/object:Gem::Requirement
|
|
50
65
|
none: false
|
|
51
66
|
requirements:
|
|
52
67
|
- - ! '>='
|
|
@@ -54,10 +69,15 @@ dependencies:
|
|
|
54
69
|
version: '0'
|
|
55
70
|
type: :development
|
|
56
71
|
prerelease: false
|
|
57
|
-
version_requirements:
|
|
72
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
73
|
+
none: false
|
|
74
|
+
requirements:
|
|
75
|
+
- - ! '>='
|
|
76
|
+
- !ruby/object:Gem::Version
|
|
77
|
+
version: '0'
|
|
58
78
|
- !ruby/object:Gem::Dependency
|
|
59
79
|
name: rake
|
|
60
|
-
requirement:
|
|
80
|
+
requirement: !ruby/object:Gem::Requirement
|
|
61
81
|
none: false
|
|
62
82
|
requirements:
|
|
63
83
|
- - ! '>='
|
|
@@ -65,7 +85,12 @@ dependencies:
|
|
|
65
85
|
version: '0'
|
|
66
86
|
type: :development
|
|
67
87
|
prerelease: false
|
|
68
|
-
version_requirements:
|
|
88
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
89
|
+
none: false
|
|
90
|
+
requirements:
|
|
91
|
+
- - ! '>='
|
|
92
|
+
- !ruby/object:Gem::Version
|
|
93
|
+
version: '0'
|
|
69
94
|
description: A Redis-backed concurrency management system
|
|
70
95
|
email:
|
|
71
96
|
- mikepackdev@gmail.com
|
|
@@ -84,6 +109,8 @@ files:
|
|
|
84
109
|
- lib/pipes.rb
|
|
85
110
|
- lib/pipes/resque_hooks.rb
|
|
86
111
|
- lib/pipes/runner.rb
|
|
112
|
+
- lib/pipes/server.rb
|
|
113
|
+
- lib/pipes/server/views/pipes.erb
|
|
87
114
|
- lib/pipes/stage_parser.rb
|
|
88
115
|
- lib/pipes/store.rb
|
|
89
116
|
- lib/pipes/utils.rb
|
|
@@ -117,7 +144,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
117
144
|
version: '0'
|
|
118
145
|
requirements: []
|
|
119
146
|
rubyforge_project:
|
|
120
|
-
rubygems_version: 1.8.
|
|
147
|
+
rubygems_version: 1.8.24
|
|
121
148
|
signing_key:
|
|
122
149
|
specification_version: 3
|
|
123
150
|
summary: A Redis-backed concurrency management system
|
|
@@ -130,4 +157,3 @@ test_files:
|
|
|
130
157
|
- spec/pipes/utils_spec.rb
|
|
131
158
|
- spec/pipes_spec.rb
|
|
132
159
|
- spec/spec_helper.rb
|
|
133
|
-
has_rdoc:
|