yapra 0.1.1 → 0.1.2
Sign up to get free protection for your applications and to get access to all the features.
- data/History.txt +9 -0
- data/Manifest.txt +5 -0
- data/bin/yapra +22 -13
- data/legacy_plugins/RSS/save.rb +1 -1
- data/lib/yapra/pipeline.rb +53 -9
- data/lib/yapra/plugin/mechanize_base.rb +8 -1
- data/lib/yapra/runtime.rb +7 -1
- data/lib/yapra/version.rb +1 -1
- data/lib/yapra.rb +1 -5
- data/lib-plugins/yapra/plugin/feed/load.rb +3 -3
- data/lib-plugins/yapra/plugin/filter/deduped.rb +57 -0
- data/lib-plugins/yapra/plugin/test/raise_error.rb +10 -0
- data/lib-plugins/yapra/plugin/test/test.rb +14 -0
- data/lib-plugins/yapra/plugin/test/test2.rb +14 -0
- data/plugins/Filter/deduped.rb +1 -1
- data/spec/yapra/pipeline_spec.rb +63 -0
- data/spec/yapra_spec.rb +3 -1
- metadata +18 -4
data/History.txt
CHANGED
data/Manifest.txt
CHANGED
@@ -64,12 +64,16 @@ lib-plugins/yapra/plugin/config/web_post.rb
|
|
64
64
|
lib-plugins/yapra/plugin/feed/custom.rb
|
65
65
|
lib-plugins/yapra/plugin/feed/load.rb
|
66
66
|
lib-plugins/yapra/plugin/filter/apply_template.rb
|
67
|
+
lib-plugins/yapra/plugin/filter/deduped.rb
|
67
68
|
lib-plugins/yapra/plugin/filter/entry_full_text.rb
|
68
69
|
lib-plugins/yapra/plugin/publish/file_download.rb
|
69
70
|
lib-plugins/yapra/plugin/publish/gmail.rb
|
70
71
|
lib-plugins/yapra/plugin/publish/imap.rb
|
71
72
|
lib-plugins/yapra/plugin/publish/on_memory_download.rb
|
72
73
|
lib-plugins/yapra/plugin/test/append_entry.rb
|
74
|
+
lib-plugins/yapra/plugin/test/raise_error.rb
|
75
|
+
lib-plugins/yapra/plugin/test/test.rb
|
76
|
+
lib-plugins/yapra/plugin/test/test2.rb
|
73
77
|
lib/yapra.rb
|
74
78
|
lib/yapra/config.rb
|
75
79
|
lib/yapra/inflector.rb
|
@@ -101,6 +105,7 @@ spec/spec_helper.rb
|
|
101
105
|
spec/yapra/config_spec.rb
|
102
106
|
spec/yapra/legacy_plugin/base_spec.rb
|
103
107
|
spec/yapra/legacy_plugin/registry_factory_spec.rb
|
108
|
+
spec/yapra/pipeline_spec.rb
|
104
109
|
spec/yapra_spec.rb
|
105
110
|
tasks/deployment.rake
|
106
111
|
tasks/environment.rake
|
data/bin/yapra
CHANGED
@@ -49,18 +49,27 @@ config_files << 'config.yaml' if config_files.size == 0
|
|
49
49
|
|
50
50
|
legacy_plugin_registry_factory = Yapra::LegacyPlugin::RegistryFactory.new(legacy_plugin_directory_paths, mode)
|
51
51
|
|
52
|
+
yapra = nil
|
52
53
|
config_files.each do |config_file|
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
'
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
54
|
+
begin
|
55
|
+
config = YAML.load(File.read(config_file).toutf8.gsub(/base64::([\w+\/]+=*)/){ Base64.decode64($1) })
|
56
|
+
config = Yapra::Config.new(config)
|
57
|
+
config.env.update({
|
58
|
+
'log' => {
|
59
|
+
'level' => loglebel
|
60
|
+
}
|
61
|
+
}) if loglebel
|
62
|
+
Yapra::Runtime.logger = config.create_logger
|
63
|
+
yapra = Yapra::Runtime.new(
|
64
|
+
config.env,
|
65
|
+
legacy_plugin_registry_factory
|
66
|
+
)
|
67
|
+
yapra.execute(config.pipeline_commands)
|
68
|
+
rescue => ex
|
69
|
+
STDERR.puts("* Pipeline, '#{yapra.current_pipeline.name}'") if yapra && yapra.current_pipeline
|
70
|
+
STDERR.puts("#{ex.class.name} in '#{ex.message}'")
|
71
|
+
ex.backtrace.each do |t|
|
72
|
+
STDERR.puts(t)
|
73
|
+
end
|
74
|
+
end
|
66
75
|
end
|
data/legacy_plugins/RSS/save.rb
CHANGED
@@ -18,7 +18,7 @@ def save(config,data)
|
|
18
18
|
rss = RSS::Maker.make("1.0") do |maker|
|
19
19
|
maker.channel.about = config['about'] || config['link'] || "http://example.net/"
|
20
20
|
maker.channel.title = config['title'] || "Pragger output"
|
21
|
-
maker.channel.description = config['description'] || ""
|
21
|
+
maker.channel.description = config['description'] || "Generated by Yapra."
|
22
22
|
maker.channel.link = config['link'] || "http://example.net/"
|
23
23
|
|
24
24
|
data.each do |i|
|
data/lib/yapra/pipeline.rb
CHANGED
@@ -1,22 +1,29 @@
|
|
1
1
|
require 'yapra'
|
2
|
+
require 'yapra/runtime'
|
2
3
|
require 'yapra/inflector'
|
3
4
|
require 'yapra/legacy_plugin/base'
|
4
5
|
|
5
6
|
class Yapra::Pipeline
|
6
7
|
attr_reader :yapra, :context
|
8
|
+
attr_writer :logger
|
7
9
|
attr_accessor :legacy_plugin_registry
|
8
10
|
|
9
11
|
UPPER_CASE = /[A-Z]/
|
10
12
|
|
11
|
-
def initialize
|
13
|
+
def initialize pipeline_name, yapra=Yapra::Runtime.new
|
14
|
+
@logger = nil
|
12
15
|
@yapra = yapra
|
13
16
|
@context = { 'pipeline_name' => pipeline_name }
|
14
17
|
|
15
18
|
@module_name_prefix = construct_module_name_prefix yapra.env
|
16
19
|
end
|
17
20
|
|
21
|
+
def name
|
22
|
+
self.context[ 'pipeline_name' ]
|
23
|
+
end
|
24
|
+
|
18
25
|
def logger
|
19
|
-
Yapra::Runtime.logger
|
26
|
+
return @logger || Yapra::Runtime.logger
|
20
27
|
end
|
21
28
|
|
22
29
|
# start pipeline from commands.
|
@@ -41,8 +48,20 @@ class Yapra::Pipeline
|
|
41
48
|
# }
|
42
49
|
# ])
|
43
50
|
def run pipeline_command, data=[]
|
44
|
-
|
45
|
-
|
51
|
+
@plugins = []
|
52
|
+
begin
|
53
|
+
pipeline_command.inject(data) do |data, command|
|
54
|
+
execute_plugin(command, data.clone)
|
55
|
+
end
|
56
|
+
rescue => ex
|
57
|
+
@plugins.each do |plugin|
|
58
|
+
begin
|
59
|
+
plugin.on_error(ex) if plugin.respond_to?('on_error')
|
60
|
+
rescue => exx
|
61
|
+
self.logger.error("error is occured when error handling: #{exx.message}")
|
62
|
+
end
|
63
|
+
end
|
64
|
+
raise ex
|
46
65
|
end
|
47
66
|
end
|
48
67
|
|
@@ -62,23 +81,48 @@ class Yapra::Pipeline
|
|
62
81
|
|
63
82
|
def run_legacy_plugin command, data
|
64
83
|
self.logger.debug("evaluate plugin as legacy")
|
65
|
-
legacy_plugin_registry.get(command['module'])._yapra_run_as_legacy_plugin(command['config'], data)
|
84
|
+
data = legacy_plugin_registry.get(command['module'])._yapra_run_as_legacy_plugin(command['config'], data)
|
85
|
+
return data
|
66
86
|
end
|
67
87
|
|
68
88
|
def run_class_based_plugin command, data
|
69
89
|
self.logger.debug("evaluate plugin as class based")
|
90
|
+
load_error_stack = []
|
70
91
|
plugin_class = nil
|
71
92
|
@module_name_prefix.each do |prefix|
|
72
93
|
yapra_module_name = "#{prefix}#{command['module']}"
|
73
|
-
|
74
|
-
|
94
|
+
begin
|
95
|
+
plugin_class = Yapra.load_class_constant(yapra_module_name)
|
96
|
+
break if plugin_class
|
97
|
+
rescue LoadError, NameError => ex
|
98
|
+
load_error_stack << ex
|
99
|
+
end
|
75
100
|
end
|
76
|
-
|
101
|
+
raise_load_error(load_error_stack, command) unless plugin_class
|
102
|
+
|
103
|
+
plugin = initialize_plugin(plugin_class, command)
|
104
|
+
@plugins << plugin
|
105
|
+
data = plugin.run(data)
|
106
|
+
return data
|
107
|
+
end
|
108
|
+
|
109
|
+
def raise_load_error load_error_stack, command
|
110
|
+
load_error = LoadError.new("#{command['module']} module is not found.")
|
111
|
+
backtrace = load_error.backtrace || []
|
112
|
+
load_error_stack.each do |e|
|
113
|
+
backtrace << "#{e.class.name} in '#{e.message}'"
|
114
|
+
backtrace = backtrace + e.backtrace
|
115
|
+
end
|
116
|
+
load_error.set_backtrace(backtrace)
|
117
|
+
raise load_error
|
118
|
+
end
|
119
|
+
|
120
|
+
def initialize_plugin plugin_class, command
|
77
121
|
plugin = plugin_class.new
|
78
122
|
plugin.yapra = yapra if plugin.respond_to?('yapra=')
|
79
123
|
plugin.pipeline = self if plugin.respond_to?('pipeline=')
|
80
124
|
plugin.plugin_config = command['config'] if plugin.respond_to?('plugin_config=')
|
81
|
-
plugin
|
125
|
+
plugin
|
82
126
|
end
|
83
127
|
|
84
128
|
def construct_module_name_prefix env
|
@@ -11,7 +11,14 @@ class Yapra::Plugin::MechanizeBase < Yapra::Plugin::Base
|
|
11
11
|
def extract_attribute_from element, item
|
12
12
|
if plugin_config['extract_xpath']
|
13
13
|
plugin_config['extract_xpath'].each do |k, v|
|
14
|
-
value =
|
14
|
+
value = nil
|
15
|
+
case v.class.to_s
|
16
|
+
when 'String'
|
17
|
+
value = element.search(v).to_html.toutf8
|
18
|
+
when 'Hash'
|
19
|
+
ele = element.at( v['first_node'] )
|
20
|
+
value = ( ele.nil? ) ? nil : ele.get_attribute( v['attr'] )
|
21
|
+
end
|
15
22
|
set_attribute_to item, k, value
|
16
23
|
end
|
17
24
|
end
|
data/lib/yapra/runtime.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
require 'logger'
|
1
2
|
require 'yapra'
|
2
3
|
require 'yapra/pipeline'
|
3
4
|
require 'yapra/config'
|
@@ -20,6 +21,9 @@ require 'yapra/inflector'
|
|
20
21
|
class Yapra::Runtime
|
21
22
|
attr_reader :env
|
22
23
|
attr_reader :legacy_plugin_registry_factory
|
24
|
+
attr_reader :current_pipeline
|
25
|
+
|
26
|
+
@@logger = Logger.new(STDOUT)
|
23
27
|
|
24
28
|
def initialize env={}, legacy_plugin_registry_factory=nil
|
25
29
|
@env = env
|
@@ -37,9 +41,11 @@ class Yapra::Runtime
|
|
37
41
|
# execute one pipeline.
|
38
42
|
def execute_pipeline pipeline_name, command_array, data=[]
|
39
43
|
self.class.logger.info("# pipeline '#{pipeline_name}' is started...")
|
40
|
-
pipeline = Yapra::Pipeline.new(
|
44
|
+
pipeline = Yapra::Pipeline.new(pipeline_name, self)
|
45
|
+
@current_pipeline = pipeline
|
41
46
|
legacy_plugin_registory = legacy_plugin_registry_factory.create(pipeline) if legacy_plugin_registry_factory
|
42
47
|
pipeline.run(command_array, data)
|
48
|
+
@current_pipeline = nil
|
43
49
|
end
|
44
50
|
|
45
51
|
def self.logger
|
data/lib/yapra/version.rb
CHANGED
data/lib/yapra.rb
CHANGED
@@ -30,11 +30,7 @@ module Yapra
|
|
30
30
|
# TODO create util class, and move this method.
|
31
31
|
def load_class_constant module_name
|
32
32
|
require Yapra::Inflector.underscore(module_name)
|
33
|
-
Yapra::Inflector.constantize(module_name)
|
34
|
-
rescue LoadError
|
35
|
-
nil
|
36
|
-
rescue NameError
|
37
|
-
nil
|
33
|
+
return Yapra::Inflector.constantize(module_name)
|
38
34
|
end
|
39
35
|
end
|
40
36
|
require 'rss/1.0'
|
@@ -19,14 +19,14 @@ module Yapra::Plugin::Feed
|
|
19
19
|
[ config['url'] ]
|
20
20
|
end
|
21
21
|
|
22
|
-
urls.each
|
22
|
+
urls.each do |url|
|
23
23
|
logger.debug("Process: #{url}")
|
24
24
|
source = agent.get(url).body
|
25
25
|
rss = nil
|
26
26
|
begin
|
27
|
-
rss = RSS::Parser.parse(
|
27
|
+
rss = RSS::Parser.parse(source)
|
28
28
|
rescue
|
29
|
-
rss = RSS::Parser.parse(
|
29
|
+
rss = RSS::Parser.parse(source, false)
|
30
30
|
end
|
31
31
|
rss.items.each do |item|
|
32
32
|
data << item
|
@@ -0,0 +1,57 @@
|
|
1
|
+
require 'yapra/plugin/base'
|
2
|
+
require 'fileutils'
|
3
|
+
require 'pathname'
|
4
|
+
require 'digest/md5'
|
5
|
+
|
6
|
+
module Yapra::Plugin::Filter
|
7
|
+
# Filter::Deduped - Plugin to get Deduped entries -- original by emergent
|
8
|
+
#
|
9
|
+
# Plugin to get Deduped entries
|
10
|
+
# Cache path can be set.
|
11
|
+
#
|
12
|
+
# - module: Filter::Deduped
|
13
|
+
# config:
|
14
|
+
# path: /tmp/cache/hoge
|
15
|
+
#
|
16
|
+
class Deduped < Yapra::Plugin::Base
|
17
|
+
def run(data)
|
18
|
+
cacheroot = Pathname.new(File.join(File.dirname(__FILE__), '..', '..', '..', '..', 'cache'))
|
19
|
+
cachepath = Pathname.new(config['path']) || cacheroot
|
20
|
+
if cachepath.relative?
|
21
|
+
cachepath = cacheroot + cachepath
|
22
|
+
end
|
23
|
+
|
24
|
+
unless File.exists?(cachepath)
|
25
|
+
FileUtils.mkdir_p(cachepath)
|
26
|
+
end
|
27
|
+
|
28
|
+
attribute = config['attribute']
|
29
|
+
|
30
|
+
@cache_paths = []
|
31
|
+
deduped_data = data.select {|d|
|
32
|
+
v = d.link rescue d.to_s
|
33
|
+
if attribute && d.respond_to?(attribute)
|
34
|
+
v = d.__send__(attribute).to_s
|
35
|
+
end
|
36
|
+
hashpath = File.join(cachepath.to_s, Digest::MD5.hexdigest(v))
|
37
|
+
if File.exists?(hashpath)
|
38
|
+
false
|
39
|
+
else
|
40
|
+
begin
|
41
|
+
File.open(hashpath, "wb").write(v)
|
42
|
+
@cache_paths << hashpath
|
43
|
+
true
|
44
|
+
rescue
|
45
|
+
false
|
46
|
+
end
|
47
|
+
end
|
48
|
+
}
|
49
|
+
return deduped_data
|
50
|
+
end
|
51
|
+
|
52
|
+
def on_error(ex)
|
53
|
+
logger.debug('error is occured.')
|
54
|
+
FileUtils.rm(@cache_paths, {:force => true})
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
data/plugins/Filter/deduped.rb
CHANGED
@@ -24,7 +24,7 @@ end
|
|
24
24
|
|
25
25
|
def deduped config, data
|
26
26
|
cacheroot = Pathname(__FILE__).parent.parent.parent.realpath + 'cache'
|
27
|
-
cachepath = Pathname.new(config['path']) ||
|
27
|
+
cachepath = Pathname.new(config['path']) || cacheroot
|
28
28
|
if cachepath.relative?
|
29
29
|
cachepath = cacheroot + cachepath
|
30
30
|
end
|
@@ -0,0 +1,63 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/../spec_helper.rb'
|
2
|
+
require 'yapra/pipeline'
|
3
|
+
|
4
|
+
describe Yapra::Pipeline do
|
5
|
+
before do
|
6
|
+
require 'yapra/plugin/test/test'
|
7
|
+
@pipeline = Yapra::Pipeline.new('spec test')
|
8
|
+
@test_plugin = Yapra::Plugin::Test::Test.new
|
9
|
+
end
|
10
|
+
|
11
|
+
it 'execute pipeline from commands.' do
|
12
|
+
@pipeline.run([
|
13
|
+
{
|
14
|
+
'module' => 'Test::Test'
|
15
|
+
}
|
16
|
+
])
|
17
|
+
end
|
18
|
+
|
19
|
+
it 'call run method of plugin, when pipeline is running.' do
|
20
|
+
data = []
|
21
|
+
Yapra::Plugin::Test::Test.should_receive(:new).and_return(@test_plugin)
|
22
|
+
@test_plugin.should_receive(:run).with(data).and_return([])
|
23
|
+
@pipeline.run([
|
24
|
+
{
|
25
|
+
'module' => 'Test::Test'
|
26
|
+
}
|
27
|
+
], data)
|
28
|
+
end
|
29
|
+
|
30
|
+
it 'transfer previous plugin return object to next plugin.' do
|
31
|
+
require 'yapra/plugin/test/test2'
|
32
|
+
@test_plugin2 = Yapra::Plugin::Test::Test2.new
|
33
|
+
data = []
|
34
|
+
modified_data = [ 'modified' ]
|
35
|
+
Yapra::Plugin::Test::Test.should_receive(:new).and_return(@test_plugin)
|
36
|
+
@test_plugin.should_receive(:run).with(data).and_return(modified_data)
|
37
|
+
Yapra::Plugin::Test::Test2.should_receive(:new).and_return(@test_plugin2)
|
38
|
+
@test_plugin2.should_receive(:run).with(modified_data)
|
39
|
+
@pipeline.run([
|
40
|
+
{
|
41
|
+
'module' => 'Test::Test'
|
42
|
+
},
|
43
|
+
{
|
44
|
+
'module' => 'Test::Test2'
|
45
|
+
}
|
46
|
+
], data)
|
47
|
+
end
|
48
|
+
|
49
|
+
it 'call on_error method of plugin, when Error has occured.' do
|
50
|
+
Yapra::Plugin::Test::Test.should_receive(:new).and_return(@test_plugin)
|
51
|
+
@test_plugin.should_receive(:on_error)
|
52
|
+
lambda {
|
53
|
+
@pipeline.run([
|
54
|
+
{
|
55
|
+
'module' => 'Test::Test'
|
56
|
+
},
|
57
|
+
{
|
58
|
+
'module' => 'Test::RaiseError'
|
59
|
+
}
|
60
|
+
])
|
61
|
+
}.should raise_error
|
62
|
+
end
|
63
|
+
end
|
data/spec/yapra_spec.rb
CHANGED
@@ -9,7 +9,9 @@ describe Yapra do
|
|
9
9
|
end
|
10
10
|
|
11
11
|
it 'can\'t load constant from invalid name.' do
|
12
|
-
|
12
|
+
lambda {
|
13
|
+
Yapra.load_class_constant('_arheiuhri_333***').should be_nil
|
14
|
+
}.should raise_error(LoadError)
|
13
15
|
end
|
14
16
|
|
15
17
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: yapra
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yuanying
|
@@ -9,10 +9,19 @@ autorequire:
|
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
11
|
|
12
|
-
date: 2008-
|
12
|
+
date: 2008-10-14 00:00:00 +09:00
|
13
13
|
default_executable:
|
14
|
-
dependencies:
|
15
|
-
|
14
|
+
dependencies:
|
15
|
+
- !ruby/object:Gem::Dependency
|
16
|
+
name: hoe
|
17
|
+
type: :development
|
18
|
+
version_requirement:
|
19
|
+
version_requirements: !ruby/object:Gem::Requirement
|
20
|
+
requirements:
|
21
|
+
- - ">="
|
22
|
+
- !ruby/object:Gem::Version
|
23
|
+
version: 1.7.0
|
24
|
+
version:
|
16
25
|
description: Yet another pragger implementation.
|
17
26
|
email:
|
18
27
|
- yuanying at fraction dot jp
|
@@ -94,12 +103,16 @@ files:
|
|
94
103
|
- lib-plugins/yapra/plugin/feed/custom.rb
|
95
104
|
- lib-plugins/yapra/plugin/feed/load.rb
|
96
105
|
- lib-plugins/yapra/plugin/filter/apply_template.rb
|
106
|
+
- lib-plugins/yapra/plugin/filter/deduped.rb
|
97
107
|
- lib-plugins/yapra/plugin/filter/entry_full_text.rb
|
98
108
|
- lib-plugins/yapra/plugin/publish/file_download.rb
|
99
109
|
- lib-plugins/yapra/plugin/publish/gmail.rb
|
100
110
|
- lib-plugins/yapra/plugin/publish/imap.rb
|
101
111
|
- lib-plugins/yapra/plugin/publish/on_memory_download.rb
|
102
112
|
- lib-plugins/yapra/plugin/test/append_entry.rb
|
113
|
+
- lib-plugins/yapra/plugin/test/raise_error.rb
|
114
|
+
- lib-plugins/yapra/plugin/test/test.rb
|
115
|
+
- lib-plugins/yapra/plugin/test/test2.rb
|
103
116
|
- lib/yapra.rb
|
104
117
|
- lib/yapra/config.rb
|
105
118
|
- lib/yapra/inflector.rb
|
@@ -131,6 +144,7 @@ files:
|
|
131
144
|
- spec/yapra/config_spec.rb
|
132
145
|
- spec/yapra/legacy_plugin/base_spec.rb
|
133
146
|
- spec/yapra/legacy_plugin/registry_factory_spec.rb
|
147
|
+
- spec/yapra/pipeline_spec.rb
|
134
148
|
- spec/yapra_spec.rb
|
135
149
|
- tasks/deployment.rake
|
136
150
|
- tasks/environment.rake
|