yapra 0.1.1 → 0.1.2

Sign up to get free protection for your applications and to get access to all the features.
data/History.txt CHANGED
@@ -1,3 +1,12 @@
1
+ == 0.1.2
2
+
3
+ * 1 bug fix:
4
+ * rss 1.0 format bug is fixed. (RSS::save)
5
+
6
+ * 2 majar enhancement:
7
+ * Error handling of plugin is added.
8
+ * Plugin load error has more detailed informations.
9
+
1
10
  == 0.1.1
2
11
 
3
12
  * 1 bug fix:
data/Manifest.txt CHANGED
@@ -64,12 +64,16 @@ lib-plugins/yapra/plugin/config/web_post.rb
64
64
  lib-plugins/yapra/plugin/feed/custom.rb
65
65
  lib-plugins/yapra/plugin/feed/load.rb
66
66
  lib-plugins/yapra/plugin/filter/apply_template.rb
67
+ lib-plugins/yapra/plugin/filter/deduped.rb
67
68
  lib-plugins/yapra/plugin/filter/entry_full_text.rb
68
69
  lib-plugins/yapra/plugin/publish/file_download.rb
69
70
  lib-plugins/yapra/plugin/publish/gmail.rb
70
71
  lib-plugins/yapra/plugin/publish/imap.rb
71
72
  lib-plugins/yapra/plugin/publish/on_memory_download.rb
72
73
  lib-plugins/yapra/plugin/test/append_entry.rb
74
+ lib-plugins/yapra/plugin/test/raise_error.rb
75
+ lib-plugins/yapra/plugin/test/test.rb
76
+ lib-plugins/yapra/plugin/test/test2.rb
73
77
  lib/yapra.rb
74
78
  lib/yapra/config.rb
75
79
  lib/yapra/inflector.rb
@@ -101,6 +105,7 @@ spec/spec_helper.rb
101
105
  spec/yapra/config_spec.rb
102
106
  spec/yapra/legacy_plugin/base_spec.rb
103
107
  spec/yapra/legacy_plugin/registry_factory_spec.rb
108
+ spec/yapra/pipeline_spec.rb
104
109
  spec/yapra_spec.rb
105
110
  tasks/deployment.rake
106
111
  tasks/environment.rake
data/bin/yapra CHANGED
@@ -49,18 +49,27 @@ config_files << 'config.yaml' if config_files.size == 0
49
49
 
50
50
  legacy_plugin_registry_factory = Yapra::LegacyPlugin::RegistryFactory.new(legacy_plugin_directory_paths, mode)
51
51
 
52
+ yapra = nil
52
53
  config_files.each do |config_file|
53
- config = YAML.load(File.read(config_file).toutf8.gsub(/base64::([\w+\/]+=*)/){ Base64.decode64($1) })
54
- config = Yapra::Config.new(config)
55
- config.env.update({
56
- 'log' => {
57
- 'level' => loglebel
58
- }
59
- }) if loglebel
60
- Yapra::Runtime.logger = config.create_logger
61
- yapra = Yapra::Runtime.new(
62
- config.env,
63
- legacy_plugin_registry_factory
64
- )
65
- yapra.execute(config.pipeline_commands)
54
+ begin
55
+ config = YAML.load(File.read(config_file).toutf8.gsub(/base64::([\w+\/]+=*)/){ Base64.decode64($1) })
56
+ config = Yapra::Config.new(config)
57
+ config.env.update({
58
+ 'log' => {
59
+ 'level' => loglebel
60
+ }
61
+ }) if loglebel
62
+ Yapra::Runtime.logger = config.create_logger
63
+ yapra = Yapra::Runtime.new(
64
+ config.env,
65
+ legacy_plugin_registry_factory
66
+ )
67
+ yapra.execute(config.pipeline_commands)
68
+ rescue => ex
69
+ STDERR.puts("* Pipeline, '#{yapra.current_pipeline.name}'") if yapra && yapra.current_pipeline
70
+ STDERR.puts("#{ex.class.name} in '#{ex.message}'")
71
+ ex.backtrace.each do |t|
72
+ STDERR.puts(t)
73
+ end
74
+ end
66
75
  end
@@ -18,7 +18,7 @@ def save(config,data)
18
18
  rss = RSS::Maker.make("1.0") do |maker|
19
19
  maker.channel.about = config['about'] || config['link'] || "http://example.net/"
20
20
  maker.channel.title = config['title'] || "Pragger output"
21
- maker.channel.description = config['description'] || ""
21
+ maker.channel.description = config['description'] || "Generated by Yapra."
22
22
  maker.channel.link = config['link'] || "http://example.net/"
23
23
 
24
24
  data.each do |i|
@@ -1,22 +1,29 @@
1
1
  require 'yapra'
2
+ require 'yapra/runtime'
2
3
  require 'yapra/inflector'
3
4
  require 'yapra/legacy_plugin/base'
4
5
 
5
6
  class Yapra::Pipeline
6
7
  attr_reader :yapra, :context
8
+ attr_writer :logger
7
9
  attr_accessor :legacy_plugin_registry
8
10
 
9
11
  UPPER_CASE = /[A-Z]/
10
12
 
11
- def initialize yapra, pipeline_name
13
+ def initialize pipeline_name, yapra=Yapra::Runtime.new
14
+ @logger = nil
12
15
  @yapra = yapra
13
16
  @context = { 'pipeline_name' => pipeline_name }
14
17
 
15
18
  @module_name_prefix = construct_module_name_prefix yapra.env
16
19
  end
17
20
 
21
+ def name
22
+ self.context[ 'pipeline_name' ]
23
+ end
24
+
18
25
  def logger
19
- Yapra::Runtime.logger
26
+ return @logger || Yapra::Runtime.logger
20
27
  end
21
28
 
22
29
  # start pipeline from commands.
@@ -41,8 +48,20 @@ class Yapra::Pipeline
41
48
  # }
42
49
  # ])
43
50
  def run pipeline_command, data=[]
44
- pipeline_command.inject(data) do |data, command|
45
- execute_plugin(command, data.clone)
51
+ @plugins = []
52
+ begin
53
+ pipeline_command.inject(data) do |data, command|
54
+ execute_plugin(command, data.clone)
55
+ end
56
+ rescue => ex
57
+ @plugins.each do |plugin|
58
+ begin
59
+ plugin.on_error(ex) if plugin.respond_to?('on_error')
60
+ rescue => exx
61
+ self.logger.error("error is occured when error handling: #{exx.message}")
62
+ end
63
+ end
64
+ raise ex
46
65
  end
47
66
  end
48
67
 
@@ -62,23 +81,48 @@ class Yapra::Pipeline
62
81
 
63
82
  def run_legacy_plugin command, data
64
83
  self.logger.debug("evaluate plugin as legacy")
65
- legacy_plugin_registry.get(command['module'])._yapra_run_as_legacy_plugin(command['config'], data)
84
+ data = legacy_plugin_registry.get(command['module'])._yapra_run_as_legacy_plugin(command['config'], data)
85
+ return data
66
86
  end
67
87
 
68
88
  def run_class_based_plugin command, data
69
89
  self.logger.debug("evaluate plugin as class based")
90
+ load_error_stack = []
70
91
  plugin_class = nil
71
92
  @module_name_prefix.each do |prefix|
72
93
  yapra_module_name = "#{prefix}#{command['module']}"
73
- plugin_class = Yapra.load_class_constant(yapra_module_name)
74
- break if plugin_class
94
+ begin
95
+ plugin_class = Yapra.load_class_constant(yapra_module_name)
96
+ break if plugin_class
97
+ rescue LoadError, NameError => ex
98
+ load_error_stack << ex
99
+ end
75
100
  end
76
- raise LoadError unless plugin_class
101
+ raise_load_error(load_error_stack, command) unless plugin_class
102
+
103
+ plugin = initialize_plugin(plugin_class, command)
104
+ @plugins << plugin
105
+ data = plugin.run(data)
106
+ return data
107
+ end
108
+
109
+ def raise_load_error load_error_stack, command
110
+ load_error = LoadError.new("#{command['module']} module is not found.")
111
+ backtrace = load_error.backtrace || []
112
+ load_error_stack.each do |e|
113
+ backtrace << "#{e.class.name} in '#{e.message}'"
114
+ backtrace = backtrace + e.backtrace
115
+ end
116
+ load_error.set_backtrace(backtrace)
117
+ raise load_error
118
+ end
119
+
120
+ def initialize_plugin plugin_class, command
77
121
  plugin = plugin_class.new
78
122
  plugin.yapra = yapra if plugin.respond_to?('yapra=')
79
123
  plugin.pipeline = self if plugin.respond_to?('pipeline=')
80
124
  plugin.plugin_config = command['config'] if plugin.respond_to?('plugin_config=')
81
- plugin.run(data)
125
+ plugin
82
126
  end
83
127
 
84
128
  def construct_module_name_prefix env
@@ -11,7 +11,14 @@ class Yapra::Plugin::MechanizeBase < Yapra::Plugin::Base
11
11
  def extract_attribute_from element, item
12
12
  if plugin_config['extract_xpath']
13
13
  plugin_config['extract_xpath'].each do |k, v|
14
- value = element.search(v).to_html.toutf8
14
+ value = nil
15
+ case v.class.to_s
16
+ when 'String'
17
+ value = element.search(v).to_html.toutf8
18
+ when 'Hash'
19
+ ele = element.at( v['first_node'] )
20
+ value = ( ele.nil? ) ? nil : ele.get_attribute( v['attr'] )
21
+ end
15
22
  set_attribute_to item, k, value
16
23
  end
17
24
  end
data/lib/yapra/runtime.rb CHANGED
@@ -1,3 +1,4 @@
1
+ require 'logger'
1
2
  require 'yapra'
2
3
  require 'yapra/pipeline'
3
4
  require 'yapra/config'
@@ -20,6 +21,9 @@ require 'yapra/inflector'
20
21
  class Yapra::Runtime
21
22
  attr_reader :env
22
23
  attr_reader :legacy_plugin_registry_factory
24
+ attr_reader :current_pipeline
25
+
26
+ @@logger = Logger.new(STDOUT)
23
27
 
24
28
  def initialize env={}, legacy_plugin_registry_factory=nil
25
29
  @env = env
@@ -37,9 +41,11 @@ class Yapra::Runtime
37
41
  # execute one pipeline.
38
42
  def execute_pipeline pipeline_name, command_array, data=[]
39
43
  self.class.logger.info("# pipeline '#{pipeline_name}' is started...")
40
- pipeline = Yapra::Pipeline.new(self, pipeline_name)
44
+ pipeline = Yapra::Pipeline.new(pipeline_name, self)
45
+ @current_pipeline = pipeline
41
46
  legacy_plugin_registory = legacy_plugin_registry_factory.create(pipeline) if legacy_plugin_registry_factory
42
47
  pipeline.run(command_array, data)
48
+ @current_pipeline = nil
43
49
  end
44
50
 
45
51
  def self.logger
data/lib/yapra/version.rb CHANGED
@@ -2,7 +2,7 @@ module Yapra
2
2
  module VERSION #:nodoc:
3
3
  MAJOR = 0
4
4
  MINOR = 1
5
- TINY = 1
5
+ TINY = 2
6
6
 
7
7
  STRING = [MAJOR, MINOR, TINY].join('.')
8
8
  end
data/lib/yapra.rb CHANGED
@@ -30,11 +30,7 @@ module Yapra
30
30
  # TODO create util class, and move this method.
31
31
  def load_class_constant module_name
32
32
  require Yapra::Inflector.underscore(module_name)
33
- Yapra::Inflector.constantize(module_name)
34
- rescue LoadError
35
- nil
36
- rescue NameError
37
- nil
33
+ return Yapra::Inflector.constantize(module_name)
38
34
  end
39
35
  end
40
36
  require 'rss/1.0'
@@ -19,14 +19,14 @@ module Yapra::Plugin::Feed
19
19
  [ config['url'] ]
20
20
  end
21
21
 
22
- urls.each.do |url|
22
+ urls.each do |url|
23
23
  logger.debug("Process: #{url}")
24
24
  source = agent.get(url).body
25
25
  rss = nil
26
26
  begin
27
- rss = RSS::Parser.parse(cont)
27
+ rss = RSS::Parser.parse(source)
28
28
  rescue
29
- rss = RSS::Parser.parse(cont, false)
29
+ rss = RSS::Parser.parse(source, false)
30
30
  end
31
31
  rss.items.each do |item|
32
32
  data << item
@@ -0,0 +1,57 @@
1
+ require 'yapra/plugin/base'
2
+ require 'fileutils'
3
+ require 'pathname'
4
+ require 'digest/md5'
5
+
6
+ module Yapra::Plugin::Filter
7
+ # Filter::Deduped - Plugin to get Deduped entries -- original by emergent
8
+ #
9
+ # Plugin to get Deduped entries
10
+ # Cache path can be set.
11
+ #
12
+ # - module: Filter::Deduped
13
+ # config:
14
+ # path: /tmp/cache/hoge
15
+ #
16
+ class Deduped < Yapra::Plugin::Base
17
+ def run(data)
18
+ cacheroot = Pathname.new(File.join(File.dirname(__FILE__), '..', '..', '..', '..', 'cache'))
19
+ cachepath = Pathname.new(config['path']) || cacheroot
20
+ if cachepath.relative?
21
+ cachepath = cacheroot + cachepath
22
+ end
23
+
24
+ unless File.exists?(cachepath)
25
+ FileUtils.mkdir_p(cachepath)
26
+ end
27
+
28
+ attribute = config['attribute']
29
+
30
+ @cache_paths = []
31
+ deduped_data = data.select {|d|
32
+ v = d.link rescue d.to_s
33
+ if attribute && d.respond_to?(attribute)
34
+ v = d.__send__(attribute).to_s
35
+ end
36
+ hashpath = File.join(cachepath.to_s, Digest::MD5.hexdigest(v))
37
+ if File.exists?(hashpath)
38
+ false
39
+ else
40
+ begin
41
+ File.open(hashpath, "wb").write(v)
42
+ @cache_paths << hashpath
43
+ true
44
+ rescue
45
+ false
46
+ end
47
+ end
48
+ }
49
+ return deduped_data
50
+ end
51
+
52
+ def on_error(ex)
53
+ logger.debug('error is occured.')
54
+ FileUtils.rm(@cache_paths, {:force => true})
55
+ end
56
+ end
57
+ end
@@ -0,0 +1,10 @@
1
+ require 'yapra/plugin/base'
2
+
3
+ module Yapra::Plugin::Test
4
+ class RaiseError < Yapra::Plugin::Base
5
+ def run(data)
6
+ message = config['message'] || 'Error.'
7
+ raise message
8
+ end
9
+ end
10
+ end
@@ -0,0 +1,14 @@
1
+ require 'yapra/plugin'
2
+
3
+ module Yapra::Plugin::Test
4
+ class Test
5
+ def run(data)
6
+ puts 'test!!'
7
+ data
8
+ end
9
+
10
+ def on_error(ex)
11
+ puts 'on error!!'
12
+ end
13
+ end
14
+ end
@@ -0,0 +1,14 @@
1
+ require 'yapra/plugin/base'
2
+
3
+ module Yapra::Plugin::Test
4
+ class Test2 < Yapra::Plugin::Base
5
+ def run(data)
6
+ logger.debug 'test2!!'
7
+ data
8
+ end
9
+
10
+ def on_error(ex)
11
+ logger.debug 'on error2!!'
12
+ end
13
+ end
14
+ end
@@ -24,7 +24,7 @@ end
24
24
 
25
25
  def deduped config, data
26
26
  cacheroot = Pathname(__FILE__).parent.parent.parent.realpath + 'cache'
27
- cachepath = Pathname.new(config['path']) || root
27
+ cachepath = Pathname.new(config['path']) || cacheroot
28
28
  if cachepath.relative?
29
29
  cachepath = cacheroot + cachepath
30
30
  end
@@ -0,0 +1,63 @@
1
+ require File.dirname(__FILE__) + '/../spec_helper.rb'
2
+ require 'yapra/pipeline'
3
+
4
+ describe Yapra::Pipeline do
5
+ before do
6
+ require 'yapra/plugin/test/test'
7
+ @pipeline = Yapra::Pipeline.new('spec test')
8
+ @test_plugin = Yapra::Plugin::Test::Test.new
9
+ end
10
+
11
+ it 'execute pipeline from commands.' do
12
+ @pipeline.run([
13
+ {
14
+ 'module' => 'Test::Test'
15
+ }
16
+ ])
17
+ end
18
+
19
+ it 'call run method of plugin, when pipeline is running.' do
20
+ data = []
21
+ Yapra::Plugin::Test::Test.should_receive(:new).and_return(@test_plugin)
22
+ @test_plugin.should_receive(:run).with(data).and_return([])
23
+ @pipeline.run([
24
+ {
25
+ 'module' => 'Test::Test'
26
+ }
27
+ ], data)
28
+ end
29
+
30
+ it 'transfer previous plugin return object to next plugin.' do
31
+ require 'yapra/plugin/test/test2'
32
+ @test_plugin2 = Yapra::Plugin::Test::Test2.new
33
+ data = []
34
+ modified_data = [ 'modified' ]
35
+ Yapra::Plugin::Test::Test.should_receive(:new).and_return(@test_plugin)
36
+ @test_plugin.should_receive(:run).with(data).and_return(modified_data)
37
+ Yapra::Plugin::Test::Test2.should_receive(:new).and_return(@test_plugin2)
38
+ @test_plugin2.should_receive(:run).with(modified_data)
39
+ @pipeline.run([
40
+ {
41
+ 'module' => 'Test::Test'
42
+ },
43
+ {
44
+ 'module' => 'Test::Test2'
45
+ }
46
+ ], data)
47
+ end
48
+
49
+ it 'call on_error method of plugin, when Error has occured.' do
50
+ Yapra::Plugin::Test::Test.should_receive(:new).and_return(@test_plugin)
51
+ @test_plugin.should_receive(:on_error)
52
+ lambda {
53
+ @pipeline.run([
54
+ {
55
+ 'module' => 'Test::Test'
56
+ },
57
+ {
58
+ 'module' => 'Test::RaiseError'
59
+ }
60
+ ])
61
+ }.should raise_error
62
+ end
63
+ end
data/spec/yapra_spec.rb CHANGED
@@ -9,7 +9,9 @@ describe Yapra do
9
9
  end
10
10
 
11
11
  it 'can\'t load constant from invalid name.' do
12
- Yapra.load_class_constant('_arheiuhri_333***').should be_nil
12
+ lambda {
13
+ Yapra.load_class_constant('_arheiuhri_333***').should be_nil
14
+ }.should raise_error(LoadError)
13
15
  end
14
16
 
15
17
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: yapra
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.1
4
+ version: 0.1.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Yuanying
@@ -9,10 +9,19 @@ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
11
 
12
- date: 2008-06-26 00:00:00 +09:00
12
+ date: 2008-10-14 00:00:00 +09:00
13
13
  default_executable:
14
- dependencies: []
15
-
14
+ dependencies:
15
+ - !ruby/object:Gem::Dependency
16
+ name: hoe
17
+ type: :development
18
+ version_requirement:
19
+ version_requirements: !ruby/object:Gem::Requirement
20
+ requirements:
21
+ - - ">="
22
+ - !ruby/object:Gem::Version
23
+ version: 1.7.0
24
+ version:
16
25
  description: Yet another pragger implementation.
17
26
  email:
18
27
  - yuanying at fraction dot jp
@@ -94,12 +103,16 @@ files:
94
103
  - lib-plugins/yapra/plugin/feed/custom.rb
95
104
  - lib-plugins/yapra/plugin/feed/load.rb
96
105
  - lib-plugins/yapra/plugin/filter/apply_template.rb
106
+ - lib-plugins/yapra/plugin/filter/deduped.rb
97
107
  - lib-plugins/yapra/plugin/filter/entry_full_text.rb
98
108
  - lib-plugins/yapra/plugin/publish/file_download.rb
99
109
  - lib-plugins/yapra/plugin/publish/gmail.rb
100
110
  - lib-plugins/yapra/plugin/publish/imap.rb
101
111
  - lib-plugins/yapra/plugin/publish/on_memory_download.rb
102
112
  - lib-plugins/yapra/plugin/test/append_entry.rb
113
+ - lib-plugins/yapra/plugin/test/raise_error.rb
114
+ - lib-plugins/yapra/plugin/test/test.rb
115
+ - lib-plugins/yapra/plugin/test/test2.rb
103
116
  - lib/yapra.rb
104
117
  - lib/yapra/config.rb
105
118
  - lib/yapra/inflector.rb
@@ -131,6 +144,7 @@ files:
131
144
  - spec/yapra/config_spec.rb
132
145
  - spec/yapra/legacy_plugin/base_spec.rb
133
146
  - spec/yapra/legacy_plugin/registry_factory_spec.rb
147
+ - spec/yapra/pipeline_spec.rb
134
148
  - spec/yapra_spec.rb
135
149
  - tasks/deployment.rake
136
150
  - tasks/environment.rake