yapra 0.1.0 → 0.1.1

data/History.txt

@@ -1,3 +1,15 @@
+== 0.1.1
+
+* 1 bug fix:
+  * Publish::Imap, mailbox select fixed.
+  * Spec files are added.
+
+* 2 majar enhancement:
+  * A few plugins are added.
+  * RDoc is improved.
+  * -d option is added.
+  * Source code management is migrated to github.com.
+
 == 0.1.0 2008-06-20
 
 * 1 major enhancement:

data/Manifest.txt

@@ -5,8 +5,13 @@ Manifest.txt
 PostInstall.txt
 README.txt
 Rakefile
+bin/yapra
 config/hoe.rb
 config/requirements.rb
+fixtures/config/habu_like.yml
+fixtures/config/mixed_type.yml
+fixtures/config/pragger_like.yml
+fixtures/legacy_plugin/legacy_test_plugin.rb
 legacy_plugins/Download/nicovideo.rb
 legacy_plugins/Feed/custom_feed.rb
 legacy_plugins/Feed/google_calendar.rb
@@ -58,10 +63,12 @@ lib-plugins/yapra/plugin/config/basic_auth.rb
 lib-plugins/yapra/plugin/config/web_post.rb
 lib-plugins/yapra/plugin/feed/custom.rb
 lib-plugins/yapra/plugin/feed/load.rb
+lib-plugins/yapra/plugin/filter/apply_template.rb
 lib-plugins/yapra/plugin/filter/entry_full_text.rb
 lib-plugins/yapra/plugin/publish/file_download.rb
 lib-plugins/yapra/plugin/publish/gmail.rb
 lib-plugins/yapra/plugin/publish/imap.rb
+lib-plugins/yapra/plugin/publish/on_memory_download.rb
 lib-plugins/yapra/plugin/test/append_entry.rb
 lib/yapra.rb
 lib/yapra/config.rb
@@ -81,9 +88,9 @@ lib/yapra/plugin/mechanize_base.rb
 lib/yapra/runtime.rb
 lib/yapra/version.rb
 plugins/Filter/deduped.rb
+plugins/Filter/grep.rb
 plugins/Filter/sort.rb
 plugins/Filter/subs.rb
-bin/yapra
 script/console
 script/destroy
 script/generate
@@ -91,12 +98,14 @@ script/txt2html
 setup.rb
 spec/spec.opts
 spec/spec_helper.rb
+spec/yapra/config_spec.rb
+spec/yapra/legacy_plugin/base_spec.rb
+spec/yapra/legacy_plugin/registry_factory_spec.rb
 spec/yapra_spec.rb
 tasks/deployment.rake
 tasks/environment.rake
 tasks/rspec.rake
 tasks/website.rake
-website/index.html
 website/index.txt
 website/javascripts/rounded_corners_lite.inc.js
 website/stylesheets/screen.css

data/PostInstall.txt

@@ -1,7 +0,0 @@
-
-For more information on yapra, see http://yapra.rubyforge.org
-
-NOTE: Change this information in PostInstall.txt 
-You can also delete it if you don't want it.
-
-

data/README.txt

@@ -1,6 +1,7 @@
 = yapra
 
-* http://www.fraction.jp/trac/rana2/wiki/Yapra
+* http://yapra.rubyforge.org/
+* http://github.com/yuanying/yapra/tree/master
 
 == DESCRIPTION:
 

data/bin/yapra

@@ -14,21 +14,30 @@ $:.insert(0, *[
 ])
 
 legacy_plugin_directory_paths = [
-  Pathname.new(YAPRA_ROOT) + 'legacy_plugins',
-  Pathname.new(YAPRA_ROOT) + 'plugins'
+  File.join(YAPRA_ROOT, 'legacy_plugins'),
+  File.join(YAPRA_ROOT, 'plugins')
 ]
 
+require 'yapra/version'
 require 'yapra/runtime'
 require 'yapra/config'
 require 'yapra/legacy_plugin/registry_factory'
 
-Version     = '0.1.0'
-mode        = 'compatible'
-config_file = "config.yaml"
-loglebel    = nil
+Version       = Yapra::VERSION::STRING
+mode          = 'compatible'
+config_files  = []#"config.yaml"
+loglebel      = nil
 opt = OptionParser.new
-opt.on("-c", "--configfile CONFIGFILE") {|v| config_file = v }
-opt.on("-p", "--plugindir PLUGINDIR") {|v| legacy_plugin_directory_paths << Pathname.new(v) }
+opt.on("-c", "--configfile CONFIGFILE") {|v| config_files << v }
+opt.on("-d", "--configfile-directory CONFIGFILE_DIRECTORY") { |v|
+  Dir.glob(File.join(v, '**/*.yml')).each do |file|
+    config_files << file
+  end
+  Dir.glob(File.join(v, '**/*.yaml')).each do |file|
+    config_files << file
+  end
+}
+opt.on("-p", "--plugindir PLUGINDIR") {|v| legacy_plugin_directory_paths << v }
 opt.on("-m", "--mode MODE", 'compatible / advance') { |v| mode = v }
 opt.on("--log-level LOGLEVEL", 'fatal / error / warn / info / debug') { |v| loglebel = v }
 # opt.on("-u", "--pluginusage PLUGINNAME") {|v| $plugins[v].source.gsub(/^## ?(.*)/){ puts $1 }; exit }
@@ -36,17 +45,22 @@ opt.on("--log-level LOGLEVEL", 'fatal / error / warn / info / debug') { |v| logl
 # opt.on("-w", "--where") { puts(Pathname.new(__FILE__).parent + "plugin"); exit }
 opt.parse!
 
+config_files << 'config.yaml' if config_files.size == 0
+
 legacy_plugin_registry_factory = Yapra::LegacyPlugin::RegistryFactory.new(legacy_plugin_directory_paths, mode)
-config = YAML.load(File.read(config_file).toutf8.gsub(/base64::([\w+\/]+=*)/){ Base64.decode64($1) })
-config = Yapra::Config.new(config)
-config.env.update({
-  'log' => {
-    'level' => loglebel
-  }
-}) if loglebel
-Yapra::Runtime.logger = config.create_logger
-yapra = Yapra::Runtime.new(
-  config.env,
-  legacy_plugin_registry_factory
-)
-yapra.execute(config.pipeline_commands)
+
+config_files.each do |config_file|
+  config = YAML.load(File.read(config_file).toutf8.gsub(/base64::([\w+\/]+=*)/){ Base64.decode64($1) })
+  config = Yapra::Config.new(config)
+  config.env.update({
+    'log' => {
+      'level' => loglebel
+    }
+  }) if loglebel
+  Yapra::Runtime.logger = config.create_logger
+  yapra = Yapra::Runtime.new(
+    config.env,
+    legacy_plugin_registry_factory
+  )
+  yapra.execute(config.pipeline_commands)
+end

data/fixtures/config/habu_like.yml

@@ -0,0 +1,43 @@
+global:
+  log:
+    out: stderr
+    level: debug
+      
+pipeline:
+  pipeline1:
+    - module: RSS::load
+      config:
+        url:
+          - http://www.fraction.jp/log/feed/rdf
+          - http://d.hatena.ne.jp/yuanying/rss
+          - http://yuanying.oeilvert.org/books.rss
+
+    - module: Filter::sort
+      config:
+        method: date
+
+    - module: reverse
+
+    - module: head
+      config:
+        n: 10
+
+    - module: RSS::save
+      config:
+        filename: feed.xml
+        link: http://www.fraction.jp/
+        title: BONNOH FRACTION
+  
+  pipeline2:
+    - module: Feed::pixiv_tag
+      config:
+        id: yuanying
+        password: kojiro
+        tag: C74
+
+    - module: RSS::save
+      config:
+        #filename: /Users/Shared/Sites/fraction/feed.xml
+        filename: pixiv.xml
+        link: http://www.pixiv.net/tags.php?tag=C74
+        title: Pixiv - C74

data/fixtures/config/mixed_type.yml

@@ -0,0 +1,18 @@
+global:
+  log:
+    out: stderr
+    level: debug
+      
+pipeline:
+  - module: Feed::pixiv_tag
+    config:
+      id: yuanying
+      password: kojiro
+      tag: C74
+
+  - module: RSS::save
+    config:
+      #filename: /Users/Shared/Sites/fraction/feed.xml
+      filename: pixiv.xml
+      link: http://www.pixiv.net/tags.php?tag=C74
+      title: Pixiv - C74

data/fixtures/config/pragger_like.yml

@@ -0,0 +1,12 @@
+- module: Feed::pixiv_tag
+  config:
+    id: yuanying
+    password: kojiro
+    tag: C74
+
+- module: RSS::save
+  config:
+    #filename: /Users/Shared/Sites/fraction/feed.xml
+    filename: pixiv.xml
+    link: http://www.pixiv.net/tags.php?tag=C74
+    title: Pixiv - C74

data/fixtures/legacy_plugin/legacy_test_plugin.rb

@@ -0,0 +1,4 @@
+
+def legacy_test_plugin(config, data)
+  data
+end

data/lib/yapra/config.rb

@@ -1,4 +1,9 @@
+require 'logger'
+require 'yapra'
+
 # = Config Examples
+#
+# Config file for yapra.
 # 
 # == Format 1: Pragger like.
 # A simplest. You can run one pipeline without global config.
@@ -54,9 +59,6 @@
 #       config:
 #         a: b
 #
-require 'logger'
-require 'yapra'
-
 class Yapra::Config
   attr_reader :env
   attr_reader :pipeline_commands

data/lib/yapra/legacy_plugin/advance_mode_registry.rb

@@ -2,12 +2,16 @@ require 'yapra/legacy_plugin'
 require 'yapra/inflector'
 
 module Yapra::LegacyPlugin
-  
+  #
+  # AdvanceModeRegistry load legacy plugin at one by one.
+  #
   class AdvanceModeRegistry
     attr_accessor :legacy_plugins
     attr_accessor :plugin_paths
     attr_accessor :pipeline
     
+    # _paths_ :: Directory paths which contain legacy plugins.
+    # _pipeline_ :: Runtime pipline.
     def initialize paths, pipeline
       self.legacy_plugins = {}
       self.plugin_paths = paths.reverse
@@ -18,12 +22,19 @@ module Yapra::LegacyPlugin
       Yapra::Runtime.logger
     end
     
+    # load plugin from module name.
+    # 
+    # example:
+    # 
+    #     registry = Yapra::LegacyPlugin::AdvanceModeRegistry.new(paths, pipeline)
+    #     feed_load_plugin = registry.get('Feed::load')
+    #
     def get module_name
       plugin = legacy_plugins[module_name]
       unless plugin
         plugin_paths.each do |folder|
-          file = folder + (module_name.gsub('::', '/') + '.rb')
-          if file.file?()
+          file = File.join(folder, (module_name.gsub('::', '/') + '.rb'))
+          if File.file?(file)
             plugin = Yapra::LegacyPlugin::Base.new(pipeline, file)
             legacy_plugins[ module_name ] = plugin
             logger.debug "#{module_name} is loaded from #{file}"

data/lib/yapra/legacy_plugin/base.rb

@@ -1,4 +1,5 @@
 # require 'pathname'
+require 'kconv'
 require 'yapra/legacy_plugin'
 
 class Yapra::LegacyPlugin::Base

data/lib/yapra/legacy_plugin/compatible_mode_registry.rb

@@ -4,12 +4,15 @@ require 'yapra/legacy_plugin'
 module Yapra::LegacyPlugin
   class CompatibleModeRegistry
     attr_accessor :legacy_plugins
-      
+    
+    # _paths_ :: Directory paths which contain legacy plugins.
+    # _pipeline_ :: Runtime pipline.
     def initialize paths, pipeline
       self.legacy_plugins = {}
       
       paths.each do |folder|
-        Pathname.glob(folder + "**/*.rb").sort.each do |file|
+        folder = Pathname.new(folder)
+        Pathname.glob(File.join(folder, "**/*.rb")).sort.each do |file|
           module_name = file.relative_path_from(folder).to_s.gsub("/","::")[0..-4]
           begin
             legacy_plugins[ module_name ] = Yapra::LegacyPlugin::Base.new(pipeline, file)
@@ -25,6 +28,13 @@ module Yapra::LegacyPlugin
       Yapra::Runtime.logger
     end
     
+    # load plugin from module name.
+    # 
+    # example:
+    # 
+    #     registry = Yapra::LegacyPlugin::CompatibleModeRegistry.new(paths, pipeline)
+    #     feed_load_plugin = registry.get('Feed::load')
+    #
     def get module_name
       plugin = legacy_plugins[module_name]
       raise "#{module_name} is not registered." unless plugin

data/lib/yapra/legacy_plugin/registry_factory.rb

@@ -1,10 +1,17 @@
 require 'yapra/legacy_plugin'
 require 'yapra/inflector'
 
+# RegistryFactory is factory class for LegacyPluginRegistry.
+# 
+# LegacyPluginRegistry is created by mode.
+# 
+# If mode is compatible, this factory create a CompatibeModeRegistry.
 class Yapra::LegacyPlugin::RegistryFactory
   attr_reader :plugin_paths
   attr_reader :registry_class
   
+  # plugin_paths :: directory paths which are contains legacy plugins.
+  # mode :: 'compatible' / 'advance'
   def initialize plugin_paths, mode = 'compatible'
     registry_name = "Yapra::LegacyPlugin::#{Yapra::Inflector.camelize(mode)}ModeRegistry"
     @registry_class = Yapra.load_class_constant(registry_name)

data/lib/yapra/pipeline.rb

@@ -19,6 +19,27 @@ class Yapra::Pipeline
     Yapra::Runtime.logger
   end
   
+  # start pipeline from commands.
+  # 
+  # example:
+  # 
+  #     pipeline.run([
+  #       {
+  #         'module' => 'Config::agent',
+  #         'config' => {
+  #           'user_agent_alias' => 'Windows IE 6'
+  #         }
+  #       },
+  #       {
+  #         'module' => 'RSS::load',
+  #         'config' => {
+  #           'uri' => 'http://www.example.com/hoge.rdf'
+  #         }
+  #       },
+  #       {
+  #         'module' => 'print'
+  #       }
+  #     ])
   def run pipeline_command, data=[]
     pipeline_command.inject(data) do |data, command|
       execute_plugin(command, data.clone)

data/lib/yapra/plugin/base.rb

@@ -3,6 +3,10 @@ require 'yapra/plugin/context_aware'
 require 'yapra/plugin/erb_applier'
 require 'yapra/plugin/feed_item_operator'
 
+# Yapra plugin base class.
+# 
+# Subclass this base is not required to be Yapra plugin.
+# But a few utility module is added in this base.
 class Yapra::Plugin::Base
   include Yapra::Plugin::ContextAware
   include Yapra::Plugin::FeedItemOperator

data/lib/yapra/plugin/erb_applier.rb

@@ -10,6 +10,6 @@ module Yapra::Plugin::ErbApplier
         template = io.read
       end
     end
-    ERB.new(template).result(apply_binding)
+    ERB.new(template, nil, '-').result(apply_binding)
   end
 end

data/lib/yapra/plugin/mechanize_base.rb

@@ -9,15 +9,15 @@ class Yapra::Plugin::MechanizeBase < Yapra::Plugin::Base
   end
   
   def extract_attribute_from element, item
-    if config['extract_xpath']
-      config['extract_xpath'].each do |k, v|
+    if plugin_config['extract_xpath']
+      plugin_config['extract_xpath'].each do |k, v|
         value = element.search(v).to_html.toutf8
         set_attribute_to item, k, value
       end
     end
 
-    if config['apply_template_after_extracted']
-      config['apply_template_after_extracted'].each do |k, template|
+    if plugin_config['apply_template_after_extracted']
+      plugin_config['apply_template_after_extracted'].each do |k, template|
         value = apply_template template, binding
         set_attribute_to item, k, value
       end

data/lib/yapra/runtime.rb

@@ -3,6 +3,20 @@ require 'yapra/pipeline'
 require 'yapra/config'
 require 'yapra/inflector'
 
+# = How to use
+# 
+#     require 'yapra/runtime'
+#     require 'yapra/config'
+# 
+#     config = YAML.load(config_file)
+#     config = Yapra::Config.new(config)
+# 
+#     Yapra::Runtime.logger = Logger.new(STDOUT)
+# 
+#     yapra = Yapra::Runtime.new(config.env)
+#     yapra.execute(config.pipeline_commands)
+# 
+# config_file format written in Yapra::Config document.
 class Yapra::Runtime
   attr_reader :env
   attr_reader :legacy_plugin_registry_factory
@@ -13,12 +27,14 @@ class Yapra::Runtime
     @legacy_plugin_registry_factory        = legacy_plugin_registry_factory
   end
   
+  # execute pipelines from commands.
   def execute pipeline_commands
     pipeline_commands.each do |k, v|
       execute_pipeline k, v, []
     end
   end
   
+  # execute one pipeline.
   def execute_pipeline pipeline_name, command_array, data=[]
     self.class.logger.info("# pipeline '#{pipeline_name}' is started...")
     pipeline = Yapra::Pipeline.new(self, pipeline_name)

data/lib/yapra/version.rb

@@ -2,7 +2,7 @@ module Yapra
   module VERSION #:nodoc:
     MAJOR = 0
     MINOR = 1
-    TINY  = 0
+    TINY  = 1
 
     STRING = [MAJOR, MINOR, TINY].join('.')
   end

data/lib/yapra.rb

@@ -42,3 +42,5 @@ require 'rss/2.0'
 require 'rss/dublincore'
 require 'rss/maker'
 require 'rubygems'
+
+require 'yapra/inflector'

data/lib-plugins/yapra/plugin/config/agent.rb

@@ -1,19 +1,19 @@
-## Yapra::Config::Agent -- Yuanying
-##
-## add WWW::Mechanize agent to context.
-## 
-## - module: Config::Agent
-##   config:
-##     user_agent_alias: Windows IE 6
-##     proxy:
-##       addr: localhost
-##       port: 8080
-##       user: username
-##       password: password
-##
 require 'yapra/plugin/mechanize_base'
 
 module Yapra::Plugin::Config
+  # Yapra::Config::Agent -- Yuanying
+  # 
+  # add WWW::Mechanize agent to context.
+  # 
+  #     - module: Config::Agent
+  #       config:
+  #         user_agent_alias: Windows IE 6
+  #         proxy:
+  #           addr: localhost
+  #           port: 8080
+  #           user: username
+  #           password: password
+  #
   class Agent < Yapra::Plugin::MechanizeBase
 
     def run(data)

data/lib-plugins/yapra/plugin/config/basic_auth.rb

@@ -1,15 +1,15 @@
-## Config::BasicAuth -- Yuanying
-##
-## post to web page with WWW::Mechanize agent.
-## 
-## - module: Config::BasicAuth
-##   config:
-##     user: yuanying
-##     password: password-dayo
-##
 require 'yapra/plugin/mechanize_base'
 
 module Yapra::Plugin::Config
+  # Config::BasicAuth -- Yuanying
+  # 
+  # post to web page with WWW::Mechanize agent.
+  # 
+  #     - module: Config::BasicAuth
+  #       config:
+  #         user: yuanying
+  #         password: password-dayo
+  
   class BasicAuth < Yapra::Plugin::MechanizeBase
     def run(data)
       agent.basic_auth(config['user'], config['password'])

data/lib-plugins/yapra/plugin/config/web_post.rb

@@ -1,17 +1,17 @@
-## Config::WebPost -- Yuanying
-##
-## post to web page with WWW::Mechanize agent.
-## 
-## - module: Config::WebPost
-##   config:
-##     url: http://www.pixiv.net/index.php
-##     params:
-##       pixiv_id: yuanying
-##       pass: password-dayo
-##
 require 'yapra/plugin/mechanize_base'
 
 module Yapra::Plugin::Config
+  # Config::WebPost -- Yuanying
+  # 
+  #     post to web page with WWW::Mechanize agent.
+  # 
+  #     - module: Config::WebPost
+  #       config:
+  #         url: http://www.pixiv.net/index.php
+  #         params:
+  #           pixiv_id: yuanying
+  #           pass: password-dayo
+  #
   class WebPost < Yapra::Plugin::MechanizeBase
     def run(data)
       agent.post(config['url'], config['params'])

data/lib-plugins/yapra/plugin/feed/custom.rb

@@ -1,18 +1,23 @@
-## - module: Feed::Custom
-##   config:
-##     url: 'http://www.fraction.jp/'
-##     extract_xpath:
-##       capture: '//div'
-##       split: '//div[@class="test"]'
-##       description: '//div'
-##       link: '//li[2]'
-##       title: '//p'
-##     apply_template_after_extracted:
-##       content_encoded: '<div><%= title %></div>'
-##
 require 'yapra/plugin/mechanize_base'
 
 module Yapra::Plugin::Feed
+  # = Feed::Custom
+  # 
+  # generate rss feed from web page.
+  # 
+  # example:
+  # 
+  #     - module: Feed::Custom
+  #       config:
+  #         url: 'http://www.fraction.jp/'
+  #         extract_xpath:
+  #           capture: '//div'
+  #           split: '//div[@class="test"]'
+  #           description: '//div'
+  #           link: '//li[2]'
+  #           title: '//p'
+  #         apply_template_after_extracted:
+  #           content_encoded: '<div><%= title %></div>'
   class Custom < Yapra::Plugin::MechanizeBase
     def run(data)
       page    = agent.get(config['url'])

data/lib-plugins/yapra/plugin/feed/load.rb

@@ -1,6 +1,15 @@
 require 'yapra/plugin/mechanize_base'
 
 module Yapra::Plugin::Feed
+  # = Load RSS from given URLs
+  # 
+  # Load RSS from given URLs.
+  # If URL is an Array, all URLs in the array will be loaded.
+  # 
+  #     - module: RSS::load
+  #       config:
+  #         uri: http://www.example.com/hoge.rdf
+  #
   class Load < Yapra::Plugin::MechanizeBase
     def run(data)
       urls = 
@@ -11,6 +20,7 @@ module Yapra::Plugin::Feed
         end
       
       urls.each.do |url|
+        logger.debug("Process: #{url}")
         source = agent.get(url).body
         rss = nil
         begin