yapra 0.1.0

data/lib/yapra/config.rb

@@ -0,0 +1,103 @@
+# = Config Examples
+# 
+# == Format 1: Pragger like.
+# A simplest. You can run one pipeline without global config.
+#
+#    - module: Module:name
+#      config:
+#        a: 3
+#    - module: Module:name2
+#      config:
+#        a: b
+#    - module: Module:name3
+#      config:
+#        a: 88
+#
+# == Format 2: Python habu like.
+# You can run a lot of pipelines with global config.
+#
+#   global:
+#     log:
+#       out: stderr
+#       level: warn
+#
+#   pipeline:
+#     pipeline1:
+#       - module: Module:name
+#         config:
+#           a: 3
+#       - module: Module:name2
+#         config:
+#           a: b
+#
+#     pipeline2:
+#       - module: Module:name
+#         config:
+#           a: 3
+#       - module: Module:name2
+#         config:
+#           a: b
+#
+# == Format 3: Mixed type.
+# You can run sigle pipeline with global config.
+#
+#   global:
+#     log:
+#       out: stderr
+#       level: warn
+#
+#   pipeline:
+#     - module: Module:name
+#       config:
+#         a: 3
+#     - module: Module:name2
+#       config:
+#         a: b
+#
+require 'logger'
+require 'yapra'
+
+class Yapra::Config
+  attr_reader :env
+  attr_reader :pipeline_commands
+  
+  def initialize config={}
+    if config.kind_of?(Hash)
+      @env = config['global'] || {}
+      if config['pipeline']
+        if config['pipeline'].kind_of?(Hash)
+          @pipeline_commands = config['pipeline']
+        elsif config['pipeline'].kind_of?(Array)
+          @pipeline_commands = { 'default' => config['pipeline'] }
+        end
+      end
+      raise 'config["global"]["pipeline"] is invalid!' unless @pipeline_commands
+    elsif config.kind_of?(Array)
+      @env        = {}
+      @pipeline_commands  = { 'default' => config }
+    else
+      raise 'config file is invalid!'
+    end
+  end
+  
+  def create_logger
+    logger = nil
+    if env['log'] && env['log']['out']
+      if env['log']['out'].index('file://')
+        logger = Logger.new(URI.parse(env['log']['out']).path)
+      else
+        logger = Logger.new(Yapra::Inflector.constantize(env['log']['out'].upcase))
+      end
+    else
+      logger = Logger.new(STDOUT)
+    end
+    
+    if env['log'] && env['log']['level']
+      logger.level = Yapra::Inflector.constantize("Logger::#{env['log']['level'].upcase}")
+    else
+      logger.level = Logger::WARN
+    end
+    logger
+  end
+  
+end

data/lib/yapra/inflector.rb

@@ -0,0 +1,55 @@
+# original from ActiveSupport-2.0.2
+# MIT Licence
+require 'yapra'
+
+module Yapra::Inflector
+  extend self
+  
+  # By default, camelize converts strings to UpperCamelCase. If the argument to camelize
+  # is set to ":lower" then camelize produces lowerCamelCase.
+  #
+  # camelize will also convert '/' to '::' which is useful for converting paths to namespaces
+  #
+  # Examples
+  #   "active_record".camelize #=> "ActiveRecord"
+  #   "active_record".camelize(:lower) #=> "activeRecord"
+  #   "active_record/errors".camelize #=> "ActiveRecord::Errors"
+  #   "active_record/errors".camelize(:lower) #=> "activeRecord::Errors"
+  def camelize(lower_case_and_underscored_word, first_letter_in_uppercase = true)
+    if first_letter_in_uppercase
+      lower_case_and_underscored_word.to_s.gsub(/\/(.?)/) { "::" + $1.upcase }.gsub(/(^|_)(.)/) { $2.upcase }
+    else
+      lower_case_and_underscored_word.first + camelize(lower_case_and_underscored_word)[1..-1]
+    end
+  end
+  
+  # The reverse of +camelize+. Makes an underscored form from the expression in the string.
+  #
+  # Changes '::' to '/' to convert namespaces to paths.
+  #
+  # Examples
+  #   "ActiveRecord".underscore #=> "active_record"
+  #   "ActiveRecord::Errors".underscore #=> active_record/errors
+  def underscore(camel_cased_word)
+    camel_cased_word.to_s.gsub(/::/, '/').
+      gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
+      gsub(/([a-z\d])([A-Z])/,'\1_\2').
+      tr("-", "_").
+      downcase
+  end
+  
+  # Constantize tries to find a declared constant with the name specified
+  # in the string. It raises a NameError when the name is not in CamelCase
+  # or is not initialized.
+  #
+  # Examples
+  #   "Module".constantize #=> Module
+  #   "Class".constantize #=> Class
+  def constantize(camel_cased_word)
+    unless /\A(?:::)?([A-Z]\w*(?:::[A-Z]\w*)*)\z/ =~ camel_cased_word
+      raise NameError, "#{camel_cased_word.inspect} is not a valid constant name!"
+    end
+
+    Object.module_eval("::#{$1}", __FILE__, __LINE__)
+  end
+end

data/lib/yapra/legacy_plugin/advance_mode_registry.rb

@@ -0,0 +1,38 @@
+require 'yapra/legacy_plugin'
+require 'yapra/inflector'
+
+module Yapra::LegacyPlugin
+  
+  class AdvanceModeRegistry
+    attr_accessor :legacy_plugins
+    attr_accessor :plugin_paths
+    attr_accessor :pipeline
+    
+    def initialize paths, pipeline
+      self.legacy_plugins = {}
+      self.plugin_paths = paths.reverse
+      self.pipeline = pipeline
+    end
+    
+    def logger
+      Yapra::Runtime.logger
+    end
+    
+    def get module_name
+      plugin = legacy_plugins[module_name]
+      unless plugin
+        plugin_paths.each do |folder|
+          file = folder + (module_name.gsub('::', '/') + '.rb')
+          if file.file?()
+            plugin = Yapra::LegacyPlugin::Base.new(pipeline, file)
+            legacy_plugins[ module_name ] = plugin
+            logger.debug "#{module_name} is loaded from #{file}"
+            break
+          end
+        end
+      end
+      raise "#{module_name} is not registered." unless plugin
+      plugin
+    end
+  end
+end

data/lib/yapra/legacy_plugin/base.rb

@@ -0,0 +1,26 @@
+# require 'pathname'
+require 'yapra/legacy_plugin'
+
+class Yapra::LegacyPlugin::Base
+  attr_reader :source
+  attr_reader :_yapra_run_method
+  attr_reader :_yapra_pipeline
+  
+  def initialize(pipeline, plugin_path)
+    @_yapra_pipeline     = pipeline
+    @_yapra_run_method  = File.basename(plugin_path, '.*')
+    instance_eval( @source = File.read(plugin_path).toutf8, plugin_path, 1)
+  end
+  
+  def logger
+    Yapra::Runtime.logger
+  end
+  
+  def eval_pragger(command_array, data)
+    pipeline.execute_plugins(command_array, data)
+  end
+  
+  def _yapra_run_as_legacy_plugin(config, data)
+    self.__send__(self._yapra_run_method, config, data)
+  end
+end

data/lib/yapra/legacy_plugin/compatible_mode_registry.rb

@@ -0,0 +1,34 @@
+require 'pathname'
+require 'yapra/legacy_plugin'
+
+module Yapra::LegacyPlugin
+  class CompatibleModeRegistry
+    attr_accessor :legacy_plugins
+      
+    def initialize paths, pipeline
+      self.legacy_plugins = {}
+      
+      paths.each do |folder|
+        Pathname.glob(folder + "**/*.rb").sort.each do |file|
+          module_name = file.relative_path_from(folder).to_s.gsub("/","::")[0..-4]
+          begin
+            legacy_plugins[ module_name ] = Yapra::LegacyPlugin::Base.new(pipeline, file)
+            logger.debug "#{module_name} is loaded from #{file}"
+          rescue LoadError => ex
+            logger.warn "#{module_name} can't load, because: #{ex.message}"
+          end
+        end
+      end
+    end
+    
+    def logger
+      Yapra::Runtime.logger
+    end
+    
+    def get module_name
+      plugin = legacy_plugins[module_name]
+      raise "#{module_name} is not registered." unless plugin
+      plugin
+    end
+  end
+end

data/lib/yapra/legacy_plugin/registry_factory.rb

@@ -0,0 +1,21 @@
+require 'yapra/legacy_plugin'
+require 'yapra/inflector'
+
+class Yapra::LegacyPlugin::RegistryFactory
+  attr_reader :plugin_paths
+  attr_reader :registry_class
+  
+  def initialize plugin_paths, mode = 'compatible'
+    registry_name = "Yapra::LegacyPlugin::#{Yapra::Inflector.camelize(mode)}ModeRegistry"
+    @registry_class = Yapra.load_class_constant(registry_name)
+    raise "'#{mode}' mode is not supported." unless @registry_class
+    
+    @plugin_paths = plugin_paths
+  end
+  
+  def create pipeline
+    registry = registry_class.new(plugin_paths, pipeline)
+    pipeline.legacy_plugin_registry = registry
+    registry
+  end
+end

data/lib/yapra/legacy_plugin.rb

@@ -0,0 +1,5 @@
+# require 'pathname'
+require 'yapra'
+
+module Yapra::LegacyPlugin
+end

data/lib/yapra/pipeline.rb

@@ -0,0 +1,74 @@
+require 'yapra'
+require 'yapra/inflector'
+require 'yapra/legacy_plugin/base'
+
+class Yapra::Pipeline
+  attr_reader :yapra, :context
+  attr_accessor :legacy_plugin_registry
+  
+  UPPER_CASE = /[A-Z]/
+  
+  def initialize yapra, pipeline_name
+    @yapra              = yapra
+    @context            = { 'pipeline_name' => pipeline_name }
+    
+    @module_name_prefix = construct_module_name_prefix yapra.env
+  end
+  
+  def logger
+    Yapra::Runtime.logger
+  end
+  
+  def run pipeline_command, data=[]
+    pipeline_command.inject(data) do |data, command|
+      execute_plugin(command, data.clone)
+    end
+  end
+  
+  def execute_plugin command, data
+    self.logger.info("exec plugin #{command["module"]}")
+    if class_based_plugin?(command['module'])
+      run_class_based_plugin command, data
+    else
+      run_legacy_plugin command, data
+    end
+  end
+  
+  protected
+  def class_based_plugin? command_name
+    UPPER_CASE =~ command_name.split('::').last[0, 1]
+  end
+  
+  def run_legacy_plugin command, data
+    self.logger.debug("evaluate plugin as legacy")
+    legacy_plugin_registry.get(command['module'])._yapra_run_as_legacy_plugin(command['config'], data)
+  end
+  
+  def run_class_based_plugin command, data
+    self.logger.debug("evaluate plugin as class based")
+    plugin_class = nil
+    @module_name_prefix.each do |prefix|
+      yapra_module_name = "#{prefix}#{command['module']}"
+      plugin_class      = Yapra.load_class_constant(yapra_module_name)
+      break if plugin_class
+    end
+    raise LoadError unless plugin_class
+    plugin                = plugin_class.new
+    plugin.yapra          = yapra if plugin.respond_to?('yapra=')
+    plugin.pipeline       = self  if plugin.respond_to?('pipeline=')
+    plugin.plugin_config  = command['config'] if plugin.respond_to?('plugin_config=')
+    plugin.run(data)
+  end
+  
+  def construct_module_name_prefix env
+    module_name_prefix = [ 'Yapra::Plugin::', '' ]
+    if env['module_name_prefix']
+      if env['module_name_prefix'].kind_of?(Array)
+        module_name_prefix = env['module_name_prefix']
+      else
+        module_name_prefix = [ env['module_name_prefix'] ]
+      end
+    end
+    module_name_prefix
+  end
+end

data/lib/yapra/plugin/base.rb

@@ -0,0 +1,14 @@
+require 'yapra/plugin'
+require 'yapra/plugin/context_aware'
+require 'yapra/plugin/erb_applier'
+require 'yapra/plugin/feed_item_operator'
+
+class Yapra::Plugin::Base
+  include Yapra::Plugin::ContextAware
+  include Yapra::Plugin::FeedItemOperator
+  include Yapra::Plugin::ErbApplier
+  
+  def logger
+    Yapra::Runtime.logger
+  end
+end

data/lib/yapra/plugin/context_aware.rb

@@ -0,0 +1,14 @@
+require 'yapra/plugin'
+
+module Yapra::Plugin::ContextAware
+  attr_accessor :yapra, :pipeline, :plugin_config
+  
+  def config
+    @config ||= nil
+    unless @config
+      @config = {}.update(yapra.env).update(pipeline.context)
+      @config.update(plugin_config) if plugin_config
+    end
+    @config
+  end
+end

data/lib/yapra/plugin/erb_applier.rb

@@ -0,0 +1,15 @@
+require 'yapra/plugin'
+require 'erb'
+require 'uri'
+
+module Yapra::Plugin::ErbApplier
+  def apply_template template, apply_binding=binding
+    if template.index('file://') == 0
+      path = URI.parse(template).path
+      open(path) do |io|
+        template = io.read
+      end
+    end
+    ERB.new(template).result(apply_binding)
+  end
+end

data/lib/yapra/plugin/feed_item_operator.rb

@@ -0,0 +1,21 @@
+require 'yapra/plugin'
+
+module Yapra::Plugin::FeedItemOperator
+  LOCAL_VAL_RE = /[a-z_][0-9A-Za-z_]/
+  
+  def set_attribute_to item, k, value
+    raise NameError unless LOCAL_VAL_RE =~ k
+    unless item.respond_to?("#{k}=")
+      item.instance_eval %Q{
+        def #{k}
+          @#{k}
+        end
+        def #{k}= v
+          @#{k} = v
+        end
+      }, __FILE__, __LINE__
+      
+    end
+    item.__send__("#{k}=", value)
+  end
+end

data/lib/yapra/plugin/mechanize_base.rb

@@ -0,0 +1,26 @@
+require 'mechanize'
+require 'kconv'
+require 'yapra/plugin/base'
+
+class Yapra::Plugin::MechanizeBase < Yapra::Plugin::Base
+  def agent
+    pipeline.context['mechanize_agent'] ||= WWW::Mechanize.new
+    pipeline.context['mechanize_agent']
+  end
+  
+  def extract_attribute_from element, item
+    if config['extract_xpath']
+      config['extract_xpath'].each do |k, v|
+        value = element.search(v).to_html.toutf8
+        set_attribute_to item, k, value
+      end
+    end
+
+    if config['apply_template_after_extracted']
+      config['apply_template_after_extracted'].each do |k, template|
+        value = apply_template template, binding
+        set_attribute_to item, k, value
+      end
+    end
+  end
+end

data/lib/yapra/plugin.rb

@@ -0,0 +1,5 @@
+require 'yapra'
+
+module Yapra::Plugin
+
+end

data/lib/yapra/runtime.rb

@@ -0,0 +1,36 @@
+require 'yapra'
+require 'yapra/pipeline'
+require 'yapra/config'
+require 'yapra/inflector'
+
+class Yapra::Runtime
+  attr_reader :env
+  attr_reader :legacy_plugin_registry_factory
+  
+  def initialize env={}, legacy_plugin_registry_factory=nil
+    @env    = env
+    
+    @legacy_plugin_registry_factory        = legacy_plugin_registry_factory
+  end
+  
+  def execute pipeline_commands
+    pipeline_commands.each do |k, v|
+      execute_pipeline k, v, []
+    end
+  end
+  
+  def execute_pipeline pipeline_name, command_array, data=[]
+    self.class.logger.info("# pipeline '#{pipeline_name}' is started...")
+    pipeline = Yapra::Pipeline.new(self, pipeline_name)
+    legacy_plugin_registory = legacy_plugin_registry_factory.create(pipeline) if legacy_plugin_registry_factory
+    pipeline.run(command_array, data)
+  end
+  
+  def self.logger
+    @@logger
+  end
+  
+  def self.logger=logger
+    @@logger = logger
+  end
+end

data/lib/yapra/version.rb

@@ -0,0 +1,9 @@
+module Yapra
+  module VERSION #:nodoc:
+    MAJOR = 0
+    MINOR = 1
+    TINY  = 0
+
+    STRING = [MAJOR, MINOR, TINY].join('.')
+  end
+end

data/lib/yapra.rb

@@ -0,0 +1,44 @@
+#--
+# Copyright (c) 2008 fraction.jp. Yuanying, Some rights reserved.
+# Original from 2008-06-12 http://pragger.ikejisoft.com/
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+$:.unshift(File.dirname(__FILE__)) unless
+  $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
+
+module Yapra
+  extend self
+  
+  # TODO create util class, and move this method.
+  def load_class_constant module_name
+    require Yapra::Inflector.underscore(module_name)
+    Yapra::Inflector.constantize(module_name)
+  rescue LoadError
+    nil
+  rescue NameError
+    nil
+  end
+end
+require 'rss/1.0'
+require 'rss/2.0'
+require 'rss/dublincore'
+require 'rss/maker'
+require 'rubygems'

data/lib-plugins/yapra/plugin/config/agent.rb

@@ -0,0 +1,34 @@
+## Yapra::Config::Agent -- Yuanying
+##
+## add WWW::Mechanize agent to context.
+## 
+## - module: Config::Agent
+##   config:
+##     user_agent_alias: Windows IE 6
+##     proxy:
+##       addr: localhost
+##       port: 8080
+##       user: username
+##       password: password
+##
+require 'yapra/plugin/mechanize_base'
+
+module Yapra::Plugin::Config
+  class Agent < Yapra::Plugin::MechanizeBase
+
+    def run(data)
+      
+      agent.user_agent_alias = config['user_agent_alias'] || 'Windows IE 6'
+      if config['proxy']
+        agent.set_proxy(
+          config['proxy']['addr'],
+          config['proxy']['port'],
+          config['proxy']['user'],
+          config['proxy']['password']
+        )
+      end
+      
+      return data
+    end
+  end
+end

data/lib-plugins/yapra/plugin/config/basic_auth.rb

@@ -0,0 +1,19 @@
+## Config::BasicAuth -- Yuanying
+##
+## post to web page with WWW::Mechanize agent.
+## 
+## - module: Config::BasicAuth
+##   config:
+##     user: yuanying
+##     password: password-dayo
+##
+require 'yapra/plugin/mechanize_base'
+
+module Yapra::Plugin::Config
+  class BasicAuth < Yapra::Plugin::MechanizeBase
+    def run(data)
+      agent.basic_auth(config['user'], config['password'])
+      data
+    end
+  end
+end

data/lib-plugins/yapra/plugin/config/web_post.rb

@@ -0,0 +1,21 @@
+## Config::WebPost -- Yuanying
+##
+## post to web page with WWW::Mechanize agent.
+## 
+## - module: Config::WebPost
+##   config:
+##     url: http://www.pixiv.net/index.php
+##     params:
+##       pixiv_id: yuanying
+##       pass: password-dayo
+##
+require 'yapra/plugin/mechanize_base'
+
+module Yapra::Plugin::Config
+  class WebPost < Yapra::Plugin::MechanizeBase
+    def run(data)
+      agent.post(config['url'], config['params'])
+      data
+    end
+  end
+end

data/lib-plugins/yapra/plugin/feed/custom.rb

@@ -0,0 +1,41 @@
+## - module: Feed::Custom
+##   config:
+##     url: 'http://www.fraction.jp/'
+##     extract_xpath:
+##       capture: '//div'
+##       split: '//div[@class="test"]'
+##       description: '//div'
+##       link: '//li[2]'
+##       title: '//p'
+##     apply_template_after_extracted:
+##       content_encoded: '<div><%= title %></div>'
+##
+require 'yapra/plugin/mechanize_base'
+
+module Yapra::Plugin::Feed
+  class Custom < Yapra::Plugin::MechanizeBase
+    def run(data)
+      page    = agent.get(config['url'])
+      root    = page.root
+      
+      xconfig = config['extract_xpath']
+      
+      if xconfig['capture']
+        root = root.at(xconfig['capture'])
+      end
+      split = xconfig['split']
+      xconfig.delete('capture')
+      xconfig.delete('split')
+      
+      root.search(split).each do |element|
+        item = RSS::RDF::Item.new
+        
+        extract_attribute_from element, item
+
+        data << item
+      end
+      
+      data
+    end
+  end
+end