yapra 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/History.txt +12 -0
- data/Manifest.txt +11 -2
- data/PostInstall.txt +0 -7
- data/README.txt +2 -1
- data/bin/yapra +35 -21
- data/fixtures/config/habu_like.yml +43 -0
- data/fixtures/config/mixed_type.yml +18 -0
- data/fixtures/config/pragger_like.yml +12 -0
- data/fixtures/legacy_plugin/legacy_test_plugin.rb +4 -0
- data/lib/yapra/config.rb +5 -3
- data/lib/yapra/legacy_plugin/advance_mode_registry.rb +14 -3
- data/lib/yapra/legacy_plugin/base.rb +1 -0
- data/lib/yapra/legacy_plugin/compatible_mode_registry.rb +12 -2
- data/lib/yapra/legacy_plugin/registry_factory.rb +7 -0
- data/lib/yapra/pipeline.rb +21 -0
- data/lib/yapra/plugin/base.rb +4 -0
- data/lib/yapra/plugin/erb_applier.rb +1 -1
- data/lib/yapra/plugin/mechanize_base.rb +4 -4
- data/lib/yapra/runtime.rb +16 -0
- data/lib/yapra/version.rb +1 -1
- data/lib/yapra.rb +2 -0
- data/lib-plugins/yapra/plugin/config/agent.rb +13 -13
- data/lib-plugins/yapra/plugin/config/basic_auth.rb +9 -9
- data/lib-plugins/yapra/plugin/config/web_post.rb +11 -11
- data/lib-plugins/yapra/plugin/feed/custom.rb +17 -12
- data/lib-plugins/yapra/plugin/feed/load.rb +10 -0
- data/lib-plugins/yapra/plugin/filter/apply_template.rb +52 -0
- data/lib-plugins/yapra/plugin/filter/entry_full_text.rb +16 -16
- data/lib-plugins/yapra/plugin/publish/file_download.rb +28 -77
- data/lib-plugins/yapra/plugin/publish/gmail.rb +16 -16
- data/lib-plugins/yapra/plugin/publish/imap.rb +88 -34
- data/lib-plugins/yapra/plugin/publish/on_memory_download.rb +86 -0
- data/lib-plugins/yapra/plugin/test/append_entry.rb +11 -11
- data/plugins/Filter/grep.rb +24 -0
- data/spec/spec_helper.rb +2 -0
- data/spec/yapra/config_spec.rb +72 -0
- data/spec/yapra/legacy_plugin/base_spec.rb +33 -0
- data/spec/yapra/legacy_plugin/registry_factory_spec.rb +19 -0
- data/spec/yapra_spec.rb +7 -3
- data/website/index.txt +3 -3
- metadata +15 -13
- data/website/index.html +0 -115
data/History.txt
CHANGED
@@ -1,3 +1,15 @@
|
|
1
|
+
== 0.1.1
|
2
|
+
|
3
|
+
* 1 bug fix:
|
4
|
+
* Publish::Imap, mailbox select fixed.
|
5
|
+
* Spec files are added.
|
6
|
+
|
7
|
+
* 2 majar enhancement:
|
8
|
+
* A few plugins are added.
|
9
|
+
* RDoc is improved.
|
10
|
+
* -d option is added.
|
11
|
+
* Source code management is migrated to github.com.
|
12
|
+
|
1
13
|
== 0.1.0 2008-06-20
|
2
14
|
|
3
15
|
* 1 major enhancement:
|
data/Manifest.txt
CHANGED
@@ -5,8 +5,13 @@ Manifest.txt
|
|
5
5
|
PostInstall.txt
|
6
6
|
README.txt
|
7
7
|
Rakefile
|
8
|
+
bin/yapra
|
8
9
|
config/hoe.rb
|
9
10
|
config/requirements.rb
|
11
|
+
fixtures/config/habu_like.yml
|
12
|
+
fixtures/config/mixed_type.yml
|
13
|
+
fixtures/config/pragger_like.yml
|
14
|
+
fixtures/legacy_plugin/legacy_test_plugin.rb
|
10
15
|
legacy_plugins/Download/nicovideo.rb
|
11
16
|
legacy_plugins/Feed/custom_feed.rb
|
12
17
|
legacy_plugins/Feed/google_calendar.rb
|
@@ -58,10 +63,12 @@ lib-plugins/yapra/plugin/config/basic_auth.rb
|
|
58
63
|
lib-plugins/yapra/plugin/config/web_post.rb
|
59
64
|
lib-plugins/yapra/plugin/feed/custom.rb
|
60
65
|
lib-plugins/yapra/plugin/feed/load.rb
|
66
|
+
lib-plugins/yapra/plugin/filter/apply_template.rb
|
61
67
|
lib-plugins/yapra/plugin/filter/entry_full_text.rb
|
62
68
|
lib-plugins/yapra/plugin/publish/file_download.rb
|
63
69
|
lib-plugins/yapra/plugin/publish/gmail.rb
|
64
70
|
lib-plugins/yapra/plugin/publish/imap.rb
|
71
|
+
lib-plugins/yapra/plugin/publish/on_memory_download.rb
|
65
72
|
lib-plugins/yapra/plugin/test/append_entry.rb
|
66
73
|
lib/yapra.rb
|
67
74
|
lib/yapra/config.rb
|
@@ -81,9 +88,9 @@ lib/yapra/plugin/mechanize_base.rb
|
|
81
88
|
lib/yapra/runtime.rb
|
82
89
|
lib/yapra/version.rb
|
83
90
|
plugins/Filter/deduped.rb
|
91
|
+
plugins/Filter/grep.rb
|
84
92
|
plugins/Filter/sort.rb
|
85
93
|
plugins/Filter/subs.rb
|
86
|
-
bin/yapra
|
87
94
|
script/console
|
88
95
|
script/destroy
|
89
96
|
script/generate
|
@@ -91,12 +98,14 @@ script/txt2html
|
|
91
98
|
setup.rb
|
92
99
|
spec/spec.opts
|
93
100
|
spec/spec_helper.rb
|
101
|
+
spec/yapra/config_spec.rb
|
102
|
+
spec/yapra/legacy_plugin/base_spec.rb
|
103
|
+
spec/yapra/legacy_plugin/registry_factory_spec.rb
|
94
104
|
spec/yapra_spec.rb
|
95
105
|
tasks/deployment.rake
|
96
106
|
tasks/environment.rake
|
97
107
|
tasks/rspec.rake
|
98
108
|
tasks/website.rake
|
99
|
-
website/index.html
|
100
109
|
website/index.txt
|
101
110
|
website/javascripts/rounded_corners_lite.inc.js
|
102
111
|
website/stylesheets/screen.css
|
data/PostInstall.txt
CHANGED
data/README.txt
CHANGED
data/bin/yapra
CHANGED
@@ -14,21 +14,30 @@ $:.insert(0, *[
|
|
14
14
|
])
|
15
15
|
|
16
16
|
legacy_plugin_directory_paths = [
|
17
|
-
|
18
|
-
|
17
|
+
File.join(YAPRA_ROOT, 'legacy_plugins'),
|
18
|
+
File.join(YAPRA_ROOT, 'plugins')
|
19
19
|
]
|
20
20
|
|
21
|
+
require 'yapra/version'
|
21
22
|
require 'yapra/runtime'
|
22
23
|
require 'yapra/config'
|
23
24
|
require 'yapra/legacy_plugin/registry_factory'
|
24
25
|
|
25
|
-
Version
|
26
|
-
mode
|
27
|
-
|
28
|
-
loglebel
|
26
|
+
Version = Yapra::VERSION::STRING
|
27
|
+
mode = 'compatible'
|
28
|
+
config_files = []#"config.yaml"
|
29
|
+
loglebel = nil
|
29
30
|
opt = OptionParser.new
|
30
|
-
opt.on("-c", "--configfile CONFIGFILE") {|v|
|
31
|
-
opt.on("-
|
31
|
+
opt.on("-c", "--configfile CONFIGFILE") {|v| config_files << v }
|
32
|
+
opt.on("-d", "--configfile-directory CONFIGFILE_DIRECTORY") { |v|
|
33
|
+
Dir.glob(File.join(v, '**/*.yml')).each do |file|
|
34
|
+
config_files << file
|
35
|
+
end
|
36
|
+
Dir.glob(File.join(v, '**/*.yaml')).each do |file|
|
37
|
+
config_files << file
|
38
|
+
end
|
39
|
+
}
|
40
|
+
opt.on("-p", "--plugindir PLUGINDIR") {|v| legacy_plugin_directory_paths << v }
|
32
41
|
opt.on("-m", "--mode MODE", 'compatible / advance') { |v| mode = v }
|
33
42
|
opt.on("--log-level LOGLEVEL", 'fatal / error / warn / info / debug') { |v| loglebel = v }
|
34
43
|
# opt.on("-u", "--pluginusage PLUGINNAME") {|v| $plugins[v].source.gsub(/^## ?(.*)/){ puts $1 }; exit }
|
@@ -36,17 +45,22 @@ opt.on("--log-level LOGLEVEL", 'fatal / error / warn / info / debug') { |v| logl
|
|
36
45
|
# opt.on("-w", "--where") { puts(Pathname.new(__FILE__).parent + "plugin"); exit }
|
37
46
|
opt.parse!
|
38
47
|
|
48
|
+
config_files << 'config.yaml' if config_files.size == 0
|
49
|
+
|
39
50
|
legacy_plugin_registry_factory = Yapra::LegacyPlugin::RegistryFactory.new(legacy_plugin_directory_paths, mode)
|
40
|
-
|
41
|
-
|
42
|
-
config.
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
config.
|
50
|
-
|
51
|
-
|
52
|
-
|
51
|
+
|
52
|
+
config_files.each do |config_file|
|
53
|
+
config = YAML.load(File.read(config_file).toutf8.gsub(/base64::([\w+\/]+=*)/){ Base64.decode64($1) })
|
54
|
+
config = Yapra::Config.new(config)
|
55
|
+
config.env.update({
|
56
|
+
'log' => {
|
57
|
+
'level' => loglebel
|
58
|
+
}
|
59
|
+
}) if loglebel
|
60
|
+
Yapra::Runtime.logger = config.create_logger
|
61
|
+
yapra = Yapra::Runtime.new(
|
62
|
+
config.env,
|
63
|
+
legacy_plugin_registry_factory
|
64
|
+
)
|
65
|
+
yapra.execute(config.pipeline_commands)
|
66
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
global:
|
2
|
+
log:
|
3
|
+
out: stderr
|
4
|
+
level: debug
|
5
|
+
|
6
|
+
pipeline:
|
7
|
+
pipeline1:
|
8
|
+
- module: RSS::load
|
9
|
+
config:
|
10
|
+
url:
|
11
|
+
- http://www.fraction.jp/log/feed/rdf
|
12
|
+
- http://d.hatena.ne.jp/yuanying/rss
|
13
|
+
- http://yuanying.oeilvert.org/books.rss
|
14
|
+
|
15
|
+
- module: Filter::sort
|
16
|
+
config:
|
17
|
+
method: date
|
18
|
+
|
19
|
+
- module: reverse
|
20
|
+
|
21
|
+
- module: head
|
22
|
+
config:
|
23
|
+
n: 10
|
24
|
+
|
25
|
+
- module: RSS::save
|
26
|
+
config:
|
27
|
+
filename: feed.xml
|
28
|
+
link: http://www.fraction.jp/
|
29
|
+
title: BONNOH FRACTION
|
30
|
+
|
31
|
+
pipeline2:
|
32
|
+
- module: Feed::pixiv_tag
|
33
|
+
config:
|
34
|
+
id: yuanying
|
35
|
+
password: kojiro
|
36
|
+
tag: C74
|
37
|
+
|
38
|
+
- module: RSS::save
|
39
|
+
config:
|
40
|
+
#filename: /Users/Shared/Sites/fraction/feed.xml
|
41
|
+
filename: pixiv.xml
|
42
|
+
link: http://www.pixiv.net/tags.php?tag=C74
|
43
|
+
title: Pixiv - C74
|
@@ -0,0 +1,18 @@
|
|
1
|
+
global:
|
2
|
+
log:
|
3
|
+
out: stderr
|
4
|
+
level: debug
|
5
|
+
|
6
|
+
pipeline:
|
7
|
+
- module: Feed::pixiv_tag
|
8
|
+
config:
|
9
|
+
id: yuanying
|
10
|
+
password: kojiro
|
11
|
+
tag: C74
|
12
|
+
|
13
|
+
- module: RSS::save
|
14
|
+
config:
|
15
|
+
#filename: /Users/Shared/Sites/fraction/feed.xml
|
16
|
+
filename: pixiv.xml
|
17
|
+
link: http://www.pixiv.net/tags.php?tag=C74
|
18
|
+
title: Pixiv - C74
|
@@ -0,0 +1,12 @@
|
|
1
|
+
- module: Feed::pixiv_tag
|
2
|
+
config:
|
3
|
+
id: yuanying
|
4
|
+
password: kojiro
|
5
|
+
tag: C74
|
6
|
+
|
7
|
+
- module: RSS::save
|
8
|
+
config:
|
9
|
+
#filename: /Users/Shared/Sites/fraction/feed.xml
|
10
|
+
filename: pixiv.xml
|
11
|
+
link: http://www.pixiv.net/tags.php?tag=C74
|
12
|
+
title: Pixiv - C74
|
data/lib/yapra/config.rb
CHANGED
@@ -1,4 +1,9 @@
|
|
1
|
+
require 'logger'
|
2
|
+
require 'yapra'
|
3
|
+
|
1
4
|
# = Config Examples
|
5
|
+
#
|
6
|
+
# Config file for yapra.
|
2
7
|
#
|
3
8
|
# == Format 1: Pragger like.
|
4
9
|
# A simplest. You can run one pipeline without global config.
|
@@ -54,9 +59,6 @@
|
|
54
59
|
# config:
|
55
60
|
# a: b
|
56
61
|
#
|
57
|
-
require 'logger'
|
58
|
-
require 'yapra'
|
59
|
-
|
60
62
|
class Yapra::Config
|
61
63
|
attr_reader :env
|
62
64
|
attr_reader :pipeline_commands
|
@@ -2,12 +2,16 @@ require 'yapra/legacy_plugin'
|
|
2
2
|
require 'yapra/inflector'
|
3
3
|
|
4
4
|
module Yapra::LegacyPlugin
|
5
|
-
|
5
|
+
#
|
6
|
+
# AdvanceModeRegistry load legacy plugin at one by one.
|
7
|
+
#
|
6
8
|
class AdvanceModeRegistry
|
7
9
|
attr_accessor :legacy_plugins
|
8
10
|
attr_accessor :plugin_paths
|
9
11
|
attr_accessor :pipeline
|
10
12
|
|
13
|
+
# _paths_ :: Directory paths which contain legacy plugins.
|
14
|
+
# _pipeline_ :: Runtime pipline.
|
11
15
|
def initialize paths, pipeline
|
12
16
|
self.legacy_plugins = {}
|
13
17
|
self.plugin_paths = paths.reverse
|
@@ -18,12 +22,19 @@ module Yapra::LegacyPlugin
|
|
18
22
|
Yapra::Runtime.logger
|
19
23
|
end
|
20
24
|
|
25
|
+
# load plugin from module name.
|
26
|
+
#
|
27
|
+
# example:
|
28
|
+
#
|
29
|
+
# registry = Yapra::LegacyPlugin::AdvanceModeRegistry.new(paths, pipeline)
|
30
|
+
# feed_load_plugin = registry.get('Feed::load')
|
31
|
+
#
|
21
32
|
def get module_name
|
22
33
|
plugin = legacy_plugins[module_name]
|
23
34
|
unless plugin
|
24
35
|
plugin_paths.each do |folder|
|
25
|
-
file = folder
|
26
|
-
if
|
36
|
+
file = File.join(folder, (module_name.gsub('::', '/') + '.rb'))
|
37
|
+
if File.file?(file)
|
27
38
|
plugin = Yapra::LegacyPlugin::Base.new(pipeline, file)
|
28
39
|
legacy_plugins[ module_name ] = plugin
|
29
40
|
logger.debug "#{module_name} is loaded from #{file}"
|
@@ -4,12 +4,15 @@ require 'yapra/legacy_plugin'
|
|
4
4
|
module Yapra::LegacyPlugin
|
5
5
|
class CompatibleModeRegistry
|
6
6
|
attr_accessor :legacy_plugins
|
7
|
-
|
7
|
+
|
8
|
+
# _paths_ :: Directory paths which contain legacy plugins.
|
9
|
+
# _pipeline_ :: Runtime pipline.
|
8
10
|
def initialize paths, pipeline
|
9
11
|
self.legacy_plugins = {}
|
10
12
|
|
11
13
|
paths.each do |folder|
|
12
|
-
Pathname.
|
14
|
+
folder = Pathname.new(folder)
|
15
|
+
Pathname.glob(File.join(folder, "**/*.rb")).sort.each do |file|
|
13
16
|
module_name = file.relative_path_from(folder).to_s.gsub("/","::")[0..-4]
|
14
17
|
begin
|
15
18
|
legacy_plugins[ module_name ] = Yapra::LegacyPlugin::Base.new(pipeline, file)
|
@@ -25,6 +28,13 @@ module Yapra::LegacyPlugin
|
|
25
28
|
Yapra::Runtime.logger
|
26
29
|
end
|
27
30
|
|
31
|
+
# load plugin from module name.
|
32
|
+
#
|
33
|
+
# example:
|
34
|
+
#
|
35
|
+
# registry = Yapra::LegacyPlugin::CompatibleModeRegistry.new(paths, pipeline)
|
36
|
+
# feed_load_plugin = registry.get('Feed::load')
|
37
|
+
#
|
28
38
|
def get module_name
|
29
39
|
plugin = legacy_plugins[module_name]
|
30
40
|
raise "#{module_name} is not registered." unless plugin
|
@@ -1,10 +1,17 @@
|
|
1
1
|
require 'yapra/legacy_plugin'
|
2
2
|
require 'yapra/inflector'
|
3
3
|
|
4
|
+
# RegistryFactory is factory class for LegacyPluginRegistry.
|
5
|
+
#
|
6
|
+
# LegacyPluginRegistry is created by mode.
|
7
|
+
#
|
8
|
+
# If mode is compatible, this factory create a CompatibeModeRegistry.
|
4
9
|
class Yapra::LegacyPlugin::RegistryFactory
|
5
10
|
attr_reader :plugin_paths
|
6
11
|
attr_reader :registry_class
|
7
12
|
|
13
|
+
# plugin_paths :: directory paths which are contains legacy plugins.
|
14
|
+
# mode :: 'compatible' / 'advance'
|
8
15
|
def initialize plugin_paths, mode = 'compatible'
|
9
16
|
registry_name = "Yapra::LegacyPlugin::#{Yapra::Inflector.camelize(mode)}ModeRegistry"
|
10
17
|
@registry_class = Yapra.load_class_constant(registry_name)
|
data/lib/yapra/pipeline.rb
CHANGED
@@ -19,6 +19,27 @@ class Yapra::Pipeline
|
|
19
19
|
Yapra::Runtime.logger
|
20
20
|
end
|
21
21
|
|
22
|
+
# start pipeline from commands.
|
23
|
+
#
|
24
|
+
# example:
|
25
|
+
#
|
26
|
+
# pipeline.run([
|
27
|
+
# {
|
28
|
+
# 'module' => 'Config::agent',
|
29
|
+
# 'config' => {
|
30
|
+
# 'user_agent_alias' => 'Windows IE 6'
|
31
|
+
# }
|
32
|
+
# },
|
33
|
+
# {
|
34
|
+
# 'module' => 'RSS::load',
|
35
|
+
# 'config' => {
|
36
|
+
# 'uri' => 'http://www.example.com/hoge.rdf'
|
37
|
+
# }
|
38
|
+
# },
|
39
|
+
# {
|
40
|
+
# 'module' => 'print'
|
41
|
+
# }
|
42
|
+
# ])
|
22
43
|
def run pipeline_command, data=[]
|
23
44
|
pipeline_command.inject(data) do |data, command|
|
24
45
|
execute_plugin(command, data.clone)
|
data/lib/yapra/plugin/base.rb
CHANGED
@@ -3,6 +3,10 @@ require 'yapra/plugin/context_aware'
|
|
3
3
|
require 'yapra/plugin/erb_applier'
|
4
4
|
require 'yapra/plugin/feed_item_operator'
|
5
5
|
|
6
|
+
# Yapra plugin base class.
|
7
|
+
#
|
8
|
+
# Subclass this base is not required to be Yapra plugin.
|
9
|
+
# But a few utility module is added in this base.
|
6
10
|
class Yapra::Plugin::Base
|
7
11
|
include Yapra::Plugin::ContextAware
|
8
12
|
include Yapra::Plugin::FeedItemOperator
|
@@ -9,15 +9,15 @@ class Yapra::Plugin::MechanizeBase < Yapra::Plugin::Base
|
|
9
9
|
end
|
10
10
|
|
11
11
|
def extract_attribute_from element, item
|
12
|
-
if
|
13
|
-
|
12
|
+
if plugin_config['extract_xpath']
|
13
|
+
plugin_config['extract_xpath'].each do |k, v|
|
14
14
|
value = element.search(v).to_html.toutf8
|
15
15
|
set_attribute_to item, k, value
|
16
16
|
end
|
17
17
|
end
|
18
18
|
|
19
|
-
if
|
20
|
-
|
19
|
+
if plugin_config['apply_template_after_extracted']
|
20
|
+
plugin_config['apply_template_after_extracted'].each do |k, template|
|
21
21
|
value = apply_template template, binding
|
22
22
|
set_attribute_to item, k, value
|
23
23
|
end
|
data/lib/yapra/runtime.rb
CHANGED
@@ -3,6 +3,20 @@ require 'yapra/pipeline'
|
|
3
3
|
require 'yapra/config'
|
4
4
|
require 'yapra/inflector'
|
5
5
|
|
6
|
+
# = How to use
|
7
|
+
#
|
8
|
+
# require 'yapra/runtime'
|
9
|
+
# require 'yapra/config'
|
10
|
+
#
|
11
|
+
# config = YAML.load(config_file)
|
12
|
+
# config = Yapra::Config.new(config)
|
13
|
+
#
|
14
|
+
# Yapra::Runtime.logger = Logger.new(STDOUT)
|
15
|
+
#
|
16
|
+
# yapra = Yapra::Runtime.new(config.env)
|
17
|
+
# yapra.execute(config.pipeline_commands)
|
18
|
+
#
|
19
|
+
# config_file format written in Yapra::Config document.
|
6
20
|
class Yapra::Runtime
|
7
21
|
attr_reader :env
|
8
22
|
attr_reader :legacy_plugin_registry_factory
|
@@ -13,12 +27,14 @@ class Yapra::Runtime
|
|
13
27
|
@legacy_plugin_registry_factory = legacy_plugin_registry_factory
|
14
28
|
end
|
15
29
|
|
30
|
+
# execute pipelines from commands.
|
16
31
|
def execute pipeline_commands
|
17
32
|
pipeline_commands.each do |k, v|
|
18
33
|
execute_pipeline k, v, []
|
19
34
|
end
|
20
35
|
end
|
21
36
|
|
37
|
+
# execute one pipeline.
|
22
38
|
def execute_pipeline pipeline_name, command_array, data=[]
|
23
39
|
self.class.logger.info("# pipeline '#{pipeline_name}' is started...")
|
24
40
|
pipeline = Yapra::Pipeline.new(self, pipeline_name)
|
data/lib/yapra/version.rb
CHANGED
data/lib/yapra.rb
CHANGED
@@ -1,19 +1,19 @@
|
|
1
|
-
## Yapra::Config::Agent -- Yuanying
|
2
|
-
##
|
3
|
-
## add WWW::Mechanize agent to context.
|
4
|
-
##
|
5
|
-
## - module: Config::Agent
|
6
|
-
## config:
|
7
|
-
## user_agent_alias: Windows IE 6
|
8
|
-
## proxy:
|
9
|
-
## addr: localhost
|
10
|
-
## port: 8080
|
11
|
-
## user: username
|
12
|
-
## password: password
|
13
|
-
##
|
14
1
|
require 'yapra/plugin/mechanize_base'
|
15
2
|
|
16
3
|
module Yapra::Plugin::Config
|
4
|
+
# Yapra::Config::Agent -- Yuanying
|
5
|
+
#
|
6
|
+
# add WWW::Mechanize agent to context.
|
7
|
+
#
|
8
|
+
# - module: Config::Agent
|
9
|
+
# config:
|
10
|
+
# user_agent_alias: Windows IE 6
|
11
|
+
# proxy:
|
12
|
+
# addr: localhost
|
13
|
+
# port: 8080
|
14
|
+
# user: username
|
15
|
+
# password: password
|
16
|
+
#
|
17
17
|
class Agent < Yapra::Plugin::MechanizeBase
|
18
18
|
|
19
19
|
def run(data)
|
@@ -1,15 +1,15 @@
|
|
1
|
-
## Config::BasicAuth -- Yuanying
|
2
|
-
##
|
3
|
-
## post to web page with WWW::Mechanize agent.
|
4
|
-
##
|
5
|
-
## - module: Config::BasicAuth
|
6
|
-
## config:
|
7
|
-
## user: yuanying
|
8
|
-
## password: password-dayo
|
9
|
-
##
|
10
1
|
require 'yapra/plugin/mechanize_base'
|
11
2
|
|
12
3
|
module Yapra::Plugin::Config
|
4
|
+
# Config::BasicAuth -- Yuanying
|
5
|
+
#
|
6
|
+
# post to web page with WWW::Mechanize agent.
|
7
|
+
#
|
8
|
+
# - module: Config::BasicAuth
|
9
|
+
# config:
|
10
|
+
# user: yuanying
|
11
|
+
# password: password-dayo
|
12
|
+
|
13
13
|
class BasicAuth < Yapra::Plugin::MechanizeBase
|
14
14
|
def run(data)
|
15
15
|
agent.basic_auth(config['user'], config['password'])
|
@@ -1,17 +1,17 @@
|
|
1
|
-
## Config::WebPost -- Yuanying
|
2
|
-
##
|
3
|
-
## post to web page with WWW::Mechanize agent.
|
4
|
-
##
|
5
|
-
## - module: Config::WebPost
|
6
|
-
## config:
|
7
|
-
## url: http://www.pixiv.net/index.php
|
8
|
-
## params:
|
9
|
-
## pixiv_id: yuanying
|
10
|
-
## pass: password-dayo
|
11
|
-
##
|
12
1
|
require 'yapra/plugin/mechanize_base'
|
13
2
|
|
14
3
|
module Yapra::Plugin::Config
|
4
|
+
# Config::WebPost -- Yuanying
|
5
|
+
#
|
6
|
+
# post to web page with WWW::Mechanize agent.
|
7
|
+
#
|
8
|
+
# - module: Config::WebPost
|
9
|
+
# config:
|
10
|
+
# url: http://www.pixiv.net/index.php
|
11
|
+
# params:
|
12
|
+
# pixiv_id: yuanying
|
13
|
+
# pass: password-dayo
|
14
|
+
#
|
15
15
|
class WebPost < Yapra::Plugin::MechanizeBase
|
16
16
|
def run(data)
|
17
17
|
agent.post(config['url'], config['params'])
|
@@ -1,18 +1,23 @@
|
|
1
|
-
## - module: Feed::Custom
|
2
|
-
## config:
|
3
|
-
## url: 'http://www.fraction.jp/'
|
4
|
-
## extract_xpath:
|
5
|
-
## capture: '//div'
|
6
|
-
## split: '//div[@class="test"]'
|
7
|
-
## description: '//div'
|
8
|
-
## link: '//li[2]'
|
9
|
-
## title: '//p'
|
10
|
-
## apply_template_after_extracted:
|
11
|
-
## content_encoded: '<div><%= title %></div>'
|
12
|
-
##
|
13
1
|
require 'yapra/plugin/mechanize_base'
|
14
2
|
|
15
3
|
module Yapra::Plugin::Feed
|
4
|
+
# = Feed::Custom
|
5
|
+
#
|
6
|
+
# generate rss feed from web page.
|
7
|
+
#
|
8
|
+
# example:
|
9
|
+
#
|
10
|
+
# - module: Feed::Custom
|
11
|
+
# config:
|
12
|
+
# url: 'http://www.fraction.jp/'
|
13
|
+
# extract_xpath:
|
14
|
+
# capture: '//div'
|
15
|
+
# split: '//div[@class="test"]'
|
16
|
+
# description: '//div'
|
17
|
+
# link: '//li[2]'
|
18
|
+
# title: '//p'
|
19
|
+
# apply_template_after_extracted:
|
20
|
+
# content_encoded: '<div><%= title %></div>'
|
16
21
|
class Custom < Yapra::Plugin::MechanizeBase
|
17
22
|
def run(data)
|
18
23
|
page = agent.get(config['url'])
|
@@ -1,6 +1,15 @@
|
|
1
1
|
require 'yapra/plugin/mechanize_base'
|
2
2
|
|
3
3
|
module Yapra::Plugin::Feed
|
4
|
+
# = Load RSS from given URLs
|
5
|
+
#
|
6
|
+
# Load RSS from given URLs.
|
7
|
+
# If URL is an Array, all URLs in the array will be loaded.
|
8
|
+
#
|
9
|
+
# - module: RSS::load
|
10
|
+
# config:
|
11
|
+
# uri: http://www.example.com/hoge.rdf
|
12
|
+
#
|
4
13
|
class Load < Yapra::Plugin::MechanizeBase
|
5
14
|
def run(data)
|
6
15
|
urls =
|
@@ -11,6 +20,7 @@ module Yapra::Plugin::Feed
|
|
11
20
|
end
|
12
21
|
|
13
22
|
urls.each.do |url|
|
23
|
+
logger.debug("Process: #{url}")
|
14
24
|
source = agent.get(url).body
|
15
25
|
rss = nil
|
16
26
|
begin
|