automatic 12.6.0 → 12.9.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (56) hide show
  1. data/VERSION +1 -1
  2. data/automatic.gemspec +10 -22
  3. data/bin/automatic-config +4 -4
  4. data/doc/ChangeLog +30 -1
  5. data/doc/PLUGINS +46 -178
  6. data/doc/PLUGINS.ja +46 -177
  7. data/doc/README +2 -10
  8. data/doc/README.ja +2 -9
  9. data/lib/automatic.rb +2 -3
  10. data/plugins/custom_feed/svn_log.rb +0 -1
  11. data/plugins/filter/absolute_uri.rb +39 -0
  12. data/plugins/filter/image.rb +46 -0
  13. data/plugins/filter/reverse.rb +0 -1
  14. data/plugins/store/database.rb +8 -23
  15. data/plugins/store/full_text.rb +4 -4
  16. data/plugins/store/permalink.rb +12 -9
  17. data/plugins/store/target_link.rb +3 -2
  18. data/plugins/subscription/link.rb +55 -0
  19. data/plugins/subscription/twitter.rb +57 -0
  20. data/script/build +1 -1
  21. data/spec/lib/automatic_spec.rb +3 -2
  22. data/spec/plugins/custom_feed/svn_log_spec.rb +1 -1
  23. data/spec/plugins/filter/absolute_uri_spec.rb +61 -0
  24. data/spec/plugins/filter/image_source_spec.rb +1 -1
  25. data/spec/plugins/filter/image_spec.rb +65 -0
  26. data/spec/plugins/publish/google_calendar_spec.rb +1 -1
  27. data/spec/plugins/store/full_text_spec.rb +0 -1
  28. data/spec/plugins/store/permalink_spec.rb +52 -3
  29. data/spec/plugins/store/target_link_spec.rb +1 -2
  30. data/spec/plugins/subscription/feed_spec.rb +0 -1
  31. data/spec/plugins/subscription/{uri_spec.rb → link_spec.rb} +9 -10
  32. data/spec/plugins/subscription/twitter_spec.rb +42 -0
  33. data/spec/spec_helper.rb +12 -26
  34. data/spec/user_dir/plugins/store/mock.rb +1 -1
  35. data/test/integration/test_get_image.yml +4 -6
  36. data/test/integration/test_image2local.yml +1 -1
  37. metadata +10 -22
  38. data/config/html2console.yml +0 -16
  39. data/plugins/extract/link.rb +0 -32
  40. data/plugins/filter/image_link.rb +0 -37
  41. data/plugins/publish/dump.rb +0 -24
  42. data/plugins/store/link.rb +0 -47
  43. data/plugins/store/target.rb +0 -41
  44. data/plugins/subscription/uri.rb +0 -31
  45. data/spec/fixtures/extractLink.html +0 -14
  46. data/spec/fixtures/filterImageLink.html +0 -34
  47. data/spec/fixtures/publishDump.html +0 -14
  48. data/spec/fixtures/storeLink.html +0 -34
  49. data/spec/fixtures/storeLink2.html +0 -36
  50. data/spec/fixtures/storeTarget.html +0 -11
  51. data/spec/fixtures/storeTarget2.html +0 -11
  52. data/spec/plugins/extract/link_spec.rb +0 -38
  53. data/spec/plugins/filter/image_link_spec.rb +0 -51
  54. data/spec/plugins/publish/dump_spec.rb +0 -32
  55. data/spec/plugins/store/link_spec.rb +0 -47
  56. data/spec/plugins/store/target_spec.rb +0 -41
data/config/html2console.yml DELETED
@@ -1,16 +0,0 @@
1
- global:
2
- timezone: Asia/Tokyo
3
- cache:
4
- base: /tmp
5
- log:
6
- level: info
7
-
8
- plugins:
9
- - module: SubscriptionURI
10
- config:
11
- urls:
12
- - http://id774.net/
13
-
14
- - module: ExtractLink
15
-
16
- - module: PublishDump
data/plugins/extract/link.rb DELETED
@@ -1,32 +0,0 @@
1
- # -*- coding: utf-8 -*-
2
- # Name:: Automatic::Plugin::Extract::Link
3
- # Author:: 774 <http://id774.net>
4
- # Created:: May 24, 2012
5
- # Updated:: Jun 13, 2012
6
- # Copyright:: 774 Copyright (c) 2012
7
- # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
8
-
9
- module Automatic::Plugin
10
- class ExtractLink
11
- require 'nokogiri'
12
-
13
- def initialize(config, pipeline=[])
14
- @config = config
15
- @pipeline = pipeline
16
- end
17
-
18
- def run
19
- @return_html = []
20
- @pipeline.each {|html|
21
- img_url = ""
22
- unless html.nil?
23
- doc = Nokogiri::HTML(html)
24
- (doc/:a).each {|link|
25
- @return_html << link[:href]
26
- }
27
- end
28
- }
29
- @return_html
30
- end
31
- end
32
- end
data/plugins/filter/image_link.rb DELETED
@@ -1,37 +0,0 @@
1
- # -*- coding: utf-8 -*-
2
- # Name:: Automatic::Plugin::Filter::ImageLink
3
- # Author:: 774 <http://id774.net>
4
- # Created:: May 24, 2012
5
- # Updated:: Jun 13, 2012
6
- # Copyright:: 774 Copyright (c) 2012
7
- # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
8
-
9
- module Automatic::Plugin
10
- class FilterImageLink
11
- def initialize(config, pipeline=[])
12
- @config = config
13
- @pipeline = pipeline
14
- end
15
-
16
- def run
17
- @return_html = []
18
- @pipeline.each {|html|
19
- unless html.nil?
20
- html.scan(/(.*?\.jp.*g$)/i) { |matched|
21
- @return_html << matched.join(" ")
22
- }
23
- html.scan(/(.*?\.png$)/i) { |matched|
24
- @return_html << matched.join(" ")
25
- }
26
- html.scan(/(.*?\.gif$)/i) { |matched|
27
- @return_html << matched.join(" ")
28
- }
29
- html.scan(/(.*?\.tiff$)/i) { |matched|
30
- @return_html << matched.join(" ")
31
- }
32
- end
33
- }
34
- @return_html
35
- end
36
- end
37
- end
data/plugins/publish/dump.rb DELETED
@@ -1,24 +0,0 @@
1
- # -*- coding: utf-8 -*-
2
- # Name:: Automatic::Plugin::Publish::Dump
3
- # Author:: 774 <http://id774.net>
4
- # Created:: Jun 13, 2012
5
- # Updated:: Jun 13, 2012
6
- # Copyright:: 774 Copyright (c) 2012
7
- # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
8
-
9
- module Automatic::Plugin
10
- class PublishDump
11
- require 'pp'
12
-
13
- def initialize(config, pipeline=[])
14
- @config = config
15
- @pipeline = pipeline
16
- @output = STDOUT
17
- end
18
-
19
- def run
20
- pp(@pipeline, out=@output)
21
- @pipeline
22
- end
23
- end
24
- end
data/plugins/store/link.rb DELETED
@@ -1,47 +0,0 @@
1
- # -*- coding: utf-8 -*-
2
- # Name:: Automatic::Plugin::Store::Link
3
- # Author:: 774 <http://id774.net>
4
- # Created:: May 24, 2012
5
- # Updated:: Jun 14, 2012
6
- # Copyright:: 774 Copyright (c) 2012
7
- # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
8
-
9
- require 'plugins/store/database'
10
-
11
- module Automatic::Plugin
12
- class Link < ActiveRecord::Base
13
- end
14
-
15
- class StoreLink
16
- include Automatic::Plugin::Database
17
-
18
- def initialize(config, pipeline=[])
19
- @config = config
20
- @pipeline = pipeline
21
- end
22
-
23
- def column_definition
24
- return {
25
- :url => :string,
26
- :created_at => :string
27
- }
28
- end
29
-
30
- def unique_key
31
- return :url
32
- end
33
-
34
- def model_class
35
- return Automatic::Plugin::Link
36
- end
37
-
38
- def run
39
- return for_each_new_link { |link|
40
- Link.create(
41
- :url => link,
42
- :created_at => Time.now.strftime("%Y/%m/%d %X"))
43
- Automatic::Log.puts("info", "Saving: #{link}")
44
- }
45
- end
46
- end
47
- end
data/plugins/store/target.rb DELETED
@@ -1,41 +0,0 @@
1
- #!/usr/bin/env ruby
2
- # -*- coding: utf-8 -*-
3
- # Name:: Automatic::Plugin::Store::Target
4
- # Author:: 774 <http://id774.net>
5
- # Created:: May 24, 2012
6
- # Updated:: May 28, 2012
7
- # Copyright:: 774 Copyright (c) 2012
8
- # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
9
-
10
- module Automatic::Plugin
11
- class StoreTarget
12
- def initialize(config, pipeline=[])
13
- @config = config
14
- @pipeline = pipeline
15
- end
16
-
17
- def wget(url)
18
- filename = url.split(/\//).last
19
- open(url) { |source|
20
- open(File.join(@config['path'], filename), "w+b") { |o|
21
- o.print source.read
22
- }
23
- }
24
- end
25
-
26
- def run
27
- @pipeline.each {|html|
28
- begin
29
- unless html.nil?
30
- Automatic::Log.puts("info", "Get: #{html}")
31
- wget(html)
32
- end
33
- rescue
34
- Automatic::Log.puts("error", "Error found during file download.")
35
- end
36
- sleep @config['interval'].to_i
37
- }
38
- @pipeline
39
- end
40
- end
41
- end
data/plugins/subscription/uri.rb DELETED
@@ -1,31 +0,0 @@
1
- # -*- coding: utf-8 -*-
2
- # Name:: Automatic::Plugin::Subscription::URI
3
- # Author:: 774 <http://id774.net>
4
- # Created:: May 24, 2012
5
- # Updated:: Jun 13, 2012
6
- # Copyright:: 774 Copyright (c) 2012
7
- # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
8
-
9
- module Automatic::Plugin
10
- class SubscriptionURI
11
- require 'open-uri'
12
-
13
- def initialize(config, pipeline=[])
14
- @config = config
15
- @pipeline = pipeline
16
- end
17
-
18
- def run
19
- @config['urls'].each {|url|
20
- begin
21
- Automatic::Log.puts("info", "Parsing: #{url}")
22
- doc = open(url).read
23
- @pipeline << doc
24
- rescue
25
- Automatic::Log.puts("error", "Fault in parsing: #{url}")
26
- end
27
- }
28
- return @pipeline
29
- end
30
- end
31
- end
data/spec/fixtures/extractLink.html DELETED
@@ -1,14 +0,0 @@
1
- <!DOCTYPE html>
2
- <html lang="ja">
3
- <head>
4
- <title>Sample</title>
5
- </head>
6
- <body>
7
- <p>A simple <b>test</b> string.</p>
8
- <a href="http://id774.net">id774.net</a>
9
- <a href="http://reblog.id774.net">reblog.id774.net</a>
10
- <a href="http://oh-news.net/live/wp-content/uploads/2011/04/Eila_omote.jpg">
11
- <img src="http://24.media.tumblr.com/tumblr_m5gneyJmsH1qza5ppo1_500.jpg" alt="" /></a>
12
- <a href="http://blog.id774.net/post/">blog.id774.net</a>
13
- </body>
14
- </html>
data/spec/fixtures/filterImageLink.html DELETED
@@ -1,34 +0,0 @@
1
- <!DOCTYPE html>
2
- <html lang="ja">
3
- <head>
4
- <title>Sample</title>
5
- </head>
6
- <body>
7
- <p>A simple <b>test</b> string.</p>
8
- <a href="http://id774.net">id774.net</a>
9
- <a href="http://reblog.id774.net">reblog.id774.net</a>
10
- <a href="http://link_1.jpg">
11
- <img src="http://imgsrc_1.jpg" alt="" /></a>
12
- <a href="http://link_2.jpg">
13
- <img src="http://imgsrc_2.jpg" alt="" /></a>
14
- <a href="http://link_3.JPG">
15
- <img src="http://imgsrc_3.JPG" alt="" /></a>
16
- <a href="http://link_4.png">
17
- <img src="http://imgsrc_4.png" alt="" /></a>
18
- <a href="http://link_5.jpeg">
19
- <img src="http://imgsrc_5.jpeg" alt="" /></a>
20
- <a href="http://link_6.PNG">
21
- <img src="http://imgsrc_6.PNG" alt="" /></a>
22
- <a href="http://link_7.jpega">
23
- <img src="http://imgsrc_7.jpega" alt="" /></a>
24
- <a href="http://link_8.gif">
25
- <img src="http://imgsrc_8.gif" alt="" /></a>
26
- <a href="http://link_9.GIF">
27
- <img src="http://imgsrc_9.GIF" alt="" /></a>
28
- <a href="http://link_10.tiff">
29
- <img src="http://imgsrc_10.tiff" alt="" /></a>
30
- <a href="http://link_11.TIFF">
31
- <img src="http://imgsrc_11.TIFF" alt="" /></a>
32
- <a href="http://blog.id774.net/post/">blog.id774.net</a>
33
- </body>
34
- </html>
data/spec/fixtures/publishDump.html DELETED
@@ -1,14 +0,0 @@
1
- <!DOCTYPE html>
2
- <html lang="ja">
3
- <head>
4
- <title>Sample</title>
5
- </head>
6
- <body>
7
- <p>A simple <b>test</b> string.</p>
8
- <a href="http://id774.net">id774.net</a>
9
- <a href="http://reblog.id774.net">reblog.id774.net</a>
10
- <a href="http://oh-news.net/live/wp-content/uploads/2011/04/Eila_omote.jpg">
11
- <img src="http://24.media.tumblr.com/tumblr_m5gneyJmsH1qza5ppo1_500.jpg" alt="" /></a>
12
- <a href="http://blog.id774.net/post/">blog.id774.net</a>
13
- </body>
14
- </html>
data/spec/fixtures/storeLink.html DELETED
@@ -1,34 +0,0 @@
1
- <!DOCTYPE html>
2
- <html lang="ja">
3
- <head>
4
- <title>Sample</title>
5
- </head>
6
- <body>
7
- <p>A simple <b>test</b> string.</p>
8
- <a href="http://id774.net">id774.net</a>
9
- <a href="http://reblog.id774.net">reblog.id774.net</a>
10
- <a href="http://link_1.jpg">
11
- <img src="http://imgsrc_1.jpg" alt="" /></a>
12
- <a href="http://link_2.jpg">
13
- <img src="http://imgsrc_2.jpg" alt="" /></a>
14
- <a href="http://link_3.JPG">
15
- <img src="http://imgsrc_3.JPG" alt="" /></a>
16
- <a href="http://link_4.png">
17
- <img src="http://imgsrc_4.png" alt="" /></a>
18
- <a href="http://link_5.jpeg">
19
- <img src="http://imgsrc_5.jpeg" alt="" /></a>
20
- <a href="http://link_6.PNG">
21
- <img src="http://imgsrc_6.PNG" alt="" /></a>
22
- <a href="http://link_7.jpega">
23
- <img src="http://imgsrc_7.jpega" alt="" /></a>
24
- <a href="http://link_8.gif">
25
- <img src="http://imgsrc_8.gif" alt="" /></a>
26
- <a href="http://link_9.GIF">
27
- <img src="http://imgsrc_9.GIF" alt="" /></a>
28
- <a href="http://link_10.tiff">
29
- <img src="http://imgsrc_10.tiff" alt="" /></a>
30
- <a href="http://link_11.TIFF">
31
- <img src="http://imgsrc_11.TIFF" alt="" /></a>
32
- <a href="http://blog.id774.net/post/">blog.id774.net</a>
33
- </body>
34
- </html>
data/spec/fixtures/storeLink2.html DELETED
@@ -1,36 +0,0 @@
1
- <!DOCTYPE html>
2
- <html lang="ja">
3
- <head>
4
- <title>Sample</title>
5
- </head>
6
- <body>
7
- <p>A simple <b>test</b> string.</p>
8
- <a href="http://id774.net">id774.net</a>
9
- <a href="http://reblog.id774.net">reblog.id774.net</a>
10
- <a href="http://link_1.jpg">
11
- <img src="http://imgsrc_1.jpg" alt="" /></a>
12
- <a href="http://link_2.jpg">
13
- <img src="http://imgsrc_2.jpg" alt="" /></a>
14
- <a href="http://link_3.JPG">
15
- <img src="http://imgsrc_3.JPG" alt="" /></a>
16
- <a href="http://link_4.png">
17
- <img src="http://imgsrc_4.png" alt="" /></a>
18
- <a href="http://link_5.jpeg">
19
- <img src="http://imgsrc_5.jpeg" alt="" /></a>
20
- <a href="http://link_6.PNG">
21
- <img src="http://imgsrc_6.PNG" alt="" /></a>
22
- <a href="http://link_7.jpega">
23
- <img src="http://imgsrc_7.jpega" alt="" /></a>
24
- <a href="http://link_8.gif">
25
- <img src="http://imgsrc_8.gif" alt="" /></a>
26
- <a href="http://link_9.GIF">
27
- <img src="http://imgsrc_9.GIF" alt="" /></a>
28
- <a href="http://link_10.tiff">
29
- <img src="http://imgsrc_10.tiff" alt="" /></a>
30
- <a href="http://link_11.TIFF">
31
- <img src="http://imgsrc_11.TIFF" alt="" /></a>
32
- <a href="http://link_12.jpg">
33
- <img src="http://imgsrc_12.jpg" alt="" /></a>
34
- <a href="http://blog.id774.net/post/">blog.id774.net</a>
35
- </body>
36
- </html>
data/spec/fixtures/storeTarget.html DELETED
@@ -1,11 +0,0 @@
1
- <!DOCTYPE html>
2
- <html lang="ja">
3
- <head>
4
- <title>Sample</title>
5
- </head>
6
- <body>
7
- <p>A simple <b>test</b> string.</p>
8
- <a href="http://oh-news.net/live/wp-content/uploads/2011/04/Eila_omote.jpg">
9
- <img src="http://24.media.tumblr.com/tumblr_m5gneyJmsH1qza5ppo1_500.jpg" alt="" /></a>
10
- </body>
11
- </html>
data/spec/fixtures/storeTarget2.html DELETED
@@ -1,11 +0,0 @@
1
- <!DOCTYPE html>
2
- <html lang="ja">
3
- <head>
4
- <title>Sample</title>
5
- </head>
6
- <body>
7
- <p>A simple <b>test</b> string.</p>
8
- <a href="aaa">
9
- <img src="bbb" alt="" /></a>
10
- </body>
11
- </html>
data/spec/plugins/extract/link_spec.rb DELETED
@@ -1,38 +0,0 @@
1
- # -*- coding: utf-8 -*-
2
- # Name:: Automatic::Plugin::Extract::Link
3
- # Author:: 774 <http://id774.net>
4
- # Created:: Jun 12, 2012
5
- # Updated:: Jun 14, 2012
6
- # Copyright:: 774 Copyright (c) 2012
7
- # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
8
-
9
- require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper')
10
-
11
- require 'extract/link'
12
-
13
- describe Automatic::Plugin::ExtractLink do
14
- context "with html contain link tag" do
15
- subject {
16
- Automatic::Plugin::ExtractLink.new({},
17
- AutomaticSpec.generate_pipeline {
18
- html "extractLink.html"
19
- }
20
- )}
21
-
22
- describe "#run" do
23
- its(:run) { should have(4).items }
24
- specify {
25
- subject.run
26
- expect = "http://id774.net"
27
- subject.instance_variable_get(:@return_html)[0].should == expect
28
- expect = "http://reblog.id774.net"
29
- subject.instance_variable_get(:@return_html)[1].should == expect
30
- expect = "http://oh-news.net/live/wp-content/uploads/2011/04/Eila_omote.jpg"
31
- subject.instance_variable_get(:@return_html)[2].should == expect
32
- expect = "http://blog.id774.net/post/"
33
- subject.instance_variable_get(:@return_html)[3].should == expect
34
- }
35
- end
36
- end
37
- end
38
-