maiha-dm-ys 0.3 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (16) hide show
  1. data/Rakefile +1 -1
  2. data/lib/dm-ys.rb +1 -0
  3. data/lib/dm-ys/config.rb +30 -0
  4. data/lib/dm-ys/proxy.rb +3 -0
  5. data/lib/dm-ys/scraper.rb +55 -17
  6. data/spec/composite_scraper_spec.rb +52 -0
  7. data/spec/config_spec.rb +68 -0
  8. data/spec/data/sorted1.html +262 -0
  9. data/spec/data/sorted2.html +228 -0
  10. data/spec/data/uniq1.html +258 -0
  11. data/spec/data/uniq2.html +128 -0
  12. data/spec/models/plugin.rb +39 -0
  13. data/spec/{guess_spec.rb → proxy_spec.rb} +35 -1
  14. data/spec/scraper_spec.rb +9 -0
  15. data/spec/spec_helper.rb +17 -4
  16. metadata +9 -3
data/spec/data/uniq2.html ADDED
@@ -0,0 +1,128 @@
1
+ <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
2
+ <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en-us" lang="en-us">
3
+ <head>
4
+ <title>Merbist Plugins</title>
5
+ <meta http-equiv="content-type" content="text/html; charset=utf-8" />
6
+ <meta name="verify-v1"
7
+ content="QqJ1Kmvs51kF+1Sn+7JUqkXTXbnmLVKzFctoGLRDLE8=" />
8
+ <link rel="stylesheet" href="/stylesheets/master.css" type="text/css"
9
+ media="screen" charset="utf-8" />
10
+ <script src="/javascripts/jquery.js" type="text/javascript"></script>
11
+ <link rel="alternate" type="application/rss+xml" title="Atom" href="/plugins.atom"></link>
12
+
13
+ </head>
14
+ <body>
15
+ <div id="base">
16
+ <div id="header">
17
+ <div id="navi">
18
+ <a href="/">Top</a>
19
+ <a href="/plugins">Plugins</a>
20
+ <a href="/talks">Talks</a>
21
+ <a href="/sites">Sites</a>
22
+ <a href="/users">Users</a>
23
+ </div>
24
+ <div id="menu">
25
+ <a href="/login">Login</a>
26
+ <a href="/users/new">Sing up</a>
27
+ </div>
28
+ <div class="clear"><!----></div>
29
+
30
+ </div>
31
+ <div id="body">
32
+ <div class="command">
33
+ <a href="/plugins">Index</a>
34
+ </div>
35
+
36
+
37
+ <h1>Plugin List</h1>
38
+
39
+ <table class="plugin-list" cellspacing="1" border="0">
40
+ <thead>
41
+ <tr>
42
+ <th>Name</th>
43
+ <th>Repos</th>
44
+ <th>Registered by</th>
45
+ <th>Description</th>
46
+ <th></th>
47
+ </tr>
48
+ </thead>
49
+ <tbody>
50
+ <tr class="even">
51
+ <td><a href="/plugins/16">templater-0.5.0</a></td>
52
+ <td>&dagger;</td>
53
+ <td><a href="/users/1">genki</a></td>
54
+ <td>RubiGen alternative Generator thingy</td>
55
+ <td>
56
+ </td>
57
+ </tr>
58
+ <tr class="">
59
+ <td><a href="/plugins/15">hosts_access-0.3</a></td>
60
+ <td>&dagger;</td>
61
+ <td><a href="/users/1">genki</a></td>
62
+ <td>Rails/Merb plugin that controls host access like hosts.allow</td>
63
+ <td>
64
+ </td>
65
+ </tr>
66
+ <tr class="even">
67
+ <td><a href="/plugins/36">eventmachine-0.12.5</a></td>
68
+ <td>&dagger;</td>
69
+ <td><a href="/users/1">genki</a></td>
70
+ <td>EventMachine: fast, simple event-processing library for Ruby programs</td>
71
+ <td>
72
+ </td>
73
+ </tr>
74
+ <tr class="">
75
+ <td><a href="/plugins/35">dm-last-0.0.1</a></td>
76
+ <td>&dagger;</td>
77
+ <td><a href="/users/1">genki</a></td>
78
+ <td>A DataMapper plugin that offers a short-hand for Model.all.last as Model.last</td>
79
+ <td>
80
+ </td>
81
+ </tr>
82
+ </tbody>
83
+ </table>
84
+
85
+ <div class="pagination"><span class="prev disabled">&laquo; Prev</span>
86
+ <span class="current disabled">1</span>
87
+ <span class="disabled"><a href="/plugins/index?page=2">2</a></span>
88
+ <a class="next" rel="next" href="/plugins/index?page=2">Next &raquo;</a></div>
89
+
90
+ <div class="footnote">
91
+ &dagger; You can install these gems by
92
+ <code>
93
+ gem install gem-name --source http://merbi.st
94
+ </code>
95
+ </div>
96
+
97
+
98
+ </div>
99
+ <div id="footer">
100
+ <div id="footer">
101
+ 2008
102
+ <a href="http://wota.jp/ac/">maiha</a>,
103
+ <a href="http://d.jong.gr.jp/shachi">shachi</a>
104
+ and
105
+ <a href="http://blog.s21g.com/genki">genki</a>
106
+ (<a href="http://www.s21g.com/">s21g LLC</a>).
107
+ <span class="powered-by">
108
+ Powered by
109
+ Merb-1.0.9
110
+ (Ruby-1.9.1)
111
+ </span>
112
+ </div>
113
+
114
+ </div>
115
+ </div>
116
+
117
+ <script type="text/javascript">
118
+ var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www.");
119
+ document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E"));
120
+ </script>
121
+ <script type="text/javascript">
122
+ try {
123
+ var pageTracker = _gat._getTracker("UA-2733799-11");
124
+ pageTracker._trackPageview();
125
+ } catch(err) {}</script>
126
+
127
+ </body>
128
+ </html>
data/spec/models/plugin.rb CHANGED
@@ -12,3 +12,42 @@ class Plugin2
12
12
  include DataMapper::YunkerStar
13
13
  uri "http://merbi.st/plugins/index?page=2"
14
14
  end
15
+
16
+ class SortedPlugin
17
+ include DataMapper::YunkerStar
18
+ uri "http://merbi.st/plugins/sorted?page=1*"
19
+ ys[:uniq] = false
20
+ end
21
+
22
+ class SortedPlugin1
23
+ include DataMapper::YunkerStar
24
+ uri "http://merbi.st/plugins/sorted?page=1"
25
+ ys[:uniq] = false
26
+ end
27
+
28
+ class SortedPlugin2
29
+ include DataMapper::YunkerStar
30
+ uri "http://merbi.st/plugins/sorted?page=2"
31
+ ys[:uniq] = false
32
+ end
33
+
34
+ class SortedPluginWithUniqPage
35
+ include DataMapper::YunkerStar
36
+ uri "http://merbi.st/plugins/sorted?page=1*"
37
+ ys[:uniq] = :page
38
+ end
39
+
40
+ class UniqPlugin
41
+ include DataMapper::YunkerStar
42
+ uri "http://merbi.st/plugins/uniq?page=1*"
43
+ end
44
+
45
+ class UniqPlugin1
46
+ include DataMapper::YunkerStar
47
+ uri "http://merbi.st/plugins/uniq?page=1"
48
+ end
49
+
50
+ class UniqPlugin2
51
+ include DataMapper::YunkerStar
52
+ uri "http://merbi.st/plugins/uniq?page=2"
53
+ end
data/spec/{guess_spec.rb → proxy_spec.rb} RENAMED
@@ -21,7 +21,41 @@ describe DataMapper::YunkerStar do
21
21
  thead "table.main"
22
22
  end
23
23
 
24
- it "should provide proxy" do
24
+ ######################################################################
25
+ ### Config
26
+
27
+ it "should provide .ys" do
28
+ Plugin.should respond_to(:ys)
29
+ end
30
+
31
+ describe ".ys" do
32
+ it "should return a ys config" do
33
+ Plugin.ys.should be_kind_of(DataMapper::YunkerStar::Config)
34
+ end
35
+
36
+ describe "[:max_pages]" do
37
+ class NetworkUnreachable
38
+ include DataMapper::YunkerStar
39
+ uri "http://merbi.st/plugins/index?page=1*"
40
+ ys[:max_pages] = 0
41
+ end
42
+
43
+ it "should has 100 as default value" do
44
+ Plugin.ys[:max_pages].should == 100
45
+ end
46
+
47
+ it "should raise MaxPagesOverflow when count of visited sites exceeds :max_pages value" do
48
+ lambda {
49
+ NetworkUnreachable.count
50
+ }.should raise_error(DataMapper::YunkerStar::Proxy::MaxPagesOverflow)
51
+ end
52
+ end
53
+ end
54
+
55
+ ######################################################################
56
+ ### Proxy
57
+
58
+ it "should provide .proxy" do
25
59
  BlankStyle.should respond_to(:proxy)
26
60
  end
27
61
 
data/spec/scraper_spec.rb CHANGED
@@ -29,5 +29,14 @@ describe DataMapper::YunkerStar::Proxy do
29
29
  @scraper.respond_to?(:label2name, true).should == true
30
30
  end
31
31
 
32
+ it "should provide #page_hash" do
33
+ @scraper.should respond_to(:page_hash)
34
+ end
35
+
36
+ describe "#page_hash" do
37
+ it "should return a string" do
38
+ @scraper.page_hash.should be_kind_of(String)
39
+ end
40
+ end
32
41
 
33
42
  end
data/spec/spec_helper.rb CHANGED
@@ -31,8 +31,21 @@ rescue LoadError
31
31
  exit
32
32
  end
33
33
 
34
- URI::Mapping.merge!(
34
+ mapping = {
35
+ # plugin (paginated)
35
36
  "http://merbi.st/plugins/index?page=1" => spec_data_path("plugins1.html"),
36
- "http://merbi.st/plugins/index?page=2" => spec_data_path("plugins2.html")
37
- )
38
-
37
+ "http://merbi.st/plugins/index?page=2" => spec_data_path("plugins2.html"),
38
+
39
+ # plugin + uniq
40
+ "http://merbi.st/plugins/uniq?page=1" => spec_data_path("uniq1.html"),
41
+ "http://merbi.st/plugins/uniq?page=2" => spec_data_path("uniq2.html"),
42
+
43
+ # plugin + sorted
44
+ "http://merbi.st/plugins/sorted?page=1" => spec_data_path("sorted1.html"),
45
+ "http://merbi.st/plugins/sorted?page=1&sort=name1" => spec_data_path("sorted1.html"),
46
+ "http://merbi.st/plugins/sorted?page=1&sort=name2" => spec_data_path("sorted1.html"),
47
+ "http://merbi.st/plugins/sorted?page=2" => spec_data_path("sorted2.html"),
48
+ "http://merbi.st/plugins/sorted?page=2&sort=name1" => spec_data_path("sorted2.html"),
49
+ "http://merbi.st/plugins/sorted?page=2&sort=name2" => spec_data_path("sorted2.html"),
50
+ }
51
+ URI::Mapping.merge!(mapping)
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: maiha-dm-ys
3
3
  version: !ruby/object:Gem::Version
4
- version: "0.3"
4
+ version: 0.3.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - maiha
@@ -9,7 +9,7 @@ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
11
 
12
- date: 2009-03-06 00:00:00 -08:00
12
+ date: 2009-03-07 00:00:00 -08:00
13
13
  default_executable:
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
@@ -58,16 +58,21 @@ files:
58
58
  - lib/dm-ys.rb
59
59
  - lib/dm-ys
60
60
  - lib/dm-ys/base.rb
61
+ - lib/dm-ys/config.rb
61
62
  - lib/dm-ys/indexed_property.rb
62
63
  - lib/dm-ys/memory_repository.rb
63
64
  - lib/dm-ys/scraper.rb
64
65
  - lib/dm-ys/cached_accessor.rb
65
66
  - lib/dm-ys/proxy.rb
66
- - spec/guess_spec.rb
67
+ - spec/config_spec.rb
67
68
  - spec/data
69
+ - spec/data/uniq2.html
70
+ - spec/data/sorted2.html
71
+ - spec/data/sorted1.html
68
72
  - spec/data/plugins1.html
69
73
  - spec/data/gem_maintainers.html
70
74
  - spec/data/plugins2.html
75
+ - spec/data/uniq1.html
71
76
  - spec/data/blank.html
72
77
  - spec/models
73
78
  - spec/models/gem_maintainer.rb
@@ -79,6 +84,7 @@ files:
79
84
  - spec/spec_helper.rb
80
85
  - spec/pagination_spec.rb
81
86
  - spec/scraper_utils_spec.rb
87
+ - spec/proxy_spec.rb
82
88
  has_rdoc: true
83
89
  homepage: http://github.com/maiha/dm-ys
84
90
  post_install_message: