RubyGems - pstuteville-scrubyt - Versions diffs - 0.4.31 - Mend

pstuteville-scrubyt 0.4.31

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (520) hide show

data/examples/misc/tables/grab_rows/test.rb ADDED Viewed

@@ -0,0 +1,30 @@
+$:.unshift File.join(File.dirname(__FILE__), '../../../../lib')
+require 'scrubyt'
+#Example of:
+#  - Extracting some (specific) rows from a table
+#  - Using the :generalize option
+#  - Using the select_indices function
+#
+#Goal:      Extract the first, 4th, 5th and last row from the input table
+#
+#Solution:  Since by default only the root child(ren) of the extractor
+#           is/are generalized, and we need every <tr> (so we can select the ones we need)
+#           we have to set :generalize => true.
+#           Once we have all the indices, we select the relevant ones with the 'select_indices'
+#           function.
+table_data = Scrubyt::Extractor.define do
+  #Perform the action(s)
+  fetch File.join(File.dirname(__FILE__), "input.html")
+  #Construct the wrapper
+  table do
+    row :generalize => true do
+      value('1', :generalize => true).select_indices([3..4, :first,:last])
+    end
+  end
+end
+table_data.to_xml.write($stdout, 1)
+Scrubyt::ResultDumper.print_statistics(table_data)

data/examples/misc/tables/plain_table/input.html ADDED Viewed

@@ -0,0 +1,39 @@
+<html>
+  <body>
+    <table border=1 padding=10>
+      <tr>
+        <td>1</td>
+        <td>2</td>
+      </tr>
+      <tr>
+        <td>3</td>
+        <td>4</td>
+        <td>5</td>
+      </tr>
+    </table>
+    <br/>
+    <table border=1 padding=10>
+      <tr>
+        <td>6</td>
+      </tr>
+      <tr>
+        <td>7</td>
+      </tr>
+      <tr>
+        <td>8</td>
+      </tr>
+    </table>
+    <br>
+    <table border=1 padding=10>
+      <tr>
+        <td>9</td>
+	<td>10</td>
+      </tr>
+      <tr>
+      </tr>
+      <tr>
+        <td>11</td>
+      </tr>
+    </table>
+  </body>
+</html>

data/examples/misc/tables/plain_table/test.rb ADDED Viewed

@@ -0,0 +1,35 @@
+$:.unshift File.join(File.dirname(__FILE__), '../../../../lib')
+require 'scrubyt'
+#Example of:
+#  - Extracting every td from a table
+#  - Using the :generalize option
+#
+#Goal:      Extract every td from all tables on the page
+#
+#Solution:  Since by default only the root child(ren) of the extractor
+#           is/are generalized, and we need every <td> in every <tr>, we
+#           need to generalize the patterns that extracted them, too.
+#           To achieve this, use the the :generalize option as shown below!
+#
+table_data = Scrubyt::Extractor.define do
+  #Perform the action(s)
+  fetch File.join(File.dirname(__FILE__), "input.html")
+  #Construct the wrapper
+  table do
+    row :generalize => true do
+      value '1', :generalize => true
+    end.ensure_presence_of_pattern('value')
+  end
+end
+table_data.to_xml.write($stdout, 1)
+#puts table_data.table[1].row[0].value[0].to_text
+Scrubyt::ResultDumper.print_statistics(table_data)
+#table_data.export(__FILE__)

data/examples/misc/tables/plain_table_morepages/2.html ADDED Viewed

@@ -0,0 +1,38 @@
+<html>
+  <body>
+    <table border=1 padding=10>
+      <tr>
+        <td>12</td>
+        <td>13</td>
+        <td>14</td>
+        <td>15</td>
+      </tr>
+      <tr>
+        <td>16</td>
+      </tr>
+    </table>
+    <br/>
+    <table border=1 padding=10>
+      <tr>
+        <td>17</td>
+        <td>18</td>
+      </tr>
+      <tr>
+        <td>19</td>
+      </tr>
+    </table>
+    <br>
+    <table border=1 padding=10>
+      <tr>
+        <td>20</td>
+	<td>21</td>
+      </tr>
+      <tr>
+      </tr>
+      <tr>
+        <td>22</td>
+      </tr>
+    </table>
+    <a href='3.html'>Next</a>
+  </body>
+</html>

data/examples/misc/tables/plain_table_morepages/3.html ADDED Viewed

@@ -0,0 +1,33 @@
+<html>
+  <body>
+    <table border=1 padding=10>
+      <tr>
+        <td>23</td>
+        <td>24</td>
+        <td>25</td>
+      </tr>
+    </table>
+    <br/>
+    <table border=1 padding=10>
+      <tr>
+        <td>26</td>
+      </tr>
+      <tr>
+        <td>27</td>
+        <td>28</td>
+      </tr>
+    </table>
+    <br>
+    <table border=1 padding=10>
+      <tr>
+        <td>29</td>
+	<td>30</td>
+      </tr>
+      <tr>
+      </tr>
+      <tr>
+        <td>31</td>
+      </tr>
+    </table>
+  </body>
+</html>

data/examples/misc/tables/plain_table_morepages/input.html ADDED Viewed

@@ -0,0 +1,40 @@
+<html>
+  <body>
+    <table border=1 padding=10>
+      <tr>
+        <td>1</td>
+        <td>2</td>
+      </tr>
+      <tr>
+        <td>3</td>
+        <td>4</td>
+        <td>5</td>
+      </tr>
+    </table>
+    <br/>
+    <table border=1 padding=10>
+      <tr>
+        <td>6</td>
+      </tr>
+      <tr>
+        <td>7</td>
+      </tr>
+      <tr>
+        <td>8</td>
+      </tr>
+    </table>
+    <br>
+    <table border=1 padding=10>
+      <tr>
+        <td>9</td>
+	<td>10</td>
+      </tr>
+      <tr>
+      </tr>
+      <tr>
+        <td>11</td>
+      </tr>
+    </table>
+    <a href='2.html'>Next</a>
+  </body>
+</html>

data/examples/misc/tables/plain_table_morepages/test.rb ADDED Viewed

@@ -0,0 +1,32 @@
+$:.unshift File.join(File.dirname(__FILE__), '../../../../lib')
+require 'scrubyt'
+#Example of:
+#  - Extracting every td from a table
+#  - Using the :generalize option
+#  - Crawling to 2 next pages
+#
+#Goal:      Extract every td from every table on the page, and also from
+#           2 other pages linked to this document by a 'next' link
+#
+#Solution:  This example is very similar to the 'plain_table' one (so I will
+#           not describe what's going on there once more) with one exception -
+#           crawling to the next 2 pages. This is achieved by the 'next_page'
+#           idiom.
+#
+ambigous_data = Scrubyt::Extractor.define do
+  #Perform the action(s)
+  fetch File.join(File.dirname(__FILE__), "input.html")
+  #Construct the wrapper
+  table do
+    row :generalize => true do
+      value '1', :generalize => true
+    end
+  end
+  next_page "Next"
+end
+  ambigous_data.to_xml.write($stdout, 1)

data/examples/misc/tables/plain_table_morepages_with_image/2.html ADDED Viewed

@@ -0,0 +1,40 @@
+<html>
+  <body>
+    <table border=1 padding=10>
+      <tr>
+        <td>12</td>
+        <td>13</td>
+        <td>14</td>
+        <td>15</td>
+      </tr>
+      <tr>
+        <td>16</td>
+      </tr>
+    </table>
+    <br/>
+    <table border=1 padding=10>
+      <tr>
+        <td>17</td>
+        <td>18</td>
+      </tr>
+      <tr>
+        <td>19</td>
+      </tr>
+    </table>
+    <br>
+    <table border=1 padding=10>
+      <tr>
+        <td>20</td>
+	<td>21</td>
+      </tr>
+      <tr>
+      </tr>
+      <tr>
+        <td>22</td>
+      </tr>
+    </table>
+    <br/>
+    <br/>
+    <a href='1.html'>1</a> 2 <a href='3.html'>3</a> <a href='3.html'><img src='images/right_arrow.png' alt='a nice right arrow' style='border :0px'/></a>
+  </body>
+</html>

data/examples/misc/tables/plain_table_morepages_with_image/3.html ADDED Viewed

@@ -0,0 +1,33 @@
+<html>
+  <body>
+    <table border=1 padding=10>
+      <tr>
+        <td>23</td>
+        <td>24</td>
+        <td>25</td>
+      </tr>
+    </table>
+    <br/>
+    <table border=1 padding=10>
+      <tr>
+        <td>26</td>
+      </tr>
+      <tr>
+        <td>27</td>
+        <td>28</td>
+      </tr>
+    </table>
+    <br>
+    <table border=1 padding=10>
+      <tr>
+        <td>29</td>
+	<td>30</td>
+      </tr>
+      <tr>
+      </tr>
+      <tr>
+        <td>31</td>
+      </tr>
+    </table>
+  </body>
+</html>

data/examples/misc/tables/plain_table_morepages_with_image/images/right_arrow.png ADDED Viewed

Binary file

data/examples/misc/tables/plain_table_morepages_with_image/input.html ADDED Viewed

@@ -0,0 +1,42 @@
+<html>
+  <body>
+    <table border=1 padding=10>
+      <tr>
+        <td>1</td>
+        <td>2</td>
+      </tr>
+      <tr>
+        <td>3</td>
+        <td>4</td>
+        <td>5</td>
+      </tr>
+    </table>
+    <br/>
+    <table border=1 padding=10>
+      <tr>
+        <td>6</td>
+      </tr>
+      <tr>
+        <td>7</td>
+      </tr>
+      <tr>
+        <td>8</td>
+      </tr>
+    </table>
+    <br>
+    <table border=1 padding=10>
+      <tr>
+        <td>9</td>
+	<td>10</td>
+      </tr>
+      <tr>
+      </tr>
+      <tr>
+        <td>11</td>
+      </tr>
+    </table>
+    <br/>
+    <br/>
+    1 <a href='2.html'>2</a> <a href='3.html'>3</a> <a href='2.html'><img src='images/right_arrow.png' alt='a nice right arrow' style='border :0px'/></a>
+  </body>
+</html>

data/examples/misc/tables/plain_table_morepages_with_image/test.rb ADDED Viewed

@@ -0,0 +1,32 @@
+$:.unshift File.join(File.dirname(__FILE__), '../../../../lib')
+require 'scrubyt'
+#Example of:
+#  - Extracting every td from a table
+#  - Using the :generalize option
+#  - Crawling to 2 next pages
+#
+#Goal:      Extract every td from every table on the page, and also from
+#           2 other pages linked to this document by a 'next' link
+#
+#Solution:  This example is very similar to the 'plain_table' one (so I will
+#           not describe what's going on there once more) with one exception -
+#           crawling to the next 2 pages. This is achieved by the 'next_page'
+#           idiom.
+#
+ambigous_data = Scrubyt::Extractor.define do
+  #Perform the action(s)
+  fetch File.join(File.dirname(__FILE__), "input.html")
+  #Construct the wrapper
+  table do
+    row :generalize => true do
+      value '1', :generalize => true
+    end
+  end
+  next_page "images/right_arrow.png"
+end
+  ambigous_data.to_xml.write($stdout, 1)

data/examples/misc/tables/test_select_indices/input.html ADDED Viewed

@@ -0,0 +1,46 @@
+<html>
+  <body>
+    <div>
+      <table>
+        <tr>
+          <td>
+          item 1/1
+          </td>
+          <td>
+          good item 1/2
+          </td>
+          <td>
+          crap
+          </td>
+          <td>
+          item 2/1
+          </td>
+          <td>
+          good item 2/2
+          </td>
+          <td>
+          crap
+          </td>
+          <td>
+          item 3/1
+          </td>
+          <td>
+          good item 3/2
+          </td>
+          <td>
+          crap
+          </td>
+          <td>
+          item 4/1
+          </td>
+          <td>
+          good item 4/2
+          </td>
+          <td>
+          crap
+          </td>
+        </tr>
+      </table>
+    </div>
+  </body>
+</html>

data/examples/misc/tables/test_select_indices/test.rb ADDED Viewed

@@ -0,0 +1,55 @@
+$:.unshift File.join(File.dirname(__FILE__), '../../../../lib')
+require 'scrubyt'
+#Example of:
+#  - Extracting every td from a table
+#  - Using the :generalize option
+#
+#Goal:      Extract every td from all tables on the page
+#
+#Solution:  Since by default only the root child(ren) of the extractor
+#           is/are generalized, and we need every <td> in every <tr>, we
+#           need to generalize the patterns that extracted them, too.
+#           To achieve this, use the the :generalize option as shown below!
+#
+table_data = Scrubyt::Extractor.define do
+  #Perform the action(s)
+  fetch File.join(File.dirname(__FILE__), "input.html")
+  #Construct the wrapper
+  row do
+    (item 'item 1/1', :generalize => true).select_indices(:every_third)
+    (link 'good item 1/2', :generalize => true).select_indices(:every_fourth)
+  end
+end
+table_data.to_xml.write($stdout, 1)
+#puts table_data.table[1].row[0].value[0].to_text
+Scrubyt::ResultDumper.print_statistics(table_data)
+#table_data.export(__FILE__)
+=begin
+  <root>
+    <row>
+      <item>item 1/1</item>
+      <item>item 2/1</item>
+      <item>item 3/1</item>
+      <item>item 4/1</item>
+    </row>
+  </root>
+  <root>
+    <row>
+      <link>item 1/1</link>
+      <link>good item 2/2</link>
+      <link>crap</link>
+    </row>
+  </root>
+=end

data/examples/misc/xpath_example_type/input.html ADDED Viewed

@@ -0,0 +1,15 @@
+<html>
+<body>
+<div>
+  <table>
+    <tr>
+      <td>Child</td>
+    </tr>
+    <tr>
+      <td>Parent</td>
+    </tr>
+  </table>
+  <strong>test</strong>
+</div>
+</body>
+</html>

data/examples/misc/xpath_example_type/test.rb ADDED Viewed

@@ -0,0 +1,18 @@
+$:.unshift File.join(File.dirname(__FILE__), '../../../lib')
+require 'scrubyt'
+extractor = Scrubyt::Extractor.define do
+  fetch File.join(File.dirname(__FILE__), "input.html")
+  root '//table' do
+    parent '/tr[2]', :generalize=>false do
+      name '/td'
+      child '../tr[1]', :example_type=>:xpath, :generalize=>false, :type=>:tree do
+        name './td'
+      end
+    end
+  end
+end
+extractor.to_xml.write($stdout, 1)

data/examples/misc/yahoo_finance/test.rb ADDED Viewed

@@ -0,0 +1,26 @@
+$:.unshift File.join(File.dirname(__FILE__), '../../../lib')
+require 'scrubyt'
+#Example of:
+#  - scraping stock values from finance.yahoo.com
+#
+#Goal:       from finance.yahoo.com. Scrape the actual values found there
+#            for the stock information available.
+#Solution:   Nothing special here. The examples are defined, and
+#            the ensure_presence_of_pattern constraint is used to filter out
+#            stockinfos that do not have a value specified.
+stock_data = Scrubyt::Extractor.define do
+  #Perform the action(s)
+  fetch 'http://finance.yahoo.com/'
+  #Construct the wrapper
+  stockinfo do
+    symbol  'Dow'
+    value   '15.54'
+  end.ensure_presence_of_pattern('value')
+end
+stock_data.to_xml.write($stdout, 1)
+Scrubyt::ResultDumper.print_statistics(stock_data)

data/examples/social/blog_comment/test.rb ADDED Viewed

@@ -0,0 +1,27 @@
+$:.unshift File.join(File.dirname(__FILE__), '../../..//lib')
+require 'scrubyt'
+#Example of:
+#  - filling and submitting a textfield
+#  - filling a text area
+#  - submitting the form
+#
+#Goal:       Automatically sending a comment to a wordpress blog
+#Solution:   After fetching the page, fill the 'author' and 'email' textfields,
+#            then enter your comment to the 'comment' textarea and submit the form.
+#            This example does not use a scraper - it's an example of an extractor
+#            which uses just navigation to accomplish the goal.
+#            Please don't run this too much (or change the name,email and comment
+#            to something menaningful) - I would not like to have a horde of comments
+#            from Bruce Lee :-)
+#
+#Credit: Zaheed Haque
+post_comment = Scrubyt::Extractor.define do
+  fetch 'http://scrubyt.org/your-first-extractor/'
+  fill_textfield 'author', 'Bruce Lee'
+  fill_textfield 'email', 'bruce@kung-foo.com'
+  fill_textarea 'comment', 'Yet another test comment from Bruce!'
+  submit
+end

data/examples/social/del.icio.us/test.rb ADDED Viewed

@@ -0,0 +1,22 @@
+$:.unshift File.join(File.dirname(__FILE__), '../../..//lib')
+require 'scrubyt'
+#Will comment later
+del_icio_us_data = Scrubyt::Extractor.define do
+  fetch "http://del.icio.us/search/?p=ruby"
+  item do
+    title 'Why’s (Poignant) Guide to Ruby'
+    title_url 'Why’s (Poignant) Guide to Ruby' do
+      url 'href', :type => :attribute
+    end
+    bookmarks 'saved by 6687 people', :generalize => true do
+      votes /\d+/
+    end.select_indices(:all_but_last)
+  end
+end
+del_icio_us_data.to_xml.write($stdout, 1)
+del_icio_us_data.export(__FILE__)

data/examples/social/digg/test.rb ADDED Viewed

@@ -0,0 +1,37 @@
+$:.unshift File.join(File.dirname(__FILE__), '../../..//lib')
+require 'scrubyt'
+#Example of:
+#  - Scraping a digg page; otherwise nothing special
+#
+#Goal: Extract every main item contained on the front page - an item
+#      should containt the title and the number of diggs.
+#
+#Solution:  Well, nothing special - this could even be a 'Hello, world!'
+#           type of example. However, it is far from that - since digg
+#           is changing so fast that it is practically impossible to
+#           capture an example before it gets at least one more digg on
+#           the frontpage. Solution: go to the 10th or so page, and select
+#           an example there (with the lowest possible digg, since that lowers
+#           the risk of being digged up even more) and export the stuff immediately
+#           then use the production extractor.
+# !!! NOTE !!! to run this example, you will need to change the example!
+# it is totally impossible that there is something on the frontpage with 1 diggs!
+digg_data = Scrubyt::Extractor.define do
+  fetch 'http://digg.com/'
+  fill_textfield 's', 'ruby'
+    submit
+    item do
+      title "Free Dog Training Tips"
+      diggs "1"
+    end
+end
+digg_data.to_xml.write($stdout, 1)
+Scrubyt::ResultDumper.print_statistics(digg_data)
+digg_data.export(__FILE__)