RubyGems - pstuteville-scrubyt - Versions diffs - 0.4.31 - Mend

pstuteville-scrubyt 0.4.31

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (520) hide show

data/examples/events/delta/test.rb ADDED Viewed

@@ -0,0 +1,16 @@
+$:.unshift File.join(File.dirname(__FILE__), '../../../lib')
+require 'scrubyt'
+event_data = Scrubyt::Extractor.define do
+  fetch File.join(File.dirname(__FILE__), "input.html")
+  event do
+    title 'Frühlingsgefühle'
+    begin_time '22.00'
+    admission_fee '4,00'
+  end
+end
+event_data.to_xml.write($stdout, 1)
+Scrubyt::ResultDumper.print_statistics(event_data)

data/examples/misc/auto_regex/input.html ADDED Viewed

@@ -0,0 +1,22 @@
+<html>
+  <body>
+    <div>
+      abc1
+      <i>
+        def1
+      </i>
+    </div>
+    <div>
+      abc2
+      <i>
+        def2
+      </i>
+    </div>
+    <div>
+      abc3
+      <i>
+        def3
+      </i>
+    </div>
+  </body>
+</html>

data/examples/misc/auto_regex/test.rb ADDED Viewed

@@ -0,0 +1,14 @@
+$:.unshift File.join(File.dirname(__FILE__), '../../../lib')
+require 'scrubyt'
+event_data = Scrubyt::Extractor.define do
+  fetch File.join(File.dirname(__FILE__), "input.html")
+  group do
+    e1 'f1'
+  end
+end
+event_data.to_xml.write($stdout, 1)
+Scrubyt::ResultDumper.print_statistics(event_data)

data/examples/misc/compound_example/advanced/test.rb ADDED Viewed

@@ -0,0 +1,11 @@
+$:.unshift File.join(File.dirname(__FILE__), '../../../..lib')
+require 'scrubyt'
+compo_data = Scrubyt::Extractor.define do
+   fetch(File.join(File.dirname(__FILE__),'tricky_compound.html'))
+   para({:contains => '4, 5, 6', :begins_with => '3', :ends_with => '6'}, :generalize => false)
+end
+compo_data.to_xml.write($stdout,1)

data/examples/misc/compound_example/advanced/tricky_compound.html ADDED Viewed

@@ -0,0 +1,9 @@
+<html>
+  <body>
+    <table>
+      <tr><td>1, 2, 3, 4, <b>5, 6,</b> 7, 8</td></tr>
+      <tr><td>3, 4, <b>5, 6,</b> 7</td></tr>
+      <tr><td>3, 4, 5, 6</td></tr>
+    </table>
+  </body>
+</html>

data/examples/misc/compound_example/regexp/regexp_compound.html ADDED Viewed

@@ -0,0 +1,17 @@
+<html>
+  <body>
+    <table>
+      <tr>
+        <td>
+        This is a phone number: 123-456-78. It's a dummy phone no.
+        </td>
+        <td>
+        Another dummy phone number: 111-222-333. Really dummy.
+        </td>
+        <td>
+        And for the users of scRUBYt!, yet another dumb fake-phone no: 111-111-111-111. Enough examples for now.
+        </td>
+      </tr>
+    </table>
+  </body>
+</html>

data/examples/misc/compound_example/regexp/test.rb ADDED Viewed

@@ -0,0 +1,11 @@
+$:.unshift File.join(File.dirname(__FILE__), '../../../..lib')
+require 'scrubyt'
+compo_data = Scrubyt::Extractor.define do
+   fetch(File.join(File.dirname(__FILE__),'regexp_compound.html'))
+   para({:contains => /11\d+/}, :generalize => false)
+end
+compo_data.to_xml.write($stdout,1)

data/examples/misc/compound_example/simple/compound.html ADDED Viewed

@@ -0,0 +1,5 @@
+<html>
+  <body>
+    <p>Time to go, <b>said</b> Jack</p>
+  </body>
+</html>

data/examples/misc/compound_example/simple/test.rb ADDED Viewed

@@ -0,0 +1,11 @@
+$:.unshift File.join(File.dirname(__FILE__), '../../../../lib')
+require 'scrubyt'
+compo_data = Scrubyt::Extractor.define do
+   fetch(File.join(File.dirname(__FILE__),'compound.html'))
+   para({:contains => 'said'}, :generalize => false)
+end
+compo_data.to_xml.write($stdout,1)

data/examples/misc/detail_page/detailpage.html ADDED Viewed

@@ -0,0 +1,6 @@
+<html>
+  <body>
+    <p>Here is one detail paragraph!</p>
+    <p>Yet another annoying paragraph</p>
+  </body>
+</html>

data/examples/misc/detail_page/index.html ADDED Viewed

@@ -0,0 +1,9 @@
+<html>
+  <body>
+    <table style='border :1px solid black' cellpadding=2 cellspacing=2>
+      <tr><td>This is some text</td></tr>
+      <tr><td>This is some more nonsense text</td></tr>
+      <tr><td>And here is the detail link: <a href='detailpage.html'>detail link!</a></td></tr>
+    </table>
+  </body>
+</html>

data/examples/misc/detail_page/test.rb ADDED Viewed

@@ -0,0 +1,17 @@
+$:.unshift File.join(File.dirname(__FILE__), '../../../lib')
+require 'scrubyt'
+result = Scrubyt::Extractor.define do
+  fetch          'index.html'
+  stuff 'detail link!' do
+    stuff_detail do
+      other_stuff 'Here is one detail paragraph!'
+    end
+  end
+end
+puts result.to_xml
+result.export('test')

data/examples/misc/google/test.rb ADDED Viewed

@@ -0,0 +1,39 @@
+$:.unshift File.join(File.dirname(__FILE__), '../../../lib')
+require 'scrubyt'
+#The compulsory google example :)
+#Don't you have the feeling that a google.com example in web programming
+#is becoming something like 'Hello World' in desktop programming?
+#
+#Example of:
+#  - filling and submitting a textfield
+#  - using a href attribute
+#  - recursively crawling to the next page(s)
+#
+#Goal:       Go to google.com. Enter 'ruby' into the search textfield and
+#            submit the form. Extract the url of the first 20 pages.
+#Solution:   Use the navigational commands 'fetch', 'fill_textfield', 'submit'
+#            to navigate to the page of interest. There, extract the links with
+#            the pattern 'link'. The URLs should be extracted with 'link''s child
+#            pattern, 'url' which is an attribute pattern.
+#            This will extract the first 10 results, but you need the first 20.
+#            To achieve this, the 'next_page' idiom should be used, with :limit set
+#            to 2.
+google_data = Scrubyt::Extractor.define do
+  #Perform the action(s)
+  fetch 'http://www.google.com/ncr'
+  fill_textfield 'q', 'ruby'
+  submit
+  #Construct the wrapper
+  link "Ruby Programming Language" do
+    url "href", :type => :attribute
+  end
+  next_page "Next", :limit => 2
+end
+google_data.to_xml.write($stdout, 1)
+Scrubyt::ResultDumper.print_statistics(google_data)

data/examples/misc/identical_examples/data_extractor_export.rb ADDED Viewed

@@ -0,0 +1,12 @@
+require("rubygems")
+require("scrubyt")
+data = Scrubyt::Extractor.define do
+  fetch("input.html")
+  stuff("/html[1]/body[1]/table[1]/tr[2]/td[1]", { :generalize => false }) do
+    index("index", { :type => :attribute })
+  end
+end
+data.to_xml.write($stdout, 1)

data/examples/misc/identical_examples/input.html ADDED Viewed

@@ -0,0 +1,16 @@
+<html>
+  <body>
+    <table>
+      <tr>
+        <td index="1">Identical example</td>
+        <td index="2">Identical example</td>
+      </tr>
+      <tr>
+        <td index="3">Identical example</td>
+        <td index="4">Identical example</td>
+      </tr>
+    </table>
+    <p index="5">Identical example</p>
+  </body>
+</html>

data/examples/misc/identical_examples/test.rb ADDED Viewed

@@ -0,0 +1,15 @@
+$: << File.join(File.dirname(__FILE__), '../../../lib')
+require 'scrubyt'
+include Scrubyt
+data = Extractor.define do
+  fetch 'input.html'
+  stuff 'Identical example[2]', :generalize => false do
+    index 'index', :type => :attribute
+  end
+end
+data.to_xml.write($stdout,1)
+data.export(__FILE__)

data/examples/misc/immediate_attribute_extraction/data_extractor_export.rb ADDED Viewed

@@ -0,0 +1,10 @@
+require("rubygems")
+require("scrubyt")
+data = Scrubyt::Extractor.define do
+  fetch("input.html")
+  stuff("/html/body/table/tr/td", { :generalize => true })
+end
+data.to_xml.write($stdout, 1)

data/examples/misc/immediate_attribute_extraction/input.html ADDED Viewed

@@ -0,0 +1,16 @@
+<html>
+  <body>
+    <table>
+      <tr>
+        <td index="1">Identical example</td>
+        <td index="2">Identical example</td>
+      </tr>
+      <tr>
+        <td index="3">Identical example</td>
+        <td index="4">Identical example</td>
+      </tr>
+    </table>
+    <p index="5">Identical example</p>
+  </body>
+</html>

data/examples/misc/immediate_attribute_extraction/test.rb ADDED Viewed

@@ -0,0 +1,14 @@
+$: << File.join(File.dirname(__FILE__), '../../../lib')
+require 'scrubyt'
+include Scrubyt
+data = Extractor.define do
+  fetch 'input.html'
+  stuff 'Identical example/@index'#, :generalize => false
+end
+data.to_xml.write($stdout,1)
+data.show_stats
+data.export(__FILE__)

data/examples/misc/multiple_examples/input.html ADDED Viewed

@@ -0,0 +1,7 @@
+<html>
+  <body>
+  <div>we would not like to match this div.
+    <p>However, we would like to match this paragraph, and also the following span: <span><b>hi there!</b></span></p>
+  </div>
+  </body>
+</html>

data/examples/misc/multiple_examples/test.rb ADDED Viewed

@@ -0,0 +1,22 @@
+$:.unshift File.join(File.dirname(__FILE__), '../../../lib')
+require 'scrubyt'
+#Example of:
+#Goal:
+#Solution:
+multiple_example_data = Scrubyt::Extractor.define do
+  fetch File.join(File.dirname(__FILE__), "input.html")
+  first_info 'However, we would like to match this paragraph, and also the following span: ' do
+    additional_info 'hi there!'
+  end
+end
+multiple_example_data.to_xml.write($stdout, 1)
+multiple_example_data.export(__FILE__)
+Scrubyt::ResultDumper.print_statistics(multiple_example_data)

data/examples/misc/on_click_next/next_page_link.rb ADDED Viewed

@@ -0,0 +1,42 @@
+$:.unshift File.join(File.dirname(__FILE__), '../../../lib')
+require 'scrubyt'
+require 'scrubyt/output/result_dumper'
+#Example of: Using the next_page command with onclick='' hrefs.
+### Doesn't work, as next_page doesn't click links
+data = Scrubyt::Extractor.define(:agent => :firefox) do
+  fetch("file://"+File.expand_path(File.join(File.dirname(__FILE__), "page_1.html")))
+  entry '//div'
+  next_page 'Next'
+end
+puts "First results :"
+puts data.to_xml
+puts "---------------"
+### Doesn't work, all the results are the same :(
+data = Scrubyt::Extractor.define(:agent => :firefox) do
+  fetch("file://"+File.expand_path(File.join(File.dirname(__FILE__), "page_1.html")))
+  while(true)
+    entry '//div'
+    begin
+      click_link 'Next'
+    rescue Watir::Exception::UnknownObjectException
+      puts "Reached the end.  Breaking"
+      break
+    end
+  end
+end
+puts "Second results:"
+puts data.to_xml

data/examples/misc/on_click_next/page_1.html ADDED Viewed

@@ -0,0 +1,10 @@
+<html>
+  <body>
+    <div>1</div>
+    <div>2</div>
+    <div>3</div>
+    <form id='nextPage' action='page_2.html'>
+      <a onclick='document.getElementById("nextPage").submit(); return false' href="#">Next</a>
+    </form>
+  </body>
+</html>

data/examples/misc/on_click_next/page_2.html ADDED Viewed

@@ -0,0 +1,10 @@
+<html>
+  <body>
+    <div>4</div>
+    <div>5</div>
+    <div>6</div>
+    <form id='nextPage' action='page_3.html'>
+      <a onclick='document.getElementById("nextPage").submit(); return false' href="#">Next</a>
+    </form>
+  </body>
+</html>

data/examples/misc/on_click_next/page_3.html ADDED Viewed

@@ -0,0 +1,7 @@
+<html>
+  <body>
+    <div>7</div>
+    <div>8</div>
+    <div>9</div>
+  </body>
+</html>

data/examples/misc/rubycorner/test.rb ADDED Viewed

@@ -0,0 +1,29 @@
+$:.unshift File.join(File.dirname(__FILE__), '../../../lib')
+require 'scrubyt'
+#Example of: Collecting blog addresses from rubycorner.com
+#            - extracting href attribute from an <a> tag
+#            - crawling to all of the pages
+#
+#Goal:       Go to http://rubycorner.com/blogs/lang/en. This listing
+#            contains all the English blogs aggregated by rubycorner.
+#            Scrape all of them!
+#Solution:   Extract all <a> patterns as you would normally. To further
+#            extract the href from the 'link' pattern, use an attrinute
+#            pattern (specified by :type => :attribute)
+#            Finally, crawl to all of the pages with next_page - this time
+#            without a limit since we need all the pages.
+rubycorner_data = Scrubyt::Extractor.define do
+  #Perform the action(s)
+  fetch 'http://rubycorner.com/blogs/lang/en'
+  #Construct the wrapper
+  link "Boston BarCamp Day 1" do
+    url "href", :type => :attribute
+  end
+  next_page ">"
+end
+rubycorner_data.to_xml.write($stdout, 1)
+Scrubyt::ResultDumper.print_statistics(rubycorner_data)

data/examples/misc/rubyforge_login/test.rb ADDED Viewed

@@ -0,0 +1,30 @@
+$:.unshift File.join(File.dirname(__FILE__), '../../../lib')
+require 'scrubyt'
+#Example of: - logging to an account - rubyforge.org this time
+#            - submitting the form with a button
+#
+#Goal:       Go to 'http://rubyforge.org/' and login. Once you are inside,
+#            scrape something not really interesting just to show you are in!
+#Solution:   Go to rubyforge.org and click on the 'Log In' link. This gets you to the
+#            login page, where you fill in the appropriate textfields.
+#            Submitting the textfield once everything is setup up for the show is a bit
+#            different this time - since a simple form.submit does not work. You have
+#            to submit the form by pressing a button. The solution is to specify the index
+#            of the button which should be pressed.
+rubyforge_data = Scrubyt::Extractor.define do
+  fetch          'http://rubyforge.org/'
+  click_link     'Log In'
+  fill_textfield 'form_loginname', '*login_here*'
+  fill_textfield 'form_pw', '*pass_here*'
+  submit 0
+  stuff 'My Personal Page'
+end
+rubyforge_data.to_xml.write($stdout, 1)
+Scrubyt::ResultDumper.print_statistics(rubyforge_data)

data/examples/misc/tables/ambigous_records/input.html ADDED Viewed

@@ -0,0 +1,17 @@
+<html>
+  <body>
+    <table>
+      <tr>
+        <td>Ruby</td>
+        <tb>Python</tb>
+      </tr>
+      <tr>
+        <td><b>Java</b></td>
+        <td>C#</td>
+      </tr>
+      <tr>
+      </tr>
+    </table>
+  </body>
+</html>

data/examples/misc/tables/ambigous_records/test.rb ADDED Viewed

@@ -0,0 +1,37 @@
+$:.unshift File.join(File.dirname(__FILE__), '../../../../lib')
+require 'scrubyt'
+#Example of:
+#  - Sepecyfiing multiple examples
+#
+#Goal:       From this simple table, scrape the languages and their rivals, into
+#            a structure like
+#
+#            <language_war>
+#              <lang>Ruby</lang>
+#              <rival>Python</rival>
+#            </language_war>
+#
+#            Try it as you would normally - i.e. just supply lang the example 'Ruby'
+#            and rival the example 'Python' and it won't work. This is because
+#            in the document, the other two semantically similar records (Java and
+#            C#) are contained at different places in the structure. This is a dummy
+#            example, but in the real life you have this with a lot of webshops
+#            and other pages as well.
+#
+#Solution:   Specify multiple examples as in the example below - until you get
+#            everything you need (i.e. theoretically you can specify even
+#            12345 examples, however that would be unmaintainable :-))
+ambigous_data = Scrubyt::Extractor.define do
+  fetch File.join(File.dirname(__FILE__), "input.html")
+  #Construct the wrapper
+  language_war do
+    lang  'Ruby', 'Java'
+    rival 'Python', 'C#'
+  end
+end
+ambigous_data.to_xml.write($stdout, 1)
+Scrubyt::ResultDumper.print_statistics(ambigous_data)

data/examples/misc/tables/another_plain_table/input.html ADDED Viewed

@@ -0,0 +1,15 @@
+<html>
+  <body>
+    <table border=1 padding=10>
+      <tr>
+        <td>1, 2, 3</td>
+        <td>Vancouver, Quebec</td>
+      </tr>
+      <tr>
+        <td>London and Vienna and Prague</td>
+        <td>1, Spam, 3, Ham</td>
+        <td>Just a simple text cell</td>
+      </tr>
+    </table>
+  </body>
+</html>

data/examples/misc/tables/another_plain_table/test.rb ADDED Viewed

@@ -0,0 +1,25 @@
+$:.unshift File.join(File.dirname(__FILE__), '../../../../lib')
+require 'scrubyt'
+#Example of:
+#  - using a regexp pattern
+#
+#Goal:      Extract the numbers one by one from the pattern 'cell'
+#Solution:  Add a regexp pattern to the pattern 'cell'.
+table_data = Scrubyt::Extractor.define do
+  #Perform the action(s)
+  fetch File.join(File.dirname(__FILE__), "input.html")
+  #Construct the wrapper
+  table do
+    row :generalize => true do
+      cell '1, 2, 3', :generalize => true do
+        numbers /\d+/
+      end
+    end
+  end
+end
+table_data.to_xml.write($stdout, 1)
+Scrubyt::ResultDumper.print_statistics(table_data)

data/examples/misc/tables/complex_table/input.html ADDED Viewed

@@ -0,0 +1,45 @@
+<html>
+  <body>
+    <table>
+      <tr>
+        <td>The Matrix</td>
+        <td>Neo</td>
+      </tr>
+      <tr>
+        <td>Star Wars</td>
+        <td>Luke Skywalker</td>
+      </tr>
+      <tr>
+        <td>Garfield</td>
+      </tr>
+      <tr>
+        <td>Saw</td>
+        <td>Jigsaw</td>
+      </tr>
+      <tr>
+        <td>Pirates of the Caribbean</td>
+        <td><b>Jack Sparrow<b></td>
+      </tr>
+      <tr>
+        <td>Pirates of the Amazonas</td>
+      </tr>
+      <tr></tr>
+      <tr><td>Starsky & Hutch</td></tr>
+      <tr><h1>The coolest stuff</h1>
+        <td>Test cell</td>
+        <td>
+        <table>
+          <tr>
+            <td>Kill Bill</td>
+            <td>The Bride</td>
+          </tr>
+          <tr>
+            <td>Kill Bill, vol 2.</td>
+            <td>Beatrix Kiddo</td>
+          </tr>
+        </table>
+        </td>
+      </tr>
+    </table>
+  </body>
+</html>

data/examples/misc/tables/complex_table/test.rb ADDED Viewed

@@ -0,0 +1,30 @@
+$:.unshift File.join(File.dirname(__FILE__), '../../../../lib')
+require 'scrubyt'
+#Example of:
+#  - Sepecyfying multiple examples
+#  - Narrowig the results by requiring the presence of some patterns
+#
+#Goal:      We want to extract all movies, containing their title and their
+#           main_character. Moreover, we need only those movies which
+#           have both the title and the main_character specified
+#
+#Solution:  See the ambigous_records example on why to use multiple examples.
+#           Use the ensure_presence_of_pattern constraint to extract only
+#           those movies which have child patterns named 'title' and
+#           'main_character'
+table_data_comp = Scrubyt::Extractor.define do
+  #Perform the action(s)
+  fetch File.join(File.dirname(__FILE__), "input.html")
+  #Construct the wrapper
+    movie do
+      title 'The Matrix', 'Kill Bill'
+      main_character 'Neo', 'The Bride'
+    end.ensure_presence_of_pattern('title').
+        ensure_presence_of_pattern('main_character')
+end
+table_data_comp.to_xml.write($stdout, 1)
+Scrubyt::ResultDumper.print_statistics(table_data_comp)

data/examples/misc/tables/grab_rows/input.html ADDED Viewed

@@ -0,0 +1,20 @@
+<html>
+  <body>
+    <table border=1 padding=10>
+      <tr>
+        <td>1</td>
+        <td>2</td>
+        <td>3</td>
+        <td>4</td>
+        <td>5</td>
+        <td>6</td>
+        <td>7</td>
+        <td>8</td>
+        <td>9</td>
+        <td>10</td>
+        <td>11</td>
+        <td>12</td>
+      </tr>
+    </table>
+  </body>
+</html>