scrappy 0.3.5 → 0.4.0

data/public/javascripts/annotator.js

@@ -53,8 +53,8 @@ jQuery(document).ready(function(){
   } else {
     div = "<div id='scrappy_window' title='Scrappy'>" +
           "<p>No extractor available for this URL</p>" +
-          "<p><a href='TODO'>Annotate page</a></p>" +
           "<p><a class='extractor' href='http://localhost:3434/extractors'>Generate extractor</a></p>" +
+          "<p><a class='sample' href='http://localhost:3434/samples'>Upload sample</a></p>" +
           "</div>";
   }
   

data/public/stylesheets/application.css

@@ -9,6 +9,17 @@ pre {
   border: 1px solid;
   padding: 10px;
 }
+pre.wide {
+  width: 770px;
+  max-height: 900px;
+  margin-left: auto;
+  margin-right: auto;
+  border: 1px solid;
+  font-size: 12px;
+  overflow: scroll;
+  padding: 10px;
+  font-family: monospace;
+}
 a:link, a:visited {
   color: #33f;
   text-decoration: none;
@@ -152,6 +163,9 @@ ul.detail li {
   background-color: #eee;
   margin: 1px;
 }
+ul.detail li.special {
+  background-color: #fff;
+}
 ul.detail li span {
   display: inline-block;
 }
@@ -174,6 +188,9 @@ ul.detail li span.format {
   margin-left: 10px;
   text-align: center;
 }
+ul.detail li.special span.format {
+  font-weight: normal;
+}
 ul.detail li span.date {
   float: right;
 }
@@ -196,3 +213,19 @@ ul.detail li span.date a:visited,
 ul.detail li span.date a:active {
   color: #888;
 }
+
+fieldset {
+  border:  0;
+  padding: 0;
+  margin:  0;
+  vertical-align: baseline;
+}
+
+textarea {
+  width: 100%;
+}
+
+span.type {
+  float: left;
+  width: 80px;
+}

data/scrappy.gemspec

@@ -2,11 +2,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{scrappy}
-  s.version = "0.3.5"
+  s.version = "0.4.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
   s.authors = ["Jose Ignacio"]
-  s.date = %q{2011-03-29}
+  s.date = %q{2011-06-30}
   s.default_executable = %q{scrappy}
   s.description = %q{RDF web scraper}
   s.email = %q{joseignacio.fernandez@gmail.com}
@@ -31,7 +31,7 @@ Gem::Specification.new do |s|
       s.add_runtime_dependency(%q<thin>, [">= 1.2.7"])
       s.add_runtime_dependency(%q<nokogiri>, [">= 1.4.1"])
       s.add_runtime_dependency(%q<mechanize>, [">= 1.0.0"])
-      s.add_runtime_dependency(%q<lightrdf>, [">= 0.3.7"])
+      s.add_runtime_dependency(%q<lightrdf>, [">= 0.3.9"])
       s.add_runtime_dependency(%q<i18n>, [">= 0.4.2"])
       s.add_runtime_dependency(%q<rest-client>, [">= 1.6.1"])
       s.add_runtime_dependency(%q<haml>, [">= 3.0.24"])
@@ -42,7 +42,7 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<thin>, [">= 1.2.7"])
       s.add_dependency(%q<nokogiri>, [">= 1.4.1"])
       s.add_dependency(%q<mechanize>, [">= 1.0.0"])
-      s.add_dependency(%q<lightrdf>, [">= 0.3.7"])
+      s.add_dependency(%q<lightrdf>, [">= 0.3.9"])
       s.add_dependency(%q<i18n>, [">= 0.4.2"])
       s.add_dependency(%q<rest-client>, [">= 1.6.1"])
       s.add_dependency(%q<haml>, [">= 3.0.24"])
@@ -54,7 +54,7 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<thin>, [">= 1.2.7"])
     s.add_dependency(%q<nokogiri>, [">= 1.4.1"])
     s.add_dependency(%q<mechanize>, [">= 1.0.0"])
-    s.add_dependency(%q<lightrdf>, [">= 0.3.7"])
+    s.add_dependency(%q<lightrdf>, [">= 0.3.9"])
     s.add_dependency(%q<i18n>, [">= 0.4.2"])
     s.add_dependency(%q<rest-client>, [">= 1.6.1"])
     s.add_dependency(%q<haml>, [">= 3.0.24"])

data/views/help.haml

@@ -16,5 +16,4 @@
     Drag this to your bookmarks:
     %a.bookmark{:href=>bookmark_js, :onclick=>drag_js} Scrappy
   %p
-    Then visit the web page you want to build a extractor for.
-    Click on your "Scrappy" bookmark and annotate the web page.
+    Then visit the web page you want to build a extractor for and click on your "Scrappy" bookmark.

data/views/layout.haml

@@ -4,6 +4,7 @@
     %title Scrappy
     %link{:type=>"text/css", :href=>"#{settings.base_uri}/stylesheets/application.css", :rel=>"stylesheet"}
     %script{:src=>"https://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"}
+    %script{:src=>"#{settings.base_uri}/javascripts/utils.js"}
     %script{:src=>"#{settings.base_uri}/javascripts/remote.js"}
   %body
     #bar

data/views/patterns.haml

@@ -3,17 +3,22 @@
   %p
     Patterns are visual conditions that are used to identify data in sites which do not have a defined extractor.
   %p
-    -if @uris.empty?
+    -if @patterns.empty?
       Currently, there are no patterns.
     -else
       %ul.detail
-        -@uris.each do |uri|
+        -@patterns.each do |pattern|
+          -uri = pattern.sc::type.first.to_s
           %li
             %span.action
-              %a{:href=>"#{settings.base_uri}/patterns/#{CGI::escape(uri)}", :'data-method'=>:delete, :'data-confirm'=>"Are you sure you want to delete the pattern for #{uri}?"}
+              %a{:href=>"#{settings.base_uri}/patterns/#{CGI::escape(pattern.to_s)}", :'data-method'=>:delete, :'data-confirm'=>"Are you sure you want to delete this pattern?"}
                 X
             %span.name
               -if !uri.include?('*')
-                %a{:href=>uri}=uri
+                %a{:href=>"#{settings.base_uri}/patterns/#{CGI::escape(pattern.to_s)}"}=uri
               -else
-                =uri
+                =uri
+      %p
+        %a{:href=>"#{settings.base_uri}/patterns/visual"} See patterns visually
+        |
+        %a{:href=>"#{settings.base_uri}/patterns", :'data-method'=>:delete, :'data-confirm'=>"Are you sure you want to delete all the patterns?"} Delete all patterns

data/views/samples.haml

@@ -3,28 +3,52 @@
   %p
     Sample pages are used to build extractors as well as visual patterns that can be applied to retrieve data
     from other pages.
-  %p
-    -if @samples.empty?
-      Currently, there are no samples.
-    -else
-      %ul.detail
-        -@samples.each_with_index do |sample,i|
-          %li
+  %form{:method=>:post}
+    %p
+      -if @samples.empty?
+        Currently, there are no samples.
+      -else
+        %ul.detail
+          %li.special
             %span.action
-              %a{:href=>"#{settings.base_uri}/samples/#{i}", :'data-method'=>:delete, :'data-confirm'=>"Are you sure you want to delete the sample #{sample[:uri]}?"}
-                X
-            %span.short_name
-              -if !sample[:uri].include?('*')
-                %a{:href=>sample[:uri]}=sample[:uri]
-              -else
-                =sample[:uri]
-            -[['Patterns output', :patterns], ['Extractors output', :extractors]].reverse.each do |text, action|
-              %span.format
-                %a{:href=>"#{settings.base_uri}/samples/#{i}/#{action}"}=text
             %span.format
-              %a{:href=>"#{settings.base_uri}/samples/#{i}/optimize", :'data-method'=>:post} Optimize
+              %input.checkall{:type=>:checkbox}
             %span.format
-              %a{:href=>"#{settings.base_uri}/samples/#{i}/train", :'data-method'=>:post} Train
-            %span.date
-              %a{:href=>"#{settings.base_uri}/samples/#{i}"}
-                =sample[:date].strftime("%Y/%m/%d - %H:%M")
+              Select all
+          -@samples.each_with_index do |sample,i|
+            %li
+              %span.action
+                %a{:href=>"#{settings.base_uri}/samples/#{i}", :'data-method'=>:delete, :'data-confirm'=>"Are you sure you want to delete the sample #{sample[:uri]}?"}
+                  X
+              %span.short_name
+                -if !sample[:uri].include?('*')
+                  %a{:href=>sample[:uri]}=sample[:uri]
+                -else
+                  =sample[:uri]
+              %span.format
+                %input{:type=>:checkbox, :name=>'samples[]', :value=>i}
+              -[['Patterns', :patterns], ['Extractors', :extractors], ['Annotations', :annotations]].reverse.each do |text, action|
+                %span.format
+                  %a{:href=>"#{settings.base_uri}/samples/#{i}/#{action}"}=text
+              %span.format
+                %a{:href=>"#{settings.base_uri}/samples/#{i}/raw"} RAW
+              %span.date
+                %a{:href=>"#{settings.base_uri}/samples/#{i}"}
+                  =sample[:date].strftime("%Y/%m/%d - %H:%M")
+    %p
+      %span.type General:
+      %a.checksend{:href=>"#{settings.base_uri}/samples/annotate", :title=>'This will store extractors output as the correct samples output'} Annotate
+    %p
+      %span.type Extractors:
+      %a.checksend{:href=>"#{settings.base_uri}/samples/train/extractors", :title=>'This will generate extractors for each of the selected samples'} Train
+      |
+      %a.checksend{:href=>"#{settings.base_uri}/samples/optimize/extractors", :title=>'This will generalize extractors to improve the performance on the selected samples'} Optimize
+      |
+      %a.checksend{:href=>"#{settings.base_uri}/samples/test/extractors", :title=>'This will test extractors on the selected samples'} Test
+    %p
+      %span.type Patterns:
+      %a.checksend{:href=>"#{settings.base_uri}/samples/train/patterns", :title=>'This will generate patterns for each of the selected samples'} Train
+      |
+      %a.checksend{:href=>"#{settings.base_uri}/samples/optimize/patterns", :title=>'This will generalize patterns to improve the performance on the selected samples'} Optimize
+      |
+      %a.checksend{:href=>"#{settings.base_uri}/samples/test/patterns", :title=>'This will test patterns on the selected samples'} Test

metadata

@@ -4,9 +4,9 @@ version: !ruby/object:Gem::Version
   prerelease: false
   segments: 
   - 0
-  - 3
-  - 5
-  version: 0.3.5
+  - 4
+  - 0
+  version: 0.4.0
 platform: ruby
 authors: 
 - Jose Ignacio
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-03-29 00:00:00 +02:00
+date: 2011-06-30 00:00:00 +02:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -97,8 +97,8 @@ dependencies:
         segments: 
         - 0
         - 3
-        - 7
-        version: 0.3.7
+        - 9
+        version: 0.3.9
   type: :runtime
   version_requirements: *id006
 - !ruby/object:Gem::Dependency