charles 0.0.1 → 0.0.2

data/Gemfile

@@ -2,3 +2,5 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in charles.gemspec
 gemspec
+
+gem 'shiner', :path => '/u/apps/shiner/dev'

data/bin/charles

@@ -14,6 +14,7 @@ end
 document = Charles.get(url)
 puts({
   :content => document.content,
+  :interesting_content => document.interesting_content,
   :title => document.title,
   :filtered_images => document.filtered_images.collect{|image| image[:url]}
 }.to_yaml)

data/charles.gemspec

@@ -22,4 +22,5 @@ Gem::Specification.new do |gem|
   gem.add_dependency "activesupport"
   gem.add_dependency "rack"
   gem.add_dependency "imagesize"
+  gem.add_dependency "shiner"
 end

data/lib/charles.rb

@@ -9,6 +9,7 @@ require 'mechanize'
 require 'active_support/cache'
 require 'active_support/cache/file_store'
 require 'image_size'
+require 'shiner'
 
 require 'ferret'
 Ferret.locale = "en_US.UTF-8" #if not set ferret segfaults on chinese/jap stuff randomly

data/lib/charles/document.rb

@@ -17,6 +17,10 @@ module Charles
     
     def logger; Charles.logger; end
       
+    def interesting_content(options = {:max_length => 388})
+      Shiner.shine(content, options)
+    end
+      
     def content(seeds={})
       content_node = content_node(seeds)
       return unless content_node
@@ -30,12 +34,12 @@ module Charles
     end
     
     def calculate_content_nodes(seeds={})
-      default_seeds = {:title_match=>0.145422959269808,
-  :title_match_buffer=>0.0174920023610796,
-  :length=>1100.27450832379,
-  :distance_from_top=>0.308408501217311,
-  :internal_nodes=>25.680381972181,
-  :internal_nodes_buffer=>20.2006169153009}
+      default_seeds = {:title_match=>0.0586074856962615, #0.238237272128463,0.173173520342878
+  :title_match_buffer=>0.508671373602233,
+  :length=>1246.27917099503,
+  :distance_from_top=>0.436005480844439,
+  :internal_nodes=>18.0265463704097,
+  :internal_nodes_buffer=>32.7588984705223}
       seeds = default_seeds.merge(seeds)
       
       o = []
@@ -49,7 +53,8 @@ module Charles
           :length => 1 - seeds[:length].to_f / (_n.clean_inner_tokens_text.size + seeds[:length]), #length of inner text in this node, too little = less
           :internal_nodes => seeds[:internal_nodes].to_f / (_n.internal_nodes_size + seeds[:internal_nodes] + seeds[:internal_nodes_buffer]), #number of nodes in this node, too many = less
           :distance_from_top => (1-(_rank.to_f / @nodes.size))**seeds[:distance_from_top].to_f, #how far this element is from the top of the page
-          :title_match => ((content_node_ferret_index[_i]||0.0 + seeds[:title_match_buffer]) / 1+ + seeds[:title_match_buffer])**seeds[:title_match].to_f #ferret index score, search score with page title
+          :title_match => ((content_node_ferret_index[_i]||0.0 + seeds[:title_match_buffer]) / 1 + seeds[:title_match_buffer])**seeds[:title_match].to_f, #ferret index score, search score with page title
+          #:interesting => (0.5 + _n.interesting_score) ** seeds[:interesting].to_f
           #:special_characters => (1 - (_n.inner_text.scan(/[^\s\302\240a-zA-Z]/).size.to_f / (_n.clean_inner_text.size+1)))**2 #number of special characters and numbers.. this is pretty cpu intensive!
         }
         o << {:node =>_n, :score => scores.values.inject(:*), :scores => scores}
@@ -143,6 +148,12 @@ Nokogiri::XML::Node.class_eval {
   def internal_nodes_size
     @internal_nodes_size ||= search('*').size
   end
+  #def interesting_score
+  #  @interesting_score ||= (
+  #    classifications = Shiner.classifier.classifications(clean_inner_text)
+  #    1 - classifications['Interesting'] / classifications['Uninteresting']
+  #  )
+  #end
 }
 
 

data/lib/charles/images.rb

@@ -11,7 +11,7 @@ module Charles
       return unless _node
       #logger.info _node.pretty_inspect
   
-      (_node.ancestors.size/2).times do
+      (_node.ancestors.size.to_f/2).round.times do
         o=self.calculate_image_from_node(_node)
         #logger.info o.pretty_inspect
         return o if o

data/lib/charles/version.rb

@@ -1,3 +1,3 @@
 module Charles
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: charles
 version: !ruby/object:Gem::Version 
-  hash: 29
+  hash: 27
   prerelease: 
   segments: 
   - 0
   - 0
-  - 1
-  version: 0.0.1
+  - 2
+  version: 0.0.2
 platform: ruby
 authors: 
 - Jason Ling Xiaowei
@@ -115,6 +115,20 @@ dependencies:
         version: "0"
   type: :runtime
   version_requirements: *id007
+- !ruby/object:Gem::Dependency 
+  name: shiner
+  prerelease: false
+  requirement: &id008 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id008
 description: Charles the Content Extractor
 email: 
 - jason@jeyel.com