swissparser 0.10.0 → 0.11.0

data/CHANGELOG.rdoc

@@ -1,4 +1,17 @@
-== 0.10.0 / 2009-11-14
+== 0.11.0 / 2009-11-16
+
+* 1 new feature:
+  - parsing rules can call +skip_entry!+ to skip the current entry, shortcutting other rules evaluation and entry finalizing.
+
+* 1 improvement:
+  - parsers raise an error when the last separator is missing
+
+* Code quality:
+  - Split swissparser file in several files
+  - Simplified parse method by extracting code
+  
+
+== 0.10.0 / 2009-11-15
 
 * 1 new feature:
   - SwissParser can now parse an input string or whatever input responding to the +each_line+ method

data/Rakefile

@@ -28,7 +28,7 @@ Bones {
   readme_file 'README.rdoc'
   history_file 'CHANGELOG.rdoc'
   ignore_file  '.gitignore'
-  rdoc.exclude ["examples/data"]
+  rdoc.exclude ["examples/","features/"]
 }
 
 # EOF

data/features/basic_parsing.feature

@@ -0,0 +1,30 @@
+Feature: Basic Parsing
+  I can parse from different sources
+
+  Background:
+   Given input data 
+     """
+     XX a1
+     YY b1
+     c1
+     //
+     XX a2
+     YY b2
+     c2
+     //
+     """
+ 
+  Scenario: Extension without redefinition
+    Given a simple parser
+    When I run the simple parser on data
+    Then the result should be "[{'XX'=>'a1','YY'=>'b1'},{'XX'=>'a2','YY'=>'b2'}]"
+
+  Scenario: Parsing from file
+    Given a simple parser
+    When I run it on file "input.txt"
+    Then File.open should be called with "input.txt"
+ 
+  Scenario: Parsing from URI
+    Given a simple parser
+    When I run it on a remote file "http://www.example.com/input.txt"
+    Then OpenUri.open should be called with "http://www.example.com/input.txt"

data/features/parser_extension.feature

@@ -25,7 +25,8 @@ Feature: Parser Extension
     When I extend it
       And I replace with("XX") to return always 'foo'
       And I replace with("YY") to do nothing
-    Then the parser should return "[{ 'XX' => 'foo'}, { 'XX' => 'foo'}]"   
+      And I run the extended parser on data
+    Then the result should be "[{ 'XX' => 'foo'}, { 'XX' => 'foo'}]"   
  
   Scenario: Text after replacing
     Given a simple parser
@@ -33,9 +34,10 @@ Feature: Parser Extension
       And I replace with("XX") to do nothing
       And I replace with("YY") to return always 'bar'
       And I replace with_text_after("YY") to return always 'foo'
-    Then the parser should return "[{ 'YY' => 'bar', 'txt-YY' => 'foo'}, { 'YY' => 'bar', 'txt-YY' => 'foo'}]" 
+      And I run the extended parser on data
+    Then the result should be "[{ 'YY' => 'bar', 'txt-YY' => 'foo'}, { 'YY' => 'bar', 'txt-YY' => 'foo'}]" 
 
-  Scenario: 
+  Scenario: Separator replacement
     Given a simple parser
       And input data 
         """
@@ -50,4 +52,32 @@ Feature: Parser Extension
         """
      When I extend it
        And I set the separator to '%'
-     Then the parser should return '2' entries
+       And I run the extended parser on data
+     Then the result should contain '2' entries
+
+  Scenario: Using custom entries objects
+    Given a simple parser
+    When I extend it
+      And I replace with("XX") to do nothing
+      And I replace with("YY") to do nothing
+      And I return "foo" in new entry
+      And I run the extended parser on data
+    Then the result should be "['foo','foo']"
+
+  Scenario: Changing the container
+    Given a simple parser
+    When I extend it
+      And I replace the container with a counter
+      And I run the extended parser on data
+    Then the result should be "2"
+
+  Scenario: Changing the entry finalization
+    Given a simple parser
+    When I extend it
+      And entry finalize always returns "foo"
+      And I run the extended parser on data
+    Then the result should be "['foo', 'foo']"
+
+
+
+      

data/features/parsing_context.feature

@@ -0,0 +1,48 @@
+Feature: Sharing context
+  During parsing, rules share a context object.
+
+  Background:
+   Given input data 
+     """
+     XX a1
+     YY b1
+     c1
+     //
+     XX a1
+     YY b2
+     c2
+     //
+     """
+ 
+  Scenario: Helper Method
+    Given a simple parser
+    When I extend it
+      And I define 'foo' helper
+      And I call 'foo' helper in after action
+      And I run the extended parser on data
+    Then the result should be "'foo'"
+
+  Scenario: Parsing Parameters
+    Given a simple parser
+    When I extend it
+      And I return param "foo" in after action
+      And I run the extended parser on data with param "foo" = "bar"
+    Then the result should be "'bar'"
+
+  Scenario: Instance variables
+    Given a simple parser
+    When I extend it
+      And the before action sets @foo="bar"
+      And the after action returns @foo
+      And I run the extended parser on data
+    Then the result should be "'bar'"
+
+  Scenario: Skipping entries
+    Given a simple parser
+    When I extend it
+      And set with("XX") to skip the entry
+      And I run the extended parser on data
+    Then the result should contain '0' entries
+
+
+

data/features/polite.feature

@@ -0,0 +1,16 @@
+Feature: Politeness
+  SwissParser is polite and reporst errors
+
+Scenario: Missing
+  Given input data 
+    """
+    XX a1
+    YY b1
+    c1
+    //
+    XX a2
+    YY b2
+    c2
+    """
+    And a simple parser
+  Then the simple parser should raise an error when parsing data

data/features/step_definitions/core.rb

@@ -0,0 +1,71 @@
+require 'lib/swissparser'
+require 'spec/expectations'
+require 'spec/mocks'
+
+
+Given /^a simple parser$/ do
+  @simple_parser = Swiss::Parser.define do
+    rules do
+      with("XX") {|c,e| e["XX"] = c}
+      with("YY") {|c,e| e["YY"] = c}
+    end
+  end
+end
+
+Given /^input data$/ do |string|
+  @data = string
+end
+
+When /^I extend it$/ do
+  @ext_parser = @simple_parser.extend {}
+end
+
+When /^I run the simple parser on data$/ do
+  @result = @simple_parser.parse(@data)
+end
+
+
+When /^I run the extended parser on data$/ do
+  @result = @ext_parser.parse(@data)
+end
+
+When /^I run the extended parser on data with param "([^\"]*)" = "([^\"]*)"$/ do |key, val|
+   @result = @ext_parser.parse(@data, key => val)
+end
+
+
+When /^I run it on file "([^\"]*)"$/ do |filename|
+  File.stub!(:open).and_return(@data)
+  @result = @simple_parser.parse_file( filename )
+end
+
+
+When /^I run it on a remote file "([^\"]*)"$/ do |arg1|
+  OpenURI.stub!(:open).and_return(@data)
+end
+
+Then /^the extended parser should parse it as the original one$/ do
+  @simple_parser.parse( @data ).should == @ext_parser.parse( @data )
+end
+
+Then /^the result should be "([^\"]*)"$/ do |ruby_exp|
+  result = eval(ruby_exp)
+  @result.should == result
+end
+
+Then /^the result should contain '([^\']*)' entries$/ do |n|
+  @result.size.should == n.to_i
+end
+
+
+Then /^File\.open should be called with "([^\"]*)"$/ do |filename|
+  File.should_receive(:open).with(filename,'w')
+end
+
+Then /^OpenUri\.open should be called with "([^\"]*)"$/ do |filename|
+  OpenURI.should_receive(:open).with(filename)  
+end
+
+Then /^the simple parser should raise an error when parsing data$/ do
+  lambda{@simple_parser.parse(@data)}.should raise_error
+end

data/features/step_definitions/definitions.rb

@@ -0,0 +1,68 @@
+require 'lib/swissparser'
+require 'spec/expectations'
+
+When /^I replace with\("([^\"]*)"\) to return always '([^\']*)'$/ do |key,out|
+  @ext_parser = @ext_parser.extend do
+    rules do
+      with( key ) {|c,e| e[key] = out }
+    end
+  end
+end
+
+When /^I replace with\("([^\"]*)"\) to do nothing$/ do |key|
+  @ext_parser = @ext_parser.extend do 
+    rules do
+      with( key ) {|c,e| }
+    end
+  end
+end
+
+When /^I replace with_text_after\("([^\"]*)"\) to return always '([^\']*)'$/ do |key,out|
+  text_key = "txt-#{key}"
+  @ext_parser = @ext_parser.extend do 
+    rules do
+      with_text_after( key ) {|c,e| e[text_key] = out }
+    end
+  end
+end
+
+When /^I set the separator to '([^\']*)'$/ do |sep|
+  @ext_parser = @ext_parser.extend do 
+    rules do
+      set_separator( sep )
+    end
+  end
+end
+
+When /^I return "([^\"]*)" in new entry$/ do |value|
+  @ext_parser = @ext_parser.extend do
+    new_entry { value }
+  end
+end
+
+When /^I replace the container with a counter$/ do
+  class Counter
+    def initialize
+      @n = 0
+    end
+    def <<(i)
+      @n += 1
+    end
+    def count
+      @n
+    end
+  end
+  @ext_parser = @ext_parser.extend do
+    before { Counter.new }
+    after {|c| c.count }
+  end 
+end
+
+When /^entry finalize always returns "([^\"]*)"$/ do |val|
+   @ext_parser = @ext_parser.extend do
+    finish_entry {|e,c| c << val }
+  end 
+end
+
+
+

data/features/step_definitions/extra.rb

@@ -0,0 +1,47 @@
+require 'lib/swissparser'
+require 'spec/expectations'
+
+When /^I define '([^\']*)' helper$/ do |name|
+  @ext_parser = @ext_parser.extend do 
+    helper(name.to_sym) do
+      name
+    end
+  end
+end
+
+When /^I call '([^\']*)' helper in after action$/ do |name|
+  l = eval("lambda { |x| #{name} }")
+  @ext_parser = @ext_parser.extend do 
+    after(&l)
+  end
+end
+
+When /^I return param "([^\"]*)" in after action$/ do |name|
+  l = eval("lambda { |x| param(#{name}) }")
+  @ext_parser = @ext_parser.extend do 
+    after(&l)
+  end
+
+end
+
+When /^the before action sets @foo="([^\"]*)"$/ do |val|
+   @ext_parser = @ext_parser.extend do 
+    before { @foo=val; [] }
+  end 
+end
+
+When /^the after action returns @foo$/ do
+  @ext_parser = @ext_parser.extend do 
+    after { @foo }
+  end 
+end
+
+When /^set with\("([^\"]*)"\) to skip the entry$/ do |key|
+  @ext_parser = @ext_parser.extend do 
+    rules do
+      with(key) { skip_entry! }
+    end
+  end 
+end
+
+

data/lib/swissparser.rb

@@ -18,90 +18,12 @@ along with SwissParser.  If not, see <http://www.gnu.org/licenses/>.
 =end
 
 require 'open-uri'
+require 'swissparser/parsing_context'
+require 'swissparser/parsing_rules'
 
 module Swiss
 
-  VERSION = "0.10.0"
-
-  # This class defines parsing rules. Its methods
-  # are accessible within the +rules+ section of
-  # a parser definition.
-  class ParsingRules
-    
-    attr_reader :separator, :actions
-    
-    # *Do* *not* create directly this class but access it
-    # through a +rules+ section in a parser definition.
-    def initialize
-      @actions = { :text => {} }
-    end
-    
-    # Sets the entry separator line. Default: "//"
-    def set_separator(string)
-      @separator = string
-    end
-
-    # Defines how to parse a line starting with +key+. The +proc+
-    # takes two arguments:
-    # * the rest of the line
-    # * the entry object
-    def with( key, &proc )
-      @actions[key] = proc
-    end
-
-    # Defines how to parse a line without key coming *after*
-    # a specified key. The +proc+ takes two arguments:
-    # * the rest of the line
-    # * the entry object
-    def with_text_after( key, &proc )
-      @actions[:text][key] = proc
-    end
-    
-  end
-
-  # Methods of this class are accessible to rules and actions.
-  # Methods defined in +helpers+ block are added to this class.
-  class ParsingContext
-
-    def initialize(parameters)
-      @params = parameters
-    end   
-
-    # Retrieves a parsing parameter by key. Returns nil if 
-    # there is no parameter with the provided key.
-    def param( key ) 
-      @params[key]
-    end
-
-
-    module InstanceExecHelper     #:nodoc:
-    end 
-    
-    include InstanceExecHelper
-
-    #Method instance_exec exists since version 1.9
-    if RUBY_VERSION < "1.9"
-      #Used to execute rules and action using the ParsingContext as context
-      #Stolen from http://eigenclass.org/hiki/bounded+space+instance_exec
-      def instance_exec(*args, &block)
-        begin
-          old_critical, Thread.critical = Thread.critical, true
-          n = 0
-          n += 1 while respond_to?(mname="__instance_exec#{n}")
-          InstanceExecHelper.module_eval{ define_method(mname, &block) }
-        ensure
-          Thread.critical = old_critical
-        end
-        begin
-          ret = send(mname, *args)
-        ensure
-          InstanceExecHelper.module_eval{ remove_method(mname) } rescue nil
-        end
-        ret
-      end
-    end
-    
-  end
+  VERSION = "0.11.0"
 
 
   # Parser for a typical bioinformatic flat file.
@@ -182,14 +104,8 @@ module Swiss
       r = ParsingRules.new
       r.instance_eval(&proc)
       r.actions.each do |k,v|
-        if k == :text
-          next
-        end
         @actions[k] = v
       end
-      r.actions[:text].each do |k,v|
-        @actions[:text][k] = v
-      end
       if r.separator
         @separator = r.separator
       end
@@ -197,7 +113,6 @@ module Swiss
     
 
 
-
     # Extends an existing parser by allowing to redefine rules. The
     # changes in the new parser simply replace the original defintions.
     # After extension, the new parser is independent of the original one,
@@ -243,26 +158,30 @@ module Swiss
     # value specified in the +after+ block. By default, it returns an
     # array containing _entry_ objects.
     def parse( data, params={} )
-      @ctx = ParsingContext.new( params )
-      helperModule = Module.new
-      @helpers.each do |name, proc|
-        helperModule.send( :define_method, name, proc )
-      end
-      @ctx.extend( helperModule )
+      @ctx = init_context( params )
+      state = :begin
       container = @ctx.instance_exec( &@before )
       entry = @ctx.instance_exec( &@begin )
       data.each_line do |line|
-        state = parse_line( line, entry )
-        if state == :end
-          @ctx.instance_exec( entry, container, &@end )
-          entry = @ctx.instance_exec( &@begin )
+        if @ctx.should_skip?
+          if line == @separator
+            state = :end
+            entry = init_entry
+          end
+        else
+          state = parse_line( line, entry )
+          if state == :end
+            @ctx.instance_exec( entry, container, &@end )
+            entry = init_entry
+          end
         end
       end
+      if state == :parsing 
+        raise("No separator at end of file")
+      end
       @ctx.instance_exec( container, &@after )
     end
 
-    private
-
     PROTOTYPE = Parser.new
     PROTOTYPE.instance_eval do
       before { || [] }
@@ -270,6 +189,24 @@ module Swiss
       finish_entry {|e,c| c << e }
       after {|c| c }
     end
+    PROTOTYPE.freeze
+
+    private
+
+    def init_entry()
+      @ctx.reset_skip
+      @ctx.instance_exec( &@begin )
+    end
+    
+    def init_context(params)
+      ctx = ParsingContext.new( params )
+      helperModule = Module.new
+      @helpers.each do |name, proc|
+        helperModule.send( :define_method, name, proc )
+      end
+      ctx.extend( helperModule )
+      ctx
+    end
 
     
     def parse_line( line, holder )

data/lib/swissparser/parsing_context.rb

@@ -0,0 +1,60 @@
+module Swiss
+
+  # Methods of this class are accessible to rules and actions.
+  # Methods defined in +helpers+ block are added to this class.
+  class ParsingContext
+
+    def initialize(parameters)
+      @params = parameters
+      @skip = false
+    end   
+
+    # Retrieves a parsing parameter by key. Returns nil if 
+    # there is no parameter with the provided key.
+    def param( key ) 
+      @params[key]
+    end
+
+    def skip_entry!()
+      @skip = true
+    end
+
+    def should_skip?()
+      @skip
+    end
+
+    def reset_skip()
+      @skip = false
+    end
+
+
+    module InstanceExecHelper     #:nodoc:
+    end 
+    
+    include InstanceExecHelper
+
+    #Method instance_exec exists since version 1.9
+    if RUBY_VERSION < "1.9"
+      #Used to execute rules and action using the ParsingContext as context
+      #Stolen from http://eigenclass.org/hiki/bounded+space+instance_exec
+      def instance_exec(*args, &block)
+        begin
+          old_critical, Thread.critical = Thread.critical, true
+          n = 0
+          n += 1 while respond_to?(mname="__instance_exec#{n}")
+          InstanceExecHelper.module_eval{ define_method(mname, &block) }
+        ensure
+          Thread.critical = old_critical
+        end
+        begin
+          ret = send(mname, *args)
+        ensure
+          InstanceExecHelper.module_eval{ remove_method(mname) } rescue nil
+        end
+        ret
+      end
+    end
+    
+  end
+
+end

data/lib/swissparser/parsing_rules.rb

@@ -0,0 +1,39 @@
+module Swiss
+
+  # This class defines parsing rules. Its methods
+  # are accessible within the +rules+ section of
+  # a parser definition.
+  class ParsingRules
+    
+    attr_reader :separator, :actions
+    
+    # *Do* *not* create directly this class but access it
+    # through a +rules+ section in a parser definition.
+    def initialize
+      @actions = { :text => {} }
+    end
+    
+    # Sets the entry separator line. Default: "//"
+    def set_separator(string)
+      @separator = string
+    end
+
+    # Defines how to parse a line starting with +key+. The +proc+
+    # takes two arguments:
+    # * the rest of the line
+    # * the entry object
+    def with( key, &proc )
+      @actions[key] = proc
+    end
+
+    # Defines how to parse a line without key coming *after*
+    # a specified key. The +proc+ takes two arguments:
+    # * the rest of the line
+    # * the entry object
+    def with_text_after( key, &proc )
+      @actions[:text][key] = proc
+    end
+    
+  end
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: swissparser
 version: !ruby/object:Gem::Version 
-  version: 0.10.0
+  version: 0.11.0
 platform: ruby
 authors: 
 - paradigmatic
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-11-15 00:00:00 +01:00
+date: 2009-11-16 00:00:00 +01:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -56,11 +56,17 @@ files:
 - examples/tutorial_1.rb
 - examples/tutorial_2.rb
 - examples/uniprot_param_demo.rb
+- features/basic_parsing.feature
 - features/parser_extension.feature
-- features/step_definitions/steps.rb
-- features/user_friendly.feature
+- features/parsing_context.feature
+- features/polite.feature
+- features/step_definitions/core.rb
+- features/step_definitions/definitions.rb
+- features/step_definitions/extra.rb
 - lib/swiss_parser.rb
 - lib/swissparser.rb
+- lib/swissparser/parsing_context.rb
+- lib/swissparser/parsing_rules.rb
 has_rdoc: true
 homepage: http://github.com/paradigmatic/SwissParser
 licenses: []

data/features/step_definitions/steps.rb

@@ -1,103 +0,0 @@
-require 'lib/swissparser'
-require 'spec/expectations'
-
-
-Given /^a simple parser$/ do
-  @simple_parser = Swiss::Parser.define do
-    rules do
-      with("XX") {|c,e| e["XX"] = c}
-      with("YY") {|c,e| e["YY"] = c}
-    end
-  end
-end
-
-Given /^input data$/ do |string|
-  @data = string
-end
-
-When /^I extend it$/ do
-  @ext_parser = @simple_parser.extend {}
-end
-
-When /^I replace with\("([^\"]*)"\) to return always '([^\']*)'$/ do |key,out|
-  @ext_parser = @ext_parser.extend do
-    rules do
-      with( key ) {|c,e| e[key] = out }
-    end
-  end
-end
-
-When /^I replace with\("([^\"]*)"\) to do nothing$/ do |key|
-  @ext_parser = @ext_parser.extend do 
-    rules do
-      with( key ) {|c,e| }
-    end
-  end
-end
-
-When /^I replace with_text_after\("([^\"]*)"\) to return always '([^\']*)'$/ do |key,out|
-  text_key = "txt-#{key}"
-  @ext_parser = @ext_parser.extend do 
-    rules do
-      with_text_after( key ) {|c,e| e[text_key] = out }
-    end
-  end
-end
-
-When /^I set the separator to '([^\']*)'$/ do |sep|
-  @ext_parser = @ext_parser.extend do 
-    rules do
-      set_separator( sep )
-    end
-  end
-end
-
-When /^I define '([^\']*)' helper$/ do |name|
-  @ext_parser = @ext_parser.extend do 
-    helper(name.to_sym) do
-      name
-    end
-  end
-end
-
-When /^I call '([^\']*)' helper in after action$/ do |name|
-  l = eval("lambda { |x| #{name} }")
-  @ext_parser = @ext_parser.extend do 
-    after(&l)
-  end
-end
-
-When /^I return param "([^\"]*)" in after action$/ do |name|
-  l = eval("lambda { |x| param(#{name}) }")
-  @ext_parser = @ext_parser.extend do 
-    after(&l)
-  end
-
-end
-
-When /^I call parse with param "([^\"]*)" equal to "([^\"]*)"$/ do |key, val|
-  @result = @ext_parser.parse(@data, key => val)
-end
-
-Then /^the result should be "([^\"]*)"$/ do |val|
-  @result.should == val
-end
-
-
-Then /^the parser should return '([^\']*)'$/ do |val|
-  @ext_parser.parse( @data ).should == val
-end
-
-Then /^the parser should return '(\d*)' entries$/ do |num|
-  @ext_parser.parse( @data  ).size.should == num.to_i
-end
-
-Then /^the extended parser should parse it as the original one$/ do
-  @simple_parser.parse( @data ).should == @ext_parser.parse( @data )
-end
-
-
-Then /^the parser should return "([^\"]*)"$/ do |ruby_exp|
-  result = eval(ruby_exp)
-  @ext_parser.parse( @data ).should == result
-end

data/features/user_friendly.feature

@@ -1,29 +0,0 @@
-Feature: User Friendly
-  SwissParser is nice to use.
-
-  Background:
-   Given input data 
-     """
-     XX a1
-     YY b1
-     c1
-     //
-     XX a1
-     YY b2
-     c2
-     //
-     """
- 
-  Scenario: Helper Method
-    Given a simple parser
-    When I extend it
-      And I define 'foo' helper
-      And I call 'foo' helper in after action
-    Then the parser should return 'foo'
-
-  Scenario: Helper Method
-    Given a simple parser
-    When I extend it
-      And I return param "foo" in after action
-      And I call parse with param "foo" equal to "bar"
-    Then the result should be "bar"