re2 0.2.0 → 0.3.0

data/README.md

@@ -1,4 +1,4 @@
-re2
+re2 [![Build Status](https://secure.travis-ci.org/mudge/re2.png?branch=master)](http://travis-ci.org/mudge/re2)
 ===
 
 A Ruby binding to [re2][], an "efficient, principled regular expression library".
@@ -6,7 +6,13 @@ A Ruby binding to [re2][], an "efficient, principled regular expression library"
 Installation
 ------------
 
-You will need [re2][] installed as well as a C++ compiler such as [gcc][] (on Debian and Ubuntu, this is provided by the [build-essential][] package).
+You will need [re2][] installed as well as a C++ compiler such as [gcc][] (on Debian and Ubuntu, this is provided by the [build-essential][] package). If you are using Mac OS X, I recommend installing re2 with [Homebrew][] by running the following:
+
+    $ brew install --HEAD re2
+
+If you are using Debian, you can install the [libre2-dev][] package like so:
+
+    $ sudo apt-get install libre2-dev
 
 If you are using a packaged Ruby distribution, make sure you also have the Ruby header files installed such as those provided by the [ruby-dev][] package on Debian and Ubuntu.
 
@@ -15,7 +21,9 @@ You can then install the library via RubyGems with `gem install re2` or `gem ins
 Documentation
 -------------
 
-Full documentation automatically generated from the latest version is available at <http://rdoc.info/projects/mudge/re2> and <http://yardoc.org/docs/mudge-re2/>.
+Full documentation automatically generated from the latest version is available at <http://rubydoc.info/github/mudge/re2>.
+
+Bear in mind that re2's regular expression syntax differs from PCRE, see the [official syntax page][] for more details.
 
 Usage
 -----
@@ -39,20 +47,33 @@ You can use re2 as a mostly drop-in replacement for Ruby's own [Regexp][] and [M
     > r.match("bob")
      => nil
 
+As of 0.3.0, you can use named groups:
+
+    > r = RE2::Regexp.compile('(?P<name>\w+) (?P<age>\d+)')
+     => #<RE2::Regexp /(?P<name>\w+) (?P<age>\d+)/>
+    > m = r.match("Bob 40")
+     => #<RE2::MatchData "Bob 40" 1:"Bob" 2:"40">
+    > m[:name]
+     => "Bob"
+    > m["age"]
+     => "40"
+
 Features
 --------
 
 * Pre-compiling regular expressions with [`RE2::Regexp.new(re)`](http://code.google.com/p/re2/source/browse/re2/re2.h#96), `RE2::Regexp.compile(re)` or `RE2(re)` (including specifying options, e.g. `RE2::Regexp.new("pattern", :case_sensitive => false)`
 
-  * Extracting matches with `re2.match(text)` (and an exact number of matches with `re2.match(text, number_of_matches)` such as `re2.match("123-234", 2)`)
+* Extracting matches with `re2.match(text)` (and an exact number of matches with `re2.match(text, number_of_matches)` such as `re2.match("123-234", 2)`)
+
+* Extracting matches by name (both with strings and symbols)
 
-  * Checking for matches with `re2 =~ text`, `re2 === text` (for use in `case` statements) and `re2 !~ text`
+* Checking for matches with `re2 =~ text`, `re2 === text` (for use in `case` statements) and `re2 !~ text`
 
-  * Checking regular expression compilation with `re2.ok?`, `re2.error` and `re2.error_arg`
+* Checking regular expression compilation with `re2.ok?`, `re2.error` and `re2.error_arg`
 
-  * Checking regular expression "cost" with `re2.program_size`
+* Checking regular expression "cost" with `re2.program_size`
 
-  * Checking the options for an expression with `re2.options` or individually with `re2.case_sensitive?`
+* Checking the options for an expression with `re2.options` or individually with `re2.case_sensitive?`
 
 * Performing in-place replacement with [`RE2::Replace(str, pattern, replace)`](http://code.google.com/p/re2/source/browse/re2/re2.h#335)
 
@@ -71,3 +92,6 @@ All feedback should go to the mailing list: <mailto:ruby.re2@librelist.com>
   [build-essential]: http://packages.debian.org/build-essential
   [Regexp]: http://ruby-doc.org/core/classes/Regexp.html
   [MatchData]: http://ruby-doc.org/core/classes/MatchData.html 
+  [Homebrew]: http://mxcl.github.com/homebrew
+  [libre2-dev]: http://packages.debian.org/search?keywords=libre2-dev
+  [official syntax page]: http://code.google.com/p/re2/wiki/Syntax

data/ext/re2/extconf.rb

@@ -12,6 +12,33 @@ $CFLAGS << " -Wall -Wextra -funroll-loops"
 
 have_library("stdc++")
 if have_library("re2")
+
+  # Determine which version of re2 the user has installed.
+  # Revision d9f8806c004d added an `endpos` argument to the
+  # generic Match() function.
+  #
+  # To test for this, try to compile a simple program that uses
+  # the newer form of Match() and set a flag if it is successful.
+  checking_for("RE2::Match() with endpos argument") do
+    test_re2_match_signature = <<SRC
+#include <re2/re2.h>
+
+int main() {
+  RE2 pattern("test");
+  re2::StringPiece *match;
+  pattern.Match("test", 0, 0, RE2::UNANCHORED, match, 0);
+
+  return 0;
+}
+SRC
+
+    # Pass -x c++ to force gcc to compile the test program
+    # as C++ (as it will end in .c by default).
+    if try_compile(test_re2_match_signature, "-x c++")
+      $defs.push("-DHAVE_ENDPOS_ARGUMENT")
+    end
+  end
+
   create_makefile("re2")
 else
   abort "You must have re2 installed and specified with --with-re2-dir, please see http://code.google.com/p/re2/wiki/Install"

data/ext/re2/re2.cc

@@ -7,6 +7,8 @@
  */
 
 #include <re2/re2.h>
+#include <string>
+using namespace std;
 
 extern "C" {
 
@@ -30,7 +32,7 @@ extern "C" {
   typedef struct {
     re2::StringPiece *matches;
     int number_of_matches;
-    VALUE regexp, string;
+    VALUE regexp, text;
   } re2_matchdata;
 
   VALUE re2_mRE2, re2_cRegexp, re2_cMatchData;
@@ -43,7 +45,7 @@ extern "C" {
   void re2_matchdata_mark(re2_matchdata* self)
   {
     rb_gc_mark(self->regexp);
-    rb_gc_mark(self->string);
+    rb_gc_mark(self->text);
   }
 
   void re2_matchdata_free(re2_matchdata* self)
@@ -84,7 +86,7 @@ extern "C" {
     re2_matchdata *m;
     Data_Get_Struct(self, re2_matchdata, m);
 
-    return m->string;
+    return m->text;
   }
 
   /*
@@ -141,13 +143,16 @@ extern "C" {
   {
     int i;
     re2_matchdata *m;
+    re2::StringPiece match;
+
     Data_Get_Struct(self, re2_matchdata, m);
     VALUE array = rb_ary_new2(m->number_of_matches);
     for (i = 0; i < m->number_of_matches; i++) {
       if (m->matches[i].empty()) {
         rb_ary_push(array, Qnil);
       } else {
-        rb_ary_push(array, rb_str_new(m->matches[i].data(), m->matches[i].size()));
+        match = m->matches[i];
+        rb_ary_push(array, rb_str_new(match.data(), match.size()));
       }
     }
     return array;
@@ -157,17 +162,42 @@ extern "C" {
   re2_matchdata_nth_match(int nth, VALUE self)
   {
     re2_matchdata *m;
+    re2::StringPiece match;
+
     Data_Get_Struct(self, re2_matchdata, m);
+    match = m->matches[nth];
 
-    if (nth >= m->number_of_matches || m->matches[nth].empty()) {
+    if (nth >= m->number_of_matches || match.empty()) {
       return Qnil;
     } else {
-      return rb_str_new(m->matches[nth].data(), m->matches[nth].size());
+      return rb_str_new(match.data(), match.size());
+    }
+  }
+
+  static VALUE
+  re2_matchdata_named_match(const char* name, VALUE self)
+  {
+    int idx;
+    re2_matchdata *m;
+    re2_pattern *p;
+    map<string, int> groups;
+    string name_as_string(name);
+
+    Data_Get_Struct(self, re2_matchdata, m);
+    Data_Get_Struct(m->regexp, re2_pattern, p);
+
+    groups = p->pattern->NamedCapturingGroups();
+
+    if (groups.count(name_as_string) == 1) {
+      idx = groups[name_as_string];
+      return re2_matchdata_nth_match(idx, self);
+    } else {
+      return Qnil;
     }
   }
 
   /*
-   * Retrieve zero, one or more matches by index.
+   * Retrieve zero, one or more matches by index or name.
    *
    * @return [Array<String, nil>, String, Boolean]
    *
@@ -198,6 +228,16 @@ extern "C" {
    *   @example
    *     m = RE2('(\d+)').match("bob 123")
    *     m[0..1]    #=> "[123", "123"]
+   *
+   * @overload [](name)
+   *   Access a particular match by name.
+   *
+   *   @param [String, Symbol] name the name of the match to fetch
+   *   @return [String, nil] the specific match
+   *   @example
+   *     m = RE2('(?P<number>\d+)').match("bob 123")
+   *     m["number"] #=> "123"
+   *     m[:number]  #=> "123"
    */
   static VALUE
   re2_matchdata_aref(int argc, VALUE *argv, VALUE self)
@@ -205,7 +245,11 @@ extern "C" {
     VALUE idx, rest;
     rb_scan_args(argc, argv, "11", &idx, &rest);
 
-    if (!NIL_P(rest) || !FIXNUM_P(idx) || FIX2INT(idx) < 0) {
+    if (TYPE(idx) == T_STRING) {
+      return re2_matchdata_named_match(StringValuePtr(idx), self);
+    } else if (TYPE(idx) == T_SYMBOL) {
+      return re2_matchdata_named_match(rb_id2name(SYM2ID(idx)), self);
+    } else if (!NIL_P(rest) || !FIXNUM_P(idx) || FIX2INT(idx) < 0) {
       return rb_ary_aref(argc, argv, re2_matchdata_to_a(self));
     } else {
       return re2_matchdata_nth_match(FIX2INT(idx), self);
@@ -391,9 +435,9 @@ extern "C" {
         re2_options.set_one_line(RTEST(one_line));
       }
 
-      p->pattern = new (std::nothrow) RE2(StringValuePtr(pattern), re2_options);
+      p->pattern = new (nothrow) RE2(StringValuePtr(pattern), re2_options);
     } else {
-      p->pattern = new (std::nothrow) RE2(StringValuePtr(pattern));
+      p->pattern = new (nothrow) RE2(StringValuePtr(pattern));
     }
 
     if (p->pattern == 0) {
@@ -773,6 +817,32 @@ extern "C" {
     return INT2FIX(p->pattern->NumberOfCapturingGroups());
   }
 
+  /*
+   * Returns a hash of names to capturing indices of groups.
+   *
+   * @return [Hash] a hash of names to capturing indices
+   */
+  static VALUE
+  re2_regexp_named_capturing_groups(VALUE self)
+  {
+    VALUE capturing_groups;
+    re2_pattern *p;
+    map<string, int> groups;
+    map<string, int>::iterator iterator;
+
+    Data_Get_Struct(self, re2_pattern, p);
+    groups = p->pattern->NamedCapturingGroups();
+    capturing_groups = rb_hash_new();
+
+    for (iterator = groups.begin(); iterator != groups.end(); iterator++) {
+      rb_hash_aset(capturing_groups,
+          rb_str_new(iterator->first.data(), iterator->first.size()),
+          INT2FIX(iterator->second));
+    }
+
+    return capturing_groups;
+  }
+
   /*
    * Match the pattern against the given +text+ and return either
    * a boolean (if no submatches are required) or a {RE2::MatchData}
@@ -836,10 +906,12 @@ extern "C" {
       n = p->pattern->NumberOfCapturingGroups();
     }
 
-    re2::StringPiece text_as_string_piece(StringValuePtr(text));
-
     if (n == 0) {
-      matched = p->pattern->Match(text_as_string_piece, 0, (int)RSTRING_LEN(text), RE2::UNANCHORED, 0, 0);
+#if defined(HAVE_ENDPOS_ARGUMENT)
+      matched = p->pattern->Match(StringValuePtr(text), 0, (int)RSTRING_LEN(text), RE2::UNANCHORED, 0, 0);
+#else
+      matched = p->pattern->Match(StringValuePtr(text), 0, RE2::UNANCHORED, 0, 0);
+#endif
       return BOOL2RUBY(matched);
     } else {
 
@@ -848,10 +920,10 @@ extern "C" {
 
       matchdata = rb_class_new_instance(0, 0, re2_cMatchData);
       Data_Get_Struct(matchdata, re2_matchdata, m);
-      m->matches = new (std::nothrow) re2::StringPiece[n];
+      m->matches = new (nothrow) re2::StringPiece[n];
       m->regexp = self;
-      m->string = rb_str_dup(text);
-      rb_str_freeze(m->string);
+      m->text = rb_str_dup(text);
+      rb_str_freeze(m->text);
 
       if (m->matches == 0) {
         rb_raise(rb_eNoMemError, "not enough memory to allocate StringPieces for matches");
@@ -859,7 +931,11 @@ extern "C" {
 
       m->number_of_matches = n;
 
-      matched = p->pattern->Match(text_as_string_piece, 0, (int)RSTRING_LEN(text), RE2::UNANCHORED, m->matches, n);
+#if defined(HAVE_ENDPOS_ARGUMENT)
+      matched = p->pattern->Match(StringValuePtr(text), 0, (int)RSTRING_LEN(text), RE2::UNANCHORED, m->matches, n);
+#else
+      matched = p->pattern->Match(StringValuePtr(text), 0, RE2::UNANCHORED, m->matches, n);
+#endif
 
       if (matched) {
         return matchdata;
@@ -912,19 +988,18 @@ extern "C" {
     re2_pattern *p;
 
     /* Convert all the inputs to be pumped into RE2::Replace. */
-    std::string str_as_string(StringValuePtr(str));
-    re2::StringPiece rewrite_as_string_piece(StringValuePtr(rewrite));
+    string str_as_string(StringValuePtr(str));
 
     /* Do the replacement. */
     if (rb_obj_is_kind_of(pattern, re2_cRegexp)) {
       Data_Get_Struct(pattern, re2_pattern, p);
-      RE2::Replace(&str_as_string, *p->pattern, rewrite_as_string_piece);
+      RE2::Replace(&str_as_string, *p->pattern, StringValuePtr(rewrite));
     } else {
-      RE2::Replace(&str_as_string, StringValuePtr(pattern), rewrite_as_string_piece);
+      RE2::Replace(&str_as_string, StringValuePtr(pattern), StringValuePtr(rewrite));
     }
 
     /* Save the replacement as a VALUE. */
-    repl = rb_str_new(str_as_string.c_str(), str_as_string.length());
+    repl = rb_str_new(str_as_string.data(), str_as_string.size());
 
     /* Replace the original string with the replacement. */
     if (RSTRING_LEN(str) != RSTRING_LEN(repl)) {
@@ -961,20 +1036,19 @@ extern "C" {
 
     /* Convert all the inputs to be pumped into RE2::GlobalReplace. */
     re2_pattern *p;
-    std::string str_as_string(StringValuePtr(str));
-    re2::StringPiece rewrite_as_string_piece(StringValuePtr(rewrite));
+    string str_as_string(StringValuePtr(str));
     VALUE repl;
 
     /* Do the replacement. */
     if (rb_obj_is_kind_of(pattern, re2_cRegexp)) {
       Data_Get_Struct(pattern, re2_pattern, p);
-      RE2::GlobalReplace(&str_as_string, *p->pattern, rewrite_as_string_piece);
+      RE2::GlobalReplace(&str_as_string, *p->pattern, StringValuePtr(rewrite));
     } else {
-      RE2::GlobalReplace(&str_as_string, StringValuePtr(pattern), rewrite_as_string_piece);
+      RE2::GlobalReplace(&str_as_string, StringValuePtr(pattern), StringValuePtr(rewrite));
     }
 
     /* Save the replacement as a VALUE. */
-    repl = rb_str_new(str_as_string.c_str(), str_as_string.length());
+    repl = rb_str_new(str_as_string.data(), str_as_string.size());
 
     /* Replace the original string with the replacement. */
     if (RSTRING_LEN(str) != RSTRING_LEN(repl)) {
@@ -999,8 +1073,7 @@ extern "C" {
   re2_QuoteMeta(VALUE self, VALUE unquoted)
   {
     UNUSED(self);
-    re2::StringPiece unquoted_as_string_piece(StringValuePtr(unquoted));
-    std::string quoted_string = RE2::QuoteMeta(unquoted_as_string_piece);
+    string quoted_string = RE2::QuoteMeta(StringValuePtr(unquoted));
     return rb_str_new(quoted_string.data(), quoted_string.size());
   }
 
@@ -1030,6 +1103,7 @@ extern "C" {
     rb_define_method(re2_cRegexp, "program_size", RUBY_METHOD_FUNC(re2_regexp_program_size), 0);
     rb_define_method(re2_cRegexp, "options", RUBY_METHOD_FUNC(re2_regexp_options), 0);
     rb_define_method(re2_cRegexp, "number_of_capturing_groups", RUBY_METHOD_FUNC(re2_regexp_number_of_capturing_groups), 0);
+    rb_define_method(re2_cRegexp, "named_capturing_groups", RUBY_METHOD_FUNC(re2_regexp_named_capturing_groups), 0);
     rb_define_method(re2_cRegexp, "match", RUBY_METHOD_FUNC(re2_regexp_match), -1);
     rb_define_method(re2_cRegexp, "match?", RUBY_METHOD_FUNC(re2_regexp_match_query), 1);
     rb_define_method(re2_cRegexp, "=~", RUBY_METHOD_FUNC(re2_regexp_match_query), 1);

data/test/re2_test.rb

@@ -121,11 +121,44 @@ class RE2Test < Test::Unit::TestCase
     assert_equal 2, RE2('a((b)c)').number_of_capturing_groups
   end
 
+  def test_named_capturing_groups
+    assert_equal(1, RE2('(?P<bob>a)').named_capturing_groups["bob"])
+    assert_equal(1, RE2('(?P<bob>a)(o)(?P<rob>e)').named_capturing_groups["bob"])
+    assert_equal(3, RE2('(?P<bob>a)(o)(?P<rob>e)').named_capturing_groups["rob"])
+  end
+
   def test_matching_all_subpatterns
     assert_equal ["woo", "o", "o"], RE2('w(o)(o)').match('woo').to_a
     assert_equal ["ab", nil, "a", "b"], RE2('(\d?)(a)(b)').match('ab').to_a
   end
 
+  def test_fetching_matchdata_out_of_range
+    matchdata = RE2('(\d+)').match('bob 123')
+    assert_nil matchdata[2]
+    assert_nil matchdata[3]
+  end
+
+  def test_accessing_matches_by_name
+    matchdata = RE2('(?P<numbers>\d+)').match("bob 123")
+    assert_equal "123", matchdata["numbers"]
+    assert_equal "123", matchdata[:numbers]
+  end
+
+  def test_accessing_matches_by_name_with_multiple_groups
+    matchdata = RE2('(?P<name>\w+)(\s*)(?P<numbers>\d+)').match("bob 123")
+    assert_equal "bob", matchdata["name"]
+    assert_equal "bob", matchdata[:name]
+    assert_equal " ", matchdata[2]
+    assert_equal "123", matchdata["numbers"]
+    assert_equal "123", matchdata[:numbers]
+  end
+
+  def test_accessing_matches_by_incorrect_names
+    matchdata = RE2('(?P<numbers>\d+)').match("bob 123")
+    assert_nil matchdata["missing"]
+    assert_nil matchdata[:missing]
+  end
+
   def test_matchdata
     r = RE2('(\d+)')
     text = "bob 123"
@@ -146,6 +179,7 @@ class RE2Test < Test::Unit::TestCase
     assert_equal "123", m.to_s
     assert_equal "123", m[0]
     assert_equal "123", m[1]
+    assert_nil m[4]
     assert_equal ["123"], m[0, 1]
     assert_equal ["123", "123"], m[0, 2]
     assert_equal ["123"], m[0...1]

metadata

@@ -1,83 +1,64 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: re2
-version: !ruby/object:Gem::Version 
-  prerelease: false
-  segments: 
-  - 0
-  - 2
-  - 0
-  version: 0.2.0
+version: !ruby/object:Gem::Version
+  version: 0.3.0
+  prerelease: 
 platform: ruby
-authors: 
+authors:
 - Paul Mucur
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2011-04-11 00:00:00 +01:00
-default_executable: 
-dependencies: 
-- !ruby/object:Gem::Dependency 
+date: 2012-02-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: rake-compiler
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
+  requirement: &70320711075760 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 0
-        version: "0"
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
-  version_requirements: *id001
+  prerelease: false
+  version_requirements: *70320711075760
 description: Ruby bindings to re2, "an efficient, principled regular expression library".
 email: ruby.re2@librelist.com
 executables: []
-
-extensions: 
+extensions:
 - ext/re2/extconf.rb
 extra_rdoc_files: []
-
-files: 
+files:
 - ext/re2/extconf.rb
 - ext/re2/re2.cc
 - LICENSE.txt
 - README.md
 - Rakefile
 - test/re2_test.rb
-- test/leak.rb
-has_rdoc: true
 homepage: http://github.com/mudge/re2
 licenses: []
-
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      segments: 
-      - 0
-      version: "0"
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: 
-rubygems_version: 1.3.7
+rubygems_version: 1.8.11
 signing_key: 
 specification_version: 3
 summary: Ruby bindings to re2.
-test_files: 
+test_files:
 - test/re2_test.rb
-- test/leak.rb
+has_rdoc: 

data/test/leak.rb

@@ -1,32 +0,0 @@
-$:.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
-require 're2'
-
-loops = if ENV["LOOPS"]
-  ENV["LOOPS"].to_i
-else
-  150
-end
-
-puts "Looping #{loops} times..."
-
-loops.times do
-  r = RE2::Regexp.new('woo(oo)(o+)(\d*)')
-  r.ok?
-  r.match('wooo')
-  r.match('wooo', 0)
-  r.match('wooo', 1)
-  r.match('wooo', 2)
-  r =~ 'woooo'
-  r =~ 'bob'
-  RE2::Replace("woo", "woo", "bob")
-  RE2::GlobalReplace("woo", "woo", "bob")
-  m = r.match('woooooo1234')
-  m[0]
-  m[1]
-  m[0,2]
-  m[1..-1]
-  m.string
-  m.regexp
-  m.inspect
-  sleep 0.1
-end