oniguruma 0.9.1 → 1.0.0

data/History.txt

@@ -1,3 +1,34 @@
+== 1.0.0 / 2007-03-27
+* Added documentation for MatchData.
+* Added ogsub, ogsub!, sub and sub! to ::String.
+* Removed ::String definitions from tests.
+* Now the minimal recommended version of oniglib is 5.5 or higher.
+* Removed ugly #if statements from c code.
+* Do not create @named_captures hash if there are no named groups for regexp -- somewhat improve speed for repetive calls
+* Fixed usage of named backreferences in gsub with non-ascii names
+* Move ORegexp#=~ to C code, make it work just like Regexp#=~, i.e. set $~. Throw ArgumentError instead of Exception if pattern does not compile
+* Fix implementation of ORegexp#===, so it now does not raise errors in case statement anymore
+  (resembles plain Ruby Regexp#=== behaviour)
+* Modified begin, end and offset methods in MatchData to handle named groups and default to group 0.
+* Exception is not longer thrown when in oregexp_make_match_data.
+* Removed references to MultiMatchData from documentation
+* Removed class MultiMatchData
+* Fix off by one error in region->num_regs usage
+* Fix dumb bug with zero-width matches that made infinite loops. now consume at least one char in gsub and scan
+* ORegexp API changes:
+  * Pass only MatchData to sub/gsub with blocks
+      oregexp.sub( str ) {|match_data| ... }
+      oregexp.gsub( str ) {|match_data| ... }
+  * Add ORegexp#scan instead of match_all
+      oregexp.scan(str) {|match_data| ... } # => MultiMatchData
+  * Friendly way to set options 
+      ORegexp.new( pattern, options_str, encoding, syntax)
+      ORegexp.new('\w+', 'imsx', 'koi8r', 'perl')
+  *  Named backreferences in substitions 
+      ORegexp.new('(?<pre>\w+)\d+(?<after>\w+)').sub('abc123def', '\<after>123\<pre>')  #=> 'def123abc'
+* couple of bugfixes with region's num_regs
+* some  docs for substitution methods added
+
 == 0.9.1 / 2007-03-25
 * FIX: Buggy resolution of numeric codes for encoding and syntax options (Nikolai Lugovoi)
 * FIX: Buggy implementation of ORegexp#gsub and  ORegexp#gsub methods. Now code is all C (Nikolai Lugovoi)

data/README.txt

@@ -8,6 +8,7 @@ Ruby bindings to the Oniguruma[http://www.geocities.jp/kosako3/oniguruma/] regul
 * Same interface than standard Regexp class (easy transition!). 
 * Support for named groups, look-ahead, look-behind, and other 
   cool features!
+* Support for other regexp syntaxes (Perl, Python, Java, etc.)
 
 == SYNOPSIS:
 
@@ -23,7 +24,7 @@ Consult the Syntax.txt[link:files/Syntax_txt.html] page.
 
 == REQUIREMENTS:
 
-* Oniguruma[http://www.geocities.jp/kosako3/oniguruma/] library v. 2.0 or greater
+* Oniguruma[http://www.geocities.jp/kosako3/oniguruma/] library v. 5.5 or higher
 
 == INSTALL:
 
@@ -43,7 +44,7 @@ sudo gem install -r oniguruma
 
 == CREDITS:
 
-* N. Lugovoi. ORegexp.sub and ORegexp.gsub code, plus other patches.
+* N. Lugovoi. ORegexp.sub and ORegexp.gsub code and lots of other stuff.
 * K. Kosako. For his great library.
 * A lot of the documentation has been copied from the original Ruby Regex documentation.
 

data/Rakefile

@@ -3,7 +3,7 @@ require 'hoe'
 
 class Hoe; def extra_deps; @extra_deps.reject { |x| Array(x).first == 'hoe' }; end end
 
-Hoe.new('oniguruma', '0.9.1') do |p|
+Hoe.new('oniguruma', '1.0.0') do |p|
   p.rubyforge_name = 'oniguruma'
   p.author = 'Dizan Vasquez'
   p.email = 'dix_ans@yahoo.com'

data/ext/oregexp.c

@@ -101,15 +101,11 @@ static int name_callback(
    regex_t* reg, 
    struct callback_packet* arg
 ) {
-   int i, gn, ref;
-   OnigRegion *region = arg->region;
+   int i, gn;
    VALUE nameHash = arg->hash;
 
    for (i = 0; i < ngroup_num; i++) {
       gn = group_nums[i];
-      ref = onig_name_to_backref_number(reg, name, name_end, region);
-      if (ref != gn )
-        return 1; 
       rb_hash_aset( nameHash, ID2SYM(rb_intern(name)), INT2FIX( gn ) );
   }
   return 0;  
@@ -124,10 +120,6 @@ static VALUE oregexp_initialize( VALUE self, VALUE pattern, VALUE options ) {
    rb_iv_set( self, "@options", options );
    UChar* pat_ptr = RSTRING(pattern_str)->ptr;
    int pat_len = RSTRING(pattern_str)->len;
-   if( pat_len == 0 ) {
-      rb_raise(rb_eArgError, "Empty pattern makes no sense.");
-   }
-
    VALUE rOptions = rb_hash_aref( options, ID2SYM( rb_intern( "options" ) ) );
    VALUE rEncoding = rb_hash_aref( options, ID2SYM( rb_intern( "encoding" ) ) );
    VALUE rSyntax = rb_hash_aref( options, ID2SYM( rb_intern( "syntax" ) ) );
@@ -142,16 +134,19 @@ static VALUE oregexp_initialize( VALUE self, VALUE pattern, VALUE options ) {
    if (r != ONIG_NORMAL) {
       char s[ONIG_MAX_ERROR_MESSAGE_LEN];
       onig_error_code_to_str(s, r, &einfo);
-      rb_raise(rb_eException, "Oniguruma Error: %s", s);
+      rb_raise(rb_eArgError, "Oniguruma Error: %s", s);
    }
    return self;
 }
 
+/* can't include re.h, since it conflicts with oniguruma typedefs */
 struct RMatch {
     struct RBasic basic;
     VALUE str;
     struct re_registers *regs;
 };
+#define RMATCH(obj)  (R_CAST(RMatch)(obj))
+void rb_match_busy _((VALUE));
 
 static VALUE oregexp_make_match_data(ORegexp * oregexp, OnigRegion * region, VALUE string_str) {
     VALUE rb_cMatch = rb_const_get(rb_cObject, rb_intern("MatchData")) ;
@@ -163,21 +158,22 @@ static VALUE oregexp_make_match_data(ORegexp * oregexp, OnigRegion * region, VAL
 
     match->str = rb_str_new4(string_str);
     match->regs = ALLOC(struct re_registers);
-    match->regs->allocated = count+1;
+    match->regs->allocated = count;
     match->regs->num_regs = count;
-    match->regs->beg = ALLOC_N(int, (count+1));
-    match->regs->end = ALLOC_N(int, (count+1));
+    match->regs->beg = ALLOC_N(int, count);
+    match->regs->end = ALLOC_N(int, count);
 
-    for ( i = 0; i <= count; i++){
+    for ( i = 0; i < count; i++){
         match->regs->beg[i] = region->beg[i];
         match->regs->end[i] = region->end[i];
     }
     rb_cv_set( kORegexp, "@@last_match", (VALUE)match );
     packet.region = region;
-    packet.hash = rb_hash_new();
-    if( onig_foreach_name(oregexp->reg, name_callback, &packet) )
-        rb_raise(rb_eException, "Oniguruma Error: group and backreference names are different");
-    rb_iv_set((VALUE)match, "@named_captures", packet.hash);
+    if( onig_number_of_names( oregexp->reg ) > 0 ) {
+        packet.hash = rb_hash_new();
+        onig_foreach_name(oregexp->reg, name_callback, &packet);
+        rb_iv_set((VALUE)match, "@named_captures", packet.hash);
+    }
     return (VALUE)match;
 }
  
@@ -201,9 +197,12 @@ static VALUE oregexp_match( VALUE self, VALUE string ) {
 
    OnigRegion *region = onig_region_new();
    int r = onig_search(oregexp->reg, str_ptr, str_ptr + str_len, str_ptr, str_ptr + str_len, region, ONIG_OPTION_NONE);
+   rb_backref_set(Qnil);
    if (r >= 0) {
       VALUE matchData = oregexp_make_match_data( oregexp, region, string_str);
       onig_region_free(region, 1 );
+      rb_backref_set(matchData);
+      rb_match_busy(matchData);
       return matchData;
    } else if (r == ONIG_MISMATCH) {
       onig_region_free(region, 1 );
@@ -212,7 +211,7 @@ static VALUE oregexp_match( VALUE self, VALUE string ) {
       onig_region_free(region, 1 );
       char s[ONIG_MAX_ERROR_MESSAGE_LEN];
       onig_error_code_to_str(s, r);
-      rb_raise(rb_eException, "Oniguruma Error: %s", s);
+      rb_raise(rb_eArgError, "Oniguruma Error: %s", s);
    }
 
 }
@@ -233,7 +232,7 @@ oregexp_get_replacement(pat, src_text, repl_text, region)
 {
     ORegexp 	*oregexp;
     VALUE 	ret;
-    int32_t  replIdx = 0;
+    int32_t  replIdx = 0, name_pos, name_start, name_end ;
     int32_t  replacementLength = RSTRING(repl_text)->len;
     UChar    *replacementText = RSTRING(repl_text)->ptr;
     UChar    *replacementEnd  = replacementText + (replacementLength-1);
@@ -254,6 +253,10 @@ oregexp_get_replacement(pat, src_text, repl_text, region)
     while (replIdx < replacementLength) {
         OnigCodePoint c = ONIGENC_MBC_TO_CODE(enc, replacementText+replIdx, replacementEnd);
 	int c_len =ONIGENC_MBC_ENC_LEN(enc, replacementText+replIdx) ; 
+        if( c_len == 0 ) {
+          rb_warn("Strange, for %d enc_len is 0", c);
+          c_len = 1;
+        }
         replIdx += c_len;
         if ( c != BACKSLASH) {
             /* Common case, no substitution, no escaping,  */
@@ -311,7 +314,7 @@ oregexp_get_replacement(pat, src_text, repl_text, region)
                                 break;
                         case '+': // last matched group
                                 replIdx += c_len;
-                                for(groupNum = region->num_regs; groupNum > 0; groupNum --) {
+                                for(groupNum = region->num_regs-1; groupNum > 0; groupNum --) {
                                         g_start = region->beg[ groupNum ];
                                         g_end   = region->end[ groupNum ];
                                         if( g_start != -1  ) {
@@ -320,7 +323,35 @@ oregexp_get_replacement(pat, src_text, repl_text, region)
                                         }
                                 }
                                 break;
-                                
+			case '<': // named group references \<name>
+				name_pos = replIdx+c_len;
+                                name_end = name_start = replIdx+c_len;
+				while(name_pos < replacementLength) {
+				   c = ONIGENC_MBC_TO_CODE(enc, replacementText+name_pos, replacementEnd);
+				   c_len = ONIGENC_MBC_ENC_LEN(enc, replacementText+name_pos) ; 
+				   name_pos += c_len;
+				   if( c == '>')  break;
+				   if( ONIGENC_IS_CODE_WORD(enc, c) ) {
+					name_end += c_len;
+				   } else {
+				        break;
+				   }
+				}
+				if( c != '>' || name_end == name_start ) {
+				   // place backslash and '<'
+				   rb_str_buf_cat(ret, replacementText+(replIdx-p_len), p_len+c_len);
+				   replIdx += c_len;
+				} else {
+				   // lookup for group and subst for that value
+				   groupNum = onig_name_to_backref_number( oregexp->reg, 
+                                        replacementText+name_start, replacementText+name_end, region);
+				   if( groupNum >= 0 ) {
+				   	rb_str_buf_cat(ret, matchText+region->beg[groupNum], 
+						   region->end[groupNum]-region->beg[groupNum]);
+				   }
+				   replIdx = name_pos;
+				}
+				break;
                         default:
                                 rb_str_buf_cat(ret, replacementText+(replIdx-p_len), p_len+c_len);
                                 replIdx += c_len;
@@ -328,7 +359,7 @@ oregexp_get_replacement(pat, src_text, repl_text, region)
                    }
              } else {
                      /* Finally, append the capture group data to the destination. */
-		     if( groupNum < region->num_regs && region->beg[groupNum] >= 0 && region->end[groupNum]>= region->beg[groupNum] ) {
+		     if( groupNum < region->num_regs && region->beg[groupNum] >= 0 ) {
 	     		rb_str_buf_cat(ret, matchText+region->beg[groupNum], region->end[groupNum]-region->beg[groupNum]);
 		     }
         }             
@@ -359,13 +390,15 @@ oregexp_gsub(self, argc, argv,  bang, once, region)
     VALUE           repl;
     long            beg,
                     end,
+                    len,
                     prev_end;
     int             tainted = 0,
 		    iter = 0;
     
     VALUE buf, curr_repl,  block_res;
     ORegexp *oregexp;
-    
+    OnigEncoding enc;
+
     if (argc == 1 && rb_block_given_p()) {
 	iter = 1;
     } else if (argc == 2) {
@@ -392,6 +425,7 @@ oregexp_gsub(self, argc, argv,  bang, once, region)
     }
     end = 0;
     buf = rb_str_buf_new(str_len);
+    enc = onig_get_encoding( oregexp->reg );
     do {
 	prev_end = end;
 	beg = region->beg[0];
@@ -400,12 +434,8 @@ oregexp_gsub(self, argc, argv,  bang, once, region)
 	if ( iter ) {
 	    VALUE match_data = oregexp_make_match_data( oregexp, region, string_str );
             rb_backref_set(match_data);
-            if( once ) 
-                block_res = rb_yield( match_data );
-            else {
-                VALUE match_string = rb_str_new( str_ptr+beg, end-beg);
-                block_res = rb_yield_values(2, match_string, match_data );
-            }
+            rb_match_busy(match_data);
+            block_res = rb_yield( match_data );
 	    str_mod_check( string_str, str_ptr, str_len);
             curr_repl = rb_obj_as_string(block_res); 
 	} else {
@@ -414,6 +444,17 @@ oregexp_gsub(self, argc, argv,  bang, once, region)
 	rb_str_append(buf, curr_repl);
 	if( once ) break;
 	// find next match
+        if( end == beg) { 
+           /*
+            * Always consume at least one character of the input string
+            * in order to prevent infinite loops.
+            */
+           if( str_len <= end )
+               break;
+           len = ONIGENC_MBC_ENC_LEN(enc, str_ptr + end);
+           rb_str_buf_cat( buf, str_ptr+end, len);
+           end += len;
+        }
 	beg=onig_search(oregexp->reg, str_ptr, str_ptr + str_len, 
 				      str_ptr+end, str_ptr + str_len, 
 				      region, ONIG_OPTION_NONE);
@@ -456,28 +497,216 @@ static VALUE oregexp_safe_gsub(self, argc, argv,  bang, once)
         gsub_packet call_args = {self, argc, argv, bang, once, region};
         return rb_ensure( oregexp_packed_gsub, (VALUE)&call_args, oregexp_cleanup_region, (VALUE)region);
 }
+
+/**
+ * call-seq:
+ *     rxp.gsub(str, replacement)
+ *     rxp.gsub(str) {|match_data| ... }
+ *     
+ * Returns a copy of _str_ with _all_ occurrences of _rxp_ pattern
+ * replaced with either _replacement_ or the value of the block. 
+ *
+ * If a string is used as the replacement, the sequences \1, \2, 
+ * and so on may be used to interpolate successive groups in the match.
+ *
+ * In the block form, the current MatchData object is passed in as a
+ * parameter. The value returned by the block will be substituted for
+ * the match on each call.
+ *
+ **/
 static VALUE oregexp_m_gsub(int argc, VALUE *argv, VALUE self) {
 	return oregexp_safe_gsub(self, argc, argv, 0, 0);
 }
+
+/**
+ * call-seq:
+ *     rxp.sub(str, replacement)
+ *     rxp.sub(str) {|match_data| ... }
+ *
+ * Returns a copy of _str_ with the _first_ occurrence of _rxp_ pattern
+ * replaced with either _replacement_ or the value of the block. 
+ *
+ * If a string is used as the replacement, the sequences \1, \2, 
+ * and so on may be used to interpolate successive groups in the match.
+ *
+ * In the block form, the current MatchData object is passed in as a
+ * parameter. The value returned by the block will be substituted for
+ * the match on each call.
+ *     
+ **/
 static VALUE oregexp_m_sub(int argc, VALUE *argv, VALUE self) {
 	return oregexp_safe_gsub(self, argc, argv, 0, 1);
 }
 
+/**
+ * call-seq:
+ *     rxp.gsub!(str, replacement)
+ *     rxp.gsub!(str) {|match_data| ... }
+ *     
+ * Performs the substitutions of ORegexp#gsub in place, returning
+ * _str_, or _nil_ if no substitutions were performed.
+ *
+ **/
 static VALUE oregexp_m_gsub_bang(int argc, VALUE *argv, VALUE self) {
 	return oregexp_safe_gsub(self, argc, argv, 1, 0);
 }
+
+/**
+ * call-seq:
+ *     oregexp.sub!(str, replacement)
+ *     oregexp.sub!(str) {|match_data| ... }
+ *
+ * Performs the substitutions of ORegexp#sub in place, returning
+ * _str_, or _nil_ if no substitutions were performed.
+ *
+ **/
 static VALUE oregexp_m_sub_bang(int argc, VALUE *argv, VALUE self) {
 	return oregexp_safe_gsub(self, argc, argv, 1, 1);
 }
 
+static VALUE
+oregexp_scan(VALUE self, VALUE str, OnigRegion * region)
+{
+    long            beg,
+                    len,
+                    end;
+    int             iter = 0;
+    
+    VALUE           matches;
+    ORegexp        *oregexp;
+    OnigEncoding    enc;
+    
+    if ( rb_block_given_p()) {
+	iter = 1;
+    } 
+    Data_Get_Struct( self, ORegexp, oregexp );
+
+    VALUE string_str = StringValue( str );
+    UChar* str_ptr = RSTRING(string_str)->ptr;
+    int str_len = RSTRING(string_str)->len;
+    beg = onig_search(oregexp->reg, str_ptr, str_ptr + str_len, str_ptr, str_ptr + str_len, region, ONIG_OPTION_NONE);
+    if (beg < 0) {
+	/* no match */
+	return Qnil;
+    }
+    matches = rb_ary_new();
+    enc = onig_get_encoding( oregexp -> reg );
+    do {
+        VALUE match_data = oregexp_make_match_data( oregexp, region, string_str );
+	end = region->end[0];
+	rb_ary_push( matches, match_data );
+	if ( iter ) 
+		rb_yield( match_data );
+	// find next match
+        if( end == beg) { 
+           /*
+            * Always consume at least one character of the input string
+            * in order to prevent infinite loops.
+            */
+           if( str_len <= end )
+               break;
+           len = ONIGENC_MBC_ENC_LEN(enc, str_ptr + end);
+           end += len;
+        }
+
+	beg=onig_search(oregexp->reg, str_ptr, str_ptr + str_len, 
+				      str_ptr+end, str_ptr + str_len, 
+				      region, ONIG_OPTION_NONE);
+    } while ( beg >= 0);
+    
+    return matches;
+}
+
+struct scan_packet {
+    VALUE self, str;
+    OnigRegion * region;
+};
+static VALUE oregexp_packed_scan( struct scan_packet * args) {
+    return oregexp_scan(args->self, args->str, args->region);
+}
+/**
+ * call-seq:
+ *     rxp.scan(str)                        # => [matchdata1, matchdata2,...] or nil
+ *     rxp.scan(str) {|match_data| ... }    # => [matchdata1, matchdata2,...] or nil
+ *
+ * Both forms iterate through _str_, matching the pattern. For each match, 
+ * a MatchData object is generated and passed to the block, and
+ * added to the resulting array of MatchData objects.
+ *
+ * If _str_ does not match pattern, _nil_ is returned.
+ *
+ **/
+static VALUE oregexp_m_scan(VALUE self, VALUE str) {
+    OnigRegion * region = onig_region_new();
+    struct scan_packet call_args = {self, str, region};
+    return rb_ensure( oregexp_packed_scan, (VALUE)&call_args, oregexp_cleanup_region, (VALUE)region);
+}
+
+/**
+ * call-seq:
+ *     rxp === str   => true or false
+ *    
+ * Case Equality---Synonym for <code>ORegexp#=~</code> used in case statements.
+ *    
+ *    a = "HELLO"
+ *    case a
+ *    when ORegexp.new('^[a-z]*$'); print "Lower case\n"
+ *    when ORegexp.new('^[A-Z]*$'); print "Upper case\n"
+ *    else;                         print "Mixed case\n"
+ *    end
+ *    
+ * <em>produces:</em>
+ *    
+ *    Upper case      
+ *
+ **/
+ 
+static VALUE oregexp_m_eqq(VALUE self, VALUE str) {
+    VALUE match;
+
+    if (TYPE(str) != T_STRING) {
+	str = rb_check_string_type(str);
+	if (NIL_P(str)) {
+	    return Qfalse;
+	}
+    }
+    StringValue(str);
+    match = oregexp_match(self, str);
+    if (Qnil == match) {
+	return Qfalse;
+    }
+    return Qtrue;
+}
+/*
+* call-seq:
+*    rxp =~ string  => int or nil
+*
+* Matches <code>rxp</code> against <code>string</code>, returning the offset of the 
+* start of the match or <code>nil</code> if the match failed. Sets $~ to the corresponding 
+* <code>MatchData</code> or <code>nil</code>.
+*
+*    ORegexp.new( 'SIT' ) =~ "insensitive"                                 #=>    nil
+*    ORegexp.new( 'SIT', :options => OPTION_IGNORECASE ) =~ "insensitive"  #=>    5
+**/ 
+static VALUE oregexp_match_op(VALUE self, VALUE str) {
+   VALUE ret = oregexp_match(self, str);
+   if(ret == Qnil)
+      return Qnil;
+   return INT2FIX(RMATCH(ret)->regs->beg[0]);
+}
+
 void Init_oregexp() {
    mOniguruma = rb_define_module("Oniguruma");
    VALUE cORegexp = rb_define_class_under(mOniguruma, "ORegexp", rb_cObject);
    rb_define_alloc_func(cORegexp, oregexp_allocate);
    rb_define_method( cORegexp, "initialize", oregexp_initialize, 2 );
    rb_define_method( cORegexp, "match", oregexp_match, 1 );
+   rb_define_method( cORegexp, "=~", oregexp_match_op, 1 );
    rb_define_method( cORegexp, "gsub", oregexp_m_gsub, -1 );
    rb_define_method( cORegexp, "sub",  oregexp_m_sub,  -1 );
    rb_define_method( cORegexp, "gsub!", oregexp_m_gsub_bang, -1 );
    rb_define_method( cORegexp, "sub!",  oregexp_m_sub_bang,  -1 );
+   rb_define_method( cORegexp, "scan",  oregexp_m_scan,  1 );
+   rb_define_method( cORegexp, "===",  oregexp_m_eqq,  1 );
+   rb_define_const( mOniguruma, "VERSION", rb_str_new2(onig_version()) );
 }

data/lib/oniguruma.rb

@@ -17,6 +17,20 @@ module Oniguruma
    OPTION_MAXBIT               = OPTION_POSIX_REGION
    OPTION_DEFAULT              = OPTION_NONE
 
+   OPTIONS_SHORTCUTS = {
+    'i' => OPTION_IGNORECASE,
+    'x' => OPTION_EXTEND,
+    'm' => OPTION_MULTILINE,
+    's' => OPTION_SINGLELINE,
+    'l' => OPTION_FIND_LONGEST,
+    'E' => OPTION_FIND_NOT_EMPTY,
+    'S' => OPTION_NEGATE_SINGLELINE,
+    'G' => OPTION_DONT_CAPTURE_GROUP,
+    'g' => OPTION_CAPTURE_GROUP,
+    'B' => OPTION_NOTBOL,
+    'E' => OPTION_NOTEOL,
+   }
+
    SYNTAX_ASIS                 = 0
    SYNTAX_POSIX_BASIC          = 1
    SYNTAX_POSIX_EXTENDED       = 2
@@ -117,8 +131,12 @@ module Oniguruma
       alias old_initialize initialize 
       # :startdoc:
       
+      # call-seq:
+      #     ORegexp.new( pattern, options_hash )
+      #     ORegexp.new( pattern, option_str, encoding_str=nil, syntax_str=nil) 
+      #
       # Constructs a new regular expression from <i>pattern</i>, which is a 
-      # <code>String</code>. The paramter <i>options</i> is a <code>Hash</code> 
+      # <code>String</code>. The second parameter <i></i> may be a <code>Hash</code> 
       # of the form:
       #
       # <code>{ :options => option_value, :encoding => encoding_value, :syntax => syntax_value }</code>
@@ -135,9 +153,27 @@ module Oniguruma
       #
       #     #Accept java syntax on SJIS encoding:
       #     r4 = ORegexp.new('ape', :syntax  => SYNTAX_JAVA, :encoding => ENCODING_SJIS) #=> /ape/ 
+      #    
+      # Second form uses string shortcuts to set options and encoding:
+      #     r = ORegexp.new('cat', 'i', 'utf8', 'java')
       
-      def initialize( pattern, options = {} )
+      def initialize( pattern, *args )
          defaults = { :options => OPTION_DEFAULT, :encoding => ENCODING_ASCII, :syntax => SYNTAX_DEFAULT}
+	 if args[0].is_a?(String)
+           options = {}
+	   option_str, encoding_str, syntax_str = *args
+	   opt = 0
+	   option_str.each_byte {|x| opt |= (OPTIONS_SHORTCUTS[x.chr] || 0) }
+	   options[:options] = opt
+	   if encoding_str && Oniguruma::const_defined?("ENCODING_#{encoding_str.upcase}")
+             options[:encoding] = Oniguruma::const_get("ENCODING_#{encoding_str.upcase}")
+	   end
+	   if syntax_str && Oniguruma::const_defined?("SYNTAX_#{syntax_str.upcase}")
+             options[:syntax] = Oniguruma::const_get("SYNTAX_#{syntax_str.upcase}")
+	   end
+	 else
+           options = args[0] || {}
+	 end
          old_initialize( pattern,  defaults.merge( options ).freeze )
       end
       
@@ -241,131 +277,203 @@ module Oniguruma
       end
       
       # call-seq:
-      #    rxp =~ string  => int or nil
-      #
-      # Matches <code>rxp</code> against <code>string</code>, returning the offset of the 
-      # start of the match or <code>nil</code> if the match failed. Sets $~ to the corresponding 
-      # <code>MatchData</code> or <code>nil</code>.
-      #
-      #    ORegexp.new( 'SIT' ) =~ "insensitive"                                 #=>    nil
-      #    ORegexp.new( 'SIT', :options => OPTION_IGNORECASE ) =~ "insensitive"  #=>    5
-      
-      def =~ string
-         return nil unless string
-         m = match( string )
-         return nil unless m
-         m.begin(0)
-      end
-      
-      # call-seq:
-      #    rxp === str   => true or false
+      #    rxp.source   => str
       # 
-      # Case Equality---Synonym for <code>Regexp#=~</code> used in case statements.
-      #    
-      #    a = "HELLO"
-      #    case a
-      #    when ORegexp.new('^[a-z]*$'); print "Lower case\n"
-      #    when ORegexp.new('^[A-Z]*$'); print "Upper case\n"
-      #    else;                         print "Mixed case\n"
-      #    end
-      #    
-      # <em>produces:</em>
-      #    
-      #    Upper case      
-      
-      alias === =~
-      
+      # Returns the original string of the pattern.
+      # 
+      #    ORegex.new( 'ab+c', 'ix' ).source   #=> "ab+c"
       def source
          @pattern.freeze
       end
       
-      def match_all string
-         matches = []
-         positions = []
-         position = 0
-         tmp_string = string
-         while tmp_string != ""
-            if m = match( tmp_string )
-               matches << m
-               positions << position
-               tmp_string = m.post_match
-               position += m.end(0)
-               #if m.end == m.begin
-               #   tmp_string = tmp_string[1..-1]
-               #   position += 1
-               #end
-            else
-               break
-            end
-         end
-         if matches.size > 0
-            MultiMatchData.new( string, matches, positions )
-         else
-            nil
-         end
-      end
+      alias match_all scan
+      
    end
    
-   class MultiMatchData
-      def initialize( string, matches, positions )
-         @matches = matches
-         @positions = positions
-         @string = string
-      end
-      
-      def position index
-         @positions[index]
-      end
-      
-      def [] ( value1, value2 = nil )
-         unless value2
-            @matches[value1]
-         else
-            @matches[value1, value2]
-         end
-      end
-      
-      def begin index
-         @matches[index].begin(0) + @positions[index]
-      end
-      
-      def end index
-         @matches[index].end(0) + @positions[index]
-      end
-      
-      def length
-         @matches.size
-      end
-      alias size length
-      
-      def offset index
-         [self.begin(index), self.end(index) ]
-      end
-      
-      def string
-         @string.freeze
-      end
-      
-      def to_a
-         @matches
-      end
-      
-      def each
-         @matches.size.times do |i|
-            yield @matches[i], @positions[i]
-         end
-      end
+end
+
+class ::String
+   # Calls <code>Oniguruma::ORegexp#gsub</code> on this string.
+   def ogsub(*args)
+      Oniguruma::ORegexp.new(args.shift).gsub(self, *args)
    end
    
+   # Calls <code>Oniguruma::ORegexp#gsub!</code> on this string.
+   def ogsub!(*args)
+      Oniguruma::ORegexp.new(args.shift).gsub!(self, *args)
+   end
+   
+   # Calls <code>Oniguruma::ORegexp#sub</code> on this string.
+   def osub(re, *args)
+      Oniguruma::ORegexp.new( re ).sub(self, *args)
+   end
+   
+   # Calls <code>Oniguruma::ORegexp#sub!</code> on this string.
+   def osub!(re, *args)
+      Oniguruma::ORegexp.new( re ).sub(self, *args)
+   end
 end
+
 class ::MatchData
-  alias old_aref :[]
-  def [](*idx)
-    if idx[0].is_a?(Symbol) 
-      k = @named_captures && @named_captures[idx[0]]
-      k && old_aref(k)
-    else
-      old_aref(*idx)
-    end
-  end
+   # call-seq:
+   #    to_index[symbol]      => int or nil
+   # 
+   # Returns the group index for the corresponding named group, or 
+   # <code>nil</code> if the group does not exist.
+   #    
+   #    m = ORegexp.new( '(?<begin>^.*?)(?<middle>\d)(?<end>.*)' ).match("THX1138")
+   #    m.to_index[:begin]    #=> 1
+   #    m.to_index[:unknown]  #=> nil
+   def to_index symbol
+      @named_captures && @named_captures[symbol]
+   end
+
+   alias old_aref :[]
+   
+   # call-seq:
+   #    mtch[i]               => obj
+   #    mtch[start, length]   => array
+   #    mtch[range]           => array
+   #    mtch[symbol]          => obj
+   # 
+   # <code>MatchData</code> acts as an array, and may be
+   # accessed using the normal array indexing techniques.  <i>mtch</i>[0] is
+   # equivalent to the special variable <code>$&</code>, and returns the entire
+   # matched string.  <i>mtch</i>[1], <i>mtch</i>[2], and so on return the values
+   # of the matched backreferences (portions of the pattern between parentheses).
+   #    
+   #    m = ORegexp.new( '(.)(.)(\d+)(\d)' ).match("THX1138.")
+   #    m[0]       #=> "HX1138"
+   #    m[1, 2]    #=> ["H", "X"]
+   #    m[1..3]    #=> ["H", "X", "113"]
+   #    m[-3, 2]   #=> ["X", "113"]
+   #
+   # If a symbol is used as index, the corresponding named group is returned,
+   # or <code>nil</code> if such a group does not exist.
+   #
+   #    m = ORegexp.new( '(?<begin>^.*?)(?<middle>\d)(?<end>.*)' ).match("THX1138")
+   #    m[:begin]  #=> "THX"
+   #    m[:moddle]  #=> "1"
+   #    m[:end]  #=> "138"
+   
+   def [](*idx)
+      if idx[0].is_a?(Symbol) 
+         k = to_index( idx[0] )
+         k && old_aref(k)
+       else
+         old_aref(*idx)
+       end
+   end
+   
+   alias old_begin :begin
+   
+   # call-seq:
+   #    mtch.begin(n)        => integer
+   #    mtch.begin           => integer
+   #    mtch.begin(symbol)   => integer
+   # 
+   # Returns the offset of the start of the <em>n</em>th element of the match
+   # array in the string.
+   #    
+   #    m = ORegexp.new( '(.)(.)(\d+)(\d)' ).match("THX1138.")
+   #    m.begin(0)   #=> 1
+   #    m.begin(2)   #=> 2
+   #
+   # If no arguments are given, the index of the
+   # first matching character is returned.
+   #
+   #    m = ORegexp.new( '(.)(.)(\d+)(\d)' ).match("THX1138.")
+   #    m.begin      #=> 1
+   #
+   # If the argument is a symbol, then the beginning of the 
+   # corresponding named group is returned, or <code>nil</code>
+   # if the group does not exist.
+   #
+   #    m = ORegexp.new( '(?<begin>^.*?)(?<middle>\d)(?<end>.*)' ).match("THX1138")
+   #    m.begin(:middle) #=> 3
+   
+   def begin(*idx)
+      if idx[0].is_a?(Symbol) 
+         k = to_index( idx[0] )
+         k && old_begin(k)
+      elsif idx.empty?
+         old_begin( 0 )
+      else
+         old_begin(*idx)
+      end
+   end
+   
+   alias old_end :end
+   
+   # call-seq:
+   #    mtch.end(n)   => integer
+   # 
+   # Returns the offset of the character immediately following the end of the
+   # <em>n</em>th element of the match array in the string.
+   #    
+   #    m = ORegexp.new( '(.)(.)(\d+)(\d)' ).match("THX1138.")
+   #    m.end(0)   #=> 7
+   #    m.end(2)   #=> 3
+   #
+   # If no arguments are given, the index of the
+   # last matching character is returned.
+   #
+   #    m = ORegexp.new( '(.)(.)(\d+)(\d)' ).match("THX1138.")
+   #    m.last      #=> 7
+   #
+   # If the argument is a symbol, then the beginning of the 
+   # corresponding named group is returned, or <code>nil</code>
+   # if the group does not exist.
+   #
+   #    m = ORegexp.new( '(?<begin>^.*?)(?<middle>\d)(?<end>.*)' ).match("THX1138")
+   #    m.end(:middle) #=> 4
+   
+   def end(*idx)
+      if idx[0].is_a?(Symbol) 
+         k = to_index( idx[0] )
+         k && old_end(k)
+      elsif idx.empty?
+         old_end( 0 )
+      else
+         old_end(*idx)
+      end
+   end
+   
+   alias old_offset :offset
+   
+   # call-seq:
+   #    mtch.offset(n)      => array
+   #    mtch.offset         => array
+   #    mtch.offset(symbol) => array
+   # 
+   # Returns a two-element array containing the beginning and ending offsets of
+   # the <em>n</em>th match.
+   #    
+   #    m = ORegexp.new( '(.)(.)(\d+)(\d)' ).match("THX1138.")
+   #    m.offset(0)   #=> [1, 7]
+   #    m.offset(4)   #=> [6, 7]   
+   #
+   # If no arguments are given, the offsets of the entire
+   # sequence are returned.
+   #
+   #    m = ORegexp.new( '(.)(.)(\d+)(\d)' ).match("THX1138.")
+   #    m.offset      #=> [1, 7]
+   #
+   # If the argument is a symbol, then the offsets of the 
+   # corresponding named group are returned, or <code>nil</code>
+   # if the group does not exist.
+   #
+   #    m = ORegexp.new( '(?<begin>^.*?)(?<middle>\d)(?<end>.*)' ).match("THX1138")
+   #    m.end(:middle) #=> [3, 4]
+   
+   def offset(*idx)
+      if idx[0].is_a?(Symbol) 
+         k = to_index( idx[0] )
+         k && old_offset(k)
+      elsif idx.empty?
+         old_offset( 0 )
+      else
+         old_offset(*idx)
+      end
+   end
 end

data/test/test_oniguruma.rb

@@ -27,7 +27,7 @@ class ORegexpTestCase < Test::Unit::TestCase
    end
 
    def test_bad_initialization
-      assert_raises(Exception) do
+      assert_raises(ArgumentError) do
          reg  = Oniguruma::ORegexp.new( "(3.)(.*)(3.))" )
       end
    end
@@ -53,7 +53,7 @@ class ORegexpTestCase < Test::Unit::TestCase
       string = 'My favorite fruits are (?#fruit1), (?#fruit2), and (?#fruit3)'
       assert_equal( "My favorite fruits are *, *, and *", reg.gsub( string, '*' ) )
       fruits = { "fruit1" => "apples", "fruit2" => "bananas", "fruit3" => "grapes" }
-      assert_equal( "My favorite fruits are apples, bananas, and grapes", reg.gsub( string ) { |text, match| fruits[match[1]]} )
+      assert_equal( "My favorite fruits are apples, bananas, and grapes", reg.gsub( string ) { |match| fruits[match[1]]} )
    end
    
    def test_eql
@@ -74,10 +74,23 @@ class ORegexpTestCase < Test::Unit::TestCase
       
       assert_equal( "Upper case\n", result )
    end
-   
+
+   def test_case_eql_compat
+      # === method should not raise when used in case statements
+      a = Time.now
+      result = ""
+      case a
+        when /./ ; result =  "rgx"
+        when Oniguruma::ORegexp.new('.'); result =  "ore"
+        else;                         result =  "else"
+      end
+      assert_equal( "else", result )
+   end
+  
    def test_operator_match
       assert_equal( nil, Oniguruma::ORegexp.new( 'SIT' ) =~ "insensitive" )
       assert_equal( 5, Oniguruma::ORegexp.new( 'SIT', :options => Oniguruma::OPTION_IGNORECASE ) =~ "insensitive" )
+      assert_equal( 5, Oniguruma::ORegexp.new( 'SIT', 'i' ) =~ "insensitive" )
    end
    
 #    def test_operator_match_2
@@ -96,6 +109,8 @@ class ORegexpTestCase < Test::Unit::TestCase
    def test_kcode
       reg  = Oniguruma::ORegexp.new( "(3.)(.*)(3.)" )
       assert_equal( Oniguruma::ENCODING_ASCII, reg.kcode )
+      reg  = Oniguruma::ORegexp.new( "(3.)(.*)(3.)", '', 'SJIS' )
+      assert_equal( Oniguruma::ENCODING_SJIS, reg.kcode )
    end
    
    def test_options
@@ -106,6 +121,40 @@ class ORegexpTestCase < Test::Unit::TestCase
       string = '(?<=\n)\\.*ocatarinetabelachitchix'
       assert_equal( string, Oniguruma::ORegexp.new( string ).source )
    end
+
+   def test_named_sub_backrefs
+      re = Oniguruma::ORegexp.new('(?<pre>\w+?)\d+(?<after>\w+)')
+      assert_equal(' def123abc ', re.sub('abc123def', ' \<after>123\<pre> ') )
+   end
+
+   def test_named_sub_backrefs_dupes
+      re = Oniguruma::ORegexp.new('(?<pre>\w+?)\d+(?<pre>\w+)')
+      assert_equal('123def', re.sub('abc123def', '123\<pre>') )
+   end
+
+   def test_backref_set_for_match
+     re = Oniguruma::ORegexp.new('Date:(\d{4})/(\d{2})/(\d{2})')
+     assert re.match( "Date:2007/03/25" )
+     assert_not_nil $~
+     assert_equal "2007", $1
+     assert_equal "03", $2
+     assert_equal "25", $3
+   end
+
+   def test_backref_set_for_match_op
+     re = Oniguruma::ORegexp.new('Date:(\d{4})/(\d{2})/(\d{2})')
+     assert re =~ "Date:2007/03/25"
+     assert_not_nil $~
+     assert_equal "2007", $1
+     assert_equal "03", $2
+     assert_equal "25", $3
+   end
+
+   def test_multibyte_named_backrefs 
+     r = Oniguruma::ORegexp.new('(?<группа>test).+(\k<группа>)', :encoding => Oniguruma::ENCODING_UTF8)
+     assert_equal "should !test!", r.sub("should test this damned test", '!\<группа>!') 
+   end
+
 end
 
 class MatchDataTestCase < Test::Unit::TestCase
@@ -123,6 +172,7 @@ class MatchDataTestCase < Test::Unit::TestCase
    
    def test_begin
       matches = @reg.match( "THX1138." )
+      assert_equal( 1, matches.begin )
       assert_equal( 1, matches.begin(0) )
       assert_equal( 2, matches.begin(2) )
    end
@@ -134,6 +184,7 @@ class MatchDataTestCase < Test::Unit::TestCase
    
    def test_end
       matches = @reg.match( "THX1138." )
+      assert_equal( 7, matches.end )
       assert_equal( 7, matches.end(0) )
       assert_equal( 3, matches.end(2) )
    end
@@ -146,6 +197,7 @@ class MatchDataTestCase < Test::Unit::TestCase
    
    def test_offset
       matches = @reg.match( "THX1138." )
+      assert_equal( [1, 7], matches.offset )
       assert_equal( [1, 7], matches.offset(0) )
       assert_equal( [6, 7], matches.offset(4) )
    end
@@ -189,9 +241,20 @@ class MatchDataTestCase < Test::Unit::TestCase
    def test_match_all
       reg = Oniguruma::ORegexp.new( 'ca' )
       matches = reg.match_all( 'ocatacachaca' )
+      a = []
+      matches.each { |m| a << m.offset(0) }
+      assert_equal( [ [1,3], [5,7], [10,12] ], a)
       assert_equal( 3, matches.size )
-      assert_equal( 7, matches.position(2) )
-      assert_equal( "ca", matches.string[matches.begin(1)...matches.end(1)])
+      assert_equal( 10, matches[2].begin( 0 ) )
+      assert_equal( "ca", matches[1].string[matches[1].begin( 0 )...matches[1].end( 0 )])
+   end
+   
+   def test_scan
+      reg = Oniguruma::ORegexp.new( 'ca' )
+      a = []
+      matches = reg.match_all( 'ocatacachaca' ) { |m| a << m.offset(0) }
+      #assert_kind_of(Oniguruma::MultiMatchData, matches)
+      assert_equal( [ [1,3], [5,7], [10,12] ], a)
    end
    
    def test_match_empty_string
@@ -205,12 +268,32 @@ class MatchDataTestCase < Test::Unit::TestCase
       reg = Oniguruma::ORegexp.new( '(?<begin>\()(?<body>.*)(?<end>\))', :options => Oniguruma::OPTION_MULTILINE )
       matches = reg.match( "blah (content) blah" )
       assert_not_nil( matches )
+      assert_equal $~, matches
       assert_equal( '(', matches[:begin] )
       assert_equal( 'content', matches[:body] )
       assert_equal( ')', matches[:end] )
       assert_equal( nil, matches[:inexistent])
    end
 
+   def test_multibyte_named_backrefs 
+     r = Oniguruma::ORegexp.new('(?<имя>test).+(\k<имя>)', :encoding => Oniguruma::ENCODING_UTF8)
+     assert_equal "should TEST", r.sub("should test this damned test") {|m| m[:"имя"].upcase } 
+   end
+
+   def test_no_named_backrefs
+     r = Oniguruma::ORegexp.new('(.+).+(.+)')
+     r.match("text")
+     assert_not_nil $~
+     assert_equal 0, $~.instance_variables.size
+     r = Oniguruma::ORegexp.new('(?<a>.+).+(?<b>.+)')
+     r.match("text")
+     assert_not_nil $~
+     assert_equal 1, $~.instance_variables.size
+
+   end
+
+# casefolding for full Unicode set is not present in versions prior to 5.
+if Oniguruma::VERSION >= '5.0.0'
    def test_utf8_ignore_case
      reg = Oniguruma::ORegexp.new( '([а-я])+', :options => Oniguruma::OPTION_IGNORECASE, :encoding => Oniguruma::ENCODING_UTF8 )
      matches = reg.match("Text: Ехал Грека Через Реку")
@@ -222,16 +305,17 @@ class MatchDataTestCase < Test::Unit::TestCase
    
    def test_utf8_gsub
      reg = Oniguruma::ORegexp.new( '([а-я])([а-я])([а-я]+)', :options => Oniguruma::OPTION_IGNORECASE, :encoding => Oniguruma::ENCODING_UTF8 )
-     new_str = reg.gsub("Text: Ехал Грека Через Реку") {|s,m| m[1]*2+m[2]*2+m[3] }
+     new_str = reg.gsub("Text: Ехал Грека Через Реку") {|m| m[1]*2+m[2]*2+m[3] }
      assert_equal("Text: ЕЕххал ГГррека ЧЧеерез РРееку", new_str)
    end
    
    def test_utf8_gsub2
      reg = Oniguruma::ORegexp.new( '[а-я]', :options => Oniguruma::OPTION_IGNORECASE, :encoding => Oniguruma::ENCODING_UTF8 )
-     new_str = reg.gsub("Text: Ехал Грека Через Реку") {|s,m| s*2 }
+     new_str = reg.gsub("Text: Ехал Грека Через Реку") {|m| m[0]*2 }
      assert_equal("Text: ЕЕххаалл ГГррееккаа ЧЧеерреезз РРееккуу", new_str)
    end
-   
+end
+
    def test_sub_compatibility
      $x = "a.gif"
      assert_equal("b.gif", $x.osub('.*\.([^\.]+)$', 'b.\1'))
@@ -242,36 +326,36 @@ class MatchDataTestCase < Test::Unit::TestCase
      assert_equal("<a.gif>", $x.osub('.*\.([^\.]+)$', '<\&>'))
      assert_equal("a.a.", $x.osub('(gif)', '\`') )
    end
-   
-  class ::String
-    def ogsub(*args)
-      Oniguruma::ORegexp.new(args.shift).gsub(self, *args)
-    end
-    def ogsub!(*args)
-      Oniguruma::ORegexp.new(args.shift).gsub!(self, *args)
-    end
-    def osub(re, *args)
-      Oniguruma::ORegexp.new( re ).sub(self, *args)
-    end
-  end
  
   def test_gsub_compat
     assert_equal("hello".ogsub('[aeiou]', '*')              , "h*ll*")
     assert_equal("hello".ogsub('([aeiou])', '<\1>')         , "h<e>ll<o>")
     i = 0 
-    assert_equal("12345" , Oniguruma::ORegexp.new('.').gsub("hello") {|s,m| i+=1; i.to_s})
-    assert_equal("214365", Oniguruma::ORegexp.new('(.)(.)').gsub("123456") {|s,m| m[2] + m[1] }) 
+    assert_equal("12345" , Oniguruma::ORegexp.new('.').gsub("hello") {|m| i+=1; i.to_s})
+    assert_equal("214365", Oniguruma::ORegexp.new('(.)(.)').gsub("123456") {|m| m[2] + m[1] }) 
     a = "test"
     a.ogsub!('t', a)
     assert_equal("testestest", a)
   end
 
   def test_match_compat
-    t = Oniguruma::ORegexp.new('(.)(.)').gsub("123456") {|s,m| "#$2#$1" }
+    t = Oniguruma::ORegexp.new('(.)(.)').gsub("123456") {|m| "#$2#$1" }
     assert_equal("214365", t ) 
-    t = Oniguruma::ORegexp.new('([aeiou])').gsub("hello") {|s,m| "<#$1>" }
+    t = Oniguruma::ORegexp.new('([aeiou])').gsub("hello") {|m| "<#$1>" }
     assert_equal( "h<e>ll<o>", t)
   end
 
+  def _u16(str)
+    str.unpack("U*").pack("n*") 
+  end
+puts Oniguruma::VERSION 
+if Oniguruma::VERSION >= '4.0.0'
+  def test_utf16_gsub
+    r = Oniguruma::ORegexp.new( _u16('[aeiou]'), :encoding => Oniguruma::ENCODING_UTF16_BE)
+    assert_equal( _u16("h*ll*"), r.gsub( _u16("hello"), _u16('*')) )
+    r = Oniguruma::ORegexp.new( _u16('([aeiou])'), :encoding => Oniguruma::ENCODING_UTF16_BE)
+    assert_equal( _u16("h<e>\\ll<o>\\"), r.gsub( _u16("hello"), _u16('<\1>\\')) )
+  end
+end
 
 end

metadata

@@ -3,8 +3,8 @@ rubygems_version: 0.9.2
 specification_version: 1
 name: oniguruma
 version: !ruby/object:Gem::Version 
-  version: 0.9.1
-date: 2007-03-25 00:00:00 +01:00
+  version: 1.0.0
+date: 2007-03-27 00:00:00 +02:00
 summary: Bindings for the oniguruma regular expression library
 require_paths: 
 - lib