oniguruma 0.9.0 → 0.9.1

data/History.txt

@@ -1,4 +1,10 @@
-== 1.0.0 / 2007-03-19
+== 0.9.1 / 2007-03-25
+* FIX: Buggy resolution of numeric codes for encoding and syntax options (Nikolai Lugovoi)
+* FIX: Buggy implementation of ORegexp#gsub and  ORegexp#gsub methods. Now code is all C (Nikolai Lugovoi)
+* Added documentation for class ORegexp
+* Added regexp syntax documentation.
+
+== 0.9.0 / 2007-03-19
 
 * 1 major enhancement
   * Birthday!

data/Manifest.txt

@@ -1,6 +1,7 @@
 History.txt
 Manifest.txt
 README.txt
+Syntax.txt
 Rakefile
 lib/oniguruma.rb
 ext/oregexp.c

data/README.txt

@@ -16,6 +16,10 @@ Ruby bindings to the Oniguruma[http://www.geocities.jp/kosako3/oniguruma/] regul
  puts match[0]         <= 'terraforming'
  puts match[:before]   <= 'terr'
  puts match[:after]    <= 'forming'
+ 
+== SYNTAX
+
+Consult the Syntax.txt[link:files/Syntax_txt.html] page.
 
 == REQUIREMENTS:
 
@@ -39,8 +43,9 @@ sudo gem install -r oniguruma
 
 == CREDITS:
 
-* K.Kosako, for his great library.
-* A lot of the documentation has been copied from the orininal Ruby Regex documentation.
+* N. Lugovoi. ORegexp.sub and ORegexp.gsub code, plus other patches.
+* K. Kosako. For his great library.
+* A lot of the documentation has been copied from the original Ruby Regex documentation.
 
 == LICENSE:
 

data/Rakefile

@@ -3,7 +3,7 @@ require 'hoe'
 
 class Hoe; def extra_deps; @extra_deps.reject { |x| Array(x).first == 'hoe' }; end end
 
-Hoe.new('oniguruma', '0.9.0') do |p|
+Hoe.new('oniguruma', '0.9.1') do |p|
   p.rubyforge_name = 'oniguruma'
   p.author = 'Dizan Vasquez'
   p.email = 'dix_ans@yahoo.com'

data/Syntax.txt

@@ -0,0 +1,396 @@
+= RUBY REGULAR EXPRESSION SYNTAX
+
+
+== Syntax Elements
+
+[\]     escape (enable or disable meta character meaning)
+[|]     alternation
+[(...)] group
+[[...]] character class  
+
+
+== Characters
+
+[\t]           horizontal tab (0x09)
+[\v]           vertical tab   (0x0B)
+[\n]           newline        (0x0A)
+[\r]           return         (0x0D)
+[\b]           back space     (0x08)
+
+               \b is effective in character class [...] only
+[\f]           form feed      (0x0C)
+[\a]           bell           (0x07)
+[\e]           escape         (0x1B)
+[\nnn]         octal char            (encoded byte value)
+[\xHH]         hexadecimal char      (encoded byte value)
+[\x{7HHHHHHH}] wide hexadecimal char (character code point value)
+[\cx]          control char          (character code point value)
+[\C-x]         control char          (character code point value)
+[\M-x]         meta  (x|0x80)        (character code point value)
+[\M-\C-x]      meta control char     (character code point value)
+
+
+
+== Character types
+
+[.]        any character (except newline)
+[\w]       word character
+
+           Not Unicode:
+           * alphanumeric, "_" and multibyte char. 
+           Unicode:
+           * General_Category -- (Letter|Mark|Number|Connector_Punctuation)
+[\W]       non word char
+[\s]       whitespace char
+
+           Not Unicode:
+           * \t, \n, \v, \f, \r, \x20
+           Unicode:
+           * 0009, 000A, 000B, 000C, 000D, 0085(NEL), 
+           * General_Category:
+             * -- Line_Separator
+             * -- Paragraph_Separator
+             * -- Space_Separator
+[\S]       non whitespace char
+[\d]       decimal digit char
+
+           Unicode: General_Category -- Decimal_Number
+[\D]       non decimal digit char
+[\h]       hexadecimal digit char   [0-9a-fA-F]
+[\H]       non hexadecimal digit char
+
+
+== Character Properties
+
+ \p{property-name}
+ \p{^property-name}    (negative)
+ \P{property-name}     (negative)
+
+=== property-name:
+
+Works on all encodings:
+* Alnum, Alpha, Blank, Cntrl, Digit, Graph, Lower,
+  Print, Punct, Space, Upper, XDigit, Word, ASCII,
+Works on EUC_JP, Shift_JIS:
+* Hiragana, Katakana
+Works on UTF8, UTF16, UTF32:
+* Any, Assigned, C, Cc, Cf, Cn, Co, Cs, L, Ll, Lm, Lo, Lt, Lu,
+  M, Mc, Me, Mn, N, Nd, Nl, No, P, Pc, Pd, Pe, Pf, Pi, Po, Ps,
+  S, Sc, Sk, Sm, So, Z, Zl, Zp, Zs, 
+  Arabic, Armenian, Bengali, Bopomofo, Braille, Buginese,
+  Buhid, Canadian_Aboriginal, Cherokee, Common, Coptic,
+  Cypriot, Cyrillic, Deseret, Devanagari, Ethiopic, Georgian,
+  Glagolitic, Gothic, Greek, Gujarati, Gurmukhi, Han, Hangul,
+  Hanunoo, Hebrew, Hiragana, Inherited, Kannada, Katakana,
+  Kharoshthi, Khmer, Lao, Latin, Limbu, Linear_B, Malayalam,
+  Mongolian, Myanmar, New_Tai_Lue, Ogham, Old_Italic, Old_Persian,
+  Oriya, Osmanya, Runic, Shavian, Sinhala, Syloti_Nagri, Syriac,
+  Tagalog, Tagbanwa, Tai_Le, Tamil, Telugu, Thaana, Thai, Tibetan,
+  Tifinagh, Ugaritic, Yi
+
+== Quantifiers
+
+=== Greedy
+
+[?]       1 or 0 times
+[*]       0 or more times
+[+]       1 or more times
+[{n,m}]   at least n but not more than m times
+[{n,}]    at least n times
+[{,n}]    at least 0 but not more than n times ({0,n})
+[{n}]     n times
+
+=== Reluctant
+
+[??]      1 or 0 times
+[*?]      0 or more times
+[+?]      1 or more times
+[{n,m}?]  at least n but not more than m times  
+[{n,}?]   at least n times
+[{,n}?]   at least 0 but not more than n times (== {0,n}?)
+
+=== Possessive (greedy and does not backtrack after repeated)
+
+[?+]      1 or 0 times
+[*+]      0 or more times
+[++]      1 or more times
+
+({n,m}+, {n,}+, {n}+ are possessive op. in ONIG_SYNTAX_JAVA only)
+
+
+== Anchors
+
+[^]       beginning of the line
+[$]       end of the line
+[\b]      word boundary
+[\B]      not word boundary
+[\A]      beginning of string
+[\Z]      end of string, or before newline at the end
+[\z]      end of string
+[\G]      matching start position 
+
+
+== Character class
+
+[^...]    negative class (lowest precedence operator)
+[x-y]     range from x to y
+[[...]]   set (character class in character class)
+[..&&..]  intersection (low precedence at the next of ^)
+          
+If you want to use '[', '-', ']' as a normal character
+in a character class, you should escape these characters by '\'.
+
+
+POSIX bracket ([:xxxxx:], negate [:^xxxxx:])
+
+=== Not Unicode Case:
+
+[alnum]    alphabet or digit char
+[alpha]    alphabet
+[ascii]    code value: [0 - 127]
+[blank]    \t, \x20
+[cntrl]    control
+[digit]    0-9
+[graph]    include all of multibyte encoded characters
+[lower]    lower case
+[print]    include all of multibyte encoded characters
+[punct]    punctuation
+[space]    \t, \n, \v, \f, \r, \x20
+[upper]    upper case
+[xdigit]   0-9, a-f, A-F
+[word]     alphanumeric, "_" and multibyte characters
+
+
+=== Unicode Case:
+
+[alnum]    Letter | Mark | Decimal_Number
+[alpha]    Letter | Mark
+[ascii]    0000 - 007F
+[blank]    Space_Separator | 0009
+[cntrl]    Control | Format | Unassigned | Private_Use | Surrogate
+[digit]    Decimal_Number
+[graph]    [[:^space:]] && ^Control && ^Unassigned && ^Surrogate
+[lower]    Lowercase_Letter
+[print]    [[:graph:]] | [[:space:]]
+[punct]    Connector_Punctuation | Dash_Punctuation | Close_Punctuation |
+           Final_Punctuation | Initial_Punctuation | Other_Punctuation |
+           Open_Punctuation
+[space]    Space_Separator | Line_Separator | Paragraph_Separator |
+           0009 | 000A | 000B | 000C | 000D | 0085
+[upper]    Uppercase_Letter
+[xdigit]   0030 - 0039 | 0041 - 0046 | 0061 - 0066
+           (0-9, a-f, A-F)
+[word]     Letter | Mark | Decimal_Number | Connector_Punctuation
+
+
+
+== Extended groups
+
+[(?#...)]            comment
+[(?imx-imx)]         option on/off:
+                     * i: ignore case
+                     * m: multi-line (dot(.) match newline)
+                     * x: extended form
+[(?imx-imx:subexp)]  option on/off for subexp
+[(?:subexp)]         not captured group
+[(subexp)]           captured group
+[(?=subexp)]         look-ahead
+[(?!subexp)]         negative look-ahead
+[(?<=subexp)]        look-behind
+[(?<!subexp)]        negative look-behind
+
+                     Subexp of look-behind must be fixed character length.
+                     But different character length is allowed in top level
+                     alternatives only.
+                     ex. (?<=a|bc) is OK. (?<=aaa(?:b|cd)) is not allowed.
+                     
+                     In negative-look-behind, captured group isn't allowed, 
+                     but shy group(?:) is allowed.
+[(?>subexp)]         atomic group
+                     don't backtrack in subexp.
+[(?<name>subexp)]    define named group
+                     (All characters of the name must be a word character.)
+                     
+                     Not only a name but a number is assigned like a captured
+                     group.
+
+                     Assigning the same name as two or more subexps is allowed.
+                     In this case, a subexp call can not be performed although
+                     the back reference is possible.
+
+
+== Back reference
+
+[\n]          back reference by group number (n >= 1)
+[\k<name>]    back reference by group name
+In the back reference by the multiplex definition name,
+a subexp with a large number is referred to preferentially.
+(When not matched, a group of the small number is referred to.)
+
+* Back reference by group number is forbidden if named group is defined 
+  in the pattern and ONIG_OPTION_CAPTURE_GROUP is not setted.
+
+
+=== Back reference with nest level
+
+[\k<name+n>]     n: 0, 1, 2, ...
+[\k<name-n>]     n: 0, 1, 2, ...
+
+Destinate relative nest level from back reference position.    
+
+Examples:
+   /\A(?<a>|.|(?:(?<b>.)\g<a>\k<b+0>))\z/.match("reer")
+
+   r = ORegexp.compile(<<'__REGEXP__'.strip, :options => Oniguruma::EXTENDED)
+   (?<element> \g<stag> \g<content>* \g<etag> ){0}
+   (?<stag> < \g<name> \s* > ){0}
+   (?<name> [a-zA-Z_:]+ ){0}
+   (?<content> [^<&]+ (\g<element> | [^<&]+)* ){0}
+   (?<etag> </ \k<name+1> >){0}
+   \g<element>
+   __REGEXP__
+   
+   p r.match('<foo>f<bar>bbb</bar>f</foo>').captures
+
+
+
+=== Subexp call ("Tanaka Akira special")
+
+[\g<name>]    call by group name
+[\g<n>]       call by group number (n >= 1)
+
+* left-most recursive call is not allowed.
+
+  Example:
+    (?<name>a|\g<name>b)   => error
+    (?<name>a|b\g<name>c)  => OK
+* Call by group number is forbidden if named group is defined in the pattern
+  and Oniguruma::OPTION_CAPTURE_GROUP is not set.
+* If the option status of called group is different from calling position
+  then the group's option is effective.
+  
+  Example:
+    (?-i:\g<name>)(?i:(?<name>a)){0}  <i>matches "A"</i>
+
+
+== Captured group
+
+Behavior of the no-named group (...) changes with the following conditions.
+(But named group is not changed.)
+
+[case 1]                         <code>ORegexp.new( '...' )</code>  (named group is not used, no option)
+
+                                 ... is treated as a captured group.
+[case 2]                         <code>ORegexp.new( '...', :options => OPTION_DONT_CAPTURE_GROUP )</code> (named group is not used, 'g' option)
+
+                                 ... is treated as a no-captured group (?:...).
+
+[case 3]                         <code>ORegexp.new( '...(?<name>...)...' )</code>  (named group is used, no option)
+
+                                 (?<name>...) is treated as a no-captured group (?:...)
+                                 
+                                 numbered-backref/call is not allowed.
+
+[case 2]                         <code>ORegexp.new( '...', :options => OPTION_CAPTURE_GROUP )</code> (named group is used, 'G' option)
+
+                                 (?<name>...) is treated as a captured group (?:...)
+
+                                 numbered-backref/call is allowed.
+
+where
+* g: OPTION_DONT_CAPTURE_GROUP
+* G: OPTION_CAPTURE_GROUP
+
+('g' and 'G' options are argued in ruby-dev ML)
+
+
+== Syntax dependent options
+
+=== ONIG_SYNTAX_RUBY
+
+[(?m)] dot(.) match newline
+
+=== ONIG_SYNTAX_PERL and ONIG_SYNTAX_JAVA
+
+[(?s)] dot(.) match newline
+[(?m)] ^ match after newline, $ match before newline
+
+== Original extensions
+
+* hexadecimal digit char type  \h, \H
+* named group                  (?<name>...)
+* named backref                \k<name>
+* subexp call                  \g<name>, \g<group-num>
+
+
+== Lacking features compare with perl 5.8.0
+
+* \N{name}
+* \l,\u,\L,\U, \X, \C
+* (?{code})
+* (??{code})
+* (?(condition)yes-pat|no-pat)
+* \Q...\E
+
+  This is effective on ONIG_SYNTAX_PERL and ONIG_SYNTAX_JAVA.
+
+
+== Differences with Japanized GNU regex(version 0.12) of Ruby 1.8
+
+* add character property (\p{property}, \P{property})
+* add hexadecimal digit char type (\h, \H)
+* add look-behind
+
+  (?<=fixed-char-length-pattern), (?<!fixed-char-length-pattern)
+* add possessive quantifier. ?+, *+, ++
+* add operations in character class. [], &&
+
+  ('[' must be escaped as an usual char in character class.)
+* add named group and subexp call.
+* octal or hexadecimal number sequence can be treated as 
+  a multibyte code char in character class if multibyte encoding
+  is specified.
+  
+  (ex. <code>[\xa1\xa2], [\xa1\xa7-\xa4\xa1]</code>)
+* allow the range of single byte char and multibyte char in character
+  class.
+  
+  ex. <code>[a-<<any EUC-JP character>>]</code> in EUC-JP encoding.
+* effect range of isolated option is to next ')'.
+  ex. (?:(?i)a|b) is interpreted as (?:(?i:a|b)), not (?:(?i:a)|b).
+* isolated option is not transparent to previous pattern.
+  ex. <code>a(?i)*</code> is a syntax error pattern.
+* allowed incompleted left brace as an usual string.
+  ex. /{/, /({)/, /a{2,3/ etc...
+* negative POSIX bracket [:^xxxx:] is supported.
+* POSIX bracket [:ascii:] is added.
+* repeat of look-ahead is not allowed.
+  ex. <code>(?=a)*</code>, <code>(?!b){5}</code>
+* Ignore case option is effective to numbered character.
+  ex. <code>/\x61/i =~ "A"<code>
+* In the range quantifier, the number of the minimum is omissible.
+
+  <code>/a{,n}/ == /a{0,n}/<code>
+  
+  The simultanious abbreviation of the number of times of the minimum
+  and the maximum is not allowed. (/a{,}/)
+* <code>a{n}?<code> is not a non-greedy operator.
+  <code>/a{n}?/ == /(?:a{n})?/<code>
+* invalid back reference is checked and cause error.
+  /\1/, /(a)\2/
+* Zero-length match in infinite repeat stops the repeat,
+  then changes of the capture group status are checked as stop condition.
+   /(?:()|())*\1\2/ =~ ""
+   /(?:\1a|())*/ =~ "a"
+
+
+== Problems
+
+* Invalid encoding byte sequence is not checked in UTF-8.
+
+* Invalid first byte is treated as a character.
+   /./u =~ "\xa3"
+
+* Incomplete byte sequence is not checked.
+   /\w+/ =~ "a\xf3\x8ec"
+

data/ext/extconf.rb

@@ -1,3 +1,4 @@
 require 'mkmf'
 have_library("onig")
+$CFLAGS='-Wall'
 create_makefile( "oregexp" )

data/ext/oregexp.c

@@ -24,7 +24,10 @@ static VALUE oregexp_allocate( VALUE klass ) {
 }
 
 
-static OnigEncodingType * int2encoding( int index ) {
+static OnigEncodingType * int2encoding( VALUE v_index ) {
+   int index;
+   if( ! NIL_P(v_index) ) {
+     index = FIX2INT(v_index);
    switch( index ) {
       case 0: return ONIG_ENCODING_ASCII;
       case 1: return ONIG_ENCODING_ISO_8859_1;
@@ -60,10 +63,14 @@ static OnigEncodingType * int2encoding( int index ) {
       case 31: return ONIG_ENCODING_GB18030;
       case 32: return ONIG_ENCODING_UNDEF;
    }
+   }
    return ONIG_ENCODING_UNDEF;
 }
 
-static OnigSyntaxType * int2syntax( int index ) {
+static OnigSyntaxType * int2syntax( VALUE v_index ) {
+   int index;
+   if( ! NIL_P(v_index) ) {
+     index = FIX2INT(v_index);
    switch( index ) {
       case 0: return ONIG_SYNTAX_ASIS;
       case 1: return ONIG_SYNTAX_POSIX_BASIC;
@@ -77,25 +84,32 @@ static OnigSyntaxType * int2syntax( int index ) {
       case 9: return ONIG_SYNTAX_RUBY;
       case 10: return ONIG_SYNTAX_DEFAULT;
    }
+   }
    return ONIG_SYNTAX_DEFAULT;
 }
 
+struct callback_packet {
+    VALUE hash;
+    OnigRegion * region;
+};
+
 static int name_callback(
    const UChar* name, 
    const UChar* name_end,
    int ngroup_num, 
    int* group_nums,
    regex_t* reg, 
-   void* arg
+   struct callback_packet* arg
 ) {
    int i, gn, ref;
-   OnigRegion *region = (OnigRegion* )arg;
+   OnigRegion *region = arg->region;
+   VALUE nameHash = arg->hash;
 
    for (i = 0; i < ngroup_num; i++) {
       gn = group_nums[i];
       ref = onig_name_to_backref_number(reg, name, name_end, region);
       if (ref != gn )
-         rb_raise(rb_eException, "Oniguruma Error: group and backreference names are different");
+        return 1; 
       rb_hash_aset( nameHash, ID2SYM(rb_intern(name)), INT2FIX( gn ) );
   }
   return 0;  
@@ -110,13 +124,16 @@ static VALUE oregexp_initialize( VALUE self, VALUE pattern, VALUE options ) {
    rb_iv_set( self, "@options", options );
    UChar* pat_ptr = RSTRING(pattern_str)->ptr;
    int pat_len = RSTRING(pattern_str)->len;
+   if( pat_len == 0 ) {
+      rb_raise(rb_eArgError, "Empty pattern makes no sense.");
+   }
 
    VALUE rOptions = rb_hash_aref( options, ID2SYM( rb_intern( "options" ) ) );
    VALUE rEncoding = rb_hash_aref( options, ID2SYM( rb_intern( "encoding" ) ) );
    VALUE rSyntax = rb_hash_aref( options, ID2SYM( rb_intern( "syntax" ) ) );
    int iOptions = NUM2INT( rOptions );
-   int iEncoding = int2encoding( rEncoding );
-   int iSyntax = int2syntax( rSyntax );
+   OnigEncodingType * iEncoding = int2encoding( rEncoding );
+   OnigSyntaxType * iSyntax = int2syntax( rSyntax );
 
 
    int r;
@@ -130,6 +147,40 @@ static VALUE oregexp_initialize( VALUE self, VALUE pattern, VALUE options ) {
    return self;
 }
 
+struct RMatch {
+    struct RBasic basic;
+    VALUE str;
+    struct re_registers *regs;
+};
+
+static VALUE oregexp_make_match_data(ORegexp * oregexp, OnigRegion * region, VALUE string_str) {
+    VALUE rb_cMatch = rb_const_get(rb_cObject, rb_intern("MatchData")) ;
+    NEWOBJ(match, struct RMatch);
+    OBJSETUP(match, rb_cMatch, T_MATCH);
+    VALUE kORegexp = rb_const_get( mOniguruma, rb_intern( "ORegexp" ) ) ;
+    int i , count = region->num_regs;
+    struct callback_packet packet;
+
+    match->str = rb_str_new4(string_str);
+    match->regs = ALLOC(struct re_registers);
+    match->regs->allocated = count+1;
+    match->regs->num_regs = count;
+    match->regs->beg = ALLOC_N(int, (count+1));
+    match->regs->end = ALLOC_N(int, (count+1));
+
+    for ( i = 0; i <= count; i++){
+        match->regs->beg[i] = region->beg[i];
+        match->regs->end[i] = region->end[i];
+    }
+    rb_cv_set( kORegexp, "@@last_match", (VALUE)match );
+    packet.region = region;
+    packet.hash = rb_hash_new();
+    if( onig_foreach_name(oregexp->reg, name_callback, &packet) )
+        rb_raise(rb_eException, "Oniguruma Error: group and backreference names are different");
+    rb_iv_set((VALUE)match, "@named_captures", packet.hash);
+    return (VALUE)match;
+}
+ 
 /*
  *  call-seq:
  *     rxp.match(str)   => matchdata or nil
@@ -151,25 +202,7 @@ static VALUE oregexp_match( VALUE self, VALUE string ) {
    OnigRegion *region = onig_region_new();
    int r = onig_search(oregexp->reg, str_ptr, str_ptr + str_len, str_ptr, str_ptr + str_len, region, ONIG_OPTION_NONE);
    if (r >= 0) {
-   
-      VALUE begins = rb_ary_new();
-      VALUE ends = rb_ary_new();
-      nameHash = rb_hash_new();
-      
-      onig_foreach_name(oregexp->reg, name_callback, (void* )region);
-
- 
-      int i;
-
-      for (i = 0; i < region->num_regs; i++) {
-         rb_ary_push( begins, INT2FIX( region->beg[i] ) );
-         rb_ary_push( ends, INT2FIX( region->end[i] ) );
-      }
-      VALUE kMatchData = rb_const_get( mOniguruma, rb_intern( "MatchData" ) );
-      VALUE kORegexp = rb_const_get( mOniguruma, rb_intern( "ORegexp" ) );
-      VALUE matchData = rb_funcall(kMatchData, rb_intern("new"), 4, string_str, begins, ends, nameHash ); 
-      rb_cv_set( kORegexp, "@@last_match", matchData );
-
+      VALUE matchData = oregexp_make_match_data( oregexp, region, string_str);
       onig_region_free(region, 1 );
       return matchData;
    } else if (r == ONIG_MISMATCH) {
@@ -184,11 +217,267 @@ static VALUE oregexp_match( VALUE self, VALUE string ) {
 
 }
 
+static const UChar BACKSLASH  = 0x5c;
+
+/* Additional backslash sequences work in substitution strings: \& (last match), \+ (last
+matched group), \` (string prior to match), \' (string after match), and \\ (a literal
+backslash). */
+
+/* scan the replacement text, looking for substitutions (\n) and \escapes. */
+static VALUE
+oregexp_get_replacement(pat, src_text, repl_text, region)
+     VALUE           pat,
+     		     src_text,
+                     repl_text;
+     OnigRegion * region;
+{
+    ORegexp 	*oregexp;
+    VALUE 	ret;
+    int32_t  replIdx = 0;
+    int32_t  replacementLength = RSTRING(repl_text)->len;
+    UChar    *replacementText = RSTRING(repl_text)->ptr;
+    UChar    *replacementEnd  = replacementText + (replacementLength-1);
+    long numDigits = 0;
+    long groupNum  = 0, g_start, g_end;
+    OnigCodePoint digitC;
+    OnigEncoding  enc;
+    const UChar * matchText;
+    long  matchLen;
+    
+    matchText = RSTRING(src_text)->ptr;
+    matchLen  = RSTRING(src_text)->len;
+    Data_Get_Struct( pat, ORegexp, oregexp );
+    enc = onig_get_encoding( oregexp->reg );
+
+    ret = rb_str_buf_new(RSTRING(repl_text)->len);
+
+    while (replIdx < replacementLength) {
+        OnigCodePoint c = ONIGENC_MBC_TO_CODE(enc, replacementText+replIdx, replacementEnd);
+	int c_len =ONIGENC_MBC_ENC_LEN(enc, replacementText+replIdx) ; 
+        replIdx += c_len;
+        if ( c != BACKSLASH) {
+            /* Common case, no substitution, no escaping,  */
+            /*  just copy the char to the dest buf. */
+	    rb_str_buf_cat( ret, replacementText+replIdx-c_len, c_len);
+            continue;
+        }
+             if (replIdx >= replacementLength) {
+                rb_str_buf_cat(ret, replacementText+(replIdx-c_len), c_len);
+                break;
+             }
+             /* Pick up a capture group number if one follows. */
+            numDigits = 0;
+            groupNum  = 0;
+            for (;;) {
+                    if (replIdx >= replacementLength) {
+                            break;
+                    }
+		    digitC = ONIGENC_MBC_TO_CODE(enc, replacementText+replIdx, replacementEnd);
+		    c_len = ONIGENC_MBC_ENC_LEN(enc, replacementText+replIdx) ; 
+                    if ( ! ONIGENC_IS_CODE_DIGIT(enc, digitC) ) {
+                            break;
+                    }
+		    replIdx += c_len;
+                    groupNum=groupNum*10 + (digitC - '0');
+                    numDigits++;
+                    if (numDigits >= 2) { /* limit 99 groups */
+                       break;
+                    }
+             }
+             if (numDigits == 0) {
+                        /* Additional backslash sequences work in substitution strings: \& (last match), \+ (last
+                        matched group), \` (string prior to match), \' (string after match), and \\ (a literal
+                        backslash). */
+		   int p_len = c_len;
+		   c = ONIGENC_MBC_TO_CODE(enc, replacementText+replIdx, replacementEnd);
+		   c_len = ONIGENC_MBC_ENC_LEN(enc, replacementText+replIdx) ; 
+                   switch(c) {
+                        case '&' :  // matched substring
+				rb_str_buf_cat(ret, matchText+region->beg[0], region->end[0] - region->beg[0]);
+                                replIdx += c_len;
+                                break;
+                        case '`' : // prematch
+                                rb_str_buf_cat(ret, matchText, region->beg[0]);
+                                replIdx += c_len;
+                                break;
+                        case '\'': // postmatch 
+				rb_str_buf_cat(ret, matchText+region->end[0], matchLen - region->end[0]);
+                                replIdx += c_len;
+                                break;
+                        case '\\': // literal backslash
+                                // place single backslash 
+				rb_str_buf_cat(ret, replacementText+replIdx, c_len);
+                                replIdx += c_len;
+                                break;
+                        case '+': // last matched group
+                                replIdx += c_len;
+                                for(groupNum = region->num_regs; groupNum > 0; groupNum --) {
+                                        g_start = region->beg[ groupNum ];
+                                        g_end   = region->end[ groupNum ];
+                                        if( g_start != -1  ) {
+					    rb_str_buf_cat(ret, matchText+g_start, g_end-g_start);
+                                            break;
+                                        }
+                                }
+                                break;
+                                
+                        default:
+                                rb_str_buf_cat(ret, replacementText+(replIdx-p_len), p_len+c_len);
+                                replIdx += c_len;
+                               
+                   }
+             } else {
+                     /* Finally, append the capture group data to the destination. */
+		     if( groupNum < region->num_regs && region->beg[groupNum] >= 0 && region->end[groupNum]>= region->beg[groupNum] ) {
+	     		rb_str_buf_cat(ret, matchText+region->beg[groupNum], region->end[groupNum]-region->beg[groupNum]);
+		     }
+        }             
+    }
+    return ret;
+}
+
+static inline void
+str_mod_check(s, p, len)
+	VALUE s;
+	char *p;
+	long len;
+{
+	if (RSTRING(s)->ptr != p || RSTRING(s)->len != len) {
+		rb_raise(rb_eRuntimeError, "string modified");
+	}
+}
+
+static VALUE
+oregexp_gsub(self, argc, argv,  bang, once, region)
+     VALUE	     self; // pattern
+     int             argc; // should be 1 if block given
+     VALUE          *argv; // either replacement string 
+     int             bang;
+     int	     once;
+     OnigRegion     *region;
+{
+    VALUE           repl;
+    long            beg,
+                    end,
+                    prev_end;
+    int             tainted = 0,
+		    iter = 0;
+    
+    VALUE buf, curr_repl,  block_res;
+    ORegexp *oregexp;
+    
+    if (argc == 1 && rb_block_given_p()) {
+	iter = 1;
+    } else if (argc == 2) {
+	repl = argv[1];
+	Check_Type(repl, T_STRING);
+	if (OBJ_TAINTED(argv[1]))
+	    tainted = 1;
+    } else {
+	rb_raise(rb_eArgError, "wrong number of arguments (%d for 2)", argc);
+    }
+    Data_Get_Struct( self, ORegexp, oregexp );
+
+    VALUE string_str = StringValue( argv[0] );
+    UChar* str_ptr = RSTRING(string_str)->ptr;
+    int str_len = RSTRING(string_str)->len;
+
+    beg = onig_search(oregexp->reg, str_ptr, str_ptr + str_len, str_ptr, str_ptr + str_len, region, ONIG_OPTION_NONE);
+
+    if (beg < 0) {
+	/* no match */
+	if (bang)
+	    return Qnil;
+	return rb_str_dup(string_str);
+    }
+    end = 0;
+    buf = rb_str_buf_new(str_len);
+    do {
+	prev_end = end;
+	beg = region->beg[0];
+	end = region->end[0];
+	rb_str_buf_cat(buf, str_ptr+prev_end, beg-prev_end);
+	if ( iter ) {
+	    VALUE match_data = oregexp_make_match_data( oregexp, region, string_str );
+            rb_backref_set(match_data);
+            if( once ) 
+                block_res = rb_yield( match_data );
+            else {
+                VALUE match_string = rb_str_new( str_ptr+beg, end-beg);
+                block_res = rb_yield_values(2, match_string, match_data );
+            }
+	    str_mod_check( string_str, str_ptr, str_len);
+            curr_repl = rb_obj_as_string(block_res); 
+	} else {
+	    curr_repl = oregexp_get_replacement(self, string_str, repl, region);
+	}
+	rb_str_append(buf, curr_repl);
+	if( once ) break;
+	// find next match
+	beg=onig_search(oregexp->reg, str_ptr, str_ptr + str_len, 
+				      str_ptr+end, str_ptr + str_len, 
+				      region, ONIG_OPTION_NONE);
+    } while ( beg >= 0);
+    rb_str_buf_cat( buf, str_ptr+end, str_len - end);
+
+    if(tainted) 
+	    OBJ_INFECT(buf, repl);
+    OBJ_INFECT(buf, string_str);
+    if (bang) {
+	rb_funcall(string_str, rb_intern("replace"), 1, buf);
+	return string_str;
+    } else {
+	return buf;
+    }
+}
+
+typedef struct gsub_packet_t {
+     VALUE	     self; // pattern
+     int             argc; // should be 1 if block given
+     VALUE          *argv; // either replacement string 
+     int             bang;
+     int	     once;
+     OnigRegion     *region;
+} gsub_packet;
+static VALUE oregexp_packed_gsub( gsub_packet* args ) {
+    return oregexp_gsub(args->self, args->argc, args->argv, args->bang, args->once, args->region);
+}
+void oregexp_cleanup_region(OnigRegion * region){
+    onig_region_free(region, 1);       
+}
+static VALUE oregexp_safe_gsub(self, argc, argv,  bang, once)
+     VALUE	     self; // pattern
+     int             argc; // should be 1 if block given
+     VALUE          *argv; // either replacement string 
+     int             bang;
+     int	     once;
+{
+        OnigRegion * region = onig_region_new();
+        gsub_packet call_args = {self, argc, argv, bang, once, region};
+        return rb_ensure( oregexp_packed_gsub, (VALUE)&call_args, oregexp_cleanup_region, (VALUE)region);
+}
+static VALUE oregexp_m_gsub(int argc, VALUE *argv, VALUE self) {
+	return oregexp_safe_gsub(self, argc, argv, 0, 0);
+}
+static VALUE oregexp_m_sub(int argc, VALUE *argv, VALUE self) {
+	return oregexp_safe_gsub(self, argc, argv, 0, 1);
+}
+
+static VALUE oregexp_m_gsub_bang(int argc, VALUE *argv, VALUE self) {
+	return oregexp_safe_gsub(self, argc, argv, 1, 0);
+}
+static VALUE oregexp_m_sub_bang(int argc, VALUE *argv, VALUE self) {
+	return oregexp_safe_gsub(self, argc, argv, 1, 1);
+}
+
 void Init_oregexp() {
    mOniguruma = rb_define_module("Oniguruma");
    VALUE cORegexp = rb_define_class_under(mOniguruma, "ORegexp", rb_cObject);
    rb_define_alloc_func(cORegexp, oregexp_allocate);
    rb_define_method( cORegexp, "initialize", oregexp_initialize, 2 );
    rb_define_method( cORegexp, "match", oregexp_match, 1 );
-   
+   rb_define_method( cORegexp, "gsub", oregexp_m_gsub, -1 );
+   rb_define_method( cORegexp, "sub",  oregexp_m_sub,  -1 );
+   rb_define_method( cORegexp, "gsub!", oregexp_m_gsub_bang, -1 );
+   rb_define_method( cORegexp, "sub!",  oregexp_m_sub_bang,  -1 );
 }

data/lib/oniguruma.rb

@@ -254,7 +254,7 @@ module Oniguruma
          return nil unless string
          m = match( string )
          return nil unless m
-         m.begin
+         m.begin(0)
       end
       
       # call-seq:
@@ -289,7 +289,7 @@ module Oniguruma
                matches << m
                positions << position
                tmp_string = m.post_match
-               position += m.end
+               position += m.end(0)
                #if m.end == m.begin
                #   tmp_string = tmp_string[1..-1]
                #   position += 1
@@ -304,51 +304,6 @@ module Oniguruma
             nil
          end
       end
-      
-      def sub string, replacement = nil
-         matches = match( string )
-         if matches
-            replacement = yield matches[0] unless replacement
-            string.sub( matches[0], replacement )
-         else
-            return string
-         end
-      end
-      
-      def gsub string, replacement = nil
-         result = string
-         matches = match_all( string )
-         string_replace = replacement
-         if matches
-            matches.each do |m, p|
-               replacement = yield( m[0], m ) unless string_replace
-               result = result.sub( m[0], replacement )
-            end
-         end
-         result
-      end
-      
-      def sub! string, replacement = nil
-         matches = match( string )
-         if matches
-            replacement = yield matches[0] unless replacement
-            string.sub!( matches[0], replacement )
-         else
-            return string
-         end
-      end
-      
-      def gsub! string, replacement = nil
-         matches = match_all( string )
-         string_replace = replacement
-         if matches
-            matches.each do |m, p|
-               replacement = yield( m[0], m ) unless string_replace
-               string.sub!( m[0], replacement )
-            end
-         end
-         string
-      end
    end
    
    class MultiMatchData
@@ -371,11 +326,11 @@ module Oniguruma
       end
       
       def begin index
-         @matches[index].begin + @positions[index]
+         @matches[index].begin(0) + @positions[index]
       end
       
       def end index
-         @matches[index].end + @positions[index]
+         @matches[index].end(0) + @positions[index]
       end
       
       def length
@@ -402,90 +357,15 @@ module Oniguruma
       end
    end
    
-   class MatchData
-      def initialize( string, starts, ends, names )
-         @string = string
-         @starts = starts
-         @ends = ends
-         @matches = []
-         @starts.size.times do |i|
-            @matches << @string[@starts[i]...@ends[i]]
-         end
-         @match_count = @matches.size
-         @start_pos = 0
-         @names = names
-      end
-      
-      def [] ( value1, value2 = nil )
-         unless value2
-            if index = to_index( value1 )
-               @matches[index]
-            else
-               nil
-            end
-         else
-            @matches[value1, value2]
-         end
-      end
-      
-      def to_index name 
-         if name.is_a? Symbol
-            @names[name]
-         else
-            name
-         end
-      end
-      
-      def begin index = 0
-         @starts[to_index( index )]
-      end
-      
-      def end index = 0
-         @ends[to_index( index )]
-      end
-      
-      def captures
-         @matches[1..-1]
-      end
-      
-      def length
-         @match_count
-      end
-      alias size length
-      
-      def offset index = 0
-         [@starts[to_index( index )], @ends[to_index( index )]]
-      end
-      
-      def post_match
-         @string[@ends[0], @string.length]
-      end
-      
-      def pre_match
-         @string[0, @starts[0]]
-      end
-      
-      def select &block
-         @matches.select( &block )
-      end
-      
-      def string
-         @string.freeze
-      end
-      
-      def to_a
-         @matches
-      end
-      
-      def to_s
-         @matches[0]
-      end
-      
-      def values_at *values
-         result = []
-         values.each { |v| result << @matches[v] }
-         result
-      end
-   end
 end
-
+class ::MatchData
+  alias old_aref :[]
+  def [](*idx)
+    if idx[0].is_a?(Symbol) 
+      k = @named_captures && @named_captures[idx[0]]
+      k && old_aref(k)
+    else
+      old_aref(*idx)
+    end
+  end
+end

data/test/test_oniguruma.rb

@@ -210,5 +210,68 @@ class MatchDataTestCase < Test::Unit::TestCase
       assert_equal( ')', matches[:end] )
       assert_equal( nil, matches[:inexistent])
    end
+
+   def test_utf8_ignore_case
+     reg = Oniguruma::ORegexp.new( '([а-я])+', :options => Oniguruma::OPTION_IGNORECASE, :encoding => Oniguruma::ENCODING_UTF8 )
+     matches = reg.match("Text: Ехал Грека Через Реку")
+     assert_not_nil( matches )
+     assert_equal("Ехал", matches[0])
+     reg = Oniguruma::ORegexp.new( 'р(уби.*)', :options => Oniguruma::OPTION_IGNORECASE, :encoding => Oniguruma::ENCODING_UTF8 )
+     assert_equal("*убил бы*", reg.gsub("Руби", '*\1л бы*') )
+   end
+   
+   def test_utf8_gsub
+     reg = Oniguruma::ORegexp.new( '([а-я])([а-я])([а-я]+)', :options => Oniguruma::OPTION_IGNORECASE, :encoding => Oniguruma::ENCODING_UTF8 )
+     new_str = reg.gsub("Text: Ехал Грека Через Реку") {|s,m| m[1]*2+m[2]*2+m[3] }
+     assert_equal("Text: ЕЕххал ГГррека ЧЧеерез РРееку", new_str)
+   end
+   
+   def test_utf8_gsub2
+     reg = Oniguruma::ORegexp.new( '[а-я]', :options => Oniguruma::OPTION_IGNORECASE, :encoding => Oniguruma::ENCODING_UTF8 )
+     new_str = reg.gsub("Text: Ехал Грека Через Реку") {|s,m| s*2 }
+     assert_equal("Text: ЕЕххаалл ГГррееккаа ЧЧеерреезз РРееккуу", new_str)
+   end
    
+   def test_sub_compatibility
+     $x = "a.gif"
+     assert_equal("b.gif", $x.osub('.*\.([^\.]+)$', 'b.\1'))
+     assert_equal("\\.gif", $x.osub('.*\.([^\.]+)$', '\\.\1'))
+     assert_equal("gif", $x.osub('.*\.([^\.]+)$', '\1'))
+     assert_equal("", $x.osub('.*\.([^\.]+)$', '\2'))
+     assert_equal("ab", $x.osub('.*\.([^\.]+)$', 'a\2b'))
+     assert_equal("<a.gif>", $x.osub('.*\.([^\.]+)$', '<\&>'))
+     assert_equal("a.a.", $x.osub('(gif)', '\`') )
+   end
+   
+  class ::String
+    def ogsub(*args)
+      Oniguruma::ORegexp.new(args.shift).gsub(self, *args)
+    end
+    def ogsub!(*args)
+      Oniguruma::ORegexp.new(args.shift).gsub!(self, *args)
+    end
+    def osub(re, *args)
+      Oniguruma::ORegexp.new( re ).sub(self, *args)
+    end
+  end
+ 
+  def test_gsub_compat
+    assert_equal("hello".ogsub('[aeiou]', '*')              , "h*ll*")
+    assert_equal("hello".ogsub('([aeiou])', '<\1>')         , "h<e>ll<o>")
+    i = 0 
+    assert_equal("12345" , Oniguruma::ORegexp.new('.').gsub("hello") {|s,m| i+=1; i.to_s})
+    assert_equal("214365", Oniguruma::ORegexp.new('(.)(.)').gsub("123456") {|s,m| m[2] + m[1] }) 
+    a = "test"
+    a.ogsub!('t', a)
+    assert_equal("testestest", a)
+  end
+
+  def test_match_compat
+    t = Oniguruma::ORegexp.new('(.)(.)').gsub("123456") {|s,m| "#$2#$1" }
+    assert_equal("214365", t ) 
+    t = Oniguruma::ORegexp.new('([aeiou])').gsub("hello") {|s,m| "<#$1>" }
+    assert_equal( "h<e>ll<o>", t)
+  end
+
+
 end

metadata

@@ -3,8 +3,8 @@ rubygems_version: 0.9.2
 specification_version: 1
 name: oniguruma
 version: !ruby/object:Gem::Version 
-  version: 0.9.0
-date: 2007-03-22 00:00:00 +01:00
+  version: 0.9.1
+date: 2007-03-25 00:00:00 +01:00
 summary: Bindings for the oniguruma regular expression library
 require_paths: 
 - lib
@@ -33,6 +33,7 @@ files:
 - History.txt
 - Manifest.txt
 - README.txt
+- Syntax.txt
 - Rakefile
 - lib/oniguruma.rb
 - ext/oregexp.c