japetheape-tmail 1.2.3.4

data/CHANGES

@@ -0,0 +1,74 @@
+=== 1.2.3.1 / 2008-04-11
+
+* Closed #19429 - Installing TMail on Windows with the gem
+
+=== 1.2.3 / 2008-04-11
+
+* Closed #18881 - TMail goes into an endless loop if sent an crafted email
+* Closed #19203 - TMail errors in Ruby 1.9.1 on invalid multibyte chars.
+* Closed #18814 - Fixed attchment.rb failing on mail part that had a nil content-type
+* Closed #18516 - Fix TMail::Mail#preamble, and add #preamble= (Charles Lowe) (233)
+* Applied patch #18515 to remove ftools from test case (Charles Lowe) (232)
+* A lot of documentation patches to the mail and utils and net files.  Also added "log/BugTrackingLog.txt" for myself as my bug log got clobbered by the new changelog.txt format. (231)
+
+=== 1.2.2 / 2008-03-07
+
+* Fixed install bug with gem pacakge (1.2.1 was not compiling on gem install)
+* A _LOT_ more documentation...!
+* More documentation - (Mikel)
+* Applied Ruby 1.9 patches to the library - All tests passing now - (Mikel)
+* Closed #17719 - Fixed UNIXMbox code - readonly was not working and raising an exception.  Now works.
+* Closed #18038 - Multiple froms not being parsed correctly, added a test case to cover this and show the correct handling - (Mikel)
+
+=== 1.2.1 / 2008-01-11
+
+* More documentation (Mikel)
+* Added 15 test cases from the Rails ActionMailer to TMail
+* Changed mailscanner to tmailscanner (mailscanner is copyrighted)
+* Closed Bug -  Handled quote boundary being gready on content-type header (M. Mondragon)
+* Closed #16025 - Fixed scanner.rb so it passes same tests as scanner.c
+* Closed #16283 - Handled incorrect decoding of attachments (M. Aoki - garyo)
+* Closed #16899 - HeaderField.new_from_port and added test cases to cover this code (Maarten O.)
+* Closed #16900 - UNIXMbox.fromaddr missing port param and does not return Envelope Sender (Maarten O.)
+
+=== 1.2.0 / 2007-11-29
+
+* 5 major enhancements:
+  * Extensive documentation work. (mikel)
+  * Renamed scanner_c.c to mailscanner.c. (trans)
+  * Removed base64 c extension. It's speed benefit was negligable (only 0.2 sec over 10000 sizable encode/decode runs) (trans)
+  * Closed 15445 - TMail::Mail#create_forward now returns a new Mail object that has the original mail as an encoded 7 bit multipart attachment.  Also moved create_forward and create_reply from tmail/net into tmail/interface as it makes more sense to have it there. (mikel)
+  * Closed 15643 - TMail::Mail#reply_addresses was returning an empty array if reply_to was set to nil (ie, the header field existed but was empty) instead of returning the from address or default. (mikel)
+  * Closed 16025 - TMail scanner.rb would not parse ATOM chars correctly making it fail tests where the C version passed them.  Fixed this by updating the Scanner.rb version to be in step with the C version (there was an extra @ symbol in the ATOM CHARS definition that was not in the C version.) (mikel)
+  * Fixed scanner.rb so that it would pass the same tests that the C version does - had a sundry @ symbol inside of the ATOM CHARS
+* 3 minor enhancements:
+  * Renamed scanner_c.c to tmailscanner.c (trans)
+  * Changed TMail::Mail#sender to have a default "default" value of nil to be in alignment with all the other interface methods (mikel)
+  * Made base64_decode and base64_encode into ! versions as they are destructive on the mail body (encodes the body to base64 or decodes it), made aliases for base64_decode and base64_encode to point back to the bang versions.  Doing this with a view to change base64_encode to a non destructive version (returns the encoded body) in future versions. (mikel)
+
+=== 1.1.1 / 2007-11-05
+
+* 3 major enhancement:
+  * Created unified package, for installation in any platform.
+  * Added require_arch.rb to facilitate multi-platform support.
+  * If compilation fails, set NORUBYEXT="true" and reinstall.
+* 3 minor enhancement:
+  * Fixed line wrapping of long header fields so that they wrap at the correct whitespace points.
+  * Fixed bug where re-assigning the mail.body to existing mail object that already had a parsed body would not re-parse the body.
+  * Started documenting the source code... lots more to do.
+
+=== 1.1.0 / 2007-10-28
+
+* 1 minor enhancements:
+  * Changed the quoting of paramaters in the header fields to wrap double quotes around fields that are needed to be quoted.
+  * Removed keeping double quotes around a filename that does not need double quotes per RFC 1521
+  * More clean up and getting tests passing.  Now standing at 2 failures out of 3366 assertions.  One is the incorrect handling of "@"@test.com (returns @@test.com) and the other is a japanese encoding issue.
+
+=== 1.0.0 / 2007-10-28
+
+* 1 major enhancement:
+  * TMail is now released as a GEM!
+* 2 minor enhancements:
+  * Fixed bug 15077 - TMail now recognizes attachments as soon as they are added to the body.
+  * Refactored handling of quotations in header fields - now cleaner
+

data/LICENSE

@@ -0,0 +1,21 @@
+Copyright (c) 2001-2007 Minero Aoki
+Changes Copyright (c) 2007 Mikel Lindsaar
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/NOTES

@@ -0,0 +1,7 @@
+[ANN] TMail 1.2.3.1 - Minor Fix to Installer
+
+This release just handles a bug with installing the gem on a Windows platform.
+
+It is otherwise identical to 1.2.3
+
+Mikel

data/README

@@ -0,0 +1,169 @@
+= TMail
+
+    http://tmail.rubyforge.org/
+
+    Mikel Lindsaar   maintainer
+    Trans            assitant developer
+    Minero Aoki      original developer   
+
+== DESCRIPTION:
+
+TMail is a mail handling library for Ruby.  It abstracts a mail message into a usable object allowing you to read, set, add and delete headers and the mail body.
+
+TMail is used by the Ruby on Rails web framework as the Email abstraction layer for their ActionMailer module.  It is also used by the Nitro framework and many other applications on and off the web.
+
+The goal of the TMail handling library is to be able to parse and handle raw Email sources and produce RFC compliant Emails as a result.  If you find something that TMail does that violates an RFC, we want to know and we'll get it fixed fast.
+
+== DOCUMENTATION:
+
+The place you will want to look first is the TMail::Mail class.  This has the vast majority of methods you will be using to talk to your TMail object.
+
+== FEATURES/PROBLEMS:
+
+TMail is fairly RFC compliant on the handling of emails.
+
+There are also some problems in the header handling, but for 99.9% of email, you will be fine.  Usually, the problems revolve around parsing incomming emails and making sense of them.
+
+I really welcome any examples of Emails that "didn't work" with TMail so I can use them as test cases.
+
+== SYNOPSIS:
+
+TMail is very easy to use.  You simply require the library and then pass a raw email text message into the TMail::Mail.parse method.  This returns a TMail::Mail object which you can now query and run methods against to modify, inspect or add to the Email.
+
+You can find almost all of the methods that you will use to talk to and update a TMail instance in the TMail::Mail class.  I am constantly updating this code, with comments, added a fair bit and have a lot more to go!.
+
+=== Short Version:
+
+  irb(main):001:0> require 'tmail'
+  irb(main):002:0> raw_email = File.open("my_raw_email", 'r') { |f| @mail = f.read }
+  irb(main):003:0> email = TMail::Mail.parse(raw_email)
+  irb(main):004:0> puts email['to']
+  mikel@example.com
+  => nil
+  irb(main):005:0> email['to'] = 'mikel@somewhere.else.com'
+  => "mikel@somewhere.else.com"
+  irb(main):006:0> puts email['to']
+  mikel@somewhere.else.com
+  => nil
+
+=== Longer Version:
+
+Assuming you have a single raw email in the variable my_message, you can do the following:
+
+  require 'tmail'
+  email = TMail::Mail.parse(my_message)
+  
+This will give you a TMail::Mail class containing your parsed message.  There are other methods of opening emails through Ports.
+
+You can view this email by a simple puts:
+
+  puts email
+
+  Return-Path: <mikel@nowhere.com>
+  Date: Sun, 21 Oct 2007 19:38:13 +1000
+  From: Mikel Lindsaar <mikel@nowhere.com>
+  To: mikel@somewhere.com
+  Message-Id: <009601c813c6$19df3510$0437d30a@mikel091a>
+  Subject: Testing Email
+
+  Hello Mikel
+  
+Easy right?
+
+=== Adding a header to the EMail:
+
+Say now that you have opened your message, you want to put in a Reply-To field.  You do this like so:
+
+  email['reply-to'] = "My Email Address <my_address@anotherplace.com>"
+
+Is it really there?  Well, find out with a puts:
+
+  puts email
+
+  Return-Path: <mikel@nowhere.com>
+  Date: Sun, 21 Oct 2007 19:38:13 +1000
+  From: Mikel Lindsaar <mikel@nowhere.com>
+  Reply-To: My Email Address <my_address@anotherplace.com>
+  To: mikel@somewhere.com
+  Message-Id: <009601c813c6$19df3510$0437d30a@mikel091a>
+  Subject: Testing Email
+
+  Hello Mikel
+
+Yup looks good.
+
+=== Inspecting a header:
+
+You can then inspect your added header by doing:
+
+  email['reply-to'] # => #<TMail::AddressHeader "My Email Address <my_address@anotherplace.com>">
+
+If you just want to the actual value, not the AddressHeader object, pass to_s to this.
+
+  email['reply-to'].to_s # => "My Email Address <my_address@anotherplace.com>"
+
+=== Deleting a header:
+
+One way of deleting a header from an Email is just assigning it nil like so:
+
+  email['reply-to'] = nil # => nil
+  
+If you now puts the email again, it will not be included:
+
+  puts email
+  
+  Return-Path: <mikel@nowhere.com>
+  Date: Sun, 21 Oct 2007 19:38:13 +1000
+  From: Mikel Lindsaar <mikel@nowhere.com>
+  To: mikel@somewhere.com
+  Message-Id: <009601c813c6$19df3510$0437d30a@mikel091a>
+  Subject: Testing Email
+
+  Hello Mikel
+
+=== Writing out an Email:
+
+You can just call to_s on any email to have it serialized out as a single string with the right number of line breaks and encodings.
+
+== CONTRIBUTING:
+
+You can visit the {Contributing to TMail}[link:http://tmail.rubyforge.org/contributing/] to find out how to contribute to TMail, developers are welcome and wanted!
+
+== REQUIREMENTS:
+
+* C compiler if you want the Ruby extension for Scanner
+* Ruby 1.8 or later
+
+== INSTALLATION:
+
+* sudo gem install tmail
+
+Or manually,
+
+* sudo script/setup
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2007 FIX
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+

data/Rakefile

@@ -0,0 +1,2 @@
+require File.join(File.dirname(__FILE__), "setup.rb")
+

data/ext/Makefile

@@ -0,0 +1,20 @@
+#
+# ext/tmail/Makefile
+#
+
+.PHONY: tmailscanner clean distclean
+
+all: tmailscanner
+
+tmailscanner: tmailscanner/tmail/Makefile
+	cd tmailscanner/tmail; $(MAKE)
+
+tmailscanner/tmail/Makefile: tmailscanner/tmail/extconf.rb
+	cd tmailscanner/tmail; ruby extconf.rb
+
+clean:
+	cd tmailscanner/tmail; $(MAKE) clean
+
+distclean:
+	cd tmailscanner/tmail; $(MAKE) distclean
+

data/ext/tmailscanner/tmail/MANIFEST

@@ -0,0 +1,4 @@
+MANIFEST
+tmailscanner.c
+extconf.rb
+depend

data/ext/tmailscanner/tmail/depend

@@ -0,0 +1 @@
+tmailscanner.o: tmailscanner.c $(hdrdir)/ruby.h $(topdir)/config.h $(hdrdir)/defines.h Makefile

data/ext/tmailscanner/tmail/extconf.rb

@@ -0,0 +1,33 @@
+require 'mkmf'
+require 'rbconfig'
+
+extension_name = 'tmailscanner'
+
+windows = (/djgpp|(cyg|ms|bcc)win|mingw/ =~ RUBY_PLATFORM)
+
+# For now use pure Ruby tmailscanner if on Windows, since 
+# most Window's users don't have developer tools needed.
+
+if (ENV['NORUBYEXT'] == 'true' || windows)
+  if windows
+    File.open('make.bat', 'w') do |f|
+      f << 'echo Native extension will be omitted.'
+    end
+    File.open('nmake.bat', 'w') do |f|
+      f << 'echo Native extension will be omitted.'
+    end
+  end
+  File.open('Makefile', 'w') do |f|
+    f << "all:\n"
+    f << "install:\n"
+  end
+else
+  #dir_config(extension_name)
+  if windows && ENV['make'].nil?
+    $LIBS += " msvcprt.lib"
+  else
+    $CFLAGS += " -D_FILE_OFFSET_BITS=64"  #???
+  end
+  create_makefile(extension_name)
+end
+

data/ext/tmailscanner/tmail/tmailscanner.c

@@ -0,0 +1,607 @@
+/*
+
+    tmailscanner.c
+
+    Copyright (c) 1998-2007 Minero Aoki
+
+    This program is free software.
+    You can distribute/modify this program under the terms of
+    the GNU Lesser General Public License version 2.1.
+
+*/
+
+#include <stdio.h>
+#ifdef __STDC__
+# include <stdlib.h>
+#endif
+
+#include "ruby.h"
+
+#include "ruby/encoding.h"
+
+#define TMAIL_VERSION "1.2.3"
+
+static VALUE TMailScanner;
+static VALUE ScanError;
+
+struct scanner
+{
+    char *pbeg;
+    char *p;
+    char *pend;
+    unsigned int flags;
+    VALUE comments;
+};
+
+#define MODE_MIME     (1 << 0)
+#define MODE_RECV     (1 << 1)
+#define MODE_ISO2022  (1 << 2)
+#define MODE_DEBUG    (1 << 4)
+
+#define MIME_MODE_P(s)     ((s)->flags & MODE_MIME)
+#define RECV_MODE_P(s)     ((s)->flags & MODE_RECV)
+#define ISO2022_MODE_P(s)  ((s)->flags & MODE_ISO2022)
+
+#define GET_SCANNER(val, s) Data_Get_Struct(val, struct scanner, s)
+
+
+
+
+/*
+// begin @japetheape
+int
+mbclen(const char p)
+{
+    return rb_enc_mbclen(&p);
+}
+
+int
+ismbchar(const char p)
+{
+    return mbclen((p)) != 1;
+}
+// end @japethape
+*/
+
+
+static void
+mails_free(sc)
+    struct scanner *sc;
+{
+    free(sc);
+}
+
+#ifndef StringValue
+#  define StringValue(s) Check_Type(str, T_STRING);
+#endif
+
+/*
+ * Document-method: mails_s_new
+ *
+ * Creates a new mail
+ *
+ */
+static VALUE
+mails_s_new(klass, str, ident, cmt)
+    VALUE klass, str, ident, cmt;
+{
+    struct scanner *sc;
+    const char *tmp;
+
+    sc = ALLOC_N(struct scanner, 1);
+
+    StringValue(str);
+    sc->pbeg = RSTRING_PTR(str);
+    sc->p    = sc->pbeg;
+    sc->pend = sc->p + RSTRING_LEN(str);
+
+    sc->flags = 0;
+    Check_Type(ident, T_SYMBOL);
+    tmp = rb_id2name(SYM2ID(ident));
+    if      (strcmp(tmp, "RECEIVED")     == 0) sc->flags |= MODE_RECV;
+    else if (strcmp(tmp, "CTYPE")        == 0) sc->flags |= MODE_MIME;
+    else if (strcmp(tmp, "CENCODING")    == 0) sc->flags |= MODE_MIME;
+    else if (strcmp(tmp, "CDISPOSITION") == 0) sc->flags |= MODE_MIME;
+
+    tmp = rb_get_kcode();
+    if (strcmp(tmp, "EUC") == 0 || strcmp(tmp, "SJIS") == 0) {
+        sc->flags |= MODE_ISO2022;
+    }
+
+    sc->comments = Qnil;
+    if (! NIL_P(cmt)) {
+        Check_Type(cmt, T_ARRAY);
+        sc->comments = cmt;
+    }
+
+    return Data_Wrap_Struct(TMailScanner, 0, mails_free, sc);
+}
+
+/*
+ * Document-method: mails_debug_get
+ *
+ * TODO: Documentation needed
+ *
+ */
+static VALUE
+mails_debug_get(self)
+    VALUE self;
+{
+    struct scanner *sc;
+
+    GET_SCANNER(self, sc);
+    if (sc->flags & MODE_DEBUG)
+        return Qtrue;
+    else
+        return Qfalse;
+}
+
+/*
+ * Document-method: mails_debug_set
+ *
+ * TODO: Documentation needed
+ *
+ */
+static VALUE
+mails_debug_set(self, flag)
+    VALUE self, flag;
+{
+    struct scanner *sc;
+
+    GET_SCANNER(self, sc);
+    if (RTEST(flag))
+        sc->flags |= MODE_DEBUG;
+    else
+        sc->flags &= ~MODE_DEBUG;
+    return Qnil;
+}
+
+
+/*
+----------------------------------------------------------------------
+                        scanning routines
+----------------------------------------------------------------------
+*/
+
+#define ESC               '\033'
+#define ATOM_SYMBOLS      "_#!$%&'`*+-{|}~^/=?"
+#define TOKEN_SYMBOLS     "_#!$%&'`*+-{|}~^."
+#define ATOM_SPECIAL      "()<>[]@,;:\"\\."
+#define TOKEN_SPECIAL     "()<>[]@,;:\"\\/?="
+#define LWSP              " \t\r\n"
+
+#define IS_ALPHA(ch)      (('a' <= ch && ch <= 'z') || ('A' <= ch && ch <= 'Z'))
+#define IS_UPPER(ch)      ('A' <= ch && ch <= 'Z')
+#define TO_LOWER(ch)      (IS_UPPER(ch) ? ch + 32 : ch)
+#define IS_LWSP(ch)       (strchr(LWSP, ch))
+#define IS_DIGIT(ch)      ('0' <= ch && ch <= '9')
+#define IS_WORDCHAR(ch, symlist) \
+                          (IS_ALPHA(ch) || IS_DIGIT(ch) || strchr(symlist, ch))
+#define IS_ATOMCHAR(ch)   IS_WORDCHAR(ch, ATOM_SYMBOLS)
+#define IS_TOKENCHAR(ch)  IS_WORDCHAR(ch, TOKEN_SYMBOLS)
+#define IS_JCHAR(ch)      MBCLEN_CHARFOUND_LEN(ch) != 1
+
+
+
+
+
+/* I know this implement is ugly, but usually useful. */
+
+/* skip until "\e(B" (us-ascii) */
+static void
+skip_iso2022jp_string(sc)
+    struct scanner *sc;
+{
+    for (; sc->p < sc->pend; sc->p++) {
+        if (*sc->p == ESC) {
+            if (strncmp(sc->p, "\033(B", 3) == 0) {
+                sc->p += 3;
+                return;
+            }
+        }
+    }
+}
+
+static void
+skip_japanese_string(sc)
+    struct scanner *sc;
+{
+    /*
+    while (sc->p < sc->pend) {
+        if (! ismbchar(*sc->p)) return;
+        sc->p += mbclen(*sc->p);
+    }
+    */
+}
+
+
+#define scan_atom(sc) scan_word(sc, ATOM_SYMBOLS)
+#define scan_token(sc) scan_word(sc, TOKEN_SYMBOLS)
+
+static VALUE
+scan_word(sc, syms)
+    struct scanner *sc;
+    char *syms;
+{
+    char *beg = sc->p;
+
+    while (sc->p < sc->pend) {
+        if (ISO2022_MODE_P(sc) && *sc->p == ESC) {
+            skip_iso2022jp_string(sc);
+        }
+        else if (IS_JCHAR(*sc->p)) {
+            skip_japanese_string(sc);
+        }
+        else if (IS_WORDCHAR(*sc->p, syms)) {
+            sc->p++;
+        }
+        else {
+            break;
+        }
+    }
+
+    return rb_str_new(beg, sc->p - beg);
+}
+
+
+#define BUFSIZE 256
+
+static VALUE
+scan_quoted_word(sc)
+    struct scanner *sc;
+{
+    char buf[BUFSIZE];
+    char *p;
+    char *save;
+    VALUE result = rb_str_new("", 0);
+
+    sc->p++;   /* discard first dquote */
+    p = buf;
+    while (sc->p < sc->pend) {
+        if (*sc->p == '"') {
+            sc->p++;   /* discard terminal dquote */
+            rb_str_cat(result, buf, p - buf);
+            return result;
+        }
+        if (ISO2022_MODE_P(sc) && *sc->p == ESC) {
+            save = sc->p;
+            skip_iso2022jp_string(sc);
+            while (save < sc->p) {
+                *p++ = *save++;
+                if (p >= buf + BUFSIZE) {
+                    /* flush buffer */
+                    rb_str_cat(result, buf, BUFSIZE);
+                    p = buf;
+                }
+            }
+            continue;
+        }
+
+        if (*sc->p == '\\')
+            sc->p++;   /* discard quoting backslash */
+        *p++ = *sc->p++;
+        if (p >= buf + BUFSIZE) {
+            /* flush buffer */
+            rb_str_cat(result, buf, BUFSIZE);
+            p = buf;
+        }
+    }
+
+    rb_raise(ScanError, "unterminated quoted-word");
+    return Qnil;
+}
+
+static VALUE
+scan_domain_literal(sc)
+    struct scanner *sc;
+{
+    char buf[BUFSIZE];
+    char *p;
+    VALUE result = rb_str_new("", 0);
+
+    p = buf;
+    while (sc->p < sc->pend) {
+        if (*sc->p == ']') {
+            *p++ = *sc->p++;
+            rb_str_cat(result, buf, p - buf);
+            return result;
+        }
+
+        if (*sc->p == '\\')
+            sc->p++;   /* discard backslash */
+        *p++ = *sc->p++;
+        if (p >= buf + BUFSIZE) {
+            /* flush buffer */
+            rb_str_cat(result, buf, BUFSIZE);
+            p = buf;
+        }
+    }
+
+    rb_raise(ScanError, "unterminated domain literal");
+    return Qnil;
+}
+
+
+static VALUE
+scan_comment(sc)
+    struct scanner *sc;
+{
+    int nest = 1;
+    char *p;
+    VALUE ret = rb_str_new("", 0);
+
+    sc->p++;
+    p = sc->p;
+    while (sc->p < sc->pend) {
+        if (ISO2022_MODE_P(sc) && *sc->p == ESC) {
+            skip_iso2022jp_string(sc);
+        }
+        else if (IS_JCHAR(*sc->p)) {
+            skip_japanese_string(sc);
+        }
+        else {
+            switch (*sc->p) {
+            case '(':
+                nest++;
+                break;
+            case ')':
+                nest--;
+                if (nest == 0) {
+                    rb_str_cat(ret, p, sc->p - p);
+                    sc->p++;
+                    return ret;
+                }
+                break;
+            case '\\':
+                rb_str_cat(ret, p, sc->p - p);
+                sc->p++;
+                if (sc->p == sc->pend)
+                    rb_raise(ScanError, "incomplete char quote");
+                p = sc->p;
+                break;
+            default:
+                break;
+            }
+            sc->p++;
+        }
+    }
+
+    rb_raise(ScanError, "unterminated comment");
+    return Qnil;
+}
+
+
+static void
+skip_lwsp(sc)
+    struct scanner *sc;
+{
+    while (sc->p < sc->pend) {
+        if (IS_LWSP(*sc->p)) sc->p++;
+        else break;
+    }
+}
+
+static int
+nccmp(a, b)
+    char *a, *b;
+{
+    while (*a && *b) {
+        if ((*a != *b) && (TO_LOWER(*a) != TO_LOWER(*b)))
+            return 0;
+        a++; b++;
+    }
+    return (*a == *b);
+}
+
+static int
+digit_p(str)
+    VALUE str;
+{
+    char *p;
+    int i;
+
+    p = RSTRING_PTR(str);
+    for (i = 0; i < RSTRING_LEN(str); i++) {
+        if (! IS_DIGIT(RSTRING_PTR(str)[i]))
+            return 0;
+    }
+    return 1;
+}
+
+static VALUE tok_atom, tok_digit, tok_token, tok_quoted, tok_domlit;
+static VALUE tok_from, tok_by, tok_via, tok_with, tok_id, tok_for;
+
+static VALUE
+atomsym(sc, str)
+    struct scanner *sc;
+    VALUE str;
+{
+    if (digit_p(str)) {
+        return tok_digit;
+    }
+    else if (RECV_MODE_P(sc)) {
+        char *p = RSTRING_PTR(str);
+        if      (nccmp(p, "from")) return tok_from;
+        else if (nccmp(p, "by"))   return tok_by;
+        else if (nccmp(p, "via"))  return tok_via;
+        else if (nccmp(p, "with")) return tok_with;
+        else if (nccmp(p, "id"))   return tok_id;
+        else if (nccmp(p, "for"))  return tok_for;
+    }
+    return tok_atom;
+}
+
+static void
+debug_print(sc, sym, val)
+    struct scanner *sc;
+    VALUE sym, val;
+{
+    VALUE s;
+
+    s = rb_funcall(sym, rb_intern("inspect"), 0),
+    printf("%7ld %-10s token=<%s>\n",
+           (unsigned long)(sc->pend - sc->p),
+           RSTRING_PTR(s),
+           RSTRING_PTR(val));
+}
+
+#define D(expr) do {\
+    if (sc->flags & MODE_DEBUG) {expr;}\
+} while (0)
+
+static void
+pass_token(sc, sym, tok, arr)
+    struct scanner *sc;
+    VALUE sym, tok, arr;
+{
+    D(debug_print(sc, sym, tok));
+    rb_ary_store(arr, 0, sym);
+    rb_ary_store(arr, 1, tok);
+    rb_yield(arr);
+}
+
+/*
+ * Document-method: mails_scan
+ *
+ * TODO: Documentation needed
+ *
+ */
+static VALUE
+mails_scan(self)
+    VALUE self;
+{
+    struct scanner *sc;
+    VALUE arr;
+
+#define PASS(s,v) pass_token(sc,s,v,arr)
+    GET_SCANNER(self, sc);
+    if (!sc->p) {
+        rb_raise(ScanError, "Mails#scan called before reset");
+    }
+    arr = rb_assoc_new(Qnil, Qnil);
+
+    while (sc->p < sc->pend) {
+        D(puts("new loop"));
+        D(printf("char='%c'\n", *sc->p));
+        if (IS_LWSP(*sc->p)) {
+            D(puts("lwsp"));
+            skip_lwsp(sc);
+            if (sc->p >= sc->pend)
+                break;
+        }
+
+        if (MIME_MODE_P(sc)) {
+            if (IS_TOKENCHAR(*sc->p) ||
+                    (ISO2022_MODE_P(sc) && (*sc->p == ESC)) ||
+                    IS_JCHAR(*sc->p)) {
+                D(puts("token"));
+                PASS(tok_token, scan_token(sc));
+                continue;
+            }
+        }
+        else {
+            if (IS_ATOMCHAR(*sc->p) ||
+                    (ISO2022_MODE_P(sc) && (*sc->p == ESC)) ||
+                    IS_JCHAR(*sc->p)) {
+                VALUE tmp;
+                D(puts("atom"));
+                tmp = scan_atom(sc);
+                PASS(atomsym(sc, tmp), tmp);
+                continue;
+            }
+        }
+
+        if (*sc->p == '"') {
+            D(puts("quoted"));
+            PASS(tok_quoted, scan_quoted_word(sc));
+            D(puts("quoted"));
+        }
+        else if (*sc->p == '(') {
+            VALUE c;
+            D(puts("comment"));
+            c = scan_comment(sc);
+            if (! NIL_P(sc->comments))
+                rb_ary_push(sc->comments, c);
+        }
+        else if (*sc->p == '[') {
+            D(puts("domlit"));
+            PASS(tok_domlit, scan_domain_literal(sc));
+        }
+        else {
+            VALUE ch;
+            D(puts("char"));
+            ch = rb_str_new(sc->p, 1);
+            sc->p++;
+            PASS(ch, ch);
+        }
+    }
+
+    PASS(Qfalse, rb_str_new("$", 1));
+    return Qnil;
+}
+
+
+/*
+------------------------------------------------------------------
+                         ruby interface
+------------------------------------------------------------------
+*/
+
+static VALUE
+cstr2symbol(str)
+    char *str;
+{
+    ID tmp;
+
+    tmp = rb_intern(str);
+#ifdef ID2SYM
+    return ID2SYM(tmp);
+#else
+    return INT2FIX(tmp);
+#endif
+}
+
+void
+Init_tmailscanner()
+{
+    VALUE TMail;
+    VALUE tmp;
+
+    if (rb_const_defined(rb_cObject, rb_intern("TMail"))) {
+        TMail = rb_const_get(rb_cObject, rb_intern("TMail"));
+    }
+    else {
+        TMail = rb_define_module("TMail");
+    }
+    TMailScanner = rb_define_class_under(TMail, "TMailScanner", rb_cObject);
+
+    tmp = rb_str_new2(TMAIL_VERSION);
+    rb_obj_freeze(tmp);
+    rb_define_const(TMailScanner, "Version", tmp);
+
+    rb_define_singleton_method(TMailScanner, "new", mails_s_new, 3);
+    rb_define_method(TMailScanner, "scan", mails_scan, 0);
+    rb_define_method(TMailScanner, "debug", mails_debug_get, 0);
+    rb_define_method(TMailScanner, "debug=", mails_debug_set, 1);
+
+    if (rb_const_defined(TMail, rb_intern("SyntaxError"))) {
+        ScanError = rb_const_get(rb_cObject, rb_intern("SyntaxError"));
+    }
+    else {
+        ScanError = rb_define_class_under(TMail, "SyntaxError", rb_eStandardError);
+    }
+
+    tok_atom   = cstr2symbol("ATOM");
+    tok_digit  = cstr2symbol("DIGIT");
+    tok_token  = cstr2symbol("TOKEN");
+    tok_quoted = cstr2symbol("QUOTED");
+    tok_domlit = cstr2symbol("DOMLIT");
+
+    tok_from   = cstr2symbol("FROM");
+    tok_by     = cstr2symbol("BY");
+    tok_via    = cstr2symbol("VIA");
+    tok_with   = cstr2symbol("WITH");
+    tok_id     = cstr2symbol("ID");
+    tok_for    = cstr2symbol("FOR");
+}
+