qwik2md 0.1.0

data/vendor/qwik/lib/qwik/ml-quickml-server.rb

@@ -0,0 +1,122 @@
+#
+# Copyright (C) 2002-2004 Satoru Takabayashi <satoru@namazu.org> 
+# Copyright (C) 2003-2006 Kouichirou Eto
+#     All rights reserved.
+#     This is free software with ABSOLUTELY NO WARRANTY.
+#
+# You can redistribute it and/or modify it under the terms of 
+# the GNU General Public License version 2.
+#
+
+$KCODE = 's'	# FIXME: Remove $KCODE
+
+require 'fileutils'
+
+$LOAD_PATH.unshift '..' unless $LOAD_PATH.include? '..'
+require 'qwik/config'
+require 'qwik/ml-memory'
+require 'qwik/ml-server'
+require 'qwik/ml-sweeper'
+
+$ml_debug = false
+# $ml_debug = true
+
+module QuickML
+  class QuickMLServer
+    def self.main(args)
+      server = QuickMLServer.new
+
+      config = Qwik::Config.new
+      Qwik::Config.load_args_and_config(config, $0, args)
+      config.update({:debug => true, :verbose_mode => true}) if $ml_debug
+
+      ServerMemory.init_logger(config, config)
+      ServerMemory.init_mutex(config)
+      ServerMemory.init_catalog(config)
+
+      server.run(config)
+    end
+
+    def initialize
+      # Do nothing.
+    end
+
+    def run (config)
+      QuickMLServer.check_directory(config.sites_dir)
+
+      if ! config.debug
+        QuickMLServer.be_daemon
+        QuickMLServer.be_secure(config)
+      end
+
+      server  = Server.new(config)
+      sweeper = Sweeper.new(config)
+      trap(:TERM) { server.shutdown; sweeper.shutdown }
+      trap(:INT)  { server.shutdown; sweeper.shutdown }
+      if Signal.list.key?("HUP")
+        trap(:HUP)  { config.logger.reopen }
+      end
+      sweeper_thread = Thread.new { sweeper.start }
+      sweeper_thread.abort_on_exception = true
+
+      if config.debug
+        require 'qwik/autoreload'
+	AutoReload.start(1, true, 'ML')	# auto reload every sec.
+      end
+
+      server.start
+    end
+
+    private
+
+    def self.check_directory(dir)
+      error("#{dir}: No such directory") if ! File.directory?(dir) 
+      error("#{dir}: is not writable")   if ! File.writable?(dir) 
+    end
+
+    def self.error (msg)
+      STDERR.puts "#{$0}: #{msg}"
+      exit(1)
+    end
+
+    def self.be_daemon
+      exit!(0) if fork
+      Process::setsid
+      exit!(0) if fork
+      Dir::chdir('/')
+      File::umask(022)
+      STDIN.reopen('/dev/null',  'r+')
+      STDOUT.reopen('/dev/null', 'r+')
+      STDERR.reopen('/dev/null', 'r+')
+    end
+
+    def self.be_secure(config)
+      return unless Process.uid == 0
+      uid = Etc::getpwnam(config.user).uid 
+      gid = Etc::getgrnam(config.group).gid
+      FileUtils.touch(config.ml_pid_file)
+      ml_log_file = (config[:log_dir].path + Logger::ML_LOG_FILE).to_s
+      FileUtils.touch(ml_log_file)
+      File.chown(uid, gid, config.sites_dir)
+      File.chown(uid, gid, config.ml_pid_file)
+      File.chown(uid, gid, ml_log_file)
+      Process.uid  = uid
+      Process.gid  = gid
+      Process.euid = uid
+    end
+  end
+end
+
+if $0 == __FILE__
+  require 'qwik/test-module-ml'
+  $test = true
+end
+
+if defined?($test) && $test
+  class TestMLQuickMLServer < Test::Unit::TestCase
+    def test_all
+      # Just create it.
+      qml_server = QuickML::QuickMLServer.new
+    end
+  end
+end

data/vendor/qwik/lib/qwik/ml-server.rb

@@ -0,0 +1,107 @@
+#
+# Copyright (C) 2002-2004 Satoru Takabayashi <satoru@namazu.org> 
+# Copyright (C) 2003-2006 Kouichirou Eto
+#     All rights reserved.
+#     This is free software with ABSOLUTELY NO WARRANTY.
+#
+# You can redistribute it and/or modify it under the terms of 
+# the GNU General Public License version 2.
+#
+
+require 'socket'
+require 'etc'
+require 'thread'
+require 'thwait'
+require 'timeout'
+require 'time'
+require 'net/smtp'	# FIXME: Which code uses smtp?
+
+$LOAD_PATH.unshift '..' unless $LOAD_PATH.include? '..'
+require 'qwik/ml-session'
+require 'qwik/util-safe'
+require 'qwik/util-pid'
+
+module QuickML
+  class Server
+    include PidModule
+
+    def initialize (config)
+      @config = config
+      $ml_debug = true if @config.debug
+      @status = :stop
+      @logger = @config.logger
+      @server = TCPServer.new(@config.bind_address, @config.ml_port)
+    end
+
+    def start
+      raise 'server already started' if @status != :stop
+      write_pid_file(@config.ml_pid_file)
+      @logger.log sprintf('Server started at %s:%d [%d]',
+                          'localhost', @config.ml_port, Process.pid)
+      accept
+      @logger.log "Server exited [#{Process.pid}]"
+      remove_pid_file(@config.ml_pid_file)
+    end
+
+    def shutdown
+      begin
+	@server.shutdown
+      rescue Errno::ENOTCONN
+        p 'Already disconnected.'
+      end
+      @status = :shutdown
+    end
+
+    private
+
+    def accept
+      running_sessions = []
+      @status = :running
+      while @status == :running
+	begin 
+	  t = Thread.new(@server.accept) {|s|
+	    process_session(s)
+	  }
+	  t.abort_on_exception = true
+	  running_sessions.push(t)
+	rescue Errno::ECONNABORTED # caused by @server.shutdown
+	rescue Errno::EINVAL
+	end
+	running_sessions.delete_if {|t| t.status == false }
+	if running_sessions.length >= @config.max_threads
+	  ThreadsWait.new(running_sessions).next_wait
+	end
+      end
+      running_sessions.each {|t| t.join }
+    end
+
+    def process_session (socket)
+      begin
+	c = @config
+	session = Session.new(c, c.logger, c.catalog, socket)
+	session.start
+      rescue Exception => e
+	@logger.log "Unknown Session Error: #{e.class}: #{e.message}"
+	@logger.log e.backtrace
+      end
+    end
+
+  end
+end
+
+if $0 == __FILE__
+  require 'qwik/test-module-ml'
+  require 'qwik/config'
+  $test = true
+end
+
+if defined?($test) && $test
+  class TestMLServer < Test::Unit::TestCase
+    def test_all
+      #return
+      config = Qwik::Config.new
+      config[:ml_port] = 9195
+      server = QuickML::Server.new(config)
+    end
+  end
+end

data/vendor/qwik/lib/qwik/ml-session.rb

@@ -0,0 +1,348 @@
+#
+# Copyright (C) 2002-2004 Satoru Takabayashi <satoru@namazu.org> 
+# Copyright (C) 2003-2006 Kouichirou Eto
+#     All rights reserved.
+#     This is free software with ABSOLUTELY NO WARRANTY.
+#
+# You can redistribute it and/or modify it under the terms of 
+# the GNU General Public License version 2.
+#
+
+$LOAD_PATH.unshift '..' unless $LOAD_PATH.include? '..'
+require 'qwik/ml-gettext'
+require 'qwik/mail'
+require 'qwik/ml-processor'
+require 'qwik/ml-exception'
+require 'qwik/util-basic'
+require 'qwik/util-safe'
+
+require 'kconv'
+require 'net/smtp'
+require 'socket'
+#require 'qwik/util-socket'
+
+class TCPSocket
+  def address
+    return peeraddr[3]
+  end
+
+  def hostname
+    return peeraddr[2]
+  end
+end
+
+module QuickML
+  class TooLargeMail < QuickMLException; end
+
+  class Session
+    include GetText
+
+    COMMAND_TABLE = [:helo, :ehlo, :noop, :quit, :rset, :rcpt, :mail, :data]
+
+#   def initialize (config, socket)
+    def initialize (config, logger, catalog, socket)
+      @socket = socket
+      @config = config
+#      @logger = @config.logger
+#      @catalog = @config.catalog
+      @logger = logger
+      @catalog = catalog
+
+      @hello_host = 'hello.host.invalid'
+      @protocol = nil
+      @peer_hostname = @socket.hostname
+      @peer_address = @socket.address
+      @remote_host = (@peer_hostname or @peer_address)
+
+      @data_finished = false
+      @my_hostname = 'localhost'
+      @my_hostname = Socket.gethostname if @config.ml_port == 25
+      @message_charset = nil
+    end
+
+    def start
+      elapsed = calc_time {
+	_start
+      }
+      @logger.vlog "Session finished: #{elapsed} sec."
+    end
+
+    private
+
+    def calc_time
+      start_time = Time.now
+      yield
+      elapsed = Time.now - start_time
+      return elapsed
+    end
+
+    def _start
+      begin
+	def_puts(@socket)
+	connect
+	timeout(@config.timeout) {
+	  process
+	}
+      rescue TimeoutError
+	@logger.vlog "Timeout: #{@remote_host}"
+      ensure
+	close
+      end
+    end
+
+    def def_puts(socket)
+      def socket.puts(*objs)
+	objs.each {|x|
+	  begin
+	    self.print x.xchomp, "\r\n"
+	  rescue Errno::EPIPE
+	  end
+	}
+      end
+    end
+
+    def connect
+      @socket.puts "220 #{@my_hostname} ESMTP QuickML"
+      @logger.vlog "Connect: #{@remote_host}"
+    end
+
+    def process
+      until @socket.closed?
+	begin
+	  mail = Mail.new
+	  receive_mail(mail)
+	  if mail.valid?
+	    processor = Processor.new(@config, mail)
+	    processor.process
+	  end
+	rescue TooLargeMail
+	  cleanup_connection
+	  report_too_large_mail(mail) if mail.valid?
+	  @logger.log "Too Large Mail: #{mail.from}"
+	rescue TooLongLine
+	  cleanup_connection
+	  @logger.log "Too Long Line: #{mail.from}"
+	end
+      end
+    end
+
+    def receive_mail (mail)
+      while line = @socket.safe_gets
+	line = line.xchomp
+	command, arg = line.split(/\s+/, 2)
+	return if command.nil? || command.empty?
+	command = command.downcase.intern  # 'HELO' => :helo
+	if COMMAND_TABLE.include?(command)
+	  @logger.vlog "Command: #{line}"
+	  send(command, mail, arg)
+	else
+	  @logger.vlog "Unknown SMTP Command: #{command} #{arg}"
+	  @socket.puts '502 Error: command not implemented'
+	end
+	break if command == :quit or command == :data
+      end
+    end
+
+    def helo (mail, arg)
+      return if arg.nil?
+      @hello_host = arg.split.first
+      @socket.puts "250 #{@my_hostname}"
+      @protocol = 'SMTP'
+    end
+
+    def ehlo (mail, arg)
+      @hello_host = arg.split.first
+      @socket.puts "250-#{@my_hostname}"
+      @socket.puts '250 PIPELINING'
+      @protocol = 'ESMTP'
+    end
+
+    def noop (mail, arg)
+      @socket.puts '250 ok'
+    end
+
+    def quit (mail, arg)
+      @socket.puts '221 Bye'
+      close
+    end
+
+    def rset (mail, arg)
+      mail.mail_from = nil
+      mail.clear_recipients
+      @socket.puts '250 ok'
+    end
+
+    def rcpt (mail, arg)
+      if mail.mail_from.nil?
+	@socket.puts '503 Error: need MAIL command'
+      elsif /^To:\s*<(.*)>/i =~ arg or /^To:\s*(.*)/i =~ arg
+	address = $1
+	if Mail.address_of_domain?(address, @config.ml_domain)
+	  mail.add_recipient(address)
+	  @socket.puts '250 ok'
+	else
+	  @socket.puts "554 <#{address}>: Recipient address rejected"
+	  @logger.vlog "Unacceptable RCPT TO:<#{address}>"
+	end
+      else
+	@socket.puts "501 Syntax: RCPT TO: <address>"
+      end
+    end
+
+    def mail (mail, arg)
+      if @protocol.nil?
+	@socket.puts '503 Error: send HELO/EHLO first'
+      elsif /^From:\s*<(.*)>/i =~ arg or /^From:\s*(.*)/i =~ arg 
+	mail.mail_from = $1
+	@socket.puts '250 ok'
+      else
+	@socket.puts "501 Syntax: MAIL FROM: <address>"
+      end
+    end
+
+    def data (mail, arg)
+      if mail.recipients.empty?
+	@socket.puts '503 Error: need RCPT command'
+      else
+	@socket.puts '354 send the mail data, end with .';
+	begin
+	  read_mail(mail)
+	ensure
+	  @message_charset = mail.charset
+	end
+	@socket.puts '250 ok'
+      end
+    end
+
+    def read_mail (mail)
+      len = 0
+      lines = []
+      while line = @socket.safe_gets
+	break if end_of_data?(line)
+	len += line.length
+	if @config.max_mail_length < len
+	  mail.read(lines.join('')) # Generate a header for an error report.
+	  raise TooLargeMail 
+	end
+	line.sub!(/^\.\./, '.') # unescape
+	line = line.normalize_eol
+	lines << line
+	# I do not know why but constructing mail_string with
+	# String#<< here is very slow.
+	# mail_string << line  
+      end
+      mail_string = lines.join('')
+      @data_finished = true
+      mail.read(mail_string)
+      mail.unshift_field('Received', received_field)
+    end
+
+    def end_of_data? (line)
+    # line.xchomp == '.'
+      return line == ".\r\n"
+    end
+
+    def received_field
+      sprintf("from %s (%s [%s])\n" + 
+	      "	by %s (QuickML) with %s;\n" + 
+	      "	%s", 
+	      @hello_host,
+	      @peer_hostname, 
+	      @peer_address,
+	      @my_hostname,
+	      @protocol,
+	      Time.now.rfc2822)
+    end
+
+    def cleanup_connection
+      unless @data_finished
+	discard_data
+      end
+      @socket.puts '221 Bye'
+      close
+    end
+
+    def discard_data
+      begin
+	while line = @socket.safe_gets
+	  break if end_of_data?(line)
+	end
+      rescue TooLongLine
+	retry
+      end
+    end
+
+    def report_too_large_mail (mail)
+      header = []
+      subject = Mail.encode_field(_("[QuickML] Error: %s", mail['Subject']))
+      header.push(['To',	mail.from],
+		  ['From',	@config.ml_postmaster],
+		  ['Subject',	subject],
+                  ['Content-Type', content_type])
+
+      max  = @config.max_mail_length.commify
+      body =   _("Sorry, your mail exceeds the length limitation.\n")
+      body <<  _("The max length is %s bytes.\n\n", max)
+      orig_subject = codeconv(Mail.decode_subject(mail['Subject']))
+      body << "Subject: #{orig_subject}\n"
+      body << "To: #{mail['To']}\n"
+      body << "From: #{mail['From']}\n"
+      body << "Date: #{mail['Date']}\n"
+      Sendmail.send_mail(@config.smtp_host, @config.smtp_port, @logger,
+		     :mail_from => '', 
+		     :recipient => mail.from,
+		     :header => header,
+		     :body => body)
+    end
+
+    # FIXME: this is the same method of QuickML#content_type
+    def content_type
+      return Mail.content_type(@config.content_type, @message_charset)
+    end
+
+    def close
+      return if @socket.closed?
+      @socket.close
+      @logger.vlog "Closed: #{@remote_host}"
+    end
+  end
+end
+
+if $0 == __FILE__
+  require 'qwik/test-module-ml'
+  require 'qwik/config'
+  $test = true
+end
+
+if defined?($test) && $test
+  class TestMLSession < Test::Unit::TestCase
+    def test_all
+      return
+      config = Qwik::Config.new
+      hash = {
+	:logger		=> QuickML::MockLogger.new,
+	:sites_dir	=> '.',
+      }
+      config.update(hash)
+      QuickML::ServerMemory.init_mutex(config)
+      QuickML::ServerMemory.init_catalog(config)
+      old_test_memory = $test_memory
+      $test_memory = Qwik::ServerMemory.new(config)	# FIXME: Ugly.
+      socket = QuickML::MockSocket.new "HELO localhost
+MAIL FROM: user@example.net
+RCPT TO: test@example.com
+DATA
+To: test@example.com
+From: user@example.net
+Subject: create
+
+create new ML.
+.
+QUIT
+"
+      session = QuickML::Session.new(config, socket)
+      session.start
+      $test_memory = old_test_memory
+    end
+  end
+end

data/vendor/qwik/lib/qwik/ml-sweeper.rb

@@ -0,0 +1,101 @@
+#
+# Copyright (C) 2002-2004 Satoru Takabayashi <satoru@namazu.org> 
+# Copyright (C) 2003-2006 Kouichirou Eto
+#     All rights reserved.
+#     This is free software with ABSOLUTELY NO WARRANTY.
+#
+# You can redistribute it and/or modify it under the terms of 
+# the GNU General Public License version 2.
+#
+
+$LOAD_PATH.unshift '..' unless $LOAD_PATH.include? '..'
+require 'qwik/group'
+
+module QuickML
+  class Sweeper
+    def initialize (config)
+      @config = config
+      @status = :safe
+      @logger = @config.logger
+    end
+
+    def start
+      @logger.vlog 'Sweeper started'
+      loop {
+	sleep(@config.sweep_interval)
+	begin
+	  sweep
+	rescue Exception => e
+	  @logger.log "Unknown Sweep Error: #{e.class}: #{e.message}"
+	  @logger.log e.backtrace
+	end
+      }
+    end
+
+    def shutdown
+      until @status == :safe
+	sleep(0.5)
+      end
+      @logger.vlog 'Sweeper shutdown'
+    end
+
+    private
+
+    def sweep
+      @status = :sweeping
+      @logger.vlog 'Sweeper runs'
+      Dir.new(@config.sites_dir).each {|filename|
+	filename = File.join(@config.sites_dir, filename)
+	if Sweeper.ml_file?(filename)
+	  mlname = File.basename(filename)
+	  address = Sweeper.mladdress(mlname, @config.ml_domain)
+	  ServerMemory.ml_mutex(@config, address).synchronize {
+            ml = Group.new(@config, address)
+	    ml.group_config_check_exist
+	    sweep_ml(ml)
+	  }
+	end
+      }
+      @logger.vlog 'Sweeper finished'
+      @status = :safe
+    end
+
+    def self.ml_file? (filename)
+      return false if File.file?(filename)
+      return false if /\./ =~ File.basename(filename) # avoid the name with dot
+      return true
+    end
+
+    def self.mladdress (name, ml_domain)
+      return "#{name}@#{ml_domain}"
+    end
+
+    def sweep_ml (ml)
+      if ml.inactive?
+	@logger.log "[#{ml.name}]: Inactive"
+	#ml.close
+	ml.close_dummy
+      elsif ml.need_alert?
+	ml.report_ml_close_soon
+      end
+    end
+  end
+end
+
+if $0 == __FILE__
+  require 'qwik/testunit'
+  require 'qwik/test-module-ml'
+  $test = true
+end
+
+if defined?($test) && $test
+  class TestMLSweeper < Test::Unit::TestCase
+    include TestModuleML
+
+    def test_all
+      c = QuickML::Sweeper
+      eq true, c.ml_file?('/qwik/data/test')
+      eq "test@example.net", c.mladdress('test', 'example.net')
+    end
+  end
+end