milkode 0.2.2 → 0.2.4

data/HISTORY.ja.rdoc

@@ -1,3 +1,19 @@
+=== 0.2.4 2011/10/02
+
+* gem 1.3.6, 1.3.7 でインストール出来ない問題を修正
+
+=== 0.2.3 2011/09/29
+
+* webインターフェースの検索速度向上
+  * ミニファイされたJavaScript等、一行の長いファイルの検索が高速になりました。
+* 日本語ファイル名に対応
+  * #10: windows環境(mswin32)で、日本語ファイル、日英混在文字が化ける (thanks aki1999)
+  * OSXの濁点ファイル名問題を修正 (Ruby1.9.2を使えば、濁点付きの日本語ファイルを正常に処理出来ます)
+* #9: 検索文字列に特定の文字が含まれるとInternal Server Error (thanks vividtone)
+* バグ修正、特定ディレクトリ以下の検索で横のパッケージが'../milkode'のように表示されてしまう問題の修正
+* archive-zip 0.4.0に対応。vendor/archive/support/io.rb は必要なくなったので削除。
+* 既に追加済みのパッケージを再度 milk add したら milk update するように
+
 === 0.2.2 2011/08/26
 
 * ファイル内grepを追加

data/README.rdoc

@@ -2,18 +2,5 @@
 
 Line based local source code search engine & web-app.
 
-== Contributing to milkode
- 
-* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet
-* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it
-* Fork the project
-* Start a feature/bugfix branch
-* Commit and push until you are happy with your contribution
-* Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
-* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
-
-== Copyright
-
-Copyright (c) 2011 ongaeshi. See LICENSE.txt for
-further details.
-
+* {History.rdoc}[https://github.com/ongaeshi/milkode/blob/master/HISTORY.rdoc]
+* {更新履歴}[https://github.com/ongaeshi/milkode/blob/master/HISTORY.ja.rdoc]

data/Rakefile

@@ -27,9 +27,9 @@ Jeweler::Tasks.new do |gem|
   gem.add_runtime_dependency 'rack','>=1.2.1'
   gem.add_runtime_dependency 'sinatra', '>=1.2.6'
   gem.add_runtime_dependency 'launchy', '>=0.3.7'
-  gem.add_runtime_dependency 'coderay', '>=0.9.8'
+  gem.add_runtime_dependency 'coderay', '=0.9.8'
   gem.add_runtime_dependency 'thin', '>=1.2.10'
-  gem.add_runtime_dependency 'archive-zip', '>=0.3.0'
+  gem.add_runtime_dependency 'archive-zip', '>=0.4.0'
   gem.add_runtime_dependency 'haml', '>=3.1.2'
   gem.add_runtime_dependency 'sass', '>=3.1.3'
   gem.add_runtime_dependency 'nokogiri', '>=1.5.0'
@@ -58,17 +58,23 @@ Rake::TestTask.new(:test) do |test|
   test.verbose = true
 end
 
-# task :test do
-#   load "test/runner.rb"
-# end
+task :test_all do
+  puts "--- rvm 1.9.2@milkode ---"
+  system('rvm 1.9.2@milkode')
+  system('rake test')
 
-require 'rcov/rcovtask'
-Rcov::RcovTask.new do |test|
-  test.libs << 'test'
-  test.pattern = 'test/**/test_*.rb'
-  test.verbose = true
+  puts "--- rvm system ---"
+  system('rvm system')
+  system('rake test')
 end
 
+# require 'rcov/rcovtask'
+# Rcov::RcovTask.new do |test|
+#   test.libs << 'test'
+#   test.pattern = 'test/**/test_*.rb'
+#   test.verbose = true
+# end
+
 task :default => :test
 
 require 'rake/rdoctask'

data/VERSION

@@ -1 +1 @@
-0.2.2
+0.2.4

data/bin/milk

@@ -6,5 +6,5 @@
 require 'rubygems'
 require 'milkode/cdstk/cli_cdstk'
 
-Version = "0.2.2"
+Version = "0.2.4"
 Milkode::CLI_Cdstk.execute(STDOUT, ARGV)

data/lib/milkode/cdstk/cdstk.rb

@@ -34,6 +34,7 @@ module Milkode
       @db_dir = db_dir
       Database.setup(@db_dir)
       @out = io
+      # @out = $stdout # 強制出力
       clear_count
     end
 
@@ -91,11 +92,12 @@ module Milkode
       # 存在しないコンテンツがあった場合はその場で終了
       contents.each do |v|
         shortname = File.basename v
-        if (yaml.exist? shortname)
+
+        if (yaml.cant_add_directory? v)
           error_alert("already exist '#{shortname}'.")
           return
         end
-        
+          
         unless (File.exist? v)
           error_alert("not found '#{v}'.")
           return
@@ -432,13 +434,18 @@ module Milkode
     private :db_add_dir
 
     def db_add_file(stdout, filename, shortpath)
+      # ファイル名を全てUTF-8に変換
+      filename_utf8 = Util::filename_to_utf8(filename)
+      shortpath_utf8 = Util::filename_to_utf8(shortpath)
+      suffix_utf8 = File::extname(filename_utf8)
+      
       # 格納するデータ
       values = {
-        :path => filename,
-        :shortpath => shortpath,
+        :path => filename_utf8,
+        :shortpath => shortpath_utf8,
         :content => nil,
         :timestamp => File.mtime(filename),
-        :suffix => File::extname(filename),
+        :suffix => suffix_utf8,
       }
       
       # 検索するデータベース
@@ -518,7 +525,11 @@ module Milkode
     private :ignoreFile?
 
     def alert(title, msg)
-      @out.puts "#{title.ljust(10)} : #{msg}"
+      if (Util::platform_win?)
+        @out.puts "#{title.ljust(10)} : #{Kconv.kconv(msg, Kconv::SJIS)}"
+      else
+        @out.puts "#{title.ljust(10)} : #{msg}"
+      end
     end
 
     def error_alert(msg)

data/lib/milkode/cdstk/cdstk_yaml.rb

@@ -76,6 +76,13 @@ module Milkode
       @data['contents'].find {|v| File.basename(v['directory']) == shortname }   
     end
 
+    def cant_add_directory?(dir)
+      @data['contents'].find {|v|
+        v['directory'] != File.expand_path(dir) &&
+        File.basename(v['directory']) == File.basename(dir)
+      }
+    end
+
     def cleanup
       contents.delete_if do |v|
         if (!File.exist? v['directory'])

data/lib/milkode/cdweb/app.rb

@@ -18,7 +18,7 @@ require 'milkode/cdweb/lib/mkurl'
 set :haml, :format => :html5
 
 get '/' do
-  @version = "0.2.2"
+  @version = "0.2.4"
   @package_num = Database.instance.fileList('').size
   @file_num = Database.instance.fileNum
   haml :index

data/lib/milkode/cdweb/lib/coderay_wrapper.rb

@@ -9,6 +9,7 @@ require 'rubygems'
 require 'coderay'
 require 'coderay/helpers/file_type'
 require 'nokogiri'
+require 'milkode/common/util'
 
 module Milkode
   class CodeRayWrapper
@@ -22,6 +23,18 @@ module Milkode
       @line_number_start = 1
     end
 
+    def col_limit(limit_num)
+      content_a = @content.split("\n")
+
+      @content = content_a.map{|v|
+        if (v.length > limit_num)
+          v[0...limit_num] + " ..."
+        else
+          v
+        end
+      }.join("\n")
+    end
+
     def set_range(range)
       content_a = @content.split("\n")
       range = limit_range(range, content_a)
@@ -43,7 +56,13 @@ module Milkode
              :highlight_lines => @highlight_lines,
              :line_number_start => @line_number_start
              )
-      milkode_ornament(html)
+
+      if (is_ornament?)
+        html_doc = Nokogiri::HTML(html)
+        add_spanid(html_doc)
+      else
+        html
+      end
     end
 
     def to_html_anchor
@@ -55,39 +74,36 @@ module Milkode
              :highlight_lines => @highlight_lines,
              :line_number_start => @line_number_start
              )
-      
-      html_doc = Nokogiri::HTML(html)
-      anchor = create_anchorlink(html_doc.at_css("table.CodeRay td.code pre").inner_html)
-      body = milkode_ornament(html)
-      return anchor + body
+
+      if (is_ornament?)
+        html_doc = Nokogiri::HTML(html)
+        anchor = create_anchorlink(html_doc.at_css("table.CodeRay td.code pre").inner_html)
+        body = add_spanid(html_doc)
+        anchor + body
+      else
+        html
+      end
     end
 
-    def milkode_ornament(html)
-      # @todo nokogiri使う？
-      a = html.split("\n")
+    def add_spanid(html_doc)
+      table = html_doc.at_css("table.CodeRay")
+      
+      # preに<span id="行番号"> を付ける
+      pre = table.at_css("td.code pre")
+      pre.inner_html = add_spanid_in(pre.inner_html)
+      
+      # 結果を文字列で返す
+      table.to_html
+    end
 
+    def add_spanid_in(html)
+      lines = html.split("<tt>\n</tt>")
       line_number = @line_number_start
-      is_code_content = false
-
-      a.each_with_index do |l, index|
-        if (l =~ /  <td class="code"><pre (.*?)>(.*)<tt>/)
-          a[index] = "  <td class=\"code\"><pre #{$1}><span #{line_attr(line_number)}>#{$2}</span><tt>"
-          is_code_content = true
-          line_number += 1
-          next
-        elsif (l =~ %r|</tt></pre></td>|)
-          is_code_content = false
-        end
 
-        if (is_code_content)
-          if (l =~ %r|</tt>(.*)<tt>|)
-            a[index] = "</tt><span #{line_attr(line_number)}>#{$1}</span><tt>"
-            line_number += 1
-          end
-        end
-      end
-          
-      a.join("\n") + "\n"
+      lines.map {|l|
+        line_number += 1
+        "<span #{line_attr(line_number - 1)}>#{l}</span>"
+      }.join("<tt>\n</tt>") + "<tt>\n</tt>"
     end
 
     def file_type
@@ -127,6 +143,11 @@ EOF
         ""
       end
     end
+
+    def is_ornament?
+      Util::larger_than_oneline(@content)
+    end
+
   end
 end
 

data/lib/milkode/cdweb/lib/command.rb

@@ -10,6 +10,7 @@ require 'milkode/cdweb/lib/coderay_wrapper'
 require 'milkode/cdweb/lib/search_contents'
 require 'milkode/cdweb/lib/search_files'
 require 'milkode/cdweb/lib/mkurl'
+require 'milkode/common/util'
 
 module Milkode
   def view(record, params, before)
@@ -18,7 +19,7 @@ module Milkode
 
     q = params[:query] && Query.new(params[:query]) 
 
-    if (q and !q.keywords.empty?)
+    if (Util::larger_than_oneline(record.content) and q and !q.keywords.empty?)
       grep = Grep.new(record.content)
       match_lines = grep.match_lines_and(q.keywords)
       @record_content = CodeRayWrapper.new(record.content, record.shortpath, match_lines).to_html_anchor

data/lib/milkode/cdweb/lib/grep.rb

@@ -58,7 +58,7 @@ module Milkode
       strs.each do |v|
         option = 0
         option |= Regexp::IGNORECASE if (ignore)
-        regs << Regexp.new(v, option)
+        regs << Regexp.new(Regexp.escape(v), option)
       end
 
       regs

data/lib/milkode/cdweb/lib/search_contents.rb

@@ -19,9 +19,12 @@ module Milkode
     DISP_NUM = 20              # 1ページの表示数
     LIMIT_NUM = 50             # 最大検索ファイル数
     NTH = 3                    # 表示範囲
+    COL_LIMIT = 200            # 1行の桁制限
+    
 #     DISP_NUM = 1000              # 1ページの表示数
 #     LIMIT_NUM = 1000             # 最大検索ファイル数
 #     NTH = 3                      # 表示範囲
+#     COL_LIMIT = 200              # 1行の桁制限
     
     def initialize(path, params, query)
       @path = path
@@ -30,7 +33,7 @@ module Milkode
       @page = params[:page].to_i || 0
       @offset = params[:offset].to_i
       fpaths = @q.fpaths
-      fpaths << path unless path == ""
+      fpaths << path + "/" unless path == ""
       @records, @total_records, @elapsed = Database.instance.search(@q.keywords, @q.packages, fpaths, @q.suffixs, @offset, LIMIT_NUM)
       grep_contents
     end
@@ -75,9 +78,13 @@ EOF
       @next_index = @records.size
       
       @records.each_with_index do |record, index|
-        grep = Grep.new(record.content)
-        match_line = grep.one_match_and(@q.keywords)
-        @match_records << MatchRecord.new(record, match_line) if match_line
+        if (Util::larger_than_oneline(record.content))
+          grep = Grep.new(record.content)
+          match_line = grep.one_match_and(@q.keywords)
+          @match_records << MatchRecord.new(record, match_line) if match_line
+        else
+          @match_records << MatchRecord.new(record, Grep::MatchLineResult.new(0, nil))
+        end
 
         if @match_records.size >= DISP_NUM
           @next_index = index + 1
@@ -94,6 +101,7 @@ EOF
       last_index = match_line.index + NTH
 
       coderay = CodeRayWrapper.new(record.content, record.shortpath, [match_line])
+      coderay.col_limit(COL_LIMIT)
       coderay.set_range(first_index..last_index)
 
       <<EOS

data/lib/milkode/cdweb/lib/search_files.rb

@@ -24,7 +24,7 @@ module Milkode
       @offset = params[:offset].to_i
 
       fpaths = @q.fpaths
-      fpaths << path unless path == ""
+      fpaths << path + "/" unless path == ""
 
       if (fpaths.include?("*"))
         @records, @total_records = Database.instance.selectAll(@offset, DISP_NUM)

data/lib/milkode/common/archive-zip.rb

@@ -5,5 +5,6 @@
 # @author ongaeshi
 # @date   2011/08/04
 
-$LOAD_PATH.unshift File.join(File.dirname(__FILE__), '../../../vendor')
+# 0.4.0になったため、必要なくなった。
+# $LOAD_PATH.unshift File.join(File.dirname(__FILE__), '../../../vendor')
 require 'archive/zip'

data/lib/milkode/common/util.rb

@@ -4,6 +4,7 @@ require 'rubygems'
 require 'milkode/common/archive-zip'
 require 'fileutils'
 require 'pathname'
+require 'kconv'
 
 module Milkode
   module Util
@@ -48,6 +49,44 @@ module Milkode
       basedir = Pathname.new(basedir)
       path.relative_path_from(basedir)
     end
+
+    def ruby19?
+      RUBY_VERSION >= '1.9.0'
+    end
+
+    def platform_win?
+      RUBY_PLATFORM =~ /mswin(?!ce)|mingw|cygwin|bccwin/
+    end
+
+    def platform_osx?
+      RUBY_PLATFORM =~ /darwin/
+    end
+
+    def shell_kcode
+      if platform_win?
+        Kconv::SJIS             # win7? cygwin utf8?
+      else
+        Kconv::UTF8
+      end
+    end
+
+    def filename_to_utf8(str_from_file)
+      if platform_osx?
+        if (ruby19?)
+          str_from_file.encode('UTF-8', 'UTF8-MAC')
+        else
+          str_from_file
+        end
+      elsif platform_win?
+        Kconv.kconv(str_from_file, Kconv::UTF8)        
+      else
+        str_from_file
+      end
+    end
+
+    def larger_than_oneline(content)
+      content.count("\n") > 1
+    end
   end
 end
 

data/milkode.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{milkode}
-  s.version = "0.2.2"
+  s.version = "0.2.4"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["ongaeshi"]
-  s.date = %q{2011-08-26}
+  s.date = %q{2011-10-02}
   s.description = %q{Line based local source code search engine & web-app.}
   s.email = %q{ongaeshi0621@gmail.com}
   s.executables = ["cdv", "milk", "cdview"]
@@ -93,8 +93,7 @@ Gem::Specification.new do |s|
     "test/test_mkurl.rb",
     "test/test_query.rb",
     "test/test_string_snip.rb",
-    "test/test_util.rb",
-    "vendor/archive/support/io.rb"
+    "test/test_util.rb"
   ]
   s.homepage = %q{http://github.com/ongaeshi/milkode}
   s.licenses = ["MIT"]
@@ -135,9 +134,9 @@ Gem::Specification.new do |s|
       s.add_runtime_dependency(%q<rack>, [">= 1.2.1"])
       s.add_runtime_dependency(%q<sinatra>, [">= 1.2.6"])
       s.add_runtime_dependency(%q<launchy>, [">= 0.3.7"])
-      s.add_runtime_dependency(%q<coderay>, [">= 0.9.8"])
+      s.add_runtime_dependency(%q<coderay>, ["= 0.9.8"])
       s.add_runtime_dependency(%q<thin>, [">= 1.2.10"])
-      s.add_runtime_dependency(%q<archive-zip>, [">= 0.3.0"])
+      s.add_runtime_dependency(%q<archive-zip>, [">= 0.4.0"])
       s.add_runtime_dependency(%q<haml>, [">= 3.1.2"])
       s.add_runtime_dependency(%q<sass>, [">= 3.1.3"])
       s.add_runtime_dependency(%q<nokogiri>, [">= 1.5.0"])
@@ -150,9 +149,9 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<rack>, [">= 1.2.1"])
       s.add_dependency(%q<sinatra>, [">= 1.2.6"])
       s.add_dependency(%q<launchy>, [">= 0.3.7"])
-      s.add_dependency(%q<coderay>, [">= 0.9.8"])
+      s.add_dependency(%q<coderay>, ["= 0.9.8"])
       s.add_dependency(%q<thin>, [">= 1.2.10"])
-      s.add_dependency(%q<archive-zip>, [">= 0.3.0"])
+      s.add_dependency(%q<archive-zip>, [">= 0.4.0"])
       s.add_dependency(%q<haml>, [">= 3.1.2"])
       s.add_dependency(%q<sass>, [">= 3.1.3"])
       s.add_dependency(%q<nokogiri>, [">= 1.5.0"])
@@ -166,9 +165,9 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<rack>, [">= 1.2.1"])
     s.add_dependency(%q<sinatra>, [">= 1.2.6"])
     s.add_dependency(%q<launchy>, [">= 0.3.7"])
-    s.add_dependency(%q<coderay>, [">= 0.9.8"])
+    s.add_dependency(%q<coderay>, ["= 0.9.8"])
     s.add_dependency(%q<thin>, [">= 1.2.10"])
-    s.add_dependency(%q<archive-zip>, [">= 0.3.0"])
+    s.add_dependency(%q<archive-zip>, [">= 0.4.0"])
     s.add_dependency(%q<haml>, [">= 3.1.2"])
     s.add_dependency(%q<sass>, [">= 3.1.3"])
     s.add_dependency(%q<nokogiri>, [">= 1.5.0"])

data/test/test_util.rb

@@ -8,6 +8,7 @@
 require 'milkode/common/util'
 require 'test/unit'
 require 'file_test_utils'
+require 'tmpdir'
 
 class TestUtil < Test::Unit::TestCase
   include FileTestUtils
@@ -30,6 +31,38 @@ class TestUtil < Test::Unit::TestCase
     assert_equal ['abc/'], Milkode::Util::root_entrylist('../data/abc.zip')
     assert_equal ['a.txt', 'b.txt', 'c.txt'], Milkode::Util::root_entrylist('../data/nodir_abc.zip')
   end
+
+  def test_platform
+    if (Milkode::Util::platform_osx?)
+      assert_equal Milkode::Util::shell_kcode, Kconv::UTF8
+    end
+
+    if (Milkode::Util::platform_win?)
+      assert_equal Milkode::Util::shell_kcode, Kconv::SJIS
+    end
+  end
+
+  def create_filename_str(name)
+    Dir.mktmpdir do |dir|
+      FileUtils.touch( File.join(dir, name) )
+
+      Dir.foreach(dir) do |f|
+        next if (f == "." or f == "..")
+        return f
+      end
+    end
+  end
+
+  def test_filename_to_utf8
+    if (Milkode::Util::ruby19?)
+      assert_equal Encoding::UTF_8, Milkode::Util::filename_to_utf8('ダミー').encoding
+      assert_equal Encoding::UTF_8, Milkode::Util::filename_to_utf8(create_filename_str('ダミー')).encoding
+    else
+      # 実行だけはしておく
+      Milkode::Util::filename_to_utf8('ダミー')
+      Milkode::Util::filename_to_utf8(create_filename_str('ダミー'))
+    end
+  end
   
   def teardown
     teardown_custom(true)

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 2
-  - 2
-  version: 0.2.2
+  - 4
+  version: 0.2.4
 platform: ruby
 authors: 
 - ongaeshi
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-08-26 00:00:00 +09:00
+date: 2011-10-02 00:00:00 +09:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -130,7 +130,7 @@ dependencies:
 - !ruby/object:Gem::Dependency 
   version_requirements: &id009 !ruby/object:Gem::Requirement 
     requirements: 
-    - - ">="
+    - - "="
       - !ruby/object:Gem::Version 
         segments: 
         - 0
@@ -162,9 +162,9 @@ dependencies:
       - !ruby/object:Gem::Version 
         segments: 
         - 0
-        - 3
+        - 4
         - 0
-        version: 0.3.0
+        version: 0.4.0
   name: archive-zip
   requirement: *id011
   prerelease: false
@@ -299,7 +299,6 @@ files:
 - test/test_query.rb
 - test/test_string_snip.rb
 - test/test_util.rb
-- vendor/archive/support/io.rb
 has_rdoc: true
 homepage: http://github.com/ongaeshi/milkode
 licenses: 

data/vendor/archive/support/io.rb

@@ -1,53 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# @file 
-# @brief  archive-zip/lib/archive/support/io.rb patch, removed in the future.
-# @author ongaeshi
-# @date   2011/08/04
-
-begin
-  require 'readbytes'
-rescue LoadError
-  # for Ruby 1.9.2
-  class TruncatedDataError<IOError
-    def initialize(mesg, data) # :nodoc:
-      @data = data
-      super(mesg)
-    end
-
-    # The read portion of an IO#readbytes attempt.
-    attr_reader :data
-  end
-  
-  class IO
-    # Reads exactly +n+ bytes.
-    #
-    # If the data read is nil an EOFError is raised.
-    #
-    # If the data read is too short a TruncatedDataError is raised and the read
-    # data is obtainable via its #data method.
-    def readbytes(n)
-      str = read(n)
-      if str == nil
-        raise EOFError, "End of file reached"
-      end
-      if str.size < n
-        raise TruncatedDataError.new("data truncated", str) 
-      end
-      str
-    end
-  end
-end
-  
-class IO
-  # Returns +true+ if the seek method of this IO instance would succeed, +false+
-  # otherwise.
-  def seekable?
-    begin
-      pos
-      true
-    rescue SystemCallError
-      false
-    end
-  end
-end