milkode 0.9.2 → 0.9.3

data/lib/milkode/cdweb/app.rb

@@ -22,7 +22,7 @@ set :haml, :format => :html5
 
 get '/' do
   @setting = WebSetting.new
-  @version = "0.9.2"
+  @version = "0.9.3"
   @package_num = Database.instance.yaml_package_num
   @file_num = Database.instance.totalRecords
   @package_list = PackageList.new(Database.instance.grndb)
@@ -62,6 +62,20 @@ post '/search*' do
   end
 end
 
+post '/command' do
+  case params[:kind]
+  when 'update'
+    before = Time.now
+    if (params[:name] == '')
+      result = Database.instance.update_all
+      update_result_str(result, before)
+    else
+      result = Database.instance.update(params[:name])
+      update_result_str(result, before)
+    end
+  end
+end
+
 get '/home*' do |path|
   before = Time.now
   path = path.sub(/^\//, "")
@@ -158,11 +172,35 @@ EOF
   def create_headmenu(path, query, flistpath = '')
     href = Mkurl.new('/home/' + path, params).inherit_query_shead
     flist = File.join("/home/#{path}", flistpath)
+
+    package_name = ""
+    modal_body = "全てのパッケージを更新しますか？"
+
+    if (path != "")
+      package_name = path.split('/')[0]
+      modal_body = "#{package_name} を更新しますか？"
+    end
+
     <<EOF
-    #{headicon('go-home-5.png')} <a href="/home" class="headmenu">全てのパッケージ</a>
+    #{headicon('go-home-5.png')} <a href="/home" class="headmenu">ホーム</a>
     #{headicon('document-new-4.png')} <a href="#{href}" class="headmenu" onclick="window.open('#{href}'); return false;">新しい検索</a>
     #{headicon('directory.png')} <a href="#{flist}" class="headmenu">ディレクトリ</a> 
+    #{headicon('view-refresh-4.png')} <a href="#updateModal" class="headmenu" data-toggle="modal">パッケージを更新</a>
     #{headicon('help.png')} <a href="/help" class="headmenu">ヘルプ</a>
+
+    <div id="updateModal" class="modal hide fade">
+      <div class="modal-header">
+        <a href="#" class="close" data-dismiss="modal">&times;</a>
+        <h3>パッケージを更新</h3>
+      </div>
+      <div class="modal-body">
+        <h4>#{modal_body}</h4>
+      </div>
+      <div class="modal-footer">
+        <a href="#" id="updateCancel" class="btn" data-dismiss="modal">Cancel</a>
+        <a href="#" id="updateOk" class="btn btn-primary" data-loading-text="Updating..." milkode-package-name="#{package_name}"">OK</a>
+      </div>
+    </div>
 EOF
   end
 
@@ -224,6 +262,15 @@ EOF
   def filelist_title(path)
     (path == "") ? "Package List" : path
   end
+
+  def update_result_str(result, before)
+    r = []
+    r << "#{result.package_count} packages" if result.package_count > 1
+    r << "#{result.file_count} records"
+    r << "#{result.add_count} add"
+    r << "#{result.update_count} update"
+    "#{r.join(', ')} (#{Time.now - before} sec)"
+  end
 end
 
 class Array

data/lib/milkode/cdweb/lib/database.rb

@@ -13,6 +13,7 @@ require 'milkode/common/dbdir'
 require 'milkode/cdstk/yaml_file_wrapper'
 require 'milkode/database/groonga_database'
 require 'milkode/common/util'
+require 'milkode/database/updater'
 include Milkode
 
 module Milkode
@@ -168,6 +169,20 @@ module Milkode
       package, restpath = Util::divide_shortpath(path)
       @grndb.packages.touch_if(package, :viewtime) if package
     end
+
+    def update(name)
+      result = Updater::ResultAccumulator.new
+      result << update_in(yaml_load.find_name(name))
+      result
+    end
+
+    def update_all
+      result = Updater::ResultAccumulator.new
+      yaml_load.contents.each do |package|
+        result << update_in(package)
+      end
+      result
+    end
     
     private 
 
@@ -175,5 +190,14 @@ module Milkode
       YamlFileWrapper.load_if(Database.dbdir)
     end
 
+    def update_in(package)
+      updater = Updater.new(@grndb, package.name)
+      updater.set_package_ignore IgnoreSetting.new("/", package.ignore)
+      updater.enable_no_auto_ignore       if package.options[:no_auto_ignore]
+      updater.enable_update_with_git_pull if package.options[:update_with_git_pull]
+      updater.exec
+      updater.result
+    end
+
   end
 end

data/lib/milkode/cdweb/public/js/milkode.js

@@ -84,5 +84,30 @@ $(document).ready(function(){
     selectedList: 1,
     height: 350
   }).multiselectfilter();
+
+  $("#updateOk").click(function (e) {
+    update_package($("#updateOk").attr("milkode-package-name"));
+    return false;
+  });
 });
 
+function update_package(package_name)
+{
+  // click button
+  $("#updateModal .modal-body").html("<h4>更新中... <img src='/images/waiting.gif'/></h4>");
+  $("#updateCancel").addClass("hide");
+  $("#updateOk").button('loading').off('click');
+
+  // update end
+  $.post(
+    '/command',
+    {
+      kind: 'update',
+      name: package_name
+    },
+    function (data) {
+      $("#updateModal .modal-body").html("<h4>実行結果</h4>" + "<p>" + data + "</p>");
+      $("#updateOk").button('reset').attr("data-dismiss", "modal").text("Close").on('click', function () { location.reload(); });
+    }
+  );
+}

data/lib/milkode/cdweb/views/layout.haml

@@ -20,6 +20,7 @@
     %script(type='text/javascript' src='/js/jquery-ui-1.8.22.custom.min.js')
     %script(type='text/javascript' src='/js/jquery.multiselect.min.js')
     %script(type='text/javascript' src='/js/jquery.multiselect.filter.min.js')
+    %script(type='text/javascript' src='/js/bootstrap.min.js')
     %script(type='text/javascript' src='/js/milkode.js')
 
 -#  GoogleAnalyticsを使いたい場合はここにコードを挿入して下さい。

data/lib/milkode/cli.rb

@@ -32,7 +32,7 @@ Samples:
   milk add git://github.com/ongaeshi/milkode.git
 EOF
     option :ignore, :type => :array, :aliases => '-i', :desc => 'Ignore path.'
-    option :no_auto_ignore, :type => :boolean, :desc => 'Disable auto ignore (.gitignore).'
+    option :no_auto_ignore, :type => :boolean, :aliases => '-n', :desc => 'Disable auto ignore (.gitignore).'
     option :verbose, :type => :boolean, :aliases => '-v', :desc => 'Be verbose.'
     
     def add(*args)
@@ -108,9 +108,15 @@ EOF
       cdstk.mcd(options)
     end
 
-    desc "info", "Information of milkode status"
-    def info
-      cdstk.info
+    desc "info [package]", "Display package information"
+    option :all, :type => :boolean, :aliases => '-a', :desc => 'Display all packages'
+    option :detail, :type => :boolean, :aliases => '-d', :desc => 'Detail format'
+    option :table, :type => :boolean, :aliases => '-t', :desc => 'Table format'
+    option :breakdown, :type => :boolean, :aliases => '-b', :desc => 'Breakdown format'
+    option :unknown, :type => :boolean, :aliases => '-u', :desc => 'Display unknown files'
+    
+    def info(*args)
+      cdstk.info(args_or_pipe(args, $stdin), options)
     end
 
     desc "ignore [path ...]", "Ignore a file or directory"
@@ -169,10 +175,16 @@ EOF
       $stdout.puts <<EOF
 Gitomb              https://github.com/tomykaira/gitomb
 redmine_milkode     https://github.com/suer/redmine_milkode
+Milkode_Sublime     https://github.com/tsurushuu/Milkode_Sublime
 emacs-milkode       https://github.com/ongaeshi/emacs-milkode
 EOF
     end
 
+    desc "files", "Display package files"
+    def files(*args)
+      cdstk.files(args)
+    end
+
     # --------------------------------------------------------------------------
     
     no_tasks do
@@ -208,5 +220,16 @@ EOF
       end
     end
 
+    # 引数が空の場合はパイプ(標準入力)を受け取る
+    def args_or_pipe(args, stdin)
+      if !args.empty?
+        args 
+      elsif File.pipe?(stdin)
+        stdin.readlines.map{|v| v.chomp}
+      else
+        []
+      end
+    end
+    
   end
 end

data/lib/milkode/common/grenfiletest.rb

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 
 module GrenFileTest
-  IGNORE_FILE = /(\A#.*#\Z)|(~\Z)|(\A\.#)|(\.d\Z)|(\.map\Z)|(\.MAP\Z)/
+  IGNORE_FILE = /(\A#.*#\Z)|(~\Z)|(\A\.#)|(\.d\Z)|(\.map\Z)|(\.MAP\Z)|(\.xbm\Z)|(\.ppm\Z)|(\.ai\Z)|(\.png\Z)|(\.webarchive\Z)/
   IGNORE_DIR = /(\A\.svn\Z)|(\A\.git\Z)|(\ACVS\Z)/
 
   def self.ignoreDir?(fpath)

data/lib/milkode/common/plang_detector.rb

@@ -0,0 +1,111 @@
+# -*- coding: utf-8 -*-
+#
+# @file 
+# @brief  Programming Language Detector
+# @author ongaeshi
+# @date   2012/09/29
+
+module Milkode
+  class PlangDetector
+    LANGUAGES =
+      [
+       # high priority
+       { :name => 'README'        , :filepatterns  => ['README', 'readme']                              },
+
+       # normal priority
+       { :name => 'ActionScript'  , :suffixs   => ['as']                                                },
+       { :name => 'Autotools'     , :suffixs   => ['am', 'in']                                          },
+       { :name => 'AWK'           , :suffixs   => ['awk']                                               },
+       { :name => 'Batch File'    , :suffixs   => ['bat']                                               },
+       { :name => 'Bundler'       , :filenames => ['Gemfile', 'Gemfile.lock']                           },
+       { :name => 'C'             , :suffixs   => ['c', 'h']                                            },
+       { :name => 'C#'            , :suffixs   => ['cs']                                                },
+       { :name => 'C++'           , :suffixs   => ['cc', 'cpp', 'hpp']                                  },
+       { :name => 'CGI'           , :suffixs   => ['cgi']                                               },
+       { :name => 'ChangeLog'     , :filenames => ['ChangeLog']                                         },
+       { :name => 'Common Lisp'   , :suffixs   => ['cl']                                                },
+       { :name => 'CSS'           , :suffixs   => ['css']                                               },
+       { :name => 'CSV'           , :suffixs   => ['csv']                                               },
+       { :name => 'Diff'          , :suffixs   => ['diff']                                              },
+       { :name => 'Emacs Lisp'    , :suffixs   => ['el']                                                },
+       { :name => 'Erlang'        , :suffixs   => ['erl']                                               },
+       { :name => 'eRuby'         , :suffixs   => ['erb', 'rhtml']                                      },
+       { :name => 'gitignore'     , :filenames => ['.gitignore']                                        },
+       { :name => 'Haml'          , :suffixs   => ['haml']                                              },
+       { :name => 'Haskell'       , :suffixs   => ['hs']                                                },
+       { :name => 'HTML'          , :suffixs   => ['html']                                              },
+       { :name => 'Java'          , :suffixs   => ['java']                                              },
+       { :name => 'JavaScript'    , :suffixs   => ['js']                                                },
+       { :name => 'JSON'          , :suffixs   => ['json']                                              },
+       { :name => 'Lua'           , :suffixs   => ['lua']                                               },
+       { :name => 'Makefile'      , :suffixs   => ['mk']       , :filenames => ['Makefile', 'makefile'] },
+       { :name => 'Markdown'      , :suffixs   => ['md', 'markdown']                                    },
+       { :name => 'M4'            , :suffixs   => ['m4']                                                },
+       { :name => 'Objective-C'   , :suffixs   => ['m', 'mm']                                           },
+       { :name => 'PEM'           , :suffixs   => ['pem']                                               },
+       { :name => 'Perl'          , :suffixs   => ['pl', 'PL', 'pm', 't']                               },
+       { :name => 'POD'           , :suffixs   => ['pod']                                               },
+       { :name => 'PHP'           , :suffixs   => ['php']                                               },
+       { :name => 'Python'        , :suffixs   => ['py']                                                },
+       { :name => 'Rackup'        , :suffixs   => ['ru']                                                },
+       { :name => 'Rakefile'      , :suffixs   => ['rake']     , :filenames => ['Rakefile']             },
+       { :name => 'RD'            , :suffixs   => ['rd']       , :filepatterns => [/rd.ja\Z/]           },
+       { :name => 'RDoc'          , :suffixs   => ['rdoc']                                              },
+       { :name => 'Ruby'          , :suffixs   => ['rb']                                                },
+       { :name => 'RubyGems'      , :suffixs   => ['gemspec']                                           },
+       { :name => 'Scheme'        , :suffixs   => ['scm']                                               },
+       { :name => 'sed'           , :suffixs   => ['sed']                                               },
+       { :name => 'Shell'         , :suffixs   => ['sh']                                                },
+       { :name => 'SVG'           , :suffixs   => ['svg']                                               },
+       { :name => 'Tcl'           , :suffixs   => ['tcl']                                               },
+       { :name => 'Text'          , :suffixs   => ['txt']                                               },
+       { :name => 'XML'           , :suffixs   => ['xml']                                               },
+       { :name => 'Yaml'          , :suffixs   => ['yml', 'yaml']                                       },
+       # { :name => ''              , :suffixs   => []          , :filenames => [] },
+      ]
+
+    UNKNOWN          = 'unknown'
+    UNKNOWN_LANGUAGE = {:name => UNKNOWN}
+    
+    def initialize(filename)
+      suffix = File.extname(filename)
+      suffix = suffix[1..-1]
+
+      filename = File.basename(filename)
+
+      @lang = LANGUAGES.find {|v|
+        is_found = false
+
+        if v[:suffixs]
+          is_found = v[:suffixs].include?(suffix)
+        end
+        
+        if !is_found && v[:filenames]
+          is_found = v[:filenames].include?(filename)
+        end
+
+        if !is_found && v[:filepatterns]
+          v[:filepatterns].each do |pattern|
+            if filename.match pattern
+              is_found = true
+              break
+            end
+          end
+        end
+
+        is_found
+      }
+
+      @lang ||= UNKNOWN_LANGUAGE
+    end
+
+    def name
+      @lang[:name]
+    end
+
+    def unknown?
+      name == UNKNOWN
+    end
+  end
+end
+

data/lib/milkode/common/util.rb

@@ -89,7 +89,11 @@ module Milkode
     end
 
     def larger_than_oneline(content)
-      content && content.count($/) > 1      
+      begin
+        content && content.count($/) > 1      
+      rescue ArgumentError
+        true
+      end
     end
 
     def normalize_filename(str)
@@ -174,6 +178,11 @@ module Milkode
     def git_url?(src)
       (src =~ /^(:?git[:@])|(:?ssh:)/) != nil
     end
+
+    # StringIO patch
+    def pipe?(io)
+      !io.instance_of?(IO) || !File.pipe?(io) 
+    end
   end
 end
 

data/lib/milkode/database/groonga_database.rb

@@ -9,9 +9,9 @@ require 'rubygems'
 require 'groonga'
 require 'milkode/common/dbdir'
 require 'fileutils'
-require 'milkode/database/package_table.rb'
-require 'milkode/database/document_table.rb'
-require 'milkode/database/document_record.rb'
+require 'milkode/database/package_table'
+require 'milkode/database/document_table'
+require 'milkode/database/document_record'
 
 module Milkode
   class GroongaDatabase
@@ -91,7 +91,7 @@ EOF
         true
       end
     end
-    
+
     private
 
     def define_schema

data/lib/milkode/database/updater.rb

@@ -0,0 +1,242 @@
+# -*- coding: utf-8 -*-
+#
+# @file 
+# @brief
+# @author ongaeshi
+# @date   2012/09/15
+
+require 'milkode/database/groonga_database'
+require 'milkode/common/grenfiletest'
+require 'milkode/common/ignore_checker'
+require 'kconv'
+
+module Milkode
+  class Updater
+    attr_reader :result
+    
+    def initialize(grndb, package_name)
+      @grndb          = grndb
+      @package_name   = package_name
+      @package        = @grndb.packages[@package_name]
+      @result         = Result.new
+      @current_ignore = IgnoreChecker.new
+      @options        = {}
+      @out            = $stdout
+    end
+
+    def exec
+      # git pull
+      if @options[:update_with_git_pull]
+        Dir.chdir(@package.directory) { system("git pull") }        
+      end
+      
+      # cleanup
+      unless @options[:no_clean]
+        @grndb.documents.cleanup_package_name(@package_name)
+      end
+      
+      # update
+      update_dir(@package.directory)
+
+      # 更新時刻の更新
+      @grndb.packages.touch(@package_name, :updatetime)
+    end
+
+    def set_package_ignore(ignore_setting)
+      @current_ignore.add ignore_setting
+    end
+
+    def enable_no_auto_ignore
+      @options[:no_auto_ignore] = true
+    end
+
+    def enable_silent_mode
+      @options[:silent_mode] = true
+    end
+
+    def enable_display_info
+      @options[:display_info] = true
+    end
+
+    def enable_update_with_git_pull
+      @options[:update_with_git_pull] = true      
+    end
+
+    def enable_no_clean
+      @options[:no_clean] = true      
+    end
+
+    class Result
+      attr_reader :file_count
+      attr_reader :add_count
+      attr_reader :update_count
+
+      def initialize
+        @file_count = 0
+        @add_count = 0
+        @update_count = 0
+      end
+
+      def inc_file_count
+        @file_count += 1
+      end
+
+      def inc_add_count
+        @add_count += 1
+      end
+
+      def inc_update_count
+        @update_count += 1
+      end
+    end
+
+    class ResultAccumulator
+      attr_reader :package_count
+      attr_reader :file_count
+      attr_reader :add_count
+      attr_reader :update_count
+
+      def initialize
+        @package_count = 0
+        @file_count    = 0
+        @add_count     = 0
+        @update_count  = 0
+      end
+      
+      def <<(result)
+        @package_count += 1
+        @file_count    += result.file_count
+        @add_count     += result.add_count
+        @update_count  += result.update_count
+      end
+    end
+    
+    # ---------------------------------------------------------
+    private
+
+    def update_dir(dir)
+      if (!FileTest.exist?(dir))
+        warning_alert("#{dir} (Not found, skip)")
+      elsif (FileTest.directory? dir)
+        db_add_dir(dir)
+      else
+        db_add_file(File.dirname(dir), File.basename(dir), File.basename(dir)) # .bashrc/.bashrc のようになる
+      end
+    end
+
+    def db_add_dir(dir)
+      searchDirectory(dir, @package_name, "/", 0)
+    end
+
+    def db_add_file(package_dir, restpath, package_name = nil)
+      # サイレントモード
+      return if @options[:silent_mode]
+
+      # データベースには先頭の'/'を抜いて登録する
+      #   最初から'/'を抜いておけば高速化の余地あり?
+      #   ignore設定との互換性保持が必要
+      restpath = restpath.sub(/^\//, "")
+
+      # パッケージ名を設定
+      package_name = package_name || File.basename(package_dir)
+
+      # レコードの追加
+      result = @grndb.documents.add(package_dir, restpath, package_name)
+
+      # メッセージの表示
+      case result
+      when :newfile
+        @result.inc_add_count
+        alert_info("add_record", File.join(package_dir, restpath))
+      when :update
+        # @grndb.packages.touch(package_name, :updatetime)
+        @result.inc_update_count
+        alert_info("update", File.join(package_dir, restpath))
+      end
+    end
+
+    def searchDirectory(dirname, packname, path, depth)
+      # 現在位置に.gitignoreがあれば無視設定に加える
+      add_current_gitignore(dirname, path) unless @options[:no_auto_ignore]
+
+      # 子の要素を追加
+      Dir.foreach(File.join(dirname, path)) do |name|
+        next if (name == '.' || name == '..')
+
+        next_path = File.join(path, name)
+        fpath     = File.join(dirname, next_path)
+        shortpath = File.join(packname, next_path)
+
+        # 除外ディレクトリならばパス
+        next if ignoreDir?(fpath, next_path)
+
+        # 読み込み不可ならばパス
+        next unless FileTest.readable?(fpath)
+
+        # ファイルならば中身を探索、ディレクトリならば再帰
+        case File.ftype(fpath)
+        when "directory"
+          searchDirectory(dirname, packname, next_path, depth + 1)
+        when "file"
+          unless ignoreFile?(fpath, next_path)
+            db_add_file(dirname, next_path) # shortpathの先頭に'/'が付いているのが気になる
+            @result.inc_file_count
+            # @out.puts "file_count : #{@file_count}" if (@file_count % 100 == 0)
+          end
+        end
+      end
+    end
+
+    def ignoreDir?(fpath, mini_path)
+      FileTest.directory?(fpath) &&
+        (GrenFileTest::ignoreDir?(fpath) ||
+         package_ignore?(fpath, mini_path))
+    end
+
+    def ignoreFile?(fpath, mini_path)
+      GrenFileTest::ignoreFile?(fpath) ||
+        GrenFileTest::binary?(fpath) ||
+        package_ignore?(fpath, mini_path)
+    end
+
+    def package_ignore?(fpath, mini_path)
+      if @current_ignore.ignore?(mini_path)
+        alert_info("ignore", fpath)
+        true
+      else
+        false
+      end
+    end
+
+    def add_current_gitignore(dirname, path)
+      git_ignore = File.join(dirname, path, ".gitignore")
+      
+      if File.exist? git_ignore
+        alert_info("add_ignore", git_ignore)
+        
+        open(git_ignore) do |f|
+          @current_ignore.add IgnoreSetting.create_from_gitignore(path, f.read)
+        end
+      end
+    end
+
+    def alert_info(title, msg)
+      alert(title, msg) if @options[:display_info]
+    end
+
+    def alert(title, msg)
+      if (Util::platform_win?)
+        @out.puts "#{title.ljust(10)} : #{Kconv.kconv(msg, Kconv::SJIS)}"
+      else
+        @out.puts "#{title.ljust(10)} : #{msg}"
+      end
+    end
+
+    def warning_alert(msg)
+      @out.puts "[warning] #{msg}"
+    end
+
+  end
+end
+
+