milkode 0.7.1 → 0.8.0

data/Gemfile

@@ -1,11 +1,24 @@
 source "http://rubygems.org"
+
 # Add dependencies required to use your gem here.
 # Example:
 #   gem "activesupport", ">= 2.3.5"
 # gemspec
+gem 'termcolor','>= 1.2.0'
+gem 'rroonga','>= 1.1.0'
+gem 'rack','>=1.3.4'
+gem 'sinatra', '>=1.2.6'
+gem 'launchy', '>=0.3.7'
+gem 'coderay', '>=1.0.5'
+gem 'thin', '>=1.2.10'
+gem 'archive-zip', '>=0.4.0'
+gem 'haml', '>=3.1.2'
+gem 'sass', '>=3.1.3'
+gem 'thor', '~> 0.15.0'
+
 # Add dependencies to develop your gem here.
 # Include everything needed to run rake, tests, features, etc.
-# group :development do
-#   gem "bundler"
-#   gem "jeweler"
-# end
+group :development do
+  gem "bundler"
+  gem "jeweler"
+end

data/HISTORY.ja.rdoc

@@ -1,3 +1,30 @@
+=== 0.8.0 2012/06/20
+
+* milk web
+  * ホームから 最近使った、追加、更新、お気に入り のパッケージに直接ジャンプ出来るように
+  * パッケージ一覧のソート種類を指定可能に
+  * Twitter bootstrap に対応
+
+* database
+  * groongaデータベースのアクセスを一元化
+  * DocumentTable
+    * shorpath -> restpath
+    * Add package column
+    * 拡張子の検索は完全一致に
+  * PackageTable
+    * 新規追加
+
+* milk 
+  * 'milk fav' コマンドを追加。お気に入りを設定可能に
+  * milk update 時に cleanup も行うように
+  * invoke_task を再定義して help option を処理 (thanks tomykaira)
+  * milk add 出来るスキーマを追加
+    * ssh:foo@bar/baz.git (Support gitolite, thanks hazy_moon)
+    * git@example.com:foo/bar.git
+
+* build
+  * 依存関係の記述をGemfileに移動
+
 === 0.7.0 2012/05/05
 
 * milk

data/HISTORY.rdoc

@@ -1,3 +1,30 @@
+=== 0.8.0 2012/06/20
+
+* milk web
+  * Can jump to package from home. [Recent, add, update, favorite]
+  * Can specify the package list sorting type
+  * Support Twitter bootstrap
+
+* database
+  * Centralized groonga database access
+  * DocumentTable
+    * shorpath -> restpath
+    * Add package column
+    * Extension searches exact matches
+  * PackageTable
+    * New
+
+* milk 
+  * Add 'milk fav' command
+  * Also cleanup do the milk update
+  * Redefining invoke_task and handle the help option (thanks tomykaira)
+  * Add milk add 'schema'
+    * ssh:foo@bar/baz.git (Support gitolite, thanks hazy_moon)
+    * git@example.com:foo/bar.git
+
+* build
+  * Move description of the dependencies to the Gemfile
+
 === 0.7.0 2012/05/05
 
 * milk

data/LICENSE.txt

@@ -1,4 +1,8 @@
-Copyright (c) 2011 ongaeshi
+=================
+Milkode
+=================
+
+Copyright (c) 2011, 2012 ongaeshi
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the
@@ -18,3 +22,21 @@ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
 LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+=================
+Twitter Bootstrap
+=================
+
+Copyright 2012 Twitter, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/Rakefile

@@ -20,22 +20,7 @@ Jeweler::Tasks.new do |gem|
   gem.description = %Q{Line based local source code search engine & grep-command & web-app.}
   gem.email = "ongaeshi0621@gmail.com"
   gem.authors = ["ongaeshi"]
-
-  # Include your dependencies below. Runtime dependencies are required when using your gem,
-  gem.add_runtime_dependency 'termcolor','>= 1.2.0'
-  gem.add_runtime_dependency 'rroonga','>= 1.1.0'
-  gem.add_runtime_dependency 'rack','>=1.3.4'
-  gem.add_runtime_dependency 'sinatra', '>=1.2.6'
-  gem.add_runtime_dependency 'launchy', '>=0.3.7'
-  gem.add_runtime_dependency 'coderay', '>=1.0.5'
-  gem.add_runtime_dependency 'thin', '>=1.2.10'
-  gem.add_runtime_dependency 'archive-zip', '>=0.4.0'
-  gem.add_runtime_dependency 'haml', '>=3.1.2'
-  gem.add_runtime_dependency 'sass', '>=3.1.3'
-  gem.add_runtime_dependency 'thor', '~> 0.15.0'
-
-  # and development dependencies are only needed for development (ie running rake tasks, tests, etc)
-  #  gem.add_development_dependency 'rspec', '> 1.2.3'
+  # Dependencies move to 'Gemfile'
 end
 Jeweler::RubygemsDotOrgTasks.new
 

data/VERSION

@@ -1 +1 @@
-0.7.1
+0.8.0

data/bin/gmilk

@@ -6,5 +6,5 @@
 require 'rubygems'
 require 'milkode/grep/cli_grep'
 
-Version = "0.7.0"
+Version = "0.8.0"
 Milkode::CLI_Grep.execute(STDOUT, ARGV)

data/bin/milk

@@ -6,5 +6,5 @@
 require 'rubygems'
 require 'milkode/cli'
 
-Version = "0.7.0"
+Version = "0.8.0"
 Milkode::CLI.start(ARGV)

data/lib/milkode/cdstk/cdstk.rb

@@ -3,13 +3,11 @@
 require 'yaml'
 require 'pathname'
 require 'rubygems'
-require 'groonga'
 require 'fileutils'
 require 'pathname'
 require 'milkode/common/grenfiletest'
 require 'milkode/common/util'
 require 'milkode/common/dir'
-require 'milkode/findgrep/findgrep'
 include Milkode
 require 'kconv'
 begin
@@ -17,14 +15,14 @@ begin
 rescue LoadError
   $no_readline = true
 end
-require 'milkode/cdweb/lib/database'
 require 'open-uri'
-
 require 'milkode/cdstk/cdstk_command'
-
 require 'milkode/cdstk/yaml_file_wrapper'
 require 'milkode/cdstk/package'
 require 'milkode/common/ignore_checker'
+require 'milkode/database/groonga_database'
+require 'milkode/database/document_record'
+require 'milkode/common/array_diff'
 
 module Milkode
   class IgnoreError < RuntimeError ; end
@@ -40,7 +38,6 @@ module Milkode
     
     def initialize(io = $stdout, db_dir = ".")
       @db_dir = db_dir
-      Database.setup(@db_dir)
       @out = io
       # @out = $stdout # 強制出力
       @is_display_info = false     # alert_info の表示
@@ -60,7 +57,7 @@ module Milkode
       if Dir.emptydir?(@db_dir)
         @yaml = YamlFileWrapper.create(@db_dir)
         @out.puts "create     : #{yaml_file}"
-        db_create(db_file)
+        db_create
         setdb([@db_dir], {}) if (options[:setdb])
       else
         @out.puts "Can't create milkode db (Because not empty in #{db_dir_expand})"
@@ -68,24 +65,24 @@ module Milkode
     end
 
     def assert_compatible
-      db_open(db_file)
+      db_open
     end
 
-    def update_all
+    def update_all(options)
       print_result do 
-        db_open(db_file)
+        db_open
 
         @yaml.contents.each do |package|
-          update_package_in(package)
+          update_package_in(package, options)
         end
       end
     end
 
     def update(args, options)
       update_display_info(options)
-      
+
       if (options[:all])
-        update_all
+        update_all(options)
       else
         if (args.empty?)
           path = File.expand_path('.')
@@ -93,19 +90,19 @@ module Milkode
 
           if (package)
             print_result do
-              db_open(db_file)
-              update_package_in(package)
+              db_open
+              update_package_in(package, options)
             end
           else
             @out.puts "Not registered. If you want to add, 'milk add #{path}'."
           end
         else
           print_result do
-            db_open(db_file)
+            db_open
             args.each do |name|
               package = @yaml.find_name(name)
               if (package)
-                update_package_in(package)
+                update_package_in(package, options)
               else
                 @out.puts "Not found package '#{name}'."
                 return
@@ -117,7 +114,7 @@ module Milkode
     end
 
     def update_for_grep(dir)
-      db_open(db_file)
+      db_open
       update_dir_in(dir)
     end
 
@@ -126,7 +123,7 @@ module Milkode
 
       print_result do
         # データベースを開く
-        db_open(db_file)
+        db_open
 
         # メイン処理
         begin
@@ -183,7 +180,7 @@ module Milkode
 
     def add_dir(dir, no_yaml = false)
       add_yaml(Package.create(dir)) unless no_yaml
-      db_open(db_file)
+      db_open
       update_dir_in(dir)
     end
 
@@ -204,6 +201,12 @@ module Milkode
       # YAML更新
       @yaml.add(package)
       @yaml.save
+
+      # データベースを開く
+      db_open
+
+      # yamlファイルと同期する
+      @grndb.yaml_sync(@yaml.contents)
     end
 
     def convert_content(src)
@@ -251,7 +254,7 @@ module Milkode
     end
 
     def git_url?(src)
-      (src =~ /^git:/) != nil
+      Util::git_url?(src)
     end
     private :git_url?
 
@@ -294,7 +297,7 @@ module Milkode
         list([], {:verbose => true})
         
         if yes_or_no("Remove #{@yaml.contents.size} contents? (yes/no)")
-          db_open(db_file)
+          db_open
 
           @yaml.contents.each do |package|
             remove_dir(package.directory)
@@ -317,7 +320,7 @@ module Milkode
 
           if (package)
             print_result do
-              db_open(db_file)
+              db_open
               remove_dir(package.directory)
             end
           else
@@ -325,7 +328,7 @@ module Milkode
           end
         else
           print_result do
-            db_open(db_file)
+            db_open
             args.each do |name|
               package = @yaml.find_name(name)
               if (package)
@@ -359,6 +362,11 @@ module Milkode
     end
 
     def list(args, options)
+      if options[:check]
+        check_integrity
+        return
+      end
+      
       match_p = @yaml.contents.find_all do |p|
         args.all? {|k| p.name.include? k }
       end
@@ -386,6 +394,62 @@ module Milkode
       end
     end
 
+    def fav(args, options)
+      db_open
+
+      if (args.empty?)
+        @out.puts @grndb.packages.favs.map{|r| r.name}
+      else
+        is_dirty = false
+
+        args.each do |v|
+          # database
+          time = options[:delete] ? Time.at(0) : Time.now
+
+          if @grndb.packages.touch_if(v, :favtime, time)
+            # milkode_yaml
+            package = @yaml.find_name(v)
+            dst = package.options
+            unless options[:delete]
+              dst[:fav] = true
+            else
+              dst.delete(:fav)
+            end
+            package.set_options(dst)
+            @yaml.update(package)
+            is_dirty = true
+          else
+            @out.puts "Not found package '#{v}'"
+          end
+        end
+
+        @yaml.save if is_dirty
+      end
+    end
+
+    def check_integrity
+      db_open
+
+      @out.puts "Check integrity ..."
+
+      yaml = @yaml.contents.map {|p| p.name}.sort
+      grndb = @grndb.packages.map {|p| p.name}.sort
+
+      d = yaml.diff grndb
+
+      unless d[0].empty?
+        @out.puts "'milkode.yaml' has #{d[0]}, but 'PackageTable' don't have."
+      end
+
+      unless d[1].empty?
+        @out.puts "'PackageTable' has #{d[1]}, but 'milkode.yaml' don't have."
+      end
+
+      if d[0].empty? && d[1].empty?
+        @out.puts 'ok'
+      end
+    end
+
     def pwd(options)
       dir = options[:default] ? Dbdir.default_dir : db_dir_expand
       
@@ -403,9 +467,6 @@ module Milkode
     end
 
     def cleanup(options)
-      # 互換性テスト
-      db_open(db_file)
-
       # cleanup開始
       if (options[:force] or yes_or_no("cleanup contents? (yes/no)"))
         print_result do 
@@ -416,9 +477,15 @@ module Milkode
             @package_count += 1
           end
           @yaml.save
+
+          # データベースを開く
+          db_open
+
+          # yamlファイルと同期する
+          @grndb.yaml_sync(@yaml.contents)
           
           # データベースのクリーンアップ
-          Database.instance.cleanup do |record|
+          @documents.cleanup do |record|
             alert("rm_record", record.path)
             @file_count += 1
           end
@@ -430,9 +497,10 @@ module Milkode
       update_display_info(options)
       
       if (options[:all])
-        db_delete(db_file)
-        db_create(db_file)
-        update_all
+        db_delete
+        db_create
+        @grndb.yaml_sync(@yaml.contents)
+        update_all({})
       else
         if (args.empty?)
           path = File.expand_path('.')
@@ -440,7 +508,7 @@ module Milkode
 
           if (package)
             print_result do
-              db_open(db_file)
+              db_open
               remove_dir(package.directory, true)
               add_dir(package.directory, true)
             end
@@ -452,7 +520,7 @@ module Milkode
             args.each do |name|
               package = @yaml.find_name(name)
               if (package)
-                db_open(db_file)
+                db_open
                 remove_dir(package.directory, true)
                 add_dir(package.directory, true)
               else
@@ -466,11 +534,11 @@ module Milkode
     end
 
     def dump
-      db_open(db_file)
-      
-      documents = Groonga::Context.default["documents"]
-      records = documents.select
-      records.each do |record|
+      db_open
+
+      @documents.each do |grnrcd|
+        record = DocumentRecord.new grnrcd
+        
         @out.puts record.inspect
         @out.puts "path : #{record.path}"
         @out.puts "shortpath : #{record.shortpath}"
@@ -479,7 +547,8 @@ module Milkode
         @out.puts "content :", record.content ? record.content[0..64] : nil
         @out.puts
       end
-      @out.puts "total #{records.size} record."
+
+      @out.puts "total #{@documents.size} record."
     end
 
     def dir(args, options)
@@ -586,7 +655,7 @@ EOF
 
       if options[:dry_run]
         # Test mode
-        db_open(db_file)
+        db_open
         @is_display_info = true
         @is_silent = true
         update_dir_in(package.directory)
@@ -634,30 +703,31 @@ EOF
       Dbdir.expand_groonga_path(@db_dir)
     end
 
-    def db_file_expand
-      File.expand_path(db_file)
-    end
-
     def db_dir_expand
       File.expand_path(@db_dir)
     end
 
-    def custom_db?
-      db_dir_expand != Dbdir.default_dir
-    end
-
     def yaml_file
       YamlFileWrapper.yaml_file @db_dir
     end
 
-    def update_package_in(package)
+    def update_package_in(package, options)
       if package.options[:update_with_git_pull]
         Dir.chdir(package.directory) { system("git pull") }
       end
 
+      unless options[:no_clean]
+        cleanup_package_in(package)
+      end
+
       update_dir_in(package.directory)
     end
 
+    def cleanup_package_in(package)
+      db_open
+      @documents.cleanup_package_name(package.name)
+    end
+
     def update_dir_in(dir)
       alert("package", File.basename(dir) )
       @package_count += 1
@@ -669,24 +739,33 @@ EOF
       elsif (FileTest.directory? dir)
         db_add_dir(dir)
       else
-        db_add_file(STDOUT, dir, File.basename(dir))
+        db_add_file(STDOUT, File.dirname(dir), File.basename(dir), File.basename(dir)) # .bashrc/.bashrc のようになる
       end
     end
 
     def remove_dir(dir, no_yaml = false)
-      # yamlから削除
       unless (no_yaml)
-        @yaml.remove(@yaml.find_dir(dir))
+        package = @yaml.find_dir(dir)
+
+        # yamlから削除
+        @yaml.remove(package)
         @yaml.save
+
+        # PackageTableから削除
+        db_open
+        @grndb.packages.remove package.name
       end
-        
+
+      # データベース開く
+      db_open
+
       # データベースからも削除
       # dir = File.expand_path(dir)
 
       alert("rm_package", dir)
       @package_count += 1
 
-      Database.instance.remove_fpath(dir) do |record|
+      @documents.remove_match_path(dir) do |record|
         alert_info("rm_record", record.path)
         @file_count += 1
       end
@@ -716,17 +795,15 @@ EOF
     end
 
     def milkode_info
-      alert('*milkode*', "#{@yaml.contents.size} packages, #{Database.instance.totalRecords} records in #{db_file}.")
+      db_open
+      alert('*milkode*', "#{@yaml.contents.size} packages, #{@documents.size} records in #{db_file}.")
     end
 
-    def db_create(filename)
-      dbfile = Pathname(File.expand_path(filename))
-      dbdir = dbfile.dirname
-      dbdir.mkpath unless dbdir.exist?
+    def db_create
+      filename = db_file
       
-      unless dbfile.exist?
-        Groonga::Database.create(:path => dbfile.to_s)
-        db_define
+      unless File.exist? filename
+        db_open
         @out.puts "create     : #{filename} created."
       else
         @out.puts "message    : #{filename} already exist."
@@ -734,60 +811,29 @@ EOF
     end
 
     def list_info(packages)
-      option = FindGrep::FindGrep::DEFAULT_OPTION.dup
-      option.dbFile = Dbdir.groonga_path(Dbdir.default_dir)
-      option.isSilent = true
-      option.packages = packages.map{|p| p.directory}
-      findGrep = FindGrep::FindGrep.new([], option)
-      records = findGrep.pickupRecords
-      
+      db_open
+      records = @documents.search(:packages => packages.map{|p| p.name})
       alert('*milk_list*', "#{packages.size} packages, #{records.size} records in #{db_file}.")
     end
 
-    def db_define
-      Groonga::Schema.define do |schema|
-        schema.create_table("documents", :type => :hash) do |table|          
-          table.string("path")
-          table.string("shortpath")
-          table.text("content")
-          table.time("timestamp")
-          table.text("suffix")
-        end
-
-        schema.create_table("terms",
-                            :type => :patricia_trie,
-                            :key_normalize => true,
-                            :default_tokenizer => DEFAULT_TOKENIZER) do |table|
-          table.index("documents.path", :with_position => true)
-          table.index("documents.shortpath", :with_position => true)
-          table.index("documents.content", :with_position => true)
-          table.index("documents.suffix", :with_position => true)
-        end
+    def db_open
+      if !@grndb || @grndb.closed?
+        @grndb = GroongaDatabase.new
+        @grndb.open(@db_dir)
+        @documents = @grndb.documents
       end
     end
 
-    def db_open(filename)
-      dbfile = Pathname(File.expand_path(filename))
-      
-      if dbfile.exist?
-        # データベースを開く
-        Groonga::Database.open(dbfile.to_s)
-
-        # 互換性テスト
-        db_compatible?
-      else
-        raise "error      : #{dbfile.to_s} not found!!"
-      end
-    end
+    def db_delete
+      filename = db_file
 
-    def db_delete(filename)
       raise "Illegal file name : #{filename}." unless filename =~ /\.db$/
+
       Dir.glob("#{filename}*").each do |f|
         @out.puts "delete     : #{f}"
         FileUtils.rm_r(f)
       end
     end
-    private :db_delete
       
     def db_add_dir(dir)
       @current_package = @yaml.package_root(dir)
@@ -797,57 +843,30 @@ EOF
     end
     private :db_add_dir
 
-    def db_add_file(stdout, filename, shortpath)
+    def db_add_file(stdout, package_dir, restpath, package_name = nil)
       # サイレントモード
       return if @is_silent
-      
-      # ファイル名を全てUTF-8に変換
-      filename_utf8 = Util::filename_to_utf8(filename)
-      shortpath_utf8 = Util::filename_to_utf8(shortpath)
-      suffix_utf8 = File::extname(filename_utf8)
-      
-      # 格納するデータ
-      values = {
-        :path => filename_utf8,
-        :shortpath => shortpath_utf8,
-        :content => nil,
-        :timestamp => File.mtime(filename),
-        :suffix => suffix_utf8,
-      }
-      
-      # 検索するデータベース
-      documents = Groonga::Context.default["documents"]
-      
-      record = documents[ values[:path] ]
-      isNewFile = false
 
-      unless record
-        document = documents.add(values[:path])
-        isNewFile = true
-      else
-        document = record
-      end
-      
-      # タイムスタンプが新しければデータベースに格納
-      if (document[:timestamp] < values[:timestamp])
-        # 実際に使うタイミングでファイルの内容を読み込み
-        # values[:content] = open(filename).read
-        # データベース内の文字コードは'utf-8'で統一
-        values[:content] = Kconv.kconv(File.read(filename), Kconv::UTF8)
-        
-        # データベースに格納
-        values.each do |key, value|
-          if (key == :path)
-            if (isNewFile)
-              @add_count += 1
-              alert_info("add_record", value)
-            else
-              @update_count += 1
-              alert_info("update", value)
-            end
-          end
-          document[key] = value
-        end
+      # データベースには先頭の'/'を抜いて登録する
+      #   最初から'/'を抜いておけば高速化の余地あり?
+      #   ignore設定との互換性保持が必要
+      restpath = restpath.sub(/^\//, "")
+
+      # パッケージ名を設定
+      package_name = package_name || File.basename(package_dir)
+
+      # レコードの追加
+      result = @documents.add(package_dir, restpath, package_name)
+
+      # メッセージの表示
+      case result
+      when :newfile
+        @add_count += 1
+        alert_info("add_record", File.join(package_dir, restpath))
+      when :update
+        @grndb.packages.touch(package_name, :updatetime)
+        @update_count += 1
+        alert_info("update", File.join(package_dir, restpath))
       end
     end
 
@@ -875,7 +894,7 @@ EOF
           searchDirectory(stdout, dirname, packname, next_path, depth + 1)
         when "file"
           unless ignoreFile?(fpath, next_path)
-            db_add_file(stdout, fpath, shortpath)
+            db_add_file(stdout, dirname, next_path) # shortpathの先頭に'/'が付いているのが気になる
             @file_count += 1
             # @out.puts "file_count : #{@file_count}" if (@file_count % 100 == 0)
           end
@@ -934,22 +953,6 @@ EOF
       @out.puts "[fatal] #{msg}"
     end
 
-    def db_compatible?
-      begin
-        db_define
-      rescue Groonga::Schema::Error => e
-        puts <<EOF
-Milkode repository is old -> #{db_dir_expand}.
-Please rebuild repository, 
-
-  milk rebuild
-
-See 'milk --help' or http://milkode.ongaeshi.me .
-EOF
-        exit -1
-      end
-    end
-
     def update_display_info(options)
       @is_display_info = true if (options[:verbose])
     end