codestock 0.1.0

data/lib/findgrep/findgrep.rb

@@ -0,0 +1,408 @@
+# -*- coding: utf-8 -*-
+require 'find'
+require File.join(File.dirname(__FILE__), 'result')
+require 'rubygems'
+require 'termcolor'
+require 'kconv'
+require File.join(File.dirname(__FILE__), '../common/platform')
+require File.join(File.dirname(__FILE__), '../common/grenfiletest')
+require File.join(File.dirname(__FILE__), '../common/grensnip')
+require 'groonga'
+require File.join(File.dirname(__FILE__), '../common/util')
+include Gren
+require 'cgi'
+
+module FindGrep
+  class FindGrep
+    Option = Struct.new(:keywordsNot,
+                        :keywordsOr,
+                        :directory,
+                        :depth,
+                        :ignoreCase,
+                        :colorHighlight,
+                        :isSilent,
+                        :debugMode,
+                        :filePatterns,
+                        :suffixs,
+                        :ignoreFiles,
+                        :ignoreDirs,
+                        :kcode,
+                        :noSnip,
+                        :dbFile,
+                        :groongaOnly,
+                        :isMatchFile,
+                        :dispHtml)
+    
+    DEFAULT_OPTION = Option.new([],
+                                [],
+                                ".",
+                                -1,
+                                false,
+                                false,
+                                false,
+                                false,
+                                [],
+                                [],
+                                [],
+                                [],
+                                Platform.get_shell_kcode,
+                                false,
+                                nil,
+                                false,
+                                false,
+                                false)
+    
+    attr_reader :documents
+    
+    def initialize(patterns, option)
+      @patterns = patterns
+      @option = option
+      @patternRegexps = strs2regs(patterns, @option.ignoreCase)
+      @subRegexps = strs2regs(option.keywordsNot, @option.ignoreCase)
+      @orRegexps = strs2regs(option.keywordsOr, @option.ignoreCase)
+      @filePatterns = (!@option.dbFile) ? strs2regs(option.filePatterns) : []
+      @ignoreFiles = strs2regs(option.ignoreFiles)
+      @ignoreDirs = strs2regs(option.ignoreDirs)
+      @result = Result.new(option.directory)
+      open_database if (@option.dbFile)
+    end
+
+    def open_database()
+      # データベース開く
+      dbfile = Pathname(File.expand_path(@option.dbFile))
+      
+      if dbfile.exist?
+        Groonga::Database.open(dbfile.to_s)
+        puts "open    : #{dbfile} open."
+      else
+        raise "error    : #{dbfile.to_s} not found!!"
+      end
+      
+      # ドキュメントを取
+      @documents = Groonga::Context.default["documents"]
+    end
+
+    def strs2regs(strs, ignore = false)
+      regs = []
+
+      strs.each do |v|
+        option = 0
+        option |= Regexp::IGNORECASE if (ignore)
+        regs << Regexp.new(v, option)
+      end
+
+      regs
+    end
+
+    def searchAndPrint(stdout)
+      unless (@option.dbFile)
+        searchFromDir(stdout, @option.directory, 0)
+      else
+        searchFromDB(stdout, @option.directory)
+      end
+
+      @result.time_stop
+      
+      if (!@option.isSilent && !@option.dispHtml)
+        if (@option.debugMode)
+          stdout.puts
+          stdout.puts "--- search --------"
+          print_fpaths stdout, @result.search_files
+          stdout.puts "--- match --------"
+          print_fpaths stdout, @result.match_files
+          stdout.puts "--- ignore-file --------"
+          print_fpaths stdout, @result.ignore_files
+          stdout.puts "--- ignore-dir --------"
+          print_fpaths stdout, @result.prune_dirs
+          stdout.puts "--- unreadable --------"
+          print_fpaths stdout, @result.unreadable_files
+        end
+
+        unless (@option.colorHighlight)
+          stdout.puts
+        else
+          stdout.puts HighLine::REVERSE + "------------------------------------------------------------" + HighLine::CLEAR
+        end
+
+        @result.print(stdout)
+      end
+    end
+
+    def searchFromDB(stdout, dir)
+      # 全てのパターンを検索
+      table = @documents.select do |record|
+        expression = nil
+
+        # キーワード
+        @patterns.each do |word|
+          sub_expression = record.content =~ word
+          if expression.nil?
+            expression = sub_expression
+          else
+            expression &= sub_expression
+          end
+        end
+        
+        # パス
+        @option.filePatterns.each do |word|
+          sub_expression = record.path =~ word
+          if expression.nil?
+            expression = sub_expression
+          else
+            expression &= sub_expression
+          end
+        end
+
+        # 拡張子(OR)
+        se = suffix_expression(record) 
+        expression &= se if (se)
+        
+        # 検索式
+        expression
+      end
+      
+      # タイムスタンプでソート
+      records = table.sort([{:key => "_score", :order => "descending"},
+                            {:key => "timestamp", :order => "descending"}])
+
+      # データベースにヒット
+      stdout.puts "Found   : #{records.size} records." unless (@option.dispHtml)
+
+      # 検索にヒットしたファイルを実際に検索
+      records.each do |record|
+        if (@option.groongaOnly)
+          searchGroongaOnly(stdout, record)
+        else
+          searchFile(stdout, record.path, record.path) if FileTest.exist?(record.path)
+        end
+      end
+    end
+
+    def and_expression(key, list)
+      sub = nil
+      
+      list.each do |word|
+        e = key =~ word
+        if sub.nil?
+          sub = e
+        else
+          sub &= e
+        end
+      end
+
+      sub
+    end
+
+    def suffix_expression(record)
+      sub = nil
+      
+      @option.suffixs.each do |word|
+        e = record.suffix =~ word
+        if sub.nil?
+          sub = e
+        else
+          sub |= e
+        end
+      end
+
+      sub
+    end
+    private :suffix_expression
+      
+
+    def searchFromDir(stdout, dir, depth)
+      if (@option.depth != -1 && depth > @option.depth)
+        return
+      end
+      
+      Dir.foreach(dir) do |name|
+        next if (name == '.' || name == '..')
+          
+        fpath = File.join(dir,name)
+        fpath_disp = fpath.gsub(/^.\//, "")
+        
+        # 除外ディレクトリならばパス
+        if ignoreDir?(fpath)
+          @result.prune_dirs << fpath_disp if (@option.debugMode)
+          next;
+        end
+
+        # 読み込み不可ならばパス
+        unless FileTest.readable?(fpath)
+          @result.unreadable_files << fpath_disp if (@option.debugMode)
+          next
+        end
+
+        # ファイルならば中身を探索、ディレクトリならば再帰
+        case File.ftype(fpath)
+        when "directory"
+          searchFromDir(stdout, fpath, depth + 1)
+        when "file"
+          searchFile(stdout, fpath, fpath_disp)
+        end          
+      end
+    end
+    private :searchFromDir
+
+    def print_fpaths(stdout, data)
+      stdout.print data.join("\n")
+      stdout.puts if data.count > 0
+      stdout.puts "total: #{data.count}"
+      stdout.puts
+    end
+    private :print_fpaths
+
+    def ignoreDir?(fpath)
+      FileTest.directory?(fpath) &&
+      (GrenFileTest::ignoreDir?(File.basename(fpath)) || ignoreDirUser?(fpath))
+    end
+    private :ignoreDir?
+
+    def ignoreDirUser?(fpath)
+      @ignoreDirs.any? {|v| v.match File.basename(fpath) }
+    end
+    private :ignoreDirUser?
+
+    def ignoreFile?(fpath)
+      !correctFileUser?(fpath) ||
+      GrenFileTest::ignoreFile?(fpath) ||
+      ignoreFileUser?(fpath) ||
+      GrenFileTest::binary?(fpath)
+    end
+    private :ignoreFile?
+
+    def correctFileUser?(fpath)
+      @filePatterns.empty? ||
+      @filePatterns.any? {|v| v.match File.basename(fpath) }
+    end
+    private :correctFileUser?
+
+    def ignoreFileUser?(fpath)
+      @ignoreFiles.any? {|v| v.match File.basename(fpath) }
+    end
+    private :ignoreFileUser?
+
+    def searchFile(stdout, fpath, fpath_disp)
+      @result.count += 1
+      @result.size += FileTest.size(fpath)
+
+      # 除外ファイル
+      if ignoreFile?(fpath)
+        @result.ignore_files << fpath_disp if (@option.debugMode)
+        return
+      end
+      
+      @result.search_count += 1
+      @result.search_size += FileTest.size(fpath)
+
+      @result.search_files << fpath_disp if (@option.debugMode)
+
+      open(fpath, "r") do |file|
+        searchData(stdout, file2data(file), fpath_disp)
+      end
+    end
+    private :searchFile
+
+    def searchGroongaOnly(stdout, record)
+      file_size = record.content.size
+      
+      @result.count += 1
+      @result.size += file_size
+      
+      @result.search_count += 1
+      @result.search_size += file_size
+      
+      @result.search_files << record.path if (@option.debugMode)
+
+      searchData(stdout, record.content, record.path)
+    end
+    private :searchGroongaOnly
+
+    def searchData(stdout, data, path)
+      match_file = false
+
+      data.each_with_index { |line, index|
+        result, match_datas = match?(line)
+
+        if ( result )
+          unless (@option.dispHtml)
+            header = "#{path}:#{index + 1}:"
+            line = GrenSnip::snip(line, match_datas) unless (@option.noSnip)
+
+            unless (@option.colorHighlight)
+              stdout.puts header + line
+            else
+              stdout.puts HighLine::BLUE + header + HighLine::CLEAR + GrenSnip::coloring(line, match_datas)
+            end
+          else
+            line_no = index + 1
+            line = GrenSnip::snip(line, match_datas) unless (@option.noSnip)
+            
+            stdout.puts <<EOF
+<h2><a href="../::view#{path}">#{path}</a></h2>
+<pre>
+#{line_no} : #{CGI.escapeHTML(line)}
+</pre>
+EOF
+          end
+
+          unless match_file
+            @result.match_file_count += 1
+            @result.match_files << path if (@option.debugMode)
+            match_file = true
+            break if (@option.isMatchFile)
+          end
+
+          @result.match_count += 1
+        end
+      }
+    end
+    private :searchData
+
+    def file2data(file)
+        data = file.read
+
+        if (@option.kcode != Kconv::NOCONV)
+          file_kcode = Kconv::guess(data)
+
+          if (file_kcode != @option.kcode)
+#            puts "encode!! #{fpath} : #{@option.kcode} <- #{file_kcode}"
+            data = data.kconv(@option.kcode, file_kcode)
+          end
+        end
+
+        data = data.split("\n");
+    end
+    private :file2data
+
+    def match?(line)
+      match_datas = []
+      @patternRegexps.each {|v| match_datas << v.match(line)}
+
+      sub_matchs = []
+      @subRegexps.each {|v| sub_matchs << v.match(line)}
+
+      or_matchs = []
+      @orRegexps.each {|v| or_matchs << v.match(line)}
+      
+      unless (@option.isMatchFile)
+        result = match_datas.all? && !sub_matchs.any? && (or_matchs.empty? || or_matchs.any?)
+      else
+        result = first_condition(match_datas, sub_matchs, or_matchs)
+      end
+      result_match = match_datas + or_matchs
+      result_match.delete(nil)
+
+      return result, result_match
+    end
+    private :match?
+
+    def first_condition(match_datas, sub_matchs, or_matchs)
+      unless match_datas.empty?
+        match_datas[0]
+      else
+        or_matchs[0]
+      end
+    end
+  end
+end

data/lib/findgrep/result.rb

@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+require 'find'
+require File.join(File.dirname(__FILE__), '../common/util')
+include Gren
+
+module FindGrep
+  class Result
+    attr_accessor :count
+    attr_accessor :search_count
+    attr_accessor :match_file_count
+    attr_accessor :match_count
+    attr_accessor :size
+    attr_accessor :search_size
+    
+    attr_accessor :search_files
+    attr_accessor :match_files
+    attr_accessor :unreadable_files
+    attr_accessor :prune_dirs
+    attr_accessor :ignore_files
+
+    def initialize(start_dir)
+      @start_dir = File.expand_path(start_dir)
+      @count, @search_count, @match_file_count, @match_count, @size, @search_size = 0, 0, 0, 0, 0, 0
+      @start_time = Time.now
+      @search_files, @match_files, @unreadable_files, @prune_dirs, @ignore_files  = [], [], [], [], []
+    end
+
+    def time_stop
+      @end_time = Time.now
+    end
+
+    def time
+      @end_time - @start_time 
+    end
+
+    def print(stdout)
+      stdout.puts "dir   : #{@start_dir} (#{Util::time_s(time)})"
+      stdout.puts "files : #{@search_count} in #{@count} (#{Util::size_s(@search_size)} in #{Util::size_s(@size)})"
+      stdout.puts "match : #{@match_file_count} files, #{match_count} hit"
+    end
+
+  end
+end

data/lib/gren/cli.rb

@@ -0,0 +1,65 @@
+# -*- coding: utf-8 -*-
+require 'optparse'
+require File.join(File.dirname(__FILE__), '../findgrep/findgrep')
+
+module Gren
+  class CLI
+    def self.execute(stdout, arguments=[])
+      # オプション
+      option = FindGrep::FindGrep::DEFAULT_OPTION
+
+      # オプション解析
+      opt = OptionParser.new("#{File.basename($0)} [option] pattern")
+      opt.on('--not PATTERN', 'Keyword is not included.') {|v| option.keywordsNot << v}
+      opt.on('--or PATTERN', 'Either of keyword is contained.') {|v| option.keywordsOr << v}
+      opt.on('-d DIR', '--directory DIR', 'Start directory. (deafult:".")') {|v| option.directory = v}
+      opt.on('--depth DEPTH', 'Limit search depth. ') {|v| option.depth = v.to_i}
+      opt.on('--this', '"--depth 0"') {|v| option.depth = 0}
+      opt.on('-i', '--ignore', 'Ignore case.') {|v| option.ignoreCase = true}
+      opt.on('-s', '--silent', 'Silent. Display match line only.') {|v| option.isSilent = true}
+      opt.on('--debug', 'Debug display.') {|v| option.debugMode = true}
+      opt.on('-c', '--color', 'Color highlight.') {|v| option.colorHighlight = true}
+      opt.on('-f REGEXP', '--file-regexp REGEXP', 'Search file regexp. (Enable multiple call)') {|v| option.filePatterns << v}
+      opt.on('--if REGEXP', '--ignore-file REGEXP', 'Ignore file pattern. (Enable multiple call)') {|v| option.ignoreFiles << v}
+      opt.on('--id REGEXP', '--ignore-dir REGEXP', 'Ignore dir pattern. (Enable multiple call)') {|v| option.ignoreDirs << v}
+      opt.on('-e ENCODE', '--encode ENCODE', 'Specify encode(none, auto, jis, sjis, euc, ascii, utf8, utf16). Default is "auto"') {|v| setupEncodeOption(option, v) }
+      opt.on('--no-snip', 'There being a long line, it does not snip.') {|v| option.noSnip = true }
+      opt.parse!(arguments)
+
+      # 検索オブジェクトの生成
+      if (arguments.size > 0 || option.keywordsOr.size > 0)
+        findGrep = FindGrep::FindGrep.new(arguments, option)
+        findGrep.searchAndPrint(stdout)
+      else
+        stdout.print opt.help
+      end
+
+    end
+
+    def self.setupEncodeOption(option, encode)
+      case encode.downcase
+      when 'none'
+        option.kcode = Kconv::NOCONV
+      when 'auto'
+        option.kcode = Platform.get_shell_kcode
+      when 'jis'
+        option.kcode = Kconv::JIS
+      when 'sjis'
+        option.kcode = Kconv::SJIS
+      when 'euc'
+        option.kcode = Kconv::EUC
+      when 'ascii'
+        option.kcode = Kconv::ASCII
+      when 'utf8'
+        option.kcode = Kconv::UTF8
+      when 'utf16'
+        option.kcode = Kconv::UTF16
+      else
+        puts "Invalid encode."
+        puts "  none, auto, jis, sjis, euc, ascii, utf8, utf16"
+        exit(-1)
+      end
+    end
+    
+  end
+end

data/lib/gren.rb

@@ -0,0 +1,6 @@
+$:.unshift(File.dirname(__FILE__)) unless
+  $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
+
+module Gren
+  VERSION = '0.3.2'
+end

data/lib/grendb/cli.rb

@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+require 'optparse'
+require File.join(File.dirname(__FILE__), '../findgrep/findgrep')
+require File.join(File.dirname(__FILE__), '../common/dbdir')
+include CodeStock
+
+module Grendb
+  class CLI
+    def self.execute(stdout, arguments=[])
+      # オプション
+      option = FindGrep::FindGrep::DEFAULT_OPTION
+      option.dbFile = db_groonga_path(db_default_dir)
+      
+      # デフォルトのマッチモードは'File'
+      option.isMatchFile = true
+
+      # オプション解析
+      opt = OptionParser.new("#{File.basename($0)} [option] keyword1 [keyword2 ...]")
+      opt.on('--db [GREN_DB_FILE]', 'Search from the grendb database.') {|v| option.dbFile = db_groonga_path(v) }
+      opt.on('-f KEYWORD', '--file-keyword KEYWORD', 'Path keyword. (Enable multiple call)') {|v| option.filePatterns << v}
+      opt.on('-s SUFFIX', '--suffix SUFFIX',  'Search suffix.') {|v| option.suffixs << v }
+      opt.on('-i', '--ignore', 'Ignore case.') {|v| option.ignoreCase = true}
+      opt.on('-c', '--color', 'Color highlight.') {|v| option.colorHighlight = true}
+      opt.on('--no-snip', 'There being a long line, it does not snip.') {|v| option.noSnip = true }
+      opt.on('-g', '--groonga-only', 'Search only groonga db.') {|v| option.groongaOnly = true }
+      opt.on('--mf', '--match-file', 'Match file. (Default)') {|v| option.isMatchFile = true }
+      opt.on('-l', '--ml', '--match-line', 'Match line, same mode as "gren".') {|v| option.isMatchFile = false }
+
+      opt.parse!(arguments)
+
+      # 検索オブジェクトの生成
+      if (option.dbFile && (arguments.size > 0 || option.keywordsOr.size > 0))
+        findGrep = FindGrep::FindGrep.new(arguments, option)
+        findGrep.searchAndPrint(stdout)
+      else
+        stdout.print opt.help
+        stdout.puts
+        stdout.puts "please set GREN DATABSE FILE!! (--db option, or set ENV['GRENDB_DEFAULT_DB'].)" unless option.dbFile
+      end
+
+    end
+  end
+end

data/lib/grenweb/cli.rb

@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*-
+require 'rubygems'
+require 'optparse'
+require 'fileutils'
+require File.join(File.dirname(__FILE__), 'database')
+require File.join(File.dirname(__FILE__), '../common/dbdir')
+include CodeStock
+require 'rack'
+require 'launchy'
+
+module Grenweb
+  class CLI
+    def self.execute(stdout, arguments=[])
+      option = {
+        :Port => 9292,
+        :DbDir => db_default_dir,
+      }
+      
+      opt = OptionParser.new("#{File.basename($0)}")
+      opt.on('--db DB_DIR', 'Database dir (default : ~/.codestock)') {|v| option[:DbDir] = v }
+      opt.on('-p', '--port PORT', 'use PORT (default: 9292)') {|v| option[:Port] = v }
+      opt.on('--no-browser', 'No launch browser.') {|v| option[:NoBrowser] = true }
+      opt.parse!(arguments)
+
+      # webサーバー起動
+      stdout.puts <<EOF
+Start up grenweb !!
+URL : http://localhost:#{option[:Port]}
+DB  : #{option[:DbDir]}
+----------------------------------------
+EOF
+
+      # 使用するデータベースの位置設定
+      Database.setup(option[:DbDir])
+      
+      # サーバースクリプトのある場所へ移動
+      FileUtils.cd(File.dirname(__FILE__))
+      
+      # ブラウザ起動
+      Launchy.open("http://localhost:#{option[:Port]}") unless (option[:NoBrowser])
+      
+      # サーバー起動
+      Rack::Server.start(
+                         :environment => "development",
+                         :pid         => nil,
+                         :Port        => option[:Port],
+                         :Host        => "0.0.0.0",
+                         :AccessLog   => [],
+                         :config      => "grenweb.ru"
+                         )
+
+    end
+  end
+end