xapian-findex 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ad6429b138298d9caddc9f7db8be15022c1ea77f
+  data.tar.gz: 71a45fe807933edefeabfdb0c97e66af6ac4965d
+SHA512:
+  metadata.gz: 6bc6c9b451c6c10fd3d20b83db5a0385fa1f9f8d2774a93f7d6c058442e85c69e096eec882da17ba57d0b7f99a9cef0d0c4cb9028b9d5a8152b78c1d7940e7a7
+  data.tar.gz: e63c6474704a29b0f6de957ff0eadccc490224b36197a2b60bf0e8c156083244c3e0b13b74783c74c1c7ecfc9cac5ffa178bfa3b953783dbf91835a68649c213

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+/vendor/
+/*.gem

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+rvm:
+  - 2.3.0
+before_install: gem install bundler -v 1.11.2

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in findex.gemspec
+gemspec

data/README.md

@@ -0,0 +1,84 @@
+# Findex
+
+Findex is a command line file indexing tool. It performs full text indexing
+with [Xapian](http://xapian.org) as Backend and allows a great deal of
+customization via a simple DSL.
+
+## Installation
+
+    $ gem install xapian-findex
+
+### Dependencies
+
+- xapian-core
+- libmagic
+
+## Usage
+
+### Indexing
+
+    $ findex index [path]
+
+### Searching
+
+    $ findex search [path] -- some xapian search query
+
+The following prefixes are available by default:
+
+- `date` - the mtime of the document (allows [value range queries](http://xapian.org/docs/valueranges.html)
+- `path` - the relative path of the file
+
+## DSL
+
+Findex will add a directory `.findex` in the indexed path. This directory
+contains the database and a `config.rb` file.  This way you can move your
+indexed directory to another location without affecting the index. This also
+simplifies backups.
+
+You can customize the behavior of the index by editing `config.rb`.
+
+### Options
+
+Below are the available options for Findex with their defaults. It should give
+you an idea how to change them to fit your needs.
+
+``` ruby
+indexer.stem_language = 'none'
+```
+
+### Mimetypes
+
+Findex uses libmagic to determine the mimetypes of files to be indexed. To
+configure special handling for pdf files you could add the following to your
+`config.rb`:
+
+``` ruby
+indexer.on('application/pdf') do |file, index|
+  metadata = `pdfinfo "#{file}"`.lines.map { |l| l.chomp.split(/:\s+/) }.to_h
+  index[:title] = metadata["Title"]
+  index[:author] = metadata["Author"]
+  index << `pdf2text "#{file}" -`
+end
+```
+
+An example search query:
+
+    $ findex search path/to/my/library -- author:'Jules Verne'
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run
+`rake spec` to run the tests. You can also run `bin/console` for an interactive
+prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To
+release a new version, update the version number in `version.rb`, and then run
+`bundle exec rake release`, which will create a git tag for the version, push
+git commits and tags, and push the `.gem` file to
+[rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at
+https://github.com/jreinert/findex.
+

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "findex"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/exe/findex

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'findex/cli'
+
+Findex::Cli.new(ARGV)

data/lib/findex/cli.rb

@@ -0,0 +1,31 @@
+require_relative './indexer'
+require_relative './search'
+
+module Findex
+  class Cli
+    USAGE = "Usage: #{$PROGRAM_NAME} (index|search) [path] [-- query]".freeze
+
+    def initialize(args)
+      query_separator_index = args.find_index('--') || -1
+      action, path = args[0..query_separator_index]
+
+      case action
+      when 'index' then index(path || '.')
+      when 'search'
+        abort(USAGE) if query_separator_index == -1
+        search(path, args[query_separator_index..-1])
+      else abort("Unsupported action '#{action}'\n#{USAGE}")
+      end
+    end
+
+    def index(path)
+      Indexer.new(path || '.').start
+    end
+
+    def search(path, query_terms)
+      Search.new(path || '.').query(query_terms).each do |document|
+        puts document.full_path
+      end
+    end
+  end
+end

data/lib/findex/config.rb

@@ -0,0 +1,12 @@
+module Findex
+  class Config
+    attr_accessor :stem_language, :log_level
+
+    def initialize
+      @stem_language = 'none'
+      @log_level = :info
+
+      yield self if block_given?
+    end
+  end
+end

data/lib/findex/document_decorator.rb

@@ -0,0 +1,119 @@
+require 'time'
+require 'date'
+require 'xapian'
+require_relative './file_indexer'
+require_relative './term_generator_decorator'
+
+module Findex
+  # A decorator class to be used around Xapian::Documents
+  class DocumentDecorator < SimpleDelegator
+    TIME_FORMAT = '%s'.freeze
+    DATE_FORMAT = '%Y%m%d'.freeze
+    VALUE_SLOTS = {
+      path: 0,
+      mtime: 1,
+      date: 2
+    }.freeze
+
+    attr_reader :xapian_document
+
+    def initialize(xapian_document, root_path, file = nil)
+      @xapian_document = xapian_document
+      @root_path = root_path
+      super(xapian_document)
+      update_values_from(file) if file
+    end
+
+    def path
+      @path ||= value(:path)
+    end
+
+    def path=(path)
+      add_value(:path, path.to_s)
+      @path = path
+    end
+
+    def mtime
+      @mtime ||= Time.strptime(value(:mtime), TIME_FORMAT)
+    end
+
+    def actual_mtime
+      Time.at(full_path.mtime.to_i)
+    end
+
+    def mtime=(time)
+      add_value(:mtime, time.strftime(TIME_FORMAT))
+      @mtime = time
+    end
+
+    def date
+      @date ||= Date.strptime(value(:date), DATE_FORMAT)
+    end
+
+    def date=(date)
+      add_value(:date, date.strftime(DATE_FORMAT))
+      @date = date
+    end
+
+    def exists?
+      full_path.exist?
+    end
+
+    def deleted?
+      !exists?
+    end
+
+    def changed?
+      actual_mtime > mtime
+    end
+
+    def extension
+      path.extname[1..-1]
+    end
+
+    def value(slot)
+      return super if slot.is_a?(Integer)
+      super(VALUE_SLOTS[slot])
+    end
+
+    def add_value(slot, value)
+      return super if slot.is_a?(Integer)
+      super(VALUE_SLOTS[slot], value)
+    end
+
+    def update(db, term_generator)
+      index(term_generator)
+      db.replace_document(docid, xapian_document)
+    end
+
+    def insert(db, term_generator)
+      index(term_generator)
+      db.add_document(xapian_document)
+    end
+
+    def full_path
+      @root_path + path
+    end
+
+    private
+
+    def index(term_generator)
+      update_values_from(full_path)
+      clear_terms
+      term_generator.document = xapian_document
+      index_text(TermGeneratorDecorator.new(term_generator))
+    end
+
+    def index_text(term_generator)
+      file_indexer = FileIndexer.new(full_path)
+      term_generator[:path] = path
+      file_indexer.index(term_generator)
+    end
+
+    def update_values_from(file)
+      self.path = file.relative_path_from(@root_path)
+      self.mtime = full_path.mtime
+      self.date = mtime.to_date
+    end
+  end
+end

data/lib/findex/dsl.rb

@@ -0,0 +1,21 @@
+require_relative './config'
+
+module Findex
+  class DSL
+    attr_reader :definitions
+
+    def initialize
+      @definitions = {}
+      @config = Config.new
+    end
+
+    def on(extension, &block)
+      @definitions[extension] = block
+    end
+
+    def config(&block)
+      return @config unless block_given?
+      @config = Config.new(&block)
+    end
+  end
+end

data/lib/findex/file_indexer.rb

@@ -0,0 +1,20 @@
+require_relative './index'
+require 'filemagic'
+
+module Findex
+  class FileIndexer
+    TEXT_INDEXER = lambda do |file, index|
+      index << File.read(file)
+    end
+
+    def initialize(file)
+      @file = file
+      @mime = FileMagic.open(:mime_type) { |magic| magic.file(file.to_s) }
+    end
+
+    def index(term_generator)
+      indexer = Findex.definitions[@mime] || TEXT_INDEXER
+      indexer.call(@file, term_generator)
+    end
+  end
+end

data/lib/findex/index.rb

@@ -0,0 +1,25 @@
+require 'logger'
+require_relative './dsl'
+
+module Findex
+  def self.index
+    dsl = DSL.new
+    yield dsl
+    @definitions = dsl.definitions
+    @config = dsl.config
+  end
+
+  def self.definitions
+    @definitions
+  end
+
+  def self.config
+    @config
+  end
+
+  def self.logger
+    @logger ||= Logger.new(STDOUT).tap do |logger|
+      logger.level = config.log_level
+    end
+  end
+end

data/lib/findex/indexer.rb

@@ -0,0 +1,88 @@
+require 'xapian'
+require 'pathname'
+require_relative './document_decorator'
+require_relative './config'
+
+module Findex
+  # Class used for indexing files
+  class Indexer
+    DB_FLAGS = Xapian::DB_CREATE_OR_OPEN
+
+    def initialize(root_path)
+      @root_path = Pathname.new(root_path)
+      @findex_path = @root_path + '.findex'
+      setup
+    end
+
+    def start
+      @db.begin_transaction
+      existing = refresh_existing
+      files.each do |file|
+        next if existing.include?(file)
+        insert(file)
+      end
+      @db.commit_transaction
+    end
+
+    private
+
+    def insert(file)
+      xapian_document = Xapian::Document.new
+      document = DocumentDecorator.new(xapian_document, @root_path, file)
+      Findex.logger.info("adding '#{document.path}' to the index")
+      document.insert(@db, @term_generator)
+    end
+
+    def files
+      Enumerator.new do |y|
+        Pathname.glob(@root_path + '**' + '*').each do |path|
+          next if path.ascend.any? { |p| p == @findex_path }
+          next if path.directory?
+          y << Pathname.new(path)
+        end
+      end
+    end
+
+    def refresh_existing
+      documents.map do |document|
+        if document.deleted?
+          Findex.logger.info("deleting '#{document.path}' from index")
+          @db.delete_document(document.docid)
+        elsif document.changed?
+          Findex.logger.info("updating '#{document.path}'")
+          document.update(@db, @term_generator)
+        end
+
+        document.full_path
+      end
+    end
+
+    def documents
+      enquire = Xapian::Enquire.new(@db)
+      enquire.query = Xapian::Query::MatchAll
+      mset = enquire.mset(0, @db.doccount)
+      Enumerator.new do |y|
+        mset.matches.each do |match|
+          y << DocumentDecorator.new(match.document, @root_path)
+        end
+      end
+    end
+
+    def setup
+      unless @findex_path.exist?
+        @findex_path.mkdir
+        File.write(@findex_path + 'config.rb', "Findex.index do |indexer|\nend")
+      end
+
+      require((@findex_path + 'config.rb').realpath)
+      setup_db
+    end
+
+    def setup_db
+      @db = Xapian::WritableDatabase.new((@findex_path + 'db').to_s, DB_FLAGS)
+      @term_generator = Xapian::TermGenerator.new
+      @term_generator.database = @db
+      @term_generator.stemmer = Xapian::Stem.new(Findex.config.stem_language)
+    end
+  end
+end

data/lib/findex/search.rb

@@ -0,0 +1,51 @@
+module Findex
+  class Search
+    def initialize(root_path)
+      @root_path = Pathname.new(root_path)
+      @findex_path = @root_path + '.findex'
+      setup
+    end
+
+    def query(query_terms)
+      enquire = Xapian::Enquire.new(@db)
+      add_prefixes(query_terms)
+      query_terms.map! { |term| term =~ /\s+/ ? %("#{term}") : term }
+      enquire.query = @query_parser.parse_query(query_terms.join(' '))
+      mset = enquire.mset(0, @db.doccount)
+      Enumerator.new do |y|
+        mset.matches.each do |match|
+          document = DocumentDecorator.new(match.document, @root_path)
+          y << document
+        end
+      end
+    end
+
+    private
+
+    def add_prefixes(query_terms)
+      prefix_regex = /(?<prefix>\w+):/
+      query_terms.each do |term|
+        match = term.match(prefix_regex)
+        next unless match
+        prefix = match[:prefix]
+        @query_parser.add_prefix(prefix, "X#{prefix}".upcase)
+      end
+    end
+
+    def setup
+      db_path = @findex_path + 'db'
+      unless @findex_path.exist? && db_path.exist?
+        raise "No index in '#{@root_path}', run index first."
+      end
+      require((@findex_path + 'config.rb').realpath)
+      setup_db(db_path)
+    end
+
+    def setup_db(db_path)
+      @db = Xapian::Database.new(db_path.to_s)
+      @query_parser = Xapian::QueryParser.new
+      @query_parser.stemmer = Xapian::Stem.new(Findex.config.stem_language)
+      @query_parser.database = @db
+    end
+  end
+end

data/lib/findex/term_generator_decorator.rb

@@ -0,0 +1,18 @@
+module Findex
+  class TermGeneratorDecorator < SimpleDelegator
+    def initialize(term_generator)
+      @term_generator = term_generator
+      super
+    end
+
+    def []=(prefix, text)
+      prefix = prefix ? "X#{prefix}".upcase : ''
+      @term_generator.index_text(text.to_s, 1, prefix)
+      @term_generator.increase_termpos
+    end
+
+    def <<(text)
+      self[nil] = text
+    end
+  end
+end

data/lib/findex/version.rb

@@ -0,0 +1,3 @@
+module Findex
+  VERSION = "0.1.0"
+end

data/lib/findex.rb

@@ -0,0 +1,4 @@
+require 'findex/version'
+
+module Findex
+end

data/xapian-findex.gemspec

@@ -0,0 +1,31 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'findex/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'xapian-findex'
+  spec.version       = Findex::VERSION
+  spec.authors       = ['Joakim Reinert']
+  spec.email         = ['mail@jreinert.com']
+  spec.license       = 'MIT'
+
+  spec.summary       = 'A simple file indexer and full text search using Xapian'
+  spec.description   = 'Findex indexes files in a path recursively, updates ' \
+                       'existing ones and deletes removed files from the index'
+  spec.homepage      = 'https://github.com/jreinert/findex'
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f =~ %r{^(test|spec|features)/} }
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'xapian', '~> 1.2'
+  spec.add_dependency 'ruby-filemagic', '~> 0.7'
+
+  spec.add_development_dependency 'bundler', '~> 1.11'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'pry-byebug', '~> 3.3'
+  spec.add_development_dependency 'pry-doc', '~> 0.8'
+end

metadata

@@ -0,0 +1,166 @@
+--- !ruby/object:Gem::Specification
+name: xapian-findex
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Joakim Reinert
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2016-05-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: xapian
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+- !ruby/object:Gem::Dependency
+  name: ruby-filemagic
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+- !ruby/object:Gem::Dependency
+  name: pry-doc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+description: Findex indexes files in a path recursively, updates existing ones and
+  deletes removed files from the index
+email:
+- mail@jreinert.com
+executables:
+- findex
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- exe/findex
+- lib/findex.rb
+- lib/findex/cli.rb
+- lib/findex/config.rb
+- lib/findex/document_decorator.rb
+- lib/findex/dsl.rb
+- lib/findex/file_indexer.rb
+- lib/findex/index.rb
+- lib/findex/indexer.rb
+- lib/findex/search.rb
+- lib/findex/term_generator_decorator.rb
+- lib/findex/version.rb
+- xapian-findex.gemspec
+homepage: https://github.com/jreinert/findex
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: A simple file indexer and full text search using Xapian
+test_files: []
+has_rdoc: