scandex 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 8219330418a57d92510475b295080d6664feef75
+  data.tar.gz: 33e6ae442689c6839a6e60bd573ca6764d0f98d0
+SHA512:
+  metadata.gz: 132313db43cece7cc6e91dd3f21a12b8cae2d4b62f615febfc64f68efbc532b1310a77675d426be2e8bd032a02a0f485e1f96d7124c664e0b4cd16e61948b200
+  data.tar.gz: abfef9971c9d4223fe600761dfa2179dabf3b36cb0de1ec44eda99be9113a7ff093ce4d6f793be256f5d79672ff68640d32dd94a0670478e70f9dcc5910590cb

data/bin/scandex

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+require_relative '../lib/runner'
+Runner.start(ARGV)

data/lib/runner.rb

@@ -0,0 +1,50 @@
+require 'thor'
+require 'filewatcher'
+require_relative 'scandex'
+
+class Runner < Thor
+    class_option :f, :banner => "Directory where to store index"
+
+    desc "doctor", "Checks system to make sure everything is installed properly"
+    def doctor
+        ScanDex::doctor()
+    end
+
+    desc "list", "List all documents indexed"
+    def list
+        files = ScanDex::documents(options[:f])
+        files.each do |file|
+            puts "#{file[0]} #{file[1]}"
+        end
+    end
+
+    desc "index FILES", "Index scanned FILES"
+    option :force, :type => :boolean
+    def index(*files)
+        # If doctor fails then there is no point in even trying
+        if !ScanDex::doctor()
+            return
+        end
+        files.each do |file|
+            ScanDex::index_and_store(options[:f], file, options[:force])
+       end
+    end
+
+    desc "search TERM", "Search documents containing TERM"
+    def search(term)
+        files = ScanDex::search_documents(options[:f], term)
+        files.each do |file|
+            puts "#{file[0]} #{file[1]}"
+        end
+    end
+
+    desc "watch DIRECTORIES", "Watch DIRECTORIES for new files to index"
+    def watch(*directories)
+        FileWatcher.new(directories).watch do |file, event|
+            if (event == :changed || event == :new)
+                ScanDex::index_and_store(options[:f], file, true)
+            end
+        end
+    end
+end
+

data/lib/scandex.rb

@@ -0,0 +1,135 @@
+require 'rtesseract'
+require 'sqlite3'
+
+module ScanDex
+    def self.tesseract
+        `which tesseract`.strip
+    end
+
+    def self.gs
+        `which gs`.strip
+    end
+
+    def self.convert
+        `which convert`.strip
+    end
+
+    def self.doctor
+        convert = self.convert
+        if convert.empty?
+            puts "ImageMagick is missing"
+            return false
+        end
+        gs = self.gs
+        if gs.empty?
+            puts "GhostScript is missing"
+            return false
+        end
+        tesseract = self.tesseract
+        if tesseract.empty?
+            puts "Tesseract is missing"
+            return false
+        end
+        #puts "All Tools Available"
+        true
+    end
+
+    def self.index_and_store(store_path, file, force = false)
+        content = ScanDex::index(store_path, file, force)
+        if !content.nil?
+            file = File.expand_path(file)
+            ScanDex::store_document(store_path, file, content)
+        end
+    end
+
+    def self.index(store_path, source, force = false)
+        accepted_formats = ['.pdf', '.png', '.jpg', '.jpeg', '.tiff']
+        if (force || !self.has_document(store_path, source)) && accepted_formats.include?(File.extname(source).downcase)
+            puts "Indexing #{source}"
+            tmp = Dir.mktmpdir('scandex_')
+            pages = convert_to_gray_scale(source, tmp)
+            puts "Found #{pages.size} page(s)"
+            if pages.size > 0
+                ocr(pages)
+            else
+                nil
+            end
+        else
+            puts "Ignoring '#{source}'"
+            nil
+        end
+    end
+
+    def self.convert_to_gray_scale(source, destination)
+        cmd = "#{self.convert} -density 300 -depth 8 -type grayscale \"#{source}\" #{destination}/convert-%04d.jpg"
+        #puts "cmd = #{cmd}"
+        puts "Converting '#{File.basename(source)}'"
+        ret = system(cmd)
+        if !ret
+            puts "Failed to convert #{source}"
+            []
+        else
+            Dir["#{destination}/convert-*.jpg"]
+        end
+    end
+
+    def self.ocr(pages, language = "eng")
+        text = ''
+        pages.each do |page|
+            puts "OCR on '#{File.basename(page)}'"
+            text += image_to_string(page, language)
+        end
+        text
+    end
+
+    # TODO orientation and language detection
+    def self.image_to_string(image, language = "eng")
+        img = RTesseract.new(image, :lang => language)
+        img.to_s
+    end
+
+    def self.db(store_path)
+        store_path = '~/' if store_path.nil? || store_path.empty?
+        filename = File.expand_path("#{store_path}/.scandex.db")
+        migrate = !File.exists?(filename)
+        db = SQLite3::Database.new(filename)
+        if migrate
+            puts "Creating DB"
+            db.execute("CREATE TABLE documents (name VARCHAR(255), content TEXT, created TEXT, modified TEXT)")
+        end
+        db
+    end
+
+    def self.has_document(store_path, name)
+        db = self.db(store_path)
+        rows = db.execute("SELECT name FROM documents WHERE name = ?", [name])
+        rows.size > 0
+    end
+
+    def self.documents(store_path)
+        db = self.db(store_path)
+        db.execute("SELECT name, created, modified FROM documents")
+    end
+
+    def self.search_documents(store_path, text)
+        db = self.db(store_path)
+        pattern = "%#{text.downcase}%"
+        db.execute("SELECT name, created, modified FROM documents WHERE LOWER(content) LIKE ? OR LOWER(name) LIKE ?", [pattern, pattern])
+    end
+
+    def self.store_document(store_path, source, content)
+        created = File.mtime(source).utc.iso8601
+        modified = File.ctime(source).utc.iso8601
+
+        db = self.db(store_path)
+        rows = db.execute("SELECT * FROM documents WHERE name = ?", source)
+        if rows.size == 0
+            puts "Insert: #{source} #{created} #{modified}"
+            db.execute("INSERT INTO documents (name, content, created, modified) VALUES (?, ?, ?, ?)", [source, content, created, modified])
+        else
+            puts "Update: #{source} #{created} #{modified}"
+            db.execute("UPDATE documents SET content = ?, modified = ? WHERE name = ?", [content, modified, source])
+        end
+    end
+end
+

data/lib/version.rb

@@ -0,0 +1,3 @@
+module ScanDex
+    VERSION = '0.1.1'
+end

metadata

@@ -0,0 +1,134 @@
+--- !ruby/object:Gem::Specification
+name: scandex
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- Jerome Poichet
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-02-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rtesseract
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.3.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.3.2
+- !ruby/object:Gem::Dependency
+  name: rmagick
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.15.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.15.4
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.3.11
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.3.11
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.19.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.19.1
+- !ruby/object:Gem::Dependency
+  name: filewatcher
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.5.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.5.3
+- !ruby/object:Gem::Dependency
+  name: gems
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.4.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.4.6
+description: A very simple tool to index scanned documents in either PDF or image
+  format
+email: poitch@gmail.com
+executables:
+- scandex
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/scandex
+- lib/runner.rb
+- lib/scandex.rb
+- lib/version.rb
+homepage: http://github.com/poitch/scandex
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.6
+signing_key: 
+specification_version: 4
+summary: Index your scanned paper documents
+test_files: []