pdf_search 0.0.0 → 0.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5506277019033c54c84b0aeec3deae307b304af0392f17bd7e5f1147feb07ad8
-  data.tar.gz: d0488eefb43bdd4cbba1e9bfc0656eefd3e4ff41762b70d4f1e8dc0558fd73c3
+  metadata.gz: 7eaa4d55f2ab3006db01072da969f5d9ed910fbc475079ad0319c0f63cf24383
+  data.tar.gz: 824527a902950a7ae324155173665ce4837aa094e60a833df21a897254c265b6
 SHA512:
-  metadata.gz: 8a20e3fffff896a967b395bae72483d99f733bf1bc1c949c3ac0f4426bd066550741c3b9e664ce71b633fb0b4e2f5c34f958123cb758deb181aaabbd5e1b8bd0
-  data.tar.gz: a2555c899aba3123f5f99b985590d75927515044111d8a7556fae92848b6a39dc335f6aa29aa1d6a3e0dada710e2e8e07e0599277240f783faff489c3b52c699
+  metadata.gz: 8a4de679cfef7215600e55b93df197a49bdb70a70bbbcaf923c40eaeb72f732149a2e02b01650ea67d1fafcc12b17d2fbbf259ffdfc3ab5090eb9df3cb4a06de
+  data.tar.gz: 979b8c13be47593bb847514096f9d67a85958bf98f68d6eb8c5fc51779d1fe7faf8a9212b6db4fa20941ca7be5d2f45c8517342a6277fdeda435f6fef81d05fe

data/bin/pdf_search

@@ -1,18 +1,14 @@
 #!/usr/bin/env ruby 
-require 'pdfsearch'
+require 'pdf_search'
 require 'webrick'
 
-`sudo service elasticsearch start`
 
+PdfSearch.create_elasticsearch_index
+PdfSearch.start_webserver
+PdfSearch::PdfIndex.start_daemon(ARGV[0])
 
-gemDir = [File.dirname(__FILE__), '..']
-indexPdfsScriptPath = File.expand_path(File.join(gemDir + ['lib', 'indexPdfs.rb']))
-htmlDir = File.expand_path(File.join(gemDir + ['html']))
 
 
-WEBrick::HTTPServer.new(:Port => 80, :DocumentRoot => htmlDir).start
 
 
-# PdfSearch::Indexer.startDeamon
-
 

data/config/elasticsearch.yml

@@ -0,0 +1,2 @@
+cluster.routing.allocation.disk.threshold_enabled: false
+

data/html/index.html

@@ -0,0 +1,20 @@
+<input id='search'></input>
+
+<button id='doSearch'>Search</button>
+
+<ul id='results'>
+</ul>
+
+
+<script>
+  document.getElementById('doSearch').addEventListener('click', function(e) {
+    let Http = new XMLHttpRequest();
+    let url= encodeURI("http://localhost:80/search?query="+document.getElementById('search').value);
+    Http.open("GET", url);
+    Http.send();
+
+    Http.onreadystatechange=(e)=> {
+      document.getElementById('results').innerHTML = Http.responseText;
+    }
+  });
+</script>

data/lib/pdf_dir.rb

@@ -12,7 +12,7 @@ module PdfSearch
             Dir.glob(File.join(File.expand_path(@dir), '*.pdf'))
   	end
 
-	def each_pdf
+	def pdf_documents
 	  Enumerator.new do |e|
 	  	  pdf_file_paths.each do |pdf_file_path|
 	          	e << PDF::Reader.new(pdf_file_path)
@@ -20,9 +20,9 @@ module PdfSearch
 	  end
 	end
 
-	def each_page
+	def pages
 	  Enumerator.new do |e|
-		each_pdf do |pdf_reader|
+		pdf_documents.each do |pdf_reader|
 			pdf_reader.pages.each do |page|
 				e << page
 			end

data/lib/pdf_index.rb

@@ -1,5 +1,44 @@
-require 'pdfiterator'
-
-PdfIterator.new.traverse_current_directory_pdf_texts do |pdf_text|
-	`echo "#{pdf_text}" >> log/indexPdfs.log`
+require 'daemons'
+require 'elasticsearch'
+require 'pdf_dir'
+require 'digest'
+class PdfSearch::PdfIndex
+	attr_accessor :daemon
+	def initialize(pdf_dir)
+		@pdf_dir = pdf_dir
+		@els_client = ::PdfSearch::ElasticSearchClient
+	end
+	def self.start_daemon(dir)
+	  pdf_index = self.new(::PdfSearch::PdfDir.new(dir))
+    if ENV['DEBUG_PDF_INDEXING']
+      pdf_index.index_loop
+    else
+	    pdf_index.daemon = Daemons.call(multiple: true, &pdf_index.method(:index_loop))
+    end
+	  pdf_index
+	end
+  def index_loop
+	  loop do
+		  self.reindex
+	  end
+  end
+	def pid
+     daemon.pid.pid
+	end
+	def reindex
+		@pdf_dir.pages.each.with_index do |page, index|
+      begin
+        sleep 0.5
+			  @els_client.index(
+			  	index: 'pdf_pages',
+			  	type: 'document',
+			  	id: Digest::SHA256.digest(page.text),
+			  	body: {
+			  		text: page.text
+			  	}
+			  )
+      rescue
+      end
+		end
+	end
 end

data/lib/pdf_search.rb

@@ -1,5 +1,59 @@
 require 'pdf_index'
+require 'daemons'
 require 'pdf_dir'
+require 'elasticsearch'
+require 'web_server'
 
 module PdfSearch
+
+		ElasticSearchClient = Elasticsearch::Client.new log: true, host: ENV['ELASTICSEARCH_URL']
+
+  GemDir = [File.dirname(__FILE__), '..']
+
+  def self.relative_to_gem_path(path_array)
+    File.expand_path(File.join(GemDir + path_array))
+  end
+
+  def self.start_webserver
+    web_server = WebServer.new
+    web_server.start
+    return web_server.daemon.pid.pid
+  end
+
+	def self.start_elasticsearch
+		`sudo service elasticsearch start`
+	end
+
+	def self.create_elasticsearch_index
+		`curl -XPUT -H "Content-Type: application/json" http://localhost:9200/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}'`
+
+		ElasticSearchClient.indices.create(
+			index: 'pdf_pages',
+			body: {
+				mappings: {
+					document: {
+						properties: {
+							text: {
+								type: 'text'
+							}
+						}
+					}
+				}
+			}
+		)
+
+		return true
+
+	rescue Elasticsearch::Transport::Transport::Errors::BadRequest => e
+		if /"type":"(?:resource|index)_already_exists_exception"/ =~ e.message
+			return false
+		else
+			raise e
+		end
+	end
+
+  def self.wrap_elastic_request
+		`curl -XPUT -H "Content-Type: application/json" http://localhost:9200/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}'`
+    yield
+  end
 end

data/lib/web_server.rb

@@ -0,0 +1,33 @@
+require 'webrick'
+
+class PdfSearch::WebServer
+  attr_accessor :daemon
+
+  def start
+    start_server = lambda do
+      server = WEBrick::HTTPServer.new(:Port => 80, :DocumentRoot => ::PdfSearch.relative_to_gem_path(['html']))
+
+      server.mount_proc '/search' do |request, response|
+        query = request.query["query"]
+        elastic_response = Net::HTTP.get(URI.parse("#{ENV['ELASTICSEARCH_URL']}/_all/_search?q=text:#{query}"))
+        response.body = response_html(elastic_response)
+      end
+
+      server.start
+    end
+
+    if ENV['DEBUG_PDF_SEARCH'] == '1'
+      start_server.call
+    else
+	    self.daemon = Daemons.call(multiple: true, &start_server)
+    end
+  end
+
+  def response_html(elasticsearch_response)
+    results = JSON.parse(elasticsearch_response)['hits']['hits']
+
+    results.map do |result|
+      "<li>#{result["_source"]["text"]}</li>"
+    end.join("\n")
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pdf_search
 version: !ruby/object:Gem::Version
-  version: 0.0.0
+  version: 0.0.1
 platform: ruby
 authors:
 - Manuel Arno Korfmann
@@ -18,9 +18,13 @@ extensions: []
 extra_rdoc_files: []
 files:
 - bin/pdf_search
+- config/elasticsearch.yml
+- html/index.html
 - lib/pdf_dir.rb
 - lib/pdf_index.rb
 - lib/pdf_search.rb
+- lib/web_server.rb
+- log/pdf_index.log
 homepage: https://github.com/banalBI/pdfsearch
 licenses:
 - MIT