static_search 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: f9cf632c8ca6b071da82c5870adf7332e84f2e0f
+  data.tar.gz: 23274c0cc813ecfa19341810452ebdcffd9b562a
+SHA512:
+  metadata.gz: b97ce839718279eed1460052f5a8dedd6e5805717703b7fa6d014b508589e537b4456036003bebdf4e4900ece20107ec167dc7d24f7fa9fc2dcaf4a354f6c703
+  data.tar.gz: 37de39d42734f9a6e86ff4dd6977fb46c17c7de5a705a0b854070929c76c6b27fd63cee49a204e3ed8409c2149390c1ae091a583245485ee1f84a4ce0c9e0632

data/.gitignore

@@ -0,0 +1,16 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log
+spec/pages
+spec/tmp

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format documentation

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in static_search.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2015 DanielRavina
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,69 @@
+# StaticSearch
+
+StaticSearch allow you to index your static content (i.e "pages") in your Rails app easily!. Why would you want to index static pages? You can search the content within it, generate analytics etc.
+It's made of 2 modules - Indexing and Searching.
+
+#### Keep In mind:
+This gem is currently a work in progress and it is not a part of the Rubygem site. I am using it currently for a specific project and I change stuff on the fly. Once I feel it's stable enough, I will release it to the world
+
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'static-search'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install static-search
+
+Run the migration:
+
+	$ rake db:migrate
+
+## Usage
+### Indexing
+First make sure all of your static pages are in the right place:
+
+	- app/
+	  - views/
+	    - pages/
+		    - index.html
+		    - about.html.erb
+		    - faq.html
+		    - privacy_policy.html
+
+Now, you are able to run the built-in rake task:
+
+	$ rake static:index
+
+
+The output should be something like that
+
+	Indexing index
+	Indexing about
+	Indexing faq
+	Indexing privacy_policy
+	-- Completed
+
+If	you made a change in the content and you want to update the index, simply run the rake task again and it will update everything.
+
+At this point, the content (text) of the page, including parsed erb, is saved in the database *without* the html tags.
+
+### Searcing
+
+TODO: Build search module.
+
+## Contributing
+
+1. Fork it ( https://github.com/danielravina/static_search/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/app/models/static_content.rb

@@ -0,0 +1,3 @@
+class StaticContent < ActiveRecord::Base
+  self.table_name = "static_content"
+end

data/db/migrate/20150524221831_create_static_content.rb

@@ -0,0 +1,9 @@
+class CreateStaticContent < ActiveRecord::Migration
+  def change
+    create_table :static_content do |t|
+      t.text :content
+      t.string :title
+      t.timestamps
+    end
+  end
+end

data/lib/engine.rb

@@ -0,0 +1,16 @@
+# The engine file add the gem's migration to the Rails app it is located in
+
+class StaticSearch::Engine < ::Rails::Engine
+  isolate_namespace StaticSearch
+
+  config.autoload_paths += Dir["#{config.root}/lib/**/"]
+
+  initializer :append_migrations do |app|
+    unless app.root.to_s.match root.to_s
+      config.paths["db/migrate"].expanded.each do |expanded_path|
+        app.config.paths["db/migrate"] << expanded_path
+      end
+    end
+  end
+
+end

data/lib/static_search/index_builder.rb

@@ -0,0 +1,59 @@
+class StaticSearch::IndexBuilder
+
+  require File.dirname(File.expand_path '../..', __FILE__) + '/app/models/static_content.rb'
+
+  def build(pages_path, options = {})
+    Dir["#{pages_path}/**/*"].each do |fname|
+      unless File.directory? fname
+        erb      = parse_file fname
+        filename = parse_title fname
+        text     = parse_content(erb, options[:keep_tags] || {})
+        save_content(text, filename)
+      end
+    end
+    puts "-- Completed" if production?
+  end
+
+  def save_content (text, title)
+    puts "Indexing #{title} page" if production?
+    static_content = StaticContent.find_or_initialize_by(title: title)
+    static_content.update content: text
+    return static_content
+  end
+
+
+  def parse_file(fname)
+    file = File.read(fname)
+    ERB.new file, nil, "%"
+  end
+
+  def parse_content (erb, options = {})
+
+    html = Nokogiri.HTML erb.result
+    html.css('script').remove
+    if options[:keep_tags]
+      html.css("body")
+    else
+      html.css("body")
+          .text
+          .gsub("\n"," ")
+          .gsub(/(\s{2,})/," ")
+          .strip
+    end
+  end
+
+  def parse_title(fname)
+    filename = fname.split("/pages/").last
+    if filename.match(/\./)
+      return filename.split(".").first
+    else
+      return filename
+    end
+  end
+
+  private
+
+  def production?
+    defined?(Rails) || defined?(Rake)
+  end
+end

data/lib/static_search/search.rb

@@ -0,0 +1,30 @@
+class StaticSearch::Search
+
+  def self.find_content(q, options = {})
+    return [] unless q.present?
+    result = StaticContent.where("LOWER(content) like ? OR LOWER(title) like ?", "% #{q} %", "%#{q}%");
+    if options[:truncation]
+      return result.map do |page|
+        page = page.as_json
+        page["content"] = truncate_body(page["content"], q, options[:truncation])
+        page
+      end
+    else
+      return result.map(&:as_json)
+    end
+  end
+
+  def self.truncate_body content, q, truncation_size
+    content = \
+      if content.include? q
+        index = content.downcase.index(q.downcase) || 0
+        content[index..(index + truncation_size)]
+      else
+        content[0..truncation_size]
+      end
+    if content && content.size > truncation_size
+      content << "..."
+    end
+    return content
+  end
+end

data/lib/static_search/version.rb

@@ -0,0 +1,3 @@
+module StaticSearch
+  VERSION = "0.0.1"
+end

data/lib/static_search.rb

@@ -0,0 +1,15 @@
+require "static_search/version"
+require 'nokogiri'
+
+if defined?(Rails)
+  require 'engine'
+  Dir["lib/static_search/**/*.rb"].each { |ext| load ext }
+end
+
+if defined?(Rake)
+  Dir["lib/tasks/**/*.rake"].each { |ext| load ext }
+end
+
+module StaticSearch
+end
+

data/lib/tasks/add_dummy_content.rake

@@ -0,0 +1,31 @@
+namespace :static do
+
+  task :seed , [:path] => :environment do |t, args|
+
+    require 'httparty'
+    require 'static_search/index_builder'
+
+    puts "This action will override existing data.\nAre you sure you want to continue? [Y/N]"
+    user_confirm = STDIN.gets
+
+    if !!(user_confirm =~ /Y|y|Yes|yes|YES/)
+      pages_path = "#{Rails.root}/app/views/pages"
+      p "Populating random stuff..."
+
+      unless Dir.exists? pages_path
+        raise "Pages directory doesn't exists. Please create it in #{pages_path}"
+      end
+
+      random_wiki = 'http://en.wikipedia.org/wiki/Special:Random'
+
+      Dir["#{pages_path}/**/*"].each do |fname|
+        unless File.directory? fname
+          response  = HTTParty.get(random_wiki)
+          page = File.open(fname, "w")
+          page.puts response.parsed_response
+          page.close
+        end
+      end
+    end
+  end
+end

data/lib/tasks/build_index.rake

@@ -0,0 +1,15 @@
+namespace :static do
+  task :index , [:path] => :environment do |t, args|
+
+    require 'static_search/index_builder'
+
+    pages_path = "#{Rails.root}/app/views/pages"
+
+    unless Dir.exists? pages_path
+      raise "Pages directory doesn't exists. Please create it in #{pages_path}"
+    end
+
+    index_builder = StaticSearch::IndexBuilder.new
+    index_builder.build(pages_path)
+  end
+end

data/lib/tasks/clear_index.rake

@@ -0,0 +1,14 @@
+namespace :static do
+  task :clear , [:path] => :environment do |t, args|
+
+    require 'static_search/index_builder'
+
+    pages_path = "#{Rails.root}/app/views/pages"
+
+    unless Dir.exists? pages_path
+      raise "Pages directory doesn't exists. Please create it in #{pages_path}"
+    end
+
+    StaticContent.destroy_all
+  end
+end

data/lib/tasks/remove_dummy_content.rake

@@ -0,0 +1,27 @@
+namespace :static do
+
+  task :remove_seed , [:path] => :environment do |t, args|
+
+    require 'static_search/index_builder'
+
+    puts "This action will override existing data.\nAre you sure you want to continue? [Y/N]"
+    user_confirm = STDIN.gets
+
+    if !!(user_confirm =~ /Y|y|Yes|yes|YES/)
+      pages_path = "#{Rails.root}/app/views/pages"
+      p "Removing page content"
+
+      unless Dir.exists? pages_path
+        raise "Pages directory doesn't exists. Please create it in #{pages_path}"
+      end
+
+      Dir["#{pages_path}/**/*"].each do |fname|
+        unless File.directory? fname
+          page = File.open(fname, "w")
+          page.puts ""
+          page.close
+        end
+      end
+    end
+  end
+end

data/spec/index_spec.rb

@@ -0,0 +1,74 @@
+require 'spec_helper'
+require 'httparty'
+require 'static_search/index_builder'
+
+require File.dirname(File.expand_path '../', __FILE__) + '/app/models/static_content.rb'
+
+describe StaticSearch::IndexBuilder do
+  describe "build" do
+    let(:index_builder)     { StaticSearch::IndexBuilder.new }
+    let(:example_direcotry) { 'spec/pages' }
+    let(:example_file)      { "#{example_direcotry}/example.html" }
+    let(:exampl_url)        { 'http://en.wikipedia.org/wiki/Special:Random' }
+
+    before do
+      unless Dir.exists? example_direcotry
+        Dir.mkdir(example_direcotry)
+      end
+      if File.exists? (example_file)
+        File.delete(example_file)
+      end
+        response  = HTTParty.get(exampl_url)
+        temp_file = File.new(example_file, "w")
+        temp_file.puts response.parsed_response
+        temp_file.close
+    end
+
+    it "parses the file" do
+      parsed_erb = index_builder.parse_file(example_file)
+      expect(parsed_erb).to be_kind_of ERB
+    end
+
+    it "converts HTML to text" do
+      parsed_erb = index_builder.parse_file(example_file)
+      text       = index_builder.parse_content parsed_erb
+      expect(text).to be_kind_of String
+      expect(text[0]).to_not be_empty
+    end
+
+    it "Saves raw HTML" do
+      parsed_erb = index_builder.parse_file example_file
+      text       = index_builder.parse_content parsed_erb, keep_tags: true
+      title      = index_builder.parse_title example_file
+      index_builder.save_content text, title
+      static_content = StaticContent.first
+      expect(static_content.content).to include "<body"
+    end
+
+    it "saves the content" do
+      parsed_erb = index_builder.parse_file(example_file)
+      text       = index_builder.parse_content parsed_erb
+      title     = index_builder.parse_title example_file
+
+      expect(index_builder.save_content(text, title)).to be_kind_of StaticContent
+      static_content = StaticContent.first
+      expect(StaticContent.count).to eq 1
+      expect(static_content.title).to eq "example"
+      expect(static_content.content).to_not match(/<div|<script|<noscript|<a/)
+    end
+
+    it "includes the sub-directory in the file name" do
+      sub_dir  =  "sub"
+      filename = "subtest"
+      new_dir  = "#{example_direcotry}/#{sub_dir}/"
+      unless Dir.exists? new_dir
+        Dir.mkdir(new_dir)
+      end
+      temp_file = File.new(new_dir << filename + ".html", "w")
+      p
+      title     = index_builder.parse_title temp_file.path
+      expect(title).to eq sub_dir + "/" + filename
+    end
+
+  end
+end

data/spec/schema.rb

@@ -0,0 +1,11 @@
+ActiveRecord::Schema.define do
+  self.verbose = false
+
+  create_table "static_content", force: :cascade do |t|
+    t.text     "content",    limit: 65535
+    t.string   "title"
+    t.datetime "created_at"
+    t.datetime "updated_at"
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,13 @@
+require 'bundler/setup'
+Bundler.setup
+
+require 'static_search'
+require 'active_record'
+
+RSpec.configure do |config|
+  # some (optional) config here
+end
+
+
+ActiveRecord::Base.establish_connection adapter: "sqlite3", database: ":memory:"
+load File.dirname(__FILE__) + '/schema.rb'

data/static_search.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'static_search/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "static_search"
+  spec.version       = StaticSearch::VERSION
+  spec.authors       = ["DanielRavina"]
+  spec.email         = ["danielravina@gmail.com"]
+  spec.summary       = %q{Index your Rails app's static content and search it easily}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency             'nokogiri', '~> 1.6.6.2'
+  spec.add_dependency             'httparty', '~> 0.13.3'
+  spec.add_development_dependency 'activerecord', '~> 4.2.1'
+  spec.add_development_dependency 'sqlite3'
+  spec.add_development_dependency 'database_cleaner',  '~> 1.3.0'
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+end

metadata

@@ -0,0 +1,180 @@
+--- !ruby/object:Gem::Specification
+name: static_search
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- DanielRavina
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-06-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.6.6.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.6.6.2
+- !ruby/object:Gem::Dependency
+  name: httparty
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.13.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.13.3
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.2.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.2.1
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: database_cleaner
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.3.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.3.0
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: 
+email:
+- danielravina@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- app/models/static_content.rb
+- db/migrate/20150524221831_create_static_content.rb
+- lib/engine.rb
+- lib/static_search.rb
+- lib/static_search/index_builder.rb
+- lib/static_search/search.rb
+- lib/static_search/version.rb
+- lib/tasks/add_dummy_content.rake
+- lib/tasks/build_index.rake
+- lib/tasks/clear_index.rake
+- lib/tasks/remove_dummy_content.rake
+- spec/index_spec.rb
+- spec/schema.rb
+- spec/spec_helper.rb
+- static_search.gemspec
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.7
+signing_key: 
+specification_version: 4
+summary: Index your Rails app's static content and search it easily
+test_files:
+- spec/index_spec.rb
+- spec/schema.rb
+- spec/spec_helper.rb