rs_path_tokenizer 0.2.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 6655c43ba457ceabc403de89daa4c33f6371a20b
+  data.tar.gz: 9815bcc5540f0167921f6bbf6409ff5b67e26b27
+SHA512:
+  metadata.gz: e1ce0bf52544242abd6f82d075569ee55929d5c8eb78fc877221dd43c46c3f42ed6729438fb4dd06ab1b265bbd941adf72217786c2b3c89d0bef5f74e06eb403
+  data.tar.gz: 57d05b13324b25963689f1e30ac913f1b6c089f13f816424b5865421bfae21ebd4cba7e333e909398c85377907f21fa52ae2aa413275d95bc933c8a96f10d78f

data/.editorconfig

@@ -0,0 +1,25 @@
+root = true
+
+[*]
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+tab_width = 2
+indent_style = space
+indent_size = 2
+
+[**.bat]
+end_of_line = crlf
+
+[**.min.*]
+indent_style = ignore
+trim_trailing_whitespace = false
+insert_final_newline = ignore
+
+[*.slim]
+insert_final_newline = false
+trim_trailing_whitespace = false
+
+[*.txt]
+insert_final_newline = false
+trim_trailing_whitespace = false

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+.idea

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.ruby-gemset

@@ -0,0 +1 @@
+rs_path_tokenizer

data/.ruby-version

@@ -0,0 +1 @@
+2.2.1

data/.travis.yml

@@ -0,0 +1,3 @@
+language: ruby
+rvm:
+  - 2.2.1

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in rs_path_tokenizer.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Sergey Malykh
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,62 @@
+# RsPathTokenizer
+
+PathTokenizer founds predefined parts (tokens) into specified URL
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'rs_path_tokenizer'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install rs_path_tokenizer
+
+## Usage
+
+```ruby
+  # define tokens data
+  # hash key - token's URL code
+  # hash value - returned property & value (ie for SQL query)
+  tokens_data = {
+    'balashiha' => ['region', 'balashiha'],
+    'balashiha-gorodskoj-okrug' => ['region', 'balashiha-gorodskoj-okrug'],
+    'gorodskoj-okrug-drugoi' => ['region', 'gorodskoj-okrug-drugoi'],
+    # price from
+    'price-*' => ['price', nil],
+    # price from any to any (including from 0 to any)
+    'price-*-*' => ['price', nil],
+    'expensive' => ['sort', 'expensive']
+  }
+
+  tokenizer = RsPathTokenizer::Tokenizer.new(tokens_data)
+
+  # search tokens in specified URL
+  found_tokens = tokenizer.tokenize('balashiha-gorodskoj-okrug-drugoi-price-100-expensive')
+
+  # {"balashiha"=>["region", "balashiha"],
+  #  "gorodskoj-okrug-drugoi"=>["region", "gorodskoj-okrug-drugoi"],
+  #  "price-*"=>["price", "100"],
+  #  "expensive"=>["sort", "expensive"]}
+
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release` to create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/rs_path_tokenizer/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "rs_path_tokenizer"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/rs_path_tokenizer.rb

@@ -0,0 +1,3 @@
+require "rs_path_tokenizer/version"
+require "rs_path_tokenizer/error"
+require "rs_path_tokenizer/tokenizer"

data/lib/rs_path_tokenizer/error.rb

@@ -0,0 +1,3 @@
+module RsPathTokenizer
+  class Error < RuntimeError; end
+end

data/lib/rs_path_tokenizer/tokenizer.rb

@@ -0,0 +1,164 @@
+module RsPathTokenizer
+  class Tokenizer
+    PT_DEBUG = false
+    # PT_DEBUG = true
+
+    def initialize(tokens = nil)
+      return if tokens.nil?
+      @single_tokens = {}
+      tokens.keys.each do |t|
+        parts = url2token(t)
+        st = parts[0]
+        raise Error.new('Token cant starts with asterisk') if st == '*'
+        @single_tokens[st] = [] if @single_tokens[st].nil?
+        @single_tokens[st].push parts
+      end
+      @token_map = tokens
+    end
+
+    def marshal_dump
+      [@single_tokens, @token_map]
+    end
+
+    def marshal_load array
+      @single_tokens, @token_map = array
+    end
+
+    # best result
+    def tokenize(string)
+      tokens = tokenize_all(string).first
+      return if tokens.nil?
+
+      result_to_hash(tokens)
+    end
+
+    protected
+
+      # all results
+      def tokenize_all(string)
+        array = url2token(string)
+        raise Error.new('Too long URL') if array.length > 500
+        possible_tokens = Hash[@single_tokens.keys.select do |st|
+          array.include?(st)
+        end.map do |st|
+          [st, @single_tokens[st]]
+        end]
+        @out_token_map = @token_map
+        sort_results(recursive_parse(array, possible_tokens))
+      end
+
+      def sort_results(results)
+        results.sort do |a, b|
+          result = b.flatten.length <=> a.flatten.length
+          result = b.length <=> a.length if result == 0
+          puts "sorting: #{a.inspect} #{b.inspect} #{result}" if PT_DEBUG
+          result
+        end
+      end
+
+      def result_to_hash(array)
+        Hash[array.map do |e|
+          k = token2url(e)
+          [k, @out_token_map[k]]
+        end]
+      end
+
+      def recursive_parse(array, possible_tokens, limiter = 1)
+        if limiter > 30
+          raise Error.new('Too deep recursion')
+        end
+
+        st = array.first
+        return [] if st.to_s.strip == ''
+
+        tokens = possible_tokens[st]
+        if tokens.nil?
+          puts "#{"  " * limiter}NO tokens for #{st}" if PT_DEBUG
+          return recursive_parse(array.slice(1..-1), possible_tokens)
+        end
+
+        results = []
+        puts "#{"  " * limiter}possible tokens for #{st} are: #{tokens.inspect}" if PT_DEBUG
+
+        tokens.each do |token|
+          found, out, rest = try_match(token, array)
+          puts "#{"  " * limiter}matching #{token.inspect}" if PT_DEBUG
+
+          if found
+            if out != token
+              @out_token_map[token2url(token)] = out
+            end
+
+            puts "#{"  " * limiter}found a token: #{token.inspect}, parsing rest: #{rest.inspect}" if PT_DEBUG
+            more = recursive_parse(rest.dup, possible_tokens, limiter + 1)
+            results = merge_results(results, token, more)
+
+          else
+            puts "#{"  " * limiter}found none on this level, NOT parsing rest: #{rest.inspect}" if PT_DEBUG
+            more = recursive_parse(array.dup.slice(1..-1), possible_tokens, limiter + 1)
+            results = merge_results(results, nil, more)
+          end
+        end
+
+        if PT_DEBUG
+          puts "#{"  " * limiter}results:"
+          results.each do |r|
+            puts "#{"  " * limiter}  #{r.inspect}"
+          end
+        end
+
+        results
+      end
+
+      def merge_results(results, found, other)
+        if other.empty?
+          unless found.nil?
+            results.push [found]
+          end
+        else
+          if found.nil?
+            other.each do |o|
+              results.push o
+            end
+          else
+            other.each do |o|
+              results.push [found] + o
+            end
+          end
+        end
+        results.map(&:uniq).uniq
+      end
+
+      def try_match(token, array)
+        found, out = [], []
+        rest = array.dup
+
+        token.each do |token_part|
+          url_part = rest.shift
+          break if url_part.nil?
+
+          if token_part == '*'
+            out.push url_part
+            found.push token_part
+
+          elsif token_part == url_part
+            found.push token_part
+            out.push token_part
+          end
+        end
+
+        if found == token
+          [true, out, rest]
+        else
+          [false, out, array]
+        end
+      end
+
+      def token2url(token)
+        token.join('-')
+      end
+      def url2token(url)
+        url.split("-")
+      end
+  end
+end

data/lib/rs_path_tokenizer/version.rb

@@ -0,0 +1,3 @@
+module RsPathTokenizer
+  VERSION = "0.2.0"
+end

data/spec/rs_path_tokenizer_spec.rb

@@ -0,0 +1,62 @@
+require 'spec_helper'
+
+describe RsPathTokenizer do
+  before :each do
+    @tokens_data = {
+      'balashiha' => ['r', 'balashiha'],
+      'gorodskoj-okrug-balashiha-1' => ['r', 'balashiha-1'],
+      'gorodskoj-okrug-balashiha-11' => ['r', 'balashiha-11'],
+      'balashiha-gorodskoj-okrug' => ['r', 'balashiha-gorodskoj-okrug'],
+      'gorodskoj-okrug-drugoi' => ['r', 'gorodskoj-okrug-drugoi'],
+      'expensive' => ['sort', 'expensive'],
+    }
+
+    @tokenizer = RsPathTokenizer::Tokenizer.new( @tokens_data )
+  end
+
+  it 'shows best results when tokens overlaps' do
+    results = {"balashiha" => ["r", "balashiha"], "gorodskoj-okrug-drugoi" => ["r", "gorodskoj-okrug-drugoi"]}
+
+    expect( @tokenizer.tokenize( 'balashiha-gorodskoj-okrug-drugoi' ) ).to eq results
+  end
+
+  it 'shows matched result' do
+    results = {"gorodskoj-okrug-balashiha-1" => ["r", "balashiha-1"]}
+    expect( @tokenizer.tokenize( 'gorodskoj-okrug-balashiha-1' ) ).to eq results
+
+    results = {"gorodskoj-okrug-balashiha-11" => ["r", "balashiha-11"]}
+    expect( @tokenizer.tokenize( 'gorodskoj-okrug-balashiha-11' ) ).to eq results
+  end
+
+  it 'returns price range' do
+    @tokens_data['price-*-*'] = ["price", nil]
+    @tokens_data['price-*'] = ["price", nil]
+
+    @tokenizer = RsPathTokenizer::Tokenizer.new( @tokens_data )
+
+    results = {"balashiha"=>["r", "balashiha"], 'price-*-*' => ["price", "0", "100"]}
+    expect( @tokenizer.tokenize( 'balashiha-price-0-100' ) ).to eq results
+
+    results = {"balashiha"=>["r", "balashiha"], 'price-*' => ["price", "100"]}
+    expect( @tokenizer.tokenize( 'balashiha-price-100' ) ).to eq results
+
+    results = {"balashiha"=>["r", "balashiha"], "price-*-*"=>["price", "0", "100"], "expensive"=>["sort", "expensive"]}
+    expect( @tokenizer.tokenize( 'balashiha-price-0-100-expensive' ) ).to eq results
+
+    results = {"balashiha"=>["r", "balashiha"], "price-*"=>["price", "100"], "expensive"=>["sort", "expensive"]}
+    expect( @tokenizer.tokenize( 'balashiha-price-100-expensive' ) ).to eq results
+  end
+
+  it 'nothing found' do
+    expect( @tokenizer.tokenize( 'incorrect-url' ) ).to eq nil
+  end
+
+  it 'too long url' do
+    expect{ @tokenizer.tokenize( 'a-b' * 501 ) }.to raise_error RsPathTokenizer::Error
+  end
+
+  it 'incorrect token' do
+    tokens_data = {'*' => ["all", true]}
+    expect{ RsPathTokenizer::Tokenizer.new( tokens_data ) }.to raise_error RsPathTokenizer::Error
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,2 @@
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
+require 'rs_path_tokenizer'

metadata

@@ -0,0 +1,109 @@
+--- !ruby/object:Gem::Specification
+name: rs_path_tokenizer
+version: !ruby/object:Gem::Version
+  version: 0.2.0
+platform: ruby
+authors:
+- glebtv
+- Sergey Malykh
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-03-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Tokenize path from predefined tokens.
+email:
+- xronos.i.am@gmail.com
+executables:
+- console
+- setup
+extensions: []
+extra_rdoc_files: []
+files:
+- ".editorconfig"
+- ".gitignore"
+- ".rspec"
+- ".ruby-gemset"
+- ".ruby-version"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/rs_path_tokenizer.rb
+- lib/rs_path_tokenizer/error.rb
+- lib/rs_path_tokenizer/tokenizer.rb
+- lib/rs_path_tokenizer/version.rb
+- spec/rs_path_tokenizer_spec.rb
+- spec/spec_helper.rb
+homepage: http://github.com/xronos-i-am/rs_path_tokenizer
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.6
+signing_key: 
+specification_version: 4
+summary: URL path tokenizer.
+test_files:
+- spec/rs_path_tokenizer_spec.rb
+- spec/spec_helper.rb