kvg_character_recognition 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +9 -0
- data/.rspec +2 -0
- data/.travis.yml +4 -0
- data/Gemfile +8 -0
- data/LICENSE.txt +21 -0
- data/README.md +96 -0
- data/Rakefile +6 -0
- data/bin/console +14 -0
- data/bin/setup +7 -0
- data/kvg_character_recognition.gemspec +40 -0
- data/lib/kvg_character_recognition.rb +55 -0
- data/lib/kvg_character_recognition/database.rb +103 -0
- data/lib/kvg_character_recognition/feature_extractor.rb +168 -0
- data/lib/kvg_character_recognition/preprocessor.rb +214 -0
- data/lib/kvg_character_recognition/recognizer.rb +63 -0
- data/lib/kvg_character_recognition/utils.rb +310 -0
- data/lib/kvg_character_recognition/version.rb +3 -0
- metadata +152 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: a14886233c4152851248456913d18768d9c8472a
|
4
|
+
data.tar.gz: f29d6c47b40ce1eb68a4db14c3b03961c275197c
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 339db4a0b7e01108b9d105f688dfb4bba6ea81420bb4255b70f1ff8ef51c5ae04cab3424464e453632f2ea17727f2e1629a95148d06379720f5e1cae448fa4a8
|
7
|
+
data.tar.gz: 2f57226b5dac0f9311bf0c5dec2750354aa6c40c5348e2f1df858b52a470aed242aa380be1d4aadfd2d9a1753e5fbad0a1a621a232d28d5e81b3dde4b2c929d9
|
data/.gitignore
ADDED
data/.rspec
ADDED
data/.travis.yml
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2016 Jiayi Zheng
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
13
|
+
all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,96 @@
|
|
1
|
+
# KvgCharacterRecognition
|
2
|
+
KvgCharacterRecognition module contains a CJK-character recognition engine which uses pattern/template matching techniques to achieve recognitionof stroke-order and stroke-number free handwritten character patterns in the format [stroke1, stroke2 ...].
|
3
|
+
A stroke is an array of points in the format [[x1, y1], [x2, y2], ...].
|
4
|
+
For templates, we use svg data from the [KanjiVG project](http://kanjivg.tagaini.net/)
|
5
|
+
|
6
|
+
The engine takes 3 steps to perform the recognition of an input pattern.
|
7
|
+
1. Preprocessing
|
8
|
+
The preprocessing step consists of smoothing, normalizing, interpolating and downsampling of the data points.
|
9
|
+
2. Feature Extraction
|
10
|
+
Smoothed heatmap, significant points and directional feature densities are used as features.
|
11
|
+
A heatmap divides the input pattern in small grids and stores the number of data points in each grid.
|
12
|
+
Significant points are defined as start and end point of a stroke, points on curve or edge.
|
13
|
+
Directional feature densities are introduced in the paper "On-line Recognition of Freely Handwritten Japanese Character Using Directional Feature Density"
|
14
|
+
3. Matching
|
15
|
+
We use the significant points to perform a coarse recognition of the input pattern, that filters out template patterns with great distance to the input pattern. Next, a mixed distance score of directional feature density and smoothed heatmap is calculated.
|
16
|
+
## Installation
|
17
|
+
|
18
|
+
Add this line to your application's Gemfile:
|
19
|
+
|
20
|
+
```ruby
|
21
|
+
gem 'kvg_character_recognition'
|
22
|
+
```
|
23
|
+
|
24
|
+
And then execute:
|
25
|
+
|
26
|
+
$ bundle
|
27
|
+
|
28
|
+
Or install it yourself as:
|
29
|
+
|
30
|
+
$ gem install kvg_character_recognition
|
31
|
+
|
32
|
+
## Usage
|
33
|
+
|
34
|
+
1. Create a database(e.g. using sqlite3 data.db)
|
35
|
+
|
36
|
+
2. Setup the characters table in the database and populate it with kanjivg templates from the [xml release](https://github.com/KanjiVG/kanjivg/releases)
|
37
|
+
```ruby
|
38
|
+
require 'kvg_character_recognition'
|
39
|
+
|
40
|
+
KvgCharacterRecognition::Database.setup
|
41
|
+
|
42
|
+
KvgCharacterRecognition::Database.populate_from_xml "kanjivg-20150615-2.xml"
|
43
|
+
```
|
44
|
+
|
45
|
+
3. Recognition
|
46
|
+
|
47
|
+
Use an input field of size 300x300 for the best recognition accuracy. The input pattern in the example is the character 二, drawn on a 300x300 html canvas using mouse.
|
48
|
+
```ruby
|
49
|
+
strokes = [[[99.0, 108.0], [100.0, 108.0], [101.0, 108.0], [101.0, 108.0], [103.0, 108.0], [105.0, 107.0], [107.0, 107.0], [108.0, 107.0], [111.0, 106.0], [111.0, 106.0], [112.0, 106.0], [113.0, 106.0], [114.0, 106.0], [115.0, 105.0], [116.0, 105.0], [118.0, 105.0], [120.0, 105.0], [121.0, 104.0], [122.0, 104.0], [122.0, 104.0], [123.0, 104.0], [124.0, 103.0], [125.0, 103.0], [126.0, 103.0], [127.0, 103.0], [129.0, 102.0], [130.0, 102.0], [132.0, 102.0], [132.0, 101.0], [133.0, 101.0], [135.0, 101.0], [136.0, 101.0], [137.0, 101.0], [138.0, 101.0], [140.0, 101.0], [141.0, 100.0], [142.0, 100.0], [143.0, 100.0], [144.0, 100.0], [145.0, 99.0], [148.0, 99.0], [150.0, 99.0], [151.0, 98.0], [152.0, 98.0], [153.0, 98.0], [154.0, 98.0], [156.0, 97.0], [157.0, 97.0], [158.0, 97.0], [159.0, 97.0], [161.0, 97.0], [162.0, 96.0], [162.0, 96.0], [164.0, 96.0], [165.0, 96.0], [166.0, 96.0], [167.0, 96.0], [169.0, 95.0], [170.0, 95.0], [171.0, 95.0], [172.0, 95.0], [173.0, 95.0], [174.0, 95.0]], [[53.0, 190.0], [54.0, 190.0], [56.0, 190.0], [57.0, 190.0], [59.0, 190.0], [61.0, 190.0], [63.0, 189.0], [66.0, 189.0], [67.0, 189.0], [68.0, 189.0], [69.0, 189.0], [71.0, 189.0], [72.0, 188.0], [72.0, 188.0], [74.0, 188.0], [76.0, 187.0], [78.0, 187.0], [80.0, 187.0], [81.0, 187.0], [82.0, 186.0], [84.0, 186.0], [87.0, 186.0], [89.0, 185.0], [91.0, 185.0], [93.0, 185.0], [95.0, 184.0], [98.0, 184.0], [100.0, 183.0], [102.0, 183.0], [104.0, 183.0], [106.0, 183.0], [110.0, 182.0], [111.0, 182.0], [112.0, 182.0], [115.0, 182.0], [118.0, 182.0], [120.0, 182.0], [122.0, 182.0], [125.0, 182.0], [128.0, 181.0], [130.0, 181.0], [133.0, 180.0], [136.0, 180.0], [141.0, 180.0], [143.0, 179.0], [146.0, 179.0], [150.0, 179.0], [152.0, 178.0], [155.0, 178.0], [158.0, 178.0], [159.0, 178.0], [162.0, 177.0], [164.0, 177.0], [167.0, 177.0], [170.0, 177.0], [173.0, 176.0], [176.0, 176.0], [179.0, 176.0], [182.0, 175.0], [187.0, 175.0], [189.0, 174.0], [192.0, 174.0], [194.0, 174.0], [196.0, 173.0], [199.0, 173.0], [202.0, 173.0], [204.0, 172.0], [206.0, 172.0], [209.0, 172.0], [211.0, 172.0], [212.0, 172.0], [215.0, 172.0], [217.0, 172.0], [219.0, 171.0], [221.0, 171.0], [221.0, 172.0]]]
|
50
|
+
|
51
|
+
scores = KvgCharacterRecognition::Recognizer.scores strokes
|
52
|
+
|
53
|
+
irb(main):004:0> scores.take 10
|
54
|
+
=> [[1.524079282599697, 60, "二"], [2.8346163809971143, 1373, "工"], [3.0987422100694757, 7, "上"], [3.127346308294038, 365, "冫"], [3.439293212191952, 6, "三"], [3.4890481845638304, 3770, "立"], [3.541524904953307, 2721, "江"], [3.641178875851016, 569, "厂"], [3.6447144433336294, 72, "亠"], [3.7498483818966353, 2706, "氵"]]
|
55
|
+
```
|
56
|
+
|
57
|
+
## Configuration
|
58
|
+
You can try out different parameters for adapting the extracted features to your input settings i.e. other sample rate, size
|
59
|
+
Don't forget to redo the whole database step after changing the configuration.
|
60
|
+
```ruby
|
61
|
+
#this is the default configuration
|
62
|
+
config = {
|
63
|
+
size: 109, #fixed canvas size of kanjivg data
|
64
|
+
downsample_interval: 4,
|
65
|
+
interpolate_distance: 0.8,
|
66
|
+
direction_grid: 15,
|
67
|
+
smoothed_heatmap_grid: 20,
|
68
|
+
significant_points_heatmap_grid: 3
|
69
|
+
}
|
70
|
+
|
71
|
+
#from hash
|
72
|
+
Kvgcharacterrecognition.configure(config)
|
73
|
+
#from yaml file
|
74
|
+
Kvgcharacterrecognition.configure_with(path_to_yml)
|
75
|
+
|
76
|
+
#configure database with yml
|
77
|
+
#TODO why is postgres slower than sqlite?
|
78
|
+
Kvgcharacterrecognition.configure_database(path_to_yml)
|
79
|
+
```
|
80
|
+
|
81
|
+
|
82
|
+
## Development
|
83
|
+
|
84
|
+
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
85
|
+
|
86
|
+
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
87
|
+
|
88
|
+
## Contributing
|
89
|
+
|
90
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/kvg_character_recognition.
|
91
|
+
|
92
|
+
|
93
|
+
## License
|
94
|
+
|
95
|
+
The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
|
96
|
+
|
data/Rakefile
ADDED
data/bin/console
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require "bundler/setup"
|
4
|
+
require "kvg_character_recognition"
|
5
|
+
|
6
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
7
|
+
# with your gem easier. You can also use a different console, if you like.
|
8
|
+
|
9
|
+
# (If you use this, don't forget to add pry to your Gemfile!)
|
10
|
+
# require "pry"
|
11
|
+
# Pry.start
|
12
|
+
|
13
|
+
require "irb"
|
14
|
+
IRB.start
|
data/bin/setup
ADDED
@@ -0,0 +1,40 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'kvg_character_recognition/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "kvg_character_recognition"
|
8
|
+
spec.version = KvgCharacterRecognition::VERSION
|
9
|
+
spec.authors = ["Jiayi Zheng"]
|
10
|
+
spec.email = ["thebluber@gmail.com"]
|
11
|
+
|
12
|
+
spec.summary = "CJK-character recognition using template matching techniques and template data from KanjiVG project"
|
13
|
+
spec.description = %q{This gem contains a CJK-character recognition engine using pattern/template matching techniques.
|
14
|
+
It can recognize stroke-order and stroke-number free handwritten character patterns in the format [stroke1, stroke2 ...].
|
15
|
+
A stroke is an array of points in the format [[x1, y1], [x2, y2], ...].
|
16
|
+
KanjiVG data(characters in svg format) from https://github.com/KanjiVG/kanjivg/releases are used as templates.
|
17
|
+
}
|
18
|
+
spec.homepage = "https://github.com/thebluber/kvg_character_recognition"
|
19
|
+
spec.license = "MIT"
|
20
|
+
|
21
|
+
# Prevent pushing this gem to RubyGems.org by setting 'allowed_push_host', or
|
22
|
+
# delete this section to allow pushing this gem to any host.
|
23
|
+
if spec.respond_to?(:metadata)
|
24
|
+
spec.metadata['allowed_push_host'] = 'https://rubygems.org'
|
25
|
+
else
|
26
|
+
raise "RubyGems 2.0 or newer is required to protect against public gem pushes."
|
27
|
+
end
|
28
|
+
|
29
|
+
spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
30
|
+
spec.bindir = "exe"
|
31
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
32
|
+
spec.require_paths = ["lib"]
|
33
|
+
|
34
|
+
spec.add_dependency "nokogiri"
|
35
|
+
spec.add_dependency "sequel"
|
36
|
+
spec.add_dependency "sqlite3"
|
37
|
+
spec.add_dependency "bundler", "~> 1.10"
|
38
|
+
spec.add_development_dependency "rake", "~> 10.0"
|
39
|
+
spec.add_development_dependency "rspec"
|
40
|
+
end
|
@@ -0,0 +1,55 @@
|
|
1
|
+
require 'bundler'
|
2
|
+
Bundler.require
|
3
|
+
require 'yaml'
|
4
|
+
#require all files in ./lib/
|
5
|
+
Dir[File.join(File.dirname(__FILE__), '/kvg_character_recognition/*.rb')].each {|file| require file }
|
6
|
+
|
7
|
+
module KvgCharacterRecognition
|
8
|
+
|
9
|
+
@db = Sequel.connect('sqlite://characters.db')
|
10
|
+
CONFIG = {
|
11
|
+
size: 109, #fixed canvas size of kanjivg data
|
12
|
+
downsample_interval: 4,
|
13
|
+
interpolate_distance: 0.8,
|
14
|
+
direction_grid: 15,
|
15
|
+
smoothed_heatmap_grid: 20,
|
16
|
+
significant_points_heatmap_grid: 3
|
17
|
+
}
|
18
|
+
VALID_KEYS = CONFIG.keys
|
19
|
+
|
20
|
+
#Configure through hash
|
21
|
+
def self.configure(opts = {})
|
22
|
+
opts.each {|k,v| CONFIG[k.to_sym] = v if VALID_KEYS.include? k.to_sym}
|
23
|
+
end
|
24
|
+
|
25
|
+
#Configure with yaml
|
26
|
+
def self.configure_with(yml)
|
27
|
+
begin
|
28
|
+
config = YAML::load(IO.read(yml))
|
29
|
+
rescue Errno::ENOENT
|
30
|
+
log(:warning, "YAML configuration file couldn't be found. Using defaults."); return
|
31
|
+
rescue Psych::SyntaxError
|
32
|
+
log(:warning, "YAML configuration file contains invalid syntax. Using defaults."); return
|
33
|
+
end
|
34
|
+
|
35
|
+
configure(config)
|
36
|
+
end
|
37
|
+
|
38
|
+
#Configure database
|
39
|
+
def self.configure_database(yml)
|
40
|
+
begin
|
41
|
+
db_config = YAML::load(IO.read(yml))
|
42
|
+
rescue Errno::ENOENT
|
43
|
+
log(:warning, "YAML configuration file couldn't be found. Using defaults."); return
|
44
|
+
rescue Psych::SyntaxError
|
45
|
+
log(:warning, "YAML configuration file contains invalid syntax. Using defaults."); return
|
46
|
+
end
|
47
|
+
@db = Sequel.connect(yml)
|
48
|
+
end
|
49
|
+
|
50
|
+
#getter
|
51
|
+
def self.db
|
52
|
+
@db
|
53
|
+
end
|
54
|
+
|
55
|
+
end
|
@@ -0,0 +1,103 @@
|
|
1
|
+
require 'matrix'
|
2
|
+
require 'nokogiri'
|
3
|
+
|
4
|
+
module KvgCharacterRecognition
|
5
|
+
#This class contains methods for database interactions
|
6
|
+
class Database
|
7
|
+
|
8
|
+
#This method creates a database table for storing the extracted features of the templates
|
9
|
+
#Arrays of points will be serialized and stored as string
|
10
|
+
#Following fields are created:
|
11
|
+
# - primary_key :id
|
12
|
+
# - String :value
|
13
|
+
# - Integer :codepoint
|
14
|
+
# - String :serialized_strokes i.e. [stroke, x, y]
|
15
|
+
# - String :direction_e1
|
16
|
+
# - String :direction_e2
|
17
|
+
# - String :direction_e3
|
18
|
+
# - String :direction_e4
|
19
|
+
# - String :heatmap_smoothed
|
20
|
+
# - String :heatmap_significant_points
|
21
|
+
def self.setup
|
22
|
+
KvgCharacterRecognition.db.create_table :characters do
|
23
|
+
primary_key :id
|
24
|
+
String :value
|
25
|
+
Integer :codepoint
|
26
|
+
Integer :number_of_strokes
|
27
|
+
String :serialized_strokes
|
28
|
+
String :direction_e1
|
29
|
+
String :direction_e2
|
30
|
+
String :direction_e3
|
31
|
+
String :direction_e4
|
32
|
+
String :heatmap_smoothed
|
33
|
+
String :heatmap_significant_points
|
34
|
+
end
|
35
|
+
|
36
|
+
end
|
37
|
+
|
38
|
+
#Drop created table
|
39
|
+
def self.drop
|
40
|
+
KvgCharacterRecognition.db.drop_table(:characters) if KvgCharacterRecognition.db.table_exists?(:characters)
|
41
|
+
end
|
42
|
+
|
43
|
+
#This method populates the database table with parsed template patterns from the kanjivg file in xml format
|
44
|
+
#Params:
|
45
|
+
#+xml+:: download the latest xml release from https://github.com/KanjiVG/kanjivg/releases
|
46
|
+
def self.populate_from_xml xml
|
47
|
+
file = File.open(xml) { |f| Nokogiri::XML(f) }
|
48
|
+
|
49
|
+
file.xpath("//kanji").each do |kanji|
|
50
|
+
#id has format: "kvg:kanji_CODEPOINT"
|
51
|
+
codepoint = kanji.attributes["id"].value.split("_")[1]
|
52
|
+
next unless codepoint.hex >= "04e00".hex && codepoint.hex <= "09faf".hex
|
53
|
+
puts codepoint
|
54
|
+
value = [codepoint.hex].pack("U")
|
55
|
+
|
56
|
+
#Preprocessing
|
57
|
+
#--------------
|
58
|
+
#parse strokes
|
59
|
+
strokes = kanji.xpath("g//path").map{|p| p.attributes["d"].value }.map{ |stroke| KvgParser::Stroke.new(stroke).to_a }
|
60
|
+
#strokes in the format [[[x1, y1], [x2, y2] ...], [[x2, y2], [x3, y3] ...], ...]
|
61
|
+
strokes = Preprocessor.preprocess(strokes, CONFIG[:interpolate_distance], CONFIG[:downsample_interval], false)
|
62
|
+
|
63
|
+
#serialize strokes
|
64
|
+
serialized = strokes.map.with_index do |stroke, i|
|
65
|
+
stroke.map{ |p| [i, p[0], p[1]] }
|
66
|
+
end
|
67
|
+
|
68
|
+
points = strokes.flatten(1)
|
69
|
+
|
70
|
+
#Feature Extraction
|
71
|
+
#--------------
|
72
|
+
#20x20 heatmap smoothed
|
73
|
+
heatmap_smoothed = FeatureExtractor.smooth_heatmap(FeatureExtractor.heatmap(points, CONFIG[:smoothed_heatmap_grid], CONFIG[:size]))
|
74
|
+
|
75
|
+
#directional feature densities
|
76
|
+
#transposed from Mx4 to 4xM
|
77
|
+
direction = Matrix.columns(FeatureExtractor.spatial_weight_filter(FeatureExtractor.directional_feature_densities(strokes, CONFIG[:direction_grid])).to_a).to_a
|
78
|
+
|
79
|
+
#significant points
|
80
|
+
significant_points = Preprocessor.significant_points(strokes)
|
81
|
+
|
82
|
+
#3x3 heatmap of significant points for coarse recognition
|
83
|
+
heatmap_significant_points = FeatureExtractor.heatmap(significant_points, CONFIG[:significant_points_heatmap_grid], CONFIG[:size])
|
84
|
+
|
85
|
+
|
86
|
+
#Store to database
|
87
|
+
#--------------
|
88
|
+
KvgCharacterRecognition.db[:characters].insert value: value,
|
89
|
+
codepoint: codepoint.hex,
|
90
|
+
number_of_strokes: strokes.count,
|
91
|
+
serialized_strokes: serialized.join(","),
|
92
|
+
direction_e1: direction[0].join(","),
|
93
|
+
direction_e2: direction[1].join(","),
|
94
|
+
direction_e3: direction[2].join(","),
|
95
|
+
direction_e4: direction[3].join(","),
|
96
|
+
heatmap_smoothed: heatmap_smoothed.to_a.join(","),
|
97
|
+
heatmap_significant_points: heatmap_significant_points.to_a.join(",")
|
98
|
+
end
|
99
|
+
|
100
|
+
end
|
101
|
+
|
102
|
+
end
|
103
|
+
end
|
@@ -0,0 +1,168 @@
|
|
1
|
+
require 'matrix'
|
2
|
+
module KvgCharacterRecognition
|
3
|
+
#This class contains a collection of methods for extracting useful features
|
4
|
+
class FeatureExtractor
|
5
|
+
|
6
|
+
#This methods generates a heatmap for the given character pattern
|
7
|
+
#A heatmap divides the input character pattern(image of the character) into nxn grids
|
8
|
+
#We count the points in each grid and store the number in a map
|
9
|
+
#The map array can be used as feature
|
10
|
+
#Params:
|
11
|
+
#+points+:: flattened strokes i.e. [[x1, y1], [x2, y2]...] because the seperation of points in strokes is irrelevant in this case
|
12
|
+
#+grid+:: number of grids
|
13
|
+
def self.heatmap points, grid, size
|
14
|
+
|
15
|
+
grid_size = size / grid.to_f
|
16
|
+
|
17
|
+
map = Map.new grid, grid, 0
|
18
|
+
|
19
|
+
#fill the heatmap
|
20
|
+
points.each do |point|
|
21
|
+
if point[0] < size && point[1] < size
|
22
|
+
x_i = (point[0] / grid_size).floor if point[0] < size
|
23
|
+
y_i = (point[1] / grid_size).floor if point[1] < size
|
24
|
+
|
25
|
+
map[y_i, x_i] = map[y_i, x_i] + 1
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
map
|
30
|
+
end
|
31
|
+
|
32
|
+
#This method calculates the directional feature densities and stores them in a map
|
33
|
+
#The process and algorithm is described in the paper "On-line Recognition of Freely Handwritten Japanese Characters Using Directional Feature Densities" by Akinori Kawamura and co.
|
34
|
+
#Params:
|
35
|
+
#+strokes+:: [[[x1, y1], [x2, y2] ...], [[x1, y1], ...]]]
|
36
|
+
#+grid+:: number of grids in which the input character pattern should be seperated. Default is 15 as in the paper
|
37
|
+
def self.directional_feature_densities strokes, grid
|
38
|
+
#initialize a map for storing the weights in each directional space
|
39
|
+
map = Map.new grid, grid, [0, 0, 0, 0]
|
40
|
+
|
41
|
+
#step width
|
42
|
+
step = CONFIG[:size] / grid.to_f
|
43
|
+
|
44
|
+
strokes.each do |stroke|
|
45
|
+
current_p = stroke[0]
|
46
|
+
stroke.each do |point|
|
47
|
+
next if point == current_p
|
48
|
+
#map current point coordinate to map index
|
49
|
+
#i_x = xth column
|
50
|
+
#i_y = yth row
|
51
|
+
i_x = (current_p[0] / step).floor
|
52
|
+
i_y = (current_p[1] / step).floor
|
53
|
+
|
54
|
+
#direction vector V_ij = P_ij+1 - P_ij
|
55
|
+
v = [point[0] - current_p[0], point[1] - current_p[1]]
|
56
|
+
#store the sum of decomposed direction vectors in the corresponding grid
|
57
|
+
decomposed = decompose(v)
|
58
|
+
map[i_y, i_x] = [map[i_y, i_x][0] + decomposed[0],
|
59
|
+
map[i_y, i_x][1] + decomposed[1],
|
60
|
+
map[i_y, i_x][2] + decomposed[2],
|
61
|
+
map[i_y, i_x][3] + decomposed[3]]
|
62
|
+
end
|
63
|
+
end
|
64
|
+
map
|
65
|
+
end
|
66
|
+
|
67
|
+
#This method is a helper method for calculating directional feature density
|
68
|
+
#which decomposes the direction vector into predefined direction spaces
|
69
|
+
#- e1: [1, 0]
|
70
|
+
#- e2: [1/sqrt(2), 1/sqrt(2)]
|
71
|
+
#- e3: [0, 1]
|
72
|
+
#- e4: [-1/sqrt(2), 1/sqrt(2)]
|
73
|
+
#Params:
|
74
|
+
#+v+:: direction vector of 2 adjacent points V_ij = P_ij+1 - P_ij
|
75
|
+
def self.decompose v
|
76
|
+
e1 = [1, 0]
|
77
|
+
e2 = [1/Math.sqrt(2), 1/Math.sqrt(2)]
|
78
|
+
e3 = [0, 1]
|
79
|
+
e4 = [-1/Math.sqrt(2), 1/Math.sqrt(2)]
|
80
|
+
#angle between vector v and e1
|
81
|
+
#det = x1*y2 - x2*y1
|
82
|
+
#dot = x1*x2 + y1*y2
|
83
|
+
#atan2(det, dot) in range 0..180 and 0..-180
|
84
|
+
angle = (Math.atan2(v[1], v[0]) / (Math::PI / 180)).floor
|
85
|
+
if (0..44).cover?(angle) || (-180..-136).cover?(angle)
|
86
|
+
decomposed = [(Matrix.columns([e1, e2]).inverse * Vector.elements(v)).to_a, 0, 0].flatten
|
87
|
+
elsif (45..89).cover?(angle) || (-135..-91).cover?(angle)
|
88
|
+
decomposed = [0, (Matrix.columns([e2, e3]).inverse * Vector.elements(v)).to_a, 0].flatten
|
89
|
+
elsif (90..134).cover?(angle) || (-90..-44).cover?(angle)
|
90
|
+
decomposed = [0, 0, (Matrix.columns([e3, e4]).inverse * Vector.elements(v)).to_a].flatten
|
91
|
+
elsif (135..179).cover?(angle) || (-45..-1).cover?(angle)
|
92
|
+
tmp = (Matrix.columns([e4, e1]).inverse * Vector.elements(v)).to_a
|
93
|
+
decomposed = [tmp[0], 0, 0, tmp[1]]
|
94
|
+
end
|
95
|
+
|
96
|
+
decomposed
|
97
|
+
end
|
98
|
+
|
99
|
+
#This methods reduces the dimension of directonal feature densities stored in the map
|
100
|
+
#It takes every 2nd grid of directional_feature_densities map and stores the average of the weighted sum of adjacent grids around it
|
101
|
+
#weights = [1/16, 2/16, 1/16];
|
102
|
+
# [2/16, 4/16, 2/16];
|
103
|
+
# [1/16, 2/16, 1/16]
|
104
|
+
#Params:
|
105
|
+
#+map+:: directional feature densities map i.e. [[e1, e2, e3, e4], [e1, e2, e3, e4] ...] for each grid of input character pattern
|
106
|
+
def self.spatial_weight_filter map
|
107
|
+
#default grid should be 15
|
108
|
+
grid = map.size
|
109
|
+
new_grid = (grid / 2.0).ceil
|
110
|
+
new_map = Map.new(new_grid, new_grid, [0, 0, 0, 0])
|
111
|
+
|
112
|
+
(0..(grid - 1)).each_slice(2) do |i, i2|
|
113
|
+
(0..(grid - 1)).each_slice(2) do |j, j2|
|
114
|
+
#weights = [1/16, 2/16, 1/16];
|
115
|
+
# [2/16, 4/16, 2/16];
|
116
|
+
# [1/16, 2/16, 1/16]
|
117
|
+
w11 = (0..(grid-1)).cover?(i+1) && (0..(grid-1)).cover?(j-1)? map[i+1,j-1].map{|e| e * 1 / 16.0} : [0, 0, 0, 0]
|
118
|
+
w12 = (0..(grid-1)).cover?(i+1) && (0..(grid-1)).cover?(j)? map[i+1,j].map{|e| e * 2 / 16.0} : [0, 0, 0, 0]
|
119
|
+
w13 = (0..(grid-1)).cover?(i+1) && (0..(grid-1)).cover?(j+1)? map[i+1,j+1].map{|e| e * 1 / 16.0} : [0, 0, 0, 0]
|
120
|
+
w21 = (0..(grid-1)).cover?(i) && (0..(grid-1)).cover?(j-1)? map[i,j-1].map{|e| e * 2 / 16.0} : [0, 0, 0, 0]
|
121
|
+
w22 = (0..(grid-1)).cover?(i) && (0..(grid-1)).cover?(j)? map[i,j].map{|e| e * 4 / 16.0} : [0, 0, 0, 0]
|
122
|
+
w23 = (0..(grid-1)).cover?(i) && (0..(grid-1)).cover?(j+1)? map[i,j+1].map{|e| e * 2 / 16.0} : [0, 0, 0, 0]
|
123
|
+
w31 = (0..(grid-1)).cover?(i-1) && (0..(grid-1)).cover?(j-1)? map[i-1,j-1].map{|e| e * 1 / 16.0} : [0, 0, 0, 0]
|
124
|
+
w32 = (0..(grid-1)).cover?(i-1) && (0..(grid-1)).cover?(j)? map[i-1,j].map{|e| e * 2 / 16.0} : [0, 0, 0, 0]
|
125
|
+
w33 = (0..(grid-1)).cover?(i-1) && (0..(grid-1)).cover?(j+1)? map[i-1,j+1].map{|e| e * 1 / 16.0} : [0, 0, 0, 0]
|
126
|
+
|
127
|
+
new_map[i/2,j/2] = [w11[0] + w12[0] + w13[0] + w21[0] + w22[0] + w23[0] + w31[0] + w32[0] + w33[0],
|
128
|
+
w11[1] + w12[1] + w13[1] + w21[1] + w22[1] + w23[1] + w31[1] + w32[1] + w33[1],
|
129
|
+
w11[2] + w12[2] + w13[2] + w21[2] + w22[2] + w23[2] + w31[2] + w32[2] + w33[2],
|
130
|
+
w11[3] + w12[3] + w13[3] + w21[3] + w22[3] + w23[3] + w31[3] + w32[3] + w33[3]]
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
134
|
+
new_map
|
135
|
+
end
|
136
|
+
|
137
|
+
#This method smooths a heatmap using spatial_weight_filter technique
|
138
|
+
#but instead of taking every 2nd grid, it processes every grid and stores the average of the weighted sum of adjacent grids
|
139
|
+
#Params:
|
140
|
+
#+map+:: a heatmap
|
141
|
+
def self.smooth_heatmap map
|
142
|
+
grid = map.size
|
143
|
+
#map is a heatmap
|
144
|
+
new_map = Map.new(grid, grid, 0)
|
145
|
+
|
146
|
+
(0..(grid - 1)).each do |i|
|
147
|
+
(0..(grid - 1)).each do |j|
|
148
|
+
#weights = [1/16, 2/16, 1/16];
|
149
|
+
# [2/16, 4/16, 2/16];
|
150
|
+
# [1/16, 2/16, 1/16]
|
151
|
+
w11 = (0..(grid-1)).cover?(i+1) && (0..(grid-1)).cover?(j-1)? map[i+1,j-1] * 1 / 16.0 : 0
|
152
|
+
w12 = (0..(grid-1)).cover?(i+1) && (0..(grid-1)).cover?(j)? map[i+1,j] * 2 / 16.0 : 0
|
153
|
+
w13 = (0..(grid-1)).cover?(i+1) && (0..(grid-1)).cover?(j+1)? map[i+1,j+1] * 1 / 16.0 : 0
|
154
|
+
w21 = (0..(grid-1)).cover?(i) && (0..(grid-1)).cover?(j-1)? map[i,j-1] * 2 / 16.0 : 0
|
155
|
+
w22 = (0..(grid-1)).cover?(i) && (0..(grid-1)).cover?(j)? map[i,j] * 4 / 16.0 : 0
|
156
|
+
w23 = (0..(grid-1)).cover?(i) && (0..(grid-1)).cover?(j+1)? map[i,j+1] * 2 / 16.0 : 0
|
157
|
+
w31 = (0..(grid-1)).cover?(i-1) && (0..(grid-1)).cover?(j-1)? map[i-1,j-1] * 1 / 16.0 : 0
|
158
|
+
w32 = (0..(grid-1)).cover?(i-1) && (0..(grid-1)).cover?(j)? map[i-1,j] * 2 / 16.0 : 0
|
159
|
+
w33 = (0..(grid-1)).cover?(i-1) && (0..(grid-1)).cover?(j+1)? map[i-1,j+1] * 1 / 16.0 : 0
|
160
|
+
|
161
|
+
new_map[i,j] = w11 + w12 + w13 + w21 + w22 + w23 + w31 + w32 + w33
|
162
|
+
end
|
163
|
+
end
|
164
|
+
|
165
|
+
new_map
|
166
|
+
end
|
167
|
+
end
|
168
|
+
end
|
@@ -0,0 +1,214 @@
|
|
1
|
+
module KvgCharacterRecognition
|
2
|
+
#This class has a collection of methods for the preprocessing step of character recognition
|
3
|
+
class Preprocessor
|
4
|
+
|
5
|
+
#A simple smooth method using the following formula
|
6
|
+
#p'(i) = (w(-M)*p(i-M) + ... + w(0)*p(i) + ... + w(M)*p(i+M)) / S
|
7
|
+
#where the smoothed point is a weighted average of its adjacent points.
|
8
|
+
#Only the user input should be smoothed, it is not necessary for kvg data.
|
9
|
+
#Params:
|
10
|
+
#+stroke+:: array of points i.e [[x1, y1], [x2, y2] ...]
|
11
|
+
def self.smooth stroke
|
12
|
+
weights = [1,3,1]
|
13
|
+
offset = weights.length / 2
|
14
|
+
wsum = weights.inject{ |sum, x| sum + x}
|
15
|
+
|
16
|
+
return stroke if stroke.length < weights.length
|
17
|
+
|
18
|
+
copy = stroke.dup
|
19
|
+
|
20
|
+
(offset..(stroke.length - offset - 1)).each do |i|
|
21
|
+
accum = [0, 0]
|
22
|
+
|
23
|
+
weights.each_with_index do |w, j|
|
24
|
+
accum[0] += w * copy[i + j - offset][0]
|
25
|
+
accum[1] += w * copy[i + j - offset][1]
|
26
|
+
end
|
27
|
+
|
28
|
+
stroke[i] = accum.map{ |acc| (acc / wsum.to_f).round(2) }
|
29
|
+
end
|
30
|
+
stroke
|
31
|
+
end
|
32
|
+
|
33
|
+
#This method executes different preprocessing steps
|
34
|
+
#0.Normalize strokes to the size 109x109 and center the coordinates using bi moment normalization method
|
35
|
+
#1.Smooth strokes if set to true
|
36
|
+
#2.Interpolate points by given distance, in order to equalize the sample rate of input and template
|
37
|
+
#3.Downsample by given interval
|
38
|
+
def self.preprocess strokes, interpolate_distance=0.8, downsample_interval=4, smooth=true
|
39
|
+
means, diffs = means_and_diffs(strokes)
|
40
|
+
#normalize strokes
|
41
|
+
strokes = bi_moment_normalize(means, diffs, strokes)
|
42
|
+
|
43
|
+
strokes.map do |stroke|
|
44
|
+
stroke = smooth(stroke) if smooth
|
45
|
+
interpolated = interpolate(stroke, interpolate_distance)
|
46
|
+
downsample(interpolated, downsample_interval)
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
#This method calculates means and diffs of x and y coordinates in the strokes
|
51
|
+
#The return values are used in the normalization step
|
52
|
+
#means, diffs = means_and_diffs strokes
|
53
|
+
#Return values:
|
54
|
+
#+means+:: [mean_of_x, mean_of_y]
|
55
|
+
#+diffs+:: differences of the x and y coordinates to their means i.e. [[d_x1, d_x2 ...], [d_y1, d_y2 ...]]
|
56
|
+
def self.means_and_diffs strokes
|
57
|
+
points = strokes.flatten(1)
|
58
|
+
sums = points.inject([0, 0]){ |acc, point| acc = [acc[0] + point[0], acc[1] + point[1]] }
|
59
|
+
#means = [x_c, y_c]
|
60
|
+
means = sums.map{ |sum| (sum / points.length.to_f).round(2) }
|
61
|
+
|
62
|
+
diffs = points.inject([[], []]){ |acc, point| acc = [acc[0] << point[0] - means[0], acc[1] << point[1] - means[1]] }
|
63
|
+
[means, diffs]
|
64
|
+
end
|
65
|
+
|
66
|
+
#This methods normalizes the strokes using bi moment
|
67
|
+
#Params:
|
68
|
+
#+strokes+:: [[[x1, y1], [x2, y2], ...], [[x1, y1], ...]]
|
69
|
+
#+means+:: [x_c, y_c]
|
70
|
+
#+diffs+:: [d_x, d_y]; d_x = [d1, d2, ...]
|
71
|
+
def self.bi_moment_normalize means, diffs, strokes
|
72
|
+
|
73
|
+
#calculating delta values
|
74
|
+
delta = Proc.new do |diff, operator|
|
75
|
+
#d_x or d_y
|
76
|
+
#operator: >= or <
|
77
|
+
accum = 0
|
78
|
+
counter = 0
|
79
|
+
|
80
|
+
diff.each do |d|
|
81
|
+
if d.send operator, 0
|
82
|
+
accum += d ** 2
|
83
|
+
counter += 1
|
84
|
+
end
|
85
|
+
end
|
86
|
+
accum / counter
|
87
|
+
end
|
88
|
+
|
89
|
+
new_strokes = []
|
90
|
+
strokes.each do |stroke|
|
91
|
+
new_stroke = []
|
92
|
+
stroke.each do |point|
|
93
|
+
if point[0] - means[0] >= 0
|
94
|
+
new_x = ( CONFIG[:size] * (point[0] - means[0]) / (4 * Math.sqrt(delta.call(diffs[0], :>=))).round(2) ) + CONFIG[:size]/2
|
95
|
+
else
|
96
|
+
new_x = ( CONFIG[:size] * (point[0] - means[0]) / (4 * Math.sqrt(delta.call(diffs[0], :<))).round(2) ) + CONFIG[:size]/2
|
97
|
+
end
|
98
|
+
if point[1] - means[1] >= 0
|
99
|
+
new_y = ( CONFIG[:size] * (point[1] - means[1]) / (4 * Math.sqrt(delta.call(diffs[1], :>=))).round(2) ) + CONFIG[:size]/2
|
100
|
+
else
|
101
|
+
new_y = ( CONFIG[:size] * (point[1] - means[1]) / (4 * Math.sqrt(delta.call(diffs[1], :<))).round(2) ) + CONFIG[:size]/2
|
102
|
+
end
|
103
|
+
|
104
|
+
if new_x >= 0 && new_x <= CONFIG[:size] && new_y >= 0 && new_y <= CONFIG[:size]
|
105
|
+
new_stroke << [new_x.round(3), new_y.round(3)]
|
106
|
+
end
|
107
|
+
end
|
108
|
+
new_strokes << new_stroke unless new_stroke.empty?
|
109
|
+
end
|
110
|
+
new_strokes
|
111
|
+
end
|
112
|
+
|
113
|
+
#This method returns the significant points of a given character
|
114
|
+
#Significant points are:
|
115
|
+
#- Start and end point of a stroke
|
116
|
+
#- Point on curve or edge
|
117
|
+
#To determine whether a point is on curve or edge, we take the 2 adjacent points and calculate the angle between the 2 vectors
|
118
|
+
#If the angle is smaller than 150 degree, then the point should be on curve or edge
|
119
|
+
def self.significant_points strokes
|
120
|
+
points = []
|
121
|
+
strokes.each_with_index do |stroke, i|
|
122
|
+
points << stroke[0]
|
123
|
+
|
124
|
+
#collect edge points
|
125
|
+
#determine whether a point is an edge point by the internal angle between vector P_i-1 - P_i and P_i+1 - P_i
|
126
|
+
pre = stroke[0]
|
127
|
+
(1..(stroke.length - 1)).each do |j|
|
128
|
+
current = stroke[j]
|
129
|
+
nex = stroke[j+1]
|
130
|
+
if nex
|
131
|
+
v1 = [pre[0] - current[0], pre[1] - current[1]]
|
132
|
+
v2 = [nex[0] - current[0], nex[1] - current[1]]
|
133
|
+
det = v1[0] * v2[1] - (v2[0] * v1[1])
|
134
|
+
dot = v1[0] * v2[0] + (v2[1] * v1[1])
|
135
|
+
angle = Math.atan2(det, dot) / (Math::PI / 180)
|
136
|
+
|
137
|
+
if angle.abs < 150
|
138
|
+
#current point is on a curve or an edge
|
139
|
+
points << current
|
140
|
+
end
|
141
|
+
end
|
142
|
+
pre = current
|
143
|
+
end
|
144
|
+
|
145
|
+
points << stroke[stroke.length - 1]
|
146
|
+
end
|
147
|
+
|
148
|
+
points
|
149
|
+
end
|
150
|
+
|
151
|
+
#This methods calculates the euclidean distance between 2 points
|
152
|
+
#Params:
|
153
|
+
#- p1, p2: [x, y]
|
154
|
+
def self.euclidean_distance(p1, p2)
|
155
|
+
sum_of_squares = 0
|
156
|
+
p1.each_with_index do |p1_coord,index|
|
157
|
+
sum_of_squares += (p1_coord - p2[index]) ** 2
|
158
|
+
end
|
159
|
+
Math.sqrt( sum_of_squares )
|
160
|
+
end
|
161
|
+
|
162
|
+
#This method interpolates points into a stroke with given distance
|
163
|
+
#The algorithm is taken from the paper preprocessing techniques for online character recognition
|
164
|
+
def self.interpolate stroke, d=0.5
|
165
|
+
current = stroke.first
|
166
|
+
new_stroke = [current]
|
167
|
+
|
168
|
+
index = 1
|
169
|
+
last_index = 0
|
170
|
+
while index < stroke.length do
|
171
|
+
point = stroke[index]
|
172
|
+
|
173
|
+
#only consider point with greater than d distance to current point
|
174
|
+
if euclidean_distance(current, point) < d
|
175
|
+
index += 1
|
176
|
+
else
|
177
|
+
|
178
|
+
#calculate new point coordinate
|
179
|
+
new_point = []
|
180
|
+
if point[0] == current[0] # x2 == x1
|
181
|
+
if point[1] > current[1] # y2 > y1
|
182
|
+
new_point = [current[0], current[1] + d]
|
183
|
+
else # y2 < y1
|
184
|
+
new_point = [current[0], current[1] - d]
|
185
|
+
end
|
186
|
+
else # x2 != x1
|
187
|
+
slope = (point[1] - current[1]) / (point[0] - current[0]).to_f
|
188
|
+
if point[0] > current[0] # x2 > x1
|
189
|
+
new_point[0] = current[0] + Math.sqrt(d**2 / (slope**2 + 1))
|
190
|
+
else # x2 < x1
|
191
|
+
new_point[0] = current[0] - Math.sqrt(d**2 / (slope**2 + 1))
|
192
|
+
end
|
193
|
+
new_point[1] = slope * new_point[0] + point[1] - (slope * point[0])
|
194
|
+
end
|
195
|
+
|
196
|
+
new_point = new_point.map{ |num| num.round(2) }
|
197
|
+
new_stroke << new_point
|
198
|
+
|
199
|
+
current = new_point
|
200
|
+
last_index += ((index - last_index) / 2).floor
|
201
|
+
index = last_index + 1
|
202
|
+
end
|
203
|
+
end
|
204
|
+
|
205
|
+
new_stroke
|
206
|
+
end
|
207
|
+
|
208
|
+
#This methods downsamples a stroke in given interval
|
209
|
+
#The number of points in the stroke will be reduced
|
210
|
+
def self.downsample stroke, interval=3
|
211
|
+
stroke.each_slice(interval).map(&:first)
|
212
|
+
end
|
213
|
+
end
|
214
|
+
end
|
@@ -0,0 +1,63 @@
|
|
1
|
+
require 'matrix'
|
2
|
+
module KvgCharacterRecognition
|
3
|
+
#This class contains methods calculating similarity scores between input pattern and template patterns
|
4
|
+
class Recognizer
|
5
|
+
|
6
|
+
#This method selects all templates from the database which should be further examined
|
7
|
+
#It filtered out those characters with a too great difference in number of strokes to the input character
|
8
|
+
def self.select_templates strokes
|
9
|
+
min = strokes.count <= 5 ? strokes.count : strokes.count - 5
|
10
|
+
max = strokes.count + 10
|
11
|
+
KvgCharacterRecognition.db[:characters].where(:number_of_strokes => (min..max))
|
12
|
+
end
|
13
|
+
|
14
|
+
#This method uses heatmap of significant points to coarse recognize the input pattern
|
15
|
+
#Params:
|
16
|
+
#+strokes+:: strokes should be preprocessed
|
17
|
+
def self.coarse_recognize strokes
|
18
|
+
heatmap = FeatureExtractor.heatmap(Preprocessor.significant_points(strokes), CONFIG[:significant_points_heatmap_grid], CONFIG[:size]).to_a
|
19
|
+
|
20
|
+
templates = select_templates strokes
|
21
|
+
templates.map do |candidate|
|
22
|
+
candidate_heatmap = candidate[:heatmap_significant_points].split(",").map(&:to_f)
|
23
|
+
|
24
|
+
score = Preprocessor.euclidean_distance(heatmap, candidate_heatmap)
|
25
|
+
[score.round(3), candidate]
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
#This method calculates similarity scores which is an average of the somehow weighted sum of the euclidean distance of
|
30
|
+
#1. 20x20 smoothed heatmap
|
31
|
+
#2. euclidean distance of directional feature densities in average
|
32
|
+
#Params:
|
33
|
+
#+strokes+:: strokes are not preprocessed
|
34
|
+
def self.scores strokes
|
35
|
+
#preprocess strokes
|
36
|
+
#with smoothing
|
37
|
+
strokes = Preprocessor.preprocess(strokes, CONFIG[:interpolate_distance], CONFIG[:downsample_interval], true)
|
38
|
+
|
39
|
+
#feature extraction
|
40
|
+
directions = Matrix.columns(FeatureExtractor.spatial_weight_filter(FeatureExtractor.directional_feature_densities(strokes, CONFIG[:direction_grid])).to_a).to_a
|
41
|
+
heatmap_smoothed = FeatureExtractor.smooth_heatmap(FeatureExtractor.heatmap(strokes.flatten(1), CONFIG[:smoothed_heatmap_grid], CONFIG[:size])).to_a
|
42
|
+
|
43
|
+
#dump half of the templates after coarse recognition
|
44
|
+
#collection is in the form [[score, c1], [score, c2] ...]
|
45
|
+
collection = coarse_recognize(strokes).sort{ |a, b| a[0] <=> b[0] }
|
46
|
+
|
47
|
+
scores = collection.take(collection.count / 2).map do |cand|
|
48
|
+
direction_score = (Preprocessor.euclidean_distance(directions[0], cand[1][:direction_e1].split(",").map(&:to_f)) +
|
49
|
+
Preprocessor.euclidean_distance(directions[1], cand[1][:direction_e2].split(",").map(&:to_f)) +
|
50
|
+
Preprocessor.euclidean_distance(directions[2], cand[1][:direction_e3].split(",").map(&:to_f)) +
|
51
|
+
Preprocessor.euclidean_distance(directions[3], cand[1][:direction_e4].split(",").map(&:to_f)) ) / 4
|
52
|
+
|
53
|
+
heatmap_score = Preprocessor.euclidean_distance(heatmap_smoothed, cand[1][:heatmap_smoothed].split(",").map(&:to_f))
|
54
|
+
|
55
|
+
mix = (direction_score / 100) + heatmap_score
|
56
|
+
[mix/2, cand[1][:id], cand[1][:value]]
|
57
|
+
end
|
58
|
+
|
59
|
+
scores.sort{ |a, b| a[0] <=> b[0] }
|
60
|
+
end
|
61
|
+
|
62
|
+
end
|
63
|
+
end
|
@@ -0,0 +1,310 @@
|
|
1
|
+
module KvgCharacterRecognition
|
2
|
+
|
3
|
+
#This class can be used for storing heatmap count and directional feature densities
|
4
|
+
#basically it is a nxm matrix with an initial value in each cell
|
5
|
+
class Map
|
6
|
+
#Make a new map with
|
7
|
+
#Params:
|
8
|
+
#+n+:: row length
|
9
|
+
#+m+:: column length
|
10
|
+
#+initial_value+:: for heatmap initial_value = 0 and for directional feature densities initial_value = [0, 0, 0, 0] <= [weight in e1, weight in e2, ...]
|
11
|
+
def initialize n, m, initial_value
|
12
|
+
@array = Array.new(n * m, initial_value)
|
13
|
+
@n = n
|
14
|
+
@m = m
|
15
|
+
end
|
16
|
+
|
17
|
+
#Access value in the cell of i-th row and j-th column
|
18
|
+
#e.g. map[i,j]
|
19
|
+
def [](i, j)
|
20
|
+
@array[j*@n + i]
|
21
|
+
end
|
22
|
+
|
23
|
+
#Store value in the cell of i-th row and j-th column
|
24
|
+
#e.g. map[i,j] = value
|
25
|
+
def []=(i, j, value)
|
26
|
+
@array[j*@n + i] = value
|
27
|
+
end
|
28
|
+
|
29
|
+
def to_a
|
30
|
+
@array
|
31
|
+
end
|
32
|
+
|
33
|
+
#Normaly n is the same as m
|
34
|
+
def size
|
35
|
+
@n
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
|
40
|
+
#This module contains classes which can be used to parse a svg command
|
41
|
+
#The code is copied from https://github.com/rogerbraun/KVG-Tools
|
42
|
+
#Methods for generating sexp or xml outputs are removed
|
43
|
+
module KvgParser
|
44
|
+
#A Point
|
45
|
+
class Point
|
46
|
+
attr_accessor :x, :y, :color
|
47
|
+
|
48
|
+
def initialize(x,y, color = :black)
|
49
|
+
@x,@y, @color = x, y, color
|
50
|
+
end
|
51
|
+
|
52
|
+
#Basic point arithmetics
|
53
|
+
def +(p2)
|
54
|
+
return Point.new(@x + p2.x, @y + p2.y)
|
55
|
+
end
|
56
|
+
|
57
|
+
def -(p2)
|
58
|
+
return Point.new(@x - p2.x, @y - p2.y)
|
59
|
+
end
|
60
|
+
|
61
|
+
def dist(p2)
|
62
|
+
return Math.sqrt((p2.x - @x)**2 + (p2.y - @y)**2)
|
63
|
+
end
|
64
|
+
|
65
|
+
def *(number)
|
66
|
+
return Point.new(@x * number, @y * number)
|
67
|
+
end
|
68
|
+
|
69
|
+
#to array
|
70
|
+
def to_a
|
71
|
+
[@x.round(2), @y.round(2)]
|
72
|
+
end
|
73
|
+
|
74
|
+
end
|
75
|
+
|
76
|
+
# SVG_M represents the moveto command.
|
77
|
+
# SVG Syntax is:
|
78
|
+
# m x y
|
79
|
+
# It sets the current cursor to the point (x,y).
|
80
|
+
# As always, capitalization denotes absolute values.
|
81
|
+
# Takes a Point as argument.
|
82
|
+
# If given 2 Points, the second argument is treated as the current cursor.
|
83
|
+
class SVG_M
|
84
|
+
|
85
|
+
def initialize(p1, p2 = Point.new(0,0))
|
86
|
+
@p = p1 + p2
|
87
|
+
end
|
88
|
+
|
89
|
+
def to_points
|
90
|
+
return []
|
91
|
+
end
|
92
|
+
|
93
|
+
def current_cursor
|
94
|
+
return @p
|
95
|
+
end
|
96
|
+
|
97
|
+
end
|
98
|
+
|
99
|
+
# SVG_C represents the cubic Bézier curveto command.
|
100
|
+
# Syntax is:
|
101
|
+
# c x1 y1 x2 y2 x y
|
102
|
+
# It sets the current cursor to the point (x,y).
|
103
|
+
# As always, capitalization denotes absolute values.
|
104
|
+
# Takes 4 Points as argument, the fourth being the current cursor
|
105
|
+
# If constructed using SVG_C.relative, the current cursor is added to every
|
106
|
+
# point.
|
107
|
+
class SVG_C
|
108
|
+
|
109
|
+
def initialize(c1,c2,p,current_cursor)
|
110
|
+
@c1,@c2,@p,@current_cursor = c1,c2,p,current_cursor
|
111
|
+
@@c_color = :green
|
112
|
+
end
|
113
|
+
|
114
|
+
def SVG_C.relative(c1,c2,p,current_cursor)
|
115
|
+
SVG_C.new(c1 + current_cursor, c2 + current_cursor, p + current_cursor, current_cursor)
|
116
|
+
end
|
117
|
+
|
118
|
+
def second_point
|
119
|
+
@c2
|
120
|
+
end
|
121
|
+
|
122
|
+
# This implements the algorithm found here:
|
123
|
+
# http://www.cubic.org/docs/bezier.htm
|
124
|
+
# Takes 2 Points and a factor between 0 and 1
|
125
|
+
def linear_interpolation(a,b,factor)
|
126
|
+
|
127
|
+
xr = a.x + ((b.x - a.x) * factor)
|
128
|
+
yr = a.y + ((b.y - a.y) * factor)
|
129
|
+
|
130
|
+
return Point.new(xr,yr);
|
131
|
+
|
132
|
+
end
|
133
|
+
|
134
|
+
def switch_color
|
135
|
+
if @@c_color == :green
|
136
|
+
@@c_color = :red
|
137
|
+
elsif @@c_color == :red
|
138
|
+
@@c_color = :purple
|
139
|
+
else
|
140
|
+
@@c_color = :green
|
141
|
+
end
|
142
|
+
end
|
143
|
+
|
144
|
+
def make_curvepoint(factor)
|
145
|
+
ab = linear_interpolation(@current_cursor,@c1,factor)
|
146
|
+
bc = linear_interpolation(@c1,@c2,factor)
|
147
|
+
cd = linear_interpolation(@c2,@p,factor)
|
148
|
+
|
149
|
+
abbc = linear_interpolation(ab,bc,factor)
|
150
|
+
bccd = linear_interpolation(bc,cd,factor)
|
151
|
+
return linear_interpolation(abbc,bccd,factor)
|
152
|
+
end
|
153
|
+
|
154
|
+
def length(points)
|
155
|
+
old_point = @current_cursor;
|
156
|
+
length = 0.0
|
157
|
+
factor = points.to_f
|
158
|
+
|
159
|
+
(1..points).each {|point|
|
160
|
+
new_point = make_curvepoint(point/(factor.to_f))
|
161
|
+
length += old_point.dist(new_point)
|
162
|
+
old_point = new_point
|
163
|
+
}
|
164
|
+
return length
|
165
|
+
end
|
166
|
+
|
167
|
+
# This gives back an array of points on the curve. The argument given
|
168
|
+
# denotes how the distance between each point.
|
169
|
+
def make_curvepoint_array(distance)
|
170
|
+
result = Array.new
|
171
|
+
|
172
|
+
l = length(20)
|
173
|
+
points = l * distance
|
174
|
+
factor = points.to_f
|
175
|
+
|
176
|
+
(0..points).each {|point|
|
177
|
+
result.push(make_curvepoint(point/(factor.to_f)))
|
178
|
+
}
|
179
|
+
|
180
|
+
return result
|
181
|
+
end
|
182
|
+
|
183
|
+
|
184
|
+
def to_points
|
185
|
+
return make_curvepoint_array(0.3)
|
186
|
+
end
|
187
|
+
|
188
|
+
def current_cursor
|
189
|
+
@p
|
190
|
+
end
|
191
|
+
|
192
|
+
end
|
193
|
+
|
194
|
+
# SVG_S represents the smooth curveto command.
|
195
|
+
# Syntax is:
|
196
|
+
# s x2 y2 x y
|
197
|
+
# It sets the current cursor to the point (x,y).
|
198
|
+
# As always, capitalization denotes absolute values.
|
199
|
+
# Takes 3 Points as argument, the third being the current cursor
|
200
|
+
# If constructed using SVG_S.relative, the current cursor is added to every
|
201
|
+
# point.
|
202
|
+
class SVG_S < SVG_C
|
203
|
+
|
204
|
+
def initialize(c2, p, current_cursor,previous_point)
|
205
|
+
super(SVG_S.reflect(previous_point,current_cursor), c2, p, current_cursor)
|
206
|
+
end
|
207
|
+
|
208
|
+
# The reflection in this case is rather tricky. Using SVG_C.relative, the
|
209
|
+
# offset of current_cursor is added to all the positions (except current_cursor).
|
210
|
+
# The reflected point, however is already calculated in absolute values.
|
211
|
+
# Because of this, we have to subtract the current_cursor from the reflected
|
212
|
+
# point, as it is already added later. I think I got the classes somewhat wrong.
|
213
|
+
# Maybe points should get a field whether they are absolute oder relative?
|
214
|
+
# Don't know yet. It works now, though!
|
215
|
+
def SVG_S.relative(c2, p, current_cursor, previous_point)
|
216
|
+
SVG_C.relative(SVG_S.reflect(previous_point,current_cursor) - current_cursor, c2, p, current_cursor)
|
217
|
+
end
|
218
|
+
|
219
|
+
def SVG_S.reflect(p, mirror)
|
220
|
+
return mirror + (mirror - p)
|
221
|
+
end
|
222
|
+
|
223
|
+
end
|
224
|
+
|
225
|
+
|
226
|
+
# Stroke represent one stroke, which is a series of SVG commands.
|
227
|
+
class Stroke
|
228
|
+
COMMANDS = ["M", "C", "c", "s", "S"]
|
229
|
+
|
230
|
+
def initialize(stroke_as_code)
|
231
|
+
@command_list = parse(stroke_as_code)
|
232
|
+
end
|
233
|
+
|
234
|
+
def to_points
|
235
|
+
return @command_list.map{|element| element.to_points}.flatten
|
236
|
+
end
|
237
|
+
|
238
|
+
#to array
|
239
|
+
#TODO: better implementation using composite pattern
|
240
|
+
def to_a
|
241
|
+
to_points.map{|point| point.to_a}
|
242
|
+
end
|
243
|
+
|
244
|
+
def split_elements(line)
|
245
|
+
# This is magic.
|
246
|
+
return line.gsub("-",",-").gsub("s",",s,").gsub("S",",S,").gsub("c",",c,").gsub("C",",C,").gsub("m", "M").gsub("M","M,").gsub("[","").gsub(";",",;,").gsub(",,",",").gsub(" ,", ",").gsub(", ", ",").gsub(" ", ",").split(/,/);
|
247
|
+
end
|
248
|
+
|
249
|
+
def parse(stroke_as_code)
|
250
|
+
elements = split_elements(stroke_as_code).delete_if{ |e| e == "" }
|
251
|
+
command_list = Array.new
|
252
|
+
current_cursor = Point.new(0,0);
|
253
|
+
|
254
|
+
while elements != [] do
|
255
|
+
|
256
|
+
case elements.slice!(0)
|
257
|
+
when "M"
|
258
|
+
x,y = elements.slice!(0..1)
|
259
|
+
m = SVG_M.new(Point.new(x.to_f,y.to_f))
|
260
|
+
current_cursor = m.current_cursor
|
261
|
+
command_list.push(m)
|
262
|
+
|
263
|
+
when "C"
|
264
|
+
x1,y1,x2,y2,x,y = elements.slice!(0..5)
|
265
|
+
c = SVG_C.new(Point.new(x1.to_f,y1.to_f), Point.new(x2.to_f,y2.to_f), Point.new(x.to_f,y.to_f), current_cursor)
|
266
|
+
current_cursor = c.current_cursor
|
267
|
+
command_list.push(c)
|
268
|
+
|
269
|
+
#handle polybezier
|
270
|
+
unless elements.empty? || COMMANDS.include?(elements.first)
|
271
|
+
elements.unshift("C")
|
272
|
+
end
|
273
|
+
when "c"
|
274
|
+
x1,y1,x2,y2,x,y = elements.slice!(0..5)
|
275
|
+
c = SVG_C.relative(Point.new(x1.to_f,y1.to_f), Point.new(x2.to_f,y2.to_f), Point.new(x.to_f,y.to_f), current_cursor)
|
276
|
+
current_cursor = c.current_cursor
|
277
|
+
command_list.push(c)
|
278
|
+
|
279
|
+
#handle polybezier
|
280
|
+
unless elements.empty? || COMMANDS.include?(elements.first)
|
281
|
+
elements.unshift("c")
|
282
|
+
end
|
283
|
+
|
284
|
+
when "s"
|
285
|
+
x2,y2,x,y = elements.slice!(0..3)
|
286
|
+
reflected_point = command_list[-1].second_point
|
287
|
+
s = SVG_S.relative(Point.new(x2.to_f,y2.to_f), Point.new(x.to_f,y.to_f), current_cursor, reflected_point)
|
288
|
+
current_cursor = s.current_cursor
|
289
|
+
command_list.push(s)
|
290
|
+
|
291
|
+
when "S"
|
292
|
+
x2,y2,x,y = elements.slice!(0..3)
|
293
|
+
reflected_point = command_list[-1].second_point
|
294
|
+
s = SVG_S.new(Point.new(x2.to_f,y2.to_f), Point.new(x.to_f,y.to_f), current_cursor,reflected_point)
|
295
|
+
current_cursor = s.current_cursor
|
296
|
+
command_list.push(s)
|
297
|
+
|
298
|
+
else
|
299
|
+
#print "You should not be here\n"
|
300
|
+
|
301
|
+
end
|
302
|
+
|
303
|
+
end
|
304
|
+
|
305
|
+
return command_list
|
306
|
+
end
|
307
|
+
|
308
|
+
end
|
309
|
+
end
|
310
|
+
end
|
metadata
ADDED
@@ -0,0 +1,152 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: kvg_character_recognition
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Jiayi Zheng
|
8
|
+
autorequire:
|
9
|
+
bindir: exe
|
10
|
+
cert_chain: []
|
11
|
+
date: 2016-01-12 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: nokogiri
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: sequel
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: sqlite3
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: bundler
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '1.10'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '1.10'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: rake
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - "~>"
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '10.0'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - "~>"
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '10.0'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: rspec
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ">="
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0'
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0'
|
97
|
+
description: "This gem contains a CJK-character recognition engine using pattern/template
|
98
|
+
matching techniques.\n It can recognize stroke-order and stroke-number free handwritten
|
99
|
+
character patterns in the format [stroke1, stroke2 ...].\n A stroke is an array
|
100
|
+
of points in the format [[x1, y1], [x2, y2], ...].\n KanjiVG data(characters in
|
101
|
+
svg format) from https://github.com/KanjiVG/kanjivg/releases are used as templates.\n
|
102
|
+
\ "
|
103
|
+
email:
|
104
|
+
- thebluber@gmail.com
|
105
|
+
executables: []
|
106
|
+
extensions: []
|
107
|
+
extra_rdoc_files: []
|
108
|
+
files:
|
109
|
+
- ".gitignore"
|
110
|
+
- ".rspec"
|
111
|
+
- ".travis.yml"
|
112
|
+
- Gemfile
|
113
|
+
- LICENSE.txt
|
114
|
+
- README.md
|
115
|
+
- Rakefile
|
116
|
+
- bin/console
|
117
|
+
- bin/setup
|
118
|
+
- kvg_character_recognition.gemspec
|
119
|
+
- lib/kvg_character_recognition.rb
|
120
|
+
- lib/kvg_character_recognition/database.rb
|
121
|
+
- lib/kvg_character_recognition/feature_extractor.rb
|
122
|
+
- lib/kvg_character_recognition/preprocessor.rb
|
123
|
+
- lib/kvg_character_recognition/recognizer.rb
|
124
|
+
- lib/kvg_character_recognition/utils.rb
|
125
|
+
- lib/kvg_character_recognition/version.rb
|
126
|
+
homepage: https://github.com/thebluber/kvg_character_recognition
|
127
|
+
licenses:
|
128
|
+
- MIT
|
129
|
+
metadata:
|
130
|
+
allowed_push_host: https://rubygems.org
|
131
|
+
post_install_message:
|
132
|
+
rdoc_options: []
|
133
|
+
require_paths:
|
134
|
+
- lib
|
135
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
136
|
+
requirements:
|
137
|
+
- - ">="
|
138
|
+
- !ruby/object:Gem::Version
|
139
|
+
version: '0'
|
140
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
141
|
+
requirements:
|
142
|
+
- - ">="
|
143
|
+
- !ruby/object:Gem::Version
|
144
|
+
version: '0'
|
145
|
+
requirements: []
|
146
|
+
rubyforge_project:
|
147
|
+
rubygems_version: 2.4.5.1
|
148
|
+
signing_key:
|
149
|
+
specification_version: 4
|
150
|
+
summary: CJK-character recognition using template matching techniques and template
|
151
|
+
data from KanjiVG project
|
152
|
+
test_files: []
|