unicode-scripts 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 80faaef79a195000de4ade1f5b6c371f1234a1ba
+  data.tar.gz: aa006391634143063c9132a3e5f6beff03c1a6ef
+SHA512:
+  metadata.gz: 92487c1b6d5a87383c6ddea8455062231c45c99a7f2b5e711120f9e92b522a3d5688500d466c862d7a701b9aabd78b222f58de255003544bf9792eb97907c269
+  data.tar.gz: b990f3ec80f1388575912f07bcccea287d9a867bf9d5755ce382633a37ca0c536a627d3f840ccb6dccdbd7797f5d473c4761fe6cf26631d2d315740e6bf25993

data/.gitignore

@@ -0,0 +1,2 @@
+Gemfile.lock
+/pkg

data/.travis.yml

@@ -0,0 +1,21 @@
+sudo: false
+language: ruby
+
+script: bundle exec ruby spec/unicode_scripts_spec.rb
+
+rvm:
+- 2.3.0
+- 2.2
+- 2.1
+- ruby-head
+- rbx-2
+- jruby-head
+- jruby-9.0.5.0
+
+cache:
+- bundler
+
+matrix:
+  allow_failures:
+    - rvm: jruby-head
+    - rvm: rbx-2

data/CHANGELOG.md

@@ -0,0 +1,6 @@
+## CHANGELOG
+
+### 1.0.0
+
+* Inital release
+

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,74 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of experience,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at opensource@janlelis.com. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at [http://contributor-covenant.org/version/1/4][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/4/

data/Gemfile

@@ -0,0 +1,5 @@
+source 'https://rubygems.org'
+
+gemspec
+
+gem 'minitest'

data/MIT-LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2016 Jan Lelis, mail@janlelis.de
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,340 @@
+# Unicode::Scripts [![[version]](https://badge.fury.io/rb/unicode-scripts.svg)](http://badge.fury.io/rb/unicode-scripts)  [![[travis]](https://travis-ci.org/janlelis/unicode-scripts.png)](https://travis-ci.org/janlelis/unicode-scripts)
+
+Retrieve the [Unicode script(s)](https://en.wikipedia.org/wiki/Script_%28Unicode%29) a string belongs to. Can also return the *Script_Extension* property which is defined as characters which are "commonly used with more than one script, but with a limited number of scripts".
+
+Unicode version: **8.0.0**
+
+Supported Rubies: **2.3**, **2.2**, **2.1**
+
+## Gemfile
+
+```ruby
+gem "unicode-scripts"
+```
+
+## Usage
+
+```ruby
+require "unicode/scripts"
+
+Unicode::Scripts.scripts("СC") # => ["Cyrillic", "Latin"]
+
+# 4 letter script aliases
+Unicode::Scripts.scripts("СC", format: :short) # => ["Cyrl", "Latn"]
+
+# Single character
+Unicode::Scripts.script("ᴦ") # => "Greek"
+
+# Script_Extension property
+Unicode::Scripts.script_extensions("॥") # => ["Bengali", "Devanagari", "Grantha", "Gujarati",
+                                              "Gurmukhi", "Kannada", "Khudawadi", "Limbu",
+                                              "Mahajani", "Malayalam", "Oriya", "Sinhala",
+                                              "Syloti_Nagri", "Takri", "Tamil", "Telugu",
+                                              "Tirhuta"]
+```
+
+## Hints
+### Regex Matching
+
+If you have a string and want to match a substring/character from a specific Unicode script, you actually won't need this gem. Instead, you can use the [Regexp Unicode Property Syntax `\p{}`](http://ruby-doc.org/core-2.3.0/Regexp.html#class-Regexp-label-Character+Properties):
+
+```ruby
+"Coptic letter: ⲁ".scan(/\p{Coptic}/) # => ["ⲁ"]
+```
+
+### Script Names
+
+You can extract all script names from the gem like this:
+
+```ruby
+require "unicode/scripts"
+puts Unicode::Scripts.names
+
+# # # Output # # #
+
+Caucasian_Albanian
+Ahom
+Arabic
+Imperial_Aramaic
+Armenian
+Avestan
+Balinese
+Bamum
+Bassa_Vah
+Batak
+Bengali
+Bopomofo
+Brahmi
+Braille
+Buginese
+Buhid
+Chakma
+Canadian_Aboriginal
+Carian
+Cham
+Cherokee
+Coptic
+Cypriot
+Cyrillic
+Devanagari
+Deseret
+Duployan
+Egyptian_Hieroglyphs
+Elbasan
+Ethiopic
+Georgian
+Glagolitic
+Gothic
+Grantha
+Greek
+Gujarati
+Gurmukhi
+Hangul
+Han
+Hanunoo
+Hatran
+Hebrew
+Hiragana
+Anatolian_Hieroglyphs
+Pahawh_Hmong
+Katakana_Or_Hiragana
+Old_Hungarian
+Old_Italic
+Javanese
+Kayah_Li
+Katakana
+Kharoshthi
+Khmer
+Khojki
+Kannada
+Kaithi
+Tai_Tham
+Lao
+Latin
+Lepcha
+Limbu
+Linear_A
+Linear_B
+Lisu
+Lycian
+Lydian
+Mahajani
+Mandaic
+Manichaean
+Mende_Kikakui
+Meroitic_Cursive
+Meroitic_Hieroglyphs
+Malayalam
+Modi
+Mongolian
+Mro
+Meetei_Mayek
+Multani
+Myanmar
+Old_North_Arabian
+Nabataean
+Nko
+Ogham
+Ol_Chiki
+Old_Turkic
+Oriya
+Osmanya
+Palmyrene
+Pau_Cin_Hau
+Old_Permic
+Phags_Pa
+Inscriptional_Pahlavi
+Psalter_Pahlavi
+Phoenician
+Miao
+Inscriptional_Parthian
+Rejang
+Runic
+Samaritan
+Old_South_Arabian
+Saurashtra
+SignWriting
+Shavian
+Sharada
+Siddham
+Khudawadi
+Sinhala
+Sora_Sompeng
+Sundanese
+Syloti_Nagri
+Syriac
+Tagbanwa
+Takri
+Tai_Le
+New_Tai_Lue
+Tamil
+Tai_Viet
+Telugu
+Tifinagh
+Tagalog
+Thaana
+Thai
+Tibetan
+Tirhuta
+Ugaritic
+Vai
+Warang_Citi
+Old_Persian
+Cuneiform
+Yi
+Inherited
+Common
+Unknown
+```
+
+### Script Extension Names
+
+You can extract all script extensions names from the gem like this:
+
+```ruby
+require "unicode/scripts"
+puts Unicode::Scripts.extension_names
+
+# # # Output # # #
+
+Aghb
+Ahom
+Arab
+Armi
+Armn
+Avst
+Bali
+Bamu
+Bass
+Batk
+Beng
+Bopo
+Brah
+Brai
+Bugi
+Buhd
+Cakm
+Cans
+Cari
+Cham
+Cher
+Copt
+Qaac
+Cprt
+Cyrl
+Deva
+Dsrt
+Dupl
+Egyp
+Elba
+Ethi
+Geor
+Glag
+Goth
+Gran
+Grek
+Gujr
+Guru
+Hang
+Hani
+Hano
+Hatr
+Hebr
+Hira
+Hluw
+Hmng
+Hrkt
+Hung
+Ital
+Java
+Kali
+Kana
+Khar
+Khmr
+Khoj
+Knda
+Kthi
+Lana
+Laoo
+Latn
+Lepc
+Limb
+Lina
+Linb
+Lisu
+Lyci
+Lydi
+Mahj
+Mand
+Mani
+Mend
+Merc
+Mero
+Mlym
+Modi
+Mong
+Mroo
+Mtei
+Mult
+Mymr
+Narb
+Nbat
+Nkoo
+Ogam
+Olck
+Orkh
+Orya
+Osma
+Palm
+Pauc
+Perm
+Phag
+Phli
+Phlp
+Phnx
+Plrd
+Prti
+Rjng
+Runr
+Samr
+Sarb
+Saur
+Sgnw
+Shaw
+Shrd
+Sidd
+Sind
+Sinh
+Sora
+Sund
+Sylo
+Syrc
+Tagb
+Takr
+Tale
+Talu
+Taml
+Tavt
+Telu
+Tfng
+Tglg
+Thaa
+Thai
+Tibt
+Tirh
+Ugar
+Vaii
+Wara
+Xpeo
+Xsux
+Yiii
+Zinh
+Qaai
+Zyyy
+Zzzz
+```
+
+## MIT License
+
+- Copyright (C) 2016 Jan Lelis <http://janlelis.com>. Released under the MIT license.
+- Unicode data: http://www.unicode.org/copyright.html#Exhibit1

data/Rakefile

@@ -0,0 +1,37 @@
+# # #
+# Get gemspec info
+
+gemspec_file = Dir['*.gemspec'].first
+gemspec = eval File.read(gemspec_file), binding, gemspec_file
+info = "#{gemspec.name} | #{gemspec.version} | " \
+       "#{gemspec.runtime_dependencies.size} dependencies | " \
+       "#{gemspec.files.size} files"
+
+# # #
+# Gem build and install task
+
+desc info
+task :gem do
+  puts info + "\n\n"
+  print "  "; sh "gem build #{gemspec_file}"
+  FileUtils.mkdir_p 'pkg'
+  FileUtils.mv "#{gemspec.name}-#{gemspec.version}.gem", 'pkg'
+  puts; sh %{gem install --no-document pkg/#{gemspec.name}-#{gemspec.version}.gem}
+end
+
+# # #
+# Start an IRB session with the gem loaded
+
+desc "#{gemspec.name} | IRB"
+task :irb do
+  sh "irb -I ./lib -r #{gemspec.name.gsub '-','/'}"
+end
+
+# # #
+# Run Specs
+
+desc "#{gemspec.name} | Spec"
+task :spec do
+  sh "for file in spec/*.rb; do ruby $file; done"
+end
+task default: :spec

data/lib/unicode/scripts.rb

@@ -0,0 +1,57 @@
+require_relative "scripts/constants"
+
+module Unicode
+  module Scripts
+    def self.scripts(string, **options)
+      res = []
+      string.each_char{ |char|
+        script_name = script(char, **options)
+        res << script_name unless res.include?(script_name)
+      }
+      res.sort
+    end
+    class << self; alias of scripts; end
+
+    def self.script(char, format: :long)
+      require_relative 'scripts/index' unless defined? ::Unicode::Scripts::INDEX
+      codepoint_depth_offset = char.unpack("U")[0] or
+          raise(ArgumentError, "Unicode::Scripts.script must be given a valid char")
+      index_or_value = INDEX[:SCRIPTS]
+      [0x10000, 0x1000, 0x100, 0x10].each{ |depth|
+        index_or_value         = index_or_value[codepoint_depth_offset / depth]
+        codepoint_depth_offset = codepoint_depth_offset % depth
+        unless index_or_value.is_a? Array
+          res = index_or_value || INDEX[:SCRIPT_ALIASES]["Zzzz"]
+          return format == :long ? INDEX[:SCRIPT_NAMES][res] : INDEX[:SCRIPT_ALIASES].key(res)
+        end
+      }   
+
+      res = index_or_value[codepoint_depth_offset] || INDEX[:SCRIPT_ALIASES]["Zzzz"]
+      format == :long ? INDEX[:SCRIPT_NAMES][res] : INDEX[:SCRIPT_ALIASES].key(res)
+    end
+
+    def self.script_extensions(string, format: :long)
+      require_relative 'scripts/index' unless defined? ::Unicode::Scripts::INDEX
+
+      string.each_codepoint.inject([]){ |res, codepoint|
+        if new_scripts = INDEX[:SCRIPT_EXTENSIONS][codepoint]
+          script_extension_names = new_scripts.map{ |new_script|
+            format == :long ? INDEX[:SCRIPT_NAMES][new_script] : INDEX[:SCRIPT_ALIASES].key(new_script)
+          }
+        else
+          script_extension_names = scripts([codepoint].pack("U"), format: format)
+        end
+
+        res | script_extension_names
+      }.sort
+    end
+
+    def self.names(format: :long)
+      require_relative 'scripts/index' unless defined? ::Unicode::Scripts::INDEX
+      format == :long ?
+          INDEX[:SCRIPT_NAMES].sort :
+          INDEX[:SCRIPT_ALIASES].keys.sort
+    end
+  end
+end
+

data/lib/unicode/scripts/constants.rb

@@ -0,0 +1,9 @@
+module Unicode
+  module Scripts
+    VERSION = "1.0.0".freeze
+    UNICODE_VERSION = "8.0.0".freeze
+    DATA_DIRECTORY = File.expand_path(File.dirname(__FILE__) + '/../../../data/').freeze
+    INDEX_FILENAME = (DATA_DIRECTORY + '/scripts.marshal.gz').freeze
+  end
+end
+

data/lib/unicode/scripts/index.rb

@@ -0,0 +1,7 @@
+require_relative 'constants'
+
+module Unicode
+  module Scripts
+    INDEX = Marshal.load(Gem.gunzip(File.binread(INDEX_FILENAME)))
+  end
+end

data/lib/unicode/scripts/string_ext.rb

@@ -0,0 +1,12 @@
+require_relative "../scripts"
+
+class String
+  # Optional string extension for your convenience
+  def unicode_scripts
+    Unicode::Scripts.scripts(self)
+  end
+
+  def unicode_script_extensions
+    Unicode::Scripts.script_extensions(self)
+  end
+end

data/spec/unicode_scripts_spec.rb

@@ -0,0 +1,117 @@
+require_relative "../lib/unicode/scripts"
+require "minitest/autorun"
+
+describe Unicode::Scripts do
+  describe ".scripts (alias .of)" do
+    it "will always return an Array" do
+      assert_equal [], Unicode::Scripts.of("")
+    end
+
+    it "will return all scripts that characters in the string belong to" do
+      assert_equal ["Cyrillic", "Latin"], Unicode::Scripts.of("СC")
+    end
+
+    it "will return all scripts in sorted order" do
+      assert_equal ["Cyrillic", "Latin"], Unicode::Scripts.of("СA")
+      assert_equal ["Cyrillic", "Latin"], Unicode::Scripts.of("AС")
+    end
+
+    it "will call .script for every character" do
+      mocked_method = MiniTest::Mock.new
+      mocked_method.expect :call, "first script",  ["С", {}]
+      mocked_method.expect :call, "second script", ["A", {}]
+      Unicode::Scripts.stub :script, mocked_method do
+        Unicode::Scripts.of("СA")
+      end
+      mocked_method.verify
+    end
+  end
+
+  describe ".script" do
+    it "will return script for that character" do
+      assert_equal "Greek", Unicode::Scripts.script("ᴦ")
+      assert_equal "Common", Unicode::Scripts.script("�")
+    end
+
+    it "will return Unknown for characters not in any script" do
+      assert_equal "Unknown", Unicode::Scripts.script("\u{10c50}")
+    end
+
+    it "will return 4 letter script codes with format: :short" do
+      assert_equal ["Cyrl", "Latn"], Unicode::Scripts.of("СC", format: :short)
+    end
+  end
+
+  describe ".script_extensions" do
+    it "will always return an Array" do
+      assert_equal [], Unicode::Scripts.script_extensions("")
+    end
+
+    it "will return all extended scripts that characters in the string belong to" do
+      assert_equal [
+        "Bengali",
+        "Devanagari",
+        "Grantha",
+        "Gujarati",
+        "Gurmukhi",
+        "Kannada",
+        "Khudawadi",
+        "Limbu",
+        "Mahajani",
+        "Malayalam",
+        "Oriya",
+        "Sinhala",
+        "Syloti_Nagri",
+        "Takri",
+        "Tamil",
+        "Telugu",
+        "Tirhuta"
+      ], Unicode::Scripts.script_extensions("॥")
+    end
+
+    it "will return 4 letter script codes with format: :short" do
+      assert_equal [
+        "Beng",
+        "Deva",
+        "Gran",
+        "Gujr",
+        "Guru",
+        "Knda",
+        "Limb",
+        "Mahj",
+        "Mlym",
+        "Orya",
+        "Sind",
+        "Sinh",
+        "Sylo",
+        "Takr",
+        "Taml",
+        "Telu",
+        "Tirh"
+      ], Unicode::Scripts.script_extensions("॥", format: :short)
+
+    end
+
+    it "will return all extended scripts in sorted order" do
+      assert_equal ["Cyrillic", "Latin"], Unicode::Scripts.script_extensions("СA")
+      assert_equal ["Cyrillic", "Latin"], Unicode::Scripts.script_extensions("AС")
+    end
+
+    it "will call .scripts for characters that have no explicit script extension" do
+      mocked_method = MiniTest::Mock.new
+      mocked_method.expect :call, ["scripts"],  ["A", {format: :long}]
+      Unicode::Scripts.stub :scripts, mocked_method do
+        Unicode::Scripts.script_extensions("A")
+      end
+      mocked_method.verify
+    end
+  end
+
+  describe ".names" do
+    it "will return a list of all script names" do
+      assert_kind_of Array, Unicode::Scripts.names
+      assert_includes Unicode::Scripts.names, "Inscriptional_Parthian"
+    end
+  end
+end
+

data/unicode-scripts.gemspec

@@ -0,0 +1,21 @@
+# -*- encoding: utf-8 -*-
+
+require File.dirname(__FILE__) + "/lib/unicode/scripts/constants"
+
+Gem::Specification.new do |gem|
+  gem.name          = "unicode-scripts"
+  gem.version       = Unicode::Scripts::VERSION
+  gem.summary       = "Which script(s) does a Unicode string belong to?"
+  gem.description   = "[Unicode version: #{Unicode::Scripts::UNICODE_VERSION}] Retrieve the Unicode script(s) a string belongs to. Can also return the Script_Extension property which is defined as characters which are 'commonly used with more than one script, but with a limited number of scripts'. "
+  gem.authors       = ["Jan Lelis"]
+  gem.email         = ["mail@janlelis.de"]
+  gem.homepage      = "https://github.com/janlelis/unicode-scripts"
+  gem.license       = "MIT"
+
+  gem.files         = Dir["{**/}{.*,*}"].select{ |path| File.file?(path) && path !~ /^pkg/ }
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.required_ruby_version = "~> 2.0"
+end

metadata

@@ -0,0 +1,64 @@
+--- !ruby/object:Gem::Specification
+name: unicode-scripts
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Jan Lelis
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-04-13 00:00:00.000000000 Z
+dependencies: []
+description: "[Unicode version: 8.0.0] Retrieve the Unicode script(s) a string belongs
+  to. Can also return the Script_Extension property which is defined as characters
+  which are 'commonly used with more than one script, but with a limited number of
+  scripts'. "
+email:
+- mail@janlelis.de
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- CHANGELOG.md
+- CODE_OF_CONDUCT.md
+- Gemfile
+- MIT-LICENSE.txt
+- README.md
+- Rakefile
+- data/scripts.marshal.gz
+- lib/unicode/scripts.rb
+- lib/unicode/scripts/constants.rb
+- lib/unicode/scripts/index.rb
+- lib/unicode/scripts/string_ext.rb
+- spec/unicode_scripts_spec.rb
+- unicode-scripts.gemspec
+homepage: https://github.com/janlelis/unicode-scripts
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - "~>"
+    - !ruby/object:Gem::Version
+      version: '2.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.3
+signing_key: 
+specification_version: 4
+summary: Which script(s) does a Unicode string belong to?
+test_files:
+- spec/unicode_scripts_spec.rb
+has_rdoc: