scws4r 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 2f35634967bfc1105d2a014dc66399455e821a4de9e9f7f47fe805711f20239f
+  data.tar.gz: 608912dbd14c16576e1dd14465a5beea562e677a6e37b075465d2fe6b4262804
+SHA512:
+  metadata.gz: a176641cdf8518528591e5e11f803d6d99cd0cb7087af6a2830771436e45cea0b68c383dd7321abe2f1c8e7f996df436f13d58b8c3357086ac70d708100c799e
+  data.tar.gz: 26963a99702b3e1cfe82b81edf0400155fbfe3b8639633ad25902e7b221046d366200e4b7d951c0f7a72d2d077825786aa5a444cc8ef74931a8eed3977a96550

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,13 @@
+AllCops:
+  TargetRubyVersion: 2.6
+
+Style/StringLiterals:
+  Enabled: true
+  EnforcedStyle: double_quotes
+
+Style/StringLiteralsInInterpolation:
+  Enabled: true
+  EnforcedStyle: double_quotes
+
+Layout/LineLength:
+  Max: 120

data/CHANGELOG.md

@@ -0,0 +1,5 @@
+## [Unreleased]
+
+## [0.1.0] - 2022-04-08
+
+- Initial release

data/Gemfile

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in scws.gemspec
+gemspec
+
+gem "rake", "~> 13.0"
+gem "rake-compiler"
+gem "rspec", "~> 3.0"
+gem "rubocop", "~> 1.21"
+
+gem "pry", "~> 0.14.1"

data/Gemfile.lock

@@ -0,0 +1,65 @@
+PATH
+  remote: .
+  specs:
+    scws4r (0.1.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ast (2.4.2)
+    coderay (1.1.3)
+    diff-lcs (1.5.0)
+    method_source (1.0.0)
+    parallel (1.22.1)
+    parser (3.1.1.0)
+      ast (~> 2.4.1)
+    pry (0.14.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    rainbow (3.1.1)
+    rake (13.0.6)
+    rake-compiler (1.2.0)
+      rake
+    regexp_parser (2.3.0)
+    rexml (3.2.5)
+    rspec (3.11.0)
+      rspec-core (~> 3.11.0)
+      rspec-expectations (~> 3.11.0)
+      rspec-mocks (~> 3.11.0)
+    rspec-core (3.11.0)
+      rspec-support (~> 3.11.0)
+    rspec-expectations (3.11.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.11.0)
+    rspec-mocks (3.11.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.11.0)
+    rspec-support (3.11.0)
+    rubocop (1.27.0)
+      parallel (~> 1.10)
+      parser (>= 3.1.0.0)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml
+      rubocop-ast (>= 1.16.0, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.17.0)
+      parser (>= 3.1.1.0)
+    ruby-progressbar (1.11.0)
+    unicode-display_width (2.1.0)
+
+PLATFORMS
+  ruby
+  x86_64-darwin-21
+
+DEPENDENCIES
+  pry (~> 0.14.1)
+  rake (~> 13.0)
+  rake-compiler
+  rspec (~> 3.0)
+  rubocop (~> 1.21)
+  scws4r!
+
+BUNDLED WITH
+   2.3.17

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2022 张小辉
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,56 @@
+# SCWS(Simple Chinese Word Segmentation) for Ruby
+
+This is a Ruby gem integrated with [scws](https://github.com/hightman/scws/blob/master/API.md) C lib
+
+## Installation
+
+Install the gem and add to the application's Gemfile by executing:
+
+    $ bundle add scws4r
+
+If bundler is not being used to manage dependencies, install the gem by executing:
+
+    $ gem install scws4r
+
+## Usage
+
+```ruby
+2.7.2 :001 > require 'scws4r'
+ => true
+2.7.2 :002 > s = Scws4r.new
+ => #<Scws4r:0x00007fbc2b1bc878>
+2.7.2 :003 > s.load_defaults
+ => false
+2.7.2 :004 > puts s.split('保障房资金压力')
+{"offset"=>0, "idf"=>4.889999866485596, "length"=>6, "text"=>"保障", "attr"=>"vn"}
+{"offset"=>6, "idf"=>0.0, "length"=>3, "text"=>"房", "attr"=>"n"}
+{"offset"=>9, "idf"=>4.880000114440918, "length"=>6, "text"=>"资金", "attr"=>"n"}
+{"offset"=>15, "idf"=>4.900000095367432, "length"=>6, "text"=>"压力", "attr"=>"n"}
+ => nil
+2.7.2 :005 > puts s.tops('保障房资金压力', 10)
+{"times"=>1, "weight"=>4.900000095367432, "word"=>"压力", "attr"=>"n"}
+{"times"=>1, "weight"=>4.889999866485596, "word"=>"保障", "attr"=>"vn"}
+{"times"=>1, "weight"=>4.880000114440918, "word"=>"资金", "attr"=>"n"}
+ => nil
+2.7.2 :006 >
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake compile` to compile C-extension , then run `rake rspec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## references
+
+- https://docs.ruby-lang.org/en/2.4.0/extension_rdoc.html
+- https://github.com/hightman/scws
+- https://github.com/amutu/zhparser
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/scws.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+require "rubocop/rake_task"
+
+RuboCop::RakeTask.new
+
+require "rake/extensiontask"
+
+task build: :compile
+
+Rake::ExtensionTask.new("scws4r") do |ext|
+  ext.lib_dir = "lib/scws4r"
+end
+
+task default: %i[clobber compile spec rubocop]

data/defaults/rules.utf8.ini

@@ -0,0 +1,291 @@
+;
+; auto regular(utf-8)
+; $Id$
+;
+; special word, 特殊词汇
+;
+
+[special]
+C++
+C#
+R&B
+P&G
+J++
+J#
+UTF-8
+PS/2
+
+;
+; nostats
+;
+[nostats]
+about
+all
+also
+an
+and
+any
+are
+as
+at
+be
+but
+by
+both
+can
+for
+from
+have
+here
+if
+in
+is
+it
+no
+not
+of
+on
+or
+our
+out
+that
+the
+this
+to
+up
+us
+
+;
+; 词性语法规则表
+;
+[attrs]
+; c 是连词
+n + f(1) = 300
+n + m(1) = 500
+n(1) + v = 100
+n + v(1) = 10
+r + n(1) = 1000
+r(1) + n = 100
+d(1) + r = 100
+d(1) + v = 100
+v(1) + r = 100
+n + m(1) = 500
+v + f(1) = 30
+v(1) + m = 100
+v(1) + n = 3
+a + u(1) = 5
+v + n(1) = 5
+u(1) + a = 2
+c(1) + * = 50
+* + c(1) = 50
+
+;
+; 名字停用词表
+;
+[noname]
+:line = no
+给的说对在和是被最所那这有将
+你会与他为不没很了啊哦呵把去
+
+;
+; 双字节符号
+;
+[symbol]
+:type = none
+:line = no
+｀－＝［］、‘；／。，｜？》《：“｛｝＋—）（＊…％￥＃·！～
+’”〕〈〉「」『』〖〗【】＜＞
+
+;
+; 姓和外文名共同部分
+;
+[pubname]
+:type = prefix
+:line = no
+:exclude = noname,symbol,alpha,chnum2
+:znum = 1,2
+:tf = 5.0
+:idf = 3.5
+:attr = nr
+艾安贝卜戴费福盖戈古赫华霍吉贾金柯赖劳雷黎利林卢
+鲁伦罗洛马麦米莫穆齐乔冉萨沙史斯温谢尤詹诸
+
+
+[pubname2]
+:type = prefix
+:line = no
+:exclude = noname,symbol,alpha,chnum2
+:tf = 5.0
+:idf = 3.5
+:attr = nr
+伍陆
+
+[pubname3]
+:type = prefix
+:line = no
+:exclude = noname,symbol,alpha,chnum2
+:tf = 5.0
+:idf = 3.5
+:attr = nr
+万章
+
+;
+; 单姓
+;
+[surname]
+:type = prefix
+:line = no
+:exclude = noname,symbol,alpha,chnum2
+:tf = 5.0
+:idf = 3.5
+:attr = nr
+:znum = 1,2
+
+敖白班包宝保鲍毕边卞柏蔡曹岑柴昌常车陈成程迟池褚
+楚储淳崔刀邓狄刁丁董窦杜端段樊范方房斐丰封冯凤伏
+傅甘高耿龚宫勾苟辜谷顾官关管桂郭韩杭郝禾何贺衡洪
+侯胡花黄稽姬纪季简翦姜江蒋焦晋靳荆居康空孔匡邝况
+蓝郎朗乐冷李理厉励连廉练良梁廖凌刘柳隆龙楼娄吕路
+骆麻满茅毛梅孟苗缪闵明牟倪聂牛钮农潘庞裴彭皮朴平
+蒲溥浦戚祁钱强秦丘邱仇裘屈瞿权饶任荣容阮瑞芮赛单
+商邵佘申沈盛石寿舒宋苏孙邰谭谈汤唐陶滕田佟仝屠涂
+汪王危韦魏卫蔚闻翁巫邬武吴奚习夏鲜席冼项萧解辛邢
+幸熊徐许宣薛荀颜阎言严彦晏燕杨阳姚叶蚁易殷银尹应
+英游于於鱼虞俞余禹喻郁尉袁岳云臧曾查翟湛张赵甄郑
+钟周朱竺祝庄卓宗邹祖左肖
+
+;
+; 复姓
+;
+[surname2]
+:type = prefix
+:line = yes
+:exclude = noname,symbol,alpha,chnum2
+:tf = 5.0
+:idf = 3.5
+:attr = nr
+:znum = 1, 2
+东郭
+公孙
+皇甫
+慕容
+欧阳
+单于
+司空
+司马
+司徒
+澹台
+诸葛
+
+;
+; 地点名称
+;
+[areaname]
+:type = suffix
+:znum = 2
+:exclude = noname,symbol,alpha,chnum2
+:tf = 4.5
+:idf = 3.0
+:attr = ns
+:line = no
+
+县市镇村乡区
+
+;
+; 双字地点名称
+;
+[areaname2]
+:type = suffix
+:znum = 2
+:exclude = noname,symbol,alpha,chnum2
+:tf = 4.5
+:idf = 3.0
+:attr = ns
+:line = yes
+东路
+西路
+支路
+街道
+南路
+北路
+
+
+[munit]
+:type = none
+:line = no
+萬亿零年点分秒回节名个多届次集
+
+[chnum0]
+:type = prefix
+:line = no
+:tf = 2.5
+:idf = 1.0
+:attr = mt
+:include = chnum2,chnum3,munit,pubname3
+０
+
+[chnum1]
+:type = prefix
+:include = chnum0,chnum1,munit,pubname3
+:tf = 3.0
+:idf = 1.0
+:attr = mt
+:line = no
+一二三四五六七八九十百千
+
+[chnum2]
+:type = prefix
+:line = no
+:tf = 3.0
+:idf = 1.0
+:attr = mt
+:include = chnum0,chnum2,chnum3,munit,pubname3
+１２３４５６７８９
+
+[chnum3]
+:type = none
+:line = no
+．
+
+[chnum4]
+:type = prefix
+:line = no
+:tf = 3.0
+:idf = 1.0
+:attr = mt
+:include = chnum4,munit,pubname2,pubname3
+
+壹贰叁肆柒捌玖拾佰仟
+
+[chnum5]
+:type = prefix
+:line = no
+:tf = 3.5
+:idf = 2.0
+:attr = nz
+:include = chnum1,munit,pubname3,chnum2
+
+第每
+
+[alpha]
+:type = prefix
+:line = no
+:tf = 2.5
+:idf = 1.0
+:attr = en
+:include = alpha
+
+ａｂｃｄｅｆｇｈｉｊｋｌｍｎｏｐｑｒｓｔｕｖｗｘｙｚ
+ＡＢＣＤＥＦＧＨＩＪＫＬＭＮＯＰＱＲＳＴＵＶＷＸＹＺ＇
+
+[foregin]
+:type = prefix
+:line = no
+:tf = 4.0
+:idf = 3.0
+:attr = nr
+:include = foregin,pubname,pubname2,pubname3
+阿克拉加内亚巴尔姆爱兰西伊杰纳布可夫勒特坦芬尼根登都
+伯泰胥俄科索沃森奥瓦茨普蒂塞维大莱德冈墨哥弗库澳哈兹
+乌奇切诺里基延达塔卡雅来波迈蓬什比摩曼乃休合娜迪凯帕
+桑佩蒙博托格泽及希匹印埃努烈累法图喀土腓耶逊宾