blingfire 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: a93f90584141a618ff5bb73860068e2ea662c568a07e2f877a5f2be7e265919e
+  data.tar.gz: 61b49a4bd7eb304f44c041730bce733102f1936c9340cd9c0c3f1390bd9e853a
+SHA512:
+  metadata.gz: fd0242fc13696c620ffd3a7383b9088aaddade1f7280c4f35c54d5165f66e26b783e88fa2d65ced002e8f233af0637e43e9adb68ad7556a16cfdcbb6aff9ba7e
+  data.tar.gz: a1206eaa056ed93639c00c03eb16be5fbbec6b077e1677a916b69c6244c26d3c2cce51299dc0394335c54d12de500a9648f30fa6f3785ef749be528904d5ae60

data/CHANGELOG.md

@@ -0,0 +1,3 @@
+## 0.1.0 (2020-02-24)
+
+- First release

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2020 Andrew Kane
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,76 @@
+# BlingFire
+
+[BlingFire](https://github.com/microsoft/BlingFire) - high speed text tokenization - for Ruby
+
+## Installation
+
+Add this line to your application’s Gemfile:
+
+```ruby
+gem 'blingfire'
+```
+
+## Getting Started
+
+Create a model
+
+```ruby
+model = BlingFire::Model.new
+```
+
+Tokenize words
+
+```ruby
+model.text_to_words(text)
+```
+
+Tokenize sentences
+
+```ruby
+model.text_to_sentences(text)
+```
+
+## Pre-trained Models
+
+BlingFire comes with a default model that follows the tokenization logic of NLTK with a few changes. You can also download other models:
+
+- [BERT Base](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/bert_base_tok.bin)
+- [BERT Base Cased](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/bert_base_cased_tok.bin)
+- [BERT Chinese](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/bert_chinese.bin)
+- [BERT Multilingual Cased](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/bert_multi_cased.bin)
+- [WBD](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/wbd_chuni.bin)
+
+Load a model
+
+```ruby
+model = BlingFire.load_model("bert_base_tok.bin")
+```
+
+Convert text to ids
+
+```ruby
+model.text_to_ids(text)
+```
+
+## History
+
+View the [changelog](https://github.com/ankane/blingfire/blob/master/CHANGELOG.md)
+
+## Contributing
+
+Everyone is encouraged to help improve this project. Here are a few ways you can help:
+
+- [Report bugs](https://github.com/ankane/blingfire/issues)
+- Fix bugs and [submit pull requests](https://github.com/ankane/blingfire/pulls)
+- Write, clarify, or fix documentation
+- Suggest or add new features
+
+To get started with development:
+
+```sh
+git clone https://github.com/ankane/blingfire.git
+cd blingfire
+bundle install
+bundle exec rake vendor:all
+bundle exec rake test
+```

data/lib/blingfire.rb

@@ -0,0 +1,91 @@
+# stdlib
+require "fiddle/import"
+
+# modules
+require "blingfire/model"
+require "blingfire/version"
+
+module BlingFire
+  class Error < StandardError; end
+
+  class << self
+    attr_accessor :ffi_lib
+  end
+  lib_name =
+    if Gem.win_platform?
+      "blingfiretokdll.dll"
+    elsif RbConfig::CONFIG["host_os"] =~ /darwin/i
+      "libblingfiretokdll.dylib"
+    else
+      "libblingfiretokdll.so"
+    end
+  vendor_lib = File.expand_path("../vendor/#{lib_name}", __dir__)
+  self.ffi_lib = [vendor_lib]
+
+  # friendlier error message
+  autoload :FFI, "blingfire/ffi"
+
+  class << self
+    def lib_version
+      FFI.GetBlingFireTokVersion
+    end
+
+    def load_model(path)
+      Model.new(path)
+    end
+
+    def text_to_words(text)
+      text = encode_utf8(text.dup) unless text.encoding == Encoding::UTF_8
+      out = Fiddle::Pointer.malloc(text.bytesize * 3)
+      out_size = FFI.TextToWords(text, text.bytesize, out, out.size)
+      check_status out_size
+      encode_utf8(out[0, out_size - 1]).split(" ")
+    end
+
+    def text_to_words_with_model(model, text)
+      text = encode_utf8(text.dup) unless text.encoding == Encoding::UTF_8
+      out = Fiddle::Pointer.malloc(text.bytesize * 3)
+      out_size = FFI.TextToWordsWithModel(text, text.bytesize, out, out.size, model)
+      check_status out_size
+      encode_utf8(out[0, out_size - 1]).split(" ")
+    end
+
+    def text_to_sentences(text)
+      text = encode_utf8(text.dup) unless text.encoding == Encoding::UTF_8
+      out = Fiddle::Pointer.malloc(text.bytesize * 3)
+      out_size = FFI.TextToSentences(text, text.bytesize, out, out.size)
+      check_status out_size
+      encode_utf8(out[0, out_size - 1]).split("\n")
+    end
+
+    def text_to_sentences_with_model(model, text)
+      text = encode_utf8(text.dup) unless text.encoding == Encoding::UTF_8
+      out = Fiddle::Pointer.malloc(text.bytesize * 3)
+      out_size = FFI.TextToSentencesWithModel(text, text.bytesize, out, out.size, model)
+      check_status out_size
+      encode_utf8(out[0, out_size - 1]).split("\n")
+    end
+
+    def text_to_ids(model, text, max_len = nil, unk_id = 0)
+      text = encode_utf8(text.dup) unless text.encoding == Encoding::UTF_8
+      ids = Fiddle::Pointer.malloc((max_len || text.size) * Fiddle::SIZEOF_INT)
+      out_size = FFI.TextToIds(model, text, text.bytesize, ids, ids.size, unk_id)
+      check_status out_size
+      ids[0, (max_len || out_size) * Fiddle::SIZEOF_INT].unpack("i!*")
+    end
+
+    def free_model(model)
+      FFI.FreeModel(model)
+    end
+
+    private
+
+    def check_status(ret)
+      raise Error, "Bad status" if ret == -1
+    end
+
+    def encode_utf8(text)
+      text.force_encoding(Encoding::UTF_8)
+    end
+  end
+end

data/lib/blingfire/ffi.rb

@@ -0,0 +1,22 @@
+module BlingFire
+  module FFI
+    extend Fiddle::Importer
+
+    libs = Array(BlingFire.ffi_lib).dup
+    begin
+      dlload Fiddle.dlopen(libs.shift)
+    rescue Fiddle::DLError => e
+      retry if libs.any?
+      raise e
+    end
+
+    extern "int GetBlingFireTokVersion()"
+    extern "void* LoadModel(char * pszLdbFileName)"
+    extern "int FreeModel(void* ModelPtr)"
+    extern "int TextToWordsWithModel(char * pInUtf8Str, int InUtf8StrByteCount, char * pOutUtf8Str, int MaxOutUtf8StrByteCount, void * hModel)"
+    extern "int TextToWords(char * pInUtf8Str, int InUtf8StrByteCount, char * pOutUtf8Str, int MaxOutUtf8StrByteCount)"
+    extern "int TextToIds(void* ModelPtr, char * pInUtf8Str, int InUtf8StrByteCount, int32_t * pIdsArr, int MaxIdsArrLength, int UnkId)"
+    extern "int TextToSentences(char * pInUtf8Str, int InUtf8StrByteCount, char * pOutUtf8Str, int MaxOutUtf8StrByteCount)"
+    extern "int TextToSentencesWithModel(char * pInUtf8Str, int InUtf8StrByteCount, char * pOutUtf8Str, int MaxOutUtf8StrByteCount, void * hModel)"
+  end
+end

data/lib/blingfire/model.rb

@@ -0,0 +1,43 @@
+module BlingFire
+  class Model
+    def initialize(path = nil)
+      if path
+        @handle = FFI.LoadModel(path)
+        ObjectSpace.define_finalizer(self, self.class.finalize(@handle))
+      end
+    end
+
+    def text_to_words(text)
+      if @handle
+        BlingFire.text_to_words_with_model(@handle, text)
+      else
+        BlingFire.text_to_words(text)
+      end
+    end
+
+    def text_to_sentences(text)
+      if @handle
+        BlingFire.text_to_sentences_with_model(@handle, text)
+      else
+        BlingFire.text_to_sentences(text)
+      end
+    end
+
+    def text_to_ids(text, max_len = nil, unk_id = 0)
+      if @handle
+        BlingFire.text_to_ids(@handle, text, max_len, unk_id)
+      else
+        raise "Not implemented"
+      end
+    end
+
+    def to_ptr
+      @handle
+    end
+
+    def self.finalize(pointer)
+      # must use proc instead of stabby lambda
+      proc { FFI.FreeModel(pointer) }
+    end
+  end
+end

data/lib/blingfire/version.rb

@@ -0,0 +1,3 @@
+module BlingFire
+  VERSION = "0.1.0"
+end

data/vendor/LICENSE

@@ -0,0 +1,21 @@
+    MIT License
+
+    Copyright (c) Microsoft Corporation. All rights reserved.
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE

metadata

@@ -0,0 +1,95 @@
+--- !ruby/object:Gem::Specification
+name: blingfire
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Andrew Kane
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-02-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5'
+description: 
+email: andrew@chartkick.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- LICENSE.txt
+- README.md
+- lib/blingfire.rb
+- lib/blingfire/ffi.rb
+- lib/blingfire/model.rb
+- lib/blingfire/version.rb
+- vendor/LICENSE
+- vendor/blingfiretokdll.dll
+- vendor/libblingfiretokdll.dylib
+- vendor/libblingfiretokdll.so
+homepage: https://github.com/ankane/blingfire
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.4'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.2
+signing_key: 
+specification_version: 4
+summary: High speed text tokenization for Ruby
+test_files: []