blingfire 0.1.0 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a93f90584141a618ff5bb73860068e2ea662c568a07e2f877a5f2be7e265919e
-  data.tar.gz: 61b49a4bd7eb304f44c041730bce733102f1936c9340cd9c0c3f1390bd9e853a
+  metadata.gz: 2585ca0684a0d6af6beaae99ab7b5250eb928c5200cbfb6274dcd2cbc914dccc
+  data.tar.gz: fe4681abb0c63e7d8fd0e8a777abaed95e578bbb0cfd7238bacbe9beecd3d07c
 SHA512:
-  metadata.gz: fd0242fc13696c620ffd3a7383b9088aaddade1f7280c4f35c54d5165f66e26b783e88fa2d65ced002e8f233af0637e43e9adb68ad7556a16cfdcbb6aff9ba7e
-  data.tar.gz: a1206eaa056ed93639c00c03eb16be5fbbec6b077e1677a916b69c6244c26d3c2cce51299dc0394335c54d12de500a9648f30fa6f3785ef749be528904d5ae60
+  metadata.gz: 9a02c3f87eea7ea989f73032388f4abe69b30055f1ea7a96aa302385a2f6038eb833cffe00b15e5b69124d7a11bf8dd031067a4a16b415999955b8ce393f25d4
+  data.tar.gz: e696df2343cab6ee82af95bedcfc45457b69287113f819514851ef5f943a2f0f8c165b21abff0f944dcd96d4cab45b8faa1190940bf36c294c9c98d55bd9d980

data/CHANGELOG.md

@@ -1,3 +1,28 @@
+## 0.1.5 (2021-03-14)
+
+- Updated Bling Fire to 0.1.5
+- Added ARM shared library for Linux
+
+## 0.1.4 (2020-12-28)
+
+- Added ARM shared library for Mac
+
+## 0.1.3 (2020-10-01)
+
+- Added `text_to_words_with_offsets` method
+- Added `text_to_sentences_with_offsets` method
+- Added `text_to_ids_with_offsets` method
+- Added `normalize_spaces` method
+
+## 0.1.2 (2020-06-25)
+
+- Updated Bling Fire to 0.1.3
+
+## 0.1.1 (2020-05-01)
+
+- Updated Bling Fire to 0.1.1
+- Improved error message when model not found
+
 ## 0.1.0 (2020-02-24)
 
 - First release

data/LICENSE.txt

@@ -1,22 +1,22 @@
-Copyright (c) 2020 Andrew Kane
-
 MIT License
 
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
+Copyright (c) Microsoft Corporation. All rights reserved.
+Copyright (c) 2020 Andrew Kane
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
 
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
 
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE

data/README.md

@@ -1,6 +1,8 @@
-# BlingFire
+# Bling Fire
 
-[BlingFire](https://github.com/microsoft/BlingFire) - high speed text tokenization - for Ruby
+[Bling Fire](https://github.com/microsoft/BlingFire) - high speed text tokenization - for Ruby
+
+[![Build Status](https://github.com/ankane/blingfire/workflows/build/badge.svg?branch=master)](https://github.com/ankane/blingfire/actions)
 
 ## Installation
 
@@ -30,14 +32,30 @@ Tokenize sentences
 model.text_to_sentences(text)
 ```
 
+Get offsets for words
+
+```ruby
+words, start_offsets, end_offsets = model.text_to_words_with_offsets(text)
+```
+
+Get offsets for sentences
+
+```ruby
+sentences, start_offsets, end_offsets = model.text_to_sentences_with_offsets(text)
+```
+
 ## Pre-trained Models
 
-BlingFire comes with a default model that follows the tokenization logic of NLTK with a few changes. You can also download other models:
+Bling Fire comes with a default model that follows the tokenization logic of NLTK with a few changes. You can also download other models:
 
-- [BERT Base](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/bert_base_tok.bin)
-- [BERT Base Cased](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/bert_base_cased_tok.bin)
-- [BERT Chinese](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/bert_chinese.bin)
-- [BERT Multilingual Cased](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/bert_multi_cased.bin)
+- [BERT Base](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/bert_base_tok.bin), [BERT Base Cased](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/bert_base_cased_tok.bin), [BERT Chinese](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/bert_chinese.bin), [BERT Multilingual Cased](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/bert_multi_cased.bin)
+- [GPT-2](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/gpt2.bin)
+- [Laser 100k](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/laser100k.bin), [Laser 250k](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/laser250k.bin), [Laser 500k](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/laser500k.bin)
+- [RoBERTa](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/roberta.bin)
+- [Syllab](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/syllab.bin)
+- [URI 100k](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/uri100k.bin), [URI 250k](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/uri250k.bin), [URI 500k](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/uri500k.bin)
+- [XLM-RoBERTa](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/xlm_roberta_base.bin)
+- [XLNet](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/xlnet.bin), [XLNet No Norm](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/xlnet_nonorm.bin)
 - [WBD](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/wbd_chuni.bin)
 
 Load a model
@@ -52,6 +70,12 @@ Convert text to ids
 model.text_to_ids(text)
 ```
 
+Get offsets for ids
+
+```ruby
+ids, start_offsets, end_offsets = model.text_to_ids_with_offsets(text)
+```
+
 ## History
 
 View the [changelog](https://github.com/ankane/blingfire/blob/master/CHANGELOG.md)
@@ -71,6 +95,6 @@ To get started with development:
 git clone https://github.com/ankane/blingfire.git
 cd blingfire
 bundle install
-bundle exec rake vendor:all
+bundle exec rake vendor:all download:models
 bundle exec rake test
 ```

data/lib/blingfire.rb

@@ -15,9 +15,17 @@ module BlingFire
     if Gem.win_platform?
       "blingfiretokdll.dll"
     elsif RbConfig::CONFIG["host_os"] =~ /darwin/i
-      "libblingfiretokdll.dylib"
+      if RbConfig::CONFIG["host_cpu"] =~ /arm/i
+        "libblingfiretokdll.arm64.dylib"
+      else
+        "libblingfiretokdll.dylib"
+      end
     else
-      "libblingfiretokdll.so"
+      if RbConfig::CONFIG["host_cpu"] =~ /aarch64/i
+        "libblingfiretokdll.arm64.so"
+      else
+        "libblingfiretokdll.so"
+      end
     end
   vendor_lib = File.expand_path("../vendor/#{lib_name}", __dir__)
   self.ffi_lib = [vendor_lib]
@@ -35,57 +43,144 @@ module BlingFire
     end
 
     def text_to_words(text)
-      text = encode_utf8(text.dup) unless text.encoding == Encoding::UTF_8
-      out = Fiddle::Pointer.malloc(text.bytesize * 3)
-      out_size = FFI.TextToWords(text, text.bytesize, out, out.size)
-      check_status out_size
-      encode_utf8(out[0, out_size - 1]).split(" ")
+      text_to(text, " ") do |t, out|
+        FFI.TextToWords(t, t.bytesize, out, out.size)
+      end
     end
 
     def text_to_words_with_model(model, text)
-      text = encode_utf8(text.dup) unless text.encoding == Encoding::UTF_8
-      out = Fiddle::Pointer.malloc(text.bytesize * 3)
-      out_size = FFI.TextToWordsWithModel(text, text.bytesize, out, out.size, model)
-      check_status out_size
-      encode_utf8(out[0, out_size - 1]).split(" ")
+      text_to(text, " ") do |t, out|
+        FFI.TextToWordsWithModel(t, t.bytesize, out, out.size, model)
+      end
+    end
+
+    def text_to_words_with_offsets(text)
+      text_to_with_offsets(text, " ") do |t, out, start_offsets, end_offsets|
+        FFI.TextToWordsWithOffsets(t, t.bytesize, out, start_offsets, end_offsets, out.size)
+      end
+    end
+
+    def text_to_words_with_offsets_with_model(model, text)
+      text_to_with_offsets(text, " ") do |t, out, start_offsets, end_offsets|
+        FFI.TextToWordsWithOffsetsWithModel(t, t.bytesize, out, start_offsets, end_offsets, out.size, model)
+      end
     end
 
     def text_to_sentences(text)
-      text = encode_utf8(text.dup) unless text.encoding == Encoding::UTF_8
-      out = Fiddle::Pointer.malloc(text.bytesize * 3)
-      out_size = FFI.TextToSentences(text, text.bytesize, out, out.size)
-      check_status out_size
-      encode_utf8(out[0, out_size - 1]).split("\n")
+      text_to(text, "\n") do |t, out|
+        FFI.TextToSentences(t, t.bytesize, out, out.size)
+      end
     end
 
     def text_to_sentences_with_model(model, text)
-      text = encode_utf8(text.dup) unless text.encoding == Encoding::UTF_8
-      out = Fiddle::Pointer.malloc(text.bytesize * 3)
-      out_size = FFI.TextToSentencesWithModel(text, text.bytesize, out, out.size, model)
-      check_status out_size
-      encode_utf8(out[0, out_size - 1]).split("\n")
+      text_to(text, "\n") do |t, out|
+        FFI.TextToSentencesWithModel(t, t.bytesize, out, out.size, model)
+      end
+    end
+
+    def text_to_sentences_with_offsets(text)
+      text_to_with_offsets(text, "\n") do |t, out, start_offsets, end_offsets|
+        FFI.TextToSentencesWithOffsets(t, t.bytesize, out, start_offsets, end_offsets, out.size)
+      end
+    end
+
+    def text_to_sentences_with_offsets_with_model(model, text)
+      text_to_with_offsets(text, "\n") do |t, out, start_offsets, end_offsets|
+        FFI.TextToSentencesWithOffsetsWithModel(t, t.bytesize, out, start_offsets, end_offsets, out.size, model)
+      end
     end
 
     def text_to_ids(model, text, max_len = nil, unk_id = 0)
       text = encode_utf8(text.dup) unless text.encoding == Encoding::UTF_8
       ids = Fiddle::Pointer.malloc((max_len || text.size) * Fiddle::SIZEOF_INT)
       out_size = FFI.TextToIds(model, text, text.bytesize, ids, ids.size, unk_id)
-      check_status out_size
+      check_status out_size, ids
       ids[0, (max_len || out_size) * Fiddle::SIZEOF_INT].unpack("i!*")
     end
 
+    def text_to_ids_with_offsets(model, text, max_len = nil, unk_id = 0)
+      text = encode_utf8(text.dup) unless text.encoding == Encoding::UTF_8
+      ids = Fiddle::Pointer.malloc((max_len || text.size) * Fiddle::SIZEOF_INT)
+
+      start_offsets = Fiddle::Pointer.malloc(Fiddle::SIZEOF_INT * ids.size)
+      end_offsets = Fiddle::Pointer.malloc(Fiddle::SIZEOF_INT * ids.size)
+
+      out_size = FFI.TextToIdsWithOffsets(model, text, text.bytesize, ids, start_offsets, end_offsets, ids.size, unk_id)
+
+      check_status out_size, ids
+
+      result = ids[0, (max_len || out_size) * Fiddle::SIZEOF_INT].unpack("i!*")
+      [result].concat(unpack_offsets(start_offsets, end_offsets, result, text))
+    end
+
     def free_model(model)
       FFI.FreeModel(model)
     end
 
+    def normalize_spaces(text)
+      u_space = 0x20
+      text = encode_utf8(text.dup) unless text.encoding == Encoding::UTF_8
+      out = Fiddle::Pointer.malloc([text.bytesize * 1.5, 20].max)
+      out_size = FFI.NormalizeSpaces(text, text.bytesize, out, out.size, u_space)
+      check_status out_size, out
+      encode_utf8(out.to_str(out_size))
+    end
+
     private
 
-    def check_status(ret)
-      raise Error, "Bad status" if ret == -1
+    def check_status(ret, ptr)
+      raise Error, "Not enough memory allocated" if ret == -1 || ret > ptr.size
+    end
+
+    def text_to(text, sep)
+      text = encode_utf8(text.dup) unless text.encoding == Encoding::UTF_8
+      # TODO allocate less, and try again if needed
+      out = Fiddle::Pointer.malloc([text.bytesize * 3, 20].max)
+      out_size = yield(text, out)
+      check_status out_size, out
+      encode_utf8(out.to_str(out_size - 1)).split(sep)
+    end
+
+    def text_to_with_offsets(text, sep)
+      text = encode_utf8(text.dup) unless text.encoding == Encoding::UTF_8
+      # TODO allocate less, and try again if needed
+      out = Fiddle::Pointer.malloc([text.bytesize * 3, 20].max)
+
+      start_offsets = Fiddle::Pointer.malloc(Fiddle::SIZEOF_INT * out.size)
+      end_offsets = Fiddle::Pointer.malloc(Fiddle::SIZEOF_INT * out.size)
+
+      out_size = yield(text, out, start_offsets, end_offsets)
+
+      check_status out_size, out
+
+      result = encode_utf8(out.to_str(out_size - 1)).split(sep)
+      [result].concat(unpack_offsets(start_offsets, end_offsets, result, text))
     end
 
     def encode_utf8(text)
       text.force_encoding(Encoding::UTF_8)
     end
+
+    def unpack_offsets(start_offsets, end_offsets, result, text)
+      start_bytes = start_offsets.to_s(Fiddle::SIZEOF_INT * result.size).unpack("i*")
+      end_bytes = end_offsets.to_s(Fiddle::SIZEOF_INT * result.size).unpack("i*")
+      starts = []
+      ends = []
+
+      # convert byte offsets to character offsets
+      # TODO see if more efficient to store next_pos in variable
+      pos = 0
+      text.each_char.with_index do |c, i|
+        while pos == start_bytes[starts.size]
+          starts << i
+        end
+        pos += c.bytesize
+        while pos - 1 == end_bytes[ends.size]
+          ends << i + 1
+        end
+      end
+
+      [starts, ends]
+    end
   end
 end

data/lib/blingfire/ffi.rb

@@ -10,13 +10,35 @@ module BlingFire
       raise e
     end
 
+    # https://github.com/microsoft/BlingFire/blob/master/blingfiretools/blingfiretokdll/blingfiretokdll.cpp
+
+    # version
     extern "int GetBlingFireTokVersion()"
-    extern "void* LoadModel(char * pszLdbFileName)"
-    extern "int FreeModel(void* ModelPtr)"
+
+    # text to sentences
+    extern "int TextToSentencesWithOffsetsWithModel(char * pInUtf8Str, int InUtf8StrByteCount, char * pOutUtf8Str, int * pStartOffsets, int * pEndOffsets, int MaxOutUtf8StrByteCount, void * hModel)"
+    extern "int TextToSentencesWithOffsets(char * pInUtf8Str, int InUtf8StrByteCount, char * pOutUtf8Str, int * pStartOffsets, int * pEndOffsets, int MaxOutUtf8StrByteCount)"
+    extern "int TextToSentencesWithModel(char * pInUtf8Str, int InUtf8StrByteCount, char * pOutUtf8Str, int MaxOutUtf8StrByteCount, void * hModel)"
+    extern "int TextToSentences(char * pInUtf8Str, int InUtf8StrByteCount, char * pOutUtf8Str, int MaxOutUtf8StrByteCount)"
+
+    # text to words
+    extern "int TextToWordsWithOffsetsWithModel(char * pInUtf8Str, int InUtf8StrByteCount, char * pOutUtf8Str, int * pStartOffsets, int * pEndOffsets, int MaxOutUtf8StrByteCount, void * hModel)"
+    extern "int TextToWordsWithOffsets(char * pInUtf8Str, int InUtf8StrByteCount, char * pOutUtf8Str, int * pStartOffsets, int * pEndOffsets, int MaxOutUtf8StrByteCount)"
     extern "int TextToWordsWithModel(char * pInUtf8Str, int InUtf8StrByteCount, char * pOutUtf8Str, int MaxOutUtf8StrByteCount, void * hModel)"
     extern "int TextToWords(char * pInUtf8Str, int InUtf8StrByteCount, char * pOutUtf8Str, int MaxOutUtf8StrByteCount)"
+
+    # misc
+    extern "int NormalizeSpaces(char * pInUtf8Str, int InUtf8StrByteCount, char * pOutUtf8Str, int MaxOutUtf8StrByteCount, int uSpace)"
+    extern "int TextToHashes(char * pInUtf8Str, int InUtf8StrByteCount, int32_t * pHashArr, int MaxHashArrLength, int wordNgrams, int bucketSize)"
+
+    # model
+    extern "void* LoadModel(char * pszLdbFileName)"
+
+    # text to ids
+    extern "int TextToIdsWithOffsets(void* ModelPtr, char * pInUtf8Str, int InUtf8StrByteCount, int32_t * pIdsArr, int * pStartOffsets, int * pEndOffsets, int MaxIdsArrLength, int UnkId)"
     extern "int TextToIds(void* ModelPtr, char * pInUtf8Str, int InUtf8StrByteCount, int32_t * pIdsArr, int MaxIdsArrLength, int UnkId)"
-    extern "int TextToSentences(char * pInUtf8Str, int InUtf8StrByteCount, char * pOutUtf8Str, int MaxOutUtf8StrByteCount)"
-    extern "int TextToSentencesWithModel(char * pInUtf8Str, int InUtf8StrByteCount, char * pOutUtf8Str, int MaxOutUtf8StrByteCount, void * hModel)"
+
+    # free model
+    extern "int FreeModel(void* ModelPtr)"
   end
 end

data/lib/blingfire/model.rb

@@ -1,7 +1,9 @@
 module BlingFire
   class Model
     def initialize(path = nil)
+      @handle = nil
       if path
+        raise Error, "Model not found" unless File.exist?(path)
         @handle = FFI.LoadModel(path)
         ObjectSpace.define_finalizer(self, self.class.finalize(@handle))
       end
@@ -15,6 +17,14 @@ module BlingFire
       end
     end
 
+    def text_to_words_with_offsets(text)
+      if @handle
+        BlingFire.text_to_words_with_offsets_with_model(@handle, text)
+      else
+        BlingFire.text_to_words_with_offsets(text)
+      end
+    end
+
     def text_to_sentences(text)
       if @handle
         BlingFire.text_to_sentences_with_model(@handle, text)
@@ -23,6 +33,14 @@ module BlingFire
       end
     end
 
+    def text_to_sentences_with_offsets(text)
+      if @handle
+        BlingFire.text_to_sentences_with_offsets_with_model(@handle, text)
+      else
+        BlingFire.text_to_sentences_with_offsets(text)
+      end
+    end
+
     def text_to_ids(text, max_len = nil, unk_id = 0)
       if @handle
         BlingFire.text_to_ids(@handle, text, max_len, unk_id)
@@ -31,6 +49,14 @@ module BlingFire
       end
     end
 
+    def text_to_ids_with_offsets(text, max_len = nil, unk_id = 0)
+      if @handle
+        BlingFire.text_to_ids_with_offsets(@handle, text, max_len, unk_id)
+      else
+        raise "Not implemented"
+      end
+    end
+
     def to_ptr
       @handle
     end

data/lib/blingfire/version.rb

@@ -1,3 +1,3 @@
 module BlingFire
-  VERSION = "0.1.0"
+  VERSION = "0.1.5"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: blingfire
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.5
 platform: ruby
 authors:
 - Andrew Kane
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-02-24 00:00:00.000000000 Z
+date: 2021-03-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -52,7 +52,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '5'
-description: 
+description:
 email: andrew@chartkick.com
 executables: []
 extensions: []
@@ -67,13 +67,15 @@ files:
 - lib/blingfire/version.rb
 - vendor/LICENSE
 - vendor/blingfiretokdll.dll
+- vendor/libblingfiretokdll.arm64.dylib
+- vendor/libblingfiretokdll.arm64.so
 - vendor/libblingfiretokdll.dylib
 - vendor/libblingfiretokdll.so
 homepage: https://github.com/ankane/blingfire
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -88,8 +90,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.2.3
+signing_key:
 specification_version: 4
 summary: High speed text tokenization for Ruby
 test_files: []