blingfire 0.1.6 → 0.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fa9f7ebf09e5745b6865d4c5645b36f42b7c484ed667ce493febee80887e89ad
-  data.tar.gz: 7fb6e8716138c35396964081b8ad60d7c01db88e71e4fbe219d0ea27a6eff94b
+  metadata.gz: 466b528fca6404415ad072b597cbb6fc96628a325504aba0e7becc4793b35b3f
+  data.tar.gz: c1e02e1f8c48578c6407ac9f7d7a996ecd78205ffdcaa3158144012f61d8922e
 SHA512:
-  metadata.gz: 4f1e20f3520f4be17af2df5db4b3e05b169756f1667dccb08dbafb89ee07f633862d16649fa8e3be505073ca58a11989cc5c9b99d3f571aae2eaff3d4053dd8c
-  data.tar.gz: 8f70ecb06ddcad466508a0debe63fd99e9a5cbab35e3702296c8ff522ffbb5075ad52b2ce68d2f316de6685256b756ff098bc5a2b29e807652b89de84f9b4661
+  metadata.gz: cbfd60e8e54b42a1e7258afcebb51d27029bf66a2969daf4badc62f3117d7113b8d02f99c3445513dc37f08698205c21ba97787ba761512166257438b73d3956
+  data.tar.gz: 1aa595a55116d5e084d31e087180ed04392d1b7d91932a875d53cab88bc72b3a1ae179c15094b9679aa9fa78dc4419afc70f144ba971343d0c465a7842ca634e

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+## 0.1.8 (2023-02-01)
+
+- Improved ARM detection
+
+## 0.1.7 (2021-09-24)
+
+- Updated Bling Fire to 0.1.8
+- Added `ids_to_text` method
+
 ## 0.1.6 (2021-06-07)
 
 - Updated Bling Fire to 0.1.7

data/README.md

@@ -1,15 +1,15 @@
-# Bling Fire
+# Bling Fire Ruby
 
 [Bling Fire](https://github.com/microsoft/BlingFire) - high speed text tokenization - for Ruby
 
-[![Build Status](https://github.com/ankane/blingfire/workflows/build/badge.svg?branch=master)](https://github.com/ankane/blingfire/actions)
+[![Build Status](https://github.com/ankane/blingfire-ruby/workflows/build/badge.svg?branch=master)](https://github.com/ankane/blingfire-ruby/actions)
 
 ## Installation
 
 Add this line to your application’s Gemfile:
 
 ```ruby
-gem 'blingfire'
+gem "blingfire"
 ```
 
 ## Getting Started
@@ -82,24 +82,38 @@ Disable prefix space
 model = BlingFire.load_model("roberta.bin", prefix: false)
 ```
 
+## Ids to Text [experimental]
+
+Load a model
+
+```ruby
+model = BlingFire.load_model("bert_base_tok.i2w")
+```
+
+Convert ids to text
+
+```ruby
+model.ids_to_text(ids)
+```
+
 ## History
 
-View the [changelog](https://github.com/ankane/blingfire/blob/master/CHANGELOG.md)
+View the [changelog](https://github.com/ankane/blingfire-ruby/blob/master/CHANGELOG.md)
 
 ## Contributing
 
 Everyone is encouraged to help improve this project. Here are a few ways you can help:
 
-- [Report bugs](https://github.com/ankane/blingfire/issues)
-- Fix bugs and [submit pull requests](https://github.com/ankane/blingfire/pulls)
+- [Report bugs](https://github.com/ankane/blingfire-ruby/issues)
+- Fix bugs and [submit pull requests](https://github.com/ankane/blingfire-ruby/pulls)
 - Write, clarify, or fix documentation
 - Suggest or add new features
 
 To get started with development:
 
 ```sh
-git clone https://github.com/ankane/blingfire.git
-cd blingfire
+git clone https://github.com/ankane/blingfire-ruby.git
+cd blingfire-ruby
 bundle install
 bundle exec rake vendor:all download:models
 bundle exec rake test

data/lib/blingfire/ffi.rb

@@ -45,5 +45,8 @@ module BlingFire
 
     # prefix
     extern "int SetNoDummyPrefix(void* ModelPtr, bool fNoDummyPrefix)"
+
+    # ids to text
+    extern "int IdsToText(void* ModelPtr, int32_t * pIdsArr, int IdsCount, char * pOutUtf8Str, int MaxOutUtf8StrByteCount, bool SkipSpecialTokens)"
   end
 end

data/lib/blingfire/model.rb

@@ -61,6 +61,14 @@ module BlingFire
       end
     end
 
+    def ids_to_text(ids, skip_special_tokens: true, output_buffer_size: nil)
+      if @handle
+        BlingFire.ids_to_text(@handle, ids, skip_special_tokens: skip_special_tokens, output_buffer_size: output_buffer_size)
+      else
+        raise "Not implemented"
+      end
+    end
+
     def to_ptr
       @handle
     end

data/lib/blingfire/version.rb

@@ -1,3 +1,3 @@
 module BlingFire
-  VERSION = "0.1.6"
+  VERSION = "0.1.8"
 end

data/lib/blingfire.rb

@@ -15,13 +15,13 @@ module BlingFire
     if Gem.win_platform?
       "blingfiretokdll.dll"
     elsif RbConfig::CONFIG["host_os"] =~ /darwin/i
-      if RbConfig::CONFIG["host_cpu"] =~ /arm/i
+      if RbConfig::CONFIG["host_cpu"] =~ /arm|aarch64/i
         "libblingfiretokdll.arm64.dylib"
       else
         "libblingfiretokdll.dylib"
       end
     else
-      if RbConfig::CONFIG["host_cpu"] =~ /aarch64/i
+      if RbConfig::CONFIG["host_cpu"] =~ /arm|aarch64/i
         "libblingfiretokdll.arm64.so"
       else
         "libblingfiretokdll.so"
@@ -113,6 +113,15 @@ module BlingFire
       [result].concat(unpack_offsets(start_offsets, end_offsets, result, text))
     end
 
+    def ids_to_text(model, ids, skip_special_tokens: true, output_buffer_size: nil)
+      output_buffer_size ||= ids.size * 32
+      c_ids = Fiddle::Pointer[ids.pack("i*")]
+      out = Fiddle::Pointer.malloc(output_buffer_size)
+      out_size = FFI.IdsToText(model, c_ids, ids.size, out, output_buffer_size, skip_special_tokens ? 1 : 0)
+      check_status out_size, out
+      encode_utf8(out.to_str(out_size - 1))
+    end
+
     def free_model(model)
       FFI.FreeModel(model)
     end

metadata

@@ -1,59 +1,17 @@
 --- !ruby/object:Gem::Specification
 name: blingfire
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.8
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-06-07 00:00:00.000000000 Z
-dependencies:
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: minitest
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '5'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '5'
+date: 2023-02-02 00:00:00.000000000 Z
+dependencies: []
 description:
-email: andrew@chartkick.com
+email: andrew@ankane.org
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -71,7 +29,7 @@ files:
 - vendor/libblingfiretokdll.arm64.so
 - vendor/libblingfiretokdll.dylib
 - vendor/libblingfiretokdll.so
-homepage: https://github.com/ankane/blingfire
+homepage: https://github.com/ankane/blingfire-ruby
 licenses:
 - MIT
 metadata: {}
@@ -90,7 +48,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.3
+rubygems_version: 3.4.1
 signing_key:
 specification_version: 4
 summary: High speed text tokenization for Ruby