sentencepiece 0.0.2 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 52d4bb0f1a0b4a68c9db252911fe0661b7d8042d2d976a17506e0eaf0005d48f
-  data.tar.gz: 793b8b3e47cb6a9c1ab6b05b4cbef264b16eef7b632699983de9f8c40056d9ac
+  metadata.gz: 45cc838c2108753b87e03e3aee4bf3665f1e8c6e0cd407e5a68355b44657c582
+  data.tar.gz: 14d29d47f2a78b57218ceaf45c12a30de270f000c5d6d756b6b66a767a2ee128
 SHA512:
-  metadata.gz: 9702468ce33efdf7ae2ac3735cec983ae58a50677357a2d6cbff88089e73bf40b81ef9872b9860474c948fed3f5920a093252e90fbdea2d5e188c41057c7923e
-  data.tar.gz: ba262d347b32364255ecbbce8306b2d126296b46e6aac7828903c0dc9d6485702ef153ed7aea83a79eb69b5bfc655482787f6bb1c3ad086215f9106658a3a3e8
+  metadata.gz: ac0b780ec734901f030bbfdcf91949401b40064d89480fa54d303d7d83f707e42e0ba897cf342ea4650402b350deac2da4c1017cac7c278adb1272e5cee0834e
+  data.tar.gz: b03e10826137a4d3d1002906af0e8e46014eafed08da76fb5494e7b5311deaf002b40b7771b7299ef1b39cb2349e1fa9f5f3262020479d2655a2ede9c3db4f03

data/CHANGELOG.md

@@ -1,6 +1,13 @@
-## [Unreleased]
+## [[0.2.0](https://github.com/yoshoku/sentencepiece.rb/compare/v0.1.0...v0.2.0)] - 2025-12-29
 
-## [0.0.2] - 2023-03-26
+- Change to specify `c++17` with -std option as recent sentencepiece uses `std::string_view`.
+
+## [[0.1.0](https://github.com/yoshoku/sentencepiece.rb/compare/v0.0.2...v0.1.0)] - 2023-03-26
+
+- Add API documentation.
+- Add type signatures.
+
+## [[0.0.2](https://github.com/yoshoku/sentencepiece.rb/compare/v0.0.1...v0.0.2)] - 2023-03-26
 
 - Add SentencePieceTrainer class.
 - Add some encoding and decoding methods to SentencePieceProcessor.

data/README.md

@@ -3,12 +3,11 @@
 [![Build Status](https://github.com/yoshoku/sentencepiece.rb/actions/workflows/main.yml/badge.svg)](https://github.com/yoshoku/sentencepiece.rb/actions/workflows/main.yml)
 [![Gem Version](https://badge.fury.io/rb/sentencepiece.svg)](https://badge.fury.io/rb/sentencepiece)
 [![License](https://img.shields.io/badge/License-Apache%202.0-yellowgreen.svg)](https://github.com/yoshoku/sentencepiece.rb/blob/main/LICENSE.txt)
+[![Documentation](https://img.shields.io/badge/api-reference-blue.svg)](https://yoshoku.github.io/sentencepiece.rb/doc/)
 
 sentencepiece.rb provides Ruby bindings for the [SentencePiece](https://github.com/google/sentencepiece),
 an unsupervised text tokenizer and detokenizer for neural network-based text generation.
 
-It is still **under development** and may undergo many changes in the future.
-
 ## Installation
 
 Install SentencePiece using your OS package manager;

data/ext/sentencepiece/extconf.rb

@@ -9,6 +9,10 @@ abort 'libsentencepiece_train is not found.' unless have_library('sentencepiece_
 # abort 'sentencepiece_processor.h is not found.' unless have_header('sentencepiece_processor.h')
 # abort 'sentencepiece_trainer.h is not found.' unless have_header('sentencepiece_trainer.h')
 
-$CXXFLAGS << ' -std=c++11'
+$CXXFLAGS << if /mswin/ =~ RUBY_PLATFORM
+               '/std:c++17'
+             else
+               ' -std=c++17'
+             end
 
 create_makefile('sentencepiece/sentencepiece')

data/ext/sentencepiece/sentencepiece.cpp

@@ -1,7 +1,7 @@
 /**
  * sentencepiece.rb provides Ruby bindings for the SentencePiece.
  *
- * Copyright (c) 2023 Atsushi Tatsuma
+ * Copyright (c) 2023-2025 Atsushi Tatsuma
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

data/ext/sentencepiece/sentencepiece.hpp

@@ -1,7 +1,7 @@
 /**
  * sentencepiece.rb provides Ruby bindings for the SentencePiece.
  *
- * Copyright (c) 2023 Atsushi Tatsuma
+ * Copyright (c) 2023-2025 Atsushi Tatsuma
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -79,6 +79,7 @@ public:
     rb_define_method(rb_cSentencePieceProcessor, "bos_id", RUBY_METHOD_FUNC(_sentencepiece_processor_bos_id), 0);
     rb_define_method(rb_cSentencePieceProcessor, "eos_id", RUBY_METHOD_FUNC(_sentencepiece_processor_eos_id), 0);
     rb_define_method(rb_cSentencePieceProcessor, "pad_id", RUBY_METHOD_FUNC(_sentencepiece_processor_pad_id), 0);
+    rb_define_alias(rb_cSentencePieceProcessor, "vocab_size", "piece_size");
     return rb_cSentencePieceProcessor;
   };
 

data/lib/sentencepiece/version.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+# sentencepiece.rb provides Ruby bindings for the SentencePiece.
 module SentencePiece
-  VERSION = '0.0.2'
+  # The version of sentencepiece.rb you install.
+  VERSION = '0.2.0'
 end

data/sig/sentencepiece.rbs

@@ -1,4 +1,63 @@
 module SentencePiece
   VERSION: String
-  # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+
+  class SentencePieceTrainer
+    def self.train: (String args) -> void
+  end
+
+  class SentencePieceProcessor
+    def initialize: (?model_file: String model_file) -> void
+
+    def load: (String model_file) -> void
+
+    def encode: (String text, ?out_type: String out_type) -> (Array[Integer] | Array[String])
+              | (Array[String] text, ?out_type: String out_type) -> (Array[Array[Integer]] | Array[Array[String]])
+
+    def encode_as_ids: (String text) -> Array[Integer]
+
+    def encode_as_pieces: (String text) -> Array[String]
+
+    def encode_as_serialized_proto: (String text) -> String
+
+    def nbest_encode_as_ids: (String text, nbest_size: Integer nbest_size) -> Array[Array[Integer]]
+
+    def nbest_encode_as_pieces: (String text, nbest_size: Integer nbest_size) -> Array[Array[String]]
+
+    def nbest_encode_as_serialized_proto: (String text, nbest_size: Integer nbest_size) -> String
+
+    def sample_encode_as_ids: (String text, nbest_size: Integer nbest_size, alpha: Float alpha) -> Array[Integer]
+
+    def sample_encode_as_pieces: (String text, nbest_size: Integer nbest_size, alpha: Float alpha) -> Array[String]
+
+    def sample_encode_as_serialized_proto: (String text, nbest_size: Integer nbest_size, alpha: Float alpha) -> String
+
+    def decode: (Array[Integer], ?out_type: String out_type) -> String
+              | (Array[Array[Integer]], ?out_type: String out_type) -> Array[String]
+              | (Array[String], ?out_type: String out_type) -> String
+              | (Array[Array[String]], ?out_type: String out_type) -> Array[String]
+
+    def decode_ids: (Array[Integer]) -> String
+
+    def decode_ids_as_serialized_proto: (Array[Integer] ids) -> String
+
+    def decode_pieces: (Array[String]) -> String
+
+    def decode_pieces_as_serialized_proto: (Array[String] pieces) -> String
+
+    def id_to_piece: (Integer id) -> String
+                   | (Array[Integer] ids) -> Array[String]
+
+    def piece_to_id: (String piece) -> Integer
+                   | (Array[String] pieces) -> Array[Integer]
+
+    def piece_size: () -> Integer
+
+    def bos_id: () -> Integer
+
+    def eos_id: () -> Integer
+
+    def pad_id: () -> Integer
+
+    def unk_id: () -> Integer
+  end
 end

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: sentencepiece
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.2.0
 platform: ruby
 authors:
 - yoshoku
-autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-03-26 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies: []
 description: |
   sentencepiece.rb provides Ruby bindings for the SentencePiece,
@@ -36,8 +35,8 @@ metadata:
   homepage_uri: https://github.com/yoshoku/sentencepiece.rb
   source_code_uri: https://github.com/yoshoku/sentencepiece.rb
   changelog_uri: https://github.com/yoshoku/sentencepiece.rb/blob/main/CHANGELOG.md
+  documentation_uri: https://yoshoku.github.io/sentencepiece.rb/doc/
   rubygems_mfa_required: 'true'
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -52,8 +51,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.26
-signing_key:
+rubygems_version: 4.0.3
 specification_version: 4
 summary: Ruby bindings for the SentencePiece
 test_files: []