sentencepiece 0.0.2 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 52d4bb0f1a0b4a68c9db252911fe0661b7d8042d2d976a17506e0eaf0005d48f
-  data.tar.gz: 793b8b3e47cb6a9c1ab6b05b4cbef264b16eef7b632699983de9f8c40056d9ac
+  metadata.gz: 63aaf69a72781f0087072db8197b9ae3a126b63e632b52bded5b483ff7b2c6b4
+  data.tar.gz: 5ffbd48ed587ef57fd3b427615cc21f4d79804ea90598f9db2178ceff702ca57
 SHA512:
-  metadata.gz: 9702468ce33efdf7ae2ac3735cec983ae58a50677357a2d6cbff88089e73bf40b81ef9872b9860474c948fed3f5920a093252e90fbdea2d5e188c41057c7923e
-  data.tar.gz: ba262d347b32364255ecbbce8306b2d126296b46e6aac7828903c0dc9d6485702ef153ed7aea83a79eb69b5bfc655482787f6bb1c3ad086215f9106658a3a3e8
+  metadata.gz: 61b91c838bbdb2f7c47166036b890b545060361e91a66e6cf416206f53d84595cc74052a38df083ad507a6f219d0b2b8ba3667095d2d3514b5ba0c3660967ee5
+  data.tar.gz: 467c2f252e23fbfe684486d42af9fe0eedd058dd99361fc5652bb83412ae801571e5e25879b2f1bb6734c997b9c04f4287378c595afa3698b06885631c8fdcae

data/CHANGELOG.md

@@ -1,5 +1,10 @@
 ## [Unreleased]
 
+## [0.1.0] - 2023-03-26
+
+- Add API documentation.
+- Add type signatures.
+
 ## [0.0.2] - 2023-03-26
 
 - Add SentencePieceTrainer class.

data/README.md

@@ -3,12 +3,11 @@
 [![Build Status](https://github.com/yoshoku/sentencepiece.rb/actions/workflows/main.yml/badge.svg)](https://github.com/yoshoku/sentencepiece.rb/actions/workflows/main.yml)
 [![Gem Version](https://badge.fury.io/rb/sentencepiece.svg)](https://badge.fury.io/rb/sentencepiece)
 [![License](https://img.shields.io/badge/License-Apache%202.0-yellowgreen.svg)](https://github.com/yoshoku/sentencepiece.rb/blob/main/LICENSE.txt)
+[![Documentation](https://img.shields.io/badge/api-reference-blue.svg)](https://yoshoku.github.io/sentencepiece.rb/doc/)
 
 sentencepiece.rb provides Ruby bindings for the [SentencePiece](https://github.com/google/sentencepiece),
 an unsupervised text tokenizer and detokenizer for neural network-based text generation.
 
-It is still **under development** and may undergo many changes in the future.
-
 ## Installation
 
 Install SentencePiece using your OS package manager;

data/lib/sentencepiece/version.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+# sentencepiece.rb provides Ruby bindings for the SentencePiece.
 module SentencePiece
-  VERSION = '0.0.2'
+  # The version of sentencepiece.rb you install.
+  VERSION = '0.1.0'
 end

data/sig/sentencepiece.rbs

@@ -1,4 +1,63 @@
 module SentencePiece
   VERSION: String
-  # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+
+  class SentencePieceTrainer
+    def self.train: (String args) -> void
+  end
+
+  class SentencePieceProcessor
+    def initialize: (?model_file: String model_file) -> void
+
+    def load: (String model_file) -> void
+
+    def encode: (String text, ?out_type: String out_type) -> (Array[Integer] | Array[String])
+              | (Array[String] text, ?out_type: String out_type) -> (Array[Array[Integer]] | Array[Array[String]])
+
+    def encode_as_ids: (String text) -> Array[Integer]
+
+    def encode_as_pieces: (String text) -> Array[String]
+
+    def encode_as_serialized_proto: (String text) -> String
+
+    def nbest_encode_as_ids: (String text, nbest_size: Integer nbest_size) -> Array[Array[Integer]]
+
+    def nbest_encode_as_pieces: (String text, nbest_size: Integer nbest_size) -> Array[Array[String]]
+
+    def nbest_encode_as_serialized_proto: (String text, nbest_size: Integer nbest_size) -> String
+
+    def sample_encode_as_ids: (String text, nbest_size: Integer nbest_size, alpha: Float alpha) -> Array[Integer]
+
+    def sample_encode_as_pieces: (String text, nbest_size: Integer nbest_size, alpha: Float alpha) -> Array[String]
+
+    def sample_encode_as_serialized_proto: (String text, nbest_size: Integer nbest_size, alpha: Float alpha) -> String
+
+    def decode: (Array[Integer], ?out_type: String out_type) -> String
+              | (Array[Array[Integer]], ?out_type: String out_type) -> Array[String]
+              | (Array[String], ?out_type: String out_type) -> String
+              | (Array[Array[String]], ?out_type: String out_type) -> Array[String]
+
+    def decode_ids: (Array[Integer]) -> String
+
+    def decode_ids_as_serialized_proto: (Array[Integer] ids) -> String
+
+    def decode_pieces: (Array[String]) -> String
+
+    def decode_pieces_as_serialized_proto: (Array[String] pieces) -> String
+
+    def id_to_piece: (Integer id) -> String
+                   | (Array[Integer] ids) -> Array[String]
+
+    def piece_to_id: (String piece) -> Integer
+                   | (Array[String] pieces) -> Array[Integer]
+
+    def piece_size: () -> Integer
+
+    def bos_id: () -> Integer
+
+    def eos_id: () -> Integer
+
+    def pad_id: () -> Integer
+
+    def unk_id: () -> Integer
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sentencepiece
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.1.0
 platform: ruby
 authors:
 - yoshoku
@@ -36,6 +36,7 @@ metadata:
   homepage_uri: https://github.com/yoshoku/sentencepiece.rb
   source_code_uri: https://github.com/yoshoku/sentencepiece.rb
   changelog_uri: https://github.com/yoshoku/sentencepiece.rb/blob/main/CHANGELOG.md
+  documentation_uri: https://yoshoku.github.io/sentencepiece.rb/doc/
   rubygems_mfa_required: 'true'
 post_install_message:
 rdoc_options: []