jekyll-chatgpt-translate 0.0.37 → 0.0.39

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c0f1eb7be508b40853f8fc04d94cd0c6f30ddfe2be81b6d3fc7f6a45101d10b1
-  data.tar.gz: cb4d6e55f8d93e00307135f52cfba873a7ece62a01cb785f0d0e4ada293aaa20
+  metadata.gz: b4ab44177f3ba1b8eb9fb9f705734c130a5aa50447fa3e40f746d43a0fbb2c74
+  data.tar.gz: 2516780138d056a38a1f8aac70fad9e4606043fb6d9808fdf7224d4e3a3e08c4
 SHA512:
-  metadata.gz: 053d40f293680a7b4ec3f78f5b09b1ddef17183b1dbfce93ab10fffca3cbb1b1384644e4da3377c5c1abc0ff96fdb1b55d8f58d7e30788aa7ddeba44dc6e0d65
-  data.tar.gz: 0cc7dd560f374a400b8c8ff2b7782daca793a2db48eac1fc698806506f0341713751241a2feb739d485145efbe57ba6c5d2d43d188208fa0e8142993447a6928
+  metadata.gz: 7b3dba562f9860cfb1c00198770ed637a75c85216e87b9f36bb4520d212ed8cdfb9b3332601571addab38b81011b92a24408c4142fe31130d75f94d09a4f62d6
+  data.tar.gz: b679a82e6e19e3954c3676c6b81ef240fadda14aa06c490577bcb51d2c5f9d39c7ee8b708cc998617101be95c4d3747a29976b006e7cdb8ed68dedb72d4ce482

data/README.md

@@ -79,6 +79,9 @@ Full list of options available to specify in `_config.yml`:
     doesn't translate short paragraphs pretty enough. It's better to keep this
     number big enough, to avoid silly translations. The default is 128.
 
+  * `window_length` (optional) — maximum number of words to be sent to OpenAI API in one
+    request. The default is 2048.
+
   * `layout` (optional) — is name of the file in `_layouts` directory, without the extension. 
     This layout will be specified for the pages generated by this plugin.
     The default value is `translated` (expecting you to have `_layouts/translated.html` file available).

data/jekyll-chatgpt-translate.gemspec

@@ -28,7 +28,7 @@ Gem::Specification.new do |s|
   s.required_rubygems_version = Gem::Requirement.new('>= 0') if s.respond_to? :required_rubygems_version=
   s.required_ruby_version = '>= 2.6'
   s.name = 'jekyll-chatgpt-translate'
-  s.version = '0.0.37'
+  s.version = '0.0.39'
   s.license = 'MIT'
   s.summary = 'Translate Jekyll Pages Through ChatGPT'
   s.description = [
@@ -48,5 +48,6 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'jekyll', '>= 3'
   s.add_runtime_dependency 'redcarpet', '>= 3'
   s.add_runtime_dependency 'ruby-openai', '>= 5'
+  s.add_runtime_dependency 'tiktoken_ruby', '>= 0.0.6'
   s.metadata['rubygems_mfa_required'] = 'true'
 end

data/lib/jekyll-chatgpt-translate/chatgpt.rb

@@ -25,6 +25,7 @@
 require 'jekyll'
 require 'openai'
 require 'iso-639'
+require 'tiktoken_ruby'
 require_relative 'pars'
 require_relative 'prompt'
 
@@ -92,11 +93,16 @@ class GptTranslate::ChatGPT
       end
       accum = []
       until later[i].nil?
+        already = Tiktoken.encoding_for_model('gpt-4').encode(accum.join).length
+        if already > window_length
+          Jekyll.logger.debug("Already #{already} words, over the window_length of #{window_length}")
+          break
+        end
         accum << later[i]
-        break if accum.join.split.count > window_length
         i += 1
       end
       out << translate_pars(accum)
+      i += 1
     end
     out.join("\n\n")
   end

data/lib/jekyll-chatgpt-translate/generator.rb

@@ -44,6 +44,7 @@ class GptTranslate::Generator < Jekyll::Generator
 
   # Main plugin action, called by Jekyll-core
   def generate(site)
+    Jekyll.logger.info("jekyll-chatgpt-translate #{GptTranslate::VERSION} starting...")
     config ||= site.config['chatgpt-translate'] || {}
     home = config['tmpdir'] || '_chatgpt-translate'
     key = api_key(config)
@@ -113,7 +114,7 @@ class GptTranslate::Generator < Jekyll::Generator
             config['source'] || 'en',
             lang
           )
-          foreign = gpt.translate(plain, min: min_chars)
+          foreign = gpt.translate(plain, min: min_chars, window_length: config['window_length'] || '2048')
           File.write(
             path,
             [

data/lib/jekyll-chatgpt-translate/version.rb

@@ -23,5 +23,5 @@
 # SOFTWARE.
 
 module GptTranslate
-  VERSION = '0.0.37'
+  VERSION = '0.0.39'
 end

data/test/test_chatgpt.rb

@@ -97,6 +97,18 @@ class GptTranslate::ChatGPTTest < Minitest::Test
     assert_equal('done!', chat.translate('This is the text to send to OpenAI'))
   end
 
+  def test_through_small_window
+    stub_it!
+    chat = GptTranslate::ChatGPT.new('fake-key', 'gpt-3.5-turbo', 'en', 'ru')
+    assert_equal(
+      "done!\n\ndone!",
+      chat.translate(
+        "This is the first paragraph\n\nThis is second\n\nThis is third",
+        min: 1, window_length: 4
+      )
+    )
+  end
+
   private
 
   def stub_it!

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: jekyll-chatgpt-translate
 version: !ruby/object:Gem::Version
-  version: 0.0.37
+  version: 0.0.39
 platform: ruby
 authors:
 - Yegor Bugayenko
@@ -94,6 +94,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '5'
+- !ruby/object:Gem::Dependency
+  name: tiktoken_ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.6
 description: Add this plugin to your Jekyll site and all posts will be automatically
   translated to the languages of your choice through ChatGPT
 email: yegor256@gmail.com