jekyll-chatgpt-translate 0.0.1 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 903d9dcdc683aff4816e02d7e49b699d3b8393a64716b162c5b46509efa77a7a
-  data.tar.gz: a202b7a057a6294046ba081ab8432ee6ef2a3c398a93805bf95b87c8fde76731
+  metadata.gz: 12d3fb457c7565b3726be719790ec577aa436e4725380aa1f7ce45b1dcc008d1
+  data.tar.gz: 03ce94a20742b6489b018f01bc19c309bc0eed8891a35e1c3737bf744187ed2f
 SHA512:
-  metadata.gz: 1e5ae442015d83a9b7f9a5408826dc1c0ed808abb1bf3b9b7339f87d40d7ba6b0e6518547d9bc707a00b7a98c30bc1d2e3fcf41ed576b175e215f764ed8dcf7d
-  data.tar.gz: 19e9825b50ecd0b72bf9acde8f330d47f1a00964516570e967d3dbe0e3d1077069dddcb6daac7eb1851bd8fcc2d52be0044a060320c83621fff3dac5a36a9da0
+  metadata.gz: 175718e03a2a9993edb00583b676132705a9af77ce16d0a460f28dcdc9a5bf2dfedf52d370519907e086997022a130589c0e95c74058911107fba8fb05f79fcb
+  data.tar.gz: 1d853b8ca20c5107dcceda0c2cb37e2eb83d50a968ebe902a0c9ba58862e68ef8fdc91ac887a666a6f155f819fa56cc2014a85b60bf48108e0b5062cccd98486

data/.rubocop.yml

@@ -18,11 +18,11 @@ Layout/MultilineMethodCallIndentation:
 Metrics/AbcSize:
   Enabled: false
 Metrics/BlockLength:
-  Max: 30
+  Max: 50
 Metrics/CyclomaticComplexity:
-  Max: 10
+  Max: 15
 Metrics/PerceivedComplexity:
-  Max: 10
+  Max: 15
 Layout/EmptyLineAfterGuardClause:
   Enabled: false
 Naming/FileName:

data/Gemfile

@@ -26,7 +26,7 @@ source 'https://rubygems.org'
 gemspec
 
 gem 'cucumber', '8.0.0', require: false
-gem 'jekyll', '3.9.2', require: false
+gem 'jekyll', '3.9.3', require: false
 gem 'kramdown-parser-gfm', '1.1.0', require: false
 gem 'minitest', '5.19.0', require: false
 gem 'rake', '13.0.6', require: false

data/README.md

@@ -1,6 +1,9 @@
 [![rake](https://github.com/yegor256/jekyll-chatgpt-translate/actions/workflows/rake.yml/badge.svg)](https://github.com/yegor256/jekyll-chatgpt-translate/actions/workflows/rake.yml)
 [![Gem Version](https://badge.fury.io/rb/jekyll-chatgpt-translate.svg)](http://badge.fury.io/rb/jekyll-chatgpt-translate)
 
+If you have a [Jekyll](https://jekyllrb.com/) static site, this plugin may help you automatically
+translate its pages to another language, through [ChatGPT](https://chat.openai.com/).
+
 Install it first:
 
 ```
@@ -11,6 +14,7 @@ Then, add this to `_config.yml`:
 
 ```yaml
 chatgpt-translate:
+  model: gpt-3.5-turbo
   source: en
   layout: translated
   targets: 
@@ -34,7 +38,28 @@ The plugin is compatible with
 [Jekyll 3.9.3](https://jekyllrb.com/news/2023/01/29/jekyll-3-9-3-released/) and 
 [Jekyll 4.3.2](https://jekyllrb.com/news/2023/01/20/jekyll-4-3-2-released/). 
 
-# How to Contribute
+## Options
+
+Full list of options available to specify in `_config.yml`:
+
+  * `model` (optional) --- specifies the model to use by ChatGPT.
+
+  * `source` (optional) --- is the ISO-839-1 code of the source language.
+
+  * `layout` (optional) --- is name of the file in `_layouts` directory, without the extension. 
+This layout will be specified for the pages generated by this plugin.
+
+  * `targets` (mandatory) --- an array of target languages, each of which has the following attributes
+
+    * `language` (mandatory) --- ISO-839-1 code of the target language
+
+    * `permalink` (mandatory) --- template to use for newly generated pages
+
+    * `layout` (optional) --- the name of the file in the `_layouts` directory
+
+  * `threshold` (optional) --- maximum number of pages to generate in one build cycle.
+
+## How to Contribute
 
 Test is like this:
 

data/jekyll-chatgpt-translate.gemspec

@@ -28,7 +28,7 @@ Gem::Specification.new do |s|
   s.required_rubygems_version = Gem::Requirement.new('>= 0') if s.respond_to? :required_rubygems_version=
   s.required_ruby_version = '>= 2.6'
   s.name = 'jekyll-chatgpt-translate'
-  s.version = '0.0.1'
+  s.version = '0.0.3'
   s.license = 'MIT'
   s.summary = 'Translate Jekyll Pages Through ChatGPT'
   s.description = [
@@ -42,9 +42,11 @@ Gem::Specification.new do |s|
   s.executables = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
   s.rdoc_options = ['--charset=UTF-8']
   s.extra_rdoc_files = %w[README.md LICENSE.txt]
-  s.add_runtime_dependency 'iso-639', '~> 0.3.6'
-  s.add_runtime_dependency 'jekyll', '~> 3.9.2'
-  s.add_runtime_dependency 'redcarpet', '~> 3.6.0'
-  s.add_runtime_dependency 'ruby-openai', '~> 5.0.0'
+  s.add_runtime_dependency 'iri', '~> 0.6'
+  s.add_runtime_dependency 'iso-639', '~> 0.3'
+  s.add_runtime_dependency 'jekyll', '~> 3.9'
+  s.add_runtime_dependency 'liquid', '~> 4.0'
+  s.add_runtime_dependency 'redcarpet', '~> 3.6'
+  s.add_runtime_dependency 'ruby-openai', '~> 5.0'
   s.metadata['rubygems_mfa_required'] = 'true'
 end

data/lib/jekyll-chatgpt-translate/chatgpt.rb

@@ -38,40 +38,40 @@ class GptTranslate::ChatGPT
   # +key+ OpenAI API Key, which can't be nil, but can be empty string, which means dry mode (no calls to OpenAI)
   # +source+ The language to translate from
   # +target+ The language to translate into
-  def initialize(key, source, target)
+  def initialize(key, model, source, target)
     raise 'OpenAI key cannot be nil' if key.nil?
     @key = key
+    @model = model
     @source = source
     @target = target
   end
 
   def translate(text)
-    model = 'gpt-3.5-turbo'
     text.split("\n\n").compact.map do |par|
       if par.length <= 32
         Jekyll.logger.debug("Not translating this, b/c too short: \"#{par}\"")
         par
-      elsif par !~ /^[А-Я]/
+      elsif par !~ /^[a-zA-Zа-яА-Я]/
         Jekyll.logger.debug("Not translating this, b/c it's not a plain text: \"#{par}\"")
         par
       elsif @key.empty?
         par
       else
-        translate_par(par, model)
+        translate_par(par)
       end
     end.join("\n\n")
   end
 
   private
 
-  def translate_par(par, model)
+  def translate_par(par)
     Time.now
     t = nil
     attempt = 0
     begin
       response = OpenAI::Client.new(access_token: @key).chat(
         parameters: {
-          model: model,
+          model: @model,
           messages: [{
             role: 'user',
             content: "#{prompt}:\n\n#{par}"
@@ -91,7 +91,7 @@ class GptTranslate::ChatGPT
     else
       Jekyll.logger.debug("Translated #{par.split.count} #{@source.upcase} words
 to #{t.split.count} #{@target.upcase} words
-through #{model} in #{(Time.now - pstart).round(2)}s")
+through #{@model} in #{(Time.now - pstart).round(2)}s")
       t
     end
   end

data/lib/jekyll-chatgpt-translate/generator.rb

@@ -24,7 +24,9 @@
 
 require 'jekyll'
 require_relative 'chatgpt'
+require_relative 'ping'
 require_relative 'plain'
+require_relative 'version'
 
 # The module we are in.
 module GptTranslate; end
@@ -52,14 +54,27 @@ but pages will be generated')
       return
     end
     layout = config['layout'] || 'translated'
+    threshold = config['threshold'] || 1_000_000_000
     start = Time.now
     total = 0
     site.posts.docs.each do |doc|
       plain = GptTranslate::Plain.new(doc.content).to_s
       config['targets'].each do |target|
+        link = permalink(doc, target['permalink'])
+        next if GptTranslate::Ping.new(site, link).exists?
         lang = target['language']
         raise 'Language must be defined for each target' if target.nil?
-        gpt = GptTranslate::ChatGPT.new(key, config['source'] || 'en', lang)
+        model = config['model'] || 'gpt-3.5-turbo'
+        if total >= threshold
+          Jekyll.logger.info("Already generated #{total} pages, that's enough for today")
+          break
+        end
+        gpt = GptTranslate::ChatGPT.new(
+          key,
+          model,
+          config['source'] || 'en',
+          lang
+        )
         translated = gpt.translate(plain)
         path = "_chatgpt-translated/#{doc.basename}"
         FileUtils.mkdir_p(File.dirname(path))
@@ -69,17 +84,20 @@ but pages will be generated')
             '---',
             "layout: #{target['layout'] || layout}",
             "title: #{doc.data['title']}",
-            "permalink: #{permalink(doc, target['permalink'])}",
+            "permalink: #{link}",
             '---',
             '',
-            translated
+            translated,
+            '',
+            "Translated by ChatGPT #{model}/#{GptTranslate::VERSION}\n{: .jekyll-chatgpt-translate}"
           ].join("\n")
         )
         site.pages << Jekyll::Page.new(site, site.source, File.dirname(path), File.basename(path))
         total += 1
       end
+      break if total >= threshold
     end
-    puts "#{total} pages generated in #{(Time.now - start).round(2)}s"
+    Jekyll.logger.info("#{total} pages generated in #{(Time.now - start).round(2)}s")
   end
 
   private

data/lib/jekyll-chatgpt-translate/ping.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+# (The MIT License)
+#
+# Copyright (c) 2023 Yegor Bugayenko
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the 'Software'), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+require 'iri'
+require 'net/http'
+require 'uri'
+require_relative 'version'
+
+# see https://stackoverflow.com/a/6048451/187141
+require 'openssl'
+OpenSSL::SSL::VERIFY_PEER = OpenSSL::SSL::VERIFY_NONE
+
+# The module we are in.
+module GptTranslate; end
+
+# Ping one page of a site.
+# Author:: Yegor Bugayenko (yegor256@gmail.com)
+# Copyright:: Copyright (c) 2023 Yegor Bugayenko
+# License:: MIT
+class GptTranslate::Ping
+  # Ctor.
+  def initialize(site, link)
+    @site = site
+    @link = link
+  end
+
+  def exists?
+    home = @site.config['url']
+    return false if home.nil?
+    uri = Iri.new(home).path(@link)
+    before = Net::HTTP.get_response(URI(uri.to_s))
+    if before.is_a?(Net::HTTPSuccess) && before.body.include?("/#{GptTranslate::VERSION}")
+      Jekyll.logger.info("No need to translate, page exists at #{uri} (#{before.body.split.count} words)")
+      return true
+    end
+    Jekyll.logger.debug("GET #{uri}: #{before.code}")
+    false
+  end
+end

data/lib/jekyll-chatgpt-translate/plain.rb

@@ -23,6 +23,7 @@
 # SOFTWARE.
 
 require 'redcarpet'
+require 'liquid'
 
 # The module we are in.
 module GptTranslate; end
@@ -42,19 +43,30 @@ class GptTranslate::Plain
       par.gsub!("\n", ' ')
       par.gsub!(/\s{2,}/, ' ')
       par.strip!
-      next if par.start_with?('<')
-      next if par.start_with?('{%')
+      liquid = Liquid::Template.parse(par)
+      par = liquid.render({}, drop: NullDrop.new, registers: { removed: true })
       Redcarpet::Markdown.new(Strip).render(par)
     end.join("\n\n").gsub(/\n{2,}/, "\n\n").strip
   end
 
+  # To ignore/remove Liquid tags.
+  class NullDrop < Liquid::Drop
+    def method_missing(*)
+      nil
+    end
+
+    def respond_to_missing?(*)
+      true
+    end
+  end
+
   # Markdown to pain text.
   class Strip < Redcarpet::Render::Base
     %i[
       block_code block_quote
       block_html
       autolink codespan double_emphasis
-      emphasis underline raw_html
+      emphasis underline
       triple_emphasis strikethrough
       superscript highlight quote
       footnotes footnote_def footnote_ref
@@ -65,6 +77,10 @@ class GptTranslate::Plain
       end
     end
 
+    def raw_html(_content)
+      'HTML'
+    end
+
     def list(content, _type)
       content
     end

data/lib/jekyll-chatgpt-translate/version.rb

@@ -22,6 +22,6 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
-module Jekyll::Translate
+module GptTranslate
   VERSION = '0.0.1'
 end

data/test/test_chatgpt.rb

@@ -31,17 +31,17 @@ require_relative '../lib/jekyll-chatgpt-translate/chatgpt'
 # License:: MIT
 class GptTranslate::ChatGPTTest < Minitest::Test
   def test_short_text
-    chat = GptTranslate::ChatGPT.new('fake-key', 'en', 'ru')
+    chat = GptTranslate::ChatGPT.new('fake-key', 'gpt-3.5-turbo', 'en', 'ru')
     assert_equal('Hello, world!', chat.translate('Hello, world!'))
   end
 
   def test_dry_mode
-    chat = GptTranslate::ChatGPT.new('', 'en', 'ru')
+    chat = GptTranslate::ChatGPT.new('', 'gpt-3.5-turbo', 'en', 'ru')
     assert_equal(38, chat.translate('This text should not be sent to OpenAI').length)
   end
 
   def test_markup
-    chat = GptTranslate::ChatGPT.new('fake-key', 'en', 'ru')
+    chat = GptTranslate::ChatGPT.new('fake-key', 'gpt-3.5-turbo', 'en', 'ru')
     assert_equal('<img src="a"/>', chat.translate('<img src="a"/>'))
   end
 end

data/test/test_ping.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+# (The MIT License)
+#
+# Copyright (c) 2023 Yegor Bugayenko
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the 'Software'), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+require 'minitest/autorun'
+require 'jekyll'
+require_relative '../lib/jekyll-chatgpt-translate/ping'
+
+# Ping test.
+# Author:: Yegor Bugayenko (yegor256@gmail.com)
+# Copyright:: Copyright (c) 2023 Yegor Bugayenko
+# License:: MIT
+class GptTranslate::PingTest < Minitest::Test
+  class FakeSite
+    attr_reader :config
+
+    def initialize(config)
+      @config = config
+    end
+  end
+
+  def test_when_exists
+    site = FakeSite.new({ 'url' => 'https://www.yegor256.com/' })
+    ping = GptTranslate::Ping.new(site, '/about-me.html')
+    assert(!ping.exists?)
+  end
+end

data/test/test_plain.rb

@@ -53,4 +53,33 @@ class GptTranslate::PlainTest < Minitest::Test
       GptTranslate::Plain.new('Hello, [dude](https://www.google.com)!').to_s
     )
   end
+
+  def test_code
+    assert_equal(
+      'Hello, Java!',
+      GptTranslate::Plain.new('Hello, `Java`!').to_s
+    )
+  end
+
+  def test_code_block
+    assert_equal(
+      "Hello:\n\nJava",
+      GptTranslate::Plain.new("Hello:\n\n```\nJava\n```\n").to_s
+    )
+  end
+
+  def test_html
+    assert_equal(
+      'This is picture: HTML!',
+      GptTranslate::Plain.new('This is picture: <img src="a"/>!').to_s
+    )
+    assert_equal('HTML', GptTranslate::Plain.new('<img src="a"/>').to_s)
+  end
+
+  def test_liquid_tags
+    assert_equal(
+      'Hello, !',
+      GptTranslate::Plain.new('Hello, {{ Java }}!').to_s
+    )
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: jekyll-chatgpt-translate
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.3
 platform: ruby
 authors:
 - Yegor Bugayenko
@@ -10,62 +10,90 @@ bindir: bin
 cert_chain: []
 date: 2023-08-20 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: iri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
 - !ruby/object:Gem::Dependency
   name: iso-639
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.6
+        version: '0.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.6
+        version: '0.3'
 - !ruby/object:Gem::Dependency
   name: jekyll
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.9.2
+        version: '3.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.9'
+- !ruby/object:Gem::Dependency
+  name: liquid
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.9.2
+        version: '4.0'
 - !ruby/object:Gem::Dependency
   name: redcarpet
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.6.0
+        version: '3.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.6.0
+        version: '3.6'
 - !ruby/object:Gem::Dependency
   name: ruby-openai
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.0.0
+        version: '5.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.0.0
+        version: '5.0'
 description: Add this plugin to your Jekyll site and all posts will be automatically
   translated to the languages of your choice through ChatGPT
 email: yegor256@gmail.com
@@ -93,12 +121,14 @@ files:
 - lib/jekyll-chatgpt-translate.rb
 - lib/jekyll-chatgpt-translate/chatgpt.rb
 - lib/jekyll-chatgpt-translate/generator.rb
+- lib/jekyll-chatgpt-translate/ping.rb
 - lib/jekyll-chatgpt-translate/plain.rb
 - lib/jekyll-chatgpt-translate/version.rb
 - renovate.json
 - test/test__helper.rb
 - test/test_chatgpt.rb
 - test/test_generator.rb
+- test/test_ping.rb
 - test/test_plain.rb
 homepage: https://github.com/yegor256/jekyll-chatgpt-translate
 licenses: