jekyll-chatgpt-translate 0.0.13 → 0.0.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 864696015c8e2b7ee1abba3e5bd8aee3310d7d9d3d85721b5a082b45eef6cd0b
-  data.tar.gz: cfc8b8d09865af57aea8202be714c0ee7211da0fc2121873b8004f37cf40a507
+  metadata.gz: 0104da23aeb28e61fb3c2eb4599684accff5945175001b86a49cc821f5e49be6
+  data.tar.gz: 05a05b676a664ec01567f15c49c24bbc2809bd3aeb4dd11666eeea123ec19a7e
 SHA512:
-  metadata.gz: 83cbb3ae370b246351ce64302e48b0d5c4b94fcc985fb7c239f36f7c2767e5779fa45998ae6b808b8c8e973369ee6d6f21c9a62b241cb5459fb235eb9a424d01
-  data.tar.gz: 9400ee361c82cf27582e416a607d292f16850269fa204a837ff40f052917692dbb0a8d9f3aade9d21d0c2d74870028d6c8c4aec50b4b0efd177f6efc6a220e5c
+  metadata.gz: f9e400ffbf49216617b0a10cf6600a8786c891dbb73d950da2bdd363acf2c67e1d8e3e86c119a26be56aaed2226bd3b64d1dfb136c7b9272ac4e993ec7c83e81
+  data.tar.gz: 1b4d7a1da595f9bf4829b739d786268c66bac31d9b3b42d9cd9a422bcb1b626af9aa54eeb0088be7e33c2fe39aaa787e972d6c136df20769e55f39f290d6285d

data/.gitignore

@@ -2,4 +2,5 @@ Gemfile.lock
 *.gem
 .bundle/
 .DS_Store
-coverage/
+coverage/
+_chatgpt-translated/

data/features/cli.feature

@@ -40,11 +40,12 @@ Feature: Simple site building
     Hello, world!
     """
     Then I build Jekyll site
-    Then File "_chatgpt-translated/cn/2023-01-01-hello-cn.md" exists
-    Then File "_site/2023/01/01/hello.html" exists
-    Then File "_site/2023/01/01/hello.html" contains "The Chinese: /2023-01-01-hello-chinese.html"
-    Then File "_site/2023-01-01-hello-chinese.html" exists
-    Then File "_site/2023-01-01-hello-chinese.html" contains "The original: /2023/01/01/hello.html"
-    Then File "_site/2023/hello-french.html" exists
+    And File "_chatgpt-translated/cn/2023-01-01-hello-cn.md" exists
+    And File "_chatgpt-translated/cn/2023-01-01-hello-cn.md" contains "/2023-01-01-hello-chinese.html"
+    And File "_site/2023/01/01/hello.html" exists
+    And File "_site/2023/01/01/hello.html" contains "The Chinese: /2023-01-01-hello-chinese.html"
+    And File "_site/2023-01-01-hello-chinese.html" exists
+    And File "_site/2023-01-01-hello-chinese.html" contains "The original: /2023/01/01/hello.html"
+    And File "_site/2023/hello-french.html" exists
     And Exit code is zero
 

data/features/step_definitions/steps.rb

@@ -50,10 +50,11 @@ Then(/^Stdout contains "([^"]*)"$/) do |txt|
 end
 
 Then(/^File "([^"]*)" exists$/) do |name|
-  raise "The file \"#{name}\" is absent:\n#{`tree`}" unless File.exist?(name)
+  raise "The file \"#{name}\" is absent:\n#{`tree -s`}" unless File.exist?(name)
 end
 
 Then(/^File "([^"]*)" contains "([^"]*)"$/) do |name, text|
+  raise "The file \"#{name}\" is absent" unless File.exist?(name)
   content = File.read(name)
   raise "The file \"#{name}\" doesn't contain \"#{text}\":\n#{content}" unless content.include?(text)
 end

data/jekyll-chatgpt-translate.gemspec

@@ -28,7 +28,7 @@ Gem::Specification.new do |s|
   s.required_rubygems_version = Gem::Requirement.new('>= 0') if s.respond_to? :required_rubygems_version=
   s.required_ruby_version = '>= 2.6'
   s.name = 'jekyll-chatgpt-translate'
-  s.version = '0.0.13'
+  s.version = '0.0.15'
   s.license = 'MIT'
   s.summary = 'Translate Jekyll Pages Through ChatGPT'
   s.description = [

data/lib/jekyll-chatgpt-translate/chatgpt.rb

@@ -25,6 +25,7 @@
 require 'jekyll'
 require 'openai'
 require 'iso-639'
+require_relative 'prompt'
 
 # The module we are in.
 module GptTranslate; end
@@ -66,43 +67,27 @@ class GptTranslate::ChatGPT
 
   def translate_par(par)
     start = Time.now
-    t = nil
+    answer = nil
     attempt = 0
     begin
+      prompt = GptTranslate::Prompt.new(par, @source, @target).to_s
       response = OpenAI::Client.new(access_token: @key).chat(
         parameters: {
           model: @model,
-          messages: [{
-            role: 'user',
-            content: "#{prompt}:\n\n#{par}"
-          }],
+          messages: [{ role: 'user', content: prompt }],
           temperature: 0.7
         }
       )
-      t = response.dig('choices', 0, 'message', 'content')
+      answer = response.dig('choices', 0, 'message', 'content')
+      Jekyll.logger.debug("ChatGPT prompt: \"#{prompt}\", ChatGPT answer: \"#{answer}\"")
     rescue StandardError => e
       attempt += 1
       retry if attempt < 4
       raise e
     end
     Jekyll.logger.info("Translated #{par.split.count} #{@source.upcase} words \
-to #{t.split.count} #{@target.upcase} words \
+to #{answer.split.count} #{@target.upcase} words \
 through #{@model} in #{(Time.now - start).round(2)}s")
-    t
-  end
-
-  def prompt
-    if (@source == 'ru') && (@target == 'en')
-      'Пожалуйста, переведи этот параграф на английский язык'
-    elsif (@source == 'en') && (@target == 'ru')
-      'Please, translate this paragraph to Russian'
-    else
-      [
-        'Please, translate this paragraph from',
-        ISO_639.find_by_code(@source),
-        'to',
-        ISO_639.find_by_code(@target)
-      ].join(' ')
-    end
+    answer
   end
 end

data/lib/jekyll-chatgpt-translate/generator.rb

@@ -23,6 +23,7 @@
 # SOFTWARE.
 
 require 'jekyll'
+require 'json'
 require_relative 'chatgpt'
 require_relative 'permalink'
 require_relative 'ping'
@@ -60,13 +61,19 @@ class GptTranslate::Generator < Jekyll::Generator
       plain = GptTranslate::Plain.new(doc.content).to_s
       config['targets'].each do |target|
         link = GptTranslate::Permalink.new(doc, target['permalink']).to_path
-        next if GptTranslate::Ping.new(site, link).found?(doc.path, version)
         lang = target['language']
         raise 'Language must be defined for each target' if target.nil?
         if total >= threshold
           Jekyll.logger.info("Already generated #{total} pages, that's enough for today")
           break
         end
+        path = File.join(home, lang, doc.basename.gsub(/\.md$/, "-#{lang}.md"))
+        FileUtils.mkdir_p(File.dirname(path))
+        File.write(path, '') # in order to surpress warnings in Page ctor
+        dest = Jekyll::Page.new(site, site.source, File.dirname(path), File.basename(path)).destination(site.dest)
+        doc.data["translated-#{lang}-url"] = link
+        doc.data['chatgpt-model'] = model
+        next if GptTranslate::Ping.new(site, link).found?(dest, version)
         gpt = GptTranslate::ChatGPT.new(
           key,
           model,
@@ -74,26 +81,23 @@ class GptTranslate::Generator < Jekyll::Generator
           lang
         )
         translated = gpt.translate(plain)
-        path = File.join(home, lang, doc.basename.gsub(/\.md$/, "-#{lang}.md"))
-        FileUtils.mkdir_p(File.dirname(path))
         File.write(
           path,
           [
             '---',
             "layout: #{target['layout'] || layout}",
-            "title: #{doc.data['title']}",
-            "permalink: #{link}",
-            "translated-original-url: #{doc.url}",
-            "chatgpt-model: #{model}",
+            "title: #{doc['title'].to_json}",
+            "description: #{doc['description'].to_json}",
+            "permalink: #{link.to_json}",
+            "translated-original-url: #{doc.url.to_json}",
+            "chatgpt-model: #{model.to_json}",
             '---',
             '',
             translated,
             '',
-            "#{marker}\n{: .jekyll-chatgpt-translate}"
+            "#{marker} on #{Time.now.strftime('%d/%m/%Y %H:%M')}\n{: .jekyll-chatgpt-translate}"
           ].join("\n")
         )
-        doc.data["translated-#{lang}-url"] = link
-        doc.data['chatgpt-model'] = model
         site.pages << Jekyll::Page.new(site, site.source, File.dirname(path), File.basename(path))
         total += 1
         Jekyll.logger.info("Translated via ChatGPT: #{path}")

data/lib/jekyll-chatgpt-translate/ping.rb

@@ -57,7 +57,7 @@ class GptTranslate::Ping
         html = before.body
         if html.include?(marker)
           Jekyll.logger.info("No need to translate, the page exists at \
-  #{uri} (#{html.split.count} words), saved to #{file}")
+\"#{uri}\" (#{html.split.count} words), saved to \"#{file}\"")
           FileUtils.mkdir_p(File.dirname(file))
           File.write(file, html)
           return true
@@ -68,7 +68,8 @@ class GptTranslate::Ping
       end
       Jekyll.logger.debug("GET \"#{uri}\": #{before.code}")
     rescue StandardError => e
-      Jekyll.logger.info("Failed to ping \"#{uri}\": #{e.message}")
+      Jekyll.logger.debug("Failed to ping \"#{uri}\": #{e.message}")
+      Jekyll.logger.info("The page is absent: \"#{uri}\"")
     end
     false
   end

data/lib/jekyll-chatgpt-translate/plain.rb

@@ -38,28 +38,31 @@ class GptTranslate::Plain
   end
 
   def to_s
-    @markdown.split(/\n{2,}/).compact.map do |par|
-      par.gsub!("\n", ' ')
-      par.gsub!(/\s{2,}/, ' ')
+    # To turn compact lists into proper lists
+    @markdown.gsub(/([^\n])\n(\s*\*)/, "\\1\n\n\\2").split(/\n{2,}/).compact.map do |par|
+      par.gsub!("\t", ' ')
+      par.gsub!(/\n+/, ' ')
+      par.gsub!(/ {2,}/, ' ')
       # Liquid tags are removed, but this implementation is primitive
       # Seehttps://stackoverflow.com/questions/
       par.gsub!(/{{[^}]+}}/, '')
-      par.gsub!(/{%[^%]+%}/, '')
+      par.gsub!(/{%.+?%}/, '')
+      par.gsub!(/<!--.+?-->/m, '')
       par.strip!
       Redcarpet::Markdown.new(Strip).render(par)
     end.join("\n\n").gsub(/\n{2,}/, "\n\n").strip
   end
 
-  # To ignore/remove Liquid tags.
-  class NullDrop < Liquid::Drop
-    def method_missing(*)
-      nil
-    end
+  # # To ignore/remove Liquid tags.
+  # class NullDrop < Liquid::Drop
+  #   def method_missing(*)
+  #     nil
+  #   end
 
-    def respond_to_missing?(*)
-      true
-    end
-  end
+  #   def respond_to_missing?(*)
+  #     true
+  #   end
+  # end
 
   # Markdown to pain text.
   class Strip < Redcarpet::Render::Base

data/lib/jekyll-chatgpt-translate/prompt.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+# (The MIT License)
+#
+# Copyright (c) 2023 Yegor Bugayenko
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the 'Software'), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+require 'iso-639'
+
+# The module we are in.
+module GptTranslate; end
+
+# Prompt for ChatGPT.
+# Author:: Yegor Bugayenko (yegor256@gmail.com)
+# Copyright:: Copyright (c) 2023 Yegor Bugayenko
+# License:: MIT
+class GptTranslate::Prompt
+  # Ctor.
+  # +par+ Text to translate
+  # +source+ The language to translate from
+  # +target+ The language to translate into
+  def initialize(par, source, target)
+    @par = par
+    @source = source
+    @target = target
+  end
+
+  def to_s
+    from = ISO_639.find_by_code(@source)
+    raise "Unknown source language ISO-639 code: \"#{@source}\"" if from.nil?
+    to = ISO_639.find_by_code(@target)
+    raise "Unknown source language ISO-639 code: \"#{@target}\"" if to.nil?
+    head = [
+      'Please, translate the following paragraph from ',
+      from[3],
+      ' to ',
+      to[3],
+      ', don\'t change proper nouns'
+    ].join
+    "#{head}:\n\n#{@par}"
+  end
+end

data/lib/jekyll-chatgpt-translate/version.rb

@@ -23,5 +23,5 @@
 # SOFTWARE.
 
 module GptTranslate
-  VERSION = '0.0.13'
+  VERSION = '0.0.15'
 end

data/test/test_generator.rb

@@ -23,6 +23,8 @@
 # SOFTWARE.
 
 require 'minitest/autorun'
+require 'tmpdir'
+require 'webmock/minitest'
 require_relative '../lib/jekyll-chatgpt-translate/generator'
 
 # Generator test.
@@ -30,7 +32,123 @@ require_relative '../lib/jekyll-chatgpt-translate/generator'
 # Copyright:: Copyright (c) 2023 Yegor Bugayenko
 # License:: MIT
 class GptTranslate::GeneratorTest < Minitest::Test
+  class FakeSite
+    attr_reader :config
+
+    def initialize(config, doc)
+      @config = config
+      @doc = doc
+    end
+
+    def posts
+      FakePosts.new(@doc)
+    end
+
+    def pages
+      []
+    end
+
+    def permalink_style
+      ''
+    end
+
+    def frontmatter_defaults
+      Jekyll::FrontmatterDefaults.new(self)
+    end
+
+    def converters
+      [Jekyll::Converters::Markdown.new({ 'markdown_ext' => 'md' })]
+    end
+
+    def source
+      ''
+    end
+
+    def dest
+      ''
+    end
+
+    def in_theme_dir(base, _foo = nil, _bar = nil)
+      base
+    end
+
+    def in_dest_dir(*paths)
+      paths[0].dup
+    end
+  end
+
+  class FakeDocument
+    def initialize(path)
+      @path = path
+    end
+
+    def content
+      'Hello, world!'
+    end
+
+    def data
+      {}
+    end
+
+    def []=(key, value); end
+
+    def [](key)
+      if key == 'date'
+        Time.now
+      elsif key == 'title'
+        'Hello!'
+      else
+        ''
+      end
+    end
+
+    def relative_path
+      @path
+    end
+
+    def url
+      '2023-01-01-hello.html'
+    end
+
+    def basename
+      '2023-01-01-hello.md'
+    end
+  end
+
+  class FakePosts
+    attr_reader :config
+
+    def initialize(doc)
+      @doc = doc
+    end
+
+    def docs
+      [FakeDocument.new(@doc)]
+    end
+  end
+
   def test_simple_scenario
-    # assert(stdout.include?('DEBUG'))
+    Dir.mktmpdir do |home|
+      post = File.join(home, '2023-01-01-hello.md')
+      File.write(post, "---\ntitle: Hello\n---\n\nHello, world!")
+      site = FakeSite.new(
+        {
+          'url' => 'https://www.yegor256.com/',
+          'chatgpt-translate' => {
+            'targets' => [
+              {
+                'language' => 'cn',
+                'layout' => 'chinese',
+                'permalink' => ':slug.html'
+              }
+            ]
+          }
+        },
+        FakeDocument.new({})
+      )
+      gen = GptTranslate::Generator.new
+      stub_request(:get, 'https://www.yegor256.com/.html').to_return(body: '')
+      gen.generate(site)
+    end
   end
 end

data/test/test_plain.rb

@@ -34,6 +34,7 @@ class GptTranslate::PlainTest < Minitest::Test
     assert_equal('Hello, world!', GptTranslate::Plain.new('Hello, **world**!').to_s)
     assert_equal('Hello, Jeff!', GptTranslate::Plain.new('Hello, _Jeff_!').to_s)
     assert_equal("Hi\n\nBye", GptTranslate::Plain.new("  Hi\n\nBye\n\n\n").to_s)
+    assert_equal('Hi, dude!', GptTranslate::Plain.new("  Hi,\ndude!\n").to_s)
   end
 
   def test_lists
@@ -47,6 +48,20 @@ class GptTranslate::PlainTest < Minitest::Test
     )
   end
 
+  def test_ordered_list
+    assert_equal(
+      "first\n\nsecond\n\nthird",
+      GptTranslate::Plain.new("1. first\n\n2. second\n\n3. third").to_s
+    )
+  end
+
+  def test_compact_list
+    assert_equal(
+      "first\n\nsecond\n\nthird",
+      GptTranslate::Plain.new("* first\n* second\n* third").to_s
+    )
+  end
+
   def test_links
     assert_equal(
       'Hello, dude!',
@@ -89,5 +104,20 @@ class GptTranslate::PlainTest < Minitest::Test
       'Hello, !',
       GptTranslate::Plain.new('Hello, {% Java %}!').to_s
     )
+    assert_equal(
+      'Hello, !',
+      GptTranslate::Plain.new('Hello, {% plantuml "width=50%" %}!').to_s
+    )
+  end
+
+  def test_html_comments
+    assert_equal(
+      'Hello, !',
+      GptTranslate::Plain.new('Hello, <!-- Java -->!').to_s
+    )
+    assert_equal(
+      'Hello, !',
+      GptTranslate::Plain.new("Hello, <!-- \nJava\n -->!").to_s
+    )
   end
 end

data/test/test_prompt.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+# (The MIT License)
+#
+# Copyright (c) 2023 Yegor Bugayenko
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the 'Software'), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+require 'minitest/autorun'
+require_relative '../lib/jekyll-chatgpt-translate/prompt'
+
+# Prompt test.
+# Author:: Yegor Bugayenko (yegor256@gmail.com)
+# Copyright:: Copyright (c) 2023 Yegor Bugayenko
+# License:: MIT
+class GptTranslate::PromptTest < Minitest::Test
+  def par(body, source, target)
+    "Please, translate the following paragraph from #{source} to #{target}, don't change proper nouns:\n\n#{body}"
+  end
+
+  def test_english_to_russian
+    assert_equal(
+      par('Hello, dude!', 'English', 'Russian'),
+      GptTranslate::Prompt.new('Hello, dude!', 'en', 'ru').to_s
+    )
+  end
+
+  def test_english_to_chinese
+    assert_equal(
+      par('Hello, Jeff!', 'English', 'Chinese'),
+      GptTranslate::Prompt.new('Hello, Jeff!', 'en', 'zh').to_s
+    )
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: jekyll-chatgpt-translate
 version: !ruby/object:Gem::Version
-  version: 0.0.13
+  version: 0.0.15
 platform: ruby
 authors:
 - Yegor Bugayenko
@@ -110,6 +110,7 @@ files:
 - lib/jekyll-chatgpt-translate/permalink.rb
 - lib/jekyll-chatgpt-translate/ping.rb
 - lib/jekyll-chatgpt-translate/plain.rb
+- lib/jekyll-chatgpt-translate/prompt.rb
 - lib/jekyll-chatgpt-translate/version.rb
 - logo.png
 - renovate.json
@@ -119,6 +120,7 @@ files:
 - test/test_permalink.rb
 - test/test_ping.rb
 - test/test_plain.rb
+- test/test_prompt.rb
 homepage: https://github.com/yegor256/jekyll-chatgpt-translate
 licenses:
 - MIT