jekyll-chatgpt-translate 0.0.15 → 0.0.17

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0104da23aeb28e61fb3c2eb4599684accff5945175001b86a49cc821f5e49be6
-  data.tar.gz: 05a05b676a664ec01567f15c49c24bbc2809bd3aeb4dd11666eeea123ec19a7e
+  metadata.gz: f48ffb6334ada2deb878ede0622065326b23ed765946070aff20c9ce5888715d
+  data.tar.gz: df98890a24fb215b1ba1efc2794a6e4bf1cb0b9949efb94648b267c343052562
 SHA512:
-  metadata.gz: f9e400ffbf49216617b0a10cf6600a8786c891dbb73d950da2bdd363acf2c67e1d8e3e86c119a26be56aaed2226bd3b64d1dfb136c7b9272ac4e993ec7c83e81
-  data.tar.gz: 1b4d7a1da595f9bf4829b739d786268c66bac31d9b3b42d9cd9a422bcb1b626af9aa54eeb0088be7e33c2fe39aaa787e972d6c136df20769e55f39f290d6285d
+  metadata.gz: 158df2fdeb7459cace117be39d3e94f9830c852034719ae049777b863bbbc61dc869c362534cc651f53e21af1fa37203865ec7b295254768274921a406cf858f
+  data.tar.gz: f8e34b0bd3432d598b9e7b2f5582ede75231f0c8c9e3dd71db348a5711ddf56106f7685adbf28c02a84f8ef84c7589f4a4909c650afb08d9a110751a317b8bbd

data/README.md

@@ -7,7 +7,7 @@ If you have a [Jekyll](https://jekyllrb.com/) static site, this plugin may help
 translate its pages to another language, through [ChatGPT](https://chat.openai.com/). See how it 
 works for [my blog](https://github.com/yegor256/ru.yegor256.com), 
 for example [this page](https://ru.yegor256.com/2023-08-13-dictators.html) is translated to 
-[English]().
+[English](https://ru.yegor256.com/english/2023-08-13-dictators.html).
 
 Install it first:
 
@@ -18,13 +18,16 @@ gem install jekyll-chatgpt-translate
 Then, add this to `_config.yml`:
 
 ```yaml
+plugins:
+  - ... your other plugins here ...
+  - jekyll-chatgpt-translate
 chatgpt-translate:
   model: gpt-3.5-turbo
   source: en
   layout: translated
   targets: 
     - 
-      language: cn
+      language: zh
       permalink: :year-:month-:day-:slug-chinese.html
       layout: chinese-translated
     - 
@@ -32,7 +35,7 @@ chatgpt-translate:
       permalink: :year-:month-:day-:title-french.html
 ```
 
-Here, the source language is English (`en`), the target one is Chinese (`cn`),
+Here, the source language is English (`en`), the target one is Chinese (`zh`),
 the layout is `_layout/translated.html` (you must have this file).
 
 OpenAI API KEY must be set in `OPENAI_API_KEY` environment variable, otherwise
@@ -40,9 +43,12 @@ the plugin will not do any translation and won't generate translated pages.
 You can get your key [here](https://help.openai.com/en/articles/4936850-where-do-i-find-my-secret-api-key).
 
 Inside the original page you can use `{{ page.translated-XX-url }}` in order to render the URL
-of the translated page, where `XX` is the ISO-839-1 code of the target language..
+of the translated page, where `XX` is the [ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) 
+code of the target language.
 Inside the translated page you can use `{{ page.translated-original-url }}` in order
-to get the URL of the page that was translated. You can also use `{{ page.chatgpt-model }}`
+to get the URL of the page that was translated. 
+
+You can also use `{{ page.chatgpt-model }}`
 inside both the original page and the translated one, to refer to the model of ChatGPT.
 
 ## Options

data/features/cli.feature

@@ -13,7 +13,7 @@ Feature: Simple site building
       layout: translated
       targets:
         -
-          language: cn
+          language: zh
           permalink: :year-:month-:day-:slug-chinese.html
           layout: chinese-translated
         -
@@ -22,7 +22,7 @@ Feature: Simple site building
     """
     And I have a "_layouts/default.html" file with content:
     """
-    The Chinese: {{ page.translated-cn-url }}
+    The Chinese: {{ page.translated-zh-url }}
     The French: {{ page.translated-fr-url }}
     {{ content }}
     """
@@ -40,8 +40,9 @@ Feature: Simple site building
     Hello, world!
     """
     Then I build Jekyll site
-    And File "_chatgpt-translated/cn/2023-01-01-hello-cn.md" exists
-    And File "_chatgpt-translated/cn/2023-01-01-hello-cn.md" contains "/2023-01-01-hello-chinese.html"
+    And File "_chatgpt-translated/zh/2023-01-01-hello-zh.md" exists
+    And File "_chatgpt-translated/zh/2023-01-01-hello-zh.md" contains "/2023-01-01-hello-chinese.html"
+    And File "_chatgpt-translated/zh/2023-01-01-hello-zh.md" contains "translated-language: \"zh\""
     And File "_site/2023/01/01/hello.html" exists
     And File "_site/2023/01/01/hello.html" contains "The Chinese: /2023-01-01-hello-chinese.html"
     And File "_site/2023-01-01-hello-chinese.html" exists

data/features/gem_package.feature

@@ -3,7 +3,7 @@ Feature: Gem Package
   package the Gem into .gem file
 
   Scenario: Gem can be packaged
-    When it is Unix
+    When It is Unix
     Given I have a "execs.rb" file with content:
     """
     #!/usr/bin/env ruby

data/features/step_definitions/steps.rb

@@ -45,18 +45,18 @@ When(/^I build Jekyll site$/) do
   @exitstatus = $CHILD_STATUS.exitstatus
 end
 
-Then(/^Stdout contains "([^"]*)"$/) do |txt|
-  raise "STDOUT doesn't contain '#{txt}':\n#{@stdout}" unless @stdout.include?(txt)
+Then('Stdout contains {string}') do |string|
+  raise "STDOUT doesn't contain '#{string}':\n#{@stdout}" unless @stdout.include?(string)
 end
 
-Then(/^File "([^"]*)" exists$/) do |name|
-  raise "The file \"#{name}\" is absent:\n#{`tree -s`}" unless File.exist?(name)
+Then('File {string} exists') do |string|
+  raise "The file \"#{string}\" is absent:\n#{`tree -s`}" unless File.exist?(string)
 end
 
-Then(/^File "([^"]*)" contains "([^"]*)"$/) do |name, text|
-  raise "The file \"#{name}\" is absent" unless File.exist?(name)
-  content = File.read(name)
-  raise "The file \"#{name}\" doesn't contain \"#{text}\":\n#{content}" unless content.include?(text)
+Then('File {string} contains {string}') do |string, string2|
+  raise "The file \"#{string}\" is absent" unless File.exist?(string)
+  content = File.read(string)
+  raise "The file \"#{string}\" doesn't contain \"#{string2}\":\n#{content}" unless content.include?(string2)
 end
 
 Then(/^Stdout is empty$/) do
@@ -85,10 +85,10 @@ When(/^I copy this gem into temp dir$/) do
   FileUtils.copy_entry(@cwd, File.join(@dir, 'jekyll-chatgpt-translate'))
 end
 
-Given(/^It is Unix$/) do
+Given('It is Unix') do
   pending if Gem.win_platform?
 end
 
-Given(/^It is Windows$/) do
+Given('It is Windows') do
   pending unless Gem.win_platform?
 end

data/jekyll-chatgpt-translate.gemspec

@@ -28,7 +28,7 @@ Gem::Specification.new do |s|
   s.required_rubygems_version = Gem::Requirement.new('>= 0') if s.respond_to? :required_rubygems_version=
   s.required_ruby_version = '>= 2.6'
   s.name = 'jekyll-chatgpt-translate'
-  s.version = '0.0.15'
+  s.version = '0.0.17'
   s.license = 'MIT'
   s.summary = 'Translate Jekyll Pages Through ChatGPT'
   s.description = [

data/lib/jekyll-chatgpt-translate/chatgpt.rb

@@ -52,7 +52,7 @@ class GptTranslate::ChatGPT
       if par.length <= 32
         Jekyll.logger.debug("Not translating this, b/c too short: \"#{par}\"")
         par
-      elsif par !~ /^[a-zA-Zа-яА-Я]/
+      elsif par !~ /^[[:alpha:]]/
         Jekyll.logger.debug("Not translating this, b/c it's not a plain text: \"#{par}\"")
         par
       elsif @key.empty?

data/lib/jekyll-chatgpt-translate/generator.rb

@@ -54,7 +54,8 @@ class GptTranslate::Generator < Jekyll::Generator
     version = config['version'] || GptTranslate::VERSION
     threshold = config['threshold'] || 1024
     start = Time.now
-    total = 0
+    translated = 0
+    copied = 0
     model = config['model'] || 'gpt-3.5-turbo'
     marker = "Translated by ChatGPT #{model}/#{version}"
     site.posts.docs.shuffle.each do |doc|
@@ -63,24 +64,24 @@ class GptTranslate::Generator < Jekyll::Generator
         link = GptTranslate::Permalink.new(doc, target['permalink']).to_path
         lang = target['language']
         raise 'Language must be defined for each target' if target.nil?
-        if total >= threshold
-          Jekyll.logger.info("Already generated #{total} pages, that's enough for today")
-          break
-        end
         path = File.join(home, lang, doc.basename.gsub(/\.md$/, "-#{lang}.md"))
         FileUtils.mkdir_p(File.dirname(path))
         File.write(path, '') # in order to surpress warnings in Page ctor
         dest = Jekyll::Page.new(site, site.source, File.dirname(path), File.basename(path)).destination(site.dest)
         doc.data["translated-#{lang}-url"] = link
         doc.data['chatgpt-model'] = model
-        next if GptTranslate::Ping.new(site, link).found?(dest, version)
+        if GptTranslate::Ping.new(site, link).found?(dest, version)
+          copied += 1
+          next
+        end
+        next if translated >= threshold
         gpt = GptTranslate::ChatGPT.new(
           key,
           model,
           config['source'] || 'en',
           lang
         )
-        translated = gpt.translate(plain)
+        foreign = gpt.translate(plain)
         File.write(
           path,
           [
@@ -90,21 +91,21 @@ class GptTranslate::Generator < Jekyll::Generator
             "description: #{doc['description'].to_json}",
             "permalink: #{link.to_json}",
             "translated-original-url: #{doc.url.to_json}",
+            "translated-language: #{lang.to_json}",
             "chatgpt-model: #{model.to_json}",
             '---',
             '',
-            translated,
+            foreign,
             '',
             "#{marker} on #{Time.now.strftime('%d/%m/%Y %H:%M')}\n{: .jekyll-chatgpt-translate}"
           ].join("\n")
         )
         site.pages << Jekyll::Page.new(site, site.source, File.dirname(path), File.basename(path))
-        total += 1
-        Jekyll.logger.info("Translated via ChatGPT: #{path}")
+        translated += 1
+        Jekyll.logger.info("Translated via ChatGPT: #{path} (#{File.size(path)} bytes)")
       end
-      break if total >= threshold
     end
-    Jekyll.logger.info("#{total} pages translated in #{(Time.now - start).round(2)}s")
+    Jekyll.logger.info("#{translated} pages translated and #{copied} pages copied in #{(Time.now - start).round(2)}s")
   end
 
   private

data/lib/jekyll-chatgpt-translate/plain.rb

@@ -40,36 +40,26 @@ class GptTranslate::Plain
   def to_s
     # To turn compact lists into proper lists
     @markdown.gsub(/([^\n])\n(\s*\*)/, "\\1\n\n\\2").split(/\n{2,}/).compact.map do |par|
-      par.gsub!("\t", ' ')
-      par.gsub!(/\n+/, ' ')
-      par.gsub!(/ {2,}/, ' ')
+      par.strip!
       # Liquid tags are removed, but this implementation is primitive
       # Seehttps://stackoverflow.com/questions/
       par.gsub!(/{{[^}]+}}/, '')
       par.gsub!(/{%.+?%}/, '')
       par.gsub!(/<!--.+?-->/m, '')
-      par.strip!
-      Redcarpet::Markdown.new(Strip).render(par)
+      par = Redcarpet::Markdown.new(Strip).render(par)
+      par.gsub!("\t", '  ')
+      par.gsub!(/\n+/, ' ') unless par.start_with?('```')
+      par.gsub!(/ {2,}/, ' ') unless par.start_with?('```')
+      par.strip
     end.join("\n\n").gsub(/\n{2,}/, "\n\n").strip
   end
 
-  # # To ignore/remove Liquid tags.
-  # class NullDrop < Liquid::Drop
-  #   def method_missing(*)
-  #     nil
-  #   end
-
-  #   def respond_to_missing?(*)
-  #     true
-  #   end
-  # end
-
   # Markdown to pain text.
   class Strip < Redcarpet::Render::Base
     %i[
       block_code block_quote
       block_html
-      autolink codespan double_emphasis
+      autolink double_emphasis
       emphasis underline
       triple_emphasis strikethrough
       superscript highlight quote
@@ -81,8 +71,18 @@ class GptTranslate::Plain
       end
     end
 
-    def raw_html(_content)
-      'HTML'
+    def codespan(content)
+      if content.start_with?("\n")
+        "```#{content}```"
+      elsif content.end_with?("\n")
+        "```\n#{content.split("\n", 2)[1]}```"
+      else
+        content
+      end
+    end
+
+    def raw_html(content)
+      content
     end
 
     def list(content, _type)

data/lib/jekyll-chatgpt-translate/version.rb

@@ -23,5 +23,5 @@
 # SOFTWARE.
 
 module GptTranslate
-  VERSION = '0.0.15'
+  VERSION = '0.0.17'
 end

data/test/test_generator.rb

@@ -33,19 +33,16 @@ require_relative '../lib/jekyll-chatgpt-translate/generator'
 # License:: MIT
 class GptTranslate::GeneratorTest < Minitest::Test
   class FakeSite
-    attr_reader :config
+    attr_reader :config, :pages
 
-    def initialize(config, doc)
+    def initialize(config, docs)
       @config = config
-      @doc = doc
+      @docs = docs
+      @pages = []
     end
 
     def posts
-      FakePosts.new(@doc)
-    end
-
-    def pages
-      []
+      FakePosts.new(@docs)
     end
 
     def permalink_style
@@ -65,7 +62,7 @@ class GptTranslate::GeneratorTest < Minitest::Test
     end
 
     def dest
-      ''
+      File.dirname(@docs[0])
     end
 
     def in_theme_dir(base, _foo = nil, _bar = nil)
@@ -118,12 +115,12 @@ class GptTranslate::GeneratorTest < Minitest::Test
   class FakePosts
     attr_reader :config
 
-    def initialize(doc)
-      @doc = doc
+    def initialize(docs)
+      @docs = docs
     end
 
     def docs
-      [FakeDocument.new(@doc)]
+      @docs.map { |d| FakeDocument.new(d) }
     end
   end
 
@@ -137,18 +134,48 @@ class GptTranslate::GeneratorTest < Minitest::Test
           'chatgpt-translate' => {
             'targets' => [
               {
-                'language' => 'cn',
+                'language' => 'zh',
                 'layout' => 'chinese',
                 'permalink' => ':slug.html'
               }
             ]
           }
         },
-        FakeDocument.new({})
+        [post]
+      )
+      gen = GptTranslate::Generator.new
+      stub_request(:get, 'https://www.yegor256.com/.html').to_return(body: '')
+      gen.generate(site)
+      assert_equal(1, site.pages.count)
+    end
+  end
+
+  def test_threshold_stops
+    Dir.mktmpdir do |home|
+      post = File.join(home, '2023-01-01-hello.md')
+      File.write(post, "---\ntitle: Hello\n---\n\nHello, world!")
+      site = FakeSite.new(
+        {
+          'chatgpt-translate' => {
+            'threshold' => 1,
+            'targets' => [
+              {
+                'language' => 'zh',
+                'permalink' => ':slug.html'
+              },
+              {
+                'language' => 'fr',
+                'permalink' => ':year/:slug.html'
+              }
+            ]
+          }
+        },
+        [post, post]
       )
       gen = GptTranslate::Generator.new
       stub_request(:get, 'https://www.yegor256.com/.html').to_return(body: '')
       gen.generate(site)
+      assert_equal(1, site.pages.count)
     end
   end
 end

data/test/test_plain.rb

@@ -78,17 +78,28 @@ class GptTranslate::PlainTest < Minitest::Test
 
   def test_code_block
     assert_equal(
-      "Hello:\n\nJava",
+      "Hello:\n\n```\nJava\n```",
       GptTranslate::Plain.new("Hello:\n\n```\nJava\n```\n").to_s
     )
   end
 
   def test_html
     assert_equal(
-      'This is picture: HTML!',
+      'This is picture: <img src="a"/>!',
       GptTranslate::Plain.new('This is picture: <img src="a"/>!').to_s
     )
-    assert_equal('HTML', GptTranslate::Plain.new('<img src="a"/>').to_s)
+    assert_equal('<img src="a"/>', GptTranslate::Plain.new('<img src="a"/>').to_s)
+  end
+
+  def test_big_code
+    assert_equal(
+      "```\nHello\n```",
+      GptTranslate::Plain.new("```\nHello\n```").to_s
+    )
+    assert_equal(
+      "```\nprint('hi!')\n```",
+      GptTranslate::Plain.new("```java\nprint('hi!')\n```").to_s
+    )
   end
 
   def test_liquid_tags
@@ -97,7 +108,7 @@ class GptTranslate::PlainTest < Minitest::Test
       GptTranslate::Plain.new('Hello, {{ Java }}!').to_s
     )
     assert_equal(
-      'Hello,  dude !',
+      'Hello, dude !',
       GptTranslate::Plain.new('Hello, {% if a %} dude {% endif %}!').to_s
     )
     assert_equal(

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: jekyll-chatgpt-translate
 version: !ruby/object:Gem::Version
-  version: 0.0.15
+  version: 0.0.17
 platform: ruby
 authors:
 - Yegor Bugayenko