jekyll-chatgpt-translate 0.0.29 → 0.0.31

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dc3e9a30a7d065c15ac18a15b5047f94881809857f65ed77330351f95a129d18
-  data.tar.gz: f6254349103e49e105451d773ac459be452027a5bdafc5a7a004a1d926e7203e
+  metadata.gz: 8c50e5ce734d7a12ddcedbbcc4602453d5c0a7e78352a6e12bfbe04abfcf149e
+  data.tar.gz: 66f6a5668e0d37fd32a1f9573e34036b3155cf864238afb48116c8be4cd19e64
 SHA512:
-  metadata.gz: 15a82d1c87dda2069162db76f00c3a0ade671a76806eeefeeef0239ba3dff9c62c109a1b5772e2fa002ee8da13c26ac8a0b8466235c32a88df557f5896008329
-  data.tar.gz: 61637d5cd8a3ce93f8e10b5df542b6a3319d23ddd168a3293df44e062a95b000b0137cc1d6e336ef24cc4cc7b4025ec095dbea3960d2099cbdfc517016efc921
+  metadata.gz: b5adeb2316daff862f639e9318f5636d3651c69ea5fd50c42c3ddc86e13a7ac97406491fe7f0acf5bfe5d0550135a0568abeed790d1b70ccb3a593a89fc279b6
+  data.tar.gz: cf233bb2b67710883367515aaffffe4304d4c7f2b4c9571e276ed1323c90793bbddef1045067d5e84ca04dbb93fb946614bb5d05bbe71571bbfc589011af3c66

data/.gitignore

@@ -3,4 +3,4 @@ Gemfile.lock
 .bundle/
 .DS_Store
 coverage/
-_chatgpt-translated/
+_chatgpt-translate/

data/Gemfile

@@ -29,7 +29,7 @@ gem 'cucumber', '8.0.0', require: false
 gem 'kramdown-parser-gfm', '1.1.0', require: false
 gem 'minitest', '5.19.0', require: false
 gem 'rake', '13.0.6', require: false
-gem 'rubocop', '1.56.0', require: false
+gem 'rubocop', '1.56.2', require: false
 gem 'rubocop-rspec', '2.23.2', require: false
 gem 'simplecov', '0.22.0', require: false
 gem 'webmock', '3.18.1', require: false

data/README.md

@@ -42,13 +42,13 @@ OpenAI API KEY must be set in `OPENAI_API_KEY` environment variable, otherwise
 the plugin will not do any translation and won't generate translated pages. 
 You can get your key [here](https://help.openai.com/en/articles/4936850-where-do-i-find-my-secret-api-key).
 
-Inside the original page you can use `{{ page.translated-XX-url }}` in order to render the URL
+Inside the original page you can use `{{ page.chatgpt-translate.urls[XX] }}` in order to render the URL
 of the translated page, where `XX` is the [ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) 
 code of the target language.
-Inside the translated page you can use `{{ page.translated-original-url }}` in order
+Inside the translated page you can use `{{ page.chatgpt-translate.original-url }}` in order
 to get the URL of the page that was translated. 
 
-You can also use `{{ page.chatgpt-model }}`
+You can also use `{{ page.chatgpt-translate.model }}`
 inside both the original page and the translated one, to refer to the model of ChatGPT.
 The presence of this attribute in the `{{ page }}` means that the
 page was translated or the translated HTML was downloaded and placed into the `_site` directory.
@@ -80,7 +80,8 @@ Full list of options available to specify in `_config.yml`:
     number big enough, to avoid silly translations. The default is 128.
 
   * `layout` (optional) — is name of the file in `_layouts` directory, without the extension. 
-This layout will be specified for the pages generated by this plugin.
+    This layout will be specified for the pages generated by this plugin.
+    The default value is `translated` (expecting you to have `_layouts/translated.html` file available).
 
   * `targets` (mandatory) — an array of target languages, each of which has the following attributes
 
@@ -100,6 +101,9 @@ This layout will be specified for the pages generated by this plugin.
     when the `version` is changed on another hand. By default, the version of
     this plugin will be used, unless you set your own value.
 
+  * `tmpdir` (optional) — the name of the directory where to keep temporary files,
+    `_chatgpt-translate` is the default value.
+
 ## How to Contribute
 
 Make a fork and then test it locally like this:

data/features/cli.feature

@@ -22,19 +22,19 @@ Feature: Simple site building
     """
     And I have a "_layouts/default.html" file with content:
     """
-    The Chinese: {{ page.translated-zh-url }}
-    The French: {{ page.translated-fr-url }}
+    The Chinese: {{ page.chatgpt-translate.urls['zh'] }}
+    The French: {{ page.chatgpt-translate.urls['fr'] }}
     {{ content }}
     """
     And I have a "_layouts/chinese-translated.html" file with content:
     """
     Chinese: {{ content }}
-    The original: {{ page.translated-original-url }}
+    The original: {{ page.chatgpt-translate.original-url }}
     """
     And I have a "_layouts/translated.html" file with content:
     """
     French: {{ content }}
-    The original: {{ page.translated-original-url }}
+    The original: {{ page.chatgpt-translate.original-url }}
     """
     And I have a "_posts/2023-01-01-hello.md" file with content:
     """
@@ -46,9 +46,9 @@ Feature: Simple site building
     """
     Then I build Jekyll site
     And Exit code is zero
-    And File "_chatgpt-translated/zh/2023-01-01-hello-zh.md" exists
-    And File "_chatgpt-translated/zh/2023-01-01-hello-zh.md" contains "/2023-01-01-hello-chinese.html"
-    And File "_chatgpt-translated/zh/2023-01-01-hello-zh.md" contains "translated-language: \"zh\""
+    And File "_chatgpt-translate/zh/2023-01-01-hello-zh.md" exists
+    And File "_chatgpt-translate/zh/2023-01-01-hello-zh.md" contains "/2023-01-01-hello-chinese.html"
+    And File "_chatgpt-translate/zh/2023-01-01-hello-zh.md" contains "language: \"zh\""
     And File "_site/2023/01/01/hello.html" exists
     And File "_site/2023/01/01/hello.html" contains "The Chinese: /2023-01-01-hello-chinese.html"
     And File "_site/2023-01-01-hello-chinese.html" exists
@@ -69,7 +69,7 @@ Feature: Simple site building
       layout: should-not-be-used
       targets:
         -
-          language: en
+          language: ru
           permalink: about-me.html
     """
     And I have a "_posts/2023-01-01-hello.md" file with content:
@@ -77,12 +77,13 @@ Feature: Simple site building
     ---
     title: foo
     ---
-    foo
+    see translated page: {{ page.chatgpt-translate.urls['ru'] }}
     """
     Then I build Jekyll site
     And Exit code is zero
-    And Stdout contains "No need to translate, the page exists"
+    And Stdout contains "Re-translation not required, since version is empty"
     And File "_site/2023/01/01/hello.html" exists
+    And File "_site/2023/01/01/hello.html" contains "see translated page: /about-me.html"
     And File "_site/about-me.html" exists
     And File "_site/about-me.html" contains "Yegor Bugayenko"
 
@@ -125,3 +126,45 @@ Feature: Simple site building
     And File "_site/about-me.html" exists
     And File "_site/about-me.html" contains "foo-file-foo"
     And File "_site/boom.html" exists
+
+  Scenario: Simple translation with links to other pages
+    Given I have a "_config.yml" file with content:
+    """
+    url: https://www.yegor256.com
+    markdown: kramdown
+    plugins:
+      - jekyll-chatgpt-translate
+    chatgpt-translate:
+      source: en
+      api_key: "it-is-not-used, because EN to EN translation"
+      layout: default
+      targets:
+        -
+          language: en
+          permalink: :slug.html
+    """
+    And I have a "_layouts/default.html" file with content:
+    """
+    {{ content }}
+    """
+    And I have a "_posts/2023-01-01-hello.md" file with content:
+    """
+    ---
+    title: foo
+    ---
+    See {% post_url 2023-02-02-bye %}
+    """
+    And I have a "_posts/2023-02-02-bye.md" file with content:
+    """
+    ---
+    title: foo
+    ---
+    See {% post_url 2023-01-01-hello %}
+    """
+    Then I build Jekyll site
+    And Exit code is zero
+    And Stdout contains "The page is absent, need to translate"
+    And File "_site/2023/01/01/hello.html" exists
+    And File "_site/2023/01/01/hello.html" contains "/bye.html"
+    And File "_site/2023/02/02/bye.html" exists
+    And File "_site/2023/02/02/bye.html" contains "/hello.html"

data/jekyll-chatgpt-translate.gemspec

@@ -28,7 +28,7 @@ Gem::Specification.new do |s|
   s.required_rubygems_version = Gem::Requirement.new('>= 0') if s.respond_to? :required_rubygems_version=
   s.required_ruby_version = '>= 2.6'
   s.name = 'jekyll-chatgpt-translate'
-  s.version = '0.0.29'
+  s.version = '0.0.31'
   s.license = 'MIT'
   s.summary = 'Translate Jekyll Pages Through ChatGPT'
   s.description = [

data/lib/jekyll-chatgpt-translate/generator.rb

@@ -23,6 +23,7 @@
 # SOFTWARE.
 
 require 'jekyll'
+require 'fileutils'
 require 'json'
 require_relative 'chatgpt'
 require_relative 'permalink'
@@ -44,7 +45,7 @@ class GptTranslate::Generator < Jekyll::Generator
   # Main plugin action, called by Jekyll-core
   def generate(site)
     config ||= site.config['chatgpt-translate'] || {}
-    home = '_chatgpt-translated'
+    home = config['tmpdir'] || '_chatgpt-translate'
     key = api_key(config)
     if key.nil?
       Jekyll.logger.info('jekyll-chatgpt-translate requires OPENAI_API_KEY environment variable')
@@ -76,18 +77,34 @@ class GptTranslate::Generator < Jekyll::Generator
             "title: #{doc['title'].to_json}",
             "description: #{doc['description'].to_json}",
             "permalink: #{link.to_json}",
-            "translated-original-url: #{doc.url.to_json}",
-            "translated-language: #{lang.to_json}",
-            "chatgpt-model: #{model.to_json}",
+            'chatgpt-translate:',
+            "  original-url: #{doc.url.to_json}",
+            "  language: #{lang.to_json}",
+            "  model: #{model.to_json}",
             '---'
           ].join("\n")
         )
-        ping = GptTranslate::Ping.new(site, link)
-        if config['no_download'].nil? && ping.found?(version.empty? ? '' : marker)
-          copied += 1
-        elsif translated >= threshold
-          next
+        html = config['no_download'].nil? ? GptTranslate::Ping.new(site, link).download : nil
+        needed = false
+        if html.nil?
+          Jekyll.logger.info("The page is absent, need to translate #{link.inspect}")
+          needed = true
         else
+          copied += 1
+          site.static_files << DownloadedFile.new(site, link, html)
+          if version.empty?
+            Jekyll.logger.info("Re-translation not required, since version is empty: #{link.inspect}")
+          elsif html.include?(marker)
+            Jekyll.logger.info("No need to translate, the page exists at \
+#{link.inspect} (#{html.split.count} words)")
+          else
+            Jekyll.logger.info("Re-translation required for #{link.inspect}")
+            needed = true
+          end
+        end
+        if translated >= threshold
+          Jekyll.logger.info("We are over the threshold: #{translated} > #{threshold}")
+        elsif needed
           gpt = GptTranslate::ChatGPT.new(
             key,
             model,
@@ -106,19 +123,39 @@ class GptTranslate::Generator < Jekyll::Generator
             mode: 'a+'
           )
           site.pages << Jekyll::Page.new(site, site.source, File.dirname(path), File.basename(path))
-          site.static_files.delete_if { |f| f.is_a?(GptTranslate::Ping::DownloadedFile) && f.link == link }
+          site.static_files.delete_if { |f| f.is_a?(DownloadedFile) && f.link == link }
           translated += 1
           Jekyll.logger.info("Translated via ChatGPT \
 in #{(Time.now - start).round(2)}s: #{path} (#{File.size(path)} bytes)")
         end
-        doc.data["translated-#{lang}-url"] = link
-        doc.data['chatgpt-model'] = model
+        doc.data['chatgpt-translate'] ||= {}
+        doc.data['chatgpt-translate']['model'] ||= model
+        doc.data['chatgpt-translate']['urls'] ||= {}
+        doc.data['chatgpt-translate']['urls'][lang] = link
       end
     end
     Jekyll.logger.info("jekyll-chatgpt-translate #{GptTranslate::VERSION}: \
 #{translated} pages translated and #{copied} pages copied in #{(Time.now - start).round(2)}s")
   end
 
+  # The file we just downloaded.
+  class DownloadedFile < Jekyll::StaticFile
+    attr_reader :link
+
+    def initialize(site, link, html)
+      super(site, site.dest, '', link)
+      @html = html
+      @link = link
+    end
+
+    def write(_dest)
+      FileUtils.mkdir_p(File.dirname(path))
+      File.write(path, @html)
+      Jekyll.logger.info("Saved #{@html.split.count} words to #{path.inspect}")
+      true
+    end
+  end
+
   private
 
   # Try to find the KEY, either in the environment, a file, etc.

data/lib/jekyll-chatgpt-translate/ping.rb

@@ -25,8 +25,6 @@
 require 'iri'
 require 'net/http'
 require 'uri'
-require 'jekyll'
-require 'fileutils'
 require_relative 'version'
 
 # see https://stackoverflow.com/a/6048451/187141
@@ -48,47 +46,20 @@ class GptTranslate::Ping
     @path = path
   end
 
-  def found?(marker)
+  # Downloads the page from the Internet and returns HTML or NIL, if the page is absent
+  def download
     home = @site.config['url']
-    return false if home.nil?
+    return nil if home.nil?
     uri = Iri.new(home).path(@path).to_s
+    html = nil
     begin
-      before = Net::HTTP.get_response(URI(uri))
-      if before.is_a?(Net::HTTPSuccess)
-        html = before.body
-        @site.static_files << DownloadedFile.new(@site, @path, html)
-        if html.include?(marker)
-          Jekyll.logger.info("No need to translate, the page exists at \
-#{uri.inspect} (#{html.split.count} words)")
-          return true
-        end
-        Jekyll.logger.info("Re-translation required for #{uri.inspect}")
-      else
-        Jekyll.logger.info("The page is absent, will translate #{uri.inspect} (#{before.code})")
-      end
-      Jekyll.logger.debug("GET #{uri.inspect}: #{before.code}")
+      response = Net::HTTP.get_response(URI(uri))
+      html = response.body if response.is_a?(Net::HTTPSuccess)
+      Jekyll.logger.debug("GET #{uri.inspect}: #{response.code}")
     rescue Errno::ECONNREFUSED, Errno::EADDRNOTAVAIL => e
       Jekyll.logger.debug("Failed to ping #{uri.inspect}: #{e.message}")
       Jekyll.logger.info("The page is absent (#{e.class.name}): #{uri.inspect}")
     end
-    false
-  end
-
-  # The file we just downloaded.
-  class DownloadedFile < Jekyll::StaticFile
-    attr_reader :link
-
-    def initialize(site, link, html)
-      super(site, site.dest, '', link)
-      @html = html
-      @link = link
-    end
-
-    def write(_dest)
-      FileUtils.mkdir_p(File.dirname(path))
-      File.write(path, @html)
-      Jekyll.logger.info("Saved #{@html.split.count} words to #{path.inspect}")
-      true
-    end
+    html
   end
 end

data/lib/jekyll-chatgpt-translate/plain.rb

@@ -46,6 +46,8 @@ class GptTranslate::Plain
         .gsub(/<!--.+?-->/m, '')
         .gsub(/{{[^}]+}}/, '')
         .gsub(/{%.+?%}/, '')
+        .gsub(/^\{.+?\}\n/, '')
+        .gsub(/\n\{.+?\}$/, '')
     ).strip
   end
 
@@ -119,7 +121,11 @@ class GptTranslate::Plain
     end
 
     def link(link, _title, content)
-      "[#{content}](#{link})"
+      if !link.nil? && link.start_with?('/', 'https://', 'http://')
+        "[#{content}](#{link})"
+      else
+        content
+      end
     end
   end
 end

data/lib/jekyll-chatgpt-translate/version.rb

@@ -23,5 +23,5 @@
 # SOFTWARE.
 
 module GptTranslate
-  VERSION = '0.0.29'
+  VERSION = '0.0.31'
 end

data/test/test_chatgpt.rb

@@ -38,22 +38,19 @@ class GptTranslate::ChatGPTTest < Minitest::Test
   end
 
   def test_start_with_link
-    stub_request(:any, 'https://api.openai.com/v1/chat/completions')
-      .to_return(body: '{"choices":[{"message":{"content": "done!"}}]}')
+    stub_it!
     chat = GptTranslate::ChatGPT.new('fake-key', 'gpt-3.5-turbo', 'en', 'ru')
     assert_equal('done!', chat.translate('[OpenAI](https://openai.com) is the creator of ChatGPT', min: 10))
   end
 
   def test_unordered_list_item
-    stub_request(:any, 'https://api.openai.com/v1/chat/completions')
-      .to_return(body: '{"choices":[{"message":{"content": "done!"}}]}')
+    stub_it!
     chat = GptTranslate::ChatGPT.new('fake-key', 'gpt-3.5-turbo', 'en', 'ru')
     assert_equal("* done!\n\n* done!", chat.translate("* First\n\n* Second", min: 1))
   end
 
   def test_ordered_list_item
-    stub_request(:any, 'https://api.openai.com/v1/chat/completions')
-      .to_return(body: '{"choices":[{"message":{"content": "done!"}}]}')
+    stub_it!
     chat = GptTranslate::ChatGPT.new('fake-key', 'gpt-3.5-turbo', 'en', 'ru')
     assert_equal("1. done!\n\n1. done!", chat.translate("1. First\n\n2. Second", min: 1))
   end
@@ -94,9 +91,15 @@ class GptTranslate::ChatGPTTest < Minitest::Test
   end
 
   def test_through_webmock
-    stub_request(:any, 'https://api.openai.com/v1/chat/completions')
-      .to_return(body: '{"choices":[{"message":{"content": "boom!"}}]}')
+    stub_it!
     chat = GptTranslate::ChatGPT.new('fake-key', 'gpt-3.5-turbo', 'en', 'ru')
-    assert_equal('boom!', chat.translate('This is the text to send to OpenAI'))
+    assert_equal('done!', chat.translate('This is the text to send to OpenAI'))
+  end
+
+  private
+
+  def stub_it!
+    stub_request(:any, 'https://api.openai.com/v1/chat/completions')
+      .to_return(body: '{"choices":[{"message":{"content": "done!"}}]}')
   end
 end

data/test/test_ping.rb

@@ -38,24 +38,21 @@ class GptTranslate::PingTest < Minitest::Test
     stub_request(:any, 'https://www.yegor256.com/about-me.html').to_return(body: 'Hello!')
     site = GptTranslate::FakeSite.new({ 'url' => 'https://www.yegor256.com/' })
     ping = GptTranslate::Ping.new(site, '/about-me.html')
-    assert(ping.found?(''))
-    assert_equal(1, site.static_files.size)
+    assert(!ping.download.nil?)
   end
 
   def test_when_not_exists
     stub_request(:any, 'https://www.yegor256.com/absent.html').to_return(status: 404)
     site = GptTranslate::FakeSite.new({ 'url' => 'https://www.yegor256.com/' })
     ping = GptTranslate::Ping.new(site, '/absent.html')
-    assert(!ping.found?(''))
-    assert_equal(0, site.static_files.size)
+    assert(ping.download.nil?)
   end
 
   def test_wrong_address
     WebMock.allow_net_connect!
     site = GptTranslate::FakeSite.new({ 'url' => 'https://localhost:1/' })
     ping = GptTranslate::Ping.new(site, '/boom.html')
-    assert(!ping.found?(''))
-    assert_equal(0, site.static_files.size)
+    assert(ping.download.nil?)
   end
 
   def test_relative_path

data/test/test_plain.rb

@@ -40,6 +40,11 @@ class GptTranslate::PlainTest < Minitest::Test
     assert_equal('Hi, dude!', GptTranslate::Plain.new("  Hi,\ndude!\n").to_s)
   end
 
+  def test_strip_meta_markup
+    assert_equal('Hello, world!', GptTranslate::Plain.new("{:name='boom'}\nHello, world!").to_s)
+    assert_equal('Hello, world!', GptTranslate::Plain.new("Hello, world!\n{: .foo-class}").to_s)
+  end
+
   def test_lists
     assert_equal(
       "* first\n\n* second\n\n* third",
@@ -70,6 +75,14 @@ class GptTranslate::PlainTest < Minitest::Test
       'Hello, [dude](/a.html)!',
       GptTranslate::Plain.new('Hello, [dude](/a.html)!').to_s
     )
+    assert_equal(
+      'Hello, dude!',
+      GptTranslate::Plain.new('Hello, [dude]()!').to_s
+    )
+    assert_equal(
+      'Hello, dude!',
+      GptTranslate::Plain.new('Hello, [dude]({% post_url 2023-01-01-hello %})!').to_s
+    )
   end
 
   def test_code

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: jekyll-chatgpt-translate
 version: !ruby/object:Gem::Version
-  version: 0.0.29
+  version: 0.0.31
 platform: ruby
 authors:
 - Yegor Bugayenko
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-08-24 00:00:00.000000000 Z
+date: 2023-08-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: iri