sutty-migration 0.4.0rc0 → 0.4.0rc2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 33b04f68ec23be8ad61143db87c0496fc1a01a84fdb3351b9f76a2782f79342f
-  data.tar.gz: 411e0050ee48d300c7bcda874f1943b33d867f4c090f72650c0122a69027faa2
+  metadata.gz: 7e6f379b5ad8bdb17dbe4353ac4436f244cbcbee4db468070df6da38befde458
+  data.tar.gz: 26e38ae35fdef4c9d702b959ed24ea51c1cf41d457a89c4914a7757a5a942289
 SHA512:
-  metadata.gz: 708dff82c233c3819acbe9ccf3901d8345463282ba9df953adb8ac0f15f9646560e514c87499921d13ac6bfa2693c71b0d634e03e1f04e87d0e148174bd91c50
-  data.tar.gz: 0faa1dc6c107f0999d8a52b7c28b08ad1540e44cff8fd488bfcfae55d2d22548fe0afe61fb54520f4b1d8552674079ba65d617a361139cbb1a5c761581e6df7f
+  metadata.gz: 73cf557ebcf3747e6878e56fb045586573d8b6b2883717fc092a4a85572c3e93361234877deaf4f5aa44af4ae47ac3b6b2c007a707b3cbb57db03afa0af5234b
+  data.tar.gz: 8182bf944bee9bdf1afb61517a700aaa87e99b45a2a19f6c405d1840ca1adc5eb866fca462b94cf96af97d7cad227f88e4a259f6c50a7737d6926af437a9fa4a

data/README.md

@@ -119,48 +119,17 @@ a backup, put them there so the plugin finds them.
 ### WordPress XML
 
 If you have the XML dump from a WordPress site, you can migrate content
-by writing a migration plugin.
+by using the XML migration plugin and hooking to its steps.
 
 ```ruby
+# _plugins/wordpress.rb
 # frozen_string_literal: true
 
-require 'sutty_migration/jekyll/document_creator'
-require 'sutty_migration/wordpress_xml'
-require 'jekyll-write-and-commit-changes'
-require 'securerandom'
-
-# Run after reading the site
-Jekyll::Hooks.register :site, :post_read do |site|
-  # Put the XML dump at _files/wordpress.xml
-  xml = SuttyMigration::WordpressXml.new site: site, file: '_files/wordpress.xml'
-
-  # Download all files
-  xml.attachments.values.map(&:download)
-
-  # Migrate posts.  You can move metadata around and recover
-  # relationships or any info your theme requires.
-  xml.posts.values.each do |post|
-    # Update documents already migrated.
-    doc = Jekyll::Document.find_or_create(site: site, collection: locale, title: post.title, slug: post.slug, date: post.date)
-    # Don't change the UUIDv4
-    d.data['uuid'] ||= SecureRandom.uuid
-    d.data['draft'] = post.draft?
-    d.data['layout'] = 'post'
-    d.data['last_modified_at'] = post.last_modified_at
-
-    d.data['categories'] = post.categories.map { |c| c[:title] }
-    d.data['tags'] = post.tags.map { |t| t[:title] }
-
-    d.data['author'] = post.author[:email]
-    d.data['description'] = post.description
-    d.content = post.content
-
-    doc.save
-  rescue => e
-    Jekyll.logger.warn "Couldn't migrate #{post.title}"
-  end
-
-  exit # Stop here
+require 'jekyll-wordpress-xml-migration'
+require 'pry'
+
+Jekyll::Hooks.register :wordpress_post, :post_init do |item|
+  binding.pry
 end
 ```
 

data/lib/jekyll-wordpress-xml-migration.rb

@@ -10,10 +10,81 @@
 # can use `wordpress_url`.
 #
 # JEKYLL_ENV=migration bundle exec jekyll build
-Jekyll::Hooks.register :site, :post_read, priority: :high do |_site|
+Jekyll::Hooks.register :site, :post_read, priority: :high do |site|
   next unless Jekyll.env == 'migration'
 
-  wordpress_xml = '_files/wordpress.xml'
+  file = '_files/wordpress.xml'
 
-  Jekyll.logger.abort_with 'Migration:', "#{wordpress_xml} is missing!" unless File.exists wordpress_xml
+  Jekyll.logger.abort_with 'Migration:', "#{file} is missing!" unless File.exist? file
+
+  require 'sutty_migration/wordpress_xml'
+  require 'sutty_migration/jekyll/document_creator'
+  require 'jekyll-write-and-commit-changes'
+
+  wordpress = SuttyMigration::WordpressXml.new(site:, file:)
+  migrated_posts = []
+
+  Jekyll::Hooks.trigger :migration, :post_init, wordpress
+
+  wordpress.attachments.values.uniq.each do |attachment|
+    Jekyll::Hooks.trigger :wordpress_attachment, :post_init, attachment.item
+
+    Jekyll::Hooks.trigger :wordpress_attachment, :pre_render, attachment
+    # Download and convert to Sutty's public/
+    attachment.public_path
+    Jekyll::Hooks.trigger :wordpress_attachment, :post_convert, attachment
+    Jekyll::Hooks.trigger :wordpress_attachment, :post_render, attachment
+    Jekyll::Hooks.trigger :wordpress_attachment, :post_write, attachment
+  end
+
+  Jekyll::Hooks.trigger :migration, :pre_render, wordpress
+
+  wordpress.posts.each_value do |post|
+    Jekyll::Hooks.trigger :wordpress_post, :post_init, post.item
+    Jekyll::Hooks.trigger :wordpress_post, :pre_render, post
+    Jekyll::Document.find_or_create(site:, slug: post.slug, title: post.title, date: post.date,
+                                    collection: 'posts').tap do |doc|
+      # If you want to customize the layout, change it with a :post_init
+      # hook.
+      doc.data['layout'] = 'post'
+      doc.data['permalink'] = post.permalink
+      doc.data['created_at'] = post.date
+      doc.data['last_modified_at'] = post.last_modified_at&.localtime
+      doc.data['uuid'] ||= SecureRandom.uuid
+      doc.data['categories'] = post.categories.map { _1[:title] }
+      doc.data['tags'] = post.tags.map { _1[:title] }
+      doc.data['draft'] = post.draft?
+
+      unless (attachment_id = post.meta['_thumbnail_id'].to_i).zero?
+        attachment = wordpress.attachments[attachment_id]
+
+        if attachment
+          doc.data['image'] = {
+            'path' => attachment.public_path,
+            'description' => ''
+          }
+        end
+      end
+
+      Jekyll::Hooks.trigger :wordpress_post, :post_convert, doc
+
+      doc.content = post.beautiful_content
+
+      Jekyll::Hooks.trigger :wordpress_post, :post_render, doc
+
+      migrated_posts << doc
+    end
+  end
+
+  Jekyll::Hooks.trigger :migration, :post_convert, wordpress
+  Jekyll::Hooks.trigger :migration, :post_render, migrated_posts
+
+  # Save them all at the end
+  migrated_posts.each do |doc|
+    doc.save
+
+    Jekyll::Hooks.trigger :wordpress_post, :post_write, doc
+  end
+
+  Jekyll::Hooks.trigger :migration, :post_write, migrated_posts
 end

data/lib/sutty_migration/jekyll/document_creator.rb

@@ -83,4 +83,4 @@ module SuttyMigration
   end
 end
 
-::Jekyll::Document.include SuttyMigration::Jekyll::DocumentCreator
+Jekyll::Document.include SuttyMigration::Jekyll::DocumentCreator

data/lib/sutty_migration/wordpress_xml/attachment.rb

@@ -10,6 +10,16 @@ module SuttyMigration
   class WordpressXml
     # Represents an attachment or uploaded file.
     class Attachment < Post
+      @@connection_failed = false
+
+      def self.connection_failed
+        @@connection_failed
+      end
+
+      def self.connection_failed=(value)
+        @@connection_failed = value
+      end
+
       # File URL
       #
       # @return [String]
@@ -17,11 +27,19 @@ module SuttyMigration
         @attachment_url ||= attribute_value('attachment_url') || attribute_value('guid')
       end
 
+      def parsed_url
+        @parsed_url ||= Addressable::URI.parse(attachment_url)
+      end
+
+      def normalized_url
+        @normalized_url ||= parsed_url.normalize
+      end
+
       # File destination
       #
       # @return [String]
       def dest
-        @dest ||= CGI.unescape(Addressable::URI.parse(attachment_url).path.sub(%r{\A/}, '').unicode_normalize)
+        @dest ||= CGI.unescape(parsed_url.path.sub(%r{\A/}, '').unicode_normalize)
       end
 
       # Metadata, with file information as a Hash
@@ -33,6 +51,8 @@ module SuttyMigration
         end
       end
 
+      attr_reader :head
+
       # Download the file if it doesn't exist.  Optionally show a
       # progress bar.
       #
@@ -40,33 +60,41 @@ module SuttyMigration
       # @return [Boolean]
       def download(progress: true)
         return true if File.exist? dest
+        return false if self.class.connection_failed
 
-        ::Jekyll.logger.info 'Downloading:', dest
-
-        FileUtils.mkdir_p File.dirname(dest)
-
-        File.open(dest, 'w') do |f|
-          head = Faraday.head(attachment_url)
-
-          if head.status > 399
-            ::Jekyll.logger.warn 'Error:', head.status.to_s
+        @head =
+          begin
+            Faraday.head(normalized_url)
+          rescue Faraday::ConnectionFailed, Faraday::SSLError, Faraday::ServerError => e
+            ::Jekyll.logger.warn 'Error:', e.message
+            self.class.connection_failed = true
+            return false
+          rescue Faraday::Error => e
+            ::Jekyll.logger.warn 'Error:', e.message
             return false
           end
 
-          content_length = head.headers['content-length'].to_i
+        if head.status > 399
+          ::Jekyll.logger.warn 'Error:', "#{head.status} #{normalized_url}"
+          return false
+        end
 
-          if content_length.zero?
-            ::Jekyll.logger.warn 'Error:', 'Content length is zero'
-            return false
-          end
+        content_length = head.headers['content-length'].to_i
 
-          if progress
-            progress_bar = ProgressBar.create(title: File.basename(dest), total: content_length,
-                                              output: $stderr)
-          end
+        if content_length.zero?
+          ::Jekyll.logger.warn 'Error:', "Content length is zero for #{normalized_url}"
+          return false
+        end
 
-          begin
-            Faraday.get(attachment_url) do |req|
+        ::Jekyll.logger.info 'Downloading:', dest
+
+        progress_bar =
+          (ProgressBar.create(title: File.basename(dest), total: content_length, output: $stderr) if progress)
+
+        begin
+          FileUtils.mkdir_p File.dirname(dest)
+          File.open(dest, 'w') do |f|
+            Faraday.get(normalized_url) do |req|
               req.options.on_data = proc do |chunk, downloaded_bytes|
                 if downloaded_bytes > content_length
                   raise Faraday::Error, 'File is bigger than informed, aborting download'
@@ -74,16 +102,18 @@ module SuttyMigration
 
                 f.write chunk
 
-                progress_bar.progress = downloaded_bytes if progress
+                progress_bar&.progress = downloaded_bytes
               end
             end
-          rescue Faraday::Error => e
-            ::Jekyll.logger.warn 'Error:', e.message
           end
+        rescue Faraday::Error => e
+          ::Jekyll.logger.warn 'Error:', e.message
+          FileUtils.rm_f(File.dirname(dest))
         end
 
-        progress_bar.stop if progress
-        File.exist? dest
+        progress_bar&.stop
+
+        File.exist?(dest)
       end
 
       # Local path following Sutty's conventions.  Instead of producing
@@ -94,35 +124,34 @@ module SuttyMigration
       #
       # @return [String]
       def public_path
+        return unless digest
+
         @public_path ||= File.join('public', digest[..27], dest.split('/').last).tap do
-          next if File.exist? _1
-
-          # If the directory exists and it's not empty, but the file is
-          # not there, we either have the same file with different names
-          # or old thumbnails.
-          dir = File.dirname(_1)
-          if File.directory?(dir) && !Dir.empty?(dir)
-            Jekyll::Logger.warn 'Migration:',
-                                "#{dir} is not empty, you probably have the same file with different names"
+          # If there's a file with the same digest, we replace the
+          # public_path so we deduplicate them.
+          if (other = wordpress.attachments[digest]) == self
+            next if File.exist? _1
+
+            FileUtils.mkdir_p(File.dirname(_1))
+            FileUtils.ln(dest, _1)
+          else
+            _1.replace other.public_path
           end
-
-          FileUtils.mkdir_p(File.dirname(_1))
-          FileUtils.ln(dest, _1)
         end
       end
 
-      # Download and produce a digest. Produce an error if the file
-      # can't be downloaded.
+      # Download and produce a digest if the file can be downloaded.
+      # Index by digest too.
       #
       # @return [String]
       def digest
         @digest ||=
-          begin
+          if download
             require 'digest'
 
-            raise Faraday::Error unless download
-
-            Digest::SHA2.file(dest).hexdigest
+            Digest::SHA2.file(dest).hexdigest.tap do |digest|
+              wordpress.attachments[digest] ||= self
+            end
           end
       end
     end

data/lib/sutty_migration/wordpress_xml/post.rb

@@ -25,7 +25,7 @@ module SuttyMigration
                       data-turbo href id name referrerpolicy rel reversed rowspan src start target title type]
       # rubocop:enable Style/MutableConstant
       SLASH = '/'
-      THUMBNAIL_PREFIX = /(-scaled)?(-\d+x\d+)?(\.....?)\z/
+      THUMBNAIL_PREFIX = /-\d+x\d+(\.....?)\z/
       # TODO: Move to wordpress-formatting
       WORDPRESS_ELEMENT_START = /\[(\w+ [^\]]+)\]/
       WORDPRESS_ELEMENT_END = %r{\[(/\w+)\]}
@@ -50,13 +50,13 @@ module SuttyMigration
 
       # Permalink. Absolute URL to the post.
       #
-      # @return [String]
+      # @return [String, nil]
       def permalink
         @permalink ||=
           begin
-            p = attribute_value('link')&.delete(wordpress.url)
+            p = attribute_value('link')&.sub(wordpress.url, '')&.split('?', 2)&.first
 
-            unless p.nil?
+            unless p.nil? || p.blank?
               p << SLASH if !p.end_with?(SLASH) && File.extname(p).empty?
 
               p.squeeze(SLASH)
@@ -104,10 +104,10 @@ module SuttyMigration
       #
       # @return [Time, nil]
       def last_modified_at
-        @last_modified_at ||=
-          if (date = attribute_value('post_modified_gmt'))
-            ::Jekyll::Utils.parse_date(date)
-          end
+        @last_modified_at ||= %w[post_modified_gmt post_modified].map do |date_attr|
+          ::Jekyll::Utils.parse_date attribute_value(date_attr)
+        rescue StandardError
+        end.compact.first
       end
 
       # Content as an HTML string, converting Wordpress [tags] to <tags>
@@ -135,10 +135,17 @@ module SuttyMigration
         @parsed_content ||= Nokogiri::HTML.fragment(sanitized_content).tap do |html|
           html.css('[src]').each do |element|
             # Remove the thumbnail suffix, we only want the original files
-            element['src'] = element['src'].sub(THUMBNAIL_PREFIX, '\\3')
+            element['src'] = element['src'].sub(THUMBNAIL_PREFIX, '\\1')
 
             if element['src'].start_with? wordpress.url
-              element['src'] = wordpress.attachments[CGI.unescape element['src']].public_path
+              attachment = wordpress.attachments[CGI.unescape element['src']]
+
+              unless attachment
+                ::Jekyll.logger.warn 'Error:', "Not found #{element['src']}"
+                next
+              end
+
+              element['src'] = attachment.public_path
 
               # Move the element into a <figure> like our editor does
               unless element.parent && element.parent.name == '#document-fragment'

data/lib/sutty_migration/wordpress_xml.rb

@@ -15,7 +15,13 @@ module SuttyMigration
     def initialize(site:, file:)
       @site = site
       @file = file
-      @xml  = Nokogiri::XML File.read(file)
+      @xml  = Nokogiri::XML File.read(file).delete("\b").delete("\u0002")
+
+      if xml.errors.empty?
+        ::Jekyll.logger.info 'Migration:', "#{file} parsed correctly"
+      else
+        ::Jekyll.logger.warn 'Migration:', "#{xml.errors.count} parsing errors found on #{file}!"
+      end
 
       # Make things easier by removing namespaces.
       xml.remove_namespaces!
@@ -108,8 +114,8 @@ module SuttyMigration
     #
     # @return [Hash]
     def posts
-      @posts ||= items_find_by('post_type', 'post').map do |post|
-        { attribute_value(post, 'post_id').to_i => Post.new(wordpress: self, item: post) }
+      @posts ||= items_find_by('post_type', 'post').map do |item|
+        { attribute_value(item, 'post_id').to_i => Post.new(wordpress: self, item:) }
       end.reduce(&:merge)
     end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sutty-migration
 version: !ruby/object:Gem::Version
-  version: 0.4.0rc0
+  version: 0.4.0rc2
 platform: ruby
 authors:
 - f
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-10-16 00:00:00.000000000 Z
+date: 2026-10-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: jekyll