sutty-migration 0.3.5 → 0.4.0rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f1fd9ce789787d3a77c484a988bc3a97d687d2db636ced588954578c2a069b60
-  data.tar.gz: 848ae95f5a846a9ad8d7102a3bfbfc2c8813bf8f656e40bbae79a66eb41f482f
+  metadata.gz: 41536e494891ca8e5b5a6ea4f98dcf2cdd780bb8f1a16cdfc10bcee9f5195eb6
+  data.tar.gz: d73e41285e4bfd4080aad2beb3b9b3a29332ee0f31e64b4438fd0bcc7a479952
 SHA512:
-  metadata.gz: 1b3908174927eb8034605f60c56e40c2455514f1e4a707d425c1eab65dc892f9223ffa2eec055d97f4810795f7cf4599daf0803ae823c62eafb7cbc793d4b858
-  data.tar.gz: 767de3717e12d798fba13af63c9f889e26e155228784b07faadfdb8aeddc88a5930437760305a146422366239debe3dd3f13e8b6935a1951aff89ac7008cb0c1
+  metadata.gz: b7e64d51403a731be28d4eae8385747dee6abc99e3e5c7b8220240a673b1dee555584706ef0d10f203751eacb51b89022c02ec1d9019b3854a547650f9489e1d
+  data.tar.gz: f7ae65bde83caa3404d7de2b825fe751fdc0490fda3c399f4c98f2cf320e037323172b065ec82e9e7af4a5e09ecc5c27e095f3b955c19268e1fe0cc55522a557

data/README.md

@@ -91,75 +91,45 @@ SQLite syntax, since `mysql2sqlite` doesn't support this yet.
 Wordpress websites can include lots of posts and metadata, depending on
 the amount of plugins installed.  We don't have an official way of
 dumping everything into Jekyll, because you will probably want to move
-things around.  You can write a plugin like this:
+things around.  You can write a plugin like this to inspect what's
+happening on every stage:
 
 ```ruby
 # _plugins/wordpress.rb
 # frozen_string_literal: true
 
-require 'sutty_migration/wordpress'
-require 'sutty_migration/jekyll/document_creator'
-require 'jekyll-write-and-commit-changes'
+require 'jekyll-wordpress-sql-migration'
+require 'pry'
 
-Jekyll::Hooks.register :site, :post_read, priority: :low do |site|
-  wp = SuttyMigration::Wordpress.new(site: site, database: 'wordpress.sqlite3', prefix: 'wp_', url: 'https://wordpre.ss')
-
-  # Download all files
-  wp.download_all
-
-  wp.posts(layout: 'post').each do |post|
-    doc = Jekyll::Document.create(site: site, title: post[:post_title], date: post[:post_date], collection: 'posts')
-    doc.content = post[:content]
-    doc.save
-  end
+Jekyll::Hooks.register :wordpress_post, :post_init do |item|
+  binding.pry
 end
 ```
 
+Available hooks are `:wordpress_attachment`, `:wordpress_post` and
+`:migration` for the `:post_init`, `:pre_render`, `:post_convert`,
+`:post_render` and `:post_write` stages.
+
+And then set `JEKYLL_ENV=migration` for the migration to run.
+
+If the website is still available (although probably doesn't),
+attachments will be downloaded to `wp-content/`.  If you have them from
+a backup, put them there so the plugin finds them.
+
 ### WordPress XML
 
 If you have the XML dump from a WordPress site, you can migrate content
-by writing a migration plugin.
+by using the XML migration plugin and hooking to its steps.
 
 ```ruby
+# _plugins/wordpress.rb
 # frozen_string_literal: true
 
-require 'sutty_migration/jekyll/document_creator'
-require 'sutty_migration/wordpress_xml'
-require 'jekyll-write-and-commit-changes'
-require 'securerandom'
-
-# Run after reading the site
-Jekyll::Hooks.register :site, :post_read do |site|
-  # Put the XML dump at _files/wordpress.xml
-  xml = SuttyMigration::WordpressXml.new site: site, file: '_files/wordpress.xml'
-
-  # Download all files
-  xml.attachments.values.map(&:download)
-
-  # Migrate posts.  You can move metadata around and recover
-  # relationships or any info your theme requires.
-  xml.posts.values.each do |post|
-    # Update documents already migrated.
-    doc = Jekyll::Document.find_or_create(site: site, collection: locale, title: post.title, slug: post.slug, date: post.date)
-    # Don't change the UUIDv4
-    d.data['uuid'] ||= SecureRandom.uuid
-    d.data['draft'] = post.draft?
-    d.data['layout'] = 'post'
-    d.data['last_modified_at'] = post.last_modified_at
-
-    d.data['categories'] = post.categories.map { |c| c[:title] }
-    d.data['tags'] = post.tags.map { |t| t[:title] }
-
-    d.data['author'] = post.author[:email]
-    d.data['description'] = post.description
-    d.content = post.content
-
-    doc.save
-  rescue => e
-    Jekyll.logger.warn "Couldn't migrate #{post.title}"
-  end
-
-  exit # Stop here
+require 'jekyll-wordpress-xml-migration'
+require 'pry'
+
+Jekyll::Hooks.register :wordpress_post, :post_init do |item|
+  binding.pry
 end
 ```
 

data/lib/jekyll-wordpress-sql-migration.rb

@@ -0,0 +1,132 @@
+# frozen_string_literal: true
+
+# Enable this plugin if you have an SQL dump backup of your Wordpress
+# site.  To avoid having to run a MySQL/MariaDB database, we convert the
+# dump into an SQLite database with our fork of
+# [mysql2sqlite](https://git.sutty.abyaya.la/misc/mysql2sqlite)
+#
+# Put the SQLite file on _files/wordpress.sqlite3 and set the URL on the
+# `url` field on `_config.yml`.  If the new website doesn't match the
+# URL, you can use `wordpress_url`.
+#
+# If the prefix for Wordpress tables is different than "wp_", set
+# `wordpress_prefix` on `_config.yml`.
+#
+# ```sh
+# JEKYLL_ENV=migration bundle exec jekyll build
+# ```
+#
+# ## Hooks
+#
+# Hook to `:migration` if you need to access and modify data during the
+# migration process:
+#
+# * `:post_init`: will receive the `Hash`
+# * `:pre_render`: will receive the `SuttyMigration::WordpressXml::Post`
+# * `:post_convert`: will receive the `Jekyll::Document` after setting front matter
+# * `:post_render`: will receive the `Jekyll::Document` after sanitizing the content
+# * `:post_write`: will receive the `Jekyll::Document` after writing to disk
+
+Jekyll::Hooks.register :site, :post_read, priority: :high do |site|
+  next unless Jekyll.env == 'migration'
+
+  database = '_files/wordpress.sqlite3'
+
+  Jekyll.logger.abort_with 'Migration:', "#{database} doesn't exist!" unless File.exist? database
+
+  require 'sutty_migration/wordpress'
+  require 'sutty_migration/wordpress_xml/post'
+  require 'sutty_migration/wordpress_xml/attachment'
+  require 'sutty_migration/jekyll/document_creator'
+  require 'jekyll-write-and-commit-changes'
+
+  url = site.config['wordpress_url'] || site.config['url']
+  prefix = site.config['wordpress_prefix'] || 'wp_'
+
+  wordpress = SuttyMigration::Wordpress.new(site:, url:, database:, prefix:)
+  migrated_posts = []
+
+  Jekyll::Hooks.trigger :migration, :post_init, wordpress
+
+  wordpress.posts(layout: 'attachment').each do |item|
+    Jekyll::Hooks.trigger :wordpress_attachment, :post_init, item
+
+    attachment = SuttyMigration::WordpressXml::Attachment.new(wordpress:, item:)
+
+    # Download and convert to Sutty's public/
+    Jekyll::Hooks.trigger :wordpress_attachment, :pre_render, attachment
+    # Index attachments by id, public path and original destination so
+    # we can find them later.
+    wordpress.attachments[attachment.attachment_url] =
+      wordpress.attachments[attachment.public_path] =
+        wordpress.attachments[item[:id]] =
+          attachment
+    Jekyll::Hooks.trigger :wordpress_attachment, :post_convert, attachment
+    Jekyll::Hooks.trigger :wordpress_attachment, :post_render, attachment
+
+    Jekyll::Hooks.trigger :wordpress_attachment, :post_write, attachment
+  end
+
+  Jekyll::Hooks.trigger :migration, :pre_render, wordpress
+
+  # We have to instantiate Posts from a Hash instead of an XML object.
+  wordpress.posts(layout: 'post').each do |item|
+    Jekyll::Hooks.trigger :wordpress_post, :post_init, item
+
+    item[:title] = 'Draft' if item[:title].empty?
+    item[:slug] = Jekyll::Utils.slugify(item[:title], mode: 'latin') if item[:slug].empty?
+
+    post = SuttyMigration::WordpressXml::Post.new(wordpress:, item:).tap do |p|
+      p.categories = [item.dig(:terms, :category)].flatten.compact
+      p.tags = [item.dig(:terms, :post_tag)].flatten.compact
+      p.date = item[:date] || item[:last_modified_at]
+      p.last_modified_at = item[:last_modified_at]
+    end
+
+    Jekyll::Hooks.trigger :wordpress_post, :pre_render, post
+
+    Jekyll::Document.find_or_create(site:, slug: post.slug, title: post.title, date: post.date,
+                                    collection: 'posts').tap do |doc|
+      # If you want to customize the layout, change it with a :post_init
+      # hook.
+      doc.data['layout'] = item[:layout]
+      doc.data['created_at'] = post.date
+      doc.data['last_modified_at'] = post.last_modified_at&.localtime
+      doc.data['uuid'] ||= SecureRandom.uuid
+      doc.data['categories'] = post.categories
+      doc.data['tags'] = post.tags
+      doc.data['draft'] = post.draft?
+
+      unless (attachment_id = item.dig(:front_matter, :_thumbnail_id).to_i).zero?
+        attachment = wordpress.attachments[attachment_id]
+
+        if attachment
+          doc.data['image'] = {
+            'path' => attachment.public_path,
+            'description' => ''
+          }
+        end
+      end
+
+      Jekyll::Hooks.trigger :wordpress_post, :post_convert, doc
+
+      doc.content = post.beautiful_content
+
+      Jekyll::Hooks.trigger :wordpress_post, :post_render, doc
+
+      migrated_posts << doc
+    end
+  end
+
+  Jekyll::Hooks.trigger :migration, :post_convert, wordpress
+  Jekyll::Hooks.trigger :migration, :post_render, migrated_posts
+
+  # Save them all at the end
+  migrated_posts.each do |doc|
+    doc.save
+
+    Jekyll::Hooks.trigger :wordpress_post, :post_write, doc
+  end
+
+  Jekyll::Hooks.trigger :migration, :post_write, migrated_posts
+end

data/lib/jekyll-wordpress-xml-migration.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+# Enable this plugin if you have an XML backup of your Wordpress site.
+# Although this is the easiest path, it's not always possible to have
+# one, either because the website is not working or producing the XML
+# dump gets canceled by PHP resource usage limits.
+#
+# Put the XML file on _files/wordpress.xml and set the URL on the `url`
+# field on _config.yml.  If the new website doesn't match the URL, you
+# can use `wordpress_url`.
+#
+# JEKYLL_ENV=migration bundle exec jekyll build
+Jekyll::Hooks.register :site, :post_read, priority: :high do |site|
+  next unless Jekyll.env == 'migration'
+
+  file = '_files/wordpress.xml'
+
+  Jekyll.logger.abort_with 'Migration:', "#{file} is missing!" unless File.exist? file
+
+  require 'sutty_migration/wordpress_xml'
+  require 'sutty_migration/jekyll/document_creator'
+  require 'jekyll-write-and-commit-changes'
+
+  wordpress = SuttyMigration::WordpressXml.new(site:, file:)
+  migrated_posts = []
+
+  Jekyll::Hooks.trigger :migration, :post_init, wordpress
+
+  wordpress.attachments.values.uniq.each do |attachment|
+    Jekyll::Hooks.trigger :wordpress_attachment, :post_init, attachment.item
+
+    Jekyll::Hooks.trigger :wordpress_attachment, :pre_render, attachment
+    # Download and convert to Sutty's public/
+    attachment.public_path
+    Jekyll::Hooks.trigger :wordpress_attachment, :post_convert, attachment
+    Jekyll::Hooks.trigger :wordpress_attachment, :post_render, attachment
+    Jekyll::Hooks.trigger :wordpress_attachment, :post_write, attachment
+  end
+
+  wordpress.posts.each_value do |post|
+    Jekyll::Hooks.trigger :wordpress_post, :post_init, post.item
+    Jekyll::Hooks.trigger :wordpress_post, :pre_render, post
+    Jekyll::Document.find_or_create(site:, slug: post.slug, title: post.title, date: post.date,
+                                    collection: 'posts').tap do |doc|
+      # If you want to customize the layout, change it with a :post_init
+      # hook.
+      doc.data['layout'] = 'post'
+      doc.data['permalink'] = post.permalink
+      doc.data['created_at'] = post.date
+      doc.data['last_modified_at'] = post.last_modified_at&.localtime
+      doc.data['uuid'] ||= SecureRandom.uuid
+      doc.data['categories'] = post.categories.map { _1[:title] }
+      doc.data['tags'] = post.tags.map { _1[:title] }
+      doc.data['draft'] = post.draft?
+
+      unless (attachment_id = post.meta['_thumbnail_id'].to_i).zero?
+        attachment = wordpress.attachments[attachment_id]
+
+        if attachment
+          doc.data['image'] = {
+            'path' => attachment.public_path,
+            'description' => ''
+          }
+        end
+      end
+
+      Jekyll::Hooks.trigger :wordpress_post, :post_convert, doc
+
+      doc.content = post.beautiful_content
+
+      Jekyll::Hooks.trigger :wordpress_post, :post_render, doc
+
+      migrated_posts << doc
+    end
+  end
+
+  Jekyll::Hooks.trigger :migration, :post_convert, wordpress
+  Jekyll::Hooks.trigger :migration, :post_render, migrated_posts
+
+  # Save them all at the end
+  migrated_posts.each do |doc|
+    doc.save
+
+    Jekyll::Hooks.trigger :wordpress_post, :post_write, doc
+  end
+
+  Jekyll::Hooks.trigger :migration, :post_write, migrated_posts
+end

data/lib/sutty_migration/data.rb

@@ -27,9 +27,9 @@ Jekyll::Hooks.register :site, :post_read, priority: :low do |site|
       end
 
       document ||= begin
-                     data = row.slice(*%w[date slug title]).transform_keys(&:to_sym)
-                     Jekyll::Document.find_or_create(site: site, collection: 'posts', **data)
-                   end
+        data = row.slice(*%w[date slug title]).transform_keys(&:to_sym)
+        Jekyll::Document.find_or_create(site: site, collection: 'posts', **data)
+      end
       next unless document
 
       row.each do |attribute, value|

data/lib/sutty_migration/wordpress.rb

@@ -17,7 +17,7 @@ module SuttyMigration
   # migration plugin where you can convert data by yourself.  We may add
   # this feature in the future.
   class Wordpress
-    attr_reader :site, :prefix, :limit, :url, :wp, :database, :multisite
+    attr_reader :site, :prefix, :limit, :url, :wp, :database, :multisite, :attachments
 
     # @param :site [Jekyll::Site] Jekyll site
     # @param :url [String] Wordpress site URL (must be up for downloads)
@@ -32,13 +32,14 @@ module SuttyMigration
       @url = url.freeze
       @database = database || File.join(site.source, '_data', 'wordpress.sqlite3')
       @multisite = multisite
+      @attachments = {}
     end
 
     # Generate database connections for a multisite WP
     #
     # @return [Hash] { "ID" => SuttyMigration::Wordpress }
     def blogs
-      @blogs ||= wp["select * from #{prefix}blogs"].to_a.map do |blog|
+      @blogs ||= wp["select * from #{prefix}blogs"].to_a.to_h do |blog|
         url   = "https://#{blog[:domain]}#{blog[:path]}"
         pfx   = "#{prefix}#{blog[:blog_id]}_" if blog[:blog_id] > 1
         pfx ||= prefix
@@ -46,11 +47,11 @@ module SuttyMigration
         [blog[:blog_id],
          blog.merge(db: self.class.new(site: site, url: url, prefix: pfx, database: database, limit: limit,
                                        multisite: self))]
-      end.to_h
+      end
     end
 
     def options
-      @options ||= wp["select option_name, option_value from #{prefix}options"].to_a.map(&:values).to_h.transform_keys(&:to_sym)
+      @options ||= wp["select option_name, option_value from #{prefix}options"].to_a.to_h(&:values).transform_keys(&:to_sym)
     end
 
     # Open the database.
@@ -62,53 +63,6 @@ module SuttyMigration
       end
     end
 
-    # Download all attachments.  Adds the local path to them.
-    #
-    # @param :progress [Boolean] Toggle progress bar
-    # @return [Nil]
-    def download_all(progress: true)
-      posts(layout: 'attachment').each do |attachment|
-        attachment[:front_matter]['file_path'] = download(url: attachment[:guid], progress: progress)
-      end
-    end
-
-    # Downloads a file if needed, optionally showing a progress bar.
-    #
-    # @param :url [String] File URL
-    # @param :progress [Boolean] Toggle progress bar
-    # @return [String] File local path
-    def download(url:, progress: true)
-      uri = URI(url)
-      dest = uri.path.sub(%r{\A/}, '')
-      full = File.join(site.source, dest)
-
-      return dest if File.exist? full
-
-      ::Jekyll.logger.info "Downloading #{dest}"
-
-      FileUtils.mkdir_p File.dirname(full)
-
-      File.open(full, 'w') do |f|
-        if progress
-          head = Faraday.head(url)
-          content_length = head.headers['content-length'].to_i
-          progress = ProgressBar.create(title: File.basename(dest), total: content_length, output: $stderr)
-        end
-
-        Faraday.get(url) do |req|
-          req.options.on_data = proc do |chunk, downloaded_bytes|
-            f.write chunk
-
-            if progress
-              progress.progress = downloaded_bytes > content_length ? content_length : downloaded_bytes
-            end
-          end
-        end
-      end
-
-      dest
-    end
-
     # List post types
     #
     # @return [Array]
@@ -129,7 +83,7 @@ module SuttyMigration
       end
 
       wp[post_query(**options)].each_page(limit).to_a.map(&:to_a).flatten.tap do |p|
-        p.map do |post|
+        p.each do |post|
           # Sequel parses dates on localtime
           post[:date] = ::Jekyll::Utils.parse_date(post[:date]) unless post[:date].blank?
           unless post[:last_modified_at].blank?

data/lib/sutty_migration/wordpress_xml/attachment.rb

@@ -4,6 +4,7 @@ require_relative 'post'
 require 'php-serialize'
 require 'faraday'
 require 'progressbar'
+require 'addressable'
 
 module SuttyMigration
   class WordpressXml
@@ -13,14 +14,22 @@ module SuttyMigration
       #
       # @return [String]
       def attachment_url
-        @attachment_url ||= attribute_value 'attachment_url'
+        @attachment_url ||= attribute_value('attachment_url') || attribute_value('guid')
+      end
+
+      def parsed_url
+        @parsed_url ||= Addressable::URI.parse(attachment_url)
+      end
+
+      def normalized_url
+        @normalized_url ||= parsed_url.normalize
       end
 
       # File destination
       #
       # @return [String]
       def dest
-        @dest ||= CGI.unescape(URI(attachment_url).path.sub(%r{\A/}, ''))
+        @dest ||= CGI.unescape(parsed_url.path.sub(%r{\A/}, '').unicode_normalize)
       end
 
       # Metadata, with file information as a Hash
@@ -40,30 +49,89 @@ module SuttyMigration
       def download(progress: true)
         return true if File.exist? dest
 
-        ::Jekyll.logger.info 'Downloading:', dest
-
         FileUtils.mkdir_p File.dirname(dest)
 
         File.open(dest, 'w') do |f|
-          if progress
-            head = Faraday.head(attachment_url)
-            content_length = head.headers['content-length'].to_i
-            progress = ProgressBar.create(title: File.basename(dest), total: content_length, output: $stderr)
+          head = Faraday.head(normalized_url)
+
+          if head.status > 399
+            ::Jekyll.logger.warn 'Error:', "#{head.status} #{normalized_url}"
+            return false
           end
 
-          Faraday.get(attachment_url) do |req|
-            req.options.on_data = proc do |chunk, downloaded_bytes|
-              f.write chunk
+          content_length = head.headers['content-length'].to_i
+
+          if content_length.zero?
+            ::Jekyll.logger.warn 'Error:', "Content length is zero for #{normalized_url}"
+            return false
+          end
 
-              if progress
-                progress.progress = downloaded_bytes > content_length ? content_length : downloaded_bytes
+          ::Jekyll.logger.info 'Downloading:', dest
+
+          progress_bar =
+            (ProgressBar.create(title: File.basename(dest), total: content_length, output: $stderr) if progress)
+
+          begin
+            Faraday.get(normalized_url) do |req|
+              req.options.on_data = proc do |chunk, downloaded_bytes|
+                if downloaded_bytes > content_length
+                  raise Faraday::Error, 'File is bigger than informed, aborting download'
+                end
+
+                f.write chunk
+
+                progress_bar&.progress = downloaded_bytes
               end
             end
+          rescue Faraday::Error => e
+            ::Jekyll.logger.warn 'Error:', e.message
           end
+
+          progress_bar&.stop
         end
 
         File.exist? dest
       end
+
+      # Local path following Sutty's conventions.  Instead of producing
+      # a SecureRandom.base36 key, we use the first 28 characters of the
+      # digest to obtain constand names.
+      #
+      # It then hardlinks the original file.
+      #
+      # @return [String]
+      def public_path
+        return unless digest
+
+        @public_path ||= File.join('public', digest[..27], dest.split('/').last).tap do
+          next if File.exist? _1
+
+          # If the directory exists and it's not empty, but the file is
+          # not there, we either have the same file with different names
+          # or old thumbnails.
+          dir = File.dirname(_1)
+          if File.directory?(dir) && !Dir.empty?(dir)
+            ::Jekyll.logger.warn 'Migration:',
+                                 "#{dir} is not empty, you probably have the same file with different names"
+          end
+
+          FileUtils.mkdir_p(File.dirname(_1))
+          FileUtils.ln(dest, _1)
+        end
+      end
+
+      # Download and produce a digest. Produce an error if the file
+      # can't be downloaded.
+      #
+      # @return [String]
+      def digest
+        @digest ||=
+          begin
+            require 'digest'
+
+            Digest::SHA2.file(dest).hexdigest if download
+          end
+      end
     end
   end
 end

data/lib/sutty_migration/wordpress_xml/post.rb

@@ -1,18 +1,37 @@
 # frozen_string_literal: true
 
+require 'rails-html-sanitizer'
 require 'wordpress_formatting/wpautop'
+require 'htmlbeautifier'
 require 'jekyll/utils'
+require 'nokogiri'
 
 module SuttyMigration
   class WordpressXml
     # Represents a WordPress post
     class Post
       attr_reader :wordpress, :item
+      attr_writer :tags, :categories, :author, :meta, :date, :last_modified_at
 
+      SANITIZER = Rails::HTML::Sanitizer.safe_list_sanitizer.new
+      # This follows https://www.npmjs.com/package/@suttyweb/editor and
+      # also allows some known WP tags, you can add your own during
+      # plugin initilization.
+      #
+      # rubocop:disable Style/MutableConstant
+      TAGS = %w[a aside audio blockquote br caption code del em figcaption figure h1 h2 h3 h4 h5 h6 hr iframe img li mark ol p
+                pre small strong sub sup table tbody td tfoot th thead tr u ul video]
+      ATTRIBUTES = %w[alt colspan controls data-align data-id data-lang data-multimedia data-multimedia-inner
+                      data-turbo href id name referrerpolicy rel reversed rowspan src start target title type]
+      # rubocop:enable Style/MutableConstant
       SLASH = '/'
+      THUMBNAIL_PREFIX = /-\d+x\d+(\.....?)\z/
+      # TODO: Move to wordpress-formatting
+      WORDPRESS_ELEMENT_START = /\[(\w+ [^\]]+)\]/
+      WORDPRESS_ELEMENT_END = %r{\[(/\w+)\]}
 
       # @param :wordpress [SuttyMigration::WordpressXml]
-      # @param :item [Nokogiri::XML::Element]
+      # @param :item [Nokogiri::XML::Element, Hash]
       def initialize(wordpress:, item:)
         @wordpress = wordpress
         @item = item
@@ -31,17 +50,17 @@ module SuttyMigration
 
       # Permalink. Absolute URL to the post.
       #
-      # @return [String]
+      # @return [String, nil]
       def permalink
         @permalink ||=
           begin
-            p = attribute_value('link').sub(wordpress.url, '')
+            p = attribute_value('link')&.sub(wordpress.url, '')&.split('?', 2)&.first
 
-            if !p.end_with?(SLASH) && File.extname(p).empty?
-              p << SLASH
-            end
+            unless p.nil? || p.blank?
+              p << SLASH if !p.end_with?(SLASH) && File.extname(p).empty?
 
-            p.squeeze(SLASH)
+              p.squeeze(SLASH)
+            end
           end
       end
 
@@ -63,7 +82,9 @@ module SuttyMigration
       #
       # @return [String]
       def slug
-        @slug ||= attribute_value('post_name').tr(SLASH, '-').squeeze('-').sub(/\A-+/, '').sub(/-+\z/, '')
+        @slug ||= (attribute_value('post_name') || attribute_value('slug')).to_s.tr(SLASH, '-').squeeze('-').sub(/\A-+/, '').sub(
+          /-+\z/, ''
+        )
       end
 
       # Publication date.
@@ -83,19 +104,93 @@ module SuttyMigration
       #
       # @return [Time, nil]
       def last_modified_at
-        @last_modified_at ||=
-          if (date = attribute_value('post_modified_gmt'))
-            ::Jekyll::Utils.parse_date(date)
-          end
+        @last_modified_at ||= %w[post_modified_gmt post_modified].map do |date_attr|
+          ::Jekyll::Utils.parse_date attribute_value(date_attr)
+        rescue StandardError
+        end.compact.first
       end
 
-      # Content as HTML, with site URL removed.
+      # Content as an HTML string, converting Wordpress [tags] to <tags>
       #
       # @return [String]
       def content
-        @content ||= WordpressFormatting::Wpautop.wpautop(attribute_value('encoded')).gsub(
-          / (href|src)="#{wordpress.url}/, ' \\1="'
-        )
+        @content ||= WordpressFormatting::Wpautop.wpautop((attribute_value('encoded') || attribute_value('content')).gsub(WORDPRESS_ELEMENT_START, '<\\1>').gsub(
+                                                            WORDPRESS_ELEMENT_END, '<\\1>'
+                                                          ))
+      end
+
+      # Remove extraneous code
+      #
+      # @return [String]
+      def sanitized_content
+        @sanitized_content ||= SANITIZER.sanitize(content.to_s, tags: TAGS, attributes: ATTRIBUTES)
+      end
+
+      # Returns a fragment that can be queried using Nokogiri and
+      # transformed for final migration.  Applies common transformations
+      # too.
+      #
+      # @return [Nokogiri::HTML]
+      def parsed_content
+        @parsed_content ||= Nokogiri::HTML.fragment(sanitized_content).tap do |html|
+          html.css('[src]').each do |element|
+            # Remove the thumbnail suffix, we only want the original files
+            element['src'] = element['src'].sub(THUMBNAIL_PREFIX, '\\1')
+
+            if element['src'].start_with? wordpress.url
+              attachment = wordpress.attachments[CGI.unescape element['src']]
+
+              unless attachment
+                ::Jekyll.logger.warn 'Error:', "Not found #{element['src']}"
+                next
+              end
+
+              element['src'] = attachment.public_path
+
+              # Move the element into a <figure> like our editor does
+              unless element.parent && element.parent.name == '#document-fragment'
+                parent = element.parent
+
+                parent = parent.parent until parent.parent.nil? || parent.parent.name == '#document-fragment'
+
+                figure = parent.add_previous_sibling('<figure></figure>').first
+                figure.add_child element
+                # This was the [caption] WP element
+                figcaption = html.css('caption').first
+
+                if figcaption
+                  figcaption.name = 'figcaption'
+                  figure.add_child figcaption
+                  element['alt'] = figcaption.text.strip
+                end
+              end
+
+            # Iframes are converted to an autolinked paragraph so
+            # jekyll-embed-urls can pick them up during build
+            elsif element.name == 'iframe'
+              element.parent.name = 'p'
+              element.name = 'a'
+              element['href'] = element['src']
+              if element['href'].include? 'youtube.com/embed/'
+                element['href'] = Addressable::URI.parse(element['href']).tap do |uri|
+                  uri.query = URI.encode_www_form(URI.decode_www_form(uri.query.to_s).tap do |query|
+                    query << ['v', uri.path.split('/').last]
+                  end)
+                  uri.path = '/watch'
+                end.to_s
+              end
+              element.delete 'src'
+              element.inner_html = element['href']
+            end
+          end
+        end
+      end
+
+      # Beautifies the parsed content.
+      #
+      # @return [String]
+      def beautiful_content
+        HtmlBeautifier.beautify(parsed_content.to_s.squeeze(' ').gsub(/>\s+</, '><'))
       end
 
       # Author attributes.
@@ -176,9 +271,16 @@ module SuttyMigration
 
       # Get a value from the attribute
       #
+      # @param key [String, Symbol]
       # @return [String, nil]
       def attribute_value(key)
-        item.at_css(key)&.text
+        (case item
+         when Hash
+           item[key.to_sym]
+         else
+           item.at_css(key.to_s)&.text
+         end
+        )&.unicode_normalize&.strip
       end
     end
   end

data/lib/sutty_migration/wordpress_xml.rb

@@ -15,7 +15,13 @@ module SuttyMigration
     def initialize(site:, file:)
       @site = site
       @file = file
-      @xml  = Nokogiri::XML File.read(file)
+      @xml  = Nokogiri::XML File.read(file).delete("\b").delete("\u0002")
+
+      if xml.errors.empty?
+        ::Jekyll.logger.info 'Migration:', "#{file} parsed correctly"
+      else
+        ::Jekyll.logger.warn 'Migration:', "#{xml.errors.count} parsing errors found on #{file}!"
+      end
 
       # Make things easier by removing namespaces.
       xml.remove_namespaces!
@@ -108,8 +114,8 @@ module SuttyMigration
     #
     # @return [Hash]
     def posts
-      @posts ||= items_find_by('post_type', 'post').map do |post|
-        { attribute_value(post, 'post_id').to_i => Post.new(wordpress: self, item: post) }
+      @posts ||= items_find_by('post_type', 'post').map do |item|
+        { attribute_value(item, 'post_id').to_i => Post.new(wordpress: self, item:) }
       end.reduce(&:merge)
     end
 
@@ -117,18 +123,23 @@ module SuttyMigration
     #
     # @return [Hash]
     def pages
-      @pages ||= items_find_by('post_type', 'page').map do |page|
-        { attribute_value(page, 'post_id').to_i => Post.new(wordpress: self, item: page) }
-      end.reduce(&:merge)
+      @pages ||= items_find_by('post_type', 'page').to_h do |item|
+        [attribute_value(page, 'post_id').to_i, Post.new(wordpress: self, item:)]
+      end
     end
 
-    # Attachments, indexed by ID
+    # Attachments, indexed by URL and ID
     #
     # @return [Hash]
     def attachments
-      @attachments ||= items_find_by('post_type', 'attachment').map do |attachment|
-        { attribute_value(attachment, 'post_id').to_i => Attachment.new(wordpress: self, item: attachment) }
-      end.reduce(&:merge)
+      @attachments ||= items_find_by('post_type', 'attachment').map do |item|
+        attachment = Attachment.new(wordpress: self, item:)
+
+        [
+          [attribute_value(item, 'post_id').to_i, attachment],
+          [attribute_value(item, 'attachment_url'), attachment]
+        ]
+      end.flatten(1).to_h
     end
 
     # Find items by attribute and value

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sutty-migration
 version: !ruby/object:Gem::Version
-  version: 0.3.5
+  version: 0.4.0rc1
 platform: ruby
 authors:
 - f
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-09-30 00:00:00.000000000 Z
+date: 2026-10-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: jekyll
@@ -38,6 +38,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.1'
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: fast_blank
   requirement: !ruby/object:Gem::Requirement
@@ -150,6 +164,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.3.0
+- !ruby/object:Gem::Dependency
+  name: htmlbeautifier
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rails-html-sanitizer
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
@@ -164,7 +206,21 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: Takes datafiles and converts them into posts
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Takes datafiles, Wordpress backups and converts them into Jekyll posts
 email:
 - f@sutty.nl
 executables: []
@@ -175,6 +231,8 @@ extra_rdoc_files:
 files:
 - LICENSE.txt
 - README.md
+- lib/jekyll-wordpress-sql-migration.rb
+- lib/jekyll-wordpress-xml-migration.rb
 - lib/sutty-migration.rb
 - lib/sutty_migration/core_extensions.rb
 - lib/sutty_migration/data.rb
@@ -183,19 +241,18 @@ files:
 - lib/sutty_migration/wordpress_xml.rb
 - lib/sutty_migration/wordpress_xml/attachment.rb
 - lib/sutty_migration/wordpress_xml/post.rb
-homepage: https://0xacab.org/sutty/jekyll/sutty-migration
+homepage: https://git.sutty.abyaya.la/jekyll/sutty-migration
 licenses:
 - GPL-3.0
 metadata:
-  bug_tracker_uri: https://0xacab.org/sutty/jekyll/sutty-migration/issues
-  homepage_uri: https://0xacab.org/sutty/jekyll/sutty-migration
-  source_code_uri: https://0xacab.org/sutty/jekyll/sutty-migration
-  changelog_uri: https://0xacab.org/sutty/jekyll/sutty-migration/-/blob/master/CHANGELOG.md
+  bug_tracker_uri: https://git.sutty.abyaya.la/jekyll/sutty-migration/issues
+  homepage_uri: https://git.sutty.abyaya.la/jekyll/sutty-migration
+  source_code_uri: https://git.sutty.abyaya.la/jekyll/sutty-migration
   documentation_uri: https://rubydoc.info/gems/sutty-migration
 post_install_message:
 rdoc_options:
 - "--title"
-- sutty-migration - Jekyll data migration for Sutty
+- sutty-migration - CSV, Wordpress SQL and Wordpress XML migration
 - "--main"
 - README.md
 - "--line-numbers"
@@ -207,15 +264,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.6.0
+      version: '3.1'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubygems_version: 3.3.27
 signing_key:
 specification_version: 4
-summary: Jekyll data migration for Sutty
+summary: CSV, Wordpress SQL and Wordpress XML migration
 test_files: []