sutty-migration 0.1.2 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +48 -0
- data/lib/sutty-migration.rb +1 -69
- data/lib/sutty_migration/core_extensions.rb +26 -0
- data/lib/sutty_migration/data.rb +77 -0
- data/lib/sutty_migration/jekyll/document_creator.rb +41 -0
- data/lib/sutty_migration/wordpress.rb +230 -0
- data/lib/wordpress.rb +64 -46
- metadata +76 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 046cf945de1c0736e329224151a4b331c87e44cab6e6c2bc1e69f22b7639fe68
|
4
|
+
data.tar.gz: 58effbee202ab51c7ff1ed4e8e98498cd5a65f618c6ae6fc4f924fe6aed2e0e1
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a21cb549bddd9bc55218c0633932300811e547d2d4cfde2525fbcdcaf8a2ff3bb89f0542813b46fdb9b90c9d739166ce37e303fb4de76b918b52bee9402fcb6d
|
7
|
+
data.tar.gz: 544f18359b4e9996c07f6828643bb8d4856d55457b4a6134b2ef2cedf01d4471d294effc6024598ef66f9b1d2258345f44ac370e1678166a2b9d19b5c4a55e74
|
data/README.md
CHANGED
@@ -66,6 +66,54 @@ To start migration just build your site:
|
|
66
66
|
bundle exec jekyll build
|
67
67
|
```
|
68
68
|
|
69
|
+
**Tip:** Files can also be JSON, TSV and YAML, since they're all
|
70
|
+
supported by Jekyll.
|
71
|
+
|
72
|
+
### Wordpress
|
73
|
+
|
74
|
+
Instead of requiring you to install and configure MariaDB/MySQL, you can
|
75
|
+
convert the database into SQLite3 like this:
|
76
|
+
|
77
|
+
```bash
|
78
|
+
git clone https://0xacab.org/sutty/mysql2sqlite.git
|
79
|
+
cd mysql2sqlite
|
80
|
+
./mysql2sqlite /path/to/database/dump.sql |
|
81
|
+
sed -re "s/, 0x([0-9a-f]+),/, X'\1',/i" |
|
82
|
+
sqlite3 wordpress.sqlite3
|
83
|
+
```
|
84
|
+
|
85
|
+
It will probably show some errors.
|
86
|
+
|
87
|
+
Note the `sed` command is required to convert hexadecimal values into
|
88
|
+
SQLite syntax, since `mysql2sqlite` doesn't support this yet.
|
89
|
+
|
90
|
+
Wordpress websites can include lots of posts and metadata, depending on
|
91
|
+
the amount of plugins installed. We don't have an official way of
|
92
|
+
dumping everything into Jekyll, because you will probably want to move
|
93
|
+
things around. You can write a plugin like this:
|
94
|
+
|
95
|
+
```ruby
|
96
|
+
# _plugins/wordpress.rb
|
97
|
+
# frozen_string_literal: true
|
98
|
+
|
99
|
+
require 'sutty_migration/wordpress'
|
100
|
+
require 'sutty_migration/jekyll/document_creator'
|
101
|
+
require 'jekyll-write-and-commit-changes'
|
102
|
+
|
103
|
+
Jekyll::Hooks.register :site, :post_read, priority: :low do |site|
|
104
|
+
wp = SuttyMigration::Wordpress.new(site: site, database: 'wordpress.sqlite3', prefix: 'wp_', url: 'https://wordpre.ss')
|
105
|
+
|
106
|
+
# Download all files
|
107
|
+
wp.download_all
|
108
|
+
|
109
|
+
wp.posts(layout: 'post').each do |post|
|
110
|
+
doc = Jekyll::Document.create(site: site, title: post[:post_title], date: post[:post_date], collection: 'posts')
|
111
|
+
doc.content = post[:content]
|
112
|
+
doc.save
|
113
|
+
end
|
114
|
+
end
|
115
|
+
```
|
116
|
+
|
69
117
|
## Contributing
|
70
118
|
|
71
119
|
Bug reports and pull requests are welcome on 0xacab.org at
|
data/lib/sutty-migration.rb
CHANGED
@@ -1,71 +1,3 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
|
4
|
-
require 'fast_blank'
|
5
|
-
require 'jekyll-write-and-commit-changes'
|
6
|
-
|
7
|
-
Jekyll::Hooks.register :site, :post_read, priority: :low do |site|
|
8
|
-
documents = site.documents
|
9
|
-
|
10
|
-
site.data['layouts']&.each do |name, layout|
|
11
|
-
site.data.dig('migration', name)&.each do |row|
|
12
|
-
row['date'] = Jekyll::Utils.parse_date(row['date']) unless row['date'].blank?
|
13
|
-
|
14
|
-
if row['id']
|
15
|
-
document = documents.find do |doc|
|
16
|
-
doc.data['id'] == row['id']
|
17
|
-
end
|
18
|
-
end
|
19
|
-
|
20
|
-
document ||=
|
21
|
-
begin
|
22
|
-
base = "#{row['date'] || Date.today.to_s}-#{Jekyll::Utils.slugify(row['title'], mode: 'latin')}.markdown"
|
23
|
-
path = File.join(site.source, '_posts', base)
|
24
|
-
|
25
|
-
raise ArgumentError, "Row #{row['id']} duplicates file #{base}" if File.exist? path
|
26
|
-
|
27
|
-
doc = Jekyll::Document.new(path, site: site, collection: site.collections['posts'])
|
28
|
-
site.collections['posts'] << doc
|
29
|
-
|
30
|
-
doc
|
31
|
-
end
|
32
|
-
|
33
|
-
row.each do |attribute, value|
|
34
|
-
row[attribute] =
|
35
|
-
case layout.dig(attribute, 'type')
|
36
|
-
when 'string' then value
|
37
|
-
when 'text' then value
|
38
|
-
when 'tel' then value
|
39
|
-
when 'color' then value # TODO: validar
|
40
|
-
when 'date' then Jekyll::Utils.parse_date(value)
|
41
|
-
when 'email' then value # TODO: validar
|
42
|
-
when 'url' then value # TODO: validar
|
43
|
-
when 'content' then value
|
44
|
-
when 'markdown_content' then value
|
45
|
-
when 'markdown' then value
|
46
|
-
when 'number' then value.to_i
|
47
|
-
when 'order' then value.to_i
|
48
|
-
when 'boolean' then !value.strip.empty?
|
49
|
-
when 'array' then value.split(',').map(&:strip)
|
50
|
-
# TODO: procesar los valores en base a los valores predefinidos
|
51
|
-
when 'predefined_array' then value.split(',').map(&:strip)
|
52
|
-
when 'image' then { 'path' => value, 'description' => '' }
|
53
|
-
when 'file' then { 'path' => value, 'description' => '' }
|
54
|
-
when 'geo' then %w[lat lng].zip(value.split(',', 2).map(&:to_f)).to_h
|
55
|
-
when 'belongs_to' then value
|
56
|
-
when 'has_many' then value.split(',').map(&:strip)
|
57
|
-
when 'has_and_belongs_to_many' then value.split(',').map(&:strip)
|
58
|
-
when 'related_posts' then value.split(',').map(&:strip)
|
59
|
-
when 'locales' then value.split(',').map(&:strip)
|
60
|
-
else value
|
61
|
-
end
|
62
|
-
end
|
63
|
-
|
64
|
-
document.data['uuid'] ||= SecureRandom.uuid
|
65
|
-
document.content = row.delete('content')
|
66
|
-
|
67
|
-
document.data.merge! row
|
68
|
-
document.save
|
69
|
-
end
|
70
|
-
end
|
71
|
-
end
|
3
|
+
require_relative 'sutty_migration/data'
|
@@ -0,0 +1,26 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# Expandir String para poder verificar si está vacía
|
4
|
+
require 'fast_blank'
|
5
|
+
|
6
|
+
# Verificar que los valores nulos estén vacíos
|
7
|
+
class NilClass
|
8
|
+
def blank?
|
9
|
+
true
|
10
|
+
end
|
11
|
+
|
12
|
+
def present?
|
13
|
+
false
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
# Verificar que una fecha está vacía
|
18
|
+
class Time
|
19
|
+
def blank?
|
20
|
+
false
|
21
|
+
end
|
22
|
+
|
23
|
+
def present?
|
24
|
+
true
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,77 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'securerandom'
|
4
|
+
require_relative 'core_extensions'
|
5
|
+
require_relative 'jekyll/document_creator'
|
6
|
+
|
7
|
+
# Registers a plugin for converting CSV files into posts following
|
8
|
+
# Sutty's layout definition.
|
9
|
+
#
|
10
|
+
# If jekyll-write-and-commit-changes is enabled, documents will be saved
|
11
|
+
# on disk and commited is the build command is run with
|
12
|
+
# JEKYLL_ENV=production
|
13
|
+
Jekyll::Hooks.register :site, :post_read, priority: :low do |site|
|
14
|
+
documents = site.documents
|
15
|
+
|
16
|
+
site.data['layouts']&.each do |name, layout|
|
17
|
+
site.data.dig('migration', name)&.each do |row|
|
18
|
+
row['date'] = Jekyll::Utils.parse_date(row['date']) unless row['date'].blank?
|
19
|
+
row['date'] ||= Time.now
|
20
|
+
|
21
|
+
unless row['id'].blank?
|
22
|
+
document = documents.find do |doc|
|
23
|
+
doc.data['id'] == row['id']
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
document ||= Jekyll::Document.create(site: site, collection: 'posts', **row.slice(*%w[date slug title]).transform_keys(&:to_sym))
|
28
|
+
|
29
|
+
row.each do |attribute, value|
|
30
|
+
next unless value.blank?
|
31
|
+
|
32
|
+
row[attribute] =
|
33
|
+
case layout.dig(attribute, 'type')
|
34
|
+
when 'string' then value
|
35
|
+
when 'text' then value
|
36
|
+
when 'tel' then value
|
37
|
+
# TODO: validate
|
38
|
+
when 'color' then value
|
39
|
+
when 'date' then Jekyll::Utils.parse_date(value)
|
40
|
+
# TODO: validate
|
41
|
+
when 'email' then value
|
42
|
+
# TODO: validate
|
43
|
+
when 'url' then value
|
44
|
+
when 'content' then value
|
45
|
+
when 'markdown_content' then value
|
46
|
+
when 'markdown' then value
|
47
|
+
when 'number' then value.to_i
|
48
|
+
when 'order' then value.to_i
|
49
|
+
when 'boolean' then !value.strip.empty?
|
50
|
+
when 'array' then value.split(',').map(&:strip)
|
51
|
+
# TODO: process values from the default array
|
52
|
+
when 'predefined_array' then value.split(',').map(&:strip)
|
53
|
+
when 'image' then { 'path' => value, 'description' => '' }
|
54
|
+
when 'file' then { 'path' => value, 'description' => '' }
|
55
|
+
when 'geo' then %w[lat lng].zip(value.split(',', 2).map(&:to_f)).to_h
|
56
|
+
when 'belongs_to' then value
|
57
|
+
when 'has_many' then value.split(',').map(&:strip)
|
58
|
+
when 'has_and_belongs_to_many' then value.split(',').map(&:strip)
|
59
|
+
when 'related_posts' then value.split(',').map(&:strip)
|
60
|
+
when 'locales' then value.split(',').map(&:strip)
|
61
|
+
else value
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
document.data['uuid'] ||= SecureRandom.uuid
|
66
|
+
document.content = row.delete('content')
|
67
|
+
|
68
|
+
document.data.merge! row
|
69
|
+
document.save if document.respond_to? :save
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
next unless site.respond_to?(:repository)
|
74
|
+
next unless ENV['JEKYLL_ENV'] == 'production'
|
75
|
+
|
76
|
+
site.repository.commit 'CSV Migration'
|
77
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'jekyll/utils'
|
4
|
+
require_relative '../core_extensions'
|
5
|
+
|
6
|
+
module SuttyMigration
|
7
|
+
module Jekyll
|
8
|
+
module DocumentCreator
|
9
|
+
class DocumentExists < ArgumentError; end
|
10
|
+
def self.included(base)
|
11
|
+
base.class_eval do
|
12
|
+
|
13
|
+
# Creates a new document in a collection or fails if it already
|
14
|
+
# exists.
|
15
|
+
#
|
16
|
+
# @param :site [Jekyll::Site] Jekyll site
|
17
|
+
# @param :date [Time] Post date
|
18
|
+
# @param :title [String] Post title
|
19
|
+
# @param :slug [String] Post slug, slugified title if empty
|
20
|
+
# @param :collection [Jekyll::Collection,String] Collection label or collection
|
21
|
+
# @return [Jekyll::Document] A new document
|
22
|
+
def self.create(site:, date:, title:, slug: nil, collection:)
|
23
|
+
collection = site.collections[collection] if collection.is_a? String
|
24
|
+
slug = ::Jekyll::Utils.slugify(title, mode: 'latin') if slug.blank?
|
25
|
+
basename = "#{date.strftime('%F')}-#{slug}.markdown"
|
26
|
+
path = File.join(collection.directory, basename)
|
27
|
+
|
28
|
+
raise DocumentExists, "#{path} already exists" if File.exist? path
|
29
|
+
|
30
|
+
::Jekyll::Document.new(path, site: site, collection: collection).tap do |document|
|
31
|
+
collection.docs << document
|
32
|
+
document.data['title'] = title
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
::Jekyll::Document.include SuttyMigration::Jekyll::DocumentCreator
|
@@ -0,0 +1,230 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'time'
|
4
|
+
require 'securerandom'
|
5
|
+
require 'sequel'
|
6
|
+
require 'sqlite3'
|
7
|
+
require 'json'
|
8
|
+
require 'faraday'
|
9
|
+
require 'progressbar'
|
10
|
+
require 'jekyll/utils'
|
11
|
+
|
12
|
+
module SuttyMigration
|
13
|
+
# Brings posts and attachments from a SQLite3 database. You can
|
14
|
+
# convert a MySQL/MariaDB dump by using `mysql2sqlite`.
|
15
|
+
#
|
16
|
+
# It doesn't convert them into Jekyll posts but allows you to write a
|
17
|
+
# migration plugin where you can convert data by yourself. We may add
|
18
|
+
# this feature in the future.
|
19
|
+
class Wordpress
|
20
|
+
attr_reader :site, :prefix, :limit, :url, :wp, :database, :multisite
|
21
|
+
|
22
|
+
# @param :site [Jekyll::Site] Jekyll site
|
23
|
+
# @param :url [String] Wordpress site URL (must be up for downloads)
|
24
|
+
# @param :database [String] Database path, by default `_data/wordpress.sqlite3`
|
25
|
+
# @param :prefix [String] WP table prefix
|
26
|
+
# @param :limit [Integer] Page length
|
27
|
+
# @param :multisite [Boolean] Site is multisite
|
28
|
+
def initialize(site:, url:, database: nil, prefix: 'wp_', limit: 10, multisite: nil)
|
29
|
+
@site = site
|
30
|
+
@prefix = prefix.freeze
|
31
|
+
@limit = limit.freeze
|
32
|
+
@url = url.freeze
|
33
|
+
@database = database || File.join(site.source, '_data', 'wordpress.sqlite3')
|
34
|
+
@multisite = multisite
|
35
|
+
end
|
36
|
+
|
37
|
+
# Generate database connections for a multisite WP
|
38
|
+
#
|
39
|
+
# @return [Hash] { "ID" => SuttyMigration::Wordpress }
|
40
|
+
def blogs
|
41
|
+
@blogs ||= wp["select blog_id as id, domain, path from #{prefix}blogs"].to_a.map do |blog|
|
42
|
+
url = "https://#{blog[:domain]}#{blog[:path]}"
|
43
|
+
pfx = "#{prefix}#{blog[:id]}_" if blog[:id] > 1
|
44
|
+
pfx ||= prefix
|
45
|
+
|
46
|
+
[ blog[:id], self.class.new(site: site, url: url, prefix: pfx, database: database, limit: limit, multisite: self) ]
|
47
|
+
end.to_h
|
48
|
+
end
|
49
|
+
|
50
|
+
# Open the database.
|
51
|
+
#
|
52
|
+
# @return [Sequel::SQLite::Database]
|
53
|
+
def wp
|
54
|
+
@wp ||= Sequel.sqlite(database).tap do |db|
|
55
|
+
db.extension :pagination
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
# Download all attachments. Adds the local path to them.
|
60
|
+
#
|
61
|
+
# @param :progress [Boolean] Toggle progress bar
|
62
|
+
# @return [Nil]
|
63
|
+
def download_all(progress: true)
|
64
|
+
posts(layout: 'attachment').each do |attachment|
|
65
|
+
attachment[:front_matter]['file_path'] = download(url: attachment[:guid], progress: progress)
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
# Downloads a file if needed, optionally showing a progress bar.
|
70
|
+
#
|
71
|
+
# @param :url [String] File URL
|
72
|
+
# @param :progress [Boolean] Toggle progress bar
|
73
|
+
# @return [String] File local path
|
74
|
+
def download(url:, progress: true)
|
75
|
+
uri = URI(url)
|
76
|
+
dest = uri.path.sub(%r{\A/}, '')
|
77
|
+
full = File.join(site.source, dest)
|
78
|
+
|
79
|
+
return dest if File.exist? full
|
80
|
+
|
81
|
+
::Jekyll.logger.info "Downloading #{dest}"
|
82
|
+
|
83
|
+
FileUtils.mkdir_p File.dirname(full)
|
84
|
+
|
85
|
+
File.open(full, 'w') do |f|
|
86
|
+
if progress
|
87
|
+
head = Faraday.head(url)
|
88
|
+
content_length = head.headers['content-length'].to_i
|
89
|
+
progress = ProgressBar.create(title: File.basename(dest), total: content_length, output: $stderr)
|
90
|
+
end
|
91
|
+
|
92
|
+
Faraday.get(url) do |req|
|
93
|
+
req.options.on_data = Proc.new do |chunk, downloaded_bytes|
|
94
|
+
f.write chunk
|
95
|
+
|
96
|
+
if progress
|
97
|
+
progress.progress = (downloaded_bytes > content_length) ? content_length : downloaded_bytes
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
dest
|
104
|
+
end
|
105
|
+
|
106
|
+
# List post types
|
107
|
+
#
|
108
|
+
# @return [Array]
|
109
|
+
def layouts
|
110
|
+
@layouts ||= wp["select distinct post_type from #{prefix}posts"].to_a.map(&:values).flatten
|
111
|
+
end
|
112
|
+
|
113
|
+
# Finds all posts optionally filtering by post type. This is not
|
114
|
+
# the official Sequel syntax, but it retrieves metadata as objects
|
115
|
+
# with a single query (and a sub-query).
|
116
|
+
#
|
117
|
+
# @param :layout [String] Layout name, one of #layouts
|
118
|
+
# @param :with_meta [Boolean] Toggle metadata pulling and conversion
|
119
|
+
# @return [Enumerator]
|
120
|
+
def posts(**options)
|
121
|
+
unless options[:layout].blank? || layouts.include?(options[:layout])
|
122
|
+
raise ArgumentError, "#{options[:layout]} must be one of #{layouts.join(', ')}"
|
123
|
+
end
|
124
|
+
|
125
|
+
wp[post_query(**options)].each_page(limit).to_a.map(&:to_a).flatten.tap do |p|
|
126
|
+
p.map do |post|
|
127
|
+
# Sequel parses dates on localtime
|
128
|
+
post[:date] = ::Jekyll::Utils.parse_date(post[:date]) unless post[:date].blank?
|
129
|
+
post[:last_modified_at] = ::Jekyll::Utils.parse_date(post[:last_modified_at]) unless post[:last_modified_at].blank?
|
130
|
+
|
131
|
+
post[:front_matter] = JSON.parse(post[:front_matter]).transform_keys(&:to_sym) unless post[:front_matter].blank?
|
132
|
+
post[:terms] = JSON.parse(post[:terms]).transform_keys(&:to_sym) unless post[:terms].blank?
|
133
|
+
end
|
134
|
+
end
|
135
|
+
end
|
136
|
+
|
137
|
+
# Brings all users.
|
138
|
+
#
|
139
|
+
# @param :with_meta [Boolean] include metadata
|
140
|
+
# @return [Array]
|
141
|
+
def users(**options)
|
142
|
+
options[:with_meta] = true unless options.key? :with_meta
|
143
|
+
|
144
|
+
wp[user_query(**options)].each_page(limit).to_a.map(&:to_a).flatten.tap do |u|
|
145
|
+
next unless options[:with_meta]
|
146
|
+
|
147
|
+
u.map do |user|
|
148
|
+
user[:meta] = JSON.parse(user[:meta]).transform_keys(&:to_sym) unless user[:meta].blank?
|
149
|
+
end
|
150
|
+
end
|
151
|
+
end
|
152
|
+
|
153
|
+
private
|
154
|
+
|
155
|
+
# Finds all users. If it's a multisite WP, we need to check the
|
156
|
+
# main table.
|
157
|
+
#
|
158
|
+
# @param :with_meta [Boolean] include metadata
|
159
|
+
# @return [String]
|
160
|
+
def user_query(with_meta: true)
|
161
|
+
pfx = multisite&.prefix || prefix
|
162
|
+
|
163
|
+
<<~EOQ
|
164
|
+
select
|
165
|
+
u.*
|
166
|
+
#{", json_group_object(m.meta_key, m.meta_value) as meta" if with_meta}
|
167
|
+
from #{pfx}users as u
|
168
|
+
#{"left join #{pfx}usermeta as m on m.user_id = u.id" if with_meta}
|
169
|
+
group by u.id
|
170
|
+
EOQ
|
171
|
+
end
|
172
|
+
|
173
|
+
# Query for posts, optionally bringing metadata as JSON objects.
|
174
|
+
#
|
175
|
+
# @param :layout [String] Layout name
|
176
|
+
# @param :with_meta [Boolean] Query metadata
|
177
|
+
# @return [String]
|
178
|
+
def post_query(layout: nil, with_meta: true)
|
179
|
+
<<~EOQ
|
180
|
+
select
|
181
|
+
p.ID as id,
|
182
|
+
strftime('%Y-%m-%d %H:%M:%S UTC', p.post_date_gmt) as date,
|
183
|
+
strftime('%Y-%m-%d %H:%M:%S UTC', p.post_modified_gmt) as last_modified_at,
|
184
|
+
p.post_author as author,
|
185
|
+
p.post_type as layout,
|
186
|
+
p.post_name as slug,
|
187
|
+
p.post_title as title,
|
188
|
+
p.post_content as content,
|
189
|
+
p.post_excerpt as excerpt,
|
190
|
+
p.post_status as status,
|
191
|
+
p.comment_status as comment_status,
|
192
|
+
p.ping_status as ping_status,
|
193
|
+
p.post_password as password,
|
194
|
+
p.to_ping as to_ping,
|
195
|
+
p.pinged as pinged,
|
196
|
+
p.post_content_filtered as content_filtered,
|
197
|
+
p.post_parent as parent,
|
198
|
+
p.guid as guid,
|
199
|
+
p.menu_order as menu_order,
|
200
|
+
p.post_mime_type as mime_type,
|
201
|
+
p.comment_count as comment_count
|
202
|
+
#{", json_group_object(f.meta_key, f.meta_value) as front_matter" if with_meta}
|
203
|
+
#{", t.terms as terms" if with_meta}
|
204
|
+
from #{prefix}posts as p
|
205
|
+
left join #{prefix}postmeta as f on p.ID = f.post_id
|
206
|
+
#{"left join (#{terms_query(layout: layout)}) as t on t.id = p.ID" if with_meta}
|
207
|
+
#{"where p.post_type = '#{layout}'" if layout}
|
208
|
+
group by p.ID
|
209
|
+
EOQ
|
210
|
+
end
|
211
|
+
|
212
|
+
# Term taxonomy query
|
213
|
+
#
|
214
|
+
# @param :layout [String] Layout name
|
215
|
+
# @return [String]
|
216
|
+
def terms_query(layout: nil)
|
217
|
+
<<~EOQ
|
218
|
+
select
|
219
|
+
p.ID as id,
|
220
|
+
json_group_object(tt.taxonomy, t.name) as terms
|
221
|
+
from #{prefix}posts as p
|
222
|
+
left join #{prefix}term_relationships as r on r.object_id = p.ID
|
223
|
+
left join #{prefix}term_taxonomy as tt on tt.term_taxonomy_id = r.term_taxonomy_id
|
224
|
+
left join #{prefix}terms as t on t.term_id = tt.term_id
|
225
|
+
#{"where p.post_type = '#{layout}'" if layout}
|
226
|
+
group by p.ID
|
227
|
+
EOQ
|
228
|
+
end
|
229
|
+
end
|
230
|
+
end
|
data/lib/wordpress.rb
CHANGED
@@ -1,20 +1,14 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
# Debug
|
4
|
-
require 'pry'
|
5
3
|
# Generar UUIDs
|
6
4
|
require 'securerandom'
|
7
5
|
# Traer resultados de la base de datos
|
8
6
|
require 'sequel'
|
9
7
|
require 'sqlite3'
|
10
8
|
require 'json'
|
11
|
-
# Limpieza de contenido
|
12
|
-
require 'loofah'
|
13
|
-
require 'rails/html/scrubbers'
|
14
|
-
require 'rails/html/sanitizer'
|
15
|
-
require 'reverse_markdown'
|
16
9
|
# Descargar archivos
|
17
10
|
require 'faraday'
|
11
|
+
require 'progressbar'
|
18
12
|
|
19
13
|
class Wordpress
|
20
14
|
attr_reader :site, :prefix, :limit, :url
|
@@ -33,7 +27,7 @@ class Wordpress
|
|
33
27
|
end
|
34
28
|
|
35
29
|
def download(file)
|
36
|
-
dest =
|
30
|
+
dest = "wp-content/uploads/#{file}"
|
37
31
|
full = File.join(site.source, dest)
|
38
32
|
|
39
33
|
return dest if File.exist? full
|
@@ -43,8 +37,13 @@ class Wordpress
|
|
43
37
|
FileUtils.mkdir_p File.dirname(full)
|
44
38
|
|
45
39
|
File.open(full, 'w') do |f|
|
46
|
-
|
47
|
-
|
40
|
+
url = "#{url}/#{dest}"
|
41
|
+
head = Faraday.head(url)
|
42
|
+
content_length = head.headers['content-length']
|
43
|
+
progress_bar = ProgressBar.new
|
44
|
+
|
45
|
+
Faraday.get(url) do |req|
|
46
|
+
req.options.on_data = Proc.new do |chunk, downloaded_bytes|
|
48
47
|
f.write chunk
|
49
48
|
end
|
50
49
|
end
|
@@ -53,56 +52,75 @@ class Wordpress
|
|
53
52
|
dest
|
54
53
|
end
|
55
54
|
|
55
|
+
# Obtiene todos los tipos de artículos disponibles
|
56
|
+
#
|
57
|
+
# @return [Array]
|
58
|
+
def layouts
|
59
|
+
@layouts ||= @wp["select distinct post_type from #{prefix}posts"].to_a.map(&:values).flatten
|
60
|
+
end
|
61
|
+
|
56
62
|
# Obtiene todos los posts opcionalmente filtrando por tipo de post.
|
57
63
|
# No es la forma oficial de Sequel pero no tenemos tiempo de
|
58
64
|
# aprenderla específicamente y además tenemos las opciones en formato
|
59
65
|
# JSON que no estarían soportadas.
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
66
|
+
#
|
67
|
+
# @param :layout [String] Layout name, one of #layouts
|
68
|
+
# @param :with_meta [Boolean]
|
69
|
+
# @return [Enumerator]
|
70
|
+
def posts(**options)
|
71
|
+
if options[:layout] && !layouts.include?(options[:layout])
|
72
|
+
raise ArgumentError, "#{layout} must be one of #{layouts.join(', ')}"
|
73
|
+
end
|
64
74
|
|
65
|
-
@
|
66
|
-
|
75
|
+
@posts ||= {}
|
76
|
+
@posts[options[:layout] || 'all'] ||= @wp[post_query(**options)].each_page(limit).to_a.map(&:to_a).flatten.tap do |p|
|
77
|
+
next unless options[:with_meta]
|
67
78
|
|
68
|
-
|
69
|
-
|
79
|
+
p.map do |post|
|
80
|
+
post[:front_matter] = JSON.parse(post[:front_matter]) unless post[:front_matter].nil?
|
81
|
+
post[:terms] = JSON.parse(post[:terms]) unless post[:terms].nil?
|
82
|
+
end
|
83
|
+
end
|
70
84
|
end
|
71
85
|
|
72
86
|
private
|
73
87
|
|
74
|
-
#
|
75
|
-
|
88
|
+
# Consulta para los posts, incluyendo metadatos en JSON. Los
|
89
|
+
# metadatos vienen en dos partes porque tienen dos
|
90
|
+
#
|
91
|
+
# @return [String]
|
92
|
+
def post_query(layout: nil, with_meta: true)
|
76
93
|
@post_query ||= <<~EOQ
|
77
|
-
select
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
94
|
+
select
|
95
|
+
p.ID as id,
|
96
|
+
p.post_title as title,
|
97
|
+
p.post_name as slug,
|
98
|
+
p.post_type as layout,
|
99
|
+
p.strftime('%Y-%m-%d', post_date) as date,
|
100
|
+
p.post_status as status,
|
101
|
+
p.post_content as content
|
102
|
+
#{", json_group_object(f.meta_key, f.meta_value) as front_matter" if with_meta}
|
103
|
+
#{", t.meta as meta" if with_meta}
|
104
|
+
from #{prefix}posts as p
|
105
|
+
left join #{prefix}postmeta as f on p.ID = f.post_id
|
106
|
+
#{"left join (#{meta_query(layout: layout)}) as as t on t.id = p.ID" if with_meta}
|
107
|
+
#{"where p.post_type = :layout" if layout}
|
108
|
+
group by p.ID
|
88
109
|
EOQ
|
89
110
|
end
|
90
111
|
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
trels.object_id = '#{id}' AND
|
104
|
-
trels.term_taxonomy_id = ttax.term_taxonomy_id AND
|
105
|
-
terms.term_id = ttax.term_id
|
112
|
+
#
|
113
|
+
def meta_query(layout: nil)
|
114
|
+
@meta_query ||= <<~EOQ
|
115
|
+
select
|
116
|
+
p.ID as id,
|
117
|
+
json_group_object(tt.taxonomy, t.name) as meta
|
118
|
+
from #{prefix}posts as p
|
119
|
+
left join #{prefix}term_relationships as r on r.object_id = p.ID
|
120
|
+
left join #{prefix}term_taxonomy as tt on tt.term_taxonomy_id = r.term_taxonomy_id
|
121
|
+
left join #{prefix}terms as t on t.term_id = tt.term_id
|
122
|
+
#{"where p.post_type = :layout" if layout}
|
123
|
+
group by p.ID
|
106
124
|
EOQ
|
107
125
|
end
|
108
126
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sutty-migration
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- f
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-
|
11
|
+
date: 2021-06-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: jekyll
|
@@ -52,6 +52,76 @@ dependencies:
|
|
52
52
|
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: '1.0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: faraday
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '1.4'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '1.4'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: progressbar
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - "~>"
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '1.11'
|
76
|
+
type: :runtime
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - "~>"
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '1.11'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: sqlite3
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - "~>"
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '1.4'
|
90
|
+
type: :runtime
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - "~>"
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '1.4'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: sequel
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - "~>"
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '5.45'
|
104
|
+
type: :runtime
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - "~>"
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '5.45'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: pry
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - ">="
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
type: :development
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - ">="
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '0'
|
55
125
|
description: Takes datafiles and converts them into posts
|
56
126
|
email:
|
57
127
|
- f@sutty.nl
|
@@ -64,6 +134,10 @@ files:
|
|
64
134
|
- LICENSE.txt
|
65
135
|
- README.md
|
66
136
|
- lib/sutty-migration.rb
|
137
|
+
- lib/sutty_migration/core_extensions.rb
|
138
|
+
- lib/sutty_migration/data.rb
|
139
|
+
- lib/sutty_migration/jekyll/document_creator.rb
|
140
|
+
- lib/sutty_migration/wordpress.rb
|
67
141
|
- lib/wordpress.rb
|
68
142
|
homepage: https://0xacab.org/sutty/jekyll/sutty-migration
|
69
143
|
licenses:
|