abqari 1.1.0

data/lib/abqari/importers/jekyll.rb

@@ -0,0 +1,256 @@
+# frozen_string_literal: true
+
+require 'yaml'
+require 'fileutils'
+
+module Abqari
+  module Importers
+    # Jekyll site importer.
+    #
+    # Reads `_posts/YYYY-MM-DD-slug.md(.markdown|.html)` files. Each
+    # file is already markdown + YAML frontmatter — the importer's
+    # job is mostly translation:
+    #
+    #   * Move from Jekyll's flat `_posts/` to Abqari's bundle
+    #     layout (`content/posts/<slug>/index.md` + colocated
+    #     assets).
+    #   * Map `categories:` → `tags:` (Jekyll has both; Abqari
+    #     just uses tags).
+    #   * Copy referenced images from `assets/posts/` (or
+    #     wherever they live) alongside the new bundle.
+    #   * Warn on Liquid template tags (`{% include %}`,
+    #     `{% post_url %}`, `{% raw %}`) — those need manual
+    #     attention.
+    #
+    # `.html` posts (Jekyll allows raw HTML) are converted via
+    # `HtmlToMarkdown` like any other importer source. Pure-markdown
+    # posts are passed through unchanged.
+    class Jekyll < Base
+      LIQUID_TAG_RE = /\{%\s*(?!comment|endcomment|raw|endraw)\w+/.freeze
+
+      def self.strip_selectors
+        [] # Jekyll authors control their own HTML; no platform chrome
+      end
+
+      def each_post
+        posts_dir = File.join(@source, '_posts')
+        raise "Jekyll source missing _posts/ at #{posts_dir}" unless File.directory?(posts_dir)
+
+        Dir.glob(File.join(posts_dir, '*')).sort.each do |path|
+          next unless File.file?(path)
+          ext = File.extname(path).downcase
+          next unless %w[.md .markdown .html].include?(ext)
+
+          parsed = parse(path)
+          next unless parsed
+
+          fm   = parsed[:frontmatter]
+          body = parsed[:body]
+
+          # Convert body if the source is HTML; otherwise keep it
+          # as markdown.
+          if ext == '.html'
+            body_html = body
+          else
+            # Wrap markdown in a single root so the base class's
+            # HtmlToMarkdown pass doesn't try to parse it — we
+            # pass the raw markdown through via `body_html` that
+            # IS already markdown, which is harmless because the
+            # base class re-runs HtmlToMarkdown over it. The
+            # safer path: bypass conversion entirely for .md files.
+            body_html = nil
+          end
+
+          yield Base::ImportedPost.new(
+            title:             fm['title'] || derive_title_from_path(path),
+            slug:              fm['slug'] || derive_slug_from_path(path),
+            date:              fm['date'].is_a?(Date) ? fm['date'] : parse_date(fm['date']) || file_date(path),
+            body_html:         body_html,
+            tags:              merge_taxonomies(fm),
+            description:       fm['description'] || fm['excerpt'],
+            feature_image_url: nil, # local-only in Jekyll; handled via raw_body below
+            source_url:        nil,
+            source_slug:       File.basename(path, '.*').sub(/\A\d{4}-\d{2}-\d{2}-/, ''),
+            extra_frontmatter: jekyll_extras(fm).merge('__raw_body' => body, '__source_path' => path)
+          )
+        end
+      end
+
+      # The base class's writer expects `body_html` to be HTML it
+      # can convert. For pure-markdown Jekyll posts, we want to
+      # skip conversion and write the body straight through. This
+      # hook lets the importer override the body-write step on a
+      # per-post basis without forking the base writer.
+      #
+      # The base writer doesn't currently call this — instead, we
+      # use `extra_frontmatter['__raw_body']` as a sentinel that
+      # the base writer ignores (it doesn't know about the key)
+      # and then post-process in the importer's `run!` override.
+      #
+      # Cleaner: extend Base::Options with a `body_passthrough`
+      # block. Future refactor.
+      def run!
+        @raw_bodies = {} # slug → raw markdown body, populated during each_post
+        each_post do |post|
+          if post.extra_frontmatter['__raw_body']
+            @raw_bodies[post.slug] = post.extra_frontmatter['__raw_body']
+            # Strip internal keys before they hit the writer.
+            post.extra_frontmatter.delete('__raw_body')
+            source_path = post.extra_frontmatter.delete('__source_path')
+            post.extra_frontmatter['_source_file'] = File.basename(source_path) if source_path
+          end
+          process(post)
+        end
+        finalise
+        @report
+      end
+
+      private
+
+      # Override the base class's body conversion when we already
+      # have markdown. Jekyll-specific behaviour, isolated here.
+      def process(post)
+        raw_md = @raw_bodies && @raw_bodies[post.slug]
+        if raw_md
+          process_markdown_post(post, raw_md)
+        else
+          super
+        end
+      end
+
+      def process_markdown_post(post, raw_md)
+        slug  = resolve_slug(post)
+        title = post.title.to_s.strip
+        title = '(untitled)' if title.empty?
+
+        bundle_dir = bundle_dir_for(slug, post.date)
+        index_path = index_path_for(bundle_dir, slug)
+
+        if File.exist?(index_path) && !@opts.force
+          @report.add(slug: slug, title: title, status: :skipped,
+                      reason: "already exists at #{rel(index_path)}")
+          @logger.call("  ⤼ skip: #{slug} (already exists)")
+          return
+        end
+
+        @logger.call("  → #{slug}")
+        warnings = []
+
+        if raw_md =~ LIQUID_TAG_RE
+          warnings << "Liquid template tag detected — needs manual conversion to ERB or removal"
+        end
+
+        # Copy colocated assets from the source post bundle (if it's
+        # already a Jekyll bundle layout) or from the conventional
+        # `assets/<slug>/` directory.
+        copy_colocated_assets(post, bundle_dir, warnings) unless @opts.dry_run
+
+        frontmatter = build_frontmatter(post, feature_image: nil)
+        content     = format_yaml(frontmatter) + "\n" + raw_md.sub(/\n*\z/, "\n")
+
+        unless @opts.dry_run
+          FileUtils.mkdir_p(bundle_dir) unless @opts.flat
+          File.write(index_path, content)
+        end
+
+        @report.add(slug: slug, title: title, status: :imported,
+                    reason: rel(index_path), warnings: warnings)
+      rescue StandardError => e
+        @report.add(slug: slug || '(unknown)', title: title || '(unknown)',
+                    status: :failed,
+                    reason: "#{e.class}: #{e.message}")
+        @logger.call("  ✗ #{slug}: #{e.class}: #{e.message}")
+      end
+
+      def copy_colocated_assets(post, bundle_dir, warnings)
+        # Look in two conventional spots for image assets:
+        #   1. assets/posts/<source_slug>/
+        #   2. assets/<source_slug>/
+        # Both patterns are common in Jekyll's wild.
+        source_slug = post.source_slug.to_s
+        return if source_slug.empty?
+
+        candidates = [
+          File.join(@source, 'assets', 'posts', source_slug),
+          File.join(@source, 'assets', source_slug)
+        ]
+        src = candidates.find { |c| File.directory?(c) }
+        return unless src
+
+        FileUtils.mkdir_p(bundle_dir)
+        Dir.glob(File.join(src, '*')).each do |file|
+          next unless File.file?(file)
+          FileUtils.cp(file, File.join(bundle_dir, File.basename(file)))
+        end
+      end
+
+      def parse(path)
+        raw = File.read(path, encoding: 'UTF-8')
+        if (m = raw.match(Abqari::FRONTMATTER_RE))
+          fm = YAML.safe_load(m[1], permitted_classes: Abqari::YAML_PERMITTED_CLASSES, aliases: true) || {}
+          { frontmatter: fm, body: m[2] }
+        else
+          { frontmatter: {}, body: raw }
+        end
+      rescue Psych::Exception => e
+        @report.add(slug: File.basename(path, '.*'), title: File.basename(path), status: :failed,
+                    reason: "YAML error: #{e.message}")
+        nil
+      end
+
+      def merge_taxonomies(fm)
+        tags       = Array(fm['tags'])
+        categories = Array(fm['categories'] || fm['category'])
+        (tags + categories).map { |t| t.to_s.strip }.reject(&:empty?).uniq
+      end
+
+      def derive_title_from_path(path)
+        base = File.basename(path, '.*').sub(/\A\d{4}-\d{2}-\d{2}-/, '')
+        base.tr('-', ' ').capitalize
+      end
+
+      def derive_slug_from_path(path)
+        File.basename(path, '.*').sub(/\A\d{4}-\d{2}-\d{2}-/, '')
+      end
+
+      def file_date(path)
+        m = File.basename(path).match(/\A(\d{4})-(\d{2})-(\d{2})-/)
+        return nil unless m
+        Date.new(m[1].to_i, m[2].to_i, m[3].to_i)
+      rescue ArgumentError
+        nil
+      end
+
+      def parse_date(value)
+        return nil if value.to_s.empty?
+        return value if value.is_a?(Date)
+        Date.parse(value.to_s)
+      rescue ArgumentError
+        nil
+      end
+
+      # Layout names imported from an untrusted export are resolved to a
+      # file path and ERB-eval'd at build time, so a traversing value is
+      # a code-execution vector. Only carry through flat/nested layout
+      # identifiers; anything with `.`, `\`, a leading `/`, or other
+      # oddities is dropped so the imported post falls back to the
+      # collection default.
+      SAFE_LAYOUT = %r{\A[A-Za-z0-9][A-Za-z0-9_\-/]*\z}.freeze
+
+      def jekyll_extras(fm)
+        # Preserve frontmatter keys Abqari understands. Anything else
+        # is dropped silently — Jekyll's frontmatter is wildly diverse
+        # and we don't want to inherit unknowns into the new site's
+        # schema.
+        keep = %w[layout permalink redirect_from author authors series share_image]
+        extras = fm.slice(*keep).reject { |_, v| v.nil? || v == '' }
+        extras.delete('layout') if extras['layout'] && !extras['layout'].to_s.match?(SAFE_LAYOUT)
+        extras
+      end
+
+      def redirects_for(_source_slug)
+        nil # Jekyll posts usually keep the same slug; nothing to forward
+      end
+    end
+  end
+end

data/lib/abqari/importers/substack.rb

@@ -0,0 +1,204 @@
+# frozen_string_literal: true
+
+require 'csv'
+require 'fileutils'
+require 'tmpdir'
+require 'uri'
+
+module Abqari
+  module Importers
+    # Substack export importer.
+    #
+    # Substack's export shape (as of 2025–2026):
+    #
+    #   substack-export/
+    #     posts.csv              # post_id, post_date, title, subtitle,
+    #                            #   type, audience, status, slug,
+    #                            #   email_sent_at, ...
+    #     posts/
+    #       12345.html
+    #       67890.html
+    #
+    # Some columns vary across export vintages (Substack rolls schema
+    # changes silently). The reader is tolerant — we look up columns
+    # by name, default missing ones to nil, and never assume column
+    # ordering.
+    #
+    # Source accepts both a `.zip` (auto-extracted to a tmp dir) and
+    # an already-unzipped directory. Zip extraction uses the shell
+    # `unzip` so we don't pull in rubyzip as a dep — the operator
+    # is running this from a CLI environment that has `unzip` on
+    # every Mac/Linux system.
+    class Substack < Base
+      # Substack chrome to strip before HTML→Markdown conversion.
+      # Each entry is a `tag.class` pair that HtmlToMarkdown can
+      # parse — see `Support::HtmlToMarkdown.parse_selector`.
+      #
+      # These selectors come from inspecting the actual HTML
+      # Substack ships in 2025 exports. New chrome blocks appear
+      # periodically; users can add their own via `--strip-selectors`
+      # without forking this list.
+      def self.strip_selectors
+        %w[
+          div.subscription-widget-wrap
+          div.subscription-widget
+          div.button-wrapper
+          div.footnote-hovercard
+          aside.poll
+          aside.subscribe-widget
+          div.embedded-publication-wrap
+          div.share-dialog
+          div.captioned-button-wrap
+        ]
+      end
+
+      def each_post
+        with_export_root do |root|
+          manifest_path = File.join(root, 'posts.csv')
+          unless File.exist?(manifest_path)
+            raise "Substack export missing posts.csv at #{manifest_path}"
+          end
+
+          CSV.foreach(manifest_path, headers: true, encoding: 'UTF-8') do |row|
+            next unless publishable?(row)
+
+            html_path = File.join(root, 'posts', "#{row['post_id']}.html")
+            unless File.exist?(html_path)
+              # Some rows in posts.csv have no corresponding HTML
+              # file (email-only sends, draft cleanup). Skip rather
+              # than fail the run.
+              next
+            end
+
+            body_html = File.read(html_path, encoding: 'UTF-8')
+            yield Base::ImportedPost.new(
+              title:             row['title'],
+              slug:              row['slug'] || derive_slug_from_title(row['title']),
+              date:              parse_date(row['post_date']),
+              body_html:         body_html,
+              tags:              [], # Substack export doesn't include tags in posts.csv
+              description:       row['subtitle'],
+              feature_image_url: nil, # Substack inlines hero images in the body
+              source_url:        row['post_url'],
+              source_slug:       row['slug'],
+              extra_frontmatter: substack_extras(row)
+            )
+          end
+        end
+      end
+
+      private
+
+      def publishable?(row)
+        status   = row['status'].to_s.downcase
+        type     = row['type'].to_s.downcase
+        audience = row['audience'].to_s.downcase
+        return false unless status.empty? || status == 'published'
+        # Skip podcast episodes and email-only thread posts — they
+        # don't have body HTML in the export shape we know about.
+        return false if %w[podcast thread].include?(type)
+        # Mark paid-only posts (we still import them, but flag in fm)
+        # via substack_extras; this method just decides import vs skip.
+        true
+      end
+
+      def substack_extras(row)
+        audience = row['audience'].to_s.downcase
+        extras = {}
+        extras['audience'] = audience if audience == 'only_paid'
+        extras
+      end
+
+      def parse_date(value)
+        return nil if value.to_s.empty?
+        # Substack uses ISO-8601 like 2024-03-15T10:00:00.000Z.
+        Date.parse(value)
+      rescue ArgumentError
+        nil
+      end
+
+      def derive_slug_from_title(title)
+        Support::SlugNormalizer.normalize(title.to_s)
+      end
+
+      # Substack URLs are `https://<sub>.substack.com/p/<slug>`. We
+      # write `/p/<slug>/` as the redirect_from so existing inbound
+      # links keep working after migration. We can't recover the
+      # subdomain reliably from the export, so the host part stays
+      # implicit — the engine matches the path-only form.
+      def redirects_for(source_slug)
+        slug = source_slug.to_s.strip
+        return nil if slug.empty?
+        ["/p/#{slug}/"]
+      end
+
+      # Yield the directory holding `posts.csv`. Handles both a
+      # directory source and a zip — extracts the zip to a tempdir,
+      # cleans up when the block returns.
+      def with_export_root(&block)
+        if File.directory?(@source)
+          yield resolve_root(@source)
+        elsif File.file?(@source) && @source.end_with?('.zip')
+          Dir.mktmpdir('abqari-substack-') do |tmp|
+            unless system('unzip', '-q', @source, '-d', tmp)
+              raise "Failed to unzip #{@source}. Is `unzip` on PATH?"
+            end
+            assert_extraction_contained!(tmp)
+            yield resolve_root(tmp)
+          end
+        else
+          raise "Unrecognised source: #{@source} (expected a directory or .zip)"
+        end
+      end
+
+      # The zip sometimes nests everything under a single top-level
+      # Verify the extraction actually stayed inside the tempdir.
+      #
+      # Defence in depth, not a known hole: the `unzip` call is argv-form
+      # so there's no shell injection, and modern Info-ZIP already
+      # refuses `../` entries and strips leading slashes. But an export
+      # zip is the least-trusted input this engine accepts, and we're
+      # delegating containment to a third-party binary whose behaviour
+      # varies by build — busybox's unzip has had traversal bugs, and
+      # "whatever `unzip` is on PATH" is not a guarantee we control.
+      # Cheap to check, so check.
+      #
+      # Symlinks are refused outright rather than resolved: a Substack
+      # export has no legitimate use for one, and refusing avoids having
+      # to reason about multi-hop chains where an intermediate link
+      # points outside.
+      def assert_extraction_contained!(tmp)
+        root = File.realpath(tmp)
+
+        Dir.glob(File.join(tmp, '**', '*'), File::FNM_DOTMATCH).each do |entry|
+          next if %w[. ..].include?(File.basename(entry))
+
+          if File.symlink?(entry)
+            raise "Refusing to import #{@source}: the archive contains a symlink " \
+                  "(#{entry.sub("#{tmp}/", '')}). Extract it yourself and import the " \
+                  'directory if you trust its contents.'
+          end
+
+          begin
+            PathSafe.assert_under(root, File.realpath(entry))
+          rescue PathSafe::Error, Errno::ENOENT
+            raise "Refusing to import #{@source}: the archive extracted a path outside " \
+                  "the temporary directory (#{entry}). This archive is not safe to import."
+          end
+        end
+      end
+
+      # directory; sometimes it dumps `posts.csv` + `posts/` directly
+      # at the root. Detect by looking for `posts.csv` at each level.
+      def resolve_root(root)
+        return root if File.exist?(File.join(root, 'posts.csv'))
+
+        nested = Dir.children(root).map { |name| File.join(root, name) }.select { |p| File.directory?(p) }
+        nested.each do |sub|
+          return sub if File.exist?(File.join(sub, 'posts.csv'))
+        end
+        root # caller will raise on the missing-CSV check
+      end
+    end
+  end
+end

data/lib/abqari/importers/support/html_to_markdown.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+module Abqari
+  module Importers
+    module Support
+      # Thin wrapper over `reverse_markdown`. The gem is lazy-required
+      # so a site that never runs `bin/import` doesn't pay the load
+      # cost on every build.
+      #
+      # Conversion tuning:
+      #
+      #   * `unknown_tags: :pass_through` keeps unrecognised HTML
+      #     elements verbatim rather than dropping them — preserves
+      #     things like Substack-specific `<figcaption>` wrappers or
+      #     Ghost's `<bookmark>` cards. The user can clean those up
+      #     manually after import; dropping them silently loses
+      #     content.
+      #
+      #   * `github_flavored: true` emits GFM table syntax and
+      #     fenced code blocks, matching how Abqari posts are
+      #     authored (kramdown + GFM extensions).
+      #
+      # Pre-clean pass: strip platform chrome before conversion so
+      # the resulting markdown doesn't contain "Subscribe now" CTAs,
+      # share buttons, etc. The selectors are passed in by the
+      # caller — each importer ships its own list because each
+      # platform's chrome is different.
+      module HtmlToMarkdown
+        # Convert an HTML string to Markdown. `strip_selectors:` is
+        # an array of CSS selectors whose matching elements are
+        # removed from the DOM before conversion. Empty array = no
+        # pre-clean.
+        def self.convert(html, strip_selectors: [])
+          return '' if html.to_s.empty?
+
+          require_reverse_markdown!
+          html = pre_clean(html, strip_selectors) unless strip_selectors.empty?
+
+          md = ReverseMarkdown.convert(
+            html,
+            unknown_tags: :pass_through,
+            github_flavored: true
+          )
+
+          # `reverse_markdown` sometimes emits 3+ consecutive blank
+          # lines around block elements. Collapse to a single blank.
+          md.gsub(/\n{3,}/, "\n\n").strip + "\n"
+        end
+
+        # `reverse_markdown` is an OPTIONAL dependency, loaded only here.
+        #
+        # It isn't in the gemspec's runtime dependencies because it pulls
+        # nokogiri, which pulls racc — a chain that costs every install a
+        # native-extension gem for a one-time migration feature most
+        # sites never run. On Ruby 3.3 that chain is worse than heavy:
+        # 3.3 bundles racc 1.7.3, Bundler resolves to 1.8.1, and 1.8.1
+        # ships no precompiled Linux binary, so a toolchain-free image
+        # (ruby:3.3-slim, most deploy bases) fails `bundle install`
+        # outright — on the version the gemspec advertises as its floor.
+        #
+        # So: importing asks for the gem at the moment it's needed,
+        # rather than every user paying for it up front.
+        def self.require_reverse_markdown!
+          require 'reverse_markdown'
+        rescue LoadError
+          raise Abqari::UserError, <<~MSG
+            Content importing needs the `reverse_markdown` gem, which Abqari
+            doesn't install by default (it pulls in nokogiri, which most
+            sites never need).
+
+            Install it, then re-run this import:
+
+                gem install reverse_markdown
+
+            Or add it to your site's Gemfile:
+
+                gem 'reverse_markdown', '~> 2.1'
+          MSG
+        end
+
+        # Remove DOM subtrees matching any of `selectors`. We use a
+        # regex-based pass for the simple cases reverse_markdown
+        # already handles internally (and to avoid pulling in a
+        # full HTML parser like Nokogiri). Patterns target whole
+        # opening-to-closing-tag spans with a class match. Good
+        # enough for the platform-chrome blocks we know about —
+        # callers should keep the selectors simple (`div.foo`,
+        # `aside.bar`).
+        def self.pre_clean(html, selectors)
+          selectors.each do |sel|
+            tag, klass = parse_selector(sel)
+            next unless tag
+
+            if klass
+              # Match <tag …class="… klass …" …>…</tag> with the
+              # class anywhere in the class attribute.
+              pattern = %r{
+                <#{Regexp.escape(tag)}\b
+                  [^>]*\bclass\s*=\s*["'][^"']*\b#{Regexp.escape(klass)}\b[^"']*["']
+                  [^>]*>
+                .*?
+                </#{Regexp.escape(tag)}>
+              }mxi
+            else
+              pattern = %r{<#{Regexp.escape(tag)}\b[^>]*>.*?</#{Regexp.escape(tag)}>}mi
+            end
+            html = html.gsub(pattern, '')
+          end
+          html
+        end
+
+        # Decompose `'div.subscription-widget'` → `['div', 'subscription-widget']`
+        # and `'aside'` → `['aside', nil]`. Returns `[nil, nil]` for
+        # patterns we don't support (id selectors, attribute selectors,
+        # descendant combinators) — those are caller errors and we
+        # ignore them rather than crashing the whole import.
+        def self.parse_selector(selector)
+          return [nil, nil] unless selector.is_a?(String)
+          m = selector.match(/\A([a-z][a-z0-9]*)(?:\.([a-z0-9_-]+))?\z/i)
+          return [nil, nil] unless m
+          [m[1], m[2]]
+        end
+      end
+    end
+  end
+end