abqari 1.1.0

data/lib/abqari/importers/support/image_downloader.rb

@@ -0,0 +1,170 @@
+# frozen_string_literal: true
+
+require 'digest'
+require 'fileutils'
+require 'uri'
+
+module Abqari
+  module Importers
+    module Support
+      # Downloads remote images into a post's bundle directory and
+      # returns a map of `{original_url => local_filename}` so the
+      # caller can rewrite references in the post body.
+      #
+      # Why download at all: source-platform CDN URLs (substackcdn.com,
+      # ghost.io's blob storage, Medium's image proxy) get rotated
+      # or revoked periodically. Sites that keep remote URLs look
+      # fine immediately and then break six to twenty-four months
+      # later when the source decides to expire the link. Downloading
+      # at import-time is slower but durable — the image is now part
+      # of the post's bundle and goes through Abqari's image pipeline
+      # at build time (AVIF + WebP variants, metadata stripped,
+      # responsive `<picture>` rewriting). The pipeline runs the same
+      # way it does for a hand-authored bundle, so an imported post
+      # ends up indistinguishable from one written natively.
+      #
+      # Failure handling: a download failure is reported as a warning
+      # and the URL is left as-is in the body. We never abort the
+      # whole post because of an image — most posts have many images
+      # and one CDN hiccup shouldn't sink the whole import.
+      class ImageDownloader
+        # Extensions we recognise as images. Anything else is left
+        # in place (svg passes through here because Abqari's image
+        # pipeline preserves it; non-image extensions like `.pdf`
+        # are intentionally ignored so we don't accidentally try
+        # to "download an image" that's actually a document).
+        IMAGE_EXTENSIONS = %w[.jpg .jpeg .png .gif .webp .avif .svg].freeze
+
+        # 20 MB cap per image. Larger than the engine's default 8 MB
+        # because some Substack hero images come through close to
+        # 10–15 MB pre-optimisation. The build's image pipeline will
+        # downsize them after import.
+        MAX_IMAGE_BYTES = 20 * 1024 * 1024
+
+        attr_reader :warnings
+
+        def initialize(bundle_dir:, logger: nil)
+          @bundle_dir = bundle_dir
+          @logger     = logger
+          @warnings   = []
+          @cache      = {} # url → local filename, so repeated refs share a file
+          @taken      = Set.new # local filenames already used in this bundle
+        end
+
+        # Resolve a remote image URL to a local filename. On success,
+        # downloads the file into the bundle dir and returns the
+        # local filename. On failure, records a warning and returns
+        # nil (caller leaves the URL alone).
+        def fetch(url)
+          return nil if url.to_s.empty?
+          return @cache[url] if @cache.key?(url)
+
+          unless url.match?(%r{\Ahttps?://}i)
+            # Not a remote URL — caller should leave it alone.
+            return nil
+          end
+
+          local = download(url)
+          @cache[url] = local
+          local
+        rescue StandardError => e
+          warn("Failed to download #{url}: #{e.class}: #{e.message}")
+          @cache[url] = nil
+          nil
+        end
+
+        private
+
+        require 'set'
+
+        def download(url)
+          uri = URI(url)
+          ext = extension_for(uri)
+          # Filename strategy: prefer the source's basename, fall
+          # back to a short content hash if the URL doesn't carry
+          # one (image-CDN URLs often look like `…/fetch/abcdef`
+          # with no extension or filename in the path).
+          base = base_for(uri)
+          filename = unique_filename("#{base}#{ext}")
+
+          # `block_private: true` — image URLs come from a third-party
+          # export (least-trusted input in the engine). Refuse ones that
+          # resolve to internal / loopback / metadata addresses.
+          res = begin
+            Abqari::HttpSafe.get(uri, max_bytes: MAX_IMAGE_BYTES,
+                                      headers: { 'User-Agent' => 'abqari-importer' },
+                                      block_private: true)
+          rescue Abqari::HttpSafe::BlockedHostError => e
+            warn(e.message)
+            return nil
+          end
+          unless res.success?
+            warn("HTTP #{res.code} for #{url}")
+            return nil
+          end
+
+          FileUtils.mkdir_p(@bundle_dir)
+          path = File.join(@bundle_dir, filename)
+          File.binwrite(path, res.body)
+          @logger&.call("    ↓ #{filename} (#{format_size(res.body.bytesize)})")
+          filename
+        end
+
+        # Pick the best available filename base from the URL. In
+        # priority order:
+        #   1. The last path segment, stripped of its extension —
+        #      gives us human-readable names where the source
+        #      includes them (e.g. `…/hero-shot.jpg` → `hero-shot`).
+        #   2. A short content hash of the URL — stable across
+        #      runs so re-imports don't churn filenames.
+        def base_for(uri)
+          last = File.basename(uri.path.to_s)
+          last = last.sub(/\.[a-z0-9]+\z/i, '') if last
+          base = SlugNormalizer.normalize(last)
+          base = "img-#{Digest::SHA1.hexdigest(uri.to_s)[0, 8]}" if base.empty?
+          base
+        end
+
+        def extension_for(uri)
+          ext = File.extname(uri.path.to_s).downcase
+          # Substack image-fetch URLs often end with `/fetch/$$$EXT-jpeg/…`
+          # or have no extension at all — default to `.jpg`. We
+          # can't reliably sniff content-type without HEAD-requesting
+          # first, and one extra round-trip per image is expensive.
+          # `.jpg` is the safe default because the image pipeline
+          # re-encodes everything to AVIF/WebP anyway.
+          ext = '.jpg' unless IMAGE_EXTENSIONS.include?(ext)
+          ext
+        end
+
+        def unique_filename(name)
+          # Two-pass uniqueness: the bundle might already contain
+          # a file with the same name from a previous post (rare —
+          # bundles are per-post) or from the import's own previous
+          # download (common — `hero.jpg` is a popular filename).
+          base = File.basename(name, '.*')
+          ext  = File.extname(name)
+          candidate = name
+          n = 2
+          while @taken.include?(candidate) || File.exist?(File.join(@bundle_dir, candidate))
+            candidate = "#{base}-#{n}#{ext}"
+            n += 1
+          end
+          @taken << candidate
+          candidate
+        end
+
+        def format_size(bytes)
+          return "#{bytes}B" if bytes < 1024
+          return "#{(bytes / 1024.0).round(1)}KB" if bytes < 1024 * 1024
+          "#{(bytes / 1_048_576.0).round(2)}MB"
+        end
+
+        def warn(message)
+          @warnings << message
+          @logger&.call("    ⚠ #{message}")
+        end
+      end
+    end
+  end
+end

data/lib/abqari/importers/support/report.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module Abqari
+  module Importers
+    module Support
+      # Accumulates per-post results during an import run and writes
+      # a markdown report (`_import_report.md`) the operator can
+      # spot-check in their editor. Gitignored — see
+      # `lib/abqari/templates/site_skeleton/.gitignore`.
+      #
+      # Each entry has a status (`:imported`, `:skipped`, `:failed`)
+      # plus a short reason. Warnings (e.g. image download failures
+      # that didn't sink the post) live alongside the entry so the
+      # operator can see exactly what went wrong with which post,
+      # rather than a flat global warning log.
+      class Report
+        Entry = Struct.new(:slug, :title, :status, :reason, :warnings,
+                           keyword_init: true)
+
+        attr_reader :entries, :source, :importer
+
+        def initialize(source:, importer:)
+          @source   = source
+          @importer = importer
+          @entries  = []
+          @started  = Time.now
+        end
+
+        def add(slug:, title:, status:, reason: nil, warnings: [])
+          @entries << Entry.new(slug: slug, title: title, status: status,
+                                reason: reason, warnings: Array(warnings))
+        end
+
+        def count(status)
+          @entries.count { |e| e.status == status }
+        end
+
+        # Returns the formatted markdown for the run. The caller writes
+        # it wherever it wants (we don't bake in a path so tests can
+        # round-trip without touching disk).
+        def to_markdown
+          lines = []
+          lines << "# Import report"
+          lines << ''
+          lines << "- **Importer:** `#{@importer}`"
+          lines << "- **Source:** `#{@source}`"
+          lines << "- **Started:** #{@started.utc.iso8601}"
+          lines << "- **Imported:** #{count(:imported)}"
+          lines << "- **Skipped:**  #{count(:skipped)}"
+          lines << "- **Failed:**   #{count(:failed)}"
+          lines << ''
+          lines << '---'
+          lines << ''
+
+          if @entries.empty?
+            lines << '_No posts were processed._'
+          else
+            @entries.each do |e|
+              lines << "## #{e.title.to_s.empty? ? '(untitled)' : e.title}"
+              lines << ''
+              lines << "- Slug: `#{e.slug}`"
+              lines << "- Status: **#{e.status}**"
+              lines << "- Reason: #{e.reason}" if e.reason && !e.reason.empty?
+              if e.warnings.any?
+                lines << '- Warnings:'
+                e.warnings.each { |w| lines << "  - #{w}" }
+              end
+              lines << ''
+            end
+          end
+
+          lines.join("\n") + "\n"
+        end
+
+        # Compact one-line summary for stdout at the end of the run.
+        def summary_line
+          warns = @entries.sum { |e| e.warnings.size }
+          parts = ["#{count(:imported)} imported"]
+          parts << "#{count(:skipped)} skipped" if count(:skipped).positive?
+          parts << "#{count(:failed)} failed"   if count(:failed).positive?
+          parts << "#{warns} warnings"          if warns.positive?
+          "✓ #{parts.join(', ')}."
+        end
+      end
+    end
+  end
+end

data/lib/abqari/importers/support/slug_normalizer.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Abqari
+  module Importers
+    module Support
+      # Slug normalisation + collision resolution shared by every
+      # importer. The slug rules match `bin/new`'s `slugify` (lowercase,
+      # `[^a-z0-9]+` collapsed to a single dash, leading/trailing dashes
+      # stripped) so imported posts are indistinguishable from posts
+      # authored fresh via the generator.
+      #
+      # Collision policy is "append a numeric suffix" — `foo`, `foo-2`,
+      # `foo-3`. The user picked this over date-prefixing because
+      # date-prefixing makes URLs uglier and the source platforms
+      # (Substack especially) sometimes reuse slugs deliberately. The
+      # appended suffix preserves the original slug as the canonical
+      # form when there's no conflict.
+      module SlugNormalizer
+        module_function
+
+        # Strip a slug down to the engine's canonical shape. ASCII-fold
+        # is intentionally omitted — non-ASCII characters are dropped
+        # rather than transliterated, matching `bin/new`'s behaviour.
+        # Callers that need transliteration (e.g. an importer pulling
+        # from a CJK source) should pre-process before calling this.
+        def normalize(text)
+          text.to_s
+              .downcase
+              .gsub(/[^a-z0-9]+/, '-')
+              .gsub(/\A-|-\z/, '')
+        end
+
+        # Resolve a slug against an in-memory set of already-taken
+        # slugs. Mutates the set as a side-effect — the resolved slug
+        # is added so subsequent calls in the same run see it. This is
+        # a pure-in-memory check, distinct from the on-disk collision
+        # check that the writer applies separately (filesystem state
+        # can change between importer invocations).
+        def resolve(slug, taken)
+          slug = 'untitled' if slug.to_s.empty?
+          candidate = slug
+          n = 2
+          while taken.include?(candidate)
+            candidate = "#{slug}-#{n}"
+            n += 1
+          end
+          taken << candidate
+          candidate
+        end
+      end
+    end
+  end
+end

data/lib/abqari/importers.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Abqari
+  # Content importers — `bin/import <platform> <source>`.
+  #
+  # Loaded lazily: the registry below maps platform names to file
+  # paths so a normal site build never pays the cost of pulling
+  # reverse_markdown + the per-platform parsers into memory. The
+  # CLI handler resolves the requested platform and requires only
+  # its file.
+  #
+  # Adding a new importer:
+  #   1. Create `lib/abqari/importers/<name>.rb` with class
+  #      `Abqari::Importers::<Name>` < `Base`.
+  #   2. Register it in `REGISTRY` below with its short name.
+  #   3. Add a one-line description for the `--help` text.
+  #
+  # Each importer implements `each_post` yielding `Base::ImportedPost`
+  # records. The base class handles the rest: HTML→Markdown,
+  # image download, slug collisions, frontmatter assembly, idempotent
+  # writes, and the run report.
+  module Importers
+    REGISTRY = {
+      'substack' => {
+        file:        'abqari/importers/substack',
+        class_name:  'Abqari::Importers::Substack',
+        description: 'Import a Substack export zip (or unzipped directory)'
+      },
+      'ghost' => {
+        file:        'abqari/importers/ghost',
+        class_name:  'Abqari::Importers::Ghost',
+        description: 'Import a Ghost JSON export'
+      },
+      'jekyll' => {
+        file:        'abqari/importers/jekyll',
+        class_name:  'Abqari::Importers::Jekyll',
+        description: 'Import a Jekyll site directory (_posts/ + assets)'
+      }
+    }.freeze
+
+    # Resolve a platform name to an importer class, lazy-loading
+    # the file on first use. Returns nil for unknown platforms.
+    def self.lookup(platform)
+      entry = REGISTRY[platform.to_s]
+      return nil unless entry
+
+      require entry[:file]
+      Object.const_get(entry[:class_name])
+    end
+
+    def self.platforms
+      REGISTRY.keys
+    end
+  end
+end
+
+# Support modules are tiny; eager-load them when any importer is
+# pulled in. The platform-specific files (substack, ghost, jekyll)
+# stay lazy.
+require_relative 'importers/support/slug_normalizer'
+require_relative 'importers/support/html_to_markdown'
+require_relative 'importers/support/image_downloader'
+require_relative 'importers/support/report'
+require_relative 'importers/base'