datalog-theme 0.10.1 → 0.11.0

data/lib/datalog/packages/refresh.rb

@@ -0,0 +1,187 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "json"
+require "net/http"
+require "uri"
+require "yaml"
+require_relative "../packages"
+require_relative "../site_config"
+require_relative "../theme/version"
+
+module Datalog
+  module Packages
+    # `datalog packages refresh`: reads the latest release of every package the
+    # site lists from each registry it names, and writes them to
+    # _data/package_releases.yml for the package layout to read. The build
+    # itself never goes to a registry, so it stays offline and two builds of
+    # one commit match. A registry that cannot be read stops the command before
+    # the file is touched. The file holds nothing but the releases, so it
+    # changes only when one does, and a scheduled run has nothing to propose
+    # otherwise.
+    class Refresh
+      class Error < StandardError; end
+
+      USER_AGENT = "datalog-theme/#{Datalog::Theme::VERSION} (+https://github.com/DiogoRibeiro7/analytics-blog-jekyll)".freeze
+      HEADER = "# The latest release of each package on each registry, written by\n" \
+               "# `bundle exec datalog packages refresh`. Run it again to update this file;\n" \
+               "# a package's registries are set in its front matter (`registry:`).\n"
+
+      attr_reader :root
+
+      # `fetch` takes a URL and returns [status, body]; the default asks the network.
+      def initialize(root:, fetch: nil)
+        @root = File.expand_path(root)
+        @fetch = fetch || method(:http_get)
+      end
+
+      def config
+        @config ||= Datalog::SiteConfig.load(File.join(root, "_config.yml"))
+      end
+
+      def data_file
+        File.join(root, config["data_dir"] || "_data", "package_releases.yml")
+      end
+
+      # {"pypi" => {"name" => release}}, read from every registry, or Error
+      # naming each registry that could not be read.
+      def releases
+        found = Hash.new { |hash, key| hash[key] = {} }
+        failures = []
+        packages.each do |file, data|
+          Packages.registries(data).each do |registry, name|
+            next unless ENDPOINTS.key?(registry)
+
+            found[registry][name] = read(registry, name)
+          rescue Error => e
+            failures << "#{file}: #{e.message}"
+          end
+        end
+        raise Error, "No release was written, because #{failures.join('; ')}" unless failures.empty?
+
+        found.sort.to_h.transform_values { |names| names.sort.to_h }
+      end
+
+      # Writes the file and returns the releases, or raises before writing.
+      # With no release to write, it leaves the file alone.
+      def run(dry_run: false)
+        data = releases
+        yaml = HEADER + YAML.dump(data).sub(/\A---\n/, "")
+        unless dry_run || data.empty?
+          FileUtils.mkdir_p(File.dirname(data_file))
+          File.write(data_file, yaml)
+        end
+        [data, yaml]
+      end
+
+      # [relative path, front matter] of each document in the packages collection.
+      def packages
+        directory = File.join(root, config["collections_dir"].to_s, "_packages")
+        Dir.glob(File.join(directory, "**", "*.{md,markdown,html}")).filter_map do |path|
+          text = File.read(path, encoding: "bom|utf-8")
+          front = text[/\A---\s*\n(.*?)^---\s*$/m, 1]
+          data = front && YAML.safe_load(front, permitted_classes: [Date, Time], aliases: true)
+          [path.delete_prefix("#{root}/"), data] if data.is_a?(Hash)
+        end
+      end
+
+      def read(registry, name)
+        url = format(ENDPOINTS.fetch(registry), URI.encode_www_form_component(name))
+        status, body = begin
+          @fetch.call(url)
+        rescue StandardError => e
+          raise Error, "could not reach #{REGISTRIES.dig(registry, :label)} at #{url} (#{e.message})"
+        end
+        if status == 404
+          raise Error, "#{REGISTRIES.dig(registry, :label)} has no package named #{name} (#{url})"
+        elsif !(200..299).cover?(status.to_i)
+          raise Error, "#{REGISTRIES.dig(registry, :label)} answered #{status} for #{name} (#{url})"
+        end
+
+        found = Releases.public_send(registry, JSON.parse(body))
+        found.merge("prerelease" => Packages.prerelease?(found["version"], registry))
+      rescue JSON::ParserError, KeyError, NoMethodError, TypeError => e
+        raise Error,
+              "#{REGISTRIES.dig(registry, :label)} sent an answer for #{name} that is not a release (#{e.message})"
+      end
+
+      def http_get(url)
+        uri = URI(url)
+        Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https", open_timeout: 10,
+                                            read_timeout: 20) do |http|
+          request = Net::HTTP::Get.new(uri)
+          request["User-Agent"] = USER_AGENT
+          request["Accept"] = "application/json"
+          response = http.request(request)
+          [response.code.to_i, response.body]
+        end
+      end
+    end
+
+    # One registry's answer as one release: version, released (a date),
+    # license, the language it needs, and whether it is a pre-release or yanked.
+    module Releases
+      module_function
+
+      def pypi(json)
+        info = json.fetch("info")
+        version = info.fetch("version")
+        files = Array(json.dig("releases", version) || json["urls"])
+        license = info["license_expression"].to_s
+        license = info["license"].to_s if license.empty? && info["license"].to_s.length.between?(1, 40)
+        license = classifier_license(info["classifiers"]) if license.empty?
+        release(version, files.map { |file| file["upload_time_iso_8601"] || file["upload_time"] }.compact.min,
+                license, "requires_python" => info["requires_python"],
+                         "yanked" => !files.empty? && files.all? { |file| file["yanked"] })
+      end
+
+      def crates(json)
+        crate = json.fetch("crate")
+        version = crate["max_stable_version"] || crate["newest_version"] || crate.fetch("max_version")
+        found = Array(json["versions"]).find { |entry| entry["num"] == version } || {}
+        release(version, found["created_at"], found["license"],
+                "rust_version" => found["rust_version"], "yanked" => found["yanked"] == true)
+      end
+
+      def rubygems(json)
+        versions = Array(json)
+        found = versions.find { |entry| entry["prerelease"] != true } || versions.fetch(0)
+        release(found.fetch("number"), found["created_at"], Array(found["licenses"]).join(", "),
+                "required_ruby_version" => found["ruby_version"])
+      end
+
+      def cran(json)
+        depends = json["Depends"]
+        r_version = depends.is_a?(Hash) ? depends["R"] : depends.to_s[/\bR\s*\(([^)]+)\)/, 1]
+        release(json.fetch("Version"), json["Date/Publication"], json["License"], "r_version" => r_version)
+      end
+
+      def npm(json)
+        version = json.dig("dist-tags", "latest") || raise(KeyError, "no latest version")
+        found = json.dig("versions", version) || {}
+        license = found["license"].is_a?(Hash) ? found.dig("license", "type") : found["license"]
+        # A deprecated version still installs, with a warning: npm has no yank.
+        release(version, json.dig("time", version), license, "node_version" => found.dig("engines", "node"))
+      end
+
+      def release(version, released, license, extra = {})
+        date = released.to_s[/\A\d{4}-\d{2}-\d{2}/]
+        found = { "version" => version.to_s, "released" => date, "license" => blank(license),
+                  "prerelease" => Packages.prerelease?(version) }
+        found.merge(extra.transform_values { |value| blank(value) })
+             .reject { |key, value| value.nil? || (key == "yanked" && !value) }
+      end
+
+      def blank(value)
+        return value if [true, false].include?(value)
+
+        text = value.to_s.strip
+        text.empty? ? nil : text
+      end
+
+      def classifier_license(classifiers)
+        Array(classifiers).filter_map { |entry| entry[/\ALicense :: (?:OSI Approved :: )?(.+)\z/, 1] }.first.to_s
+      end
+    end
+  end
+end

data/lib/datalog/packages.rb

@@ -0,0 +1,177 @@
+# frozen_string_literal: true
+
+require "date"
+require "time"
+
+module Datalog
+  # Package pages that know their registries: where each registry lists a
+  # package, how to install it from there, and how to read its latest release
+  # from the registry's API. The layout and the install panel read the first
+  # two through _plugins/packages.rb; `datalog packages refresh` the third.
+  module Packages
+    # key => label, page, install commands as [console, template], and docs.
+    REGISTRIES = {
+      "pypi" => { label: "PyPI", tab: "pip", url: "https://pypi.org/project/%s/",
+                  install: [["terminal", "pip install %s"]] },
+      "conda_forge" => { label: "conda-forge", tab: "conda", url: "https://anaconda.org/conda-forge/%s",
+                         install: [["terminal", "conda install -c conda-forge %s"]] },
+      "cran" => { label: "CRAN", tab: "cran", url: "https://cran.r-project.org/package=%s",
+                  install: [["r_console", "install.packages('%s')"]] },
+      "crates" => { label: "crates.io", tab: "cargo", url: "https://crates.io/crates/%s",
+                    install: [["terminal", "cargo add %s"]], docs: "https://docs.rs/%s" },
+      "rubygems" => { label: "RubyGems", tab: "gem", url: "https://rubygems.org/gems/%s",
+                      install: [["terminal", "gem install %s"], ["gemfile", "gem \"%s\""]] },
+      "npm" => { label: "npm", tab: "npm", url: "https://www.npmjs.com/package/%s",
+                 install: [["terminal", "npm install %s"]] },
+      "julia" => { label: "JuliaHub", tab: "julia", url: "https://juliahub.com/ui/Packages/General/%s",
+                   install: [["julia_repl", "using Pkg; Pkg.add(\"%s\")"]] }
+    }.freeze
+
+    # A tab is named for the tool that installs from the registry.
+    TAB_LABELS = { "pip" => "pip", "conda" => "conda", "cran" => "CRAN", "cargo" => "cargo", "gem" => "gem",
+                   "npm" => "npm", "julia" => "Pkg" }.freeze
+
+    # The registries `datalog packages refresh` reads, and the address of a
+    # package's latest release in each.
+    ENDPOINTS = {
+      "pypi" => "https://pypi.org/pypi/%s/json",
+      "crates" => "https://crates.io/api/v1/crates/%s",
+      "rubygems" => "https://rubygems.org/api/v1/versions/%s.json",
+      "cran" => "https://crandb.r-pkg.org/%s",
+      "npm" => "https://registry.npmjs.org/%s"
+    }.freeze
+
+    # A PEP 440 version, read leniently: an epoch, the release, then a
+    # pre-release, a post-release and a development release, each optional,
+    # then a local label. A pre-release or a development release makes it a
+    # pre-release (1.0a1, 2.0rc1, 1!1.0rc1, 1.0.post1.dev2); a post-release
+    # alone does not (1.0.post1).
+    PEP440 = /\A(?:\d+!)?\d+(?:\.\d+)*
+              (?<pre>[-_.]?(?:a|b|c|rc|alpha|beta|pre|preview)[-_.]?\d*)?
+              (?:[-_.]?(?:post|rev|r)[-_.]?\d*|-\d+)?
+              (?<dev>[-_.]?dev[-_.]?\d*)?
+              (?:\+[a-z0-9]+(?:[-_.][a-z0-9]+)*)?\z/ix
+
+    # Registries whose versions are semver, where a hyphen marks a pre-release
+    # (1.0.0-beta.2) and a plus only build metadata (1.0.0+build.5).
+    SEMVER = %w[crates npm julia].freeze
+
+    # The two addresses pages gave before `registry:`.
+    LEGACY = { "pypi" => "pypi_url", "cran" => "cran_url" }.freeze
+
+    module_function
+
+    # The registries a page names, in the order it names them: {"pypi" => "name"}.
+    # `pypi_url` and `cran_url` still count, for pages written before `registry:`.
+    def registries(data)
+      named = data["registry"].is_a?(Hash) ? data["registry"].transform_keys(&:to_s) : {}
+      named = named.select { |key, name| REGISTRIES.key?(key) && !name.to_s.strip.empty? }
+      LEGACY.each_key do |key|
+        name = legacy_name(key, legacy_url(data, key))
+        named[key] ||= name if name
+      end
+      named.transform_values { |name| name.to_s.strip }
+    end
+
+    def legacy_url(data, key)
+      url = data[LEGACY.fetch(key)].to_s.strip
+      url.empty? ? nil : url
+    end
+
+    def legacy_name(key, url)
+      return nil unless url
+
+      case key
+      when "pypi" then url[%r{pypi\.(?:python\.)?org/(?:project|pypi)/([^/?#]+)}, 1]
+      when "cran" then url[/package=([\w.]+)/, 1] || url[%r{/web/packages/([\w.]+)}, 1]
+      end
+    end
+
+    # The registry pages, and docs.rs for a crate: [{"registry", "label", "url"}].
+    # A `pypi_url` or `cran_url` is the link when it names the same package as
+    # the registry's entry, and still a link when it names none this can read.
+    def links(data)
+      named = registries(data)
+      found = named.flat_map do |key, name|
+        spec = REGISTRIES.fetch(key)
+        legacy = LEGACY.key?(key) ? legacy_url(data, key) : nil
+        legacy = nil unless legacy && legacy_name(key, legacy).to_s.casecmp?(name)
+        [{ "registry" => key, "label" => spec[:label], "url" => legacy || format(spec[:url], name) },
+         spec[:docs] && { "registry" => "docs_rs", "label" => "docs.rs", "url" => format(spec[:docs], name) }].compact
+      end
+      LEGACY.each_key do |key|
+        url = legacy_url(data, key)
+        found << { "registry" => key, "label" => REGISTRIES.dig(key, :label), "url" => url } if url && !named.key?(key)
+      end
+      found
+    end
+
+    # The install panel's tabs: one per registry, then Git. Each is
+    # {"id", "label", "commands" => [{"console", "command"}]}. A page that
+    # names no registry keeps the tabs its language always had: pip, conda
+    # and Git for Python, CRAN and GitHub for R. `overrides` replaces the
+    # command of a tab by its id, as the include's pip_command and siblings do.
+    def install_tabs(data, language: nil, git_url: nil, name: nil, overrides: {})
+      named = registries(data)
+      tabs = named.empty? ? language_tabs(language.to_s.downcase, name) : named.map { |key, package| tab(key, package) }
+      git = git_tab(language.to_s.downcase, git_url)
+      tabs << git if git
+      tabs.each do |item|
+        override = overrides[item["id"]].to_s.strip
+        item["commands"][0]["command"] = override unless override.empty?
+      end
+    end
+
+    def tab(key, package)
+      spec = REGISTRIES.fetch(key)
+      commands = spec[:install].map do |console, template|
+        { "console" => console, "command" => format(template, package) }
+      end
+      { "id" => spec[:tab], "label" => TAB_LABELS.fetch(spec[:tab]), "commands" => commands }
+    end
+
+    def language_tabs(language, name)
+      case language
+      when "python" then [terminal_tab("pip", "pip install #{name}"), terminal_tab("conda", "conda install #{name}")]
+      when "r" then [tab("cran", name)]
+      else []
+      end
+    end
+
+    def terminal_tab(id, command, console: "terminal", label: TAB_LABELS.fetch(id, id))
+      { "id" => id, "label" => label, "commands" => [{ "console" => console, "command" => command }] }
+    end
+
+    def git_tab(language, git_url)
+      return nil if git_url.to_s.empty?
+
+      if language == "r"
+        repository = git_url.sub(%r{\Ahttps?://github\.com/}, "").sub(/\.git\z/, "")
+        return terminal_tab("github", "devtools::install_github(\"#{repository}\")", console: "r_console",
+                                                                                     label: "GitHub")
+      end
+
+      # The clone's directory is the repository's name, which need not be the package's.
+      lines = ["git clone #{git_url}", "cd #{File.basename(git_url.to_s.chomp('/'), '.git')}"]
+      lines << "pip install -e ." if language == "python"
+      terminal_tab("git", lines.join("\n"), label: "Git")
+    end
+
+    # Whether a version is a pre-release by the rules of its registry: any
+    # letter for RubyGems, a hyphen for semver, PEP 440 otherwise (and semver
+    # for a version PEP 440 cannot read). CRAN has no pre-releases.
+    def prerelease?(version, registry = nil)
+      text = version.to_s.strip
+      return false if text.empty? || registry == "cran"
+      return text.match?(/[a-z]/i) if registry == "rubygems"
+      return semver_prerelease?(text) if SEMVER.include?(registry)
+
+      match = PEP440.match(text)
+      match ? !(match[:pre] || match[:dev]).nil? : semver_prerelease?(text)
+    end
+
+    def semver_prerelease?(text)
+      text.split("+", 2).first.include?("-")
+    end
+  end
+end

data/lib/datalog/plugin_system.rb

@@ -97,6 +97,9 @@ module Datalog
           next unless plugin.respond_to?(hook)
 
           plugin.public_send(hook, *args)
+        rescue Jekyll::Errors::FatalException
+          # A plugin's build error is the site's, as {% ref %}'s are: it stops the build.
+          raise
         rescue StandardError => e
           Jekyll.logger.warn("datalog plugin", "#{plugin.id} failed during #{hook}: #{e.message}")
         end
@@ -112,6 +115,8 @@ module Datalog
           next unless result.is_a?(Hash)
 
           acc.replace(deep_merge(acc, result))
+        rescue Jekyll::Errors::FatalException
+          raise
         rescue StandardError => e
           Jekyll.logger.warn("datalog plugin", "#{plugin.id} failed during #{hook}: #{e.message}")
         end