runlight 0.1.0 → 0.1.3

data/lib/runlight/dates.rb

@@ -19,8 +19,10 @@ module Runlight
     COMPARE_MODES = %w[previous year custom off].freeze
 
     MAX_BUCKETS = 1000
-    # A month of hours. Longer hourly ranges are cut off rather than refused.
+    # A month of hours. A longer range is counted by the day instead.
     MAX_HOURS = 744
+    # The next coarser interval, for a range too long for its own.
+    COARSER = { "hour" => "day", "day" => "week", "week" => "month" }.freeze
 
     # Names Intl reads as another zone where TZInfo has no name, or reads it differently.
     ALIASES = {
@@ -47,7 +49,7 @@ module Runlight
     }.freeze
 
     OFFSET = /\A([+\-]|−)([01][0-9]|2[0-3])(?::?([0-5][0-9]))?\z/
-    private_constant :MAX_BUCKETS, :MAX_HOURS, :ALIASES, :OFFSET
+    private_constant :MAX_BUCKETS, :MAX_HOURS, :COARSER, :ALIASES, :OFFSET
 
     # A zone: a TZInfo::Timezone, or the seconds of a fixed offset; false where Intl throws a RangeError.
     @zones = {}
@@ -284,8 +286,26 @@ module Runlight
       end
 
       interval = INTERVALS.include?(input["interval"]) ? input["interval"] : default_interval(from_date, to_date)
-      { "from" => start_of(from_date, timezone), "to" => start_of(add_days(to_date, 1), timezone), "fromDate" => from_date,
-        "toDate" => to_date, "interval" => interval }
+      from_ms = start_of(from_date, timezone)
+      to_ms = start_of(add_days(to_date, 1), timezone)
+      # An interval too fine for the range moves up to the next one that fits, and the range says which it is.
+      # Past 1,000 months there is none.
+      while bucket_count(from_ms, to_ms, from_date, to_date, interval) > (interval == "hour" ? MAX_HOURS : MAX_BUCKETS)
+        interval = COARSER[interval]
+        return nil if interval.nil?
+      end
+      { "from" => from_ms, "to" => to_ms, "fromDate" => from_date, "toDate" => to_date, "interval" => interval }
+    end
+
+    # How many chart buckets a range has at an interval, the first and last counted even when partial.
+    def bucket_count(from_ms, to_ms, from_date, to_date, interval)
+      return (to_ms - from_ms).fdiv(3_600_000).ceil if interval == "hour"
+      return days_between(from_date, to_date) + 1 if interval == "day"
+      return tdiv(days_between(add_days(from_date, -weekday(from_date)), to_date), 7) + 1 if interval == "week"
+
+      fy, fm = split(from_date)
+      ty, tm = split(to_date)
+      ((ty - fy) * 12) + (tm - fm) + 1
     end
 
     def add_years(date, years)
@@ -323,12 +343,15 @@ module Runlight
         "toDate" => to_date, "interval" => range["interval"] }
     end
 
-    # Chart buckets covering a range, each starting on a local boundary.
+    # Chart buckets covering a range, each starting on a local boundary and ending at the next, so only the
+    # first and last can be partial. resolve_range picks an interval that fits; a range given here with more
+    # buckets than that, such as a longer comparison, is cut off at a boundary, never merged into its last bucket.
     def buckets(range, timezone)
       starts = []
+      # One start past the limit, so the last bucket kept still ends where the next would begin.
       if range["interval"] == "hour"
         t = range["from"]
-        while t < range["to"] && starts.length < MAX_HOURS
+        while t < range["to"] && starts.length <= MAX_HOURS
           starts << t
           t += 3_600_000
         end
@@ -339,7 +362,7 @@ module Runlight
         elsif range["interval"] == "month"
           date = "#{date[0, 8]}01"
         end
-        while date <= range["toDate"] && starts.length < MAX_BUCKETS
+        while date <= range["toDate"] && starts.length <= MAX_BUCKETS
           starts << start_of(date, timezone)
           date = case range["interval"]
                  when "day" then add_days(date, 1)
@@ -348,7 +371,7 @@ module Runlight
                  end
         end
       end
-      starts.each_with_index.map do |start, i|
+      starts.first(range["interval"] == "hour" ? MAX_HOURS : MAX_BUCKETS).each_with_index.map do |start, i|
         { "start" => [start, range["from"]].max, "end" => [starts[i + 1] || range["to"], range["to"]].min }
       end
     end

data/lib/runlight/db/connect.rb

@@ -31,9 +31,15 @@ module Runlight
       def sqlite(path)
         path = path.to_s
         memory = path == ":memory:" || path.empty?
-        # A file in a folder that is not there yet gets the folder, as a fresh app's data/ often is.
-        FileUtils.mkdir_p(File.dirname(path)) if !memory && !path.start_with?("file:") && !File.directory?(File.dirname(path))
-        config = { adapter: "sqlite3", database: memory ? ":memory:" : path, timeout: 5000,
+        file = !memory && !path.start_with?("file:")
+        # A file in a folder that is not there yet gets the folder, as a fresh app's data/ often is, for its
+        # owner and group alone, as the file is.
+        if file && !File.directory?(File.dirname(path))
+          FileUtils.mkdir_p(File.dirname(path), mode: 0o770)
+          owner_and_group(File.dirname(path), 0o770)
+        end
+        made(path) if file && !File.exist?(path)
+        config = { adapter: "sqlite3", database: memory ? ":memory:" : path, timeout: Database::BUSY_TIMEOUT_MS,
                    # One connection for a database in memory, which every connection would otherwise make anew.
                    pool: memory ? 1 : 5 }
         Database.new(Pools.make(config), owned: true, dialect: "sqlite")
@@ -57,10 +63,11 @@ module Runlight
         Database.new(Pools.make(config.compact), owned: true, dialect: "postgres")
       end
 
-      # MySQL 8.4 or MariaDB 11.4 and later from a URL like mysql://user:pass@host:3306/db (or mariadb://),
+      # MySQL 8.0.20 or MariaDB 11.4 and later from a URL like mysql://user:pass@host:3306/db (or mariadb://),
       # through the trilogy gem when it is installed, else mysql2. The session is utf8mb4 with the server's own
       # SQL mode plus IGNORE_SPACE, as mysql2 opens it for the TypeScript driver (its connect flags ask for
-      # that, which neither Ruby driver can, so it is set as the session's mode). Runlight's tables carry their
+      # that, which neither Ruby driver can, so it is set as the session's mode), less NO_BACKSLASH_ESCAPES, as
+      # stores/mysql.ts drops it. Runlight's tables carry their
       # own binary collation, so text compares and sorts by code point. statement_timeout is set per
       # connection as stores/mysql.ts sets it; 0 turns it off.
       def mysql(url, statement_timeout: 120_000)
@@ -71,7 +78,9 @@ module Runlight
         db = Database.new(Pools.make(config.compact), owned: true, statement_timeout: statement_timeout.to_i, dialect: "mysql")
         version, mode = db.connection_class.connection_pool.with_connection { |c| c.select_rows("SELECT VERSION(), @@GLOBAL.sql_mode")[0] }
         db.connection_class.connection_pool.disconnect!
-        variables = { sql_mode: (mode.to_s.split(",") | ["IGNORE_SPACE"]).join(",") }
+        # Less NO_BACKSLASH_ESCAPES, since values are written with backslash escapes (Database drops it from
+        # every session too).
+        variables = { sql_mode: ((mode.to_s.split(",") - ["NO_BACKSLASH_ESCAPES"]) | ["IGNORE_SPACE"]).join(",") }
         if statement_timeout.to_i.positive?
           variables.merge!(version.to_s.match?(/mariadb/i) ? { max_statement_time: statement_timeout.to_i / 1000.0 } : { max_execution_time: statement_timeout.to_i })
         end
@@ -90,6 +99,23 @@ module Runlight
         end
       end
 
+      # A database file made here is for its owner and group alone (0660), whatever the umask: the visits in it
+      # are nobody else's to read, and a scheduled job running as another user of the same group can still
+      # write to it. It is made empty, which SQLite takes as a new database, before anything opens it, so the
+      # -wal and -shm files SQLite makes beside it later take the same mode.
+      def made(path)
+        File.open(path, File::WRONLY | File::CREAT | File::EXCL, 0o660) { nil }
+        owner_and_group(path, 0o660)
+      rescue SystemCallError
+        # Made by another process meanwhile, or a folder this one may not write to, which opening reports.
+      end
+
+      def owner_and_group(path, mode)
+        File.chmod(mode, path)
+      rescue SystemCallError, NotImplementedError
+        # Not a file this process may change, or a system with no such modes.
+      end
+
       def quote_name(name, quote)
         "#{quote}#{name.to_s.gsub(quote, quote * 2)}#{quote}"
       end

data/lib/runlight/db/db.rb

@@ -22,10 +22,38 @@ module Runlight
       # Arbitrary but fixed, so every Runlight process takes the same lock to create tables.
       MIGRATION_LOCK = 7_331_906
 
+      # The longest a process waits for another one creating or upgrading the tables before it gives up.
+      LOCK_WAIT_MS = 30_000
+
+      @lock_wait = LOCK_WAIT_MS
+
+      class << self
+        # How long the lock for creating tables is waited for: LOCK_WAIT_MS, unless a test has set another.
+        def lock_wait_ms
+          @lock_wait
+        end
+
+        # For tests: the lock is waited for this long instead, or LOCK_WAIT_MS again with nil.
+        def lock_wait_in_tests(ms)
+          @lock_wait = ms || LOCK_WAIT_MS
+        end
+      end
+
       # One lock per MySQL database, so installs sharing a server do not wait on each other. Lock names are 64
       # characters at most.
       MYSQL_LOCK = "CONCAT('runlight:', LEFT(SHA2(COALESCE(DATABASE(), ''), 256), 48))"
 
+      # The session's sql_mode as it was, less NO_BACKSLASH_ESCAPES, wherever it sits in the list, as
+      # stores/mysql.ts sets it.
+      NO_BACKSLASH_ESCAPES_OFF = "SET SESSION sql_mode = TRIM(BOTH ',' FROM REPLACE(CONCAT(',', @@SESSION.sql_mode, ','), ',NO_BACKSLASH_ESCAPES,', ','))"
+
+      # Deleted rows, such as a day's salt once it is gone, are overwritten where that costs no extra writes, so
+      # they do not linger in the file's free space. Rails' pragmas setting takes this one only as on or off.
+      SECURE_DELETE = "PRAGMA secure_delete = FAST"
+
+      # How long a SQLite connection waits for a lock another one holds, as Connect.sqlite sets it.
+      BUSY_TIMEOUT_MS = 5000
+
       attr_reader :connection_class
 
       # connection_class: an ActiveRecord::Base subclass whose pool to use (ActiveRecord::Base for the app's
@@ -38,11 +66,16 @@ module Runlight
         @statement_timeout = statement_timeout
         @dialect = dialect
         @held = 0
+        # Set on the thread inside a SQLite turn, whose BEGIN IMMEDIATE Rails does not know about.
+        @turn_key = :"runlight_turn_#{object_id}"
+        # The MySQL sessions, and the SQLite connections this database opened, already set up, by their driver
+        # connection, which a reconnect replaces.
+        @set_up = ObjectSpace::WeakMap.new
       end
 
       # "sqlite", "postgres", or "mysql".
       def dialect
-        @dialect ||= self.class.dialect_of(connection { |c| c.adapter_name })
+        @dialect ||= @connection_class.connection_pool.with_connection { |c| self.class.dialect_of(c.adapter_name) }
       end
 
       def self.dialect_of(adapter)
@@ -80,7 +113,7 @@ module Runlight
       # transaction already open (the store's, or the app's) is joined, not nested. Yields this database.
       def transaction
         connection do |c|
-          if c.transaction_open?
+          if c.transaction_open? || Thread.current[@turn_key]
             yield self
           elsif dialect == "mysql"
             # As Postgres does by default: each statement sees what was committed before it began, and InnoDB
@@ -93,7 +126,8 @@ module Runlight
       end
 
       # Runs the block while holding a database-wide lock, so two processes starting at once do not race to
-      # create the same tables. Yields this database.
+      # create the same tables. Waits for the lock at most LOCK_WAIT_MS, then raises a TablesBusyError. Yields
+      # this database.
       def exclusive(&block)
         connection do |c|
           @held += 1
@@ -105,6 +139,41 @@ module Runlight
         end
       end
 
+      # Runs the block as the only one reading and changing what it reads, across threads and processes, and
+      # yields this database. On Postgres and MySQL that is the database-wide lock. On SQLite it is a
+      # transaction begun IMMEDIATE, which takes the file's write lock before the block reads anything, so a
+      # second turn waits (up to the busy timeout) rather than reading what the first is about to change. A turn
+      # inside a turn, or inside a transaction already open, joins it.
+      def turn(&block)
+        return exclusive(&block) unless dialect == "sqlite"
+
+        connection do |c|
+          next yield(self) if Thread.current[@turn_key] || c.transaction_open?
+
+          c.execute("BEGIN IMMEDIATE", "Runlight")
+          Thread.current[@turn_key] = true
+          failed = nil
+          begin
+            yield self
+          rescue Exception => e
+            failed = e
+            raise
+          ensure
+            Thread.current[@turn_key] = nil
+            if failed
+              begin
+                c.execute("ROLLBACK", "Runlight")
+              rescue StandardError
+                # The error that got here matters more than one rolling back.
+              end
+            else
+              # Also when the block leaves early with return, break, or next.
+              c.execute("COMMIT", "Runlight")
+            end
+          end
+        end
+      end
+
       def close
         return unless @owned
 
@@ -115,7 +184,7 @@ module Runlight
       # seconds and applies it to every statement; MySQL counts milliseconds and applies it to reads.
       def limit_statements(ms)
         connection do |c|
-          @mariadb = c.select_value("SELECT VERSION() AS v").to_s.match?(/mariadb/i) if @mariadb.nil?
+          @mariadb = value(c, "SELECT VERSION() AS v").to_s.match?(/mariadb/i) if @mariadb.nil?
           c.execute(@mariadb ? "SET SESSION max_statement_time = #{Json.number(ms / 1000.0)}" : "SET SESSION max_execution_time = #{ms.to_i}")
         end
       end
@@ -184,8 +253,35 @@ module Runlight
 
       private
 
-      def connection(&block)
-        @connection_class.connection_pool.with_connection(&block)
+      def connection
+        @connection_class.connection_pool.with_connection do |c|
+          set_up(c) if dialect == "mysql" || (@owned && dialect == "sqlite")
+          yield c
+        end
+      end
+
+      # Values are escaped with backslashes, which a server set to NO_BACKSLASH_ESCAPES reads as text, so a
+      # quote could end the value early. Every MySQL session Runlight uses drops that one mode first, an app's
+      # own ActiveRecord connection's too, and again after a reconnect, which starts the session over. A SQLite
+      # file Runlight opened itself turns on secure_delete, as stores/sqlite.ts does.
+      def set_up(c)
+        raw = c.instance_variable_get(:@raw_connection)
+        return if raw && @set_up[raw]
+
+        c.execute(dialect == "sqlite" ? SECURE_DELETE : NO_BACKSLASH_ESCAPES_OFF, "Runlight")
+        raw = c.instance_variable_get(:@raw_connection)
+        wait_without_the_lock(raw) if dialect == "sqlite"
+        @set_up[raw] = true if raw
+      end
+
+      # Rails 7.2 sets SQLite's busy timeout with busy_timeout, which sleeps in C holding Ruby's global lock. A
+      # thread waiting for the file's write lock then stops the thread of the same process that holds it from
+      # ever finishing, so the wait runs out and raises "database is locked". busy_handler_timeout= (sqlite3
+      # 2.x, and what Rails 8 uses) waits the same time but lets other threads run.
+      def wait_without_the_lock(raw)
+        return unless raw.respond_to?(:busy_handler_timeout=)
+
+        raw.busy_handler_timeout = BUSY_TIMEOUT_MS
       end
 
       # A statement on a connection. A connection the server dropped (a restart, a failover, an idle timeout)
@@ -199,6 +295,7 @@ module Runlight
         warn "Runlight: a #{dialect == "mysql" ? "MySQL" : "Postgres"} connection was lost; it reconnects on the next query. #{e.message}"
         connection do |c|
           c.reconnect!
+          set_up(c) if dialect == "mysql"
           block.call(c)
         end
       end
@@ -233,7 +330,14 @@ module Runlight
         if dialect == "postgres"
           # Asked for again and again rather than waited on: a waiting statement would hold up an index being
           # built CONCURRENTLY by whoever has the lock, and the two would wait on each other for good.
-          sleep 0.1 until truthy(c.select_value("SELECT pg_try_advisory_lock(#{MIGRATION_LOCK}) AS ok"))
+          # Up to LOCK_WAIT_MS, so a request does not wait on a long index build for good.
+          wait = self.class.lock_wait_ms
+          until_at = monotonic_ms + wait
+          until truthy(value(c, "SELECT pg_try_advisory_lock(#{MIGRATION_LOCK}) AS ok"))
+            raise TablesBusyError, wait if monotonic_ms >= until_at
+
+            sleep 0.1
+          end
           begin
             return yield(self)
           ensure
@@ -244,11 +348,16 @@ module Runlight
             end
           end
         end
+        # Up to LOCK_WAIT_MS, so a request does not wait on a long index build for good.
+        wait = self.class.lock_wait_ms
+        until_at = monotonic_ms + wait
         loop do
-          ok = c.select_value("SELECT GET_LOCK(#{MYSQL_LOCK}, 5) AS ok")
+          left = ((until_at - monotonic_ms) / 1000.0).ceil.clamp(1, 5)
+          ok = value(c, "SELECT GET_LOCK(#{MYSQL_LOCK}, #{left}) AS ok")
           raise "Runlight: MySQL refused the lock for creating tables" if ok.nil?
           break if ok.to_i == 1
-          # Not got within 5 seconds: another process is creating the tables. Ask again.
+          # Not got in time: another process is creating the tables. Ask again, until the wait is up.
+          raise TablesBusyError, wait if monotonic_ms >= until_at
         end
         begin
           # An index on a big table takes a while to build, so the build may run past the statement timeout.
@@ -265,6 +374,17 @@ module Runlight
         end
       end
 
+      def monotonic_ms
+        Process.clock_gettime(Process::CLOCK_MONOTONIC, :millisecond)
+      end
+
+      # The first column of a statement's first row, always asked of the database. select_value goes through
+      # Rails' query cache, which a Rails request turns on for every pool, so a poll for a lock would see its
+      # first answer again and again; exec_query is never cached.
+      def value(c, sql)
+        c.exec_query(sql, "Runlight").rows.first&.first
+      end
+
       def truthy(value)
         value == true || value == "t" || value == 1 || value == "1" || value == "true"
       end

data/lib/runlight/db/prefixed.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module Runlight
+  module Db
+    # A database whose tables and indexes start with a prefix of the app's choosing in place of `rl_`, such as
+    # `wp_rl_` beside a WordPress database's tables, so two apps sharing one database each keep a set of their
+    # own. Runlight's SQL names every table and index `rl_` and something, and those names alone are rewritten
+    # on the way through, in each statement and in the names a schema check compares as text. A store given a
+    # prefix puts one in front of its database, so the accounts and importers that reach the database through
+    # the store follow it too. Values bound to `?` are never rewritten.
+    class Prefixed
+      # Whatever else the database underneath has (close, limit_statements, connection_class) is passed on.
+      OPTIONAL = %i[affected turn metered?].freeze
+      private_constant :OPTIONAL
+
+      attr_reader :inner, :prefix
+
+      def initialize(inner, prefix)
+        # A name in a statement, and short enough that the longest index name stays inside MySQL's 64.
+        unless prefix.is_a?(String) && prefix.match?(/\A[A-Za-z_][A-Za-z0-9_]{0,31}\z/)
+          raise ArgumentError, "Runlight: a table prefix is up to 32 letters, digits, and underscores, and does not start with a digit"
+        end
+
+        @inner = inner
+        @prefix = prefix
+      end
+
+      # A statement with each of Runlight's names under the prefix.
+      def sql(text)
+        text.gsub(/\brl_(?=[a-z])/, @prefix)
+      end
+
+      def dialect
+        @inner.dialect
+      end
+
+      def all(text, params = [])
+        @inner.all(sql(text), params)
+      end
+
+      def run(text, params = [])
+        @inner.run(sql(text), params)
+      end
+
+      def affected(text, params = [])
+        @inner.affected(sql(text), params)
+      end
+
+      def metered?
+        @inner.metered?
+      end
+
+      def transaction
+        @inner.transaction { |db| yield wrap(db) }
+      end
+
+      def exclusive
+        @inner.exclusive { |db| yield wrap(db) }
+      end
+
+      def turn
+        @inner.turn { |db| yield wrap(db) }
+      end
+
+      def close
+        @inner.close
+      end
+
+      def respond_to?(name, include_all = false)
+        return @inner.respond_to?(name, include_all) if OPTIONAL.include?(name.to_sym)
+
+        super
+      end
+
+      private
+
+      def wrap(db)
+        db.equal?(@inner) ? self : Prefixed.new(db, @prefix)
+      end
+
+      def method_missing(name, ...)
+        return super unless @inner.respond_to?(name)
+
+        @inner.public_send(name, ...)
+      end
+
+      def respond_to_missing?(name, include_all = false)
+        @inner.respond_to?(name, include_all) || super
+      end
+    end
+  end
+end

data/lib/runlight/geo.rb

@@ -102,9 +102,10 @@ module Runlight
     end
 
     # The location of a request: the platform's headers when they name a country, else the lookup's answer
-    # for the address, else nowhere.
-    def locate(headers, ip, lookup = nil)
-      from_headers = location_from_headers(headers)
+    # for the address, else nowhere. The headers are believed only with trust_headers, as the address is: without
+    # a proxy in front, a client writes them itself.
+    def locate(headers, ip, lookup = nil, trust_headers = true)
+      from_headers = trust_headers ? location_from_headers(headers) : nil
       return from_headers if !from_headers.nil? && from_headers["country"] != ""
 
       if !lookup.nil? && ip != ""

data/lib/runlight/http/fetcher.rb

@@ -58,6 +58,20 @@ module Runlight
 
       private
 
+      # Opens the connection. A name pinned to several addresses is tried address by address, as a connection
+      # by name is, so a host that listens on one of them is reached; only the failure of the last is raised.
+      def connect(http, addresses)
+        tries = addresses.empty? ? [nil] : addresses
+        tries.each_with_index do |address, at|
+          http.ipaddr = address unless address.nil?
+          begin
+            return http.start
+          rescue SystemCallError, SocketError, Net::OpenTimeout
+            raise if at == tries.length - 1
+          end
+        end
+      end
+
       def monotonic
         Process.clock_gettime(Process::CLOCK_MONOTONIC)
       end
@@ -72,16 +86,24 @@ module Runlight
         left = deadline - monotonic
         raise FetchError.new(TIMED_OUT, timed_out: true) if left <= 0
 
-        http = Net::HTTP.new(host, port)
+        # A pinned request never goes through a proxy from the environment (http_proxy and the rest), which would
+        # connect wherever the name leads now rather than to the address that was checked.
+        pinned = !Array(init["resolve"]).empty?
+        http = pinned ? Net::HTTP.new(host, port, nil) : Net::HTTP.new(host, port)
         http.use_ssl = https
         # A pin connects to the checked address, with the name kept for the Host header and the certificate.
+        addresses = []
         Array(init["resolve"]).each do |pin|
-          m = pin.to_s.match(/\A(.+):(\d+):\[?([^\],]+)\]?/)
-          next unless m && m[1].casecmp?(host) && m[2].to_i == port
+          lead = "#{host}:#{port}:"
+          next unless pin.to_s.length > lead.length && pin.to_s[0, lead.length].casecmp?(lead)
 
-          http.ipaddr = m[3]
+          addresses = pin.to_s[lead.length..].split(",").map { |address| address.delete_prefix("[").delete_suffix("]") }.reject(&:empty?)
+          http.ipaddr = addresses.first
           break
         end
+        # A pin that names another host or port would leave the name to be looked up again: refused, never
+        # fetched unpinned.
+        raise FetchError, "fetch failed" if pinned && http.ipaddr.nil?
         http.open_timeout = [left, 15].min
         http.read_timeout = left
         http.write_timeout = left
@@ -105,8 +127,9 @@ module Runlight
           # Past maxBytes the whole request is left by a throw, which closes the connection: a break would leave
           # only read_body, and Net::HTTP would then read the rest of the body after all.
           catch(:too_long) do
-            http.start do |connection|
-              connection.request(request) do |answer|
+            connect(http, addresses)
+            begin
+              http.request(request) do |answer|
                 status = answer.code.to_i
                 answer.each_capitalized_name { |name| answer_headers[name.downcase] = answer.get_fields(name) }
                 answer.read_body do |chunk|
@@ -120,6 +143,8 @@ module Runlight
                   received << chunk
                 end
               end
+            ensure
+              http.finish if http.started?
             end
           end
         rescue Net::OpenTimeout, Net::ReadTimeout, Net::WriteTimeout, Timeout::Error

data/lib/runlight/http/request.rb

@@ -32,11 +32,14 @@ module Runlight
       # the path's limit raises TooLarge once one byte past it has been read.
       def self.from_rack(env, read_body: true)
         scheme = env["rack.url_scheme"] || "http"
-        host = env["HTTP_HOST"]
-        if host.nil? || host.empty?
+        # A Host header no URL can hold (a space, a bad port, a userinfo @) is read as the server's own name, as
+        # an absent one is, so the request still has a URL; the header itself is kept as it came.
+        host = env["HTTP_HOST"].to_s
+        unless host?(scheme, host)
           port = env["SERVER_PORT"].to_s
           default = (scheme == "https" && port == "443") || (scheme == "http" && port == "80") || port.empty?
           host = "#{env["SERVER_NAME"] || "localhost"}#{default ? "" : ":#{port}"}"
+          host = "localhost" unless host?(scheme, host)
         end
         path = "#{env["SCRIPT_NAME"]}#{env["PATH_INFO"]}"
         path = "/#{path}" unless path.start_with?("/")
@@ -51,7 +54,9 @@ module Runlight
         headers.set("content-type", type) unless type.empty?
         length = env["CONTENT_LENGTH"].to_s
         headers.set("content-length", length) unless length.empty?
-        method = env["REQUEST_METHOD"].to_s.upcase
+        # Rack::MethodOverride (in every Rails app) turns a form's POST with _method=DELETE into a DELETE. Runlight
+        # answers the method the browser sent, so a form on another page cannot pass for a write that needs no JSON.
+        method = (env["rack.methodoverride.original_method"] || env["REQUEST_METHOD"]).to_s.upcase
         body = +""
         input = env["rack.input"]
         if read_body && !%w[GET HEAD].include?(method) && input
@@ -64,6 +69,12 @@ module Runlight
         new(url, method: method, headers: headers, body: body, remote_address: env["REMOTE_ADDR"].to_s)
       end
 
+      # Whether a Host header is a name (or [address]) and maybe a port, and nothing a URL would read as more.
+      def self.host?(scheme, host)
+        host.match?(%r{\A(?:\[[0-9A-Fa-f:.]+\]|[^\s/?#@\\\[\]:]+)(?::\d{1,5})?\z}) && !Url.parse("#{scheme}://#{host}/").nil?
+      end
+      private_class_method :host?
+
       # The most a body may hold at this path: little for the collect endpoint, more for everything else.
       def self.limit_for(path)
         path.end_with?("/e") ? MAX_COLLECT_BODY : MAX_BODY

data/lib/runlight/importers/csv_visits.rb

@@ -48,7 +48,7 @@ module Runlight
         ""
       end
 
-      # A row with no visitor is its own visit, keyed by its whole content so a second import gives it the same ids.
+      # A row with no visitor is its own visit, keyed by its whole content.
       def own_key(row)
         entries = row.map { |k, v| [k.to_s, v] }
         entries = entries.sort { |a, b| Js.compare(a[0], b[0]).negative? ? -1 : 1 }

data/lib/runlight/importers/index.rb

@@ -18,7 +18,9 @@ module Runlight
       #
       # Returns { "cursor", "done", "total", "links", "clicks", "skipped", "failed" }, each failure
       # { "slug", "reason", "code"?, "params"? }.
-      def import_step(runlight, site, source, credentials, cursor, done)
+      # link_domain: the domain as a site keeps it once it passes the checks a link domain added by hand does, or nil
+      # to drop it.
+      def import_step(runlight, site, source, credentials, cursor, done, link_domain = ->(_domain) {})
         name = IMPORTERS[source]
         raise ImportError.new("Runlight cannot import from #{source}", "import_source", { "source" => source }) if name.nil?
 
@@ -37,6 +39,12 @@ module Runlight
         # A total that is not a number is as good as none.
         total = result["total"]
         total = nil unless total.is_a?(Integer) || (total.is_a?(Float) && total.finite?)
+        # Each domain is checked once a step, since the check may look it up in DNS.
+        checked = {}
+        domain_for = lambda do |domain|
+          checked[domain] = link_domain.call(domain) unless checked.key?(domain)
+          checked[domain]
+        end
         step = { "cursor" => result["cursor"], "done" => done, "total" => total, "links" => 0, "clicks" => 0, "skipped" => 0, "failed" => [] }
         result["links"].each do |item|
           if Js.truthy?(item["known"])
@@ -44,7 +52,7 @@ module Runlight
             step["skipped"] += 1
             next
           end
-          written = Write.write_link(runlight, site, source, item["link"], item)
+          written = Write.write_link(runlight, site, source, item["link"], item, domain_for)
           step["done"] += 1
           case written["status"]
           when "created"