time_bucket_stream 0.1.0

data/lib/time_bucket_stream/claimer.rb

@@ -0,0 +1,263 @@
+# frozen_string_literal: true
+
+require "fileutils"
+
+class TimeBucketStream
+  class Claimer
+    DEFAULT_CLAIM_GRACE = 10
+    DEFAULT_STALE_PARTIAL_AFTER = 600
+    DEFAULT_QUARANTINE_RETENTION = Quarantine::DEFAULT_RETENTION
+
+    attr_reader :claim_grace, :paths, :quarantine_retention, :stale_partial_after
+
+    def self.normalize_claim_grace(value)
+      seconds = Integer(value)
+      return seconds if seconds >= 0
+
+      raise ArgumentError
+    rescue ArgumentError, TypeError, RangeError
+      raise ArgumentError, "claim_grace must be a non-negative number of seconds"
+    end
+
+    def self.normalize_stale_partial_after(value)
+      seconds = Integer(value)
+      return seconds if seconds >= 0
+
+      raise ArgumentError
+    rescue ArgumentError, TypeError, RangeError
+      raise ArgumentError, "stale_partial_after must be a non-negative number of seconds"
+    end
+
+    def self.normalize_quarantine_retention(value)
+      Quarantine.normalize_retention(value)
+    end
+
+    def initialize(path:, clock: Time, claim_grace: DEFAULT_CLAIM_GRACE, stale_partial_after: DEFAULT_STALE_PARTIAL_AFTER, quarantine_retention: DEFAULT_QUARANTINE_RETENTION)
+      @paths = Paths.new(path: path)
+      @clock = clock
+      @claim_grace = self.class.normalize_claim_grace(claim_grace)
+      @stale_partial_after = self.class.normalize_stale_partial_after(stale_partial_after)
+      @quarantine_retention = self.class.normalize_quarantine_retention(quarantine_retention)
+
+      ensure_directories
+    end
+
+    def claim_completed(before: claim_before_bucket)
+      before = before.to_s
+
+      ensure_directories
+      cleanup_quarantine
+      quarantine_stale_partials
+
+      claimable_log_names(before).filter_map do |log_name|
+        claim_log(log_name, before)
+      end
+    end
+
+    def delete_claimed(log_name)
+      claim_processing(log_name)&.delete
+    end
+
+    def release_claimed(log_name)
+      claim_processing(log_name)&.release
+    end
+
+    private
+
+    def claimable_log_names(before)
+      (processing_log_names + completed_log_names(before)).uniq.sort
+    end
+
+    def processing_log_names
+      Dir.children(paths.processing).grep(/\.jsonl\z/).select { |name| valid_log_name?(name) }
+    rescue SystemCallError
+      []
+    end
+
+    def completed_log_names(before)
+      Dir.children(paths.logs)
+        .grep(/\.jsonl\z/)
+        .select { |name| processable_log?(name, before) }
+    rescue SystemCallError
+      []
+    end
+
+    def partial_log_names(before)
+      Dir.children(paths.logs)
+        .grep(/\.jsonl\z/)
+        .select { |name| partial_log?(name, before) }
+    rescue SystemCallError
+      []
+    end
+
+    def claim_log(log_name, before)
+      return unless processable_log?(log_name, before)
+
+      claim_processing(log_name) || claim_pending(log_name, before)
+    end
+
+    def claim_processing(log_name)
+      return unless valid_log_name?(log_name)
+
+      claim_path = paths.processing_log_for(log_name)
+
+      with_claim_lock(log_name) do |lock_file|
+        next unless File.exist?(claim_path)
+        next unless complete_file?(claim_path)
+
+        build_claim(log_name, claim_path, lock_file)
+      end
+    rescue SystemCallError, IOError
+      nil
+    end
+
+    def quarantine_stale_partials
+      before = stale_partial_before_bucket
+
+      partial_log_names(before).each do |log_name|
+        quarantine_partial(log_name, before)
+      end
+    end
+
+    def quarantine_partial(log_name, before)
+      return unless partial_log?(log_name, before)
+
+      source_path = File.join(paths.logs, log_name)
+
+      with_claim_lock(log_name) do |lock_file|
+        next unless File.exist?(source_path)
+        next unless partial_log?(log_name, before)
+
+        Claim.new(name: log_name, path: source_path, lock_file: lock_file).quarantine(
+          paths: paths,
+          reason: "partial_trailing_line",
+          metadata: {
+            "stale_partial_after" => stale_partial_after,
+            "before_bucket" => before
+          },
+          clock: @clock
+        )
+      end
+    rescue SystemCallError, IOError
+      nil
+    end
+
+    def claim_pending(log_name, before)
+      return unless processable_log?(log_name, before)
+
+      source_path = File.join(paths.logs, log_name)
+      claim_path = paths.processing_log_for(log_name)
+
+      with_claim_lock(log_name) do |lock_file|
+        next if File.exist?(claim_path)
+        next unless File.exist?(source_path)
+        next unless processable_log?(log_name, before)
+        next unless complete_file?(source_path)
+
+        File.rename(source_path, claim_path)
+        build_claim(log_name, claim_path, lock_file)
+      end
+    rescue SystemCallError, IOError
+      nil
+    end
+
+    def build_claim(log_name, path, lock_file)
+      Claim.new(
+        name: log_name,
+        path: path,
+        lock_file: lock_file
+      )
+    end
+
+    def with_claim_lock(log_name)
+      FileUtils.mkdir_p(paths.claim_locks)
+
+      file = File.open(paths.claim_lock_for(log_name), File::RDWR | File::CREAT, 0o600)
+      return close_file(file) unless file.flock(File::LOCK_EX | File::LOCK_NB)
+
+      claim = yield file
+      return claim if claim
+
+      unlock_file(file)
+      close_file(file)
+      nil
+    rescue
+      unlock_file(file)
+      close_file(file)
+      raise
+    end
+
+    def complete_file?(path)
+      size = File.size(path)
+      return true if size.zero?
+
+      File.open(path, File::RDONLY) do |file|
+        file.seek(size - 1)
+        file.read(1) == "\n"
+      end
+    rescue SystemCallError, IOError
+      false
+    end
+
+    def ensure_directories
+      FileUtils.mkdir_p(paths.logs)
+      FileUtils.mkdir_p(paths.processing)
+      FileUtils.mkdir_p(paths.claim_locks)
+    end
+
+    def processable_log?(log_name, before)
+      bucket = log_bucket(log_name)
+      bucket && bucket < before
+    end
+
+    def partial_log?(log_name, before)
+      return false unless processable_log?(log_name, before)
+
+      !complete_file?(File.join(paths.logs, log_name))
+    end
+
+    def valid_log_name?(log_name)
+      LogName.valid?(log_name)
+    end
+
+    def log_bucket(log_name)
+      LogName.bucket(log_name)
+    end
+
+    def claim_before_bucket
+      (current_time.utc - claim_grace).strftime(Writer::BUCKET_FORMAT)
+    end
+
+    def stale_partial_before_bucket
+      (current_time.utc - stale_partial_after).strftime(Writer::BUCKET_FORMAT)
+    end
+
+    def current_time
+      @clock.respond_to?(:call) ? @clock.call : @clock.now
+    end
+
+    def cleanup_quarantine
+      quarantine.cleanup
+    end
+
+    def quarantine
+      @quarantine ||= Quarantine.new(
+        path: paths.path,
+        clock: @clock,
+        retention: quarantine_retention
+      )
+    end
+
+    def unlock_file(file)
+      file&.flock(File::LOCK_UN)
+    rescue SystemCallError, IOError
+      nil
+    end
+
+    def close_file(file)
+      file&.close
+    rescue SystemCallError, IOError
+      nil
+    end
+  end
+end

data/lib/time_bucket_stream/codecs/json.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require "json"
+
+class TimeBucketStream
+  module Codecs
+    class Json
+      def dump(value)
+        JSON.generate(value)
+      end
+
+      def load(value)
+        JSON.parse(value)
+      end
+    end
+  end
+end

data/lib/time_bucket_stream/codecs/oj.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+class TimeBucketStream
+  module Codecs
+    class Oj
+      def initialize
+        require "oj"
+      rescue LoadError
+        raise LoadError, 'TimeBucketStream::Codecs::Oj requires `gem "oj"`'
+      end
+
+      def dump(value)
+        ::Oj.dump(value, mode: :compat)
+      end
+
+      def load(value)
+        ::Oj.load(value, mode: :compat)
+      end
+    end
+  end
+end

data/lib/time_bucket_stream/codecs.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class TimeBucketStream
+  module Codecs
+    module_function
+
+    def validate!(codec)
+      return codec if codec.respond_to?(:dump) && codec.respond_to?(:load)
+
+      raise ArgumentError, "codec must respond to dump and load"
+    end
+  end
+end

data/lib/time_bucket_stream/log_name.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+class TimeBucketStream
+  module LogName
+    PATTERN = /\A(?<bucket>\d{12})-[A-Za-z0-9_.-]+-\d+-[0-9a-f]{8}\.jsonl\z/
+
+    module_function
+
+    def valid?(name)
+      name = name.to_s
+      name == File.basename(name) && PATTERN.match?(name)
+    rescue ArgumentError, EncodingError
+      false
+    end
+
+    def bucket(name)
+      match = PATTERN.match(name.to_s)
+      match && match[:bucket]
+    rescue ArgumentError, EncodingError
+      nil
+    end
+  end
+end

data/lib/time_bucket_stream/paths.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+class TimeBucketStream
+  class Paths
+    attr_reader :path
+
+    def initialize(path:)
+      @path = normalize_path(path)
+    end
+
+    def base
+      path
+    end
+
+    def logs
+      @logs ||= File.join(base, "logs")
+    end
+
+    def processing
+      @processing ||= File.join(base, "processing")
+    end
+
+    def quarantine
+      @quarantine ||= File.join(base, "quarantine")
+    end
+
+    def claim_locks
+      @claim_locks ||= File.join(base, "claim_locks")
+    end
+
+    def log_for(writer_id)
+      File.join(logs, "#{writer_id}.jsonl")
+    end
+
+    def processing_log_for(log_name)
+      File.join(processing, log_name)
+    end
+
+    def quarantine_log_for(log_name)
+      File.join(quarantine, log_name)
+    end
+
+    def quarantine_metadata_for(log_name)
+      File.join(quarantine, "#{log_name}.meta.json")
+    end
+
+    def claim_lock_for(log_name)
+      File.join(claim_locks, "#{log_name}.lock")
+    end
+
+    private
+
+    def normalize_path(path)
+      value = path.to_s
+      raise ArgumentError, "path must not be blank" if value.strip.empty?
+
+      File.expand_path(value)
+    end
+  end
+end

data/lib/time_bucket_stream/quarantine.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "json"
+require "time"
+
+class TimeBucketStream
+  class Quarantine
+    DEFAULT_RETENTION = 7 * 24 * 60 * 60
+    TIMESTAMP_PATTERN = /\.q-(\d{20})-\d+-[0-9a-f]+\.jsonl\z/
+
+    attr_reader :paths, :retention
+
+    def self.normalize_retention(value)
+      return nil if value.nil?
+
+      seconds = Integer(value)
+      return seconds if seconds >= 0
+
+      raise ArgumentError
+    rescue ArgumentError, TypeError, RangeError
+      raise ArgumentError, "quarantine_retention must be nil or a non-negative number of seconds"
+    end
+
+    def initialize(path:, clock: Time, retention: DEFAULT_RETENTION)
+      @paths = Paths.new(path: path)
+      @clock = clock
+      @retention = self.class.normalize_retention(retention)
+    end
+
+    def cleanup
+      return unless retention
+
+      cutoff = current_time.utc - retention
+
+      quarantine_log_names.each do |log_name|
+        delete_pair(log_name) if expired_log?(log_name, cutoff)
+      end
+
+      orphan_metadata_names.each do |metadata_name|
+        delete_metadata(metadata_name) if expired_metadata?(metadata_name, cutoff)
+      end
+    rescue SystemCallError, IOError, JSON::ParserError
+      nil
+    end
+
+    private
+
+    def quarantine_log_names
+      children.select { |name| name.end_with?(".jsonl") }.sort
+    end
+
+    def metadata_names
+      children.select { |name| name.end_with?(".meta.json") }.sort
+    end
+
+    def orphan_metadata_names
+      metadata_names.reject do |metadata_name|
+        File.exist?(quarantine_log_path(metadata_name.delete_suffix(".meta.json")))
+      end
+    end
+
+    def children
+      Dir.children(paths.quarantine)
+    rescue SystemCallError
+      []
+    end
+
+    def expired_log?(log_name, cutoff)
+      quarantine_time_for(log_name) < cutoff
+    end
+
+    def expired_metadata?(metadata_name, cutoff)
+      log_name = metadata_name.delete_suffix(".meta.json")
+      quarantine_time_for(log_name, metadata_name: metadata_name) < cutoff
+    end
+
+    def quarantine_time_for(log_name, metadata_name: "#{log_name}.meta.json")
+      metadata_time(metadata_path(metadata_name)) ||
+        timestamp_from_name(log_name) ||
+        file_time(quarantine_log_path(log_name)) ||
+        file_time(metadata_path(metadata_name)) ||
+        current_time.utc
+    end
+
+    def metadata_time(path)
+      return unless File.exist?(path)
+
+      value = JSON.parse(File.read(path)).fetch("quarantined_at", nil)
+      Time.parse(value).utc if value
+    rescue SystemCallError, IOError, JSON::ParserError, TypeError, ArgumentError
+      nil
+    end
+
+    def timestamp_from_name(log_name)
+      match = log_name.match(TIMESTAMP_PATTERN)
+      parse_timestamp(match[1]) if match
+    rescue ArgumentError
+      nil
+    end
+
+    def parse_timestamp(value)
+      Time.utc(
+        value[0, 4].to_i,
+        value[4, 2].to_i,
+        value[6, 2].to_i,
+        value[8, 2].to_i,
+        value[10, 2].to_i,
+        value[12, 2].to_i,
+        value[14, 6].to_i
+      )
+    end
+
+    def file_time(path)
+      File.mtime(path).utc
+    rescue SystemCallError, IOError
+      nil
+    end
+
+    def delete_pair(log_name)
+      FileUtils.rm_f(quarantine_log_path(log_name))
+      FileUtils.rm_f(metadata_path("#{log_name}.meta.json"))
+    end
+
+    def delete_metadata(metadata_name)
+      FileUtils.rm_f(metadata_path(metadata_name))
+    end
+
+    def quarantine_log_path(log_name)
+      paths.quarantine_log_for(log_name)
+    end
+
+    def metadata_path(metadata_name)
+      File.join(paths.quarantine, metadata_name)
+    end
+
+    def current_time
+      @clock.respond_to?(:call) ? @clock.call : @clock.now
+    end
+  end
+end