time_bucket_stream 0.1.0

data/lib/time_bucket_stream/stream.rb

@@ -0,0 +1,261 @@
+# frozen_string_literal: true
+
+class TimeBucketStream
+  DEFAULT_CLAIM_GRACE = Claimer::DEFAULT_CLAIM_GRACE
+  DEFAULT_STALE_PARTIAL_AFTER = Claimer::DEFAULT_STALE_PARTIAL_AFTER
+  DEFAULT_QUARANTINE_RETENTION = Claimer::DEFAULT_QUARANTINE_RETENTION
+  DEFAULT_MALFORMED_ENTRY = :quarantine
+  MALFORMED_ENTRY_MODES = %i[quarantine skip].freeze
+
+  attr_reader :claim_grace, :codec, :malformed_entry, :paths, :quarantine_retention, :stale_partial_after, :sync
+
+  def initialize(path:, sync: :flush, clock: Time, claim_grace: DEFAULT_CLAIM_GRACE, stale_partial_after: DEFAULT_STALE_PARTIAL_AFTER, quarantine_retention: DEFAULT_QUARANTINE_RETENTION, malformed_entry: DEFAULT_MALFORMED_ENTRY, codec: Codecs::Json.new)
+    @paths = Paths.new(path: path)
+    @sync = sync.respond_to?(:to_sym) ? sync.to_sym : sync
+    @clock = clock
+    @codec = Codecs.validate!(codec)
+    @claim_grace = Claimer.normalize_claim_grace(claim_grace)
+    @stale_partial_after = Claimer.normalize_stale_partial_after(stale_partial_after)
+    @quarantine_retention = Claimer.normalize_quarantine_retention(quarantine_retention)
+    @malformed_entry = normalize_malformed_entry(malformed_entry)
+    @claims_by_log_name = {}
+
+    writer
+    claimer
+  end
+
+  def append(payload)
+    writer.append(payload)
+  end
+
+  def read
+    writer.close_stale
+
+    Batch.new(
+      entries: active_claim_entries + new_claim_entries,
+      on_delete: method(:delete_entries),
+      on_release: method(:release_entries)
+    )
+  end
+
+  def drain
+    raise ArgumentError, "drain requires a block" unless block_given?
+
+    batch = read
+    completed = false
+
+    batch.each { |payload| yield payload }
+    batch.delete
+    completed = true
+
+    batch
+  ensure
+    batch.release if batch && !completed && !batch.finished?
+  end
+
+  def close
+    release_claims
+    writer.close
+  end
+
+  private
+
+  def delete_entries(ids)
+    log_names_from_entry_ids(ids).each { |log_name| delete_claim(log_name) }
+  end
+
+  def release_entries(ids)
+    log_names_from_entry_ids(ids).each { |log_name| release_claim(log_name) }
+  end
+
+  def active_claim_entries
+    @claims_by_log_name.values.flat_map do |claim|
+      entries = read_claim(claim)
+      next entries unless entries.empty?
+
+      @claims_by_log_name.delete(claim.name)
+      claim.release
+      []
+    end
+  end
+
+  def new_claim_entries
+    claimer.claim_completed.flat_map do |claim|
+      if @claims_by_log_name.key?(claim.name)
+        claim.release
+        next []
+      end
+
+      entries_from_claim(claim)
+    end
+  end
+
+  def entries_from_claim(claim)
+    entries = read_claim(claim)
+
+    if entries.empty?
+      claim.release
+    else
+      @claims_by_log_name[claim.name] = claim
+    end
+
+    entries
+  end
+
+  def read_claim(claim)
+    lines = claim.read_lines
+    return quarantine_claim(claim, reason: "empty_file", metadata: {"line_count" => 0}) if lines.empty? && empty_claim?(claim)
+    return [] if lines.empty?
+
+    entries = []
+    malformed_lines = []
+
+    lines.each_with_index do |line, index|
+      entry = parse_entry(line)
+      if entry
+        entries << [
+          encode_entry_id(claim.name, entry.fetch("id")),
+          entry.fetch("payload")
+        ]
+      else
+        malformed_lines << malformed_line_summary(line, index)
+      end
+    end
+
+    return entries if malformed_lines.empty?
+
+    if malformed_entry == :skip
+      return entries if entries.any?
+
+      return discard_fully_malformed_claim(claim)
+    end
+
+    quarantine_claim(
+      claim,
+      reason: "malformed_jsonl",
+      metadata: {
+        "line_count" => lines.length,
+        "malformed_lines" => malformed_lines
+      }
+    )
+  rescue SystemCallError, IOError
+    []
+  end
+
+  def parse_entry(line)
+    entry = codec.load(line)
+    return unless valid_entry?(entry)
+
+    entry
+  rescue
+    nil
+  end
+
+  def valid_entry?(entry)
+    entry.is_a?(Hash) &&
+      entry.fetch("id", nil).is_a?(Integer) &&
+      entry.fetch("id").positive? &&
+      entry.key?("payload")
+  end
+
+  def malformed_line_summary(line, index)
+    {
+      "line" => index + 1,
+      "bytes" => line.bytesize,
+      "sample" => line.byteslice(0, 200)
+    }
+  end
+
+  def discard_fully_malformed_claim(claim)
+    @claims_by_log_name.delete(claim.name)
+    claim.delete
+    []
+  rescue SystemCallError, IOError
+    claim.release
+    []
+  end
+
+  def normalize_malformed_entry(value)
+    mode = value.respond_to?(:to_sym) ? value.to_sym : value
+    return mode if MALFORMED_ENTRY_MODES.include?(mode)
+
+    raise ArgumentError, "malformed_entry must be :quarantine or :skip"
+  end
+
+  def encode_entry_id(log_name, entry_id)
+    "#{log_name}:#{entry_id}"
+  end
+
+  def log_names_from_entry_ids(ids)
+    Array(ids).filter_map { |id| decode_entry_id(id) }.uniq
+  end
+
+  def decode_entry_id(id)
+    value = id.to_s
+    separator = value.index(":")
+    return unless separator
+
+    log_name = value[0...separator]
+    log_name if valid_log_name?(log_name)
+  end
+
+  def valid_log_name?(log_name)
+    LogName.valid?(log_name)
+  end
+
+  def empty_claim?(claim)
+    File.zero?(claim.path)
+  rescue SystemCallError, IOError
+    false
+  end
+
+  def delete_claim(log_name)
+    claim = @claims_by_log_name.delete(log_name)
+    claim ? claim.delete : claimer.delete_claimed(log_name)
+  rescue SystemCallError, IOError
+    release_claim(log_name)
+  end
+
+  def release_claim(log_name)
+    claim = @claims_by_log_name.delete(log_name)
+    claim ? claim.release : claimer.release_claimed(log_name)
+  rescue SystemCallError, IOError
+    nil
+  end
+
+  def quarantine_claim(claim, reason:, metadata:)
+    @claims_by_log_name.delete(claim.name)
+    claim.quarantine(paths: paths, reason: reason, metadata: metadata, clock: @clock)
+    []
+  rescue SystemCallError, IOError
+    claim.release
+    []
+  end
+
+  def release_claims
+    claims = @claims_by_log_name.values
+    @claims_by_log_name = {}
+    claims.each(&:release)
+  rescue SystemCallError, IOError
+    nil
+  end
+
+  def writer
+    @writer ||= Writer.new(
+      path: paths.path,
+      sync: sync,
+      clock: @clock,
+      codec: codec
+    )
+  end
+
+  def claimer
+    @claimer ||= Claimer.new(
+      path: paths.path,
+      clock: @clock,
+      claim_grace: claim_grace,
+      stale_partial_after: stale_partial_after,
+      quarantine_retention: quarantine_retention
+    )
+  end
+end

data/lib/time_bucket_stream/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+class TimeBucketStream
+  VERSION = "0.1.0"
+end

data/lib/time_bucket_stream/writer.rb

@@ -0,0 +1,173 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "securerandom"
+require "socket"
+
+class TimeBucketStream
+  class Writer
+    BUCKET_FORMAT = "%Y%m%d%H%M"
+    SYNC_MODES = %i[none flush fsync].freeze
+
+    attr_reader :codec, :paths, :sync
+
+    def initialize(path:, sync: :flush, clock: Time, codec: Codecs::Json.new)
+      @paths = Paths.new(path: path)
+      @sync = sync.respond_to?(:to_sym) ? sync.to_sym : sync
+      @clock = clock
+      @codec = Codecs.validate!(codec)
+      @mutex = Mutex.new
+
+      validate_sync!
+      ensure_directories
+    end
+
+    def append(payload)
+      @mutex.synchronize do
+        prepare_writer
+        @sequence += 1
+
+        entry = {
+          "id" => @sequence,
+          "payload" => payload
+        }
+
+        write_line(encode_entry(entry))
+        entry.fetch("id")
+      end
+    end
+
+    def close
+      @mutex.synchronize do
+        close_writer
+      end
+    end
+
+    def close_stale
+      bucket = current_bucket
+
+      @mutex.synchronize do
+        close_writer if @writer_bucket && @writer_bucket < bucket
+      end
+    end
+
+    private
+
+    def validate_sync!
+      return if SYNC_MODES.include?(sync)
+
+      raise ArgumentError, "unsupported file sync mode: #{sync.inspect}"
+    end
+
+    def prepare_writer
+      bucket = current_bucket
+      return if writer_ready?(bucket)
+
+      close_writer
+      ensure_directories
+
+      @writer_pid = Process.pid
+      @writer_bucket = bucket
+      @writer_id = "#{bucket}-#{safe_hostname}-#{Process.pid}-#{SecureRandom.hex(4)}"
+      @sequence = 0
+      @log_path = paths.log_for(@writer_id)
+      @file = File.open(@log_path, File::WRONLY | File::CREAT | File::APPEND, 0o600)
+    end
+
+    def encode_entry(entry)
+      encoded = codec.dump(entry)
+      return encoded if encoded.is_a?(String) && !encoded.match?(/[\r\n]/)
+
+      raise ArgumentError, "codec must dump each entry to one line"
+    end
+
+    def writer_ready?(bucket)
+      active_writer? &&
+        @writer_bucket == bucket &&
+        log_file_exists?(@log_path)
+    end
+
+    def active_writer?
+      @writer_pid == Process.pid &&
+        @file &&
+        !@file.closed? &&
+        @log_path
+    end
+
+    def close_writer
+      close_file(@file)
+      @file = nil
+    end
+
+    def write_line(line)
+      data = "#{line}\n"
+
+      if sync == :none
+        syswrite_all(data)
+        return
+      end
+
+      write_all(data)
+      @file.flush
+      @file.fsync if sync == :fsync
+    end
+
+    def write_all(data)
+      bytes_written = 0
+
+      while bytes_written < data.bytesize
+        chunk = data.byteslice(bytes_written, data.bytesize - bytes_written)
+        written = @file.write(chunk)
+
+        raise IOError, "file write returned nil" unless written
+        raise IOError, "file write made no progress" unless written.positive?
+
+        bytes_written += written
+      end
+    end
+
+    def syswrite_all(data)
+      bytes_written = 0
+
+      while bytes_written < data.bytesize
+        chunk = data.byteslice(bytes_written, data.bytesize - bytes_written)
+        written = @file.syswrite(chunk)
+
+        raise IOError, "file syswrite made no progress" unless written.positive?
+
+        bytes_written += written
+      end
+    end
+
+    def ensure_directories
+      FileUtils.mkdir_p(paths.logs)
+    end
+
+    def current_bucket
+      current_time.utc.strftime(BUCKET_FORMAT)
+    end
+
+    def current_time
+      @clock.respond_to?(:call) ? @clock.call : @clock.now
+    end
+
+    def safe_hostname
+      value = Socket.gethostname.gsub(/[^A-Za-z0-9_.-]/, "_")
+      value.empty? ? "unknown-host" : value
+    rescue
+      "unknown-host"
+    end
+
+    def log_file_exists?(path)
+      File.exist?(path)
+    rescue SystemCallError
+      false
+    end
+
+    def close_file(file)
+      file&.close
+    rescue SystemCallError, IOError
+      nil
+    end
+  end
+end

data/lib/time_bucket_stream.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require_relative "time_bucket_stream/version"
+require_relative "time_bucket_stream/codecs"
+require_relative "time_bucket_stream/codecs/json"
+require_relative "time_bucket_stream/codecs/oj"
+require_relative "time_bucket_stream/paths"
+require_relative "time_bucket_stream/log_name"
+require_relative "time_bucket_stream/writer"
+require_relative "time_bucket_stream/quarantine"
+require_relative "time_bucket_stream/claim"
+require_relative "time_bucket_stream/claimer"
+require_relative "time_bucket_stream/batch"
+require_relative "time_bucket_stream/stream"

data/sig/time_bucket_stream.rbs

@@ -0,0 +1,132 @@
+class TimeBucketStream
+  VERSION: String
+  DEFAULT_CLAIM_GRACE: Integer
+  DEFAULT_MALFORMED_ENTRY: Symbol
+  DEFAULT_STALE_PARTIAL_AFTER: Integer
+  DEFAULT_QUARANTINE_RETENTION: Integer
+  MALFORMED_ENTRY_MODES: Array[Symbol]
+
+  attr_reader claim_grace: Integer
+  attr_reader codec: untyped
+  attr_reader malformed_entry: Symbol
+  attr_reader paths: Paths
+  attr_reader quarantine_retention: Integer?
+  attr_reader stale_partial_after: Integer
+  attr_reader sync: Symbol
+
+  def initialize: (path: String, ?sync: untyped, ?clock: untyped, ?claim_grace: untyped, ?stale_partial_after: untyped, ?quarantine_retention: untyped, ?malformed_entry: untyped, ?codec: untyped) -> void
+  def append: (untyped payload) -> Integer
+  def read: () -> Batch
+  def drain: () { (untyped payload) -> untyped } -> Batch
+  def close: () -> void
+
+  module Codecs
+    def self.validate!: (untyped codec) -> untyped
+
+    class Json
+      def dump: (untyped value) -> String
+      def load: (String value) -> untyped
+    end
+
+    class Oj
+      def initialize: () -> void
+      def dump: (untyped value) -> String
+      def load: (String value) -> untyped
+    end
+  end
+
+  class Batch
+    include Enumerable[untyped]
+
+    attr_reader entries: Array[[String, untyped]]
+
+    def initialize: (entries: Array[[String, untyped]], on_delete: untyped, on_release: untyped) -> void
+    def each: () { (untyped) -> void } -> Array[[String, untyped]]
+    def each: () -> Enumerator[untyped, void]
+    def size: () -> Integer
+    def length: () -> Integer
+    def empty?: () -> bool
+    def delete: () -> bool
+    def release: () -> bool
+    def finished?: () -> bool
+  end
+
+  class Writer
+    BUCKET_FORMAT: String
+    SYNC_MODES: Array[Symbol]
+
+    attr_reader codec: untyped
+    attr_reader paths: Paths
+    attr_reader sync: Symbol
+
+    def initialize: (path: String, ?sync: untyped, ?clock: untyped, ?codec: untyped) -> void
+    def append: (untyped payload) -> Integer
+    def close: () -> void
+    def close_stale: () -> void
+  end
+
+  class Quarantine
+    DEFAULT_RETENTION: Integer
+    TIMESTAMP_PATTERN: Regexp
+
+    attr_reader paths: Paths
+    attr_reader retention: Integer?
+
+    def self.normalize_retention: (untyped value) -> Integer?
+    def initialize: (path: String, ?clock: untyped, ?retention: untyped) -> void
+    def cleanup: () -> nil
+  end
+
+  module LogName
+    PATTERN: Regexp
+
+    def self.valid?: (untyped name) -> bool
+    def self.bucket: (untyped name) -> String?
+  end
+
+  class Claimer
+    DEFAULT_CLAIM_GRACE: Integer
+    DEFAULT_STALE_PARTIAL_AFTER: Integer
+    DEFAULT_QUARANTINE_RETENTION: Integer
+
+    attr_reader claim_grace: Integer
+    attr_reader paths: Paths
+    attr_reader quarantine_retention: Integer?
+    attr_reader stale_partial_after: Integer
+
+    def self.normalize_claim_grace: (untyped value) -> Integer
+    def self.normalize_stale_partial_after: (untyped value) -> Integer
+    def self.normalize_quarantine_retention: (untyped value) -> Integer?
+    def initialize: (path: String, ?clock: untyped, ?claim_grace: untyped, ?stale_partial_after: untyped, ?quarantine_retention: untyped) -> void
+    def claim_completed: (?before: String) -> Array[Claim]
+    def delete_claimed: (untyped log_name) -> untyped
+    def release_claimed: (untyped log_name) -> untyped
+  end
+
+  class Claim
+    attr_reader name: String
+    attr_reader path: String
+
+    def initialize: (name: String, path: String, lock_file: untyped) -> void
+    def read_lines: () -> Array[String]
+    def delete: () -> void
+    def quarantine: (paths: Paths, reason: untyped, ?metadata: untyped, ?clock: untyped) -> String?
+    def release: () -> void
+  end
+
+  class Paths
+    attr_reader path: String
+
+    def initialize: (path: String) -> void
+    def base: () -> String
+    def logs: () -> String
+    def processing: () -> String
+    def quarantine: () -> String
+    def claim_locks: () -> String
+    def log_for: (String writer_id) -> String
+    def processing_log_for: (String log_name) -> String
+    def quarantine_log_for: (String log_name) -> String
+    def quarantine_metadata_for: (String log_name) -> String
+    def claim_lock_for: (String log_name) -> String
+  end
+end

metadata

@@ -0,0 +1,61 @@
+--- !ruby/object:Gem::Specification
+name: time_bucket_stream
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Lim Yu Kwang
+bindir: exe
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies: []
+description: TimeBucketStream appends JSONL events into time-bucketed files and atomically
+  claims completed files for processing.
+email:
+- aaron.lim.yu.kwang@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".standard.yml"
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/time_bucket_stream.rb
+- lib/time_bucket_stream/batch.rb
+- lib/time_bucket_stream/claim.rb
+- lib/time_bucket_stream/claimer.rb
+- lib/time_bucket_stream/codecs.rb
+- lib/time_bucket_stream/codecs/json.rb
+- lib/time_bucket_stream/codecs/oj.rb
+- lib/time_bucket_stream/log_name.rb
+- lib/time_bucket_stream/paths.rb
+- lib/time_bucket_stream/quarantine.rb
+- lib/time_bucket_stream/stream.rb
+- lib/time_bucket_stream/version.rb
+- lib/time_bucket_stream/writer.rb
+- sig/time_bucket_stream.rbs
+homepage: https://github.com/aaron-lim/time_bucket_stream
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+  source_code_uri: https://github.com/aaron-lim/time_bucket_stream
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.1.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.9
+specification_version: 4
+summary: Time-bucketed file streams for Ruby.
+test_files: []