smart_rag 0.1.0 → 0.2.1

data/lib/smart_rag/core/media_processor.rb

@@ -0,0 +1,228 @@
+# frozen_string_literal: true
+
+require 'digest'
+require_relative 'media_metadata_extractor'
+require_relative 'video_semantic_extractor'
+require_relative 'transcript_normalizer'
+require_relative 'media_safety_policy'
+
+module SmartRAG
+  module Core
+    class MediaProcessor
+      CONTENT_SUMMARY_LIMIT = 4000
+
+      def initialize(document_processor:, metadata_extractor: MediaMetadataExtractor.new,
+                     video_extractor: VideoSemanticExtractor.new, safety_policy: MediaSafetyPolicy.new,
+                     content_store: nil, object_registry: nil, default_extractors: {}, logger: nil)
+        @document_processor = document_processor
+        @metadata_extractor = metadata_extractor
+        @video_extractor = video_extractor
+        @safety_policy = safety_policy
+        @content_store = content_store
+        @object_registry = object_registry
+        @default_extractors = default_extractors
+        @logger = logger || Logger.new($stdout)
+      end
+
+      def create(source, options = {})
+        options = default_extractors.merge(options)
+        file_path, materialized_file = resolve_source(source, options)
+        safety_policy.validate_file!(file_path)
+        requested_type = options[:media_type]
+        requested_type = nil if requested_type.to_s == 'auto'
+        extracted = safety_policy.with_timeout do
+          metadata_extractor.extract(file_path, media_type: requested_type)
+        end
+        safety_policy.validate_file!(file_path, duration_ms: extracted.dig(:media, :duration_ms))
+        stored = content_store&.put(file_path)
+        object_registry&.register(stored, byte_size: File.size(file_path)) if stored
+        semantic, timeline, warnings = safety_policy.with_timeout do
+          extract_semantic(file_path, extracted[:media_type], extracted[:media], options)
+        end
+        searchable_text = searchable_text_for(source, semantic, options)
+        metadata = build_metadata(file_path, extracted, semantic, searchable_text, options, stored)
+
+        chunks = timeline.empty? ? document_processor.chunk_content(searchable_text, options) : timeline_chunks(timeline)
+        document, sections = persist(source, metadata, chunks, options, stored)
+
+        all_warnings = Array(extracted[:warnings]) + warnings
+        {
+          document_id: document.id,
+          media_type: extracted[:media_type],
+          status: all_warnings.empty? ? 'success' : 'partial',
+          section_count: sections.length,
+          metadata: metadata.reject { |key, _| key == :content },
+          warnings: all_warnings
+        }
+      ensure
+        document_processor.cleanup_downloaded_file if document_processor.respond_to?(:cleanup_downloaded_file)
+        File.delete(materialized_file) if materialized_file && File.file?(materialized_file)
+      end
+
+      private
+
+      attr_reader :document_processor, :metadata_extractor, :video_extractor, :safety_policy,
+                  :content_store, :default_extractors, :logger
+
+      def resolve_source(source, options)
+        if source.to_s.match?(%r{\A(?:file|s3)://})
+          raise ArgumentError, 'configured content store cannot read media job source' unless content_store
+          path, temporary = content_store.materialize(source)
+          return [path, temporary ? path : nil]
+        end
+        if source.to_s.match?(%r{\Ahttps?://})
+          safety_policy.validate_url!(source)
+          return [document_processor.download_from_url(source, options.merge(
+            max_file_size: safety_policy.max_bytes, download_timeout: safety_policy.command_timeout
+          )), nil]
+        end
+        return [source, nil] if File.file?(source)
+
+        raise ArgumentError, "Invalid source: #{source}. Must be a valid URL or file path."
+      end
+
+      def extract_semantic(file_path, media_type, media, options)
+        warnings = []
+        captions = []
+        timeline = []
+        ocr_text = invoke_extractor(options[:ocr_extractor], file_path, warnings, 'image OCR') if media_type == 'image'
+        if media_type == 'image'
+          caption = invoke_extractor(options[:image_describer], file_path, warnings, 'image description')
+          captions << caption unless caption.to_s.strip.empty?
+        end
+        if media_type == 'audio'
+          transcript_result = invoke_extractor(options[:audio_transcriber], file_path, warnings, 'audio transcription')
+          timeline = TranscriptNormalizer.entries(transcript_result, extraction_kind: 'audio_transcript')
+          transcript = timeline.map { |entry| entry[:text] }.join("\n")
+        end
+        if media_type == 'video'
+          video_result = video_extractor.extract(file_path, duration_ms: media[:duration_ms], options: options)
+          timeline = Array(video_result[:timeline])
+          warnings.concat(Array(video_result[:warnings]))
+          transcript_entries = timeline.select { |entry| entry[:extraction_kind] == 'transcript' }
+          frame_entries = timeline.select { |entry| entry[:extraction_kind] == 'frame_description' }
+          transcript = transcript_entries.map { |entry| entry[:text] }.join("\n")
+          captions.concat(frame_entries.map { |entry| entry[:text] })
+        end
+
+        description = options[:description]
+        if media_type == 'image' && description.to_s.empty? && captions.empty? && ocr_text.to_s.empty?
+          warnings << 'no image semantic extractor configured; indexed filename and tags only'
+        elsif media_type == 'audio' && transcript.to_s.empty?
+          warnings << 'no audio transcriber configured; indexed filename and tags only'
+        elsif media_type == 'video' && timeline.empty? &&
+              (options[:transcribe] != false || options[:llm_caption] != false)
+          warnings << 'no video transcriber or frame describer configured; indexed filename and tags only'
+        end
+
+        [{ description: description, captions: captions, transcript: transcript,
+           ocr_text: ocr_text, tags: Array(options[:tags]).map(&:to_s) }, timeline, warnings]
+      end
+
+      def invoke_extractor(extractor, file_path, warnings, label)
+        return nil unless extractor
+
+        extractor.respond_to?(:call) ? extractor.call(file_path) : extractor.extract(file_path)
+      rescue StandardError => e
+        warnings << "#{label} failed: #{e.message}"
+        nil
+      end
+
+      def searchable_text_for(source, semantic, options)
+        parts = [options[:title] || File.basename(source.to_s), semantic[:description],
+                 semantic[:captions], semantic[:ocr_text], semantic[:transcript], semantic[:tags]].flatten
+        parts.map(&:to_s).map(&:strip).reject(&:empty?).uniq.join("\n\n")
+      end
+
+      def build_metadata(file_path, extracted, semantic, searchable_text, options, stored)
+        user_metadata = symbolize_keys(options[:metadata] || {})
+        user_metadata.merge(
+          schema_version: 1,
+          principal: options[:principal] || 'system',
+          file_path: file_path,
+          file_size: File.size(file_path),
+          file_type: File.extname(file_path).downcase,
+          content_hash: Digest::SHA256.file(file_path).hexdigest,
+          storage_uri: stored && stored[:storage_uri],
+          media_type: extracted[:media_type],
+          media: extracted[:media],
+          semantic: semantic,
+          title: options[:title],
+          author: options[:author],
+          description: options[:description],
+          content: searchable_text[0, CONTENT_SUMMARY_LIMIT]
+        ).compact
+      end
+
+      def timeline_chunks(timeline)
+        timeline.map.with_index do |entry, index|
+          timestamp = entry[:start_ms] || entry[:frame_timestamp_ms]
+          {
+            title: timeline_title(entry[:extraction_kind], timestamp, index),
+            content: entry[:text],
+            metadata: entry.reject { |key, _| key == :text }.merge(
+              media_type: entry[:extraction_kind] == 'audio_transcript' ? 'audio' : 'video'
+            )
+          }
+        end
+      end
+
+      def timeline_title(kind, timestamp_ms, index)
+        label = case kind
+                when 'frame_description' then 'Video frame'
+                when 'audio_transcript' then 'Audio transcript'
+                else 'Video transcript'
+                end
+        return "#{label} #{index + 1}" unless timestamp_ms
+
+        total_seconds = timestamp_ms.to_i / 1000
+        format('%s %02d:%02d:%02d', label, total_seconds / 3600, (total_seconds / 60) % 60, total_seconds % 60)
+      end
+
+      def persist(source, metadata, chunks, options, stored)
+        db = persistence_db
+        return persist_without_outer_transaction(source, metadata, chunks, options, stored) unless db
+
+        document = nil
+        sections = nil
+        db.transaction do
+          document = document_processor.create_or_update_document(source, metadata, options)
+          section_ids = db[:source_sections].where(document_id: document.id).select_map(:id)
+          db[:embeddings].where(source_id: section_ids).delete if section_ids.any? && db.table_exists?(:embeddings)
+          db[:source_sections].where(document_id: document.id).delete
+          sections = document_processor.save_sections(document, chunks, options)
+          object_registry&.attach(document.id, stored, byte_size: metadata[:file_size])
+          document.set_download_state(:completed)
+        end
+        [document, sections]
+      end
+
+      def persist_without_outer_transaction(source, metadata, chunks, options, stored)
+        document = document_processor.create_or_update_document(source, metadata, options)
+        db = document.class.db
+        sections = nil
+        db.transaction do
+          section_ids = db[:source_sections].where(document_id: document.id).select_map(:id)
+          db[:embeddings].where(source_id: section_ids).delete if section_ids.any? && db.table_exists?(:embeddings)
+          db[:source_sections].where(document_id: document.id).delete
+          sections = document_processor.save_sections(document, chunks, options)
+          object_registry&.attach(document.id, stored, byte_size: metadata[:file_size])
+          document.set_download_state(:completed)
+        end
+        [document, sections]
+      end
+
+      def persistence_db
+        ::SmartRAG::Models::SourceDocument.db
+      rescue StandardError
+        nil
+      end
+
+      def object_registry = @object_registry
+
+      def symbolize_keys(hash)
+        hash.each_with_object({}) { |(key, value), result| result[key.to_sym] = value }
+      end
+    end
+  end
+end

data/lib/smart_rag/core/media_safety_policy.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require 'ipaddr'
+require 'resolv'
+require 'timeout'
+require 'uri'
+
+module SmartRAG
+  module Core
+    class MediaSafetyPolicy
+      DEFAULT_MAX_BYTES = 50 * 1024 * 1024
+      DEFAULT_MAX_DURATION_MS = 2 * 60 * 60 * 1000
+
+      def initialize(config = {})
+        @max_bytes = config.fetch(:max_bytes, config.fetch(:max_file_size_mb, 50).to_i * 1024 * 1024)
+        @max_duration_ms = config.fetch(:max_duration_ms, DEFAULT_MAX_DURATION_MS).to_i
+        @allow_private_urls = config.fetch(:allow_private_urls, false)
+        @allowed_hosts = Array(config[:allowed_hosts]).map(&:downcase)
+        @command_timeout = config.fetch(:command_timeout_seconds, 120).to_i
+      end
+
+      attr_reader :max_bytes, :max_duration_ms, :command_timeout
+
+      def validate_file!(file_path, duration_ms: nil)
+        raise ArgumentError, "media file exceeds #{max_bytes} bytes" if File.size(file_path) > max_bytes
+        if duration_ms && duration_ms.to_i > max_duration_ms
+          raise ArgumentError, "media duration exceeds #{max_duration_ms}ms"
+        end
+        true
+      end
+
+      def validate_url!(source)
+        uri = URI.parse(source)
+        raise ArgumentError, 'media URL must use http or https' unless %w[http https].include?(uri.scheme)
+        raise ArgumentError, 'media URL must not contain credentials' if uri.userinfo
+        return true if @allowed_hosts.include?(uri.host.to_s.downcase)
+        return true if @allow_private_urls
+
+        addresses = Resolv.getaddresses(uri.host.to_s)
+        raise ArgumentError, 'media URL host could not be resolved' if addresses.empty?
+        raise ArgumentError, 'media URL resolves to a private or local address' if addresses.any? { |value| private_ip?(value) }
+        true
+      rescue URI::InvalidURIError => e
+        raise ArgumentError, "invalid media URL: #{e.message}"
+      end
+
+      def with_timeout(&block)
+        Timeout.timeout(command_timeout, &block)
+      end
+
+      private
+
+      def private_ip?(value)
+        ip = IPAddr.new(value)
+        ip.loopback? || ip.private? || ip.link_local? || ip.to_s == '0.0.0.0' || ip.to_s == '::'
+      rescue IPAddr::InvalidAddressError
+        true
+      end
+    end
+  end
+end

data/lib/smart_rag/core/s3_content_store.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+require 'digest'
+require 'tempfile'
+
+module SmartRAG
+  module Core
+    class S3ContentStore
+      def initialize(bucket:, region: nil, prefix: 'smart-rag/media', endpoint: nil,
+                     force_path_style: false, access_key_id: nil, secret_access_key: nil, client: nil)
+        @bucket = bucket.to_s
+        raise ArgumentError, 'media.content_store.bucket is required' if @bucket.empty?
+        @prefix = prefix.to_s.sub(%r{\A/+|/+$}, '')
+        @client = client || build_client(region:, endpoint:, force_path_style:, access_key_id:, secret_access_key:)
+      end
+
+      def put(file_path)
+        digest = Digest::SHA256.file(file_path).hexdigest
+        key = object_key(digest, File.extname(file_path).downcase)
+        client.put_object(bucket: bucket, key: key, body: File.open(file_path, 'rb')) unless exists?(key)
+        { content_hash: digest, storage_uri: "s3://#{bucket}/#{key}", stored_path: nil }
+      end
+
+      def delete(storage_uri)
+        key = key_from_uri(storage_uri)
+        client.delete_object(bucket: bucket, key: key)
+        true
+      end
+
+      def materialize(storage_uri)
+        key = key_from_uri(storage_uri)
+        extension = File.extname(key)
+        file = Tempfile.new(['smart-rag-object', extension])
+        file.close
+        client.get_object(bucket: bucket, key: key, response_target: file.path)
+        [file.path, true]
+      rescue StandardError
+        file&.unlink
+        raise
+      end
+
+      private
+
+      attr_reader :bucket, :prefix, :client
+
+      def build_client(region:, endpoint:, force_path_style:, access_key_id:, secret_access_key:)
+        require 'aws-sdk-s3'
+        options = { region: region || 'us-east-1', force_path_style: force_path_style }
+        options[:endpoint] = endpoint unless endpoint.to_s.empty?
+        unless access_key_id.to_s.empty?
+          options[:access_key_id] = access_key_id
+          options[:secret_access_key] = secret_access_key
+        end
+        Aws::S3::Client.new(**options)
+      rescue LoadError
+        raise LoadError, 'S3 content storage requires the aws-sdk-s3 gem'
+      end
+
+      def exists?(key)
+        client.head_object(bucket: bucket, key: key)
+        true
+      rescue StandardError => e
+        return false if e.class.name.end_with?('NotFound', 'NoSuchKey')
+        raise
+      end
+
+      def object_key(digest, extension)
+        [prefix, digest[0, 2], digest[2, 2], "#{digest}#{extension}"].reject(&:empty?).join('/')
+      end
+
+      def key_from_uri(uri)
+        prefix_value = "s3://#{bucket}/"
+        raise ArgumentError, 'storage URI does not belong to this S3 store' unless uri.to_s.start_with?(prefix_value)
+        uri.to_s.delete_prefix(prefix_value)
+      end
+    end
+  end
+end

data/lib/smart_rag/core/transcript_normalizer.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module SmartRAG
+  module Core
+    module TranscriptNormalizer
+      module_function
+
+      def entries(result, extraction_kind: 'transcript')
+        segments = if result.is_a?(Hash)
+                     result[:segments] || result['segments']
+                   elsif result.is_a?(Array)
+                     result
+                   end
+        return Array(segments).filter_map { |segment| normalize_segment(segment, extraction_kind) } if segments
+
+        text = result.is_a?(Hash) ? (result[:text] || result['text']) : result
+        text.to_s.strip.empty? ? [] : [{ extraction_kind: extraction_kind, text: text.to_s.strip }]
+      end
+
+      def normalize_segment(segment, extraction_kind)
+        return { extraction_kind: extraction_kind, text: segment.to_s.strip } unless segment.is_a?(Hash)
+
+        text = segment[:text] || segment['text']
+        return nil if text.to_s.strip.empty?
+
+        {
+          extraction_kind: extraction_kind,
+          text: text.to_s.strip,
+          start_ms: timestamp_ms(segment, :start),
+          end_ms: timestamp_ms(segment, :end),
+          speaker: segment[:speaker] || segment['speaker']
+        }.compact
+      end
+
+      def timestamp_ms(segment, key)
+        explicit = segment["#{key}_ms".to_sym] || segment["#{key}_ms"]
+        return explicit.to_f.round unless explicit.nil?
+
+        seconds = segment[key] || segment[key.to_s]
+        seconds.nil? ? nil : (seconds.to_f * 1000).round
+      end
+    end
+  end
+end

data/lib/smart_rag/core/video_semantic_extractor.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'open3'
+require 'tmpdir'
+require_relative 'transcript_normalizer'
+
+module SmartRAG
+  module Core
+    class VideoSemanticExtractor
+      DEFAULT_FRAME_INTERVAL_SECONDS = 30
+      DEFAULT_MAX_FRAMES = 12
+
+      def extract(file_path, duration_ms:, options: {})
+        warnings = []
+        timeline = []
+
+        Dir.mktmpdir('smart_rag_video') do |directory|
+          timeline.concat(transcript_entries(file_path, directory, options, warnings))
+          timeline.concat(frame_entries(file_path, directory, duration_ms, options, warnings))
+        end
+
+        { timeline: timeline.sort_by { |entry| entry[:start_ms] || entry[:frame_timestamp_ms] || 0 },
+          warnings: warnings }
+      rescue StandardError => e
+        { timeline: [], warnings: ["video semantic extraction failed: #{e.message}"] }
+      end
+
+      private
+
+      def transcript_entries(file_path, directory, options, warnings)
+        return [] if options[:transcribe] == false
+
+        transcriber = options[:video_transcriber] || options[:audio_transcriber]
+        unless transcriber
+          warnings << 'no video transcriber configured; video audio transcription skipped'
+          return []
+        end
+
+        audio_path = File.join(directory, 'audio.wav')
+        output, status = Open3.capture2e(
+          'ffmpeg', '-y', '-v', 'error', '-i', file_path, '-vn', '-ac', '1', '-ar', '16k', audio_path
+        )
+        unless status.success? && File.file?(audio_path)
+          warnings << "video audio extraction failed: #{output.strip}"
+          return []
+        end
+
+        TranscriptNormalizer.entries(invoke(transcriber, audio_path))
+      rescue Errno::ENOENT
+        warnings << 'ffmpeg not available; video audio transcription skipped'
+        []
+      rescue StandardError => e
+        warnings << "video transcription failed: #{e.message}"
+        []
+      end
+
+      def frame_entries(file_path, directory, duration_ms, options, warnings)
+        return [] if options[:llm_caption] == false
+
+        describer = options[:frame_describer] || options[:image_describer]
+        unless describer
+          warnings << 'no frame describer configured; video frame descriptions skipped'
+          return []
+        end
+
+        frame_timestamps(file_path, duration_ms, options).filter_map.with_index do |timestamp_ms, index|
+          frame_path = File.join(directory, format('frame-%03d.jpg', index))
+          output, status = Open3.capture2e(
+            'ffmpeg', '-y', '-v', 'error', '-ss', format('%.3f', timestamp_ms / 1000.0),
+            '-i', file_path, '-frames:v', '1', '-q:v', '2', frame_path
+          )
+          unless status.success? && File.file?(frame_path)
+            warnings << "video frame extraction failed at #{timestamp_ms}ms: #{output.strip}"
+            next
+          end
+
+          description = invoke(describer, frame_path, timestamp_ms)
+          next if description.to_s.strip.empty?
+
+          { extraction_kind: 'frame_description', text: description.to_s.strip,
+            frame_timestamp_ms: timestamp_ms, start_ms: timestamp_ms, end_ms: timestamp_ms }
+        rescue Errno::ENOENT
+          warnings << 'ffmpeg not available; video frame descriptions skipped'
+          break []
+        rescue StandardError => e
+          warnings << "video frame description failed at #{timestamp_ms}ms: #{e.message}"
+          nil
+        end
+      end
+
+      def frame_timestamps(file_path, duration_ms, options)
+        if options.fetch(:scene_detection, true)
+          scene_timestamps = detect_scenes(file_path, options)
+          return scene_timestamps unless scene_timestamps.empty?
+        end
+
+        interval_ms = [options.fetch(:frame_interval_seconds, DEFAULT_FRAME_INTERVAL_SECONDS).to_f * 1000, 1000].max.round
+        max_frames = [[options.fetch(:max_frames, DEFAULT_MAX_FRAMES).to_i, 1].max, 100].min
+        duration = duration_ms.to_i
+        return [0] if duration <= 0
+
+        timestamps = (0...duration).step(interval_ms).first(max_frames)
+        timestamps.empty? ? [0] : timestamps
+      end
+
+      def detect_scenes(file_path, options)
+        return [] if file_path.to_s.empty?
+
+        threshold = options.fetch(:scene_threshold, 0.35).to_f
+        max_frames = [[options.fetch(:max_frames, DEFAULT_MAX_FRAMES).to_i, 1].max, 100].min
+        output, status = Open3.capture2e(
+          'ffmpeg', '-hide_banner', '-i', file_path, '-filter:v', "select='gt(scene,#{threshold})',showinfo",
+          '-an', '-f', 'null', '-'
+        )
+        return [] unless status.success?
+
+        output.scan(/pts_time:([0-9.]+)/).flatten.map { |seconds| (seconds.to_f * 1000).round }.uniq.first(max_frames)
+      rescue Errno::ENOENT, SystemCallError
+        []
+      end
+
+      def invoke(extractor, path, timestamp_ms = nil)
+        callable = extractor.respond_to?(:call) ? extractor : extractor.method(:extract)
+        timestamp_ms.nil? || callable.arity == 1 ? callable.call(path) : callable.call(path, timestamp_ms)
+      end
+
+    end
+  end
+end

data/lib/smart_rag/http_access_policy.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+require 'digest'
+
+module SmartRAG
+  class HttpAccessPolicy
+    class Unauthorized < StandardError; end
+    class QuotaExceeded < StandardError; end
+
+    def initialize(db:, config: {}, clock: -> { Time.now })
+      @db = db
+      @config = config || {}
+      @clock = clock
+      @token_digests = configured_tokens.to_h { |principal, token| [digest(token), principal.to_s] }
+    end
+
+    def authorize!(request)
+      return 'anonymous' unless config.fetch(:enabled, false)
+
+      scheme, token = request.get_header('HTTP_AUTHORIZATION').to_s.split(' ', 2)
+      principal = token_digests[digest(token)] if scheme.to_s.casecmp('Bearer').zero? && token
+      raise Unauthorized, 'invalid or missing bearer token' unless principal
+      principal
+    end
+
+    def consume!(principal, request_bytes: 0)
+      return unless quota_enabled?
+      raise QuotaExceeded, 'quota storage is unavailable' unless db&.table_exists?(:api_rate_limits)
+
+      enforce_window!(principal, 'minute', minute_start, requests: 1,
+                       request_limit: config.dig(:quota, :requests_per_minute))
+      enforce_window!(principal, 'day', day_start, bytes: request_bytes.to_i,
+                       byte_limit: config.dig(:quota, :upload_bytes_per_day))
+    end
+
+    private
+
+    attr_reader :db, :config, :clock, :token_digests
+
+    def configured_tokens
+      tokens = config[:tokens]
+      return tokens if tokens.is_a?(Hash)
+      token = config[:token].to_s
+      token.empty? ? {} : { config.fetch(:principal, 'default') => token }
+    end
+
+    def digest(value)
+      Digest::SHA256.hexdigest(value.to_s)
+    end
+
+    def quota_enabled?
+      quota = config[:quota] || {}
+      quota[:requests_per_minute].to_i.positive? || quota[:upload_bytes_per_day].to_i.positive?
+    end
+
+    def minute_start
+      now = clock.call
+      Time.new(now.year, now.month, now.day, now.hour, now.min, 0, now.utc_offset)
+    end
+
+    def day_start
+      now = clock.call
+      Time.new(now.year, now.month, now.day, 0, 0, 0, now.utc_offset)
+    end
+
+    def enforce_window!(principal, period, window_start, requests: 0, bytes: 0,
+                        request_limit: nil, byte_limit: nil)
+      return if request_limit.to_i <= 0 && byte_limit.to_i <= 0
+
+      db.transaction do
+        dataset = db[:api_rate_limits].where(principal: principal, period: period, window_start: window_start)
+        row = dataset.for_update.first
+        unless row
+          db[:api_rate_limits].insert(principal: principal, period: period, window_start: window_start,
+                                      request_count: 0, byte_count: 0, updated_at: clock.call)
+          row = dataset.for_update.first
+        end
+        next_requests = row[:request_count].to_i + requests
+        next_bytes = row[:byte_count].to_i + bytes
+        raise QuotaExceeded, "#{period} request quota exceeded" if request_limit.to_i.positive? && next_requests > request_limit.to_i
+        raise QuotaExceeded, "#{period} upload quota exceeded" if byte_limit.to_i.positive? && next_bytes > byte_limit.to_i
+        dataset.update(request_count: next_requests, byte_count: next_bytes, updated_at: clock.call)
+      end
+    end
+  end
+end