instagram_connect 0.2.1 → 0.3.0

data/lib/instagram_connect/ingest/registry.rb

@@ -0,0 +1,93 @@
+module InstagramConnect
+  class Ingest
+    # Maps a webhook field to the handler that parses it, and infers the field
+    # for messaging events (Meta delivers all of them under `entry.messaging`
+    # regardless of which subscription triggered them, so the field has to be
+    # recovered from the event's shape).
+    module Registry
+      HANDLERS = {
+        "messages" => Handlers::Messages,
+        "message_echoes" => Handlers::Messages,
+        "messaging_postbacks" => Handlers::MessagingPostbacks,
+        "message_reactions" => Handlers::MessageReactions,
+        "messaging_seen" => Handlers::MessagingSeen,
+        "messaging_referral" => Handlers::MessagingReferral,
+        "messaging_optins" => Handlers::MessagingOptins,
+        "messaging_handover" => Handlers::MessagingHandover,
+        "comments" => Handlers::Comments,
+        "live_comments" => Handlers::Comments,
+        "mentions" => Handlers::Mentions,
+        "story_insights" => Handlers::StoryInsights
+      }.freeze
+
+      # Every field the gem knows Meta can send on the `instagram` object.
+      # Fields without a handler are still worth subscribing to: they bank as
+      # `unhandled` rows and replay once a handler ships, whereas an
+      # unsubscribed field is gone for good — Meta does not store `mentions` or
+      # `story_insights` notifications, and re-delivers nothing.
+      KNOWN_FIELDS = %w[
+        messages
+        message_echoes
+        message_reactions
+        messaging_postbacks
+        messaging_seen
+        messaging_referral
+        messaging_optins
+        messaging_handover
+        messaging_policy_enforcement
+        response_feedback
+        standby
+        comments
+        live_comments
+        mentions
+        story_insights
+      ].freeze
+
+      # Shape tests in priority order. `is_echo` must be checked before the
+      # plain `message` case, since an echo carries a message too.
+      MESSAGING_SHAPES = [
+        [ "message_echoes", ->(e) { e["message"] && e.dig("message", "is_echo") } ],
+        [ "messages", ->(e) { e["message"] } ],
+        [ "message_reactions", ->(e) { e["reaction"] } ],
+        [ "messaging_postbacks", ->(e) { e["postback"] } ],
+        [ "messaging_seen", ->(e) { e["read"] } ],
+        [ "messaging_referral", ->(e) { e["referral"] } ],
+        [ "messaging_optins", ->(e) { e["optin"] } ],
+        [ "messaging_handover", lambda { |e|
+          e["pass_thread_control"] || e["take_thread_control"] || e["request_thread_control"]
+        } ]
+      ].freeze
+
+      UNKNOWN_FIELD = "unknown".freeze
+
+      module_function
+
+      def handler_for(field)
+        HANDLERS.fetch(field, Handlers::Unknown)
+      end
+
+      def handled?(field)
+        HANDLERS.key?(field)
+      end
+
+      # The fields a host should subscribe its Page to. Everything the gem
+      # knows about, handled or not — see KNOWN_FIELDS.
+      def subscribable_fields
+        KNOWN_FIELDS
+      end
+
+      def handled_fields
+        HANDLERS.keys
+      end
+
+      # `message.referral` and `postback.referral` are nested inside their
+      # parent event and stay with it; only a standalone `referral` is a
+      # referral event in its own right, which is why the shape tests run in
+      # order rather than checking every key independently.
+      def messaging_field_for(event)
+        found = MESSAGING_SHAPES.find { |(_field, test)| test.call(event) }
+        found ? found.first : UNKNOWN_FIELD
+      end
+    end
+  end
+end

data/lib/instagram_connect/ingest/summary.rb

@@ -0,0 +1,75 @@
+module InstagramConnect
+  class Ingest
+    # What one webhook delivery produced. Behaves as a Hash so the return value
+    # of Ingest.call stays compatible with hosts written against 0.2.x, which
+    # read summary[:messages] and friends.
+    class Summary
+      COUNTERS = {
+        "messages" => :messages,
+        "message_echoes" => :messages,
+        "messaging_postbacks" => :postbacks,
+        "comments" => :comments,
+        "live_comments" => :comments
+      }.freeze
+
+      attr_accessor :messages, :comments, :postbacks, :duplicates, :unhandled,
+                    :failed, :callback_errors
+
+      def initialize
+        @messages = 0
+        @comments = 0
+        @postbacks = 0
+        @duplicates = 0
+        @unhandled = 0
+        @failed = 0
+        @callback_errors = 0
+      end
+
+      def count(field)
+        counter = COUNTERS[field]
+        return if counter.nil?
+
+        public_send(:"#{counter}=", public_send(counter) + 1)
+      end
+
+      # 0.2.x reported one `skipped` number covering both "we have seen this
+      # already" and "we do not parse this". They are different problems now —
+      # duplicates are healthy, unhandled events are a backlog to replay — but
+      # the combined figure is kept so existing hosts keep working.
+      def skipped
+        duplicates + unhandled
+      end
+
+      def to_h
+        {
+          messages: messages,
+          comments: comments,
+          postbacks: postbacks,
+          skipped: skipped,
+          duplicates: duplicates,
+          unhandled: unhandled,
+          failed: failed,
+          callback_errors: callback_errors
+        }
+      end
+
+      # Implicit conversion, not just the explicit #to_h: it is what makes a
+      # Summary behave as a Hash for callers (and matchers) written against the
+      # plain hash 0.2.x returned.
+      alias_method :to_hash, :to_h
+
+      def [](key)
+        to_h[key.to_sym]
+      end
+
+      def ==(other)
+        other.respond_to?(:to_h) ? to_h == other.to_h : super
+      end
+
+      def each(&block)
+        to_h.each(&block)
+      end
+      include Enumerable
+    end
+  end
+end

data/lib/instagram_connect/ingest.rb

@@ -1,112 +1,34 @@
 module InstagramConnect
-  # Parses a verified Meta webhook payload and persists it into the engine's
-  # chat models, deduping on Meta's message id. After persistence it fires the
-  # configured host handlers (on_message / on_comment / on_postback) so the host
-  # can layer extras (notifications, AI replies) on top.
+  # Turns a verified Meta webhook payload into persisted rows, and fires the
+  # configured host callbacks (on_message / on_comment / on_postback) so the
+  # host can layer notifications, AI drafting or CRM linking on top.
   #
-  # Returns a summary hash of what was processed.
+  # Every event is banked on InstagramConnect::WebhookEvent before anything
+  # parses it. That is what makes an unparsed or mis-parsed event recoverable:
+  # Meta neither stores nor re-delivers webhooks, so anything dropped here is
+  # gone permanently. See Ingest::Dispatcher for the walk.
+  #
+  # Returns a Summary, which reads like the plain hash 0.2.x returned.
   class Ingest
     def self.call(payload:, config: InstagramConnect.configuration)
-      new(config).call(payload)
-    end
-
-    def initialize(config)
-      @config = config
-    end
-
-    def call(payload)
-      summary = { messages: 0, comments: 0, postbacks: 0, skipped: 0 }
-      Array(payload && payload["entry"]).each do |entry|
-        account = account_for(entry)
-        if account.nil?
-          summary[:skipped] += 1
-          next
-        end
-        Array(entry["messaging"]).each { |event| ingest_messaging(account, event, summary) }
-        Array(entry["changes"]).each { |change| ingest_change(account, change, summary) }
-      end
-      summary
-    end
-
-    private
-
-    attr_reader :config
-
-    def account_for(entry)
-      id = entry["id"].to_s
-      Account.find_by(ig_user_id: id) || Account.find_by(page_id: id)
-    end
-
-    def ingest_messaging(account, event, summary)
-      if event["message"]
-        ingest_message(account, event, summary)
-      elsif event["postback"]
-        invoke(config.on_postback, build_postback(account, event))
-        summary[:postbacks] += 1
-      else
-        summary[:skipped] += 1
-      end
-    end
-
-    def ingest_message(account, event, summary)
-      msg = event["message"]
-      mid = msg["mid"].to_s
-      if mid.empty? || !InboundMessage.claim(ig_message_id: mid, account_id: account.id)
-        summary[:skipped] += 1
-        return
-      end
-
-      echo = msg["is_echo"] ? true : false
-      igsid = (echo ? event.dig("recipient", "id") : event.dig("sender", "id")).to_s
-      conversation = Conversation.locate(account: account, igsid: igsid)
-      message = Message.create!(
-        conversation: conversation,
-        direction: echo ? "outbound" : "inbound",
-        status: echo ? "sent" : "received",
-        source: echo ? "operator_app" : "inbound",
-        kind: "dm",
-        body: msg["text"],
-        ig_message_id: mid,
-        media_status: media?(msg) ? "pending" : "none"
-      )
-      conversation.register_message(message)
-      invoke(config.on_message, message) unless echo
-      summary[:messages] += 1
-    end
-
-    def ingest_change(account, change, summary)
-      unless change["field"] == "comments"
-        summary[:skipped] += 1
-        return
-      end
-      value = change["value"] || {}
-      comment = Comment.record(
-        account: account,
-        comment_id: value["id"].to_s,
-        media_id: value.dig("media", "id"),
-        text: value["text"],
-        from_username: value.dig("from", "username"),
-        parent_id: value["parent_id"]
-      )
-      invoke(config.on_comment, comment)
-      summary[:comments] += 1
-    end
-
-    def media?(msg)
-      Array(msg["attachments"]).any?
-    end
-
-    def build_postback(account, event)
-      {
-        account_id: account.id,
-        sender_id: event.dig("sender", "id"),
-        payload: event.dig("postback", "payload"),
-        title: event.dig("postback", "title")
-      }
-    end
-
-    def invoke(handler, arg)
-      handler.call(arg) if handler.respond_to?(:call)
+      Dispatcher.new(config).call(payload)
     end
   end
 end
+
+require_relative "ingest/envelope"
+require_relative "ingest/summary"
+require_relative "ingest/handlers/base"
+require_relative "ingest/handlers/messages"
+require_relative "ingest/handlers/messaging_postbacks"
+require_relative "ingest/handlers/message_reactions"
+require_relative "ingest/handlers/messaging_seen"
+require_relative "ingest/handlers/messaging_referral"
+require_relative "ingest/handlers/messaging_optins"
+require_relative "ingest/handlers/messaging_handover"
+require_relative "ingest/handlers/comments"
+require_relative "ingest/handlers/mentions"
+require_relative "ingest/handlers/story_insights"
+require_relative "ingest/handlers/unknown"
+require_relative "ingest/registry"
+require_relative "ingest/dispatcher"

data/lib/instagram_connect/media/attaching.rb

@@ -0,0 +1,57 @@
+require "marcel"
+require "stringio"
+
+module InstagramConnect
+  module Media
+    # Turns fetched bytes into a stored file, or into an honest reason why not.
+    #
+    # The rule that matters here: when the declared Content-Type and the actual
+    # magic bytes disagree, the magic bytes win. A Content-Type header is
+    # metadata anyone can set; the first few bytes of the file are what the file
+    # actually is. Trusting the header is how a host ends up serving an HTML
+    # document from an <img> tag.
+    module Attaching
+      Result = Struct.new(:ok, :mime, :size, :error, keyword_init: true) do
+        def ok? = ok
+      end
+
+      module_function
+
+      def evaluate(body:, declared_mime:, filename: nil, config: InstagramConnect.configuration)
+        bytes = body.to_s
+        return failure("empty") if bytes.empty?
+
+        mime = sniff(bytes, filename: filename, declared: declared_mime)
+        return failure("type_not_allowed:#{mime}") unless allowed?(mime, config)
+        return failure("too_large:#{bytes.bytesize}") unless within_size?(mime, bytes.bytesize, config)
+
+        Result.new(ok: true, mime: mime, size: bytes.bytesize)
+      end
+
+      def sniff(bytes, filename: nil, declared: nil)
+        sniffed = ::Marcel::MimeType.for(StringIO.new(bytes), name: filename)
+        # Marcel falls back to this when it recognises nothing, in which case the
+        # declared type is the only information available and is better than
+        # nothing.
+        return declared.to_s.split(";").first.to_s.strip if sniffed == "application/octet-stream" && declared.present?
+
+        sniffed
+      end
+
+      def allowed?(mime, config)
+        allowed = config.allowed_media_types
+        allowed.blank? || allowed.include?(mime)
+      end
+
+      # Both Meta's per-type ceiling and the host's own cap have to hold. The
+      # host's exists so an adopter can be stricter than Meta, never looser.
+      def within_size?(mime, bytes, config)
+        Limits.within_size?(mime, bytes) && bytes <= config.media_max_bytes.to_i
+      end
+
+      def failure(reason)
+        Result.new(ok: false, error: reason)
+      end
+    end
+  end
+end

data/lib/instagram_connect/media/limits.rb

@@ -0,0 +1,46 @@
+module InstagramConnect
+  module Media
+    # Meta's ceilings, in one place, read by the composer, the send path and the
+    # client so the numbers cannot drift apart.
+    module Limits
+      IMAGE_MAX_BYTES = 8 * 1024 * 1024
+      AUDIO_VIDEO_MAX_BYTES = 25 * 1024 * 1024
+      ATTACHMENTS_MAX = 10
+
+      # BYTES, not characters. An emoji-heavy 400-character reply is over the
+      # limit and Meta rejects the whole send, so anything measuring this with
+      # String#length is measuring the wrong thing.
+      TEXT_MAX_BYTES = 1000
+
+      IMAGE_TYPES = %w[image/jpeg image/png image/gif].freeze
+      AUDIO_TYPES = %w[audio/aac audio/mp4 audio/mpeg audio/ogg audio/wav].freeze
+      VIDEO_TYPES = %w[video/mp4 video/ogg video/avi video/quicktime video/webm].freeze
+
+      module_function
+
+      # The ceiling that applies to a given MIME type. Images get a tighter one
+      # than audio and video, and an unknown type is held to the tighter of the
+      # two rather than waved through.
+      def max_bytes_for(mime)
+        image?(mime) ? IMAGE_MAX_BYTES : AUDIO_VIDEO_MAX_BYTES
+      end
+
+      def image?(mime)
+        IMAGE_TYPES.include?(mime.to_s)
+      end
+
+      def within_size?(mime, bytes)
+        bytes.to_i.positive? && bytes.to_i <= max_bytes_for(mime)
+      end
+
+      # Only images can be batched; Meta accepts one audio or video per message.
+      def attachments_within_limit?(count)
+        count.to_i <= ATTACHMENTS_MAX
+      end
+
+      def text_within_limit?(text)
+        text.to_s.bytesize <= TEXT_MAX_BYTES
+      end
+    end
+  end
+end

data/lib/instagram_connect/rate_limiter.rb

@@ -0,0 +1,88 @@
+module InstagramConnect
+  # Keeps calls inside Meta's per-account ceilings.
+  #
+  # Meta publishes several limits at once and exhausting any of them throttles
+  # the whole app for hours, so the cost of being slightly conservative is far
+  # lower than the cost of being slightly wrong.
+  module RateLimiter
+    # Meta's documented per-account limits. The graph default is the floor of
+    # the 4800 × impressions formula: impressions are not knowable from the
+    # messaging API, so the floor is what is safe to assume until the usage
+    # headers say otherwise.
+    BUCKETS = {
+      graph: { limit: 4800, window: 86_400 },
+      conversations: { limit: 2, window: 1 },
+      send_text: { limit: 100, window: 1 },
+      send_media: { limit: 10, window: 1 },
+      private_reply: { limit: 750, window: 3600 },
+      publish: { limit: 50, window: 86_400 }
+    }.freeze
+
+    Outcome = Struct.new(:allowed, :retry_after, keyword_init: true) do
+      def allowed? = allowed
+    end
+
+    module_function
+
+    # Claims one call from a bucket. Returns an Outcome saying whether to
+    # proceed and, if not, how long to wait — callers re-enqueue rather than
+    # sleep, since a worker asleep on a rate limit is a worker not delivering
+    # anything else.
+    def acquire(account:, bucket:, now: Time.current)
+      spec = BUCKETS.fetch(bucket.to_sym)
+      row = budget_for(account, bucket, spec, now)
+
+      return Outcome.new(allowed: false, retry_after: (row.blocked_until - now).ceil) if row.blocked?
+
+      claimed = ApiBudget.where(id: row.id).where("used < limit_value")
+                         .update_all("used = used + 1, updated_at = #{ApiBudget.connection.quote(now)}")
+
+      if claimed == 1
+        Outcome.new(allowed: true, retry_after: nil)
+      else
+        Outcome.new(allowed: false, retry_after: (row.window_end - now).ceil.clamp(1, spec[:window]))
+      end
+    end
+
+    # Records what Meta reported after a call. Its own numbers beat any local
+    # arithmetic, so a warning halves the remaining allowance and a critical
+    # reading blocks the account outright until the stated recovery time.
+    def observe(account:, bucket:, usage:, now: Time.current)
+      return if usage.nil?
+
+      spec = BUCKETS.fetch(bucket.to_sym)
+      row = budget_for(account, bucket, spec, now)
+
+      if usage.critical?
+        row.update!(blocked_until: now + (usage.retry_after || spec[:window]))
+      elsif usage.warning?
+        row.update!(limit_value: [ row.limit_value / 2, row.used + 1 ].max)
+      end
+    end
+
+    # Called when Meta returns an explicit retry_after: one 429 pauses every
+    # job for that account, rather than each one having to discover it.
+    def block!(account:, bucket:, seconds:, now: Time.current)
+      spec = BUCKETS.fetch(bucket.to_sym)
+      budget_for(account, bucket, spec, now).update!(blocked_until: now + seconds.to_i)
+    end
+
+    def budget_for(account, bucket, spec, now)
+      start = window_start(now, spec[:window])
+      ApiBudget.find_or_create_by!(
+        account_id: account.id, bucket: bucket.to_s, window_start: start
+      ) do |row|
+        row.window_seconds = spec[:window]
+        row.limit_value = spec[:limit]
+      end
+    rescue ActiveRecord::RecordNotUnique
+      ApiBudget.find_by!(account_id: account.id, bucket: bucket.to_s, window_start: start)
+    end
+
+    # Fixed windows aligned to the epoch, so every process agrees on where a
+    # window starts without having to coordinate.
+    def window_start(now, seconds)
+      Time.at((now.to_i / seconds) * seconds).utc
+    end
+  end
+end

data/lib/instagram_connect/result.rb

@@ -3,15 +3,17 @@ module InstagramConnect
   # API-level failures — callers branch on +success?+ and read +error_code+ /
   # +error_message+. Transport/programming errors still raise.
   class Result
-    attr_reader :success, :id, :error_code, :error_message, :retry_after, :data
+    attr_reader :success, :id, :error_code, :error_message, :retry_after, :data, :usage
 
-    def initialize(success:, id: nil, error_code: nil, error_message: nil, retry_after: nil, data: {})
+    def initialize(success:, id: nil, error_code: nil, error_message: nil, retry_after: nil,
+                   data: {}, usage: nil)
       @success = success
       @id = id
       @error_code = error_code
       @error_message = error_message
       @retry_after = retry_after
       @data = data || {}
+      @usage = usage
     end
 
     def success?
@@ -22,13 +24,31 @@ module InstagramConnect
       !success?
     end
 
+    # Meta's cursor for the next page, or nil when this is the last one. Reading
+    # `paging.next` rather than incrementing an offset is the only correct way
+    # through a Graph collection — results shift under you between calls.
+    def next_cursor
+      data.dig("paging", "cursors", "after").presence if more?
+    end
+
+    def more?
+      data.is_a?(Hash) && data.dig("paging", "next").present?
+    end
+
+    # The rows of a collection response, or [] for a single-object one.
+    def records
+      list = data["data"]
+      list.is_a?(Array) ? list : []
+    end
+
     # Convenience builders so call sites read cleanly.
-    def self.ok(id: nil, data: {})
-      new(success: true, id: id, data: data)
+    def self.ok(id: nil, data: {}, usage: nil)
+      new(success: true, id: id, data: data, usage: usage)
     end
 
-    def self.error(message, error_code: nil, retry_after: nil, data: {})
-      new(success: false, error_message: message, error_code: error_code, retry_after: retry_after, data: data)
+    def self.error(message, error_code: nil, retry_after: nil, data: {}, usage: nil)
+      new(success: false, error_message: message, error_code: error_code,
+          retry_after: retry_after, data: data, usage: usage)
     end
   end
 end

data/lib/instagram_connect/text_splitter.rb

@@ -0,0 +1,51 @@
+module InstagramConnect
+  # Splits a reply that exceeds Meta's 1000-byte message ceiling into parts that
+  # fit.
+  #
+  # Splitting rather than truncating is deliberate: a customer reading half an
+  # answer is worse than reading two bubbles, and the operator who wrote it has
+  # no way to know it was cut.
+  module TextSplitter
+    module_function
+
+    # Returns the parts to send, in order. A message already within the limit
+    # comes back as a single-element array, so callers have one code path.
+    def split(text, limit: Media::Limits::TEXT_MAX_BYTES)
+      text = text.to_s
+      return [ text ] if text.bytesize <= limit
+
+      parts = []
+      remaining = text
+
+      while remaining.bytesize > limit
+        chunk = take(remaining, limit)
+        parts << chunk.rstrip
+        remaining = remaining[chunk.length..].to_s.lstrip
+      end
+
+      # Text that divides exactly on a break leaves nothing over; appending an
+      # empty part would send an empty bubble.
+      parts << remaining unless remaining.empty?
+      parts
+    end
+
+    # The longest prefix that fits, broken at whitespace where possible so words
+    # survive. Falls back to a hard character break for input with no
+    # whitespace at all — a long URL, or a script that does not use spaces.
+    def take(text, limit)
+      candidate = +""
+      last_break = nil
+
+      text.each_char do |char|
+        break if candidate.bytesize + char.bytesize > limit
+
+        candidate << char
+        last_break = candidate.length if char.match?(/\s/)
+      end
+
+      return candidate if last_break.nil?
+
+      candidate[0, last_break]
+    end
+  end
+end

data/lib/instagram_connect/usage.rb

@@ -0,0 +1,68 @@
+require "json"
+
+module InstagramConnect
+  # Meta's own account of how much of the rate budget a call just consumed,
+  # parsed from the response headers.
+  #
+  # This is worth having because the documented formula — 4800 calls per 24
+  # hours multiplied by the account's impressions — depends on a number the
+  # messaging API never returns. Any budget computed locally is a guess. These
+  # headers are Meta telling you the answer directly, so they are treated as
+  # ground truth and the arithmetic only as a fallback.
+  class Usage
+    HEADERS = %w[x-business-use-case-usage x-app-usage].freeze
+
+    # Back off well before the wall: at 100% Meta stops answering, and the
+    # penalty is measured in hours.
+    WARN_AT = 80
+    CRITICAL_AT = 95
+
+    attr_reader :call_count, :total_cputime, :total_time, :estimated_time_to_regain_access
+
+    def initialize(call_count: 0, total_cputime: 0, total_time: 0, estimated_time_to_regain_access: nil)
+      @call_count = call_count.to_i
+      @total_cputime = total_cputime.to_i
+      @total_time = total_time.to_i
+      @estimated_time_to_regain_access = estimated_time_to_regain_access
+    end
+
+    # Returns nil when Meta sent no usage headers, which is normal for some
+    # endpoints — absence is not zero usage and must not be read as headroom.
+    def self.from_headers(headers)
+      raw = HEADERS.filter_map { |name| headers&.[](name) }.first
+      return nil if raw.blank?
+
+      payload = JSON.parse(raw)
+      # The business header is keyed by object id and holds an array; the app
+      # header is a flat hash.
+      metrics = payload.is_a?(Hash) && payload.values.first.is_a?(Array) ? payload.values.first.first : payload
+      return nil unless metrics.is_a?(Hash)
+
+      new(
+        call_count: metrics["call_count"],
+        total_cputime: metrics["total_cputime"],
+        total_time: metrics["total_time"],
+        estimated_time_to_regain_access: metrics["estimated_time_to_regain_access"]
+      )
+    rescue JSON::ParserError
+      nil
+    end
+
+    # The worst of the three, since exhausting any one of them throttles the app.
+    def percentage
+      [ call_count, total_cputime, total_time ].max
+    end
+
+    def warning?
+      percentage >= WARN_AT
+    end
+
+    def critical?
+      percentage >= CRITICAL_AT
+    end
+
+    def retry_after
+      estimated_time_to_regain_access.to_i * 60 if estimated_time_to_regain_access.to_i.positive?
+    end
+  end
+end

data/lib/instagram_connect/version.rb

@@ -1,3 +1,3 @@
 module InstagramConnect
-  VERSION = "0.2.1"
+  VERSION = "0.3.0"
 end