synthient 1.0.0

data/lib/synthient/model.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+module Synthient
+  # Decoding is strict: a field the API schema requires must be present,
+  # non-null, and of its declared type. Optional fields may be absent or null
+  # and become nil. Unknown fields are ignored so that additive API changes
+  # don't break decoding.
+  #
+  # @api private
+  module Model
+    Field = Data.define(:name, :wire, :type, :required)
+
+    class Builder
+      attr_reader :fields
+
+      def initialize
+        @fields = []
+      end
+
+      def required(name, type, wire: name.to_s)
+        @fields << Field.new(name:, wire:, type:, required: true)
+      end
+
+      def optional(name, type, wire: name.to_s)
+        @fields << Field.new(name:, wire:, type:, required: false)
+      end
+    end
+
+    def self.define(&)
+      builder = Builder.new
+      builder.instance_eval(&)
+      fields = builder.fields.freeze
+      defaults = fields.reject(&:required).to_h { |field| [field.name, nil] }.freeze
+      Data.define(*fields.map(&:name)) do
+        define_singleton_method(:fields) { fields }
+        define_method(:initialize) { |**attrs| super(**defaults, **attrs) }
+      end
+    end
+
+    def self.decode(model, object, path = model.name.delete_prefix("Synthient::"))
+      raise DecodeError, "#{path} must be an object" unless object.is_a?(Hash)
+
+      attrs = model.fields.to_h do |field|
+        value = object[field.wire]
+        field_path = "#{path}.#{field.wire}"
+        raise DecodeError, "#{field_path} is required" if value.nil? && field.required
+
+        [field.name, value.nil? ? nil : decode_value(field.type, value, field_path)]
+      end
+      model.new(**attrs)
+    end
+
+    def self.decode_value(type, value, path)
+      case type
+      when :integer
+        return value if value.is_a?(Integer)
+      when :float
+        return value.to_f if (value.is_a?(Integer) || value.is_a?(Float)) && value.to_f.finite?
+      when :string
+        return value if value.is_a?(String)
+      when :bool
+        return value if [true, false].include?(value)
+      when :unix_time
+        if value.is_a?(Integer)
+          # The spec says seconds, but the Helios docs show milliseconds, so
+          # values from 1e12 up are read as milliseconds.
+          return Time.at(value, in: "UTC") if value < 1_000_000_000_000
+
+          return Time.at(value / 1000, value % 1000, :millisecond, in: "UTC")
+        end
+      when :time
+        return Time.iso8601(value) if value.is_a?(String)
+      when :date
+        return Date.iso8601(value) if value.is_a?(String) && value.match?(/\A\d{4}-\d{2}-\d{2}\z/)
+      when :bytes
+        return value.unpack1("m0").freeze if value.is_a?(String)
+      when :ip
+        return IPAddr.new(value) if value.is_a?(String) && !value.include?("/") && !value.include?("%")
+      when Array
+        if value.is_a?(Array)
+          return value.each_with_index.map { |item, index| decode_value(type.first, item, "#{path}[#{index}]") }.freeze
+        end
+      when Hash
+        return value.transform_values { |item| decode_value(type.values.first, item, path) }.freeze if value.is_a?(Hash)
+      else
+        return decode(type, value, path)
+      end
+      raise DecodeError, "#{path} is not a valid #{type.is_a?(Symbol) ? type.to_s.tr("_", " ") : type.class.name.downcase}"
+    rescue ArgumentError
+      raise DecodeError, "#{path} is not a valid #{type.to_s.tr("_", " ")}"
+    end
+  end
+end

data/lib/synthient/models/account.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Synthient
+  # A member's role in an organization.
+  module Relation
+    OWNER = "OWNER"
+    ADMIN = "ADMIN"
+    MEMBER = "MEMBER"
+  end
+
+  # API key scopes. New values may be added.
+  module Scope
+    BASIC = "BASIC"
+    PROXY_FEEDS = "PROXY_FEEDS"
+    PROXY_FIREHOSE = "PROXY_FIREHOSE"
+    ANONYMIZERS_FEED = "ANONYMIZERS_FEED"
+    ANONYMIZERS_STREAM = "ANONYMIZERS_STREAM"
+    JA3_FEED = "JA3_FEED"
+    JA3_STREAM = "JA3_STREAM"
+    TORRENTS_FEED = "TORRENTS_FEED"
+    TORRENTS_STREAM = "TORRENTS_STREAM"
+    HONEYPOT_HTTP_FEED = "HONEYPOT_HTTP_FEED"
+    HONEYPOT_HTTP_STREAM = "HONEYPOT_HTTP_STREAM"
+    HONEYPOT_HTTPS_FEED = "HONEYPOT_HTTPS_FEED"
+    HONEYPOT_HTTPS_STREAM = "HONEYPOT_HTTPS_STREAM"
+    HONEYPOT_ADB_FEED = "HONEYPOT_ADB_FEED"
+    HONEYPOT_ADB_STREAM = "HONEYPOT_ADB_STREAM"
+    HONEYPOT_DNS_FEED = "HONEYPOT_DNS_FEED"
+    HONEYPOT_DNS_STREAM = "HONEYPOT_DNS_STREAM"
+    JA4_FEED = "JA4_FEED"
+    JA4_STREAM = "JA4_STREAM"
+    JA4T_FEED = "JA4T_FEED"
+    JA4T_STREAM = "JA4T_STREAM"
+    ADMIN_PROXIES = "ADMIN_PROXIES"
+  end
+
+  # +relation+ is the account's role, one of Relation.
+  Organization = Model.define do
+    required :id, :string
+    required :name, :string
+    required :relation, :string
+  end
+
+  # +resets_in+ is the number of seconds until the credits reset.
+  LookupQuota = Model.define do
+    required :credits, :integer
+    required :resets_in, :integer
+  end
+
+  # The API key that made the request.
+  AccountAPIKey = Model.define do
+    required :id, :string
+    required :name, :string
+    required :created_at, :time
+  end
+
+  # The account that owns the API key. +scopes+ holds Scope values.
+  Account = Model.define do
+    required :email, :string
+    required :first_name, :string
+    required :last_name, :string
+    required :organization, Organization
+    required :scopes, [:string]
+    required :lookup_quota, LookupQuota
+    optional :api_key, AccountAPIKey
+  end
+end

data/lib/synthient/models/domain.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+module Synthient
+  # Whether a domain had events in the last 24 hours.
+  module DomainStatus
+    ACTIVE = "active"
+    INACTIVE = "inactive"
+  end
+
+  # The sentences that describe a domain's activity over the last 30 days.
+  module Cadence
+    NO_ACTIVITY = "No activity in the last 30 days."
+    EVENLY_DISTRIBUTED = "Activity is evenly distributed across the day."
+    BUSINESS_HOURS = "Activity follows a weekday business-hours pattern."
+    WEEKEND = "Activity follows a weekend pattern."
+    BURST_DRIVEN = "Activity is burst-driven with significant peaks."
+  end
+
+  DomainStats = Model.define do
+    required :events_24h, :integer
+    required :total_events_30d, :integer
+  end
+
+  # Distinct source addresses seen for a domain. +sparkline_24h+ holds hourly
+  # counts for the last 24 hours, oldest first.
+  UniqueIPs = Model.define do
+    required :value_24h, :integer
+    required :value_30d, :integer
+    required :sparkline_24h, [:integer]
+  end
+
+  # The events and distinct addresses seen for a domain on one day.
+  TimeSeriesPoint = Model.define do
+    required :date, :unix_time
+    required :events, :integer
+    required :unique_ips, :integer
+  end
+
+  # The autonomous system that sent the most events.
+  ASNCount = Model.define do
+    required :asn, :integer
+    required :events, :integer
+  end
+
+  SubdomainCount = Model.define do
+    required :subdomain, :string
+    required :count, :integer
+  end
+
+  PortCount = Model.define do
+    required :port, :integer
+    required :count, :integer
+  end
+
+  # The events and distinct addresses seen from one country.
+  CountryCount = Model.define do
+    required :country_code, :string
+    required :events, :integer
+    required :unique_ips, :integer
+  end
+
+  # +peak_hour+ and +quiet_hour+ are UTC hours, and +cadence+ is one of
+  # Cadence.
+  ActivityStats = Model.define do
+    required :cadence, :string
+    required :median_per_hour, :integer
+    required :p95_per_hour, :integer
+    required :peak_hour, :integer
+    required :quiet_hour, :integer
+  end
+
+  # +source_ip+ is the full source address, although the API names it
+  # source_ip_masked.
+  DomainEvent = Model.define do
+    required :timestamp, :unix_time
+    required :source_ip, :string, wire: "source_ip_masked"
+    required :target_subdomain, :string
+    required :country_code, :string
+    required :port, :integer
+  end
+
+  # The honeypot traffic Synthient has observed for a domain. A domain that
+  # was never observed is +inactive+ with zero counts. +top_asn+ is nil when
+  # there were no events. +hour_dow_heatmap+ has 7 rows, Monday first, of 24
+  # UTC hourly counts.
+  DomainInfo = Model.define do
+    required :domain, :string
+    required :status, :string
+    required :stats, DomainStats
+    required :unique_ips, UniqueIPs
+    required :time_series, [TimeSeriesPoint]
+    optional :top_asn, ASNCount
+    required :top_subdomains, [SubdomainCount]
+    required :top_ports, [PortCount]
+    required :geo_distribution, [CountryCount]
+    required :hour_dow_heatmap, [[:integer]]
+    required :activity_stats, ActivityStats
+    required :recent_events, [DomainEvent]
+  end
+end

data/lib/synthient/models/events.rb

@@ -0,0 +1,202 @@
+# frozen_string_literal: true
+
+module Synthient
+  # The kinds of service seen on an address. New values may be added.
+  module ProxyType
+    FREE_VPN = "FREE_VPN"
+    COMMERCIAL_VPN = "COMMERCIAL_VPN"
+    ENTERPRISE_VPN = "ENTERPRISE_VPN"
+    MOBILE_PROXY = "MOBILE_PROXY"
+    BLOCKCHAIN_PROXY = "BLOCKCHAIN_PROXY"
+    RESIDENTIAL_PROXY = "RESIDENTIAL_PROXY"
+    PUBLIC_PROXY = "PUBLIC_PROXY"
+    DATACENTER_PROXY = "DATACENTER_PROXY"
+    TOR_NODE = "TOR_NODE"
+    PRIVATE_RELAY = "PRIVATE_RELAY"
+    BOTNET = "BOTNET"
+    SEARCH_ENGINE = "SEARCH_ENGINE"
+    AI_CRAWLER = "AI_CRAWLER"
+    SOCIAL_MEDIA = "SOCIAL_MEDIA"
+    UPTIME_MONITOR = "UPTIME_MONITOR"
+    LINK_PREVIEW = "LINK_PREVIEW"
+    SEO_CRAWLER = "SEO_CRAWLER"
+    WEB_ARCHIVER = "WEB_ARCHIVER"
+    WEBHOOK_PROVIDER = "WEBHOOK_PROVIDER"
+    PAYMENT_PROCESSOR = "PAYMENT_PROCESSOR"
+    INTERNET_SCANNER = "INTERNET_SCANNER"
+  end
+
+  # +type+ is one of ProxyType.
+  ProxyEvent = Model.define do
+    optional :ip, :string
+    optional :provider, :string
+    optional :type, :string
+    optional :country_code, :string
+    optional :asn, :integer
+    optional :timestamp, :unix_time
+  end
+
+  # +type+ is one of ProxyType, and +country_code+ and +asn+ describe
+  # +range_start+.
+  AnonymizerEvent = Model.define do
+    optional :range_start, :string
+    optional :range_end, :string
+    optional :provider, :string
+    optional :type, :string
+    optional :country_code, :string
+    optional :asn, :integer
+    optional :timestamp, :unix_time
+  end
+
+  # +length+ is in bytes.
+  TorrentFile = Model.define do
+    optional :path, :string
+    optional :length, :integer
+  end
+
+  # +source+ is where the peer was found, such as "DHT", "PEX", or "tracker".
+  TorrentPeer = Model.define do
+    optional :ip, :string
+    optional :port, :integer
+    optional :source, :string
+    optional :encrypted, :bool
+  end
+
+  # +info_hash+ is the hex SHA-1 info hash, and sizes are in bytes.
+  TorrentEvent = Model.define do
+    optional :info_hash, :string
+    optional :name, :string
+    optional :magnet_uri, :string
+    optional :total_size, :integer
+    optional :piece_length, :integer
+    optional :file_count, :integer
+    optional :files, [TorrentFile]
+    optional :peers, [TorrentPeer]
+    optional :timestamp, :unix_time
+  end
+
+  # A JA4T TCP fingerprint sighting. +type+ is one of ProxyType.
+  JA4TEvent = Model.define do
+    optional :ip, :string
+    optional :ja4t, :string
+    optional :provider, :string
+    optional :type, :string
+    optional :country_code, :string
+    optional :asn, :integer
+    optional :timestamp, :unix_time
+  end
+
+  # The Helios honeypot sensor that captured an event. +proxy_ip+ and
+  # +provider+ are the proxy exit address and service the traffic arrived
+  # through.
+  HeliosMeta = Model.define do
+    optional :proxy_ip, :string
+    optional :server, :string
+    optional :pool_id, :string
+    optional :provider, :string
+  end
+
+  # +http_method+ is the request method. The API calls it +method+, which
+  # would shadow Object#method.
+  HTTPRequestDetails = Model.define do
+    optional :http_method, :string, wire: "method"
+    optional :uri, :string
+    optional :version, :string
+    optional :headers, { string: :string }
+  end
+
+  # +domain+ is the host the proxy tunnel was opened to, and +raw+ holds the
+  # raw request bytes.
+  HeliosHTTPEvent = Model.define do
+    required :timestamp, :unix_time
+    required :tunnel_id, :integer
+    required :domain, :string
+    required :port, :integer
+    required :protocol, :string
+    required :meta, HeliosMeta
+    optional :details, HTTPRequestDetails
+    optional :raw, :bytes
+  end
+
+  # A TLS registry code point and its name. GREASE values are named
+  # "GREASE(0x....)".
+  TLSCodeName = Model.define do
+    optional :code, :integer
+    optional :name, :string
+  end
+
+  # +length+ is the extension body length in bytes.
+  TLSExtension = Model.define do
+    optional :code, :integer
+    optional :name, :string
+    optional :length, :integer
+  end
+
+  # Fields the sensor could not parse are nil.
+  TLSClientHello = Model.define do
+    optional :record_version, :string
+    optional :handshake_version, :string
+    optional :client_random, :string
+    optional :session_id, :string
+    optional :session_id_length, :integer
+    optional :cipher_suites, [TLSCodeName]
+    optional :compression_methods, [:integer]
+    optional :extensions, [TLSExtension]
+    optional :sni, :string
+    optional :alpn, [:string]
+    optional :supported_versions, [:string]
+    optional :supported_groups, [TLSCodeName]
+    optional :signature_algorithms, [TLSCodeName]
+    optional :ec_point_formats, [:string]
+    optional :key_share_groups, [:string]
+    optional :psk_key_exchange_modes, [:string]
+    optional :compress_certificate, [:string]
+    optional :has_grease, :bool
+    optional :application_settings, :bool
+    optional :delegated_credentials, :bool
+    optional :encrypt_then_mac, :bool
+    optional :extended_master_secret, :bool
+    optional :post_handshake_auth, :bool
+    optional :renegotiation_info, :bool
+    optional :signed_certificate_timestamps, :bool
+    optional :status_request, :bool
+  end
+
+  # A TLS ClientHello captured by a Helios honeypot. +details+ is nil when the
+  # sensor could not parse the handshake. +raw+ is always nil, because the
+  # sensors do not capture handshake bytes.
+  HeliosHTTPSEvent = Model.define do
+    required :timestamp, :unix_time
+    required :tunnel_id, :integer
+    required :domain, :string
+    required :port, :integer
+    required :protocol, :string
+    required :meta, HeliosMeta
+    optional :details, TLSClientHello
+    optional :raw, :string
+  end
+
+  # A destination a Helios honeypot tunnel was asked to reach. Despite the
+  # feed's name, it is not a DNS query: names resolved inside a tunneled DNS
+  # session are not visible.
+  HeliosDNSEvent = Model.define do
+    required :timestamp, :unix_time
+    required :tunnel_id, :integer
+    required :domain, :string
+    required :port, :integer
+    required :protocol, :string
+    required :meta, HeliosMeta
+  end
+
+  # An Android Debug Bridge shell command captured by a Helios honeypot. It
+  # has no timestamp because the sensor sends none. +sequential_id+ is the
+  # 1-based position of the command in its session. +payload_hash+ is the hex
+  # SHA-256 the sensor assigns to the session's payload, sent as +hash+: it is
+  # not a hash of +command+, and every command in a session shares it.
+  HeliosADBEvent = Model.define do
+    required :session, :string
+    required :sequential_id, :integer
+    required :command, :string
+    required :payload_hash, :string, wire: "hash"
+  end
+end

data/lib/synthient/models/export.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Synthient
+  module ExportKind
+    HOURLY = "hourly"
+    DAILY = "daily"
+  end
+
+  # New values may be added.
+  module ParquetType
+    STRING = "string"
+    INT64 = "int64"
+    UINT32 = "uint32"
+    UINT64 = "uint64"
+    BOOL = "bool"
+    BYTES = "bytes"
+  end
+
+  # +hour+ is nil for a daily export. +checksum+ is the hex SHA-256 of the
+  # file. +download_path+ includes the /api/v4 prefix.
+  Export = Model.define do
+    required :id, :string
+    required :kind, :string
+    required :date, :date
+    optional :hour, :integer
+    required :size_bytes, :integer
+    required :row_count, :integer
+    required :checksum, :string
+    required :created_at, :unix_time
+    required :download_path, :string
+  end
+
+  # +next_cursor+ is nil on the last page.
+  ExportPage = Model.define do
+    required :stream, :string
+    required :exports, [Export], wire: "feeds"
+    optional :next_cursor, :string
+  end
+
+  # +type+ is one of ParquetType.
+  ParquetColumn = Model.define do
+    required :name, :string
+    required :type, :string
+  end
+
+  ParquetSchema = Model.define do
+    required :fields, [ParquetColumn]
+  end
+
+  # +hour+ is nil for a daily export. +size+ is in bytes and +checksum+ is
+  # the hex SHA-256 of the file.
+  ExportMeta = Model.define do
+    required :id, :string
+    required :stream, :string
+    required :kind, :string
+    required :format, :string
+    required :date, :unix_time
+    optional :hour, :integer
+    required :rows, :integer
+    required :size, :integer
+    required :checksum, :string
+    required :created_at, :unix_time
+    required :schema, ParquetSchema
+  end
+end

data/lib/synthient/models/lookup.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+module Synthient
+  # New values may be added.
+  module NetworkType
+    MOBILE = "MOBILE"
+    SATELLITE = "SATELLITE"
+    IN_FLIGHT_WIFI = "IN_FLIGHT_WIFI"
+    RESIDENTIAL = "RESIDENTIAL"
+    CORPORATE = "CORPORATE"
+    ACADEMIC = "ACADEMIC"
+    DATACENTER = "DATACENTER"
+    GOVERNMENT = "GOVERNMENT"
+  end
+
+  # Device operating systems. New values may be added.
+  module OS
+    ANDROID = "Android"
+    CHROME_OS = "Chrome OS"
+    FREEBSD = "FreeBSD"
+    IOS = "iOS"
+    LINUX = "Linux"
+    MAC_OS_X = "Mac OS X"
+    WINDOWS = "Windows"
+  end
+
+  # The network that announces an address. +asn+ is 0 if the address is not
+  # routed. +type+ is one of NetworkType. +org+, +domain+, +abuse_email+, and
+  # +abuse_phone+ are nil when unknown.
+  Network = Model.define do
+    required :asn, :integer
+    required :isp, :string
+    optional :org, :string
+    optional :domain, :string
+    required :type, :string
+    optional :abuse_email, :string
+    optional :abuse_phone, :string
+  end
+
+  # +country+ is an ISO 3166-1 alpha-2 code and +timezone+ an IANA time zone
+  # name.
+  Location = Model.define do
+    required :country, :string
+    required :state, :string
+    required :city, :string
+    required :timezone, :string
+    required :latitude, :float
+    required :longitude, :float
+    required :geo_hash, :string
+  end
+
+  # A proxy, VPN, or anonymizer service seen using an address. +name+ is the
+  # service tag, such as "BRIGHTDATA", and +type+ its category.
+  Provider = Model.define do
+    required :name, :string, wire: "provider"
+    required :type, :string
+    required :last_seen, :unix_time
+  end
+
+  # A device type seen behind an address. +os+ is one of OS.
+  Device = Model.define do
+    required :os, :string
+    required :version, :string
+    required :last_seen, :unix_time
+  end
+
+  # Synthient's assessment of an address. +risk_score+ runs from 0 to 100.
+  # +categories+ and +behavior+ are open sets of tags. An empty +providers+
+  # list does not mean the address is clean.
+  Intelligence = Model.define do
+    required :risk_score, :integer
+    required :categories, [:string]
+    required :behavior, [:string]
+    required :providers, [Provider]
+    required :devices, [Device]
+  end
+
+  # +network+, +location+, and +intelligence+ are nil when the API has no data
+  # for them.
+  IPInfo = Model.define do
+    required :ip, :ip
+    optional :network, Network
+    optional :location, Location
+    optional :intelligence, Intelligence
+  end
+end

data/lib/synthient/response_metadata.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Synthient
+  # The rate limit state reported in a response's RateLimit-* headers. +reset+
+  # is the number of seconds until the window resets.
+  RateLimit = Data.define(:limit, :remaining, :reset)
+
+  # Describes one API response. A client's +response_hook+ receives one for
+  # every response from the API, including retried attempts, export pages,
+  # stream handshakes, and responses that later fail to decode. Export
+  # downloads from storage are not reported.
+  #
+  # +headers+ is a frozen copy with lowercase names. +rate_limit+ is nil
+  # unless all three RateLimit-* headers are present and valid.
+  ResponseMetadata = Data.define(:status, :request_id, :rate_limit, :headers)
+end