api_models 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: dc2b42f380187648ed1eb18f3b2faa1ef2719e09e5e36245abb3441bfe90c215
+  data.tar.gz: 7db4b2aa8b21342d5fee241631d579033097db73a78e635b71d2d33091756ce2
+SHA512:
+  metadata.gz: 9eff6f1c11c250fc24d2799c75975d7f4eab183cf17fbb5da3b162239e14cf144b1dd27437bfcb4f4594886b593d5a8f14928f411cf1659fd0b36110c19ee4ac
+  data.tar.gz: edc46b089835c626227297aa2147c3383ed5a22efe210507e737113cd21568c8a48c13b2eda95e527abf55686ab8ff6a7604cc6b5eaaddffec61324dd43e27a4

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 App47
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,2 @@
+# api-models
+API Models to be shared by the API Lambda and API Sinatra apps

data/lib/api_models/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module ApiModel
+  VERSION = '0.1.0'
+end

data/lib/api_models.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require 'api_models/version'
+
+# Concerns
+require 'models/concerns/app47_app_analyzable.rb'
+require 'models/concerns/app47_app_buildable.rb'
+require 'models/concerns/app47_app_configurable.rb'
+require 'models/concerns/app47_app_policies.rb'
+require 'models/concerns/app47_cdn_url.rb'
+require 'models/concerns/app47_email_sendable.rb'
+require 'models/concerns/app47_logger.rb'
+require 'models/concerns/checkin_able.rb'
+require 'models/concerns/searchable.rb'
+
+require 'models/account.rb'
+require 'models/agent.rb'
+require 'models/app.rb'
+require 'models/app47_cache.rb'
+require 'models/app_active_build.rb'
+require 'models/app_policy.rb'
+require 'models/app_shield_policy.rb'
+require 'models/authorized_user_policy.rb'
+require 'models/b2b_app.rb'
+require 'models/product_app.rb'
+require 'models/commerce_app_store.rb'
+require 'models/configuration_constraint.rb'
+require 'models/configuration_group.rb'
+require 'models/configuration_rule.rb'
+require 'models/device.rb'
+require 'models/external_app.rb'
+require 'models/group.rb'
+require 'models/internal_app.rb'
+require 'models/metric_data_job.rb'
+require 'models/option.rb'
+require 'models/pin_code_policy.rb'
+require 'models/platform.rb'
+require 'models/platform_model.rb'
+require 'models/queue_manager.rb'
+require 'models/redis_configuration.rb'
+require 'models/sso_directory_group.rb'
+require 'models/user.rb'
+require 'models/sso_server.rb'
+require 'models/sso_directory_server.rb'
+require 'models/sso_user.rb'
+require 'models/sso_ad_group.rb'
+require 'models/sso_ad_server.rb'
+require 'models/sso_ad_user.rb'
+require 'models/sso_ldap_group.rb'
+require 'models/sso_ldap_rest_server.rb'
+require 'models/sso_ldap_server.rb'
+require 'models/sso_ldap_user.rb'
+require 'models/sso_oauth_server.rb'
+require 'models/sso_azure_server.rb'
+require 'models/sso_google_server.rb'
+require 'models/sso_saml_v2_server.rb'
+require 'models/system_configuration.rb'
+require 'models/time_bomb_policy.rb'
+require 'models/user_app_permission.rb'
+require 'models/user_device.rb'
+require 'models/version_management_policy.rb'
+require 'models/web_app.rb'
+require 'models/cat/customer.rb'
+require 'models/cat/customer_device.rb'
+require 'models/cl/customer.rb'
+require 'models/cl/customer_device.rb'
+require 'models/gehc/customer.rb'
+require 'models/gehc/customer_device.rb'

data/lib/models/account.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+#
+# Main Account object, should hold references to everything else.
+#
+class Account
+  include Mongoid::Document
+  include Mongoid::Timestamps
+
+  #
+  # Fields
+  #
+  field :name, type: String
+  field :stage, { type: String, default: 'Trial' }
+  field :api_url, type: String
+  #
+  # Relationships
+  #
+  has_many :apps do
+    #
+    # return a list of apps where all the user flag is turned on
+    #
+    def distinct_all_users
+      where(all_users: true)
+    end
+  end
+  has_many :users
+  has_many :groups
+  has_many :sso_servers
+
+  #
+  # Is the API supported for this account?
+  #
+  def api_enabled?
+    !%w[Terminated Rejected].include?(stage)
+  rescue StandardError
+    true
+  end
+end

data/lib/models/agent.rb

@@ -0,0 +1,439 @@
+# frozen_string_literal: true
+
+#
+# Represents an agent, a specific install of an app on a device.
+#
+class Agent
+  include Mongoid::Document
+  include Mongoid::Timestamps
+  include CheckinAble
+  include App47Logger
+  #
+  # Fields
+  #
+  field :agent_version, type: String
+  field :app_version, type: String
+  field :app_version_code, type: String
+  field :app_environment, type: String, default: 'production'
+  field :net_new_counted, type: Boolean, default: false
+  # field :installed_version, type: String
+  #
+  # Relationships
+  #
+  belongs_to :device
+  belongs_to :app, inverse_of: :agents, optional: true
+  #
+  # Call backs
+  #
+  after_create :notify_new_agent
+  before_save :update_app_version
+  after_find :migrate_data
+  #
+  # Delegations
+  #
+  delegate :platform, to: :device
+
+  #
+  # If the agent is capable of reporting the correct version code, basically is the agent version high enough on the
+  # respective platform to have the correct version code to report.
+  #
+  def use_version_code?
+    app.use_version_code?(platform) && (ios? && agent_version.to_f >= 4.4 || android? && agent_version.to_f >= 3.2)
+  rescue StandardError
+    false
+  end
+
+  #
+  # Is this agent an ios agent?
+  #
+  def ios?
+    platform.eql?('iOS')
+  end
+
+  #
+  # Is this agent an Android Agent
+  #
+  def android?
+    platform.eql?('Android')
+  end
+
+  #
+  # return the configuration for this agent
+  #
+  def config_data
+    data = App47Cache.get(config_cache_key)
+    if data.nil?
+      data = build_config_data
+    else
+      data[:server_time_epoch] = Time.now.to_i
+    end
+
+    App47Cache.set(config_cache_key, data, 30)
+    data
+  end
+
+  #
+  # Fetch or create an agent
+  #
+  def self.fetch(app, device, json)
+    agent = Agent.find_or_create_by(device: device, app: app)
+    device.apps << app if agent.new_record? # Add this app to the device if agent is new
+    # new_version = json[:app_version]
+    # agent.notify_new_app_version(new_version) unless new_version.nil? || new_version.eql?(agent.app_version)
+    agent.update_attributes!(json)
+    agent
+  end
+
+  #
+  # Main entry point for inserting metric data
+  #
+  def insert_metric_data(type, data, agent_ip)
+    # If the agent isn't enabled, then drop the data and return a hearty "Thank you!"
+    return true unless app_agent_enabled?
+
+    send "insert_#{type}_data", data.symbolize_keys, agent_ip
+  end
+
+  #
+  # Notify the main worker to update the app and agent configuration information with the new
+  # version
+  #
+  # We also want to force the config to regenerate for the agent as the version will
+  # yield different results.
+  #
+  def notify_new_app_version(new_version)
+    # App47Cache.delete config_cache_key
+    # App47Cache.delete policies_cache_key
+    # QueueManager.push_worker job: 'app_upgrade',
+    #                          app_id: app_id.to_s,
+    #                          agent_id: id.to_s,
+    #                          current_version: app_version,
+    #                          new_version: new_version
+  end
+
+  #
+  # Safely update attributes from the document
+  #
+  def update_attributes!(attributes = {})
+    super(filter_json(attributes))
+  end
+
+  #
+  # Safely update attributes from the document
+  #
+  def update!(attributes = {})
+    super(filter_json(attributes))
+  end
+
+  #
+  # Find the user by the device id associated with the agent's device
+  #
+  def find_user
+    account = app.account
+    device_identifier = device.unique_identifier
+    device = UserDevice.find_device_by device_identifier
+    user = device.user
+    raise 'Invalid account' unless account.eql?(user.account)
+
+    device.user
+  rescue StandardError
+    nil
+  end
+
+  #
+  # filter only the json parameters we want to take into the device. The agent
+  # sends up more than we need for the device.
+  #
+  def filter_json(json)
+    filtered_json = {}
+    %i[agent_version app_version app_version_code app_environment].each do |field|
+      filtered_json[field] = json[field] if json[field].present?
+    end
+    filtered_json
+  end
+
+  #
+  # Encrypt the data unique to this agent.
+  #
+  def encrypt_data(data)
+    cipher = OpenSSL::Cipher::AES256.new(:CBC)
+    cipher.encrypt
+    user_token = device.unique_identifier
+    user_token += user_token while user_token.length < 48
+    cipher.key = user_token[0..31]
+    cipher.iv = user_token[32..47]
+    Base64.encode64(cipher.update(data) + cipher.final)
+  end
+
+  private
+
+  #
+  # Remove analysis_states and modified_at attributes that are no longer used
+  #
+  def migrate_data
+    remove_attribute :analysis_states
+    remove_attribute :modified_at
+  end
+
+  #
+  # Basic information relavant to any metric
+  #
+  def base_metric_data(data, agent_ip, session_id = nil)
+    if data[:start_time_epoch].nil?
+      r_start_time = Time.parse(data[:start_time]).gmtime
+      record_raw_time = data[:start_time].to_s.encode('UTF-8')
+      time_zone = record_raw_time.split(' ').last
+    else
+      r_start_time = Time.at(data[:start_time_epoch].to_i).utc
+      time_zone = nil
+      record_raw_time = nil
+    end
+
+    { agent_id: id.to_s,
+      start_time: r_start_time,
+      time_zone: time_zone,
+      app_id: app_id.to_s,
+      record_date: r_start_time.strftime('%D'),
+      session_id: data[:session_id] || session_id,
+      userinfo: data[:userinfo],
+      tags: data[:tags],
+      ip_address: agent_ip,
+      device_id: device_id.to_s,
+      raw_agent_time_str: record_raw_time,
+      raw_agent_epoch_str: data[:start_time_epoch],
+      loc: {
+        lat: (data[:location_lat].nil? ? 0 : data[:location_lat].to_f),
+        long: (data[:location_long].nil? ? 0 : data[:location_long].to_f)
+      } }
+  end
+
+  #
+  # insert log data received from the agent
+  #
+  def insert_logs_data(data, agent_ip, session_id = nil)
+    return if data[:logs].blank?
+
+    data[:logs].each { |log_data| insert_log_data(log_data.symbolize_keys, agent_ip, session_id) }
+  end
+
+  #
+  # insert a single log entry
+  #
+  def insert_log_data(data, agent_ip, session_id = nil)
+    log_level = data[:level].downcase
+    QueueManager.push_log_event base_metric_data(data, agent_ip, session_id).merge(
+      log_id: data[:uuid],
+      level: log_level,
+      crash_log_format: data[:crash_log_format],
+      is_error_or_crash: %w[error crash].include?(log_level),
+      message: safely_encode(data, :message),
+      filename: safely_encode(data, :filename),
+      line_number: data[:line_number],
+      error_name: safely_encode(data, :error_name),
+      error_reason: safely_encode(data, :error_reason),
+      error_detail: safely_encode(data, :error_detail)
+    )
+  end
+
+  #
+  # Insert multiple timed events
+  #
+  def insert_timed_events_data(data, agent_ip, session_id = nil)
+    return if data[:timedevents].blank?
+
+    data[:timedevents].each { |event_data| insert_timed_event_data(event_data.symbolize_keys, agent_ip, session_id) }
+  end
+
+  #
+  # Insert a single timed event
+  #
+  def insert_timed_event_data(data, agent_ip, session_id = nil)
+    duration = data[:duration].to_f
+    return if duration <= 0
+
+    QueueManager.push_timed_event base_metric_data(data, agent_ip, session_id).merge(
+      timed_event_id: data[:uuid],
+      duration: duration,
+      name: data[:name],
+      created_at_hf: Time.now.gmtime.strftime('%Y-%m-%d %H:%M:%S.%L')
+    )
+  end
+
+  #
+  # Insert multiple generic events
+  #
+  def insert_generic_events_data(data, agent_ip, session_id = nil)
+    return if data[:events].blank?
+
+    data[:events].each { |event_data| insert_generic_event_data(event_data.symbolize_keys, agent_ip, session_id) }
+  end
+
+  #
+  # Insert a single generic event
+  #
+  def insert_generic_event_data(data, agent_ip, session_id = nil)
+    QueueManager.push_generic_event base_metric_data(data, agent_ip, session_id).merge(event_id: data[:uuid],
+                                                                                       name: data[:name])
+  end
+
+  #
+  # Insert a single session event
+  #
+  # Returns the session id for use elsewhere
+  #
+  def insert_session_data(data, agent_ip)
+    duration = data[:duration].to_f
+    return if duration <= 0
+
+    QueueManager.push_session base_metric_data(data, agent_ip).merge(session_id: data[:uuid], duration: duration)
+    data[:uuid]
+  end
+
+  #
+  # Insert bulk data which contacts a session, logs, timed events and generic events
+  #
+  def insert_bulk_data(data, agent_ip)
+    session_id = insert_session_data(data[:session].symbolize_keys, agent_ip)
+    insert_logs_data(data, agent_ip, session_id)
+    insert_generic_events_data(data, agent_ip, session_id)
+    insert_timed_events_data(data, agent_ip, session_id)
+    metrics = data[:app47logs]
+    metrics.each { |app47log| log_debug "AgentLog47: #{id}: #{app47log.inspect}" } if metrics.present?
+  end
+
+  #
+  # Build the config data from scratch
+  #
+  def build_config_data
+    data = default_config_data
+    return data if app.blank?
+
+    if app.api_enabled?
+      data[:agent_enabled] = true
+      data[:configuration_groups] = app.configuration_groups.match_agent(self)
+      data[:message] = 'App found and agent enabled'
+      if app.agent_capture_sessions?
+        data[:capture_sessions] = true
+        data[:capture_generic_events] = app.agent_capture_generic_events?
+        data[:capture_timed_events] = app.agent_capture_timed_events?
+        data[:agent_log_level] = app.agent_log_level
+        data[:session_threshold_seconds] = app.agent_session_threshold_seconds
+      end
+    end
+    data[:message] = 'App found and agent disabled'
+    data[:upload_on_exit] = app.agent_upload_on_exit
+    data[:capture_crash_format] = app.agent_capture_crash_format
+    data[:delay_data_upload_interval] = app.agent_delay_data_upload_interval
+    data[:configuration_update_frequency] = app.agent_configuration_update_frequency
+    data
+  end
+
+  #
+  # Build the default config data block
+  #
+  def default_config_data
+    { agent_id: id.to_s,
+      agent_log_level: 'None',
+      session_data_endpoint: api_url,
+      realtime_data_endpoint: api_url,
+      upload_on_exit: false,
+      show_network_activity: true,
+      capture_crash_format: 'none',
+      capture_crashes: false,
+      capture_generic_events: false,
+      agent_enabled: false,
+      session_threshold_seconds: 5,
+      message: 'Unknown app id, disabling agent',
+      capture_timed_events: false,
+      capture_sessions: false,
+      delay_data_upload_interval: 60,
+      pin_code_required: 0,
+      configuration_update_frequency: 7,
+      configuration_groups: [],
+      server_time_epoch: Time.now.to_i }.merge(policies_for_agent)
+  end
+
+  #
+  # Return either the agent's account api url or the system configuration one by default
+  #
+  def api_url
+    app.account.api_url.blank? ? SystemConfiguration.api_url : app.account.api_url
+  rescue StandardError
+    SystemConfiguration.api_url
+  end
+
+  #
+  # Key used to store the agent's config
+  #
+  def policies_cache_key
+    @policies_cache_key ||= "agent_policies::#{id}"
+  end
+
+  #
+  # Key used to store the agent's config
+  #
+  def config_cache_key
+    @config_cache_key ||= "config_cache::#{id}"
+  end
+
+  #
+  # Tell the net new worker to update analytics
+  #
+  def notify_new_agent
+    QueueManager.push_worker job: 'new_agent', app_id: app_id.to_s, agent_id: id.to_s
+  end
+
+  #
+  # Safely encode the given string, capturing any error.
+  #
+  def safely_encode(data, name)
+    data[name].force_encoding('ISO-8859-1').encode('UTF-8', invalid: :replace, undef: :replace, replace: '')
+  rescue StandardError
+    nil
+  end
+
+  #
+  # Safely test if the app is enabled, this allows the app to be missing
+  #
+  def app_agent_enabled?
+    app.agent_enabled?
+  rescue StandardError
+    false
+  end
+
+  #
+  # Clear the cache
+  #
+  def update_app_version
+    return unless changed.include?('app_version')
+
+    (from, to) = changes[:app_version]
+    return if from.eql?(to)
+
+    App47Cache.delete config_cache_key
+    App47Cache.delete policies_cache_key
+    QueueManager.push_worker job: 'app_upgrade',
+                             app_id: app_id.to_s,
+                             agent_id: id.to_s,
+                             current_version: from,
+                             new_version: to
+    true # Force a return of true so that we don't break the callback chain.
+  end
+
+  #
+  # Return a hash of security policies for this agent
+  #
+  def policies_for_agent
+    defaults = { device_enabled: 1, enforce_active_version: false }
+    current = App47Cache.get(policies_cache_key)
+    return current if current.present?
+
+    current = app.policies.inject(defaults) { |acc, elem| acc.merge(elem.enforce_policy(self)) }
+    App47Cache.set(policies_cache_key, current, 30)
+  rescue StandardError
+    # Clear it out of cache just in case that's the issue
+    App47Cache.delete(policies_cache_key)
+    # Return the default set of policies
+    { device_enabled: 1, enforce_active_version: false }
+  end
+end

data/lib/models/app.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+#
+# Base class of app
+#
+class App
+  include Mongoid::Document
+  include Mongoid::Timestamps
+  #
+  # Fields
+  #
+  field :name, type: String
+  field :all_users, type: Boolean, default: false
+  #
+  # Relationships
+  #
+  belongs_to :account
+end

data/lib/models/app47_cache.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require 'singleton'
+
+#
+# Public: Provide a common interface to redis cache
+#
+# Examples
+#
+#   RedisCacheService.delete("key name")
+#   # => the value of the key deleted
+#
+#   RedisCacheService.get("key name")
+#   # => the value of the key
+#
+class App47Cache
+  include Singleton
+
+  class << self
+    #
+    # Return the redis connection
+    #
+    def redis
+      @redis ||= Redis.new(RedisConfiguration.load(12))
+    end
+
+    #
+    # Delete a key from cache
+    #
+    # key - the key to delete
+    #
+    def delete(key)
+      redis.del key
+    end
+
+    #
+    # Get a key from cache
+    #
+    # key - the key to fetch, if key is not found, returns nil
+    #
+    def get(key)
+      return nil if ENV['RACK_ENV'].eql?('staging')
+
+      value = redis.get key
+      value.nil? ? nil : Marshal.restore(value)
+    rescue StandardError
+      nil
+    end
+
+    #
+    # Set a key with a value and TTL
+    #
+    # key - the key to delete
+    # value - the value to set
+    # ttl(optional) - optional time to live parameter, default is no ttl, live forever
+    #
+    # Returns the value passed in
+    #
+    def set(key, value, ttl = nil)
+      in_value = Marshal.dump value
+      if ttl
+        redis.setex key, ttl, in_value
+      else
+        redis.set key, in_value
+      end
+      value
+    end
+  end
+end

data/lib/models/app_active_build.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+#
+# Holds the active build for a given app
+#
+class AppActiveBuild
+  include Mongoid::Document
+  include Mongoid::Timestamps
+  include App47CdnUrl
+  #
+  # Fields
+  #
+  field :version
+  field :version_code
+  field :file_url
+  field :environment
+  field :platform
+  field :enforce, type: Boolean, default: false
+  #
+  # Relationships
+  #
+  belongs_to :app, inverse_of: :app_active_builds
+
+  #
+  # Return the selected tracking version
+  #
+  def display_version
+    send(app.tracked_version(platform)) || version
+  rescue StandardError
+    version
+  end
+end