analytics_ops 0.2.0 → 0.3.0

data/lib/analytics_ops/clients/bigquery.rb

@@ -0,0 +1,313 @@
+# frozen_string_literal: true
+
+require "date"
+require "bigdecimal"
+require "time"
+
+module AnalyticsOps
+  module Clients
+    # Optional official BigQuery client boundary. It exposes no arbitrary SQL API.
+    class BigQuery
+      PROJECT_ID = /\A[a-z][a-z0-9-]{4,28}[a-z0-9]\z/
+      DATASET = /\A[A-Za-z_][A-Za-z0-9_]{0,1023}\z/
+      RECIPES = {
+        "event_counts" => lambda do |table|
+          <<~SQL
+            SELECT event_name, COUNT(*) AS event_count
+            FROM `#{table}`
+            WHERE _TABLE_SUFFIX BETWEEN @start_suffix AND @end_suffix
+            GROUP BY event_name
+            ORDER BY event_count DESC
+            LIMIT @row_limit
+          SQL
+        end,
+        "daily_events" => lambda do |table|
+          <<~SQL
+            SELECT event_date, COUNT(*) AS event_count
+            FROM `#{table}`
+            WHERE _TABLE_SUFFIX BETWEEN @start_suffix AND @end_suffix
+            GROUP BY event_date
+            ORDER BY event_date
+            LIMIT @row_limit
+          SQL
+        end,
+        "page_views" => lambda do |table|
+          <<~SQL
+            SELECT
+              REGEXP_REPLACE(
+                (SELECT value.string_value FROM UNNEST(event_params) WHERE key = 'page_location'),
+                r'\\?.*$',
+                ''
+              ) AS page_location,
+              COUNT(*) AS page_views
+            FROM `#{table}`
+            WHERE _TABLE_SUFFIX BETWEEN @start_suffix AND @end_suffix
+              AND event_name = 'page_view'
+            GROUP BY page_location
+            ORDER BY page_views DESC
+            LIMIT @row_limit
+          SQL
+        end,
+        "event_sample" => lambda do |table|
+          <<~SQL
+            SELECT
+              event_date,
+              TIMESTAMP_MICROS(event_timestamp) AS event_time,
+              event_name,
+              platform,
+              device.category AS device_category
+            FROM `#{table}`
+            WHERE _TABLE_SUFFIX BETWEEN @start_suffix AND @end_suffix
+            ORDER BY event_timestamp DESC
+            LIMIT @row_limit
+          SQL
+        end
+      }.freeze
+
+      def initialize(settings:, service_account: nil, client: nil, timeout: nil, logger: nil)
+        @settings = validate_settings(settings)
+        @service_account = service_account
+        @client = client
+        @timeout = timeout
+        @logger = logger
+      end
+
+      def run(definition)
+        unless definition.is_a?(AnalyticsOps::BigQuery::Definition)
+          raise InvalidRequestError, "BigQuery report must be a BigQuery::Definition"
+        end
+
+        sql = RECIPES.fetch(definition.name).call(table_pattern)
+        params = {
+          start_suffix: definition.start_suffix,
+          end_suffix: definition.end_suffix,
+          row_limit: definition.limit
+        }
+        execute_query(definition.name, sql, params, definition.limit)
+      end
+
+      def run_funnel(definition, start_suffix:, end_suffix:)
+        unless definition.is_a?(Funnels::Definition)
+          raise InvalidRequestError, "funnel must be an AnalyticsOps::Funnels::Definition"
+        end
+        unless [start_suffix, end_suffix].all? { |value| value.is_a?(String) && value.match?(/\A\d{8}\z/) } &&
+               start_suffix <= end_suffix
+          raise InvalidRequestError, "Funnel date range is invalid"
+        end
+
+        sql, params = funnel_query(definition, start_suffix:, end_suffix:)
+        execute_query("funnel_#{definition.name}", sql, params, definition.steps.length)
+      end
+
+      private
+
+      def execute_query(name, sql, params, limit)
+        dry_job = query_job(sql, params:, dryrun: true)
+        bytes = job_bytes(dry_job)
+        maximum = @settings.fetch("maximum_bytes_billed")
+        if bytes > maximum
+          raise QuotaError, "BigQuery report would process #{bytes} bytes, above the configured #{maximum}-byte limit"
+        end
+
+        job = query_job(sql, params:, dryrun: false)
+        translate_errors { job.wait_until_done! }
+        raise RemoteError, "BigQuery report failed" if job.respond_to?(:failed?) && job.failed?
+
+        data = translate_errors { job.data }
+        rows = normalize_rows(data, limit)
+        headers = rows.first&.keys || schema_headers(data)
+        AnalyticsOps::BigQuery::Result.new(
+          name:,
+          headers:,
+          rows:,
+          bytes_processed: job_bytes(job, fallback: bytes),
+          maximum_bytes_billed: maximum
+        )
+      end
+
+      def funnel_query(definition, start_suffix:, end_suffix:)
+        params = {
+          start_suffix:,
+          end_suffix:,
+          event_names: definition.steps.map(&:event_name)
+        }
+        ctes = [funnel_base_cte]
+        ctes.concat(funnel_step_ctes(definition, params))
+        [funnel_sql(ctes, funnel_count_queries(definition)), params]
+      end
+
+      def funnel_base_cte
+        <<~SQL.chomp
+          base_events AS (
+            SELECT user_pseudo_id, event_name, event_timestamp
+            FROM `#{table_pattern}`
+            WHERE _TABLE_SUFFIX BETWEEN @start_suffix AND @end_suffix
+              AND user_pseudo_id IS NOT NULL
+              AND event_name IN UNNEST(@event_names)
+          )
+        SQL
+      end
+
+      def funnel_step_ctes(definition, params)
+        definition.steps.each_with_index do |step, index|
+          number = index + 1
+          params[:"step_#{number}_event"] = step.event_name
+          params[:"step_#{number}_name"] = step.name
+        end
+        definition.steps.each_index.map do |index|
+          number = index + 1
+          previous = number - 1
+          join = number == 1 ? "" : "JOIN step_#{previous} previous USING (user_pseudo_id)"
+          time_condition = number == 1 ? "" : "AND event_timestamp > previous.step_#{previous}_at"
+          <<~SQL.chomp
+            step_#{number} AS (
+              SELECT user_pseudo_id, MIN(event_timestamp) AS step_#{number}_at
+              FROM base_events
+              #{join}
+              WHERE event_name = @step_#{number}_event
+                #{time_condition}
+              GROUP BY user_pseudo_id
+            )
+          SQL
+        end
+      end
+
+      def funnel_count_queries(definition)
+        definition.steps.each_index.map do |index|
+          number = index + 1
+          "SELECT #{number} AS step, @step_#{number}_name AS step_name, COUNT(*) AS users FROM step_#{number}"
+        end
+      end
+
+      def funnel_sql(ctes, count_queries)
+        <<~SQL
+          WITH
+          #{ctes.join(",\n")}
+          , counts AS (
+            #{count_queries.join("\nUNION ALL\n")}
+          )
+          SELECT
+            step,
+            step_name,
+            users,
+            SAFE_DIVIDE(users, FIRST_VALUE(users) OVER (ORDER BY step)) AS completion_rate
+          FROM counts
+          ORDER BY step
+        SQL
+      end
+
+      def client
+        @client ||= begin
+          unless @service_account
+            raise AuthenticationError,
+                  "Analytics Ops requires configured service-account credentials"
+          end
+
+          require "google/cloud/bigquery"
+          Google::Cloud::Bigquery.new(
+            project_id: @settings.fetch("project_id"),
+            credentials: @service_account.__send__(:credentials, access: :bigquery),
+            timeout: @timeout
+          )
+        rescue LoadError
+          raise UnsupportedCapabilityError,
+                "BigQuery support is optional; add `gem \"google-cloud-bigquery\", \"~> 1.64\"` and run bundle install"
+        end
+      end
+
+      def query_job(sql, params:, dryrun:)
+        SafeLogging.write(
+          @logger,
+          :info,
+          "google_bigquery_request",
+          "dry_run" => dryrun,
+          "project" => @settings.fetch("project_id")
+        )
+        options = {
+          params:,
+          dryrun:,
+          standard_sql: true,
+          maximum_bytes_billed: @settings.fetch("maximum_bytes_billed")
+        }
+        translate_errors do
+          client.query_job(sql, **options) do |job|
+            job.location = @settings.fetch("location") if job.respond_to?(:location=)
+          end
+        end
+      end
+
+      def table_pattern
+        "#{@settings.fetch("project_id")}.#{@settings.fetch("dataset")}.events_*"
+      end
+
+      def validate_settings(value)
+        unless value.is_a?(Hash) && value["experimental"] == true &&
+               PROJECT_ID.match?(value["project_id"].to_s) &&
+               DATASET.match?(value["dataset"].to_s) &&
+               value["maximum_bytes_billed"].is_a?(Integer) &&
+               value["maximum_bytes_billed"].positive?
+          raise ConfigurationError, "BigQuery configuration is invalid"
+        end
+
+        Canonical.immutable(value)
+      end
+
+      def job_bytes(job, fallback: 0)
+        value = job.respond_to?(:bytes_processed) ? job.bytes_processed : nil
+        value ||= fallback
+        return value if value.is_a?(Integer) && !value.negative?
+
+        raise RemoteError, "BigQuery returned an invalid processed-byte count"
+      end
+
+      def normalize_rows(data, limit)
+        rows = []
+        data.each do |row|
+          raise RemoteError, "BigQuery returned too many rows" if rows.length >= limit
+          raise RemoteError, "BigQuery returned an invalid row" unless row.respond_to?(:to_h)
+
+          rows << row.to_h.to_h { |key, value| [safe_header(key), safe_value(value)] }
+        end
+        rows
+      end
+
+      def schema_headers(data)
+        return [] unless data.respond_to?(:schema) && data.schema.respond_to?(:fields)
+
+        data.schema.fields.map { |field| safe_header(field.name) }
+      end
+
+      def safe_header(value)
+        string = value.to_s
+        return string if string.match?(/\A[A-Za-z_][A-Za-z0-9_]{0,127}\z/)
+
+        raise RemoteError, "BigQuery returned an invalid field name"
+      end
+
+      def safe_value(value)
+        case value
+        when String
+          raise RemoteError, "BigQuery returned invalid UTF-8" unless value.valid_encoding?
+
+          value
+        when Integer, TrueClass, FalseClass, NilClass
+          value
+        when Float
+          raise RemoteError, "BigQuery returned a non-finite value" unless value.finite?
+
+          value
+        when Date, DateTime, Time
+          value.iso8601
+        else
+          return value.to_s("F") if defined?(BigDecimal) && value.is_a?(BigDecimal)
+
+          raise RemoteError, "BigQuery returned an unsupported value"
+        end
+      end
+
+      def translate_errors(&)
+        ErrorTranslation.call(&)
+      end
+    end
+  end
+end

data/lib/analytics_ops/clients/data.rb

@@ -1,9 +1,13 @@
 # frozen_string_literal: true
 
+require_relative "data_result_normalization"
+
 module AnalyticsOps
   module Clients
     # Narrow adapter around Google's generated Data API client.
     class Data
+      include DataResultNormalization
+
       PACKAGE_REQUIREMENT = Gem::Requirement.new("~> 0.9.0")
       MATCH_TYPES = {
         "exact" => :EXACT,
@@ -72,10 +76,51 @@ module AnalyticsOps
         definitions.zip(reports).map { |definition, report| normalize_result(definition, report) }.freeze
       end
 
+      def metadata(property_id)
+        property = property_name(property_id)
+        @metadata_cache ||= {}
+        @metadata_cache[property] ||= begin
+          response = invoke(:get_metadata, name: "#{property}/metadata")
+          Reports::Metadata.new(
+            property_id:,
+            dimensions: array_field(response, :dimensions).map { |value| normalize_field(value, "dimension") },
+            metrics: array_field(response, :metrics).map { |value| normalize_field(value, "metric") }
+          )
+        end
+      end
+
+      def check_compatibility(property_id, definition)
+        unless definition.is_a?(Reports::Definition) && !definition.realtime?
+          raise InvalidRequestError, "compatibility requires a standard report definition"
+        end
+
+        response = invoke(
+          :check_compatibility,
+          property: property_name(property_id),
+          dimensions: definition.dimensions.map { |name| { name: } },
+          metrics: definition.metrics.map { |name| { name: } },
+          dimension_filter: filter_request(definition.dimension_filter),
+          metric_filter: filter_request(definition.metric_filter)
+        )
+        dimensions = compatibility_entries(response, :dimension_compatibilities)
+        metrics = compatibility_entries(response, :metric_compatibilities)
+        unless compatibility_names(dimensions) == definition.dimensions.sort &&
+               compatibility_names(metrics) == definition.metrics.sort
+          raise RemoteError, "Google Data API returned compatibility fields that do not match the request"
+        end
+
+        Reports::Compatibility.new(
+          report_name: definition.name,
+          dimensions:,
+          metrics:
+        )
+      end
+
       def available?
         generated_client = translate_errors { client }
         generated_client.respond_to?(:run_report) && generated_client.respond_to?(:run_realtime_report) &&
-          generated_client.respond_to?(:batch_run_reports)
+          generated_client.respond_to?(:batch_run_reports) && generated_client.respond_to?(:get_metadata) &&
+          generated_client.respond_to?(:check_compatibility)
       end
 
       def compatibility
@@ -132,7 +177,7 @@ module AnalyticsOps
           dimensions: definition.dimensions.map { |name| { name: } },
           metrics: definition.metrics.map { |name| { name: } },
           dimension_filter: filter_request(definition.dimension_filter),
-          metric_filter: metric_filter_request(definition.metric_filter),
+          metric_filter: filter_request(definition.metric_filter),
           order_bys: definition.order_bys.map { |order| order_request(order) },
           limit: definition.limit,
           return_property_quota: true
@@ -142,38 +187,62 @@ module AnalyticsOps
       def filter_request(filter)
         return nil unless filter
 
-        {
-          filter: {
-            field_name: filter.fetch("field"),
-            string_filter: {
-              match_type: MATCH_TYPES.fetch(filter.fetch("match_type")),
-              value: filter.fetch("value"),
-              case_sensitive: filter.fetch("case_sensitive")
-            }
-          }
-        }
+        if filter.key?("and")
+          return { and_group: { expressions: filter.fetch("and").map do |child|
+            filter_request(child)
+          end } }
+        end
+        if filter.key?("or")
+          return { or_group: { expressions: filter.fetch("or").map do |child|
+            filter_request(child)
+          end } }
+        end
+        return { not_expression: filter_request(filter.fetch("not")) } if filter.key?("not")
+
+        leaf_filter_request(filter)
       end
 
-      def metric_filter_request(filter)
-        return nil unless filter
+      def leaf_filter_request(filter)
+        request = { field_name: filter.fetch("field") }
+        if filter.key?("match_type")
+          request[:string_filter] = {
+            match_type: MATCH_TYPES.fetch(filter.fetch("match_type")),
+            value: filter.fetch("value"),
+            case_sensitive: filter.fetch("case_sensitive")
+          }
+        end
+        if filter.key?("in_list")
+          request[:in_list_filter] = {
+            values: filter.fetch("in_list"),
+            case_sensitive: filter.fetch("case_sensitive")
+          }
+        end
+        request[:numeric_filter] = numeric_filter_request(filter) if filter.key?("operation")
+        request[:between_filter] = between_filter_request(filter) if filter.key?("between")
+        request[:empty_filter] = {} if filter.key?("null")
+        { filter: request }
+      end
 
+      def numeric_filter_request(filter)
         number = filter.fetch("value")
-        numeric_value = if number.is_a?(Integer)
-                          { int64_value: number }
-                        else
-                          { double_value: number }
-                        end
         {
-          filter: {
-            field_name: filter.fetch("field"),
-            numeric_filter: {
-              operation: NUMERIC_OPERATIONS.fetch(filter.fetch("operation")),
-              value: numeric_value
-            }
-          }
+          operation: NUMERIC_OPERATIONS.fetch(filter.fetch("operation")),
+          value: numeric_value(number)
         }
       end
 
+      def between_filter_request(filter)
+        lower, upper = filter.fetch("between")
+        {
+          from_value: numeric_value(lower),
+          to_value: numeric_value(upper)
+        }
+      end
+
+      def numeric_value(number)
+        number.is_a?(Integer) ? { int64_value: number } : { double_value: number }
+      end
+
       def order_request(order)
         selector = order.key?("metric") ? :metric : :dimension
         name_key = selector == :metric ? :metric_name : :dimension_name
@@ -184,12 +253,13 @@ module AnalyticsOps
       end
 
       def invoke(method_name, request)
+        target = request[:property] || request[:name]
         SafeLogging.write(
           @logger,
           :info,
           "google_data_request",
           "method" => method_name.to_s,
-          "property" => request.fetch(:property)
+          "property" => target
         )
         generated_client = translate_errors { client }
         unless generated_client.respond_to?(method_name)
@@ -214,45 +284,68 @@ module AnalyticsOps
         raise InvalidRequestError, "batch report definitions must have unique names"
       end
 
-      def normalize_result(definition, response)
-        dimension_headers = array_field(response, :dimension_headers).map do |header|
-          remote_string(field(header, :name), "dimension header")
+      def normalize_field(value, kind)
+        api_name = remote_string(field(value, :api_name), "#{kind} API name")
+        ui_name = optional_string(field(value, :ui_name), "#{kind} UI name")
+        description = optional_string(field(value, :description), "#{kind} description")
+        category = optional_string(field(value, :category), "#{kind} category")
+        deprecated = array_field(value, :deprecated_api_names).map do |name|
+          remote_string(name, "deprecated #{kind} API name")
         end
-        metric_headers = array_field(response, :metric_headers).map do |header|
-          remote_string(field(header, :name), "metric header")
+        blocked = array_field(value, :blocked_reasons).map { |reason| enum_name(reason) }
+        Reports::Field.new(
+          api_name:,
+          ui_name:,
+          description:,
+          category:,
+          kind:,
+          custom_definition: metadata_boolean(field(value, :custom_definition)),
+          deprecated_api_names: deprecated,
+          data_type: optional_enum_name(field(value, :type)),
+          expression: optional_string(field(value, :expression), "#{kind} expression"),
+          blocked_reasons: blocked
+        )
+      end
+
+      def compatibility_entries(response, field_name)
+        array_field(response, field_name).map do |entry|
+          metadata = field(entry, field_name == :dimension_compatibilities ? :dimension_metadata : :metric_metadata)
+          {
+            "api_name" => remote_string(field(metadata, :api_name), "compatibility API name"),
+            "compatibility" => enum_name(field(entry, :compatibility))
+          }
         end
-        expected_dimensions = definition.dimensions.dup
-        expected_dimensions << "dateRange" if definition.date_ranges.length > 1
-        unless dimension_headers == expected_dimensions && metric_headers == definition.metrics
-          raise RemoteError, "Google Data API returned headers that do not match the requested report"
+      end
+
+      def compatibility_names(entries)
+        names = entries.map { |entry| entry.fetch("api_name") }
+        unless names.uniq.length == names.length
+          raise RemoteError,
+                "Google Data API returned duplicate compatibility fields"
         end
 
-        rows = array_field(response, :rows).map do |row|
-          dimensions = values(row, :dimension_values)
-          metrics = values(row, :metric_values)
-          unless dimensions.length == dimension_headers.length && metrics.length == metric_headers.length
-            raise RemoteError, "Google Data API returned a row that does not match its headers"
-          end
+        names.sort
+      end
 
-          dimension_headers.zip(dimensions).to_h.merge(metric_headers.zip(metrics).to_h)
-        end
+      def optional_enum_name(value)
+        return nil if value.nil?
 
-        Reports::Result.new(
-          name: definition.name,
-          kind: definition.kind,
-          dimension_headers:,
-          metric_headers:,
-          rows:,
-          row_count: response_row_count(response, rows.length),
-          metadata: metadata(response)
-        )
+        enum_name(value)
+      end
+
+      def enum_name(value)
+        name = value.respond_to?(:name) ? value.name : value
+        string = name.to_s
+        raise RemoteError, "Google Data API returned an invalid enum" unless string.match?(/\A[A-Z][A-Z0-9_]*\z/)
+
+        string.downcase
       end
 
       def values(row, name)
         array_field(row, name).map { |value| remote_string(field(value, :value), "row value") }
       end
 
-      def metadata(response)
+      def normalize_metadata(response)
         response_metadata = field(response, :metadata)
         result = {}
         if response_metadata

data/lib/analytics_ops/clients/data_result_normalization.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module AnalyticsOps
+  module Clients
+    # Strict normalization of generated Data API report responses.
+    module DataResultNormalization
+      private
+
+      def normalize_result(definition, response)
+        dimension_headers = response_headers(response, :dimension_headers, "dimension header")
+        metric_headers = response_headers(response, :metric_headers, "metric header")
+        validate_result_headers!(definition, dimension_headers, metric_headers)
+        response_rows = array_field(response, :rows)
+        if response_rows.length > definition.limit
+          raise RemoteError, "Google Data API returned more rows than the requested limit"
+        end
+
+        rows = response_rows.map { |row| normalize_result_row(row, dimension_headers, metric_headers) }
+        Reports::Result.new(
+          name: definition.name,
+          kind: definition.kind,
+          dimension_headers:,
+          metric_headers:,
+          rows:,
+          row_count: response_row_count(response, rows.length),
+          metadata: normalize_metadata(response)
+        )
+      end
+
+      def response_headers(response, collection, label)
+        array_field(response, collection).map do |header|
+          remote_string(field(header, :name), label)
+        end
+      end
+
+      def validate_result_headers!(definition, dimensions, metrics)
+        expected_dimensions = definition.dimensions.dup
+        expected_dimensions << "dateRange" if definition.date_ranges.length > 1
+        return if dimensions == expected_dimensions && metrics == definition.metrics
+
+        raise RemoteError, "Google Data API returned headers that do not match the requested report"
+      end
+
+      def normalize_result_row(row, dimension_headers, metric_headers)
+        dimensions = values(row, :dimension_values)
+        metrics = values(row, :metric_values)
+        unless dimensions.length == dimension_headers.length && metrics.length == metric_headers.length
+          raise RemoteError, "Google Data API returned a row that does not match its headers"
+        end
+
+        dimension_headers.zip(dimensions).to_h.merge(metric_headers.zip(metrics).to_h)
+      end
+    end
+  end
+end

data/lib/analytics_ops/clients/error_translation.rb

@@ -19,7 +19,12 @@ module AnalyticsOps
         translated = error_class(error.class.name)
         raise unless translated
 
-        raise translated, Redaction.message(error.message)
+        raise translated.new(
+          Redaction.message(error.message),
+          remote_reason: structured_value(error, :reason),
+          remote_metadata: structured_value(error, :error_metadata),
+          remote_code: structured_value(error, :code)
+        )
       end
 
       def error_class(name)
@@ -39,6 +44,13 @@ module AnalyticsOps
         end
       end
       private_class_method :error_class
+
+      def structured_value(error, method)
+        error.public_send(method) if error.respond_to?(method)
+      rescue StandardError
+        nil
+      end
+      private_class_method :structured_value
     end
     private_constant :ErrorTranslation
   end