analytics_ops 0.2.0 → 0.3.0

data/lib/analytics_ops/portfolio.rb

@@ -0,0 +1,216 @@
+# frozen_string_literal: true
+
+module AnalyticsOps
+  # Read-only summary across every profile in one Analytics Ops configuration.
+  class Portfolio
+    # One property/period row in a portfolio summary.
+    class Entry < Resources::Value
+      fields :profile, :property_id, :period, :active_users, :sessions, :key_events
+    end
+
+    # Immutable multi-property report result.
+    class Result
+      attr_reader :entries, :date_ranges
+
+      def initialize(entries:, date_ranges:)
+        unless entries.is_a?(Array) && entries.all?(Entry)
+          raise ArgumentError, "entries must contain AnalyticsOps::Portfolio::Entry values"
+        end
+        raise ArgumentError, "date_ranges must be an array" unless date_ranges.is_a?(Array)
+
+        @entries = entries.dup.freeze
+        @date_ranges = Canonical.immutable(date_ranges)
+        freeze
+      end
+
+      def to_h
+        {
+          "entries" => entries.map(&:to_h),
+          "date_ranges" => date_ranges
+        }
+      end
+    end
+
+    # One independent property result. Failures are isolated to that property.
+    class ProfileResult < Resources::Value
+      fields :profile, :property_id, :labels, :status, :result, :error
+    end
+
+    # Immutable generalized report or health collection across profiles.
+    class Collection
+      NAME = /\A[a-z][a-z0-9_-]{0,71}\z/
+
+      attr_reader :kind, :name, :entries, :date_ranges
+
+      def initialize(kind:, name:, entries:, date_ranges:)
+        unless %w[report health].include?(kind) && name.is_a?(String) && NAME.match?(name) &&
+               entries.is_a?(Array) && entries.all?(ProfileResult)
+          raise ArgumentError, "Invalid portfolio collection"
+        end
+
+        @kind = kind.dup.freeze
+        @name = name.dup.freeze
+        @entries = entries.dup.freeze
+        @date_ranges = Canonical.immutable(date_ranges)
+        freeze
+      end
+
+      def success?
+        entries.all? { |entry| entry.status == "ok" }
+      end
+
+      def to_h
+        {
+          "kind" => kind,
+          "name" => name,
+          "success" => success?,
+          "date_ranges" => date_ranges,
+          "entries" => entries.map(&:to_h)
+        }
+      end
+    end
+
+    def initialize(config:, store: ServiceAccount::Store.new, workspace_loader: nil,
+                   service_account_loader: nil, transport: :grpc, timeout: nil, logger: nil)
+      @config = config
+      @store = store
+      @workspace_loader = workspace_loader || method(:load_workspace)
+      @service_account_loader = service_account_loader || ServiceAccount.method(:load)
+      @transport = transport
+      @timeout = timeout
+      @logger = logger
+    end
+
+    def overview(date_ranges: nil)
+      document = Configuration.load(@config)
+      definition = Reports::Catalog.overview.first
+      definition = definition.with_date_ranges(date_ranges) if date_ranges
+      entries = document.profiles.sort.flat_map do |profile, desired_state|
+        report = workspace(profile).report(definition)
+        report_entries(profile, desired_state.property_id, report)
+      end
+      Result.new(entries:, date_ranges: definition.date_ranges)
+    end
+
+    def report(name, date_ranges: nil, concurrency: 4)
+      collect(kind: "report", name: name.to_s, date_ranges:, concurrency:) do |target|
+        result = date_ranges ? target.report(name, date_ranges:) : target.report(name)
+        result.to_h
+      end
+    end
+
+    def health(date_ranges: nil, concurrency: 4)
+      collect(kind: "health", name: "health", date_ranges:, concurrency:) do |target|
+        result = date_ranges ? target.health(date_ranges:) : target.health
+        result.to_h
+      end
+    end
+
+    private
+
+    def collect(kind:, name:, date_ranges:, concurrency:)
+      unless name.is_a?(String) && Collection::NAME.match?(name)
+        raise InvalidRequestError, "Portfolio report name is invalid"
+      end
+      unless concurrency.is_a?(Integer) && concurrency.between?(1, 8)
+        raise InvalidRequestError, "Portfolio concurrency must be between 1 and 8"
+      end
+
+      profiles = Configuration.load(@config).profiles.sort
+      queue = Queue.new
+      profiles.each_with_index { |(profile, desired_state), index| queue << [index, profile, desired_state] }
+      entries = Array.new(profiles.length)
+      workers = [concurrency, profiles.length].min.times.map do
+        Thread.new do
+          loop do
+            index, profile, desired_state = queue.pop(true)
+            entries[index] = collect_profile(profile, desired_state) { yield workspace(profile) }
+          rescue ThreadError
+            break
+          end
+        end
+      end
+      workers.each(&:join)
+      Collection.new(kind:, name:, entries:, date_ranges: date_ranges || [])
+    end
+
+    def collect_profile(profile, desired_state)
+      payload = yield
+      ProfileResult.new(
+        profile:,
+        property_id: desired_state.property_id,
+        labels: desired_state.labels,
+        status: "ok",
+        result: payload,
+        error: nil
+      )
+    rescue AnalyticsOps::Error => error
+      ProfileResult.new(
+        profile:,
+        property_id: desired_state.property_id,
+        labels: desired_state.labels,
+        status: "error",
+        result: nil,
+        error: {
+          "type" => error.class.name.split("::").last,
+          "message" => Redaction.message(error.message)
+        }
+      )
+    rescue StandardError => error
+      SafeLogging.write(@logger, :error, "portfolio_profile_error", "type" => error.class.name, "profile" => profile)
+      ProfileResult.new(
+        profile:,
+        property_id: desired_state.property_id,
+        labels: desired_state.labels,
+        status: "error",
+        result: nil,
+        error: {
+          "type" => "InternalError",
+          "message" => "Analytics Ops could not complete this read-only profile request"
+        }
+      )
+    end
+
+    def workspace(profile)
+      service_account = @service_account_loader.call(
+        store: @store,
+        connection: nil,
+        config: @config,
+        profile:
+      )
+      @workspace_loader.call(
+        config: @config,
+        profile:,
+        service_account:,
+        transport: @transport,
+        timeout: @timeout,
+        logger: @logger
+      )
+    end
+
+    def load_workspace(config:, profile:, service_account:, transport:, timeout:, logger:)
+      Workspace.load(config, profile:, service_account:, transport:, timeout:, logger:)
+    end
+
+    def report_entries(profile, property_id, report)
+      unless report.is_a?(Reports::Result) &&
+             report.metric_headers == %w[activeUsers sessions keyEvents] &&
+             [[], ["dateRange"]].include?(report.dimension_headers)
+        raise RemoteError, "Portfolio report returned an unexpected shape"
+      end
+
+      rows = report.rows
+      rows = [{ "activeUsers" => "0", "sessions" => "0", "keyEvents" => "0" }] if rows.empty?
+      rows.map do |row|
+        Entry.new(
+          profile:,
+          property_id:,
+          period: row.fetch("dateRange", "current"),
+          active_users: row.fetch("activeUsers"),
+          sessions: row.fetch("sessions"),
+          key_events: row.fetch("keyEvents")
+        )
+      end
+    end
+  end
+end

data/lib/analytics_ops/rails/railtie.rb

@@ -8,9 +8,9 @@ module AnalyticsOps
     def run(command, report_name: nil)
       arguments = [
         command,
-        "--config", ENV.fetch("ANALYTICS_OPS_CONFIG", Rails.root.join("config/analytics_ops.yml").to_s),
-        "--profile", ENV.fetch("ANALYTICS_OPS_PROFILE", Rails.env.to_s)
+        "--config", ENV.fetch("ANALYTICS_OPS_CONFIG", Rails.root.join("config/analytics_ops.yml").to_s)
       ]
+      arguments.push("--profile", ENV.fetch("ANALYTICS_OPS_PROFILE")) if ENV.key?("ANALYTICS_OPS_PROFILE")
       if command == "plan"
         arguments.push("--output",
                        ENV.fetch("ANALYTICS_OPS_PLAN", Rails.root.join("tmp/analytics_ops-plan.json").to_s))
@@ -31,7 +31,16 @@ module AnalyticsOps
       audit: "Audit configured Google Analytics state without changing it",
       plan: "Write a deterministic Analytics Ops plan",
       verify: "Verify that managed Google Analytics state converges",
-      overview: "Show the selected property's Analytics overview"
+      overview: "Show the selected property's Analytics overview",
+      portfolio: "Show totals across every configured Analytics property",
+      health: "Check traffic, data quality, quota, and configuration drift",
+      governance: "Audit broader GA4 governance settings without changing them",
+      access: "Read the aggregate GA4 access report"
+    }.freeze
+    NAMED_TASKS = {
+      report: "Run a built-in report",
+      raw: "Run an optional BigQuery recipe",
+      funnel: "Run a configured funnel"
     }.freeze
 
     generators do
@@ -47,12 +56,14 @@ module AnalyticsOps
           end
         end
 
-        desc "Run a built-in report: rake analytics:report[NAME]"
-        task :report, [:name] => :environment do |_task, arguments|
-          name = arguments[:name]
-          raise Error, "analytics:report requires NAME" if name.to_s.empty?
+        NAMED_TASKS.each do |command, description|
+          desc "#{description}: rake analytics:#{command}[NAME]"
+          task command, [:name] => :environment do |_task, arguments|
+            name = arguments[:name]
+            raise Error, "analytics:#{command} requires NAME" if name.to_s.empty?
 
-          RailsTasks.run("report", report_name: name)
+            RailsTasks.run(command.to_s, report_name: name)
+          end
         end
       end
     end

data/lib/analytics_ops/redaction.rb

@@ -25,13 +25,17 @@ module AnalyticsOps
     end
 
     def message(value)
+      text(value, limit: 1_000)
+    end
+
+    def text(value, limit: nil)
       text = value.to_s.encode(Encoding::UTF_8, invalid: :replace, undef: :replace, replace: "?")
-      text.gsub(PRIVATE_KEY, "[REDACTED]")
-          .gsub(AUTHORIZATION, "Authorization: [REDACTED]")
-          .gsub(BEARER, "Bearer [REDACTED]")
-          .gsub(SECRET_ASSIGNMENT) { "#{Regexp.last_match(1)}=[REDACTED]" }
-          .gsub(CONTROL_CHARACTERS, "?")
-          .slice(0, 1_000)
+      redacted = text.gsub(PRIVATE_KEY, "[REDACTED]")
+                     .gsub(AUTHORIZATION, "Authorization: [REDACTED]")
+                     .gsub(BEARER, "Bearer [REDACTED]")
+                     .gsub(SECRET_ASSIGNMENT) { "#{Regexp.last_match(1)}=[REDACTED]" }
+                     .gsub(CONTROL_CHARACTERS, "?")
+      limit ? redacted.slice(0, limit) : redacted
     end
   end
 

data/lib/analytics_ops/reports/csv_export.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+require "csv"
+require "securerandom"
+
+module AnalyticsOps
+  module Reports
+    # Immutable receipt for one bounded CSV export.
+    class ExportResult < Resources::Value
+      fields :path, :rows, :headers
+    end
+
+    # Atomic, spreadsheet-safe streaming export for report pages.
+    module CSVExport
+      module_function
+
+      def write(path, pages)
+        destination = validated_path(path)
+        temporary = File.join(
+          File.dirname(destination),
+          ".#{File.basename(destination)}.#{Process.pid}.#{SecureRandom.hex(6)}.tmp"
+        )
+        headers = nil
+        row_count = nil
+        File.open(temporary, File::WRONLY | File::CREAT | File::EXCL, 0o600) do |file|
+          headers, row_count = write_pages(file, pages)
+          file.flush
+          file.fsync
+        end
+        File.rename(temporary, destination)
+        File.chmod(0o600, destination)
+        ExportResult.new(path: destination, rows: row_count, headers:)
+      rescue AnalyticsOps::Error
+        raise
+      rescue SystemCallError => error
+        raise ConfigurationError, "Cannot write CSV export: #{Redaction.message(error.message)}"
+      ensure
+        File.unlink(temporary) if temporary && File.exist?(temporary)
+      end
+
+      def write_pages(file, pages)
+        headers = nil
+        row_count = 0
+        pages.each do |page|
+          headers, written = write_page(file, page, headers)
+          row_count += written
+        end
+        raise RemoteError, "CSV export did not receive a report page" unless headers
+
+        [headers, row_count]
+      end
+      private_class_method :write_pages
+
+      def write_page(file, page, headers)
+        raise InvalidRequestError, "CSV export pages must be report results" unless page.is_a?(Result)
+
+        if headers
+          raise RemoteError, "Paginated report headers changed during export" unless page.headers == headers
+        else
+          headers = page.headers
+          file.write(CSV.generate_line(headers))
+        end
+        page.rows.each do |row|
+          file.write(CSV.generate_line(headers.map { |header| safe_cell(row.fetch(header)) }))
+        end
+        [headers, page.rows.length]
+      end
+      private_class_method :write_page
+
+      def safe_cell(value)
+        string = value.to_s.gsub(/[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/, "?")
+        string.match?(/\A(?:[\p{Space}\uFEFF]*[=+\-@]|[\t\r\n])/) ? "'#{string}" : string
+      end
+      private_class_method :safe_cell
+
+      def validated_path(value)
+        unless value.is_a?(String) && !value.empty? && value.length <= 4_096 &&
+               !value.match?(/[\u0000-\u001f\u007f]/)
+          raise ConfigurationError, "CSV output path is invalid"
+        end
+
+        expanded = File.expand_path(value)
+        directory = File.dirname(expanded)
+        raise ConfigurationError, "CSV output directory does not exist" unless File.directory?(directory)
+
+        expanded
+      end
+      private_class_method :validated_path
+    end
+  end
+end

data/lib/analytics_ops/reports/definition.rb

@@ -14,7 +14,12 @@ module AnalyticsOps
       RELATIVE_DATE = /\A(\d{1,4})daysAgo\z/
       MATCH_TYPES = %w[exact begins_with ends_with contains full_regexp partial_regexp].freeze
       NUMERIC_OPERATIONS = %w[equal less_than less_than_or_equal greater_than greater_than_or_equal].freeze
-      MAX_LIMIT = 100_000
+      GROUP_FILTERS = %w[and or not].freeze
+      MAX_FILTER_DEPTH = 8
+      MAX_FILTER_CHILDREN = 20
+      MAX_STANDARD_LIMIT = 250_000
+      MAX_REALTIME_LIMIT = 100_000
+      MAX_LIMIT = MAX_STANDARD_LIMIT
       INT64_RANGE = (-(2**63))..((2**63) - 1)
 
       attr_reader :name, :kind, :dimensions, :metrics, :date_ranges,
@@ -27,11 +32,12 @@ module AnalyticsOps
         @dimensions = api_names(dimensions, "dimensions", minimum: 0, maximum: realtime_kind?(kind) ? 4 : 9)
         @metrics = api_names(metrics, "metrics", minimum: 1, maximum: 10)
         @date_ranges = ranges(date_ranges)
-        @dimension_filter = string_filter(dimension_filter)
-        @metric_filter = numeric_filter(metric_filter)
+        @dimension_filter = filter_expression(dimension_filter, type: :dimension)
+        @metric_filter = filter_expression(metric_filter, type: :metric)
         @order_bys = orders(order_bys)
         @offset = integer(offset, "offset", minimum: 0, maximum: 1_000_000_000)
-        @limit = integer(limit, "limit", minimum: 1, maximum: MAX_LIMIT)
+        maximum_limit = realtime? ? MAX_REALTIME_LIMIT : MAX_STANDARD_LIMIT
+        @limit = integer(limit, "limit", minimum: 1, maximum: maximum_limit)
         validate_shape!
         freeze
       end
@@ -40,6 +46,36 @@ module AnalyticsOps
         kind == "realtime"
       end
 
+      def with_date_ranges(value)
+        self.class.new(
+          name:,
+          kind:,
+          dimensions:,
+          metrics:,
+          date_ranges: value,
+          dimension_filter:,
+          metric_filter:,
+          order_bys:,
+          offset:,
+          limit:
+        )
+      end
+
+      def with_pagination(offset:, limit:)
+        self.class.new(
+          name:,
+          kind:,
+          dimensions:,
+          metrics:,
+          date_ranges:,
+          dimension_filter:,
+          metric_filter:,
+          order_bys:,
+          offset:,
+          limit:
+        )
+      end
+
       def to_h
         {
           "name" => name,
@@ -152,16 +188,73 @@ module AnalyticsOps
         RELATIVE_DATE.match(value)&.[](1)&.to_i
       end
 
-      def string_filter(value)
+      def filter_expression(value, type:, depth: 0)
         return nil if value.nil?
 
-        hash = string_hash(value, "dimension filter", %w[field match_type value case_sensitive])
+        invalid!("Report filter nesting is too deep") if depth > MAX_FILTER_DEPTH
+
+        hash = string_hash(value, "#{type} filter", filter_allowed_keys(type))
+        groups = GROUP_FILTERS.select { |key| hash.key?(key) }
+        return filter_group(hash, type:, depth:, groups:) unless groups.empty?
+
+        type == :dimension ? string_filter(hash) : numeric_filter(hash)
+      end
+
+      def filter_allowed_keys(type)
+        primitive = if type == :dimension
+                      %w[field match_type value case_sensitive in_list null]
+                    else
+                      %w[field operation value between null]
+                    end
+        GROUP_FILTERS + primitive
+      end
+
+      def filter_group(hash, type:, depth:, groups:)
+        invalid!("A filter expression requires exactly one of and, or, or not") unless groups.length == 1
+        invalid!("A filter group cannot include primitive filter fields") unless hash.keys == groups
+
+        operator = groups.first
+        child = hash.fetch(operator)
+        if operator == "not"
+          invalid!("not must contain one filter object") unless child.is_a?(Hash)
+
+          return Canonical.immutable("not" => filter_expression(child, type:, depth: depth + 1))
+        end
+        unless child.is_a?(Array) && child.length.between?(2, MAX_FILTER_CHILDREN)
+          invalid!("#{operator} must contain 2 to #{MAX_FILTER_CHILDREN} filter objects")
+        end
+
+        Canonical.immutable(
+          operator => child.map { |item| filter_expression(item, type:, depth: depth + 1) }
+        )
+      end
+
+      def string_filter(hash)
         field = hash.fetch("field") { invalid!("Missing filter field") }
+        unless valid_api_name?(field) && dimensions.include?(field)
+          invalid!("Dimension filter is invalid or references an unselected dimension")
+        end
+
+        shapes = [
+          hash.key?("match_type") || hash.key?("value"),
+          hash.key?("in_list"),
+          hash.key?("null")
+        ]
+        unless shapes.count(true) == 1
+          invalid!("Dimension filter requires exactly one string, in_list, or null condition")
+        end
+
+        return string_match_filter(hash, field) if shapes[0]
+        return in_list_filter(hash, field) if shapes[1]
+
+        null_filter(hash, field, type: :dimension)
+      end
+
+      def string_match_filter(hash, field)
         match_type = hash.fetch("match_type") { invalid!("Missing filter match_type") }
         filter_value = hash.fetch("value") { invalid!("Missing filter value") }
-        unless valid_api_name?(field) && dimensions.include?(field) && MATCH_TYPES.include?(match_type) &&
-               printable_string?(filter_value, maximum: 1_024)
-          invalid!("Dimension filter is invalid or references an unselected dimension")
+        unless MATCH_TYPES.include?(match_type) && printable_string?(filter_value, maximum: 1_024)
+          invalid!("Dimension string filter is invalid")
         end
 
         case_sensitive = hash.key?("case_sensitive") ? boolean(hash.fetch("case_sensitive"), "case_sensitive") : false
@@ -173,15 +266,42 @@ module AnalyticsOps
         )
       end
 
-      def numeric_filter(value)
-        return nil if value.nil?
+      def in_list_filter(hash, field)
+        values = hash.fetch("in_list")
+        unless values.is_a?(Array) && values.length.between?(1, 100) &&
+               values.all? { |value| printable_string?(value, maximum: 1_024) } &&
+               values.uniq.length == values.length
+          invalid!("Dimension in_list must contain 1 to 100 unique printable strings")
+        end
+        case_sensitive = hash.key?("case_sensitive") ? boolean(hash.fetch("case_sensitive"), "case_sensitive") : false
+        Canonical.immutable("field" => field, "in_list" => values, "case_sensitive" => case_sensitive)
+      end
 
-        hash = string_hash(value, "metric filter", %w[field operation value])
+      def numeric_filter(hash)
         field = hash.fetch("field") { invalid!("Missing metric filter field") }
+        unless valid_api_name?(field) && metrics.include?(field)
+          invalid!("Metric filter is invalid or references an unselected metric")
+        end
+
+        shapes = [
+          hash.key?("operation") || hash.key?("value"),
+          hash.key?("between"),
+          hash.key?("null")
+        ]
+        unless shapes.count(true) == 1
+          invalid!("Metric filter requires exactly one numeric, between, or null condition")
+        end
+
+        return numeric_comparison_filter(hash, field) if shapes[0]
+        return between_filter(hash, field) if shapes[1]
+
+        null_filter(hash, field, type: :metric)
+      end
+
+      def numeric_comparison_filter(hash, field)
         operation = hash.fetch("operation") { invalid!("Missing metric filter operation") }
         number = hash.fetch("value") { invalid!("Missing metric filter value") }
-        valid = valid_api_name?(field) && metrics.include?(field) && NUMERIC_OPERATIONS.include?(operation)
-        invalid!("Metric filter is invalid or references an unselected metric") unless valid
+        invalid!("Metric filter operation is invalid") unless NUMERIC_OPERATIONS.include?(operation)
         unless valid_numeric_filter_value?(number)
           invalid!("Metric filter value must be a finite int64 Integer or Float")
         end
@@ -189,6 +309,24 @@ module AnalyticsOps
         Canonical.immutable("field" => field, "operation" => operation, "value" => number)
       end
 
+      def between_filter(hash, field)
+        bounds = hash.fetch("between")
+        unless bounds.is_a?(Array) && bounds.length == 2 && bounds.all? { |value| valid_numeric_filter_value?(value) }
+          invalid!("Metric between filter must contain two finite numeric values")
+        end
+        invalid!("Metric between lower bound must not exceed its upper bound") if bounds[0] > bounds[1]
+
+        Canonical.immutable("field" => field, "between" => bounds)
+      end
+
+      def null_filter(hash, field, type:)
+        invalid!("#{type} null filter must be true") unless hash.fetch("null") == true
+        allowed = %w[field null]
+        invalid!("#{type} null filter contains an unsupported option") unless (hash.keys - allowed).empty?
+
+        Canonical.immutable("field" => field, "null" => true)
+      end
+
       def valid_numeric_filter_value?(number)
         (number.is_a?(Integer) && INT64_RANGE.cover?(number)) || (number.is_a?(Float) && number.finite?)
       end