k9o 1.4.0 → 1.5.0

data/lib/k9o/audit.rb

@@ -4,6 +4,11 @@ module K9o
   # Project-plane audit log namespace — k9o.audit.*.
   # Distinct from k9o.org.audit_log.
   class Audit
+    TOPIC = 'audit.project'
+    TYPE_PATTERN = /\A[A-Za-z0-9_][A-Za-z0-9_.\-]*\z/.freeze
+    DEFAULT_REPLAY_LIMIT = 1000
+    EVENT_ID_PATTERN = /\Aevt_[0-9A-Za-z]{1,40}\z/.freeze
+
     def initialize(config)
       @config = config
     end
@@ -23,5 +28,150 @@ module K9o
       q['cursor'] = cursor unless cursor.nil?
       HttpTransport.request(config: @config, method: 'GET', path: UrlGrammar.project_path(slug, 'audit-log'), query: q)
     end
+
+    # Live tail of the project's audit events: an Enumerator of { "auditEventId", "type", "actorType", "occurredAt",
+    # "outcome", "eventId" }. The first five are the audit event (read the full row with #query by event type and
+    # time); +eventId+ is the publisher's own id for the frame, which is not the +auditEventId+.
+    #
+    # +types+ is a list of audit event types: one type subscribes to +audit.project.<type>+, none to
+    # +audit.project.*+, several to +audit.project.*+ with the SDK keeping only those types. Needs ORG_OWNER,
+    # ORG_ADMIN, PROJECT_ADMIN or PROJECT_AUDITOR (re-checked every 60 seconds; others get 403 K9O_RBAC_DENY);
+    # extensions cannot subscribe. +warden.checkpoint+ events are not in the live tail: the audit log and #replay
+    # carry them.
+    #
+    # It is the same reconnecting reader as +realtime.subscribe+: it resumes with +Last-Event-ID+ (the frame's SSE
+    # +id+), drops events it already delivered (by the frame's +eventId+ and by +auditEventId+), ignores heartbeats
+    # and abandons a connection that has been silent for +Realtime.idle_timeout+ seconds. When the gateway says the
+    # position is lost (+event: resync+, action +replay-ledger+; audit streams cannot be resumed by sequence) the tail
+    # first reconnects fresh and buffers that connection's frames, then calls #replay after the last +auditEventId+ it
+    # delivered (page by page while there are more), yields the events it missed (they also carry +sequence+, and may
+    # include +warden.checkpoint+), calls +on_resync+ with the frame's hash, then yields the buffered and live frames,
+    # each event once. Nothing is lost: the connection is open before the replay runs. If it had delivered nothing yet
+    # it only reconnects (a new connection starts at the live head; a resync that opens a fresh connection is informational and ignored). Stop with +break+ or +first(n)+.
+    def tail(types: nil, on_resync: nil)
+      topic, keep = topic_for(types)
+      slug = Studio.require_project_slug(@config, 'audit')
+      catch_up = lambda do |_info, last_event|
+        last_event ? collect_replay(topic, last_event['auditEventId']) : []
+      end
+      decode = lambda do |data, frame|
+        payload = data.is_a?(Hash) ? data['payload'] : nil
+        next nil unless payload.is_a?(Hash) && payload['auditEventId'].is_a?(String)
+
+        event = {
+          'auditEventId' => payload['auditEventId'],
+          'type' => payload['type'].is_a?(String) ? payload['type'] : (data['eventType'] || frame[:event]),
+          'actorType' => payload['actorType'],
+          'occurredAt' => payload['occurredAt'],
+          'outcome' => payload['outcome']
+        }
+        event['eventId'] = data['eventId'] if data['eventId'].is_a?(String)
+        event
+      end
+      keys = lambda do |event, _frame|
+        [event['eventId'], event['auditEventId']].compact
+      end
+      cursor = lambda do |event, frame|
+        (frame && frame[:id]) || event['eventId']
+      end
+      Realtime.new(@config).feed(UrlGrammar.project_path(slug, 'realtime/subscribe'), { 'topic' => topic }, nil,
+                                 on_resync, decode: decode, keys: keys, cursor: cursor, accept: keep,
+                                 catch_up: catch_up, fresh_head_resync: true)
+    end
+
+    # POST /@<slug>/realtime/replay for the audit tail: a batch of past audit events in order, including
+    # +warden.checkpoint+. +topic+ is +audit.project.*+ (default) or +audit.project.<type>+. Give
+    # +after_audit_event_id+ (the last +auditEventId+ you received; unknown ids answer 400 K9O_INVALID_CURSOR) or
+    # +from_sequence+ (+after_audit_event_id+ wins when both are given); +until_sequence+ and +limit+ bound the batch (+limit+ is
+    # 1000 when omitted, up to 10,000). Needs the audit log's access.
+    #
+    # +data+ is the typed page { "events" => [event], "nextCursor", "hasMore" }; each event is { "auditEventId", "type",
+    # "actorType", "occurredAt", "outcome", "sequence" } (+sequence+ is a String, as the gateway sends it).
+    def replay(topic: "#{TOPIC}.*", after_audit_event_id: nil, from_sequence: nil, until_sequence: nil, limit: nil)
+      unless topic.is_a?(String) && (topic == TOPIC || topic.start_with?("#{TOPIC}."))
+        raise ApprovalChecks.invalid('topic must be audit.project.* or audit.project.<type>', topic: topic)
+      end
+      unless after_audit_event_id.nil? || (after_audit_event_id.is_a?(String) && EVENT_ID_PATTERN.match?(after_audit_event_id))
+        raise ApprovalChecks.invalid('after_audit_event_id must look like evt_<id>', after_audit_event_id: after_audit_event_id)
+      end
+      {from_sequence: from_sequence, until_sequence: until_sequence}.each do |name, value|
+        next if value.nil? || (value.is_a?(Integer) && value >= 0)
+
+        raise ApprovalChecks.invalid("#{name} must be a whole number, 0 or more", name => value)
+      end
+      unless limit.nil? || (limit.is_a?(Integer) && limit >= 1 && limit <= 10_000)
+        raise ApprovalChecks.invalid('limit must be a whole number from 1 to 10000', limit: limit)
+      end
+
+      body = { 'topic' => topic }
+      body['afterAuditEventId'] = after_audit_event_id unless after_audit_event_id.nil?
+      body['fromSequence'] = from_sequence unless from_sequence.nil?
+      body['untilSequence'] = until_sequence unless until_sequence.nil?
+      body['limit'] = limit.nil? ? DEFAULT_REPLAY_LIMIT : limit
+      slug = Studio.require_project_slug(@config, 'audit')
+      envelope = HttpTransport.request(config: @config, method: 'POST',
+                                       path: UrlGrammar.project_path(slug, 'realtime/replay'), body: body)
+      Envelope.new(data: replay_page(envelope.data), meta: envelope.meta)
+    end
+
+    private
+
+    # [topic, filter or nil]
+    def topic_for(types)
+      list = types.nil? ? [] : Array(types).map { |t| t.is_a?(Symbol) ? t.to_s : t }
+      list.each do |t|
+        unless t.is_a?(String) && TYPE_PATTERN.match?(t)
+          raise ApprovalChecks.invalid('types must be audit event type names such as "data.write.access"', type: t)
+        end
+      end
+      list = list.uniq
+      return ["#{TOPIC}.*", nil] if list.empty?
+      return ["#{TOPIC}.#{list.first}", nil] if list.size == 1
+
+      ["#{TOPIC}.*", ->(event) { list.include?(event['type']) }]
+    end
+
+    # The replay answer with its events flattened to the tail's shape (+sequence+ kept).
+    def replay_page(data)
+      data = {} unless data.is_a?(Hash)
+      events = Array(data['events']).map { |e| flatten_replayed(e) }.compact
+      { 'events' => events, 'nextCursor' => data['nextCursor'], 'hasMore' => data['hasMore'] == true }
+    end
+
+    def flatten_replayed(raw)
+      return nil unless raw.is_a?(Hash)
+
+      payload = raw['payload'].is_a?(Hash) ? raw['payload'] : {}
+      return nil unless payload['auditEventId'].is_a?(String)
+
+      event = {
+        'auditEventId' => payload['auditEventId'],
+        'type' => payload['type'].is_a?(String) ? payload['type'] : raw['eventType'],
+        'actorType' => payload['actorType'],
+        'occurredAt' => payload['occurredAt'],
+        'outcome' => payload['outcome']
+      }
+      event['sequence'] = raw['sequence'] if raw.key?('sequence')
+      event
+    end
+
+    # Events after +last_id+ from the replay route, page after page while it says there are more. The route has no
+    # page cursor: each next page starts after the last event of the one before.
+    def collect_replay(topic, last_id)
+      out = []
+      cursor = last_id
+      loop do
+        data = replay(topic: topic, after_audit_event_id: cursor).data
+        events = data['events']
+        break if events.empty?
+
+        out.concat(events)
+        next_cursor = events.last['auditEventId']
+        break unless data['hasMore'] && next_cursor != cursor
+
+        cursor = next_cursor
+      end
+      out
+    end
   end
 end

data/lib/k9o/client.rb

@@ -6,7 +6,7 @@ module K9o
     include QueryMethods
 
     attr_reader :config, :projects, :gate, :studio, :compute, :realtime, :streams, :org,
-                :approvals, :approval_policies, :schema, :orgs, :migrations, :functions, :policies, :audit, :me, :kibo, :ai, :storage
+                :approvals, :approval_policies, :schedules, :vectors, :metadata, :rampart, :custody, :end_users, :schema, :orgs, :migrations, :functions, :policies, :audit, :me, :kibo, :ai, :storage, :analytics, :rollups, :keys, :crypto
 
     def initialize(config)
       @config = config
@@ -18,6 +18,12 @@ module K9o
       @streams = Streams.new(config)
       @approvals = Approvals.new(config)
       @approval_policies = ApprovalPolicies.new(config)
+      @schedules = Schedules.new(config)
+      @vectors = Vectors.new(config)
+      @metadata = Metadata.new(config)
+      @rampart = Rampart.new(config)
+      @custody = Custody.new(config)
+      @end_users = EndUsers.new(config)
       @schema = Schema.new(config)
       @org = Org.new(config)
       @orgs = Orgs.new(config)
@@ -29,6 +35,10 @@ module K9o
       @kibo = KiboNamespace.new(config)
       @ai = Ai.new(config)
       @storage = Storage.new(config)
+      @analytics = Analytics.new(config)
+      @rollups = Rollups.new(config)
+      @keys = Keys.new(config)
+      @crypto = Crypto.new(config)
     end
 
     def from(collection)
@@ -51,12 +61,17 @@ module K9o
   # base_url is OPTIONAL. When omitted it defaults to the public-cloud platform host (a convenience only, never used
   # to parse a host). When the host is a platform host (+api.<apex>+ or +api-staging.<apex>+, any apex) and org_slug
   # is set, the org host +<org_slug>.<host>+ is composed. Any other host passes through unchanged.
-  def self.create_client(base_url: nil, api_key: nil, org_slug: nil, project_slug: nil, default_headers: {}, http_client: nil)
+  #
+  # expected_key_refs is optional: an Array of the key identifiers (from your own cloud configuration) your BYOK or HYOK
+  # key may be. It is checked here (see Crypto.check_expected_key_refs) and handed to the client core on every crypto call.
+  def self.create_client(base_url: nil, api_key: nil, org_slug: nil, project_slug: nil, default_headers: {}, http_client: nil,
+                         expected_key_refs: nil)
+    refs = Crypto.check_expected_key_refs(expected_key_refs)
     normalised = UrlGrammar.resolve_base_url(base_url, org_slug)
     config = HttpClientConfig.new(
       base_url: normalised,
       api_key: api_key, org_slug: org_slug, project_slug: project_slug,
-      default_headers: default_headers, http_client: http_client
+      default_headers: default_headers, http_client: http_client, expected_key_refs: refs
     )
     K9oClient.new(config)
   end