lex-telemetry 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 041ec72d71e8c5ffef2fde576bdb413385e882f23594ba4cce1a9ce643ecef1c
+  data.tar.gz: 495a8e595017691e84a68fd4c1c674f740702e6935458c9cff70eda772112b4d
+SHA512:
+  metadata.gz: d1c0a40828925c7c562d9704d9baede1ae27d84c7d16040b15be14ddd7f76310ad5973c02b8720e88e9afbd62578bfbd6b7706b4d2b3c3831f96de644f861de5
+  data.tar.gz: '0951709435fdc2f9c99f5b367cd504a6b1a7f13f7fedd7701bfff17277e0556654939fbb1692070f7b39422a77cef18203b98462dded097fac1318f815924c55'

data/.github/workflows/ci.yml

@@ -0,0 +1,16 @@
+name: CI
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  ci:
+    uses: LegionIO/.github/.github/workflows/ci.yml@main
+
+  release:
+    needs: ci
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    uses: LegionIO/.github/.github/workflows/release.yml@main
+    secrets:
+      rubygems-api-key: ${{ secrets.RUBYGEMS_API_KEY }}

data/.gitignore

@@ -0,0 +1,7 @@
+/.bundle/
+/vendor/bundle
+/tmp/
+/pkg/
+*.gem
+.rspec_status
+Gemfile.lock

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,67 @@
+AllCops:
+  TargetRubyVersion: 3.4
+  NewCops: enable
+  SuggestExtensions: false
+
+Layout/LineLength:
+  Max: 160
+
+Layout/SpaceAroundEqualsInParameterDefault:
+  EnforcedStyle: space
+
+Layout/HashAlignment:
+  EnforcedHashRocketStyle: table
+  EnforcedColonStyle: table
+
+Metrics/MethodLength:
+  Max: 50
+
+Metrics/ClassLength:
+  Max: 1500
+
+Metrics/ModuleLength:
+  Max: 1500
+
+Metrics/BlockLength:
+  Max: 40
+  Exclude:
+    - 'spec/**/*'
+
+Metrics/AbcSize:
+  Max: 60
+
+Metrics/CyclomaticComplexity:
+  Max: 15
+
+Metrics/PerceivedComplexity:
+  Max: 17
+
+Style/Documentation:
+  Enabled: false
+
+Style/SymbolArray:
+  Enabled: true
+
+Style/FrozenStringLiteralComment:
+  Enabled: true
+  EnforcedStyle: always
+
+Naming/FileName:
+  Enabled: false
+
+Naming/PredicateMethod:
+  Enabled: false
+
+Gemspec/DevelopmentDependencies:
+  Enabled: false
+
+Metrics/ParameterLists:
+  Enabled: false
+
+Lint/EmptyClass:
+  Exclude:
+    - 'spec/**/*'
+
+Style/MultilineIfModifier:
+  Exclude:
+    - 'spec/**/*'

data/CHANGELOG.md

@@ -0,0 +1,24 @@
+# Changelog
+
+## [Unreleased]
+
+## [0.1.1] - 2026-03-17
+
+### Changed
+- Renamed `module Actors` to `module Actor` (singular) in collector and publisher actors
+- Updated specs to reference `Actor::Collector` and `Actor::Publisher` accordingly
+
+## [0.1.0] - 2026-03-15
+
+### Added
+- TelemetryEvent normalized event shape with 5 event types (tool_call, llm_request, error, session_start, session_end)
+- Scrubber with whitelist per tool, 3 levels (minimal/standard/paranoid), PII regex patterns
+- Claude Code JSONL parser with incremental byte-offset reads
+- Parser base interface for future tool adapters (Codex, legion-chat)
+- EventStore: in-memory buffer (10k cap) with pending queue for AMQP
+- Stats: session_summary and aggregate_stats computation
+- Runner: ingest_session, session_stats, aggregate_stats, telemetry_status, publish_pending, collect
+- Publisher actor (Every 60s): flushes pending events to telemetry.sessions AMQP exchange
+- Collector actor (Every 300s): scans ~/.claude/projects for session JSONL files
+- HighWaterMark: per-file byte offset tracking for incremental ingestion
+- AMQP transport: telemetry.sessions exchange (topic), telemetry.sessions.process queue (durable)

data/CLAUDE.md

@@ -0,0 +1,117 @@
+# lex-telemetry: Session Log Analytics for LegionIO
+
+**Repository Level 3 Documentation**
+- **Parent**: `/Users/miverso2/rubymine/legion/extensions-core/CLAUDE.md`
+- **Grandparent**: `/Users/miverso2/rubymine/legion/CLAUDE.md`
+
+## Purpose
+
+Core Legion Extension that ingests AI tool session logs (Claude Code JSONL), normalizes events into a common TelemetryEvent shape, scrubs sensitive content, buffers in memory, computes local stats, and publishes operational telemetry to AMQP for central service consumption.
+
+**GitHub**: https://github.com/LegionIO/lex-telemetry
+**License**: MIT
+**Version**: 0.1.1
+
+## Architecture
+
+```
+Legion::Extensions::Telemetry
+├── Helpers/
+│   ├── TelemetryEvent     # Event shape builder with 5-type validation
+│   ├── Scrubber           # Whitelist-based scrubbing (3 levels) + PII regex
+│   ├── EventStore         # In-memory buffer (10k cap) + pending queue for AMQP
+│   ├── Stats              # session_summary + aggregate_stats computation
+│   └── HighWaterMark      # Per-file byte offset tracking for incremental ingestion
+├── Parsers/
+│   ├── Base               # Parser interface (source_name, can_parse?, parse)
+│   └── ClaudeCode         # Claude Code JSONL parser with incremental byte-offset reads
+├── Runners/
+│   └── Telemetry          # ingest_session, session_stats, aggregate_stats, telemetry_status,
+│                          #   publish_pending, collect
+├── Actors/
+│   ├── Publisher           # Every 60s: flushes pending events to AMQP
+│   └── Collector           # Every 300s: scans session directories for new JSONL files
+└── Transport/
+    ├── Exchanges/Sessions         # telemetry.sessions topic exchange
+    ├── Queues/SessionsProcess     # telemetry.sessions.process durable queue
+    └── Messages/TelemetryMessage  # Routing key: telemetry.{source}.{event_type}
+```
+
+## TelemetryEvent Shape
+
+```ruby
+{
+  event_type:    :tool_call,      # :tool_call, :llm_request, :error, :session_start, :session_end
+  session_id:    "uuid",
+  source:        :claude_code,    # :claude_code, :codex, :legion_chat, etc.
+  timestamp:     Time,
+  tool_name:     "Read",          # nil for non-tool events
+  tool_input:    { file_path: "/path/to/file.rb" },  # scrubbed
+  duration_ms:   1000,            # computed from tool_use -> tool_result gap
+  tokens:        { input: 100, output: 50, cache_read: 200, cache_write: 0 },
+  error:         nil,
+  metadata:      {}
+}
+```
+
+## Scrub Levels
+
+| Level | Behavior |
+|-------|----------|
+| `:minimal` | PII regex only, keeps all tool inputs |
+| `:standard` (default) | Whitelist per tool + PII regex |
+| `:paranoid` | Strips everything except event_type, tool_name, timestamps, tokens |
+
+## Tool Whitelist (Standard Level)
+
+| Tool | Allowed Keys |
+|------|-------------|
+| Read | file_path, offset, limit |
+| Write | file_path |
+| Edit | file_path |
+| Glob | pattern, path |
+| Grep | pattern, path, include |
+| Bash | description, timeout |
+| Agent | description, subagent_type |
+
+Unknown tools: all input stripped at standard level.
+
+## Runner Methods
+
+| Method | Purpose |
+|--------|---------|
+| `ingest_session(file_path:, scrub_level:)` | Parse + scrub + store, returns summary |
+| `session_stats(session_id:)` | Per-session breakdown (tool counts, tokens, files, errors) |
+| `aggregate_stats` | Cross-session totals (frequencies, error rate, most-read files) |
+| `telemetry_status` | Buffer size, pending count, session count, parser list |
+| `publish_pending` | Flush pending events to AMQP (called by Publisher actor) |
+| `collect` | Scan SCAN_DIRS for new session files (called by Collector actor) |
+
+## Actors
+
+| Actor | Interval | Runner Method |
+|-------|----------|--------------|
+| Publisher | 60s | publish_pending |
+| Collector | 300s | collect |
+
+## Integration Points
+
+- **legion-transport** (optional): AMQP exchange/queue/message for telemetry publishing. If unavailable, events buffer in pending queue.
+- **LegionIO CLI**: `legion telemetry stats/ingest/status` subcommands
+- **lex-privatecore**: Scrubber carries own PII regex patterns (no hard dependency)
+
+## Design Doc
+
+`docs/work/completed/2026-03-15-session-log-analytics-design.md`
+
+## Development
+
+```bash
+bundle install
+bundle exec rspec     # 60 specs
+bundle exec rubocop   # 0 offenses
+```
+
+---
+
+**Maintained By**: Matthew Iverson (@Esity)

data/Gemfile

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+gemspec

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 LegionIO
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,43 @@
+# lex-telemetry
+
+Session log analytics pipeline for LegionIO. Ingests AI tool session logs (Claude Code JSONL),
+normalizes events, scrubs sensitive content, computes local stats, and publishes operational
+telemetry to AMQP for central consumption.
+
+## Features
+
+- **Parsers**: Claude Code JSONL (extensible to Codex, legion-chat)
+- **Scrubber**: Whitelist per tool, 3 levels (minimal/standard/paranoid), PII detection
+- **EventStore**: In-memory buffer (10k cap) with pending queue
+- **Stats**: Per-session and cross-session analytics
+- **AMQP Publishing**: Telemetry events to `telemetry.sessions` exchange
+- **Collector**: Auto-discovers session files in `~/.claude/projects/`
+
+## CLI
+
+```bash
+legion telemetry stats              # Aggregate stats across sessions
+legion telemetry stats <session_id> # Per-session breakdown
+legion telemetry ingest <path>      # Manual file ingestion
+legion telemetry status             # Buffer health and publisher state
+```
+
+## Installation
+
+Add to your Gemfile:
+
+```ruby
+gem 'lex-telemetry'
+```
+
+## Development
+
+```bash
+bundle install
+bundle exec rspec     # 60 specs
+bundle exec rubocop   # 0 offenses
+```
+
+## License
+
+MIT

data/lex-telemetry.gemspec

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'legion/extensions/telemetry/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'lex-telemetry'
+  spec.version       = Legion::Extensions::Telemetry::VERSION
+  spec.authors       = ['Esity']
+  spec.email         = ['matthewdiverson@gmail.com']
+
+  spec.summary       = 'Legion::Extensions::Telemetry'
+  spec.description   = 'Session log analytics pipeline: ingestion, normalization, scrubbing, stats, and AMQP telemetry publishing'
+  spec.homepage      = 'https://github.com/LegionIO/lex-telemetry'
+  spec.license       = 'MIT'
+  spec.required_ruby_version = '>= 3.4'
+
+  spec.metadata['homepage_uri']          = spec.homepage
+  spec.metadata['source_code_uri']       = 'https://github.com/LegionIO/lex-telemetry'
+  spec.metadata['changelog_uri']         = 'https://github.com/LegionIO/lex-telemetry/blob/main/CHANGELOG.md'
+  spec.metadata['documentation_uri']     = 'https://github.com/LegionIO/lex-telemetry'
+  spec.metadata['bug_tracker_uri']       = 'https://github.com/LegionIO/lex-telemetry/issues'
+  spec.metadata['rubygems_mfa_required'] = 'true'
+
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'rspec'
+  spec.add_development_dependency 'rubocop'
+  spec.add_development_dependency 'rubocop-rspec'
+  spec.add_development_dependency 'simplecov'
+end

data/lib/legion/extensions/telemetry/actors/collector.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Telemetry
+      module Actor
+        class Collector < Legion::Extensions::Actors::Every
+          def runner_class
+            'Legion::Extensions::Telemetry::Runners::Telemetry'
+          end
+
+          def runner_function
+            'collect'
+          end
+
+          def time
+            300
+          end
+
+          def run_now?
+            false
+          end
+
+          def use_runner?
+            false
+          end
+
+          def check_subtask?
+            false
+          end
+
+          def generate_task?
+            false
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/telemetry/actors/publisher.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Telemetry
+      module Actor
+        class Publisher < Legion::Extensions::Actors::Every
+          def runner_class
+            'Legion::Extensions::Telemetry::Runners::Telemetry'
+          end
+
+          def runner_function
+            'publish_pending'
+          end
+
+          def time
+            60
+          end
+
+          def run_now?
+            false
+          end
+
+          def use_runner?
+            false
+          end
+
+          def check_subtask?
+            false
+          end
+
+          def generate_task?
+            false
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/telemetry/helpers/event_store.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Telemetry
+      module Helpers
+        class EventStore
+          MAX_BUFFER = 10_000
+
+          attr_reader :events, :pending, :sessions
+
+          def initialize
+            @events   = []
+            @pending  = []
+            @sessions = {}
+          end
+
+          def store(event:)
+            @events.shift if @events.length >= MAX_BUFFER
+            @events << event
+            @pending << event
+
+            sid = event[:session_id]
+            @sessions[sid] ||= { first_seen: event[:timestamp], event_count: 0 }
+            @sessions[sid][:event_count] += 1
+            @sessions[sid][:last_seen] = event[:timestamp]
+          end
+
+          def flush_pending
+            flushed = @pending.dup
+            @pending.clear
+            flushed
+          end
+
+          def events_for(session_id:)
+            @events.select { |e| e[:session_id] == session_id }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/telemetry/helpers/high_water_mark.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Telemetry
+      module Helpers
+        class HighWaterMark
+          def initialize
+            @marks = {}
+            @completed = Set.new
+          end
+
+          def get(path:)
+            @marks.fetch(path, 0)
+          end
+
+          def set(path:, offset:)
+            @marks[path] = offset
+          end
+
+          def ingested?(path:)
+            @completed.include?(path)
+          end
+
+          def mark_complete(path:)
+            @completed.add(path)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/telemetry/helpers/scrubber.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Telemetry
+      module Helpers
+        module Scrubber
+          TOOL_WHITELIST = {
+            'Read'  => %i[file_path offset limit],
+            'Write' => %i[file_path],
+            'Edit'  => %i[file_path],
+            'Glob'  => %i[pattern path],
+            'Grep'  => %i[pattern path include],
+            'Bash'  => %i[description timeout],
+            'Agent' => %i[description subagent_type]
+          }.freeze
+
+          PII_PATTERNS = {
+            /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b/     => '[EMAIL]',
+            /\b\d{3}-\d{2}-\d{4}\b/                                   => '[SSN]',
+            /\b(?:\+?1[-.\s]?)?\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}\b/ => '[PHONE]'
+          }.freeze
+
+          PARANOID_KEEP = %i[event_type tool_name timestamp tokens session_id source].freeze
+
+          module_function
+
+          def scrub(event:, level: :standard)
+            scrubbed = event.dup
+
+            case level
+            when :minimal
+              scrub_pii(scrubbed)
+            when :standard
+              scrub_tool_input(scrubbed)
+              scrub_pii(scrubbed)
+            when :paranoid
+              scrub_paranoid(scrubbed)
+            end
+
+            scrubbed
+          end
+
+          def scrub_tool_input(event)
+            return unless event[:tool_input].is_a?(Hash)
+
+            whitelist = TOOL_WHITELIST[event[:tool_name]]
+            unless whitelist
+              event[:tool_input] = nil
+              return
+            end
+
+            event[:tool_input] = event[:tool_input].slice(*whitelist)
+          end
+
+          def scrub_pii(event)
+            return unless event[:tool_input].is_a?(Hash)
+
+            event[:tool_input] = event[:tool_input].transform_values { |v| scrub_pii_value(v) }
+          end
+
+          def scrub_pii_value(value)
+            return value unless value.is_a?(String)
+
+            PII_PATTERNS.each do |pattern, replacement|
+              value = value.gsub(pattern, replacement)
+            end
+            value
+          end
+
+          def scrub_paranoid(event)
+            event[:tool_input] = nil
+            event[:error] = nil
+            event[:metadata] = nil
+            event.delete_if { |k, _| !PARANOID_KEEP.include?(k) }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/telemetry/helpers/stats.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Telemetry
+      module Helpers
+        module Stats
+          module_function
+
+          def session_summary(store:, session_id:)
+            events = store.events_for(session_id: session_id)
+            tool_events = events.select { |e| e[:event_type] == :tool_call }
+            error_events = events.select { |e| e[:event_type] == :error }
+
+            tool_counts = tool_events.each_with_object(Hash.new(0)) do |e, h|
+              h[e[:tool_name]] += 1 if e[:tool_name]
+            end
+
+            tokens = sum_tokens(events)
+            files = extract_files(tool_events)
+            durations = tool_events.filter_map { |e| e[:duration_ms] }
+
+            {
+              session_id:        session_id,
+              event_count:       events.length,
+              tool_counts:       tool_counts,
+              tokens:            tokens,
+              error_count:       error_events.length,
+              unique_files:      files,
+              avg_duration_ms:   durations.empty? ? 0 : (durations.sum.to_f / durations.length).round,
+              total_duration_ms: durations.sum
+            }
+          end
+
+          def aggregate_stats(store:)
+            tool_frequency = Hash.new(0)
+            file_frequency = Hash.new(0)
+            total_tokens = { input: 0, output: 0, cache_read: 0, cache_write: 0 }
+
+            store.events.each do |e|
+              tool_frequency[e[:tool_name]] += 1 if e[:event_type] == :tool_call && e[:tool_name]
+
+              total_tokens.each_key { |k| total_tokens[k] += e[:tokens][k] || 0 } if e[:tokens].is_a?(Hash)
+
+              file_frequency[e[:tool_input][:file_path]] += 1 if e[:tool_input].is_a?(Hash) && e[:tool_input][:file_path]
+            end
+
+            error_count = store.events.count { |e| e[:event_type] == :error }
+
+            {
+              session_count:   store.sessions.length,
+              total_events:    store.events.length,
+              tool_frequency:  tool_frequency,
+              tokens:          total_tokens,
+              error_count:     error_count,
+              error_rate:      store.events.empty? ? 0.0 : (error_count.to_f / store.events.length).round(4),
+              most_read_files: file_frequency.sort_by { |_, v| -v }.first(10).to_h
+            }
+          end
+
+          def sum_tokens(events)
+            totals = { input: 0, output: 0, cache_read: 0, cache_write: 0 }
+            events.each do |e|
+              next unless e[:tokens].is_a?(Hash)
+
+              totals.each_key { |k| totals[k] += e[:tokens][k] || 0 }
+            end
+            totals
+          end
+
+          def extract_files(tool_events)
+            tool_events.filter_map { |e| e[:tool_input]&.dig(:file_path) }.uniq
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/telemetry/helpers/telemetry_event.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Telemetry
+      module Helpers
+        module TelemetryEvent
+          VALID_EVENT_TYPES = %i[tool_call llm_request error session_start session_end].freeze
+
+          module_function
+
+          def build(event_type:, session_id:, source:, timestamp:, tool_name: nil,
+                    tool_input: nil, duration_ms: nil, tokens: nil, error: nil, metadata: nil)
+            raise ArgumentError, "invalid event_type: #{event_type}" unless VALID_EVENT_TYPES.include?(event_type)
+
+            {
+              event_type:  event_type,
+              session_id:  session_id,
+              source:      source,
+              timestamp:   timestamp,
+              tool_name:   tool_name,
+              tool_input:  tool_input,
+              duration_ms: duration_ms,
+              tokens:      tokens,
+              error:       error,
+              metadata:    metadata
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/telemetry/parsers/base.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Telemetry
+      module Parsers
+        module Base
+          def source_name
+            raise NotImplementedError
+          end
+
+          def can_parse?(_path)
+            raise NotImplementedError
+          end
+
+          def parse(_path, offset: 0, &_block)
+            raise NotImplementedError
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/telemetry/parsers/claude_code.rb

@@ -0,0 +1,150 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'time'
+
+module Legion
+  module Extensions
+    module Telemetry
+      module Parsers
+        class ClaudeCode
+          include Base
+
+          def source_name
+            :claude_code
+          end
+
+          def can_parse?(path)
+            first_line = File.open(path, &:readline)
+            data = ::JSON.parse(first_line)
+            data.is_a?(Hash) && (data.key?('sessionId') || data.key?('parentUuid'))
+          rescue StandardError
+            false
+          end
+
+          def parse(path, offset: 0, &block)
+            pending_tools = {}
+            session_id = nil
+
+            File.open(path) do |f|
+              f.seek(offset) if offset.positive?
+              first_line = offset.zero?
+
+              f.each_line do |line|
+                data = ::JSON.parse(line.strip)
+                timestamp = parse_timestamp(data['timestamp'])
+
+                if first_line && offset.zero?
+                  session_id = data['sessionId'] || data['uuid'] || SecureRandom.uuid
+                  block.call(Helpers::TelemetryEvent.build(
+                               event_type: :session_start,
+                               session_id: session_id,
+                               source:     source_name,
+                               timestamp:  timestamp,
+                               metadata:   { version: data['version'] }
+                             ))
+                  first_line = false
+                  next
+                end
+
+                session_id ||= data['sessionId'] || 'unknown'
+                msg = data['message']
+                next unless msg.is_a?(Hash)
+
+                process_assistant(msg, session_id, timestamp, pending_tools, &block) if msg['role'] == 'assistant'
+                process_tool_result(msg, session_id, timestamp, pending_tools, &block) if msg['role'] == 'user'
+              rescue ::JSON::ParserError
+                next
+              end
+
+              return f.pos
+            end
+          end
+
+          private
+
+          def process_assistant(msg, session_id, timestamp, pending_tools, &block)
+            content = msg['content']
+            usage = msg['usage']
+
+            if content.is_a?(Array)
+              content.each do |item|
+                next unless item['type'] == 'tool_use'
+
+                pending_tools[item['id']] = {
+                  tool_name:  item['name'],
+                  tool_input: symbolize_keys(item['input']),
+                  timestamp:  timestamp,
+                  tokens:     extract_tokens(usage)
+                }
+              end
+            end
+
+            return unless usage
+
+            has_tool_use = content.is_a?(Array) && content.any? { |b| b['type'] == 'tool_use' }
+            return if has_tool_use
+
+            block.call(Helpers::TelemetryEvent.build(
+                         event_type: :llm_request,
+                         session_id: session_id,
+                         source:     source_name,
+                         timestamp:  timestamp,
+                         tokens:     extract_tokens(usage)
+                       ))
+          end
+
+          def process_tool_result(msg, session_id, timestamp, pending_tools, &block)
+            content = msg['content']
+            return unless content.is_a?(Array)
+
+            content.each do |item|
+              next unless item['type'] == 'tool_result'
+
+              pending = pending_tools.delete(item['tool_use_id'])
+              next unless pending
+
+              duration_ms = ((timestamp - pending[:timestamp]) * 1000).to_i if timestamp && pending[:timestamp]
+
+              block.call(Helpers::TelemetryEvent.build(
+                           event_type:  :tool_call,
+                           session_id:  session_id,
+                           source:      source_name,
+                           timestamp:   pending[:timestamp],
+                           tool_name:   pending[:tool_name],
+                           tool_input:  pending[:tool_input],
+                           duration_ms: duration_ms,
+                           tokens:      pending[:tokens]
+                         ))
+            end
+          end
+
+          def extract_tokens(usage)
+            return nil unless usage.is_a?(Hash)
+
+            {
+              input:       usage['input_tokens'] || 0,
+              output:      usage['output_tokens'] || 0,
+              cache_read:  usage['cache_read_input_tokens'] || 0,
+              cache_write: usage['cache_creation_input_tokens'] || 0
+            }
+          end
+
+          def parse_timestamp(str)
+            return nil unless str
+
+            Time.parse(str)
+          rescue ArgumentError
+            nil
+          end
+
+          def symbolize_keys(hash)
+            return nil unless hash.is_a?(Hash)
+
+            hash.transform_keys(&:to_sym)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/telemetry/runners/telemetry.rb

@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Telemetry
+      module Runners
+        module Telemetry
+          module_function
+
+          SCAN_DIRS = [
+            File.expand_path('~/.claude/projects')
+          ].freeze
+
+          def event_store
+            @event_store ||= Helpers::EventStore.new
+          end
+
+          def parsers
+            @parsers ||= [Parsers::ClaudeCode.new]
+          end
+
+          def ingest_session(file_path:, scrub_level: :standard, **_opts)
+            parser = parsers.find { |p| p.can_parse?(file_path) }
+            return { success: false, error: 'no parser found' } unless parser
+
+            session_id = nil
+            count = 0
+
+            parser.parse(file_path) do |event|
+              scrubbed = Helpers::Scrubber.scrub(event: event, level: scrub_level)
+              event_store.store(event: scrubbed)
+              session_id ||= scrubbed[:session_id]
+              count += 1
+            end
+
+            { success: true, session_id: session_id, event_count: count, file_path: file_path }
+          rescue StandardError => e
+            { success: false, error: e.message }
+          end
+
+          def session_stats(session_id:, **_opts)
+            events = event_store.events_for(session_id: session_id)
+            return { success: false, error: 'session not found' } if events.empty?
+
+            stats = Helpers::Stats.session_summary(store: event_store, session_id: session_id)
+            { success: true, stats: stats }
+          end
+
+          def aggregate_stats(**_opts)
+            stats = Helpers::Stats.aggregate_stats(store: event_store)
+            { success: true, stats: stats }
+          end
+
+          def telemetry_status(**_opts)
+            {
+              success:       true,
+              buffer_size:   event_store.events.length,
+              pending_count: event_store.pending.length,
+              session_count: event_store.sessions.length,
+              parsers:       parsers.map(&:source_name)
+            }
+          end
+
+          def publish_pending(**_opts)
+            events = event_store.flush_pending
+            return { success: true, published: 0 } if events.empty?
+
+            published = 0
+            events.each do |event|
+              if defined?(Legion::Extensions::Telemetry::Transport::Messages::TelemetryMessage)
+                routing_key = "telemetry.#{event[:source]}.#{event[:event_type]}"
+                Transport::Messages::TelemetryMessage.new.publish(event, routing_key: routing_key)
+                published += 1
+              end
+            rescue StandardError
+              event_store.pending.push(event)
+            end
+
+            { success: true, published: published, remaining: event_store.pending.length }
+          rescue StandardError => e
+            { success: false, error: e.message }
+          end
+
+          def high_water_mark
+            @high_water_mark ||= Helpers::HighWaterMark.new
+          end
+
+          def collect(**_opts)
+            files_processed = 0
+            events_ingested = 0
+
+            SCAN_DIRS.each do |dir|
+              next unless Dir.exist?(dir)
+
+              Dir.glob(File.join(dir, '**', '*.jsonl')).each do |path|
+                next if high_water_mark.ingested?(path: path)
+
+                current_size = File.size(path)
+                last_offset = high_water_mark.get(path: path)
+                next if current_size <= last_offset
+
+                parser = parsers.find { |p| p.can_parse?(path) }
+                next unless parser
+
+                count = 0
+                new_offset = parser.parse(path, offset: last_offset) do |event|
+                  scrubbed = Helpers::Scrubber.scrub(event: event, level: :standard)
+                  event_store.store(event: scrubbed)
+                  count += 1
+                end
+
+                high_water_mark.set(path: path, offset: new_offset)
+                files_processed += 1
+                events_ingested += count
+              rescue StandardError
+                next
+              end
+            end
+
+            { success: true, files_processed: files_processed, events_ingested: events_ingested }
+          rescue StandardError => e
+            { success: false, error: e.message }
+          end
+
+          def reset!
+            @event_store = nil
+            @parsers = nil
+            @high_water_mark = nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/telemetry/transport/exchanges/sessions.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Telemetry
+      module Transport
+        module Exchanges
+          class Sessions < Legion::Transport::Exchange
+            def exchange_name = 'telemetry.sessions'
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/telemetry/transport/messages/telemetry_message.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Telemetry
+      module Transport
+        module Messages
+          class TelemetryMessage < Legion::Transport::Message
+            def routing_key = 'telemetry.sessions.process'
+            def exchange    = Legion::Extensions::Telemetry::Transport::Exchanges::Sessions
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/telemetry/transport/queues/sessions_process.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Telemetry
+      module Transport
+        module Queues
+          class SessionsProcess < Legion::Transport::Queue
+            def queue_name = 'telemetry.sessions.process'
+            def queue_options = { auto_delete: false }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/telemetry/version.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Telemetry
+      VERSION = '0.1.1'
+    end
+  end
+end

data/lib/legion/extensions/telemetry.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require 'legion/extensions/telemetry/version'
+require 'legion/extensions/telemetry/helpers/telemetry_event'
+require 'legion/extensions/telemetry/helpers/scrubber'
+require 'legion/extensions/telemetry/parsers/base'
+require 'legion/extensions/telemetry/parsers/claude_code'
+require 'legion/extensions/telemetry/helpers/event_store'
+require 'legion/extensions/telemetry/helpers/stats'
+require 'legion/extensions/telemetry/helpers/high_water_mark'
+require 'legion/extensions/telemetry/runners/telemetry'
+
+module Legion
+  module Extensions
+    module Telemetry
+      extend Legion::Extensions::Core if Legion::Extensions.const_defined? :Core
+    end
+  end
+end

metadata

@@ -0,0 +1,142 @@
+--- !ruby/object:Gem::Specification
+name: lex-telemetry
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- Esity
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 'Session log analytics pipeline: ingestion, normalization, scrubbing,
+  stats, and AMQP telemetry publishing'
+email:
+- matthewdiverson@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".github/workflows/ci.yml"
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- CHANGELOG.md
+- CLAUDE.md
+- Gemfile
+- LICENSE
+- README.md
+- lex-telemetry.gemspec
+- lib/legion/extensions/telemetry.rb
+- lib/legion/extensions/telemetry/actors/collector.rb
+- lib/legion/extensions/telemetry/actors/publisher.rb
+- lib/legion/extensions/telemetry/helpers/event_store.rb
+- lib/legion/extensions/telemetry/helpers/high_water_mark.rb
+- lib/legion/extensions/telemetry/helpers/scrubber.rb
+- lib/legion/extensions/telemetry/helpers/stats.rb
+- lib/legion/extensions/telemetry/helpers/telemetry_event.rb
+- lib/legion/extensions/telemetry/parsers/base.rb
+- lib/legion/extensions/telemetry/parsers/claude_code.rb
+- lib/legion/extensions/telemetry/runners/telemetry.rb
+- lib/legion/extensions/telemetry/transport/exchanges/sessions.rb
+- lib/legion/extensions/telemetry/transport/messages/telemetry_message.rb
+- lib/legion/extensions/telemetry/transport/queues/sessions_process.rb
+- lib/legion/extensions/telemetry/version.rb
+homepage: https://github.com/LegionIO/lex-telemetry
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/LegionIO/lex-telemetry
+  source_code_uri: https://github.com/LegionIO/lex-telemetry
+  changelog_uri: https://github.com/LegionIO/lex-telemetry/blob/main/CHANGELOG.md
+  documentation_uri: https://github.com/LegionIO/lex-telemetry
+  bug_tracker_uri: https://github.com/LegionIO/lex-telemetry/issues
+  rubygems_mfa_required: 'true'
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.4'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.9
+specification_version: 4
+summary: Legion::Extensions::Telemetry
+test_files: []