lex-coldstart 0.1.0

data/lib/legion/extensions/coldstart/runners/ingest.rb

@@ -0,0 +1,174 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Coldstart
+      module Runners
+        module Ingest
+          include Legion::Extensions::Helpers::Lex if Legion::Extensions.const_defined?(:Helpers) &&
+                                                      Legion::Extensions::Helpers.const_defined?(:Lex)
+
+          # Ingest a single Claude memory or CLAUDE.md file into lex-memory traces.
+          # If lex-memory is not loaded, returns the parsed traces without storing.
+          #
+          # @param file_path [String] absolute path to the markdown file
+          # @param store_traces [Boolean] whether to store into lex-memory (default: true)
+          # @return [Hash] { file:, file_type:, traces_parsed:, traces_stored:, traces: }
+          def ingest_file(file_path:, store_traces: true, **)
+            unless File.exist?(file_path)
+              Legion::Logging.warn "[coldstart:ingest] file not found: #{file_path}"
+              return { file: file_path, error: 'file not found' }
+            end
+
+            candidates = Helpers::ClaudeParser.parse_file(file_path)
+            file_type = Helpers::ClaudeParser.detect_file_type(file_path)
+            Legion::Logging.info "[coldstart:ingest] parsed #{candidates.size} traces from #{file_path} (#{file_type})"
+
+            stored = store_traces ? store_candidates(candidates) : []
+
+            {
+              file:          file_path,
+              file_type:     file_type,
+              traces_parsed: candidates.size,
+              traces_stored: stored.size,
+              traces:        stored.empty? ? candidates : stored
+            }
+          end
+
+          # Ingest all CLAUDE.md and MEMORY.md files under a directory.
+          #
+          # @param dir_path [String] absolute path to the directory
+          # @param pattern [String] glob pattern (default: '**/{CLAUDE,MEMORY}.md')
+          # @param store_traces [Boolean] whether to store into lex-memory (default: true)
+          # @return [Hash] { directory:, files_found:, total_parsed:, total_stored:, files: }
+          def ingest_directory(dir_path:, pattern: '**/{CLAUDE,MEMORY}.md', store_traces: true, **)
+            unless Dir.exist?(dir_path)
+              Legion::Logging.warn "[coldstart:ingest] directory not found: #{dir_path}"
+              return { directory: dir_path, error: 'directory not found' }
+            end
+
+            candidates = Helpers::ClaudeParser.parse_directory(dir_path, pattern: pattern)
+            files = candidates.map { |c| c[:source_file] }.uniq
+            Legion::Logging.info "[coldstart:ingest] parsed #{candidates.size} traces from #{files.size} files in #{dir_path}"
+
+            stored = store_traces ? store_candidates(candidates) : []
+
+            {
+              directory:    dir_path,
+              files_found:  files.size,
+              total_parsed: candidates.size,
+              total_stored: stored.size,
+              files:        files
+            }
+          end
+
+          # Preview what traces would be created from a file without storing them.
+          #
+          # @param file_path [String] absolute path to the markdown file
+          # @return [Hash] { file:, file_type:, traces: }
+          def preview_ingest(file_path:, **)
+            return { file: file_path, error: 'file not found' } unless File.exist?(file_path)
+
+            candidates = Helpers::ClaudeParser.parse_file(file_path)
+            file_type = Helpers::ClaudeParser.detect_file_type(file_path)
+
+            {
+              file:      file_path,
+              file_type: file_type,
+              traces:    candidates
+            }
+          end
+
+          private
+
+          def store_candidates(candidates)
+            return [] unless memory_available?
+
+            imprint = imprint_active_now?
+            runner  = memory_runner
+            stored  = []
+
+            candidates.each do |candidate|
+              result = runner.store_trace(
+                type:                candidate[:trace_type],
+                content_payload:     candidate[:content_payload],
+                domain_tags:         candidate[:domain_tags],
+                origin:              candidate[:origin],
+                confidence:          candidate[:confidence],
+                imprint_active:      imprint,
+                emotional_valence:   candidate[:emotional_valence] || 0.0,
+                emotional_intensity: candidate[:emotional_intensity] || (candidate[:trace_type] == :firmware ? 0.8 : 0.3)
+              )
+              stored << result if result
+            rescue StandardError => e
+              Legion::Logging.warn "[coldstart:ingest] failed to store trace: #{e.message}"
+            end
+
+            # Flush the cache-backed store if it supports it
+            store = runner.send(:default_store)
+            store.flush if store.respond_to?(:flush)
+
+            Legion::Logging.info "[coldstart:ingest] stored #{stored.size} traces (imprint_active=#{imprint})"
+
+            # Co-activate traces from the same section to form Hebbian links
+            coactivate_section_traces(stored, candidates, runner)
+
+            stored
+          end
+
+          def memory_available?
+            Legion::Extensions.const_defined?(:Memory) &&
+              Legion::Extensions::Memory.const_defined?(:Runners) &&
+              Legion::Extensions::Memory::Runners.const_defined?(:Traces)
+          end
+
+          def memory_runner
+            @memory_runner ||= Object.new.extend(Legion::Extensions::Memory::Runners::Traces)
+          end
+
+          def imprint_active_now?
+            bootstrap.imprint_active?
+          rescue StandardError
+            false
+          end
+
+          def bootstrap
+            @bootstrap ||= Helpers::Bootstrap.new
+          end
+
+          def coactivate_section_traces(stored, candidates, runner)
+            return if stored.size < 2
+
+            store = runner.send(:default_store)
+
+            # Group stored trace IDs by their heading_slug domain tag
+            groups = {}
+            stored.each_with_index do |result, idx|
+              candidate = candidates[idx]
+              next unless candidate && result
+
+              slug = candidate[:domain_tags]&.find { |t| t.is_a?(String) && !%w[memory claude_md markdown].include?(t) && !t.include?('/') }
+              next unless slug
+
+              groups[slug] ||= []
+              groups[slug] << result[:trace_id]
+            end
+
+            coactivations = 0
+            groups.each_value do |trace_ids|
+              # Limit to first 10 traces per section to avoid O(n^2) explosion
+              trace_ids.first(10).combination(2).each do |id_a, id_b|
+                store.record_coactivation(id_a, id_b)
+                coactivations += 1
+              end
+            end
+
+            Legion::Logging.debug "[coldstart:ingest] co-activated #{coactivations} trace pairs across #{groups.size} sections"
+          rescue StandardError => e
+            Legion::Logging.warn "[coldstart:ingest] co-activation failed: #{e.message}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/coldstart/version.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Coldstart
+      VERSION = '0.1.0'
+    end
+  end
+end

data/lib/legion/extensions/coldstart.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require 'legion/extensions/coldstart/version'
+require 'legion/extensions/coldstart/helpers/imprint'
+require 'legion/extensions/coldstart/helpers/bootstrap'
+require 'legion/extensions/coldstart/helpers/claude_parser'
+require 'legion/extensions/coldstart/runners/coldstart'
+require 'legion/extensions/coldstart/runners/ingest'
+
+module Legion
+  module Extensions
+    module Coldstart
+      extend Legion::Extensions::Core if Legion::Extensions.const_defined? :Core
+    end
+  end
+end
+
+if defined?(Legion::Data::Local)
+  Legion::Data::Local.register_migrations(
+    name: :coldstart,
+    path: File.join(__dir__, 'coldstart', 'local_migrations')
+  )
+end

data/spec/fixtures/sample_claude.md

@@ -0,0 +1,25 @@
+# lex-example: Test Extension
+
+## Purpose
+
+A test extension for validating the Claude parser.
+
+## What is This?
+
+An async job processing engine built on RabbitMQ.
+
+## Development
+
+```bash
+bundle install
+bundle exec rspec
+```
+
+- Use `bundle exec` for all commands
+- Run rubocop before committing
+
+## Key Concepts
+
+- **Runner**: A function that processes a task
+- **Actor**: An execution mode (subscription, polling, interval)
+- **Extension**: A gem that plugs into the framework

data/spec/fixtures/sample_memory.md

@@ -0,0 +1,23 @@
+# Test Project Memory
+
+## Hard Rules
+- Never delete production data without confirmation
+- Always use snake_case for Ruby methods
+
+## Key Architecture Facts
+- Ruby gem ecosystem with auto-discovered extensions
+- RabbitMQ for task distribution via `legion-transport`
+- MySQL via `Sequel` ORM for persistence
+
+## CLI Gotchas
+- Thor reserves `run` as a method name
+- `::Process` must be explicit inside `Legion::` namespace
+- `::JSON` must be explicit inside `Legion::` namespace
+
+## Identity Auth Pattern
+- Digital Worker = Entra ID **Application** (service principal)
+- Dual-layer: OIDC client credentials + behavioral entropy
+
+## Project Structure
+- Workspace: `/tmp/test`
+- 10 repos total

data/spec/legion/extensions/coldstart/actors/imprint_spec.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+# Stub the framework actor base class since legionio gem is not available in test
+module Legion
+  module Extensions
+    module Actors
+      class Once # rubocop:disable Lint/EmptyClass
+      end
+    end
+  end
+end
+
+# Intercept the require in the actor file so it doesn't fail
+$LOADED_FEATURES << 'legion/extensions/actors/once'
+
+require 'legion/extensions/coldstart/actors/imprint'
+
+RSpec.describe Legion::Extensions::Coldstart::Actor::Imprint do
+  subject(:actor) { described_class.new }
+
+  describe '#runner_class' do
+    it 'returns the Coldstart runner module' do
+      expect(actor.runner_class).to eq(Legion::Extensions::Coldstart::Runners::Coldstart)
+    end
+  end
+
+  describe '#runner_function' do
+    it 'returns begin_imprint' do
+      expect(actor.runner_function).to eq('begin_imprint')
+    end
+  end
+
+  describe '#use_runner?' do
+    it 'returns false' do
+      expect(actor.use_runner?).to be false
+    end
+  end
+
+  describe '#check_subtask?' do
+    it 'returns false' do
+      expect(actor.check_subtask?).to be false
+    end
+  end
+
+  describe '#generate_task?' do
+    it 'returns false' do
+      expect(actor.generate_task?).to be false
+    end
+  end
+end

data/spec/legion/extensions/coldstart/client_spec.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require 'legion/extensions/coldstart/client'
+
+RSpec.describe Legion::Extensions::Coldstart::Client do
+  it 'responds to coldstart runner methods' do
+    client = described_class.new
+    expect(client).to respond_to(:begin_imprint)
+    expect(client).to respond_to(:record_observation)
+    expect(client).to respond_to(:coldstart_progress)
+    expect(client).to respond_to(:imprint_active?)
+    expect(client).to respond_to(:current_multiplier)
+  end
+end

data/spec/legion/extensions/coldstart/helpers/claude_parser_spec.rb

@@ -0,0 +1,205 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Legion::Extensions::Coldstart::Helpers::ClaudeParser do
+  let(:fixture_dir) { File.expand_path('../../../../fixtures', __dir__) }
+  let(:memory_path) { File.join(fixture_dir, 'sample_memory.md') }
+  let(:claude_path) { File.join(fixture_dir, 'sample_claude.md') }
+
+  describe '.detect_file_type' do
+    it 'detects MEMORY.md files' do
+      expect(described_class.detect_file_type('/some/path/MEMORY.md')).to eq(:memory)
+    end
+
+    it 'detects CLAUDE.md files' do
+      expect(described_class.detect_file_type('/some/path/CLAUDE.md')).to eq(:claude_md)
+    end
+
+    it 'returns :markdown for other files' do
+      expect(described_class.detect_file_type('/some/path/README.md')).to eq(:markdown)
+    end
+  end
+
+  describe '.classify_section' do
+    it 'maps Hard Rules to firmware' do
+      expect(described_class.classify_section('Hard Rules')).to eq(:firmware)
+    end
+
+    it 'maps architecture sections to semantic' do
+      expect(described_class.classify_section('Key Architecture Facts')).to eq(:semantic)
+    end
+
+    it 'maps gotchas to procedural' do
+      expect(described_class.classify_section('CLI Gotchas')).to eq(:procedural)
+    end
+
+    it 'maps identity sections to identity' do
+      expect(described_class.classify_section('Identity Auth Pattern')).to eq(:identity)
+    end
+
+    it 'defaults to semantic for unknown sections' do
+      expect(described_class.classify_section('Random Stuff')).to eq(:semantic)
+    end
+
+    it 'maps development sections to procedural' do
+      expect(described_class.classify_section('Development')).to eq(:procedural)
+    end
+
+    it 'maps API sections to procedural' do
+      expect(described_class.classify_section('REST API Routes')).to eq(:procedural)
+    end
+  end
+
+  describe '.split_sections' do
+    it 'splits on ## headers' do
+      content = "## First\nline1\n## Second\nline2\n"
+      sections = described_class.split_sections(content)
+      expect(sections.size).to eq(2)
+      expect(sections[0][:heading]).to eq('First')
+      expect(sections[1][:heading]).to eq('Second')
+    end
+
+    it 'creates slugs from headings' do
+      content = "## Key Architecture Facts\nstuff\n"
+      sections = described_class.split_sections(content)
+      expect(sections[0][:heading_slug]).to eq('key-architecture-facts')
+    end
+
+    it 'skips empty sections' do
+      content = "## Empty\n\n## HasContent\nreal stuff\n"
+      sections = described_class.split_sections(content)
+      expect(sections.size).to eq(1)
+      expect(sections[0][:heading]).to eq('HasContent')
+    end
+  end
+
+  describe '.extract_items' do
+    it 'extracts bullet points as individual items' do
+      body = "- first item\n- second item\n- third item\n"
+      items = described_class.extract_items(body)
+      expect(items).to eq(['first item', 'second item', 'third item'])
+    end
+
+    it 'merges continuation lines into bullets' do
+      body = "- first item\n  continued here\n- second item\n"
+      items = described_class.extract_items(body)
+      expect(items.size).to eq(2)
+      expect(items[0]).to include('continued here')
+    end
+
+    it 'keeps code blocks as single items' do
+      body = "- before\n```ruby\ncode here\nmore code\n```\n- after\n"
+      items = described_class.extract_items(body)
+      expect(items.any? { |i| i.include?('code here') }).to be true
+    end
+
+    it 'handles paragraphs' do
+      body = "This is a paragraph\nwith continuation.\n\nAnother paragraph.\n"
+      items = described_class.extract_items(body)
+      expect(items.size).to eq(2)
+    end
+  end
+
+  describe '.extract_inline_tags' do
+    it 'extracts backtick terms' do
+      tags = described_class.extract_inline_tags('Uses `legion-transport` for messaging')
+      expect(tags).to include('legion-transport')
+    end
+
+    it 'extracts bold terms' do
+      tags = described_class.extract_inline_tags('The **Runner** processes tasks')
+      expect(tags).to include('runner')
+    end
+
+    it 'rejects very short or very long tags' do
+      tags = described_class.extract_inline_tags("`x` and `a_very_long_tag_#{'x' * 60}`")
+      expect(tags).to be_empty
+    end
+  end
+
+  describe '.parse_file' do
+    it 'parses a MEMORY.md file into traces' do
+      traces = described_class.parse_file(memory_path)
+      expect(traces).to be_an(Array)
+      expect(traces).not_to be_empty
+    end
+
+    it 'assigns firmware type to Hard Rules' do
+      traces = described_class.parse_file(memory_path)
+      firmware = traces.select { |t| t[:trace_type] == :firmware }
+      expect(firmware).not_to be_empty
+      expect(firmware.first[:content_payload]).to include('production data')
+    end
+
+    it 'assigns procedural type to gotchas' do
+      traces = described_class.parse_file(memory_path)
+      procedural = traces.select { |t| t[:trace_type] == :procedural }
+      expect(procedural).not_to be_empty
+      expect(procedural.any? { |t| t[:content_payload].include?('Thor') }).to be true
+    end
+
+    it 'assigns identity type to identity sections' do
+      traces = described_class.parse_file(memory_path)
+      identity = traces.select { |t| t[:trace_type] == :identity }
+      expect(identity).not_to be_empty
+    end
+
+    it 'sets origin to :firmware for MEMORY.md files' do
+      traces = described_class.parse_file(memory_path)
+      # All traces from MEMORY.md get :firmware origin (imprint source)
+      expect(traces.all? { |t| t[:origin] == :firmware }).to be true
+    end
+
+    it 'sets origin to :direct_experience for CLAUDE.md files' do
+      traces = described_class.parse_file(claude_path)
+      expect(traces.all? { |t| t[:origin] == :direct_experience }).to be true
+    end
+
+    it 'sets confidence 1.0 for firmware traces' do
+      traces = described_class.parse_file(memory_path)
+      firmware = traces.select { |t| t[:trace_type] == :firmware }
+      expect(firmware.all? { |t| t[:confidence] == 1.0 }).to be true # rubocop:disable Lint/FloatComparison
+    end
+
+    it 'includes source_file in each trace' do
+      traces = described_class.parse_file(memory_path)
+      expect(traces.all? { |t| t[:source_file] == memory_path }).to be true
+    end
+
+    it 'includes domain tags' do
+      traces = described_class.parse_file(memory_path)
+      expect(traces.all? { |t| t[:domain_tags].is_a?(Array) && !t[:domain_tags].empty? }).to be true
+    end
+
+    it 'parses a CLAUDE.md file into traces' do
+      traces = described_class.parse_file(claude_path)
+      expect(traces).not_to be_empty
+      semantic = traces.select { |t| t[:trace_type] == :semantic }
+      expect(semantic).not_to be_empty
+    end
+  end
+
+  describe '.parse_directory' do
+    it 'finds and parses all matching files' do
+      traces = described_class.parse_directory(fixture_dir, pattern: '*.md')
+      expect(traces).not_to be_empty
+      source_files = traces.map { |t| t[:source_file] }.uniq
+      expect(source_files.size).to be >= 2
+    end
+  end
+
+  describe '.skip_path?' do
+    it 'skips _deprecated paths' do
+      expect(described_class.skip_path?('/foo/_deprecated/CLAUDE.md')).to be true
+    end
+
+    it 'skips references paths' do
+      expect(described_class.skip_path?('/foo/references/CLAUDE.md')).to be true
+    end
+
+    it 'allows normal paths' do
+      expect(described_class.skip_path?('/foo/lex-memory/CLAUDE.md')).to be false
+    end
+  end
+end

data/spec/legion/extensions/coldstart/runners/coldstart_spec.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require 'legion/extensions/coldstart/client'
+
+RSpec.describe Legion::Extensions::Coldstart::Runners::Coldstart do
+  let(:client) { Legion::Extensions::Coldstart::Client.new }
+
+  describe '#begin_imprint' do
+    it 'starts the imprint window' do
+      result = client.begin_imprint
+      expect(result[:started]).to be true
+      expect(result[:multiplier]).to eq(3.0)
+    end
+  end
+
+  describe '#record_observation' do
+    it 'increments observation count' do
+      client.begin_imprint
+      result = client.record_observation
+      expect(result[:observation_count]).to eq(1)
+    end
+
+    it 'transitions calibration state at baseline threshold' do
+      client.begin_imprint
+      50.times { client.record_observation }
+      result = client.record_observation
+      expect(result[:calibration_state]).to eq(:baseline_established)
+    end
+  end
+
+  describe '#coldstart_progress' do
+    it 'reports progress' do
+      client.begin_imprint
+      result = client.coldstart_progress
+      expect(result[:firmware_loaded]).to be true
+      expect(result[:imprint_active]).to be true
+      expect(result[:current_layer]).to eq(:imprint_window)
+    end
+  end
+
+  describe '#imprint_active?' do
+    it 'returns false before begin' do
+      result = client.imprint_active?
+      expect(result[:active]).to be false
+    end
+
+    it 'returns true after begin' do
+      client.begin_imprint
+      result = client.imprint_active?
+      expect(result[:active]).to be true
+    end
+  end
+
+  describe '#current_multiplier' do
+    it 'returns 1.0 when not imprinting' do
+      result = client.current_multiplier
+      expect(result[:multiplier]).to eq(1.0)
+    end
+
+    it 'returns 3.0 during imprint' do
+      client.begin_imprint
+      result = client.current_multiplier
+      expect(result[:multiplier]).to eq(3.0)
+    end
+  end
+end

data/spec/legion/extensions/coldstart/runners/ingest_spec.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Legion::Extensions::Coldstart::Runners::Ingest do
+  let(:runner) { Object.new.extend(described_class) }
+  let(:fixture_dir) { File.expand_path('../../../../fixtures', __dir__) }
+  let(:memory_path) { File.join(fixture_dir, 'sample_memory.md') }
+  let(:claude_path) { File.join(fixture_dir, 'sample_claude.md') }
+
+  describe '#ingest_file' do
+    it 'parses a MEMORY.md file' do
+      result = runner.ingest_file(file_path: memory_path, store_traces: false)
+      expect(result[:file]).to eq(memory_path)
+      expect(result[:file_type]).to eq(:memory)
+      expect(result[:traces_parsed]).to be > 0
+    end
+
+    it 'parses a CLAUDE.md file' do
+      result = runner.ingest_file(file_path: claude_path, store_traces: false)
+      expect(result[:file]).to eq(claude_path)
+      expect(result[:file_type]).to eq(:claude_md)
+      expect(result[:traces_parsed]).to be > 0
+    end
+
+    it 'returns error for missing file' do
+      result = runner.ingest_file(file_path: '/nonexistent/file.md')
+      expect(result[:error]).to eq('file not found')
+    end
+
+    it 'returns traces when store_traces is false' do
+      result = runner.ingest_file(file_path: memory_path, store_traces: false)
+      expect(result[:traces]).to be_an(Array)
+      expect(result[:traces]).not_to be_empty
+      expect(result[:traces].first).to have_key(:trace_type)
+    end
+
+    it 'includes firmware traces from Hard Rules' do
+      result = runner.ingest_file(file_path: memory_path, store_traces: false)
+      firmware = result[:traces].select { |t| t[:trace_type] == :firmware }
+      expect(firmware.size).to eq(2)
+    end
+  end
+
+  describe '#ingest_directory' do
+    it 'finds and parses files in a directory' do
+      result = runner.ingest_directory(dir_path: fixture_dir, pattern: '*.md', store_traces: false)
+      expect(result[:files_found]).to be >= 2
+      expect(result[:total_parsed]).to be > 0
+    end
+
+    it 'returns error for missing directory' do
+      result = runner.ingest_directory(dir_path: '/nonexistent/dir')
+      expect(result[:error]).to eq('directory not found')
+    end
+
+    it 'lists processed files' do
+      result = runner.ingest_directory(dir_path: fixture_dir, pattern: '*.md', store_traces: false)
+      expect(result[:files]).to be_an(Array)
+      expect(result[:files].any? { |f| f.include?('sample_memory') }).to be true
+    end
+  end
+
+  describe '#preview_ingest' do
+    it 'returns traces without storing' do
+      result = runner.preview_ingest(file_path: memory_path)
+      expect(result[:traces]).to be_an(Array)
+      expect(result[:traces]).not_to be_empty
+      expect(result[:file_type]).to eq(:memory)
+    end
+
+    it 'returns error for missing file' do
+      result = runner.preview_ingest(file_path: '/nonexistent/file.md')
+      expect(result[:error]).to eq('file not found')
+    end
+  end
+end