claude_memory 0.2.0 → 0.3.0

data/lib/claude_memory/resolve/resolver.rb

@@ -20,15 +20,19 @@ module ClaudeMemory
           provenance_created: 0
         }
 
-        entity_ids = resolve_entities(extraction.entities)
-        result[:entities_created] = entity_ids.size
-
-        extraction.facts.each do |fact_data|
-          outcome = resolve_fact(fact_data, entity_ids, content_item_id, occurred_at)
-          result[:facts_created] += outcome[:created]
-          result[:facts_superseded] += outcome[:superseded]
-          result[:conflicts_created] += outcome[:conflicts]
-          result[:provenance_created] += outcome[:provenance]
+        # Wrap entire extraction in a single transaction for better concurrency
+        # This reduces database lock time compared to per-fact transactions
+        @store.db.transaction do
+          entity_ids = resolve_entities(extraction.entities)
+          result[:entities_created] = entity_ids.size
+
+          extraction.facts.each do |fact_data|
+            outcome = resolve_fact(fact_data, entity_ids, content_item_id, occurred_at)
+            result[:facts_created] += outcome[:created]
+            result[:facts_superseded] += outcome[:superseded]
+            result[:conflicts_created] += outcome[:conflicts]
+            result[:provenance_created] += outcome[:provenance]
+          end
         end
 
         result
@@ -57,51 +61,50 @@ module ClaudeMemory
 
         existing_facts = @store.facts_for_slot(subject_id, predicate)
 
-        # Wrap all database operations in a transaction for atomicity
-        @store.db.transaction do
-          if PredicatePolicy.single?(predicate) && existing_facts.any?
-            matching = existing_facts.find { |f| values_match?(f, object_val, object_entity_id) }
-            if matching
-              add_provenance(matching[:id], content_item_id, fact_data)
-              outcome[:provenance] = 1
-              return outcome
-            elsif supersession_signal?(fact_data)
-              supersede_facts(existing_facts, occurred_at)
-              outcome[:superseded] = existing_facts.size
-            else
-              create_conflict(existing_facts.first[:id], fact_data, subject_id, content_item_id, occurred_at)
-              outcome[:conflicts] = 1
-              return outcome
-            end
+        # No transaction wrapper needed - handled by apply method
+        # This allows all facts to be processed in a single transaction
+        if PredicatePolicy.single?(predicate) && existing_facts.any?
+          matching = existing_facts.find { |f| values_match?(f, object_val, object_entity_id) }
+          if matching
+            add_provenance(matching[:id], content_item_id, fact_data)
+            outcome[:provenance] = 1
+            return outcome
+          elsif supersession_signal?(fact_data)
+            supersede_facts(existing_facts, occurred_at)
+            outcome[:superseded] = existing_facts.size
+          else
+            create_conflict(existing_facts.first[:id], fact_data, subject_id, content_item_id, occurred_at)
+            outcome[:conflicts] = 1
+            return outcome
           end
+        end
 
-          fact_scope = fact_data[:scope_hint] || @current_scope
-          fact_project = (fact_scope == "global") ? nil : @current_project_path
-
-          fact_id = @store.insert_fact(
-            subject_entity_id: subject_id,
-            predicate: predicate,
-            object_entity_id: object_entity_id,
-            object_literal: object_val,
-            polarity: fact_data[:polarity] || "positive",
-            confidence: fact_data[:confidence] || 1.0,
-            valid_from: occurred_at,
-            scope: fact_scope,
-            project_path: fact_project
-          )
-          outcome[:created] = 1
-
-          if existing_facts.any? && outcome[:superseded] > 0
-            existing_facts.each do |old_fact|
-              @store.insert_fact_link(from_fact_id: fact_id, to_fact_id: old_fact[:id], link_type: "supersedes")
-            end
-          end
+        fact_scope = fact_data[:scope_hint] || @current_scope
+        fact_project = (fact_scope == "global") ? nil : @current_project_path
 
-          add_provenance(fact_id, content_item_id, fact_data)
-          outcome[:provenance] = 1
+        fact_id = @store.insert_fact(
+          subject_entity_id: subject_id,
+          predicate: predicate,
+          object_entity_id: object_entity_id,
+          object_literal: object_val,
+          polarity: fact_data[:polarity] || "positive",
+          confidence: fact_data[:confidence] || 1.0,
+          valid_from: occurred_at,
+          scope: fact_scope,
+          project_path: fact_project
+        )
+        outcome[:created] = 1
 
-          outcome
+        if existing_facts.any? && outcome[:superseded] > 0
+          existing_facts.each do |old_fact|
+            @store.insert_fact_link(from_fact_id: fact_id, to_fact_id: old_fact[:id], link_type: "supersedes")
+          end
         end
+
+        add_provenance(fact_id, content_item_id, fact_data)
+        outcome[:provenance] = 1
+
+        outcome
       end
 
       def supersession_signal?(fact_data)

data/lib/claude_memory/store/sqlite_store.rb

@@ -1,25 +1,63 @@
 # frozen_string_literal: true
 
 require "sequel"
+require "sequel/extensions/migration"
 require "json"
+require "extralite"
+require "sequel/adapters/extralite"
 
 module ClaudeMemory
   module Store
     class SQLiteStore
-      SCHEMA_VERSION = 2
+      SCHEMA_VERSION = 7
 
       attr_reader :db
 
       def initialize(db_path)
         @db_path = db_path
-        @db = Sequel.sqlite(db_path)
+        @db = connect_database(db_path)
+
+        configure_pragmas
+
         ensure_schema!
       end
 
+      private
+
+      def connect_database(db_path)
+        # Extralite adapter: Better performance and GVL release
+        Sequel.connect("extralite:#{db_path}")
+      end
+
+      def configure_pragmas
+        # Enable WAL mode for better concurrency
+        # - Multiple readers don't block each other
+        # - Writers don't block readers
+        # - Safer concurrent hook execution
+        @db.run("PRAGMA journal_mode = WAL")
+        @db.run("PRAGMA synchronous = NORMAL")
+
+        # Set busy timeout to 30 seconds (increased from 5s)
+        # - Allows much longer wait times before raising BusyException
+        # - Critical for concurrent hook execution with MCP server
+        # - Combined with ingester retry logic, provides ~5 minutes total wait
+        # - Extralite releases GVL for better threading performance
+        @db.run("PRAGMA busy_timeout = 30000")
+      end
+
+      public
+
       def close
         @db.disconnect
       end
 
+      # Checkpoint the WAL file to prevent unlimited growth
+      # This truncates the WAL after checkpointing
+      # Should be called periodically during maintenance/sweep operations
+      def checkpoint_wal
+        @db.run("PRAGMA wal_checkpoint(TRUNCATE)")
+      end
+
       def schema_version
         @db[:meta].where(key: "schema_version").get(:value)&.to_i
       end
@@ -56,155 +94,25 @@ module ClaudeMemory
         @db[:conflicts]
       end
 
-      private
-
-      def ensure_schema!
-        create_tables!
-        run_migrations!
-        set_meta("schema_version", SCHEMA_VERSION.to_s)
-        set_meta("created_at", Time.now.utc.iso8601) unless get_meta("created_at")
+      def tool_calls
+        @db[:tool_calls]
       end
 
-      def run_migrations!
-        current = get_meta("schema_version")&.to_i || 0
-
-        migrate_to_v2! if current < 2
+      def operation_progress
+        @db[:operation_progress]
       end
 
-      def migrate_to_v2!
-        columns = @db.schema(:content_items).map(&:first)
-        unless columns.include?(:project_path)
-          @db.alter_table(:content_items) do
-            add_column :project_path, String
-          end
-        end
-
-        columns = @db.schema(:facts).map(&:first)
-        unless columns.include?(:scope)
-          @db.alter_table(:facts) do
-            add_column :scope, String, default: "project"
-            add_column :project_path, String
-            add_index :scope, name: :idx_facts_scope
-            add_index :project_path, name: :idx_facts_project
-          end
-        end
+      def schema_health
+        @db[:schema_health]
       end
 
-      def create_tables!
-        @db.create_table?(:meta) do
-          String :key, primary_key: true
-          String :value
-        end
-
-        @db.create_table?(:content_items) do
-          primary_key :id
-          String :source, null: false
-          String :session_id
-          String :transcript_path
-          String :project_path
-          String :occurred_at
-          String :ingested_at, null: false
-          String :text_hash, null: false
-          Integer :byte_len, null: false
-          String :raw_text, text: true
-          String :metadata_json, text: true
-        end
-
-        @db.create_table?(:delta_cursors) do
-          primary_key :id
-          String :session_id, null: false
-          String :transcript_path, null: false
-          Integer :last_byte_offset, null: false, default: 0
-          String :updated_at, null: false
-          unique [:session_id, :transcript_path]
-        end
-
-        @db.create_table?(:entities) do
-          primary_key :id
-          String :type, null: false
-          String :canonical_name, null: false
-          String :slug, null: false, unique: true
-          String :created_at, null: false
-        end
-
-        @db.create_table?(:entity_aliases) do
-          primary_key :id
-          foreign_key :entity_id, :entities, null: false
-          String :source
-          String :alias, null: false
-          Float :confidence, default: 1.0
-        end
-
-        @db.create_table?(:facts) do
-          primary_key :id
-          foreign_key :subject_entity_id, :entities
-          String :predicate, null: false
-          foreign_key :object_entity_id, :entities
-          String :object_literal
-          String :datatype
-          String :polarity, default: "positive"
-          String :valid_from
-          String :valid_to
-          String :status, default: "active"
-          Float :confidence, default: 1.0
-          String :created_from
-          String :created_at, null: false
-          String :scope, default: "project"
-          String :project_path
-        end
-
-        @db.create_table?(:provenance) do
-          primary_key :id
-          foreign_key :fact_id, :facts, null: false
-          foreign_key :content_item_id, :content_items
-          String :quote, text: true
-          foreign_key :attribution_entity_id, :entities
-          String :strength, default: "stated"
-        end
-
-        @db.create_table?(:fact_links) do
-          primary_key :id
-          foreign_key :from_fact_id, :facts, null: false
-          foreign_key :to_fact_id, :facts, null: false
-          String :link_type, null: false
-        end
-
-        @db.create_table?(:conflicts) do
-          primary_key :id
-          foreign_key :fact_a_id, :facts, null: false
-          foreign_key :fact_b_id, :facts, null: false
-          String :status, default: "open"
-          String :detected_at, null: false
-          String :notes, text: true
-        end
-
-        create_index_if_not_exists(:facts, :predicate, :idx_facts_predicate)
-        create_index_if_not_exists(:facts, :subject_entity_id, :idx_facts_subject)
-        create_index_if_not_exists(:facts, :status, :idx_facts_status)
-        create_index_if_not_exists(:facts, :scope, :idx_facts_scope)
-        create_index_if_not_exists(:facts, :project_path, :idx_facts_project)
-        create_index_if_not_exists(:provenance, :fact_id, :idx_provenance_fact)
-        create_index_if_not_exists(:entity_aliases, :entity_id, :idx_entity_aliases_entity)
-        create_index_if_not_exists(:content_items, :session_id, :idx_content_items_session)
-        create_index_if_not_exists(:content_items, :project_path, :idx_content_items_project)
-      end
-
-      def create_index_if_not_exists(table, column, name)
-        @db.run("CREATE INDEX IF NOT EXISTS #{name} ON #{table}(#{column})")
-      end
-
-      def set_meta(key, value)
-        @db[:meta].insert_conflict(target: :key, update: {value: value}).insert(key: key, value: value)
-      end
-
-      def get_meta(key)
-        @db[:meta].where(key: key).get(:value)
+      def ingestion_metrics
+        @db[:ingestion_metrics]
       end
 
-      public
-
       def upsert_content_item(source:, text_hash:, byte_len:, session_id: nil, transcript_path: nil,
-        project_path: nil, occurred_at: nil, raw_text: nil, metadata: nil)
+        project_path: nil, occurred_at: nil, raw_text: nil, metadata: nil,
+        git_branch: nil, cwd: nil, claude_version: nil, thinking_level: nil, source_mtime: nil)
         existing = content_items.where(text_hash: text_hash, session_id: session_id).get(:id)
         return existing if existing
 
@@ -219,10 +127,41 @@ module ClaudeMemory
           text_hash: text_hash,
           byte_len: byte_len,
           raw_text: raw_text,
-          metadata_json: metadata&.to_json
+          metadata_json: metadata&.to_json,
+          git_branch: git_branch,
+          cwd: cwd,
+          claude_version: claude_version,
+          thinking_level: thinking_level,
+          source_mtime: source_mtime
         )
       end
 
+      def content_item_by_transcript_and_mtime(transcript_path, mtime_iso8601)
+        content_items
+          .where(transcript_path: transcript_path, source_mtime: mtime_iso8601)
+          .first
+      end
+
+      def insert_tool_calls(content_item_id, tool_calls_data)
+        tool_calls_data.each do |tc|
+          tool_calls.insert(
+            content_item_id: content_item_id,
+            tool_name: tc[:tool_name],
+            tool_input: tc[:tool_input],
+            tool_result: tc[:tool_result],
+            is_error: tc[:is_error] || false,
+            timestamp: tc[:timestamp]
+          )
+        end
+      end
+
+      def tool_calls_for_content_item(content_item_id)
+        tool_calls
+          .where(content_item_id: content_item_id)
+          .order(:timestamp)
+          .all
+      end
+
       def get_delta_cursor(session_id, transcript_path)
         delta_cursors.where(session_id: session_id, transcript_path: transcript_path).get(:last_byte_offset)
       end
@@ -272,7 +211,7 @@ module ClaudeMemory
         )
       end
 
-      def update_fact(fact_id, status: nil, valid_to: nil, scope: nil, project_path: nil)
+      def update_fact(fact_id, status: nil, valid_to: nil, scope: nil, project_path: nil, embedding: nil)
         updates = {}
         updates[:status] = status if status
         updates[:valid_to] = valid_to if valid_to
@@ -282,12 +221,29 @@ module ClaudeMemory
           updates[:project_path] = (scope == "global") ? nil : project_path
         end
 
+        if embedding
+          updates[:embedding_json] = embedding.to_json
+        end
+
         return false if updates.empty?
 
         facts.where(id: fact_id).update(updates)
         true
       end
 
+      def update_fact_embedding(fact_id, embedding_vector)
+        facts.where(id: fact_id).update(embedding_json: embedding_vector.to_json)
+      end
+
+      def facts_with_embeddings(limit: 1000)
+        facts
+          .where(Sequel.~(embedding_json: nil))
+          .where(status: "active")
+          .select(:id, :subject_entity_id, :predicate, :object_literal, :embedding_json, :scope)
+          .limit(limit)
+          .all
+      end
+
       def facts_for_slot(subject_entity_id, predicate, status: "active")
         facts
           .where(subject_entity_id: subject_entity_id, predicate: predicate, status: status)
@@ -330,8 +286,118 @@ module ClaudeMemory
         fact_links.insert(from_fact_id: from_fact_id, to_fact_id: to_fact_id, link_type: link_type)
       end
 
+      # Record token usage metrics for a distillation operation
+      #
+      # @param content_item_id [Integer] The content item that was distilled
+      # @param input_tokens [Integer] Tokens sent to the API
+      # @param output_tokens [Integer] Tokens returned from the API
+      # @param facts_extracted [Integer] Number of facts extracted
+      # @return [Integer] The created metric record ID
+      def record_ingestion_metrics(content_item_id:, input_tokens:, output_tokens:, facts_extracted:)
+        ingestion_metrics.insert(
+          content_item_id: content_item_id,
+          input_tokens: input_tokens,
+          output_tokens: output_tokens,
+          facts_extracted: facts_extracted,
+          created_at: Time.now.utc.iso8601
+        )
+      end
+
+      # Get aggregate metrics across all distillation operations
+      #
+      # @return [Hash] Aggregated metrics with keys:
+      #   - total_input_tokens: Total tokens sent to API
+      #   - total_output_tokens: Total tokens returned from API
+      #   - total_facts_extracted: Total facts extracted
+      #   - total_operations: Number of distillation operations
+      #   - avg_facts_per_1k_input_tokens: Average efficiency metric
+      def aggregate_ingestion_metrics
+        # standard:disable Performance/Detect (Sequel DSL requires .select{}.first)
+        result = ingestion_metrics
+          .select {
+            [
+              sum(:input_tokens).as(:total_input),
+              sum(:output_tokens).as(:total_output),
+              sum(:facts_extracted).as(:total_facts),
+              count(:id).as(:total_ops)
+            ]
+          }
+          .first
+        # standard:enable Performance/Detect
+
+        return nil if result.nil? || result[:total_ops].to_i.zero?
+
+        total_input = result[:total_input].to_i
+        total_output = result[:total_output].to_i
+        total_facts = result[:total_facts].to_i
+        total_ops = result[:total_ops].to_i
+
+        efficiency = total_input.zero? ? 0.0 : (total_facts.to_f / total_input * 1000).round(2)
+
+        {
+          total_input_tokens: total_input,
+          total_output_tokens: total_output,
+          total_facts_extracted: total_facts,
+          total_operations: total_ops,
+          avg_facts_per_1k_input_tokens: efficiency
+        }
+      end
+
       private
 
+      def ensure_schema!
+        migrations_path = File.expand_path("../../../db/migrations", __dir__)
+
+        # Handle backward compatibility: databases created with old migration system
+        sync_legacy_schema_version!
+
+        # Run Sequel migrations to bring database to target version
+        Sequel::Migrator.run(@db, migrations_path, target: SCHEMA_VERSION)
+
+        # Set created_at timestamp on first initialization
+        set_meta("created_at", Time.now.utc.iso8601) unless get_meta("created_at")
+
+        # Sync legacy schema_version meta key with Sequel's schema_info
+        # This maintains backwards compatibility with code that reads schema_version
+        sequel_version = @db[:schema_info].get(:version) if @db.table_exists?(:schema_info)
+        set_meta("schema_version", sequel_version.to_s) if sequel_version
+      end
+
+      # Sync legacy schema_version from meta table to Sequel's schema_info
+      # Handles two cases:
+      # 1. No schema_info table exists (old system, pre-Sequel migrations)
+      # 2. schema_info exists but is out of sync with meta.schema_version
+      def sync_legacy_schema_version!
+        return unless @db.table_exists?(:meta)
+
+        meta_version = get_meta("schema_version")&.to_i
+        return unless meta_version && meta_version >= 2
+
+        # Verify database actually has v2+ schema (defensive check)
+        columns = @db.schema(:content_items).map(&:first) if @db.table_exists?(:content_items)
+        return unless columns&.include?(:project_path)
+
+        # Create or update schema_info to match meta.schema_version
+        @db.create_table?(:schema_info) do
+          Integer :version, null: false, default: 0
+        end
+
+        sequel_version = @db[:schema_info].get(:version)
+        if sequel_version.nil? || sequel_version < meta_version
+          # Update schema_info to match meta (old system's version)
+          @db[:schema_info].delete
+          @db[:schema_info].insert(version: meta_version)
+        end
+      end
+
+      def set_meta(key, value)
+        @db[:meta].insert_conflict(target: :key, update: {value: value}).insert(key: key, value: value)
+      end
+
+      def get_meta(key)
+        @db[:meta].where(key: key).get(:value)
+      end
+
       def slugify(type, name)
         "#{type}:#{name.downcase.gsub(/[^a-z0-9]+/, "_").gsub(/^_|_$/, "")}"
       end

data/lib/claude_memory/store/store_manager.rb

@@ -8,21 +8,21 @@ module ClaudeMemory
       attr_reader :global_store, :project_store, :project_path
 
       def initialize(global_db_path: nil, project_db_path: nil, project_path: nil, env: ENV)
-        @project_path = project_path || env["CLAUDE_PROJECT_DIR"] || Dir.pwd
-        @global_db_path = global_db_path || self.class.default_global_db_path(env)
-        @project_db_path = project_db_path || self.class.default_project_db_path(@project_path)
+        config = Configuration.new(env)
+        @project_path = project_path || config.project_dir
+        @global_db_path = global_db_path || config.global_db_path
+        @project_db_path = project_db_path || config.project_db_path(@project_path)
 
         @global_store = nil
         @project_store = nil
       end
 
       def self.default_global_db_path(env = ENV)
-        home = env["HOME"] || File.expand_path("~")
-        File.join(home, ".claude", "memory.sqlite3")
+        Configuration.new(env).global_db_path
       end
 
       def self.default_project_db_path(project_path = Dir.pwd)
-        File.join(project_path, ".claude", "memory.sqlite3")
+        Configuration.new.project_db_path(project_path)
       end
 
       def ensure_global!

data/lib/claude_memory/sweep/sweeper.rb

@@ -31,6 +31,7 @@ module ClaudeMemory
         expire_disputed_facts if within_budget?
         prune_orphaned_provenance if within_budget?
         prune_old_content if within_budget?
+        checkpoint_wal if within_budget?
 
         @stats[:elapsed_seconds] = Time.now - @start_time
         @stats[:budget_honored] = @stats[:elapsed_seconds] <= budget
@@ -75,6 +76,11 @@ module ClaudeMemory
           .exclude(id: referenced_ids)
           .delete
       end
+
+      def checkpoint_wal
+        @store.checkpoint_wal
+        @stats[:wal_checkpointed] = true
+      end
     end
   end
 end

data/lib/claude_memory/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ClaudeMemory
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end