phrasekit 0.2.0

data/lib/phrasekit/tagger.rb

@@ -0,0 +1,100 @@
+require "shellwords"
+
+module PhraseKit
+  class Tagger
+    class Error < StandardError; end
+
+    class << self
+      def tag(
+        input_path:,
+        output_path:,
+        artifacts_dir: nil,
+        automaton_path: nil,
+        payloads_path: nil,
+        manifest_path: nil,
+        vocab_path: nil,
+        policy: :leftmost_longest,
+        max_spans: 100,
+        label: "PHRASE",
+        config_path: nil
+      )
+        binary_path = find_binary
+
+        if config_path.nil?
+          require "tempfile"
+          require "json"
+
+          if artifacts_dir
+            automaton_path ||= File.join(artifacts_dir, "phrases.daac")
+            payloads_path ||= File.join(artifacts_dir, "payloads.bin")
+            manifest_path ||= File.join(artifacts_dir, "manifest.json")
+            vocab_path ||= File.join(artifacts_dir, "vocab.json")
+          end
+
+          unless automaton_path && payloads_path && manifest_path && vocab_path
+            raise Error, "Must provide either artifacts_dir or all artifact paths"
+          end
+
+          config_file = Tempfile.new(["tag_config", ".json"])
+          config_file.write(JSON.generate({
+            automaton_path: automaton_path.to_s,
+            payloads_path: payloads_path.to_s,
+            manifest_path: manifest_path.to_s,
+            vocab_path: vocab_path.to_s,
+            policy: policy.to_s,
+            max_spans: max_spans,
+            label: label.to_s
+          }))
+          config_file.flush
+          config_path = config_file.path
+        end
+
+        cmd = [
+          binary_path,
+          input_path.to_s,
+          config_path.to_s,
+          output_path.to_s
+        ]
+        output = `#{cmd.shelljoin} 2>&1`
+
+        unless $?.success?
+          config_file.close! if config_file
+          raise Error, "Tagging failed: #{output}"
+        end
+
+        config_file.close! if config_file
+
+        parse_stats(output)
+      end
+
+      private
+
+      def find_binary
+        base_dir = File.expand_path("../..", __dir__)
+
+        candidates = [
+          File.join(base_dir, "ext/phrasekit/target/release/phrasekit_tag"),
+          File.join(base_dir, "ext/phrasekit/target/debug/phrasekit_tag"),
+          File.join(base_dir, "bin/phrasekit_tag")
+        ]
+
+        candidates.each do |binary|
+          return binary if File.exist?(binary) && File.executable?(binary)
+        end
+
+        raise Error, "phrasekit_tag binary not found. Run: cargo build --release --bin phrasekit_tag --manifest-path ext/phrasekit/Cargo.toml"
+      end
+
+      def parse_stats(output)
+        stats = {}
+
+        output.scan(/Documents:\s+(\d+)/) { stats[:documents] = $1.to_i }
+        output.scan(/Total spans:\s+(\d+)/) { stats[:total_spans] = $1.to_i }
+        output.scan(/Documents with spans:\s+(\d+)/) { stats[:docs_with_spans] = $1.to_i }
+        output.scan(/Avg spans per document:\s+([\d.]+)/) { stats[:avg_spans_per_doc] = $1.to_f }
+
+        stats
+      end
+    end
+  end
+end

data/lib/phrasekit/version.rb

@@ -0,0 +1,3 @@
+module PhraseKit
+  VERSION = "0.2.0"
+end

data/lib/phrasekit.rb

@@ -0,0 +1,100 @@
+require "phrasekit/version"
+
+# Load the compiled Rust extension. Precompiled (platform) gems install it into a
+# Ruby-ABI-versioned subdir (lib/phrasekit/<major.minor>/phrasekit.{so,bundle}) so a
+# single fat gem can carry a binary per Ruby version; source/dev builds place it flat
+# at lib/phrasekit/phrasekit.{so,bundle}. Try the versioned path first, fall back to
+# the flat one. Resolution goes through $LOAD_PATH (`require`, never `require_relative`)
+# because RubyGems installs native extensions outside the gem's lib/ dir.
+begin
+  RUBY_VERSION =~ /(\d+\.\d+)/
+  require "phrasekit/#{Regexp.last_match(1)}/phrasekit"
+rescue LoadError
+  require "phrasekit/phrasekit"
+end
+
+require "phrasekit/miner"
+require "phrasekit/scorer"
+require "phrasekit/tagger"
+
+module PhraseKit
+  class Error < StandardError; end
+
+  class << self
+    attr_reader :vocabulary
+
+    def load!(automaton_path:, payloads_path:, manifest_path:, vocab_path: nil)
+      @matcher = NativeMatcher.new
+      begin
+        @matcher.load(automaton_path.to_s, payloads_path.to_s, manifest_path.to_s)
+      rescue RuntimeError => e
+        raise Error, e.message
+      end
+
+      if vocab_path
+        begin
+          require "json"
+          vocab_data = JSON.parse(File.read(vocab_path))
+          @vocabulary = {
+            tokens: vocab_data["tokens"],
+            special_tokens: vocab_data["special_tokens"],
+            separator_id: vocab_data["separator_id"]
+          }
+        rescue => e
+          raise Error, "Failed to load vocabulary: #{e.message}"
+        end
+      else
+        @vocabulary = nil
+      end
+    end
+
+    def match_tokens(token_ids:, policy: :leftmost_longest, max: 32)
+      raise Error, "PhraseKit not loaded. Call PhraseKit.load! first" unless @matcher
+      @matcher.match_tokens(token_ids, policy.to_s, max).map(&:symbolize_keys)
+    end
+
+    def encode_tokens(tokens)
+      raise Error, "Vocabulary not loaded. Call PhraseKit.load! with vocab_path" unless @vocabulary
+
+      unk_id = @vocabulary[:special_tokens]["<UNK>"]
+      tokens.map do |token|
+        normalized = token.to_s.downcase
+        @vocabulary[:tokens][normalized] || unk_id
+      end
+    end
+
+    def match_text_tokens(tokens:, policy: :leftmost_longest, max: 32)
+      raise Error, "PhraseKit not loaded. Call PhraseKit.load! first" unless @matcher
+      raise Error, "Vocabulary not loaded. Call PhraseKit.load! with vocab_path" unless @vocabulary
+
+      token_ids = encode_tokens(tokens)
+      match_tokens(token_ids: token_ids, policy: policy, max: max)
+    end
+
+    def stats
+      raise Error, "PhraseKit not loaded. Call PhraseKit.load! first" unless @matcher
+      begin
+        stats_hash = @matcher.stats.symbolize_keys
+        stats_hash[:loaded_at] = Time.at(stats_hash[:loaded_at] / 1000.0)
+        stats_hash
+      rescue RuntimeError => e
+        raise Error, e.message
+      end
+    end
+
+    def healthcheck
+      raise Error, "PhraseKit not loaded. Call PhraseKit.load! first" unless @matcher
+      begin
+        @matcher.healthcheck
+      rescue RuntimeError => e
+        raise Error, e.message
+      end
+    end
+  end
+end
+
+class Hash
+  def symbolize_keys
+    transform_keys { |key| key.to_sym rescue key }
+  end unless method_defined?(:symbolize_keys)
+end

data/lib/spellkit_stub.rb

@@ -0,0 +1,80 @@
+# SpellKit stub for integration example
+# This will be replaced by the actual spellkit gem (version 0.1.1+)
+
+module SpellKit
+  class Error < StandardError; end
+
+  class << self
+    attr_reader :stats
+
+    def load!(dictionary:, edit_distance: 1, frequency_threshold: 0, protected_terms: nil, skip_patterns: {})
+      @loaded = true
+      @edit_distance = edit_distance
+      @protected_terms = Set.new(protected_terms || %w[CDK10 IL6 IL-6 BRCA1 BRCA2 TP53 EGFR])
+      @stats = {
+        version: "spellkit-stub-0.1.1",
+        loaded_at: Time.now,
+        tokens_corrected: 0,
+        p50_us: 20,
+        p95_us: 60
+      }
+      puts "SpellKit loaded (stub implementation)"
+    end
+
+    def suggestions(term, max = 5)
+      return [] unless @loaded
+
+      # Stub suggestions
+      case term.downcase
+      when "sequnce"
+        [{"term" => "sequence", "distance" => 1, "freq" => 50000}]
+      when "helllo"
+        [{"term" => "hello", "distance" => 1, "freq" => 100000}]
+      when "lyssis"
+        [{"term" => "lysis", "distance" => 1, "freq" => 12345}]
+      when "protien"
+        [{"term" => "protein", "distance" => 1, "freq" => 54321}]
+      else
+        []
+      end
+    end
+
+    def correct?(term)
+      return false unless @loaded
+
+      # Protected terms are always correct
+      return true if @protected_terms.include?(term)
+
+      # Stub: check if term is in "dictionary"
+      known_terms = %w[hello world sequence gene the with to need i lysis protein oligo rat buffer western blot]
+      known_terms.include?(term.downcase) || @protected_terms.include?(term)
+    end
+
+    def correct(term)
+      return term unless @loaded
+
+      # Protected terms never get corrected
+      return term if @protected_terms.include?(term)
+
+      # Stub corrections
+      corrections = {
+        "sequnce" => "sequence",
+        "helllo" => "hello",
+        "lyssis" => "lysis",
+        "protien" => "protein"
+      }
+
+      corrections[term.downcase] || term
+    end
+
+    def correct_tokens(tokens)
+      return tokens unless @loaded
+      tokens.map { |t| correct(t) }
+    end
+
+    def healthcheck
+      raise Error, "SpellKit not loaded. Call SpellKit.load! first" unless @loaded
+      true
+    end
+  end
+end

metadata

@@ -0,0 +1,147 @@
+--- !ruby/object:Gem::Specification
+name: phrasekit
+version: !ruby/object:Gem::Version
+  version: 0.2.0
+platform: ruby
+authors:
+- PhraseKit Contributors
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rb_sys
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rake-compiler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.22'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.22'
+- !ruby/object:Gem::Dependency
+  name: spellkit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.1
+description: High-performance phrase matching using Aho-Corasick automaton with Ruby
+  bindings via Magnus
+email:
+- ''
+executables: []
+extensions:
+- ext/phrasekit/extconf.rb
+extra_rdoc_files: []
+files:
+- README.md
+- ext/phrasekit/Cargo.toml
+- ext/phrasekit/extconf.rb
+- ext/phrasekit/src/bin/fixture_builder.rs
+- ext/phrasekit/src/bin/phrasekit_build.rs
+- ext/phrasekit/src/bin/phrasekit_mine.rs
+- ext/phrasekit/src/bin/phrasekit_score.rs
+- ext/phrasekit/src/bin/phrasekit_tag.rs
+- ext/phrasekit/src/lib.rs
+- ext/phrasekit/src/manifest.rs
+- ext/phrasekit/src/matcher.rs
+- ext/phrasekit/src/payload.rs
+- ext/phrasekit/src/policy.rs
+- lib/phrasekit.rb
+- lib/phrasekit/miner.rb
+- lib/phrasekit/scorer.rb
+- lib/phrasekit/tagger.rb
+- lib/phrasekit/version.rb
+- lib/spellkit_stub.rb
+homepage: https://github.com/scientist-labs/phrasekit
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/scientist-labs/phrasekit
+  source_code_uri: https://github.com/scientist-labs/phrasekit
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.0.0
+requirements: []
+rubygems_version: 3.6.9
+specification_version: 4
+summary: Ultra-fast deterministic phrase matcher
+test_files: []