phrasekit 0.2.0-arm64-darwin

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 20b2c07b07fbbe98279a5aed095047c990c90bdf2ecb0a51ccf997b8951aca75
+  data.tar.gz: 4b857ea8548706f2df6ca2e146f9cf981d772452fc9f22bc09105c89aab96c21
+SHA512:
+  metadata.gz: af76f27a17d1356d91cb06ea45502700512eb5e55ea6c609fd10925a64afecb681a4bdec3a6b05a8fc5b35bbd4832825db9468b7ab863fc0269accceba069ca0
+  data.tar.gz: 14c7d437a0fd03d2b6981f68da3a76c3b5a6a72509ccb802582ba794bb438773d3a1aed6e2b7f6de176b1749e2e05054b1432be9f2a38f8a8bd032f75313cdf9

data/README.md

@@ -0,0 +1,131 @@
+# PhraseKit
+
+Ultra-fast deterministic phrase matching for Ruby using Rust and Aho-Corasick automaton.
+
+PhraseKit provides high-performance phrase recognition over token sequences, designed for search query understanding, NLP pipelines, and information extraction at scale.
+
+## Features
+
+- **Deterministic matching** using Double-Array Aho-Corasick (daachorse)
+- **Sub-millisecond performance** for queries with millions of phrases
+- **Hot-reloadable** artifacts with zero downtime
+- **Thread-safe** operations via Magnus/Rust
+- **Multiple matching policies**: leftmost-longest, leftmost-first, salience-max
+- **Production-ready** with health checks, stats, and observability
+
+## Installation
+
+Add to your Gemfile:
+
+```ruby
+gem 'phrasekit'
+```
+
+Or install directly:
+
+```bash
+gem install phrasekit
+```
+
+## Usage
+
+### Basic Setup
+
+```ruby
+require 'phrasekit'
+
+# Load phrase artifacts
+PhraseKit.load!(
+  automaton_path: "/path/to/phrases.daac",
+  payloads_path: "/path/to/payloads.bin",
+  manifest_path: "/path/to/phrases.json"
+)
+
+# Match tokens
+token_ids = [1012, 441, 7788, 902, 1455]  # Your tokenized input
+matches = PhraseKit.match_tokens(
+  token_ids: token_ids,
+  policy: :leftmost_longest,  # or :leftmost_first, :salience_max
+  max: 32                      # Maximum matches to return
+)
+
+# Returns array of matches:
+# [
+#   {start: 1, end: 3, phrase_id: 12345, salience: 2.13, count: 314, n: 2},
+#   {start: 3, end: 5, phrase_id: 67890, salience: 1.82, count: 271, n: 2}
+# ]
+```
+
+### Integration with SpellKit
+
+PhraseKit is designed to work with SpellKit for typo correction:
+
+```ruby
+class SearchTermExtractor
+  def call(text)
+    # 1. Tokenize
+    tokens = MyTokenizer.tokenize(text)
+
+    # 2. Spell correction (via SpellKit gem)
+    corrected = SpellKit.correct_tokens(tokens, guard: :domain)
+
+    # 3. Convert to token IDs
+    token_ids = MyTokenizer.to_ids(corrected)
+
+    # 4. Extract phrases
+    PhraseKit.match_tokens(token_ids: token_ids, policy: :leftmost_longest)
+  end
+end
+```
+
+### Monitoring
+
+```ruby
+# Check health
+PhraseKit.healthcheck  # Raises on issues
+
+# Get statistics
+PhraseKit.stats
+# => {
+#   version: "pk-2025-09-25-01",
+#   loaded_at: Time,
+#   num_patterns: 1_287_345,
+#   heap_mb: 142.3,
+#   hits_total: 892341,
+#   p50_us: 47,
+#   p95_us: 189
+# }
+```
+
+## Architecture
+
+PhraseKit uses:
+- **Rust** for core matching logic
+- **Magnus** for Ruby-Rust bindings
+- **Daachorse** for the Aho-Corasick automaton
+- **Static linking** for reliability
+
+## Performance
+
+Target performance with 1-3M phrases:
+- p50 < 100µs
+- p95 < 500µs
+- Memory < 300MB
+
+## Development
+
+```bash
+# Setup
+bundle install
+bundle exec rake compile
+
+# Run tests
+bundle exec rspec
+
+# Build gem
+gem build phrasekit.gemspec
+```
+
+## License
+
+MIT License. See LICENSE.txt for details.

data/ext/phrasekit/Cargo.toml

@@ -0,0 +1,45 @@
+[package]
+name = "phrasekit"
+version = "0.1.0"
+edition = "2021"
+
+[lib]
+crate-type = ["cdylib"]
+name = "phrasekit"
+
+[[bin]]
+name = "fixture_builder"
+path = "src/bin/fixture_builder.rs"
+
+[[bin]]
+name = "phrasekit_build"
+path = "src/bin/phrasekit_build.rs"
+
+[[bin]]
+name = "phrasekit_mine"
+path = "src/bin/phrasekit_mine.rs"
+
+[[bin]]
+name = "phrasekit_score"
+path = "src/bin/phrasekit_score.rs"
+
+[[bin]]
+name = "phrasekit_tag"
+path = "src/bin/phrasekit_tag.rs"
+
+[dependencies]
+magnus = { version = "0.7" }
+daachorse = "1.0"
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+parking_lot = "0.12"
+notify = "6.1"
+thiserror = "1.0"
+chrono = "0.4"
+
+[dependencies.rb-sys]
+version = "0.9"
+features = ["stable-api-compiled-fallback"]
+
+[dev-dependencies]
+tempfile = "3.10"

data/ext/phrasekit/extconf.rb

@@ -0,0 +1,4 @@
+require "mkmf"
+require "rb_sys/mkmf"
+
+create_rust_makefile("phrasekit/phrasekit")

data/ext/phrasekit/src/bin/fixture_builder.rs

@@ -0,0 +1,131 @@
+use daachorse::DoubleArrayAhoCorasick;
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+use std::fs::File;
+use std::path::PathBuf;
+
+#[path = "../payload.rs"]
+mod payload;
+
+#[path = "../manifest.rs"]
+mod manifest;
+
+use manifest::Manifest;
+use payload::Payload;
+
+#[derive(Debug, Serialize)]
+struct Vocabulary {
+    tokens: HashMap<String, u32>,
+    special_tokens: HashMap<String, u32>,
+    vocab_size: usize,
+    separator_id: u32,
+}
+
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let args: Vec<String> = std::env::args().collect();
+    let output_dir = if args.len() > 1 {
+        PathBuf::from(&args[1])
+    } else {
+        PathBuf::from("spec/fixtures")
+    };
+
+    std::fs::create_dir_all(&output_dir)?;
+
+    println!("Building test fixtures in: {}", output_dir.display());
+
+    // Define test patterns as byte sequences (token_id + separator)
+    let separator: u32 = 4294967294;
+
+    // Pattern 0: [100, 101] - "machine learning"
+    let pattern0 = encode_tokens(&[100, 101], separator);
+
+    // Pattern 1: [200, 101] - "deep learning"
+    let pattern1 = encode_tokens(&[200, 101], separator);
+
+    // Pattern 2: [100, 101, 102] - "machine learning algorithms"
+    let pattern2 = encode_tokens(&[100, 101, 102], separator);
+
+    let patterns = vec![pattern0, pattern1, pattern2];
+    let num_patterns = patterns.len();
+
+    // Build automaton
+    println!("Building automaton with {} patterns", num_patterns);
+    let automaton: DoubleArrayAhoCorasick<u32> = DoubleArrayAhoCorasick::new(patterns)
+        .map_err(|e| format!("Failed to build automaton: {:?}", e))?;
+
+    // Serialize automaton
+    let automaton_bytes = automaton.serialize();
+    let automaton_path = output_dir.join("phrases.daac");
+    std::fs::write(&automaton_path, &automaton_bytes)?;
+    println!("✓ Wrote automaton ({} bytes) to {}", automaton_bytes.len(), automaton_path.display());
+
+    // Create payloads
+    let payloads = vec![
+        Payload::new(100, 2.5, 150, 2),  // "machine learning" - [100, 101]
+        Payload::new(200, 2.0, 100, 2),  // "deep learning" - [200, 101]
+        Payload::new(300, 3.0, 200, 3),  // "machine learning algorithms" - [100, 101, 102]
+    ];
+
+    // Write payloads
+    let payloads_path = output_dir.join("payloads.bin");
+    let mut payloads_file = File::create(&payloads_path)?;
+    for payload in &payloads {
+        payload.write_to(&mut payloads_file)?;
+    }
+    println!("✓ Wrote {} payloads to {}", payloads.len(), payloads_path.display());
+
+    // Create manifest
+    let manifest = Manifest {
+        version: "test-v1".to_string(),
+        tokenizer: "test-tokenizer".to_string(),
+        num_patterns: num_patterns,
+        min_count: Some(10),
+        salience_threshold: Some(1.0),
+        built_at: "2025-09-25T00:00:00Z".to_string(),
+        separator_id: separator,
+    };
+
+    let manifest_path = output_dir.join("manifest.json");
+    let manifest_json = serde_json::to_string_pretty(&manifest)?;
+    std::fs::write(&manifest_path, manifest_json)?;
+    println!("✓ Wrote manifest to {}", manifest_path.display());
+
+    // Create vocabulary
+    let mut tokens = HashMap::new();
+    tokens.insert("machine".to_string(), 100);
+    tokens.insert("learning".to_string(), 101);
+    tokens.insert("algorithms".to_string(), 102);
+    tokens.insert("deep".to_string(), 200);
+
+    let mut special_tokens = HashMap::new();
+    special_tokens.insert("<UNK>".to_string(), 0);
+
+    let vocabulary = Vocabulary {
+        tokens,
+        special_tokens,
+        vocab_size: 5,
+        separator_id: separator,
+    };
+
+    let vocab_path = output_dir.join("vocab.json");
+    let vocab_json = serde_json::to_string_pretty(&vocabulary)?;
+    std::fs::write(&vocab_path, vocab_json)?;
+    println!("✓ Wrote vocabulary to {}", vocab_path.display());
+
+    println!("\n✅ Test fixtures generated successfully!");
+    println!("\nTest patterns:");
+    println!("  Pattern 0: tokens [100, 101] → phrase_id 100 (salience 2.5) - 'machine learning'");
+    println!("  Pattern 1: tokens [200, 101] → phrase_id 200 (salience 2.0) - 'deep learning'");
+    println!("  Pattern 2: tokens [100, 101, 102] → phrase_id 300 (salience 3.0) - 'machine learning algorithms'");
+
+    Ok(())
+}
+
+fn encode_tokens(tokens: &[u32], separator: u32) -> Vec<u8> {
+    let mut bytes = Vec::new();
+    for &token in tokens {
+        bytes.extend_from_slice(&token.to_le_bytes());
+        bytes.extend_from_slice(&separator.to_le_bytes());
+    }
+    bytes
+}

data/ext/phrasekit/src/bin/phrasekit_build.rs

@@ -0,0 +1,326 @@
+use daachorse::DoubleArrayAhoCorasick;
+use serde::{Deserialize, Serialize};
+use std::collections::{HashMap, HashSet};
+use std::fs::File;
+use std::io::{BufRead, BufReader, Write};
+use std::path::{Path, PathBuf};
+
+#[path = "../payload.rs"]
+mod payload;
+
+#[path = "../manifest.rs"]
+mod manifest;
+
+use manifest::Manifest;
+use payload::Payload;
+
+#[derive(Debug, Deserialize)]
+struct PhraseInput {
+    tokens: Vec<String>,
+    phrase_id: u32,
+    salience: f32,
+    #[serde(alias = "domain_count")]
+    count: u32,
+}
+
+struct ProcessedPhrase {
+    token_ids: Vec<u32>,
+    phrase_id: u32,
+    salience: f32,
+    count: u32,
+    length: u8,
+}
+
+#[derive(Debug, Deserialize)]
+struct BuildConfig {
+    version: String,
+    tokenizer: String,
+    separator_id: u32,
+    #[serde(default)]
+    min_count: Option<u32>,
+    #[serde(default)]
+    salience_threshold: Option<f32>,
+}
+
+#[derive(Debug)]
+struct BuildStats {
+    total_input: usize,
+    filtered_low_count: usize,
+    filtered_low_salience: usize,
+    duplicate_phrase_ids: usize,
+    invalid_tokens: usize,
+    built: usize,
+}
+
+#[derive(Debug, Serialize)]
+struct Vocabulary {
+    tokens: HashMap<String, u32>,
+    special_tokens: HashMap<String, u32>,
+    vocab_size: usize,
+    separator_id: u32,
+}
+
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let args: Vec<String> = std::env::args().collect();
+
+    if args.len() < 4 {
+        eprintln!("Usage: phrasekit_build <input.jsonl> <config.json> <output_dir>");
+        eprintln!("\nExample:");
+        eprintln!("  phrasekit_build phrases.jsonl config.json ./artifacts/");
+        std::process::exit(1);
+    }
+
+    let input_path = &args[1];
+    let config_path = &args[2];
+    let output_dir = PathBuf::from(&args[3]);
+
+    println!("📦 PhraseKit Artifact Builder");
+    println!("════════════════════════════════════════");
+    println!("Input:  {}", input_path);
+    println!("Config: {}", config_path);
+    println!("Output: {}", output_dir.display());
+    println!();
+
+    // Load config
+    let config = load_config(config_path)?;
+    println!("✓ Loaded config: {} (tokenizer: {})", config.version, config.tokenizer);
+
+    // Create output directory
+    std::fs::create_dir_all(&output_dir)?;
+
+    // Load and validate phrases
+    let (text_phrases, stats, unique_tokens) = load_and_validate_phrases(input_path, &config)?;
+
+    println!("\n📊 Build Statistics:");
+    println!("  Total input phrases:     {}", stats.total_input);
+    if stats.filtered_low_count > 0 {
+        println!("  Filtered (low count):    {}", stats.filtered_low_count);
+    }
+    if stats.filtered_low_salience > 0 {
+        println!("  Filtered (low salience): {}", stats.filtered_low_salience);
+    }
+    if stats.duplicate_phrase_ids > 0 {
+        println!("  Skipped (duplicate IDs): {}", stats.duplicate_phrase_ids);
+    }
+    if stats.invalid_tokens > 0 {
+        println!("  Skipped (invalid tokens): {}", stats.invalid_tokens);
+    }
+    println!("  Built patterns:          {}", stats.built);
+
+    if text_phrases.is_empty() {
+        return Err("No valid phrases to build".into());
+    }
+
+    // Build vocabulary and assign token IDs
+    println!("\n📚 Building vocabulary...");
+    let vocabulary = build_vocabulary(unique_tokens, config.separator_id);
+    println!("  ✓ Built vocabulary ({} tokens)", vocabulary.vocab_size);
+
+    // Convert text tokens to IDs
+    let mut phrases: Vec<ProcessedPhrase> = Vec::new();
+    for phrase in text_phrases {
+        let token_ids: Vec<u32> = phrase.tokens.iter()
+            .map(|t| *vocabulary.tokens.get(&t.to_lowercase()).unwrap_or(&0))
+            .collect();
+
+        phrases.push(ProcessedPhrase {
+            token_ids,
+            phrase_id: phrase.phrase_id,
+            salience: phrase.salience,
+            count: phrase.count,
+            length: phrase.tokens.len() as u8,
+        });
+    }
+
+    // Build automaton
+    println!("\n🔨 Building automaton...");
+    let patterns: Vec<Vec<u8>> = phrases.iter()
+        .map(|p| encode_tokens(&p.token_ids, config.separator_id))
+        .collect();
+
+    let automaton: DoubleArrayAhoCorasick<u32> = DoubleArrayAhoCorasick::new(patterns)
+        .map_err(|e| format!("Failed to build automaton: {:?}", e))?;
+
+    let automaton_bytes = automaton.serialize();
+    let automaton_path = output_dir.join("phrases.daac");
+    std::fs::write(&automaton_path, &automaton_bytes)?;
+    println!("  ✓ Wrote automaton ({} bytes) to {}", automaton_bytes.len(), automaton_path.display());
+
+    // Write payloads
+    println!("\n💾 Writing payloads...");
+    let payloads: Vec<Payload> = phrases.iter()
+        .map(|p| Payload::new(p.phrase_id, p.salience, p.count, p.length))
+        .collect();
+
+    let payloads_path = output_dir.join("payloads.bin");
+    let mut payloads_file = File::create(&payloads_path)?;
+    for payload in &payloads {
+        payload.write_to(&mut payloads_file)?;
+    }
+    let payloads_size = payloads.len() * 17;
+    println!("  ✓ Wrote {} payloads ({} bytes) to {}", payloads.len(), payloads_size, payloads_path.display());
+
+    // Generate manifest with checksums
+    println!("\n📝 Generating manifest...");
+    let manifest = Manifest {
+        version: config.version.clone(),
+        tokenizer: config.tokenizer.clone(),
+        num_patterns: phrases.len(),
+        min_count: config.min_count,
+        salience_threshold: config.salience_threshold,
+        built_at: chrono::Utc::now().to_rfc3339(),
+        separator_id: config.separator_id,
+    };
+
+    let manifest_path = output_dir.join("manifest.json");
+    let manifest_json = serde_json::to_string_pretty(&manifest)?;
+    std::fs::write(&manifest_path, manifest_json)?;
+    println!("  ✓ Wrote manifest to {}", manifest_path.display());
+
+    // Write vocabulary
+    println!("\n💾 Writing vocabulary...");
+    let vocab_path = output_dir.join("vocab.json");
+    let vocab_json = serde_json::to_string_pretty(&vocabulary)?;
+    std::fs::write(&vocab_path, vocab_json)?;
+    println!("  ✓ Wrote vocabulary ({} tokens) to {}", vocabulary.vocab_size, vocab_path.display());
+
+    // Summary
+    println!("\n✅ Build complete!");
+    println!("\nArtifacts:");
+    println!("  {} ({} bytes)", automaton_path.display(), automaton_bytes.len());
+    println!("  {} ({} bytes)", payloads_path.display(), payloads_size);
+    println!("  {}", manifest_path.display());
+    println!("  {}", vocab_path.display());
+
+    println!("\n🚀 To use in PhraseKit:");
+    println!("  PhraseKit.load!(");
+    println!("    automaton_path: {:?},", automaton_path.to_str().unwrap());
+    println!("    payloads_path: {:?},", payloads_path.to_str().unwrap());
+    println!("    manifest_path: {:?},", manifest_path.to_str().unwrap());
+    println!("    vocab_path: {:?}", vocab_path.to_str().unwrap());
+    println!("  )");
+
+    Ok(())
+}
+
+fn load_config(path: &str) -> Result<BuildConfig, Box<dyn std::error::Error>> {
+    let file = File::open(path)?;
+    let config: BuildConfig = serde_json::from_reader(file)?;
+    Ok(config)
+}
+
+fn load_and_validate_phrases(
+    path: &str,
+    config: &BuildConfig,
+) -> Result<(Vec<PhraseInput>, BuildStats, HashSet<String>), Box<dyn std::error::Error>> {
+    let file = File::open(path)?;
+    let reader = BufReader::new(file);
+
+    let mut phrases = Vec::new();
+    let mut seen_ids = HashSet::new();
+    let mut unique_tokens: HashSet<String> = HashSet::new();
+    let mut stats = BuildStats {
+        total_input: 0,
+        filtered_low_count: 0,
+        filtered_low_salience: 0,
+        duplicate_phrase_ids: 0,
+        invalid_tokens: 0,
+        built: 0,
+    };
+
+    println!("\n📖 Loading phrases...");
+
+    for (line_num, line) in reader.lines().enumerate() {
+        let line = line?;
+        stats.total_input += 1;
+
+        let phrase: PhraseInput = match serde_json::from_str(&line) {
+            Ok(p) => p,
+            Err(e) => {
+                eprintln!("⚠️  Line {}: Failed to parse: {}", line_num + 1, e);
+                continue;
+            }
+        };
+
+        // Validate
+        if let Some(min_count) = config.min_count {
+            if phrase.count < min_count {
+                stats.filtered_low_count += 1;
+                continue;
+            }
+        }
+
+        if let Some(threshold) = config.salience_threshold {
+            if phrase.salience < threshold {
+                stats.filtered_low_salience += 1;
+                continue;
+            }
+        }
+
+        if phrase.tokens.is_empty() {
+            eprintln!("⚠️  Line {}: Empty token sequence", line_num + 1);
+            stats.invalid_tokens += 1;
+            continue;
+        }
+
+        for token in &phrase.tokens {
+            if token.is_empty() {
+                eprintln!("⚠️  Line {}: Empty token", line_num + 1);
+                stats.invalid_tokens += 1;
+                continue;
+            }
+        }
+
+        if !seen_ids.insert(phrase.phrase_id) {
+            eprintln!("⚠️  Line {}: Duplicate phrase_id {}", line_num + 1, phrase.phrase_id);
+            stats.duplicate_phrase_ids += 1;
+            continue;
+        }
+
+        for token in &phrase.tokens {
+            unique_tokens.insert(token.to_lowercase());
+        }
+
+        phrases.push(phrase);
+        stats.built += 1;
+
+        if stats.total_input % 10000 == 0 {
+            println!("  Processed {} lines...", stats.total_input);
+        }
+    }
+
+    println!("  ✓ Loaded {} phrases", stats.total_input);
+
+    Ok((phrases, stats, unique_tokens))
+}
+
+fn encode_tokens(tokens: &[u32], separator: u32) -> Vec<u8> {
+    let mut bytes = Vec::new();
+    for &token in tokens {
+        bytes.extend_from_slice(&token.to_le_bytes());
+        bytes.extend_from_slice(&separator.to_le_bytes());
+    }
+    bytes
+}
+
+fn build_vocabulary(unique_tokens: HashSet<String>, separator_id: u32) -> Vocabulary {
+    let mut tokens = HashMap::new();
+    let mut sorted_tokens: Vec<String> = unique_tokens.into_iter().collect();
+    sorted_tokens.sort();
+
+    for (idx, token) in sorted_tokens.iter().enumerate() {
+        tokens.insert(token.clone(), (idx + 1) as u32);
+    }
+
+    let mut special_tokens = HashMap::new();
+    special_tokens.insert("<UNK>".to_string(), 0);
+
+    let vocab_size = tokens.len() + special_tokens.len();
+
+    Vocabulary {
+        tokens,
+        special_tokens,
+        vocab_size,
+        separator_id,
+    }
+}