phrasekit 0.2.0-x86_64-linux

data/ext/phrasekit/src/matcher.rs

@@ -0,0 +1,227 @@
+use crate::manifest::Manifest;
+use crate::payload::{load_payloads, Payload};
+use crate::policy::{resolve_overlaps, Match, MatchPolicy};
+use daachorse::DoubleArrayAhoCorasick;
+use std::fs::File;
+use std::io::BufReader;
+use std::path::Path;
+use std::time::SystemTime;
+use thiserror::Error;
+
+#[derive(Error, Debug)]
+pub enum MatcherError {
+    #[error("IO error: {0}")]
+    Io(#[from] std::io::Error),
+
+    #[error("Manifest error: {0}")]
+    Manifest(#[from] crate::manifest::ManifestError),
+
+    #[error("Automaton error: {0}")]
+    Automaton(String),
+
+    #[error("Matcher not loaded")]
+    #[allow(dead_code)]
+    NotLoaded,
+}
+
+pub struct Matcher {
+    automaton: DoubleArrayAhoCorasick<u32>,
+    payloads: Vec<Payload>,
+    manifest: Manifest,
+    loaded_at: SystemTime,
+}
+
+impl Matcher {
+    pub fn load<P: AsRef<Path>>(
+        automaton_path: P,
+        payloads_path: P,
+        manifest_path: P,
+    ) -> Result<Self, MatcherError> {
+        let manifest = Manifest::load(manifest_path)?;
+
+        let automaton_bytes = std::fs::read(automaton_path)?;
+        let (automaton, _): (DoubleArrayAhoCorasick<u32>, _) = unsafe {
+            DoubleArrayAhoCorasick::deserialize_unchecked(&automaton_bytes)
+        };
+
+        let payloads_file = File::open(payloads_path)?;
+        let payloads_reader = BufReader::new(payloads_file);
+        let payloads = load_payloads(payloads_reader)?;
+
+        if payloads.len() != manifest.num_patterns {
+            return Err(MatcherError::Automaton(format!(
+                "Payload count mismatch: manifest says {}, got {}",
+                manifest.num_patterns,
+                payloads.len()
+            )));
+        }
+
+        Ok(Self {
+            automaton,
+            payloads,
+            manifest,
+            loaded_at: SystemTime::now(),
+        })
+    }
+
+    pub fn match_tokens(
+        &self,
+        token_ids: &[u32],
+        policy: MatchPolicy,
+        max: usize,
+    ) -> Vec<Match> {
+        if token_ids.is_empty() {
+            return Vec::new();
+        }
+
+        let separator = self.manifest.separator_id;
+        let mut bytes = Vec::with_capacity(token_ids.len() * 5);
+        for &token_id in token_ids {
+            bytes.extend_from_slice(&token_id.to_le_bytes());
+            bytes.extend_from_slice(&separator.to_le_bytes());
+        }
+
+        let matches: Vec<Match> = self
+            .automaton
+            .find_overlapping_iter(&bytes)
+            .filter_map(|m| {
+                let pattern_id = m.value() as usize;
+                let start_token = m.start() / 8;
+                let end_token = (m.end() + 7) / 8;
+
+                self.payloads
+                    .get(pattern_id)
+                    .map(|payload| Match::new(start_token, end_token, pattern_id, payload.clone()))
+            })
+            .collect();
+
+        let mut resolved = resolve_overlaps(matches, policy);
+
+        if resolved.len() > max {
+            resolved.truncate(max);
+        }
+
+        resolved
+    }
+
+    #[allow(dead_code)]
+    pub fn manifest(&self) -> &Manifest {
+        &self.manifest
+    }
+
+    pub fn num_patterns(&self) -> usize {
+        self.payloads.len()
+    }
+
+    #[allow(dead_code)]
+    pub fn loaded_at(&self) -> SystemTime {
+        self.loaded_at
+    }
+
+    pub fn memory_usage_mb(&self) -> f64 {
+        let automaton_size = std::mem::size_of_val(&self.automaton);
+        let payloads_size = self.payloads.len() * std::mem::size_of::<Payload>();
+        ((automaton_size + payloads_size) as f64) / 1_048_576.0
+    }
+}
+
+pub struct Stats {
+    pub version: String,
+    pub loaded_at: SystemTime,
+    pub num_patterns: usize,
+    pub heap_mb: f64,
+    pub hits_total: u64,
+    pub p50_us: u64,
+    pub p95_us: u64,
+    pub p99_us: u64,
+}
+
+impl Stats {
+    pub fn from_matcher(matcher: &Matcher) -> Self {
+        Self {
+            version: matcher.manifest.version.clone(),
+            loaded_at: matcher.loaded_at,
+            num_patterns: matcher.num_patterns(),
+            heap_mb: matcher.memory_usage_mb(),
+            hits_total: 0,
+            p50_us: 0,
+            p95_us: 0,
+            p99_us: 0,
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::payload::Payload;
+    use std::io::Write;
+    use tempfile::NamedTempFile;
+
+    fn create_test_artifacts() -> (NamedTempFile, NamedTempFile, NamedTempFile) {
+        let patterns = vec![vec![1u32, 2u32], vec![2u32, 3u32]];
+
+        let automaton = DoubleArrayAhoCorasick::new(patterns).unwrap();
+        let automaton_bytes = automaton.serialize();
+
+        let mut automaton_file = NamedTempFile::new().unwrap();
+        automaton_file.write_all(&automaton_bytes).unwrap();
+        automaton_file.flush().unwrap();
+
+        let mut payloads_file = NamedTempFile::new().unwrap();
+        let payload1 = Payload::new(100, 1.5, 50, 2);
+        let payload2 = Payload::new(200, 2.0, 100, 2);
+        payload1.write_to(&mut payloads_file).unwrap();
+        payload2.write_to(&mut payloads_file).unwrap();
+        payloads_file.flush().unwrap();
+
+        let mut manifest_file = NamedTempFile::new().unwrap();
+        let manifest_json = r#"{
+            "version": "test-v1",
+            "tokenizer": "test-tokenizer",
+            "num_patterns": 2,
+            "built_at": "2025-01-01T00:00:00Z",
+            "separator_id": 4294967294
+        }"#;
+        manifest_file.write_all(manifest_json.as_bytes()).unwrap();
+        manifest_file.flush().unwrap();
+
+        (automaton_file, payloads_file, manifest_file)
+    }
+
+    #[test]
+    fn test_matcher_load() {
+        let (automaton_file, payloads_file, manifest_file) = create_test_artifacts();
+
+        let matcher = Matcher::load(
+            automaton_file.path(),
+            payloads_file.path(),
+            manifest_file.path(),
+        )
+        .unwrap();
+
+        assert_eq!(matcher.num_patterns(), 2);
+        assert_eq!(matcher.manifest().version, "test-v1");
+    }
+
+    #[test]
+    fn test_matcher_match_tokens() {
+        let (automaton_file, payloads_file, manifest_file) = create_test_artifacts();
+
+        let matcher = Matcher::load(
+            automaton_file.path(),
+            payloads_file.path(),
+            manifest_file.path(),
+        )
+        .unwrap();
+
+        let token_ids = vec![1, 2, 3, 4];
+        let matches = matcher.match_tokens(&token_ids, MatchPolicy::LeftmostLongest, 10);
+
+        assert_eq!(matches.len(), 2);
+        assert_eq!(matches[0].start, 0);
+        assert_eq!(matches[0].end, 2);
+        assert_eq!(matches[1].start, 1);
+        assert_eq!(matches[1].end, 3);
+    }
+}

data/ext/phrasekit/src/payload.rs

@@ -0,0 +1,95 @@
+use serde::{Deserialize, Serialize};
+use std::io::{Read, Write};
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct Payload {
+    pub phrase_id: u32,
+    pub salience: f32,
+    pub count: u32,
+    pub n: u8,
+}
+
+impl Payload {
+    #[allow(dead_code)]
+    pub fn new(phrase_id: u32, salience: f32, count: u32, n: u8) -> Self {
+        Self {
+            phrase_id,
+            salience,
+            count,
+            n,
+        }
+    }
+
+    pub fn salience_score(&self) -> f32 {
+        self.salience * ((self.count + 1) as f32).ln()
+    }
+
+    pub fn read_from<R: Read>(reader: &mut R) -> std::io::Result<Self> {
+        let mut buf = [0u8; 17];
+        reader.read_exact(&mut buf)?;
+
+        let phrase_id = u32::from_le_bytes([buf[0], buf[1], buf[2], buf[3]]);
+        let salience = f32::from_le_bytes([buf[4], buf[5], buf[6], buf[7]]);
+        let count = u32::from_le_bytes([buf[8], buf[9], buf[10], buf[11]]);
+        let n = buf[16];
+
+        Ok(Self {
+            phrase_id,
+            salience,
+            count,
+            n,
+        })
+    }
+
+    #[allow(dead_code)]
+    pub fn write_to<W: Write>(&self, writer: &mut W) -> std::io::Result<()> {
+        writer.write_all(&self.phrase_id.to_le_bytes())?;
+        writer.write_all(&self.salience.to_le_bytes())?;
+        writer.write_all(&self.count.to_le_bytes())?;
+        writer.write_all(&[0u8; 4])?;
+        writer.write_all(&[self.n])?;
+        Ok(())
+    }
+}
+
+pub fn load_payloads<R: Read>(mut reader: R) -> std::io::Result<Vec<Payload>> {
+    let mut payloads = Vec::new();
+
+    loop {
+        match Payload::read_from(&mut reader) {
+            Ok(payload) => payloads.push(payload),
+            Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => break,
+            Err(e) => return Err(e),
+        }
+    }
+
+    Ok(payloads)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_payload_roundtrip() {
+        let payload = Payload::new(12345, 2.13, 314, 2);
+
+        let mut buf = Vec::new();
+        payload.write_to(&mut buf).unwrap();
+
+        let mut cursor = std::io::Cursor::new(buf);
+        let loaded = Payload::read_from(&mut cursor).unwrap();
+
+        assert_eq!(loaded.phrase_id, 12345);
+        assert_eq!(loaded.count, 314);
+        assert_eq!(loaded.n, 2);
+        assert!((loaded.salience - 2.13).abs() < 0.001);
+    }
+
+    #[test]
+    fn test_salience_score() {
+        let payload = Payload::new(1, 2.0, 99, 2);
+        let score = payload.salience_score();
+        assert!((score - (2.0 * 100.0_f32.ln())).abs() < 0.001);
+    }
+}

data/ext/phrasekit/src/policy.rs

@@ -0,0 +1,190 @@
+use crate::payload::Payload;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum MatchPolicy {
+    LeftmostLongest,
+    LeftmostFirst,
+    SalienceMax,
+}
+
+impl MatchPolicy {
+    pub fn from_str(s: &str) -> Option<Self> {
+        match s {
+            "leftmost_longest" => Some(Self::LeftmostLongest),
+            "leftmost_first" => Some(Self::LeftmostFirst),
+            "salience_max" => Some(Self::SalienceMax),
+            _ => None,
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct Match {
+    pub start: usize,
+    pub end: usize,
+    #[allow(dead_code)]
+    pub pattern_id: usize,
+    pub payload: Payload,
+}
+
+impl Match {
+    pub fn new(start: usize, end: usize, pattern_id: usize, payload: Payload) -> Self {
+        Self {
+            start,
+            end,
+            pattern_id,
+            payload,
+        }
+    }
+
+    pub fn len(&self) -> usize {
+        self.end - self.start
+    }
+
+    pub fn overlaps(&self, other: &Match) -> bool {
+        !(self.end <= other.start || other.end <= self.start)
+    }
+}
+
+pub fn resolve_overlaps(mut matches: Vec<Match>, policy: MatchPolicy) -> Vec<Match> {
+    if matches.is_empty() {
+        return matches;
+    }
+
+    matches.sort_by_key(|m| m.start);
+
+    match policy {
+        MatchPolicy::LeftmostLongest => resolve_leftmost_longest(matches),
+        MatchPolicy::LeftmostFirst => resolve_leftmost_first(matches),
+        MatchPolicy::SalienceMax => resolve_salience_max(matches),
+    }
+}
+
+fn resolve_leftmost_longest(matches: Vec<Match>) -> Vec<Match> {
+    let mut result = Vec::new();
+    let mut current_end = 0;
+
+    for group_start in 0..matches.len() {
+        if matches[group_start].start < current_end {
+            continue;
+        }
+
+        let group_end = matches[group_start..]
+            .iter()
+            .position(|m| m.start != matches[group_start].start)
+            .map(|i| group_start + i)
+            .unwrap_or(matches.len());
+
+        let longest = matches[group_start..group_end]
+            .iter()
+            .max_by_key(|m| m.len())
+            .unwrap()
+            .clone();
+
+        current_end = longest.end;
+        result.push(longest);
+    }
+
+    result
+}
+
+fn resolve_leftmost_first(matches: Vec<Match>) -> Vec<Match> {
+    let mut result = Vec::new();
+    let mut current_end = 0;
+
+    for m in matches {
+        if m.start >= current_end {
+            current_end = m.end;
+            result.push(m);
+        }
+    }
+
+    result
+}
+
+fn resolve_salience_max(matches: Vec<Match>) -> Vec<Match> {
+    let mut result = Vec::new();
+    let mut i = 0;
+
+    while i < matches.len() {
+        let mut j = i + 1;
+        while j < matches.len() && matches[j].overlaps(&matches[i]) {
+            j += 1;
+        }
+
+        let best = matches[i..j]
+            .iter()
+            .max_by(|a, b| {
+                a.payload
+                    .salience_score()
+                    .partial_cmp(&b.payload.salience_score())
+                    .unwrap_or(std::cmp::Ordering::Equal)
+            })
+            .unwrap()
+            .clone();
+
+        result.push(best.clone());
+        i = matches[i..]
+            .iter()
+            .position(|m| m.start >= best.end)
+            .map(|idx| i + idx)
+            .unwrap_or(matches.len());
+    }
+
+    result
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn make_match(start: usize, end: usize, salience: f32, count: u32) -> Match {
+        Match::new(
+            start,
+            end,
+            0,
+            Payload::new(0, salience, count, (end - start) as u8),
+        )
+    }
+
+    #[test]
+    fn test_leftmost_longest() {
+        let matches = vec![
+            make_match(0, 2, 1.0, 100),
+            make_match(0, 3, 1.0, 100),
+            make_match(5, 7, 1.0, 100),
+        ];
+
+        let resolved = resolve_overlaps(matches, MatchPolicy::LeftmostLongest);
+        assert_eq!(resolved.len(), 2);
+        assert_eq!(resolved[0].len(), 3);
+        assert_eq!(resolved[1].start, 5);
+    }
+
+    #[test]
+    fn test_leftmost_first() {
+        let matches = vec![
+            make_match(0, 2, 1.0, 100),
+            make_match(1, 3, 1.0, 100),
+            make_match(3, 5, 1.0, 100),
+        ];
+
+        let resolved = resolve_overlaps(matches, MatchPolicy::LeftmostFirst);
+        assert_eq!(resolved.len(), 2);
+        assert_eq!(resolved[0].end, 2);
+        assert_eq!(resolved[1].start, 3);
+    }
+
+    #[test]
+    fn test_salience_max() {
+        let matches = vec![
+            make_match(0, 2, 1.0, 100),
+            make_match(0, 3, 2.0, 200),
+            make_match(5, 7, 1.0, 100),
+        ];
+
+        let resolved = resolve_overlaps(matches, MatchPolicy::SalienceMax);
+        assert_eq!(resolved.len(), 2);
+        assert_eq!(resolved[0].len(), 3);
+    }
+}

data/lib/phrasekit/miner.rb

@@ -0,0 +1,74 @@
+require "shellwords"
+
+module PhraseKit
+  class Miner
+    class Error < StandardError; end
+
+    class << self
+      def mine(input_path:, output_path:, min_n: 2, max_n: 5, min_count: 10, config_path: nil)
+        binary_path = find_binary
+
+        # Create temporary config if not provided
+        if config_path.nil?
+          require "tempfile"
+          require "json"
+
+          config_file = Tempfile.new(["mine_config", ".json"])
+          config_file.write(JSON.generate({
+            min_n: min_n,
+            max_n: max_n,
+            min_count: min_count
+          }))
+          config_file.flush
+          config_path = config_file.path
+        end
+
+        # Run mining
+        cmd = [binary_path, input_path.to_s, config_path.to_s, output_path.to_s]
+        output = `#{cmd.shelljoin} 2>&1`
+
+        unless $?.success?
+          config_file.close! if config_file
+          raise Error, "Mining failed: #{output}"
+        end
+
+        config_file.close! if config_file
+
+        # Parse stats from output
+        parse_stats(output)
+      end
+
+      private
+
+      def find_binary
+        # Search paths relative to this file
+        # __dir__ is lib/phrasekit, so go up 2 levels to get to gem root
+        base_dir = File.expand_path("../..", __dir__)
+
+        candidates = [
+          File.join(base_dir, "ext/phrasekit/target/release/phrasekit_mine"),
+          File.join(base_dir, "ext/phrasekit/target/debug/phrasekit_mine"),
+          # For installed gems
+          File.join(base_dir, "bin/phrasekit_mine")
+        ]
+
+        candidates.each do |binary|
+          return binary if File.exist?(binary) && File.executable?(binary)
+        end
+
+        raise Error, "phrasekit_mine binary not found. Run: cargo build --release --bin phrasekit_mine --manifest-path ext/phrasekit/Cargo.toml"
+      end
+
+      def parse_stats(output)
+        stats = {}
+
+        output.scan(/Total documents:\s+(\d+)/) { stats[:total_docs] = $1.to_i }
+        output.scan(/Total tokens:\s+(\d+)/) { stats[:total_tokens] = $1.to_i }
+        output.scan(/Unique n-grams:\s+(\d+)/) { stats[:unique_ngrams] = $1.to_i }
+        output.scan(/After min_count=\d+:\s+(\d+)/) { stats[:ngrams_after_filter] = $1.to_i }
+
+        stats
+      end
+    end
+  end
+end

data/lib/phrasekit/scorer.rb

@@ -0,0 +1,92 @@
+require "shellwords"
+
+module PhraseKit
+  class Scorer
+    class Error < StandardError; end
+
+    class << self
+      def score(
+        domain_path:,
+        background_path:,
+        output_path:,
+        method: :ratio,
+        min_salience: 2.0,
+        min_domain_count: 10,
+        assign_phrase_ids: true,
+        starting_phrase_id: 1000,
+        config_path: nil
+      )
+        binary_path = find_binary
+
+        # Create temporary config if not provided
+        if config_path.nil?
+          require "tempfile"
+          require "json"
+
+          config_file = Tempfile.new(["score_config", ".json"])
+          config_file.write(JSON.generate({
+            method: method.to_s,
+            min_salience: min_salience,
+            min_domain_count: min_domain_count,
+            assign_phrase_ids: assign_phrase_ids,
+            starting_phrase_id: starting_phrase_id
+          }))
+          config_file.flush
+          config_path = config_file.path
+        end
+
+        # Run scoring
+        cmd = [
+          binary_path,
+          domain_path.to_s,
+          background_path.to_s,
+          config_path.to_s,
+          output_path.to_s
+        ]
+        output = `#{cmd.shelljoin} 2>&1`
+
+        unless $?.success?
+          config_file.close! if config_file
+          raise Error, "Scoring failed: #{output}"
+        end
+
+        config_file.close! if config_file
+
+        # Parse stats from output
+        parse_stats(output)
+      end
+
+      private
+
+      def find_binary
+        # Search paths relative to this file
+        # __dir__ is lib/phrasekit, so go up 2 levels to get to gem root
+        base_dir = File.expand_path("../..", __dir__)
+
+        candidates = [
+          File.join(base_dir, "ext/phrasekit/target/release/phrasekit_score"),
+          File.join(base_dir, "ext/phrasekit/target/debug/phrasekit_score"),
+          # For installed gems
+          File.join(base_dir, "bin/phrasekit_score")
+        ]
+
+        candidates.each do |binary|
+          return binary if File.exist?(binary) && File.executable?(binary)
+        end
+
+        raise Error, "phrasekit_score binary not found. Run: cargo build --release --bin phrasekit_score --manifest-path ext/phrasekit/Cargo.toml"
+      end
+
+      def parse_stats(output)
+        stats = {}
+
+        output.scan(/Domain phrases:\s+(\d+)/) { stats[:domain_phrases] = $1.to_i }
+        output.scan(/Background phrases:\s+(\d+)/) { stats[:background_phrases] = $1.to_i }
+        output.scan(/After domain filter:\s+(\d+)/) { stats[:after_domain_filter] = $1.to_i }
+        output.scan(/After salience filter:\s+(\d+)/) { stats[:after_salience_filter] = $1.to_i }
+
+        stats
+      end
+    end
+  end
+end