skill-extractor 0.1.0

data/lib/skill_extractor/matcher.rb

@@ -0,0 +1,105 @@
+# Keyword matcher — a faithful port of FlashText's extract_keywords.
+# Word boundaries: any char outside [A-Za-z0-9_]. Case-insensitive,
+# longest-match-wins, non-overlapping. Identical spans across all
+# skill-extractor language implementations.
+module SkillExtractor
+  class KeywordMatcher
+    KEYWORD = :__keyword__
+    WORD_CHARS = ("a".."z").to_a + ("A".."Z").to_a + ("0".."9").to_a + ["_"]
+    WORD_SET = WORD_CHARS.to_h { |c| [c, true] }.freeze
+
+    def initialize(keywords = [])
+      @trie = {}
+      keywords.each { |kw| add(kw) }
+    end
+
+    def add(keyword)
+      node = @trie
+      kw = keyword.downcase
+      kw.each_char do |ch|
+        node = (node[ch] ||= {})
+      end
+      node[KEYWORD] = kw
+    end
+
+    # Returns [[keyword, start, end], ...] spans, matching FlashText exactly.
+    def extract(sentence)
+      out = []
+      return out if sentence.nil? || sentence.empty?
+
+      sentence = sentence.downcase
+      chars = sentence.chars
+      current = @trie
+      seq_start = 0
+      seq_end = 0
+      reset = false
+      idx = 0
+      n = chars.length
+      while idx < n
+        ch = chars[idx]
+        if !WORD_SET[ch]
+          if current.key?(KEYWORD) || current.key?(ch)
+            longest = nil
+            longer_found = false
+            if current.key?(KEYWORD)
+              longest = current[KEYWORD]
+              seq_end = idx
+            end
+            if current.key?(ch)
+              cont = current[ch]
+              idy = idx + 1
+              broke = false
+              while idy < n
+                inner = chars[idy]
+                if !WORD_SET[inner] && cont.key?(KEYWORD)
+                  longest = cont[KEYWORD]
+                  seq_end = idy
+                  longer_found = true
+                end
+                if cont.key?(inner)
+                  cont = cont[inner]
+                else
+                  broke = true
+                  break
+                end
+                idy += 1
+              end
+              if !broke && cont.key?(KEYWORD)
+                longest = cont[KEYWORD]
+                seq_end = idy
+                longer_found = true
+              end
+              idx = seq_end if longer_found
+            end
+            current = @trie
+            out << [longest, seq_start, idx] if longest
+            reset = true
+          else
+            current = @trie
+            reset = true
+          end
+        elsif current.key?(ch)
+          current = current[ch]
+        else
+          current = @trie
+          reset = true
+          idy = idx + 1
+          while idy < n
+            break unless WORD_SET[chars[idy]]
+            idy += 1
+          end
+          idx = idy
+        end
+        if idx + 1 >= n && current.key?(KEYWORD)
+          out << [current[KEYWORD], seq_start, n]
+        end
+        idx += 1
+        if reset
+          reset = false
+          seq_start = idx
+        end
+      end
+      out
+    end
+  end
+end

data/lib/skill_extractor/mlp.rb

@@ -0,0 +1,44 @@
+# MLP classifier forward pass — consumes mlp.json exported from the trained
+# sklearn MLPClassifier (base64 float32, row-major, shape [in, out]).
+require "base64"
+require "json"
+
+module SkillExtractor
+  class MLP
+    def initialize(path)
+      spec = JSON.parse(File.read(path))
+      unless spec["format"] == "mlp-weights-v1"
+        raise ArgumentError, "unsupported weights format: #{spec["format"]}"
+      end
+      @layers = spec["layers"].map do |l|
+        {
+          rows: l["shape"][0],
+          cols: l["shape"][1],
+          w: Base64.decode64(l["weights_b64"]).unpack("e*"),
+          b: Base64.decode64(l["bias_b64"]).unpack("e*")
+        }
+      end
+    end
+
+    # xs: array of 384-dim L2-normalized embeddings -> array of P(skill)
+    def predict_proba(xs)
+      last = @layers.length - 1
+      xs.map do |x|
+        h = x
+        @layers.each_with_index do |l, i|
+          cols = l[:cols]
+          w = l[:w]
+          nxt = l[:b].dup
+          h.each_with_index do |hv, r|
+            next if hv.zero?
+            off = r * cols
+            cols.times { |j| nxt[j] += hv * w[off + j] }
+          end
+          nxt.map! { |v| v.negative? ? 0.0 : v } if i < last # relu
+          h = nxt
+        end
+        1.0 / (1.0 + Math.exp(-h[0])) # logistic
+      end
+    end
+  end
+end

data/lib/skill_extractor/version.rb

@@ -0,0 +1,3 @@
+module SkillExtractor
+  VERSION = "0.1.0"
+end

data/lib/skill_extractor.rb

@@ -0,0 +1,70 @@
+# skill-extractor: extract skills from job postings and resumes.
+# 32K-skill gazetteer + MiniLM embeddings + MLP context classifier,
+# trained on 491K labeled samples (73% F1 held-out).
+require "json"
+
+require_relative "skill_extractor/matcher"
+require_relative "skill_extractor/mlp"
+require_relative "skill_extractor/version"
+
+module SkillExtractor
+  MODEL = "Xenova/all-MiniLM-L6-v2"
+  CONTEXT_BEFORE = 20
+  CONTEXT_AFTER = 21
+  DEFAULT_THRESHOLD = 0.5
+  DATA_DIR = File.expand_path("../data", __dir__)
+
+  class Extractor
+    # fp32 (default) matches the reference implementation bit-for-bit
+    def initialize(quantized: false)
+      @quantized = quantized
+      @matcher = KeywordMatcher.new(JSON.parse(File.read(File.join(DATA_DIR, "skills.json"))))
+      @mlp = MLP.new(File.join(DATA_DIR, "mlp.json"))
+      @pipe = nil
+    end
+
+    attr_reader :matcher, :mlp
+
+    # Gazetteer matches with their +/-20-word context windows.
+    def candidates(text)
+      matches = @matcher.extract(text)
+      return [] if matches.empty?
+
+      words = text.split
+      matches.map do |skill, start, _end|
+        word_idx = text[0...start].split.length
+        lo = [0, word_idx - CONTEXT_BEFORE].max
+        hi = [words.length, word_idx + CONTEXT_AFTER].min
+        { skill: skill, context: words[lo...hi].join(" ") }
+      end
+    end
+
+    # P(skill) for each candidate input string.
+    def classify(inputs)
+      @pipe ||= begin
+        require "informers"
+        Informers.pipeline("embedding", MODEL, quantized: @quantized)
+      end
+      @mlp.predict_proba(@pipe.(inputs))
+    end
+
+    # Extract confirmed skills from a job posting or resume text.
+    def extract(text, threshold: DEFAULT_THRESHOLD)
+      cands = candidates(text)
+      return [] if cands.empty?
+
+      probs = classify(cands.map { |c| "#{c[:skill]} : #{c[:context]}" })
+      found = {}
+      probs.each_with_index { |p, i| found[cands[i][:skill]] = true if p >= threshold }
+      found.keys.sort
+    end
+  end
+
+  @default = nil
+
+  # Module-level convenience wrapper around a shared Extractor.
+  def self.extract_skills(text, threshold: DEFAULT_THRESHOLD)
+    @default ||= Extractor.new
+    @default.extract(text, threshold: threshold)
+  end
+end

metadata

@@ -0,0 +1,71 @@
+--- !ruby/object:Gem::Specification
+name: skill-extractor
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Qarera
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2026-07-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: informers
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+description: Gazetteer candidates filtered by a MiniLM + MLP context classifier trained
+  on 491K labeled samples, so prose like 'can-do attitude' doesn't become a skill.
+  Runs on ONNX Runtime via the informers gem.
+email:
+- yashthenuan21@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- data/meta.json
+- data/mlp.json
+- data/skills.json
+- lib/skill_extractor.rb
+- lib/skill_extractor/matcher.rb
+- lib/skill_extractor/mlp.rb
+- lib/skill_extractor/version.rb
+homepage: https://github.com/dreamjobs-tech/skill-extractor
+licenses:
+- MIT
+metadata:
+  source_code_uri: https://github.com/dreamjobs-tech/skill-extractor/tree/main/ruby
+  homepage_uri: https://www.qarera.com
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.4.10
+signing_key:
+specification_version: 4
+summary: Extract skills from job postings and resumes — 32K-skill gazetteer + MiniLM
+  embeddings + MLP context classifier (73% F1).
+test_files: []