freql 0.1.0

data/lib/freql/fpbw.rb

@@ -0,0 +1,28 @@
+module Freql
+  module FPBW
+    # fpbw frequency per billion words.
+    # the same as fpmw but with a billion instead of million.
+
+    # The advantages over fpbw is that values are far less likely to dip below 1
+
+    class << self
+      def fpbw_to_fq fpbw
+        fpbw / 1000000000.0
+      end
+      def fpbw_to_fpmw fpbw
+        fpbw / 1000
+      end
+      def fpbw_to_zipf fpbw
+        Math.log10(fpbw)
+      end
+      def fpbw_to_cb fpbw
+        Math.log10(fpbw / 1000000000.0) * 100.0
+      end
+
+      def calc_fpbw occurances, total
+        (occurances / total.to_f) * 1000000000
+      end
+    end
+  end
+end
+

data/lib/freql/fpmw.rb

@@ -0,0 +1,41 @@
+module Freql
+  module FPMW
+    # fpmw frequency per million words.
+    # or the number of times a word occurs in one million words
+    # Practical range 53703(the) to 0.01(trella)
+    # actual range is from 1000000 to 0
+
+    # a fpmw of 1 means that word occurs 1 once on average for every million words
+    # a fpmw of 1,000,000 would mean every word/token in the corpus was the same.
+
+    # Advantages
+    # - Its straight forward to calculated and understand.
+    # - corpus size doesn't change the relative value.
+    # - Its a old standard.
+
+    # Disadvantages:
+    # - the issue with fpmw is that rare words can have a fpmw of less than 1
+    # - and its not easy for humans to compare.
+
+    class << self
+      def fpmw_to_fq fpmw
+        fpmw / 1000000.0
+      end
+      def fpmw_to_fpbw fpmw
+        fpmw * 1000
+      end
+      def fpmw_to_zipf fpmw
+        Math.log10(fpmw * 1000)
+      end
+      def fpmw_to_cb fpmw
+        Math.log10(fpmw / 1000000.0) * 100.0
+      end
+
+      def calc_fpmw occurances, total
+        (occurances / total.to_f) * 1000000
+      end
+    end
+  end
+
+end
+

data/lib/freql/fq.rb

@@ -0,0 +1,30 @@
+module Freql
+  module FQ
+    # fq frequency represented as a proportion between 0 and 1
+    # occurances in corpus divided by total words in corpus
+
+    # practicle range 0.053(the) 0.00000001(trella)
+    # actual range 0 to 1
+
+    class << self
+      def fq_to_fpmw fq
+        fq * 1000000
+      end
+      def fq_to_fpbw fq
+        fq * 1000000000
+      end
+      def fq_to_zipf fq
+        Math.log10(fq)+9
+      end
+      def fq_to_cb fq
+        Math.log10(fq) * 100.0
+      end
+
+      def calc_fq occurances, total
+        occurances / total.to_f
+      end
+    end
+
+  end
+end
+

data/lib/freql/rank.rb

@@ -0,0 +1,39 @@
+require_relative 'bindata'
+require_relative 'cb'
+
+module Freql
+  class Rank
+
+    # lookup the word rank for given words/tokens.
+
+    class << self
+      def by_lang lang, size: :small
+        rank = self.new
+        rank.extract_lang_file lang, size: size
+        return rank
+      end
+    end
+
+    attr :ranks
+
+    def lookup rank
+      @ranks[rank-1]
+    end
+
+    def top_n rank
+      @ranks[0...rank]
+    end
+
+    def extract_lang_file lang, size:
+      BinData.read_lang( lang, size: size ) do |lang_data|
+        # @ranks = lang_data.filter {|group| !group.empty? }
+        # some words really should have the same ranking as the score is the same.
+        # for now ignore that detail and build a simple flat list.
+        # Its also wirth noting that the data contains non-words like 000.
+        @ranks = lang_data.flatten
+      end
+    end
+  end
+end
+
+

data/lib/freql/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Freql
+  VERSION = "0.1.0"
+end

data/lib/freql/words.rb

@@ -0,0 +1,44 @@
+require_relative 'bindata'
+require_relative 'cb'
+
+module Freql
+  class Words
+
+    # Lookup the cb or zipf work frequency for given words/tokens
+
+    # Words may not be the best name for this class in the future.
+
+    class << self
+      def by_lang lang = :en, size: :small
+        w = self.new
+        w.extract_lang_file lang, size: size
+        return w
+      end
+    end
+
+    attr :words
+
+    def lookup word
+      @words[word]
+    end
+
+    def lookup_zipf word
+      r = lookup(word)
+      r && CB.cb_to_zipf(r)
+    end
+
+    def query *words
+      @words.slice(*words)
+    end
+
+    def query_zipf *words
+      query(*words).transform_values {|v| CB.cb_to_zipf(v) }
+    end
+
+    def extract_lang_file lang, size:
+      BinData.read_and_unpack_lang( lang, size: size ) do |lang_data|
+        @words = lang_data
+      end
+    end
+  end
+end

data/lib/freql/zipf.rb

@@ -0,0 +1,36 @@
+module Freql
+  module ZipF
+    # ZipF is log10 of frequency per billion words
+    # Named after the American linguist George Kingsley Zipf
+
+    # Practical Range 1-7ish 1.01(the) to 7.73(trella). *(wait i might have that backwards...)*
+    # Actual Range is 9.0 to 0.0(or less technically)
+
+    # Advantages
+    # - Its human readable and its a known common standerd.
+
+    # Disavantages
+    # - It requires decimials for accuracy.
+    # - Technically it can cross 0 with extremely rare items in large datasets.
+
+    class << self
+      def zipf_to_fq zipf
+        10.00**zipf / 1e9
+      end
+      def zipf_to_fpmw zipf
+        10.00 ** zipf / 1000
+      end
+      def zipf_to_fpbw zipf
+        10.00 ** zipf
+      end
+      def zipf_to_cb zipf
+        (zipf * 100.0) - 900.00
+      end
+
+      def calc_zipf occurances, total
+        Math.log10(occurances / total.to_f)+9
+      end
+    end
+  end
+end
+

data/lib/freql.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require_relative "freql/version"
+require_relative "freql/bindata"
+require_relative "freql/counter"
+require_relative "freql/words"
+
+module Freql
+
+  class Error < StandardError; end
+  # Your code goes here...
+
+end

data/sig/freql.rbs

@@ -0,0 +1,4 @@
+module Freql
+  VERSION: String
+  # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+end

metadata

@@ -0,0 +1,152 @@
+--- !ruby/object:Gem::Specification
+name: freql
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- opsaaaaa
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2023-06-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: msgpack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.5.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.5.1
+description: Right now all we do is convert fpmw to zipf and other units.
+email:
+- sean@ferney.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".rspec"
+- CHANGELOG.md
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- freql.gemspec
+- lib/freql.rb
+- lib/freql/bindata.rb
+- lib/freql/cb.rb
+- lib/freql/counter.rb
+- lib/freql/data/_chinese_mapping.msgpack.gz
+- lib/freql/data/jieba_zh.txt
+- lib/freql/data/jieba_zh_orig.txt
+- lib/freql/data/large_ar.msgpack.gz
+- lib/freql/data/large_bn.msgpack.gz
+- lib/freql/data/large_ca.msgpack.gz
+- lib/freql/data/large_cs.msgpack.gz
+- lib/freql/data/large_de.msgpack.gz
+- lib/freql/data/large_en.msgpack.gz
+- lib/freql/data/large_es.msgpack.gz
+- lib/freql/data/large_fi.msgpack.gz
+- lib/freql/data/large_fr.msgpack.gz
+- lib/freql/data/large_he.msgpack.gz
+- lib/freql/data/large_it.msgpack.gz
+- lib/freql/data/large_ja.msgpack.gz
+- lib/freql/data/large_mk.msgpack.gz
+- lib/freql/data/large_nb.msgpack.gz
+- lib/freql/data/large_nl.msgpack.gz
+- lib/freql/data/large_pl.msgpack.gz
+- lib/freql/data/large_pt.msgpack.gz
+- lib/freql/data/large_ru.msgpack.gz
+- lib/freql/data/large_sv.msgpack.gz
+- lib/freql/data/large_uk.msgpack.gz
+- lib/freql/data/large_zh.msgpack.gz
+- lib/freql/data/small_ar.msgpack.gz
+- lib/freql/data/small_bg.msgpack.gz
+- lib/freql/data/small_bn.msgpack.gz
+- lib/freql/data/small_ca.msgpack.gz
+- lib/freql/data/small_cs.msgpack.gz
+- lib/freql/data/small_da.msgpack.gz
+- lib/freql/data/small_de.msgpack.gz
+- lib/freql/data/small_el.msgpack.gz
+- lib/freql/data/small_en.msgpack.gz
+- lib/freql/data/small_es.msgpack.gz
+- lib/freql/data/small_fa.msgpack.gz
+- lib/freql/data/small_fi.msgpack.gz
+- lib/freql/data/small_fil.msgpack.gz
+- lib/freql/data/small_fr.msgpack.gz
+- lib/freql/data/small_he.msgpack.gz
+- lib/freql/data/small_hi.msgpack.gz
+- lib/freql/data/small_hu.msgpack.gz
+- lib/freql/data/small_id.msgpack.gz
+- lib/freql/data/small_is.msgpack.gz
+- lib/freql/data/small_it.msgpack.gz
+- lib/freql/data/small_ja.msgpack.gz
+- lib/freql/data/small_ko.msgpack.gz
+- lib/freql/data/small_lt.msgpack.gz
+- lib/freql/data/small_lv.msgpack.gz
+- lib/freql/data/small_mk.msgpack.gz
+- lib/freql/data/small_ms.msgpack.gz
+- lib/freql/data/small_nb.msgpack.gz
+- lib/freql/data/small_nl.msgpack.gz
+- lib/freql/data/small_pl.msgpack.gz
+- lib/freql/data/small_pt.msgpack.gz
+- lib/freql/data/small_ro.msgpack.gz
+- lib/freql/data/small_ru.msgpack.gz
+- lib/freql/data/small_sh.msgpack.gz
+- lib/freql/data/small_sk.msgpack.gz
+- lib/freql/data/small_sl.msgpack.gz
+- lib/freql/data/small_sv.msgpack.gz
+- lib/freql/data/small_ta.msgpack.gz
+- lib/freql/data/small_tr.msgpack.gz
+- lib/freql/data/small_uk.msgpack.gz
+- lib/freql/data/small_ur.msgpack.gz
+- lib/freql/data/small_vi.msgpack.gz
+- lib/freql/data/small_zh.msgpack.gz
+- lib/freql/fpbw.rb
+- lib/freql/fpmw.rb
+- lib/freql/fq.rb
+- lib/freql/rank.rb
+- lib/freql/version.rb
+- lib/freql/words.rb
+- lib/freql/zipf.rb
+- sig/freql.rbs
+homepage: https://github.com/opsaaaaa/freql
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/opsaaaaa/freql
+  source_code_uri: https://github.com/opsaaaaa/freql
+  changelog_uri: https://github.com/opsaaaaa/freql/blob/master/CHANGELOG.md
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.6.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.4.1
+signing_key: 
+specification_version: 4
+summary: A library for handling word/token freqencies units.
+test_files: []