twfilter 0.1.0

data/lib/twfilter/data/taiwan_grammar.txt

@@ -0,0 +1,20 @@
+好好吃
+好好玩
+就醬
+很會
+有來
+有做
+有到
+有去
+有吃
+有在
+有寫
+有帶
+有想
+有拿
+有玩
+有看
+有給
+有聽
+有說
+有買

data/lib/twfilter/data/taiwan_lexicon.txt

@@ -0,0 +1,51 @@
+伺服器
+便當
+光碟
+公車
+冷氣
+列印
+叫車
+品質
+宵夜
+寬頻
+寮國
+專案
+幼稚園
+影印
+影片
+捷運
+早安
+智慧型手機
+桌上型電腦
+機車
+水準
+泡麵
+滑鼠
+硬碟
+硬體
+程式
+筆記型電腦
+管道
+簡訊
+紐西蘭
+網路
+網際網路
+義大利
+腳踏車
+螢幕
+行動電話
+視訊
+計程車
+記憶體
+貓熊
+資料庫
+資訊
+超商
+身分證
+軟體
+部落格
+錄影
+隨身碟
+雪梨
+雷射
+鳳梨

data/lib/twfilter/data/taiwan_markers.txt

@@ -0,0 +1,66 @@
+中華民國
+交通部
+便利商店
+健保
+僑委會
+內政部
+勞保
+南投
+台中
+台北
+台南
+台東
+台灣
+台鐵
+司法院
+嘉義
+國中
+國小
+國防部
+基隆
+學測
+宜蘭
+屏東
+彰化
+悠遊卡
+戶政
+技專校院
+指考
+教育部
+文化部
+新北
+新台幣
+新竹
+新臺幣
+會考
+桃園
+民國
+澎湖
+環保署
+監察院
+立法院
+統一發票
+統測
+經濟部
+縣市政府
+總統府
+考試院
+臺中
+臺北
+臺南
+臺東
+臺灣
+花蓮
+苗栗
+行政院
+衛福部
+農委會
+鄉鎮市
+里長
+金管會
+金門
+陸委會
+雲林
+馬祖
+高職
+高雄

data/lib/twfilter/data/taiwan_particles.txt

@@ -0,0 +1,7 @@
+咧
+唷
+喔
+囉
+欸
+蛤
+齁

data/lib/twfilter/data/variants_used_in_taiwan.txt

@@ -0,0 +1,16 @@
+峰
+庄
+床
+昵
+晒
+洒
+灶
+痒
+痴
+秘
+粽
+羡
+群
+肴
+虱
+霉

data/lib/twfilter/data/wenyan.txt

@@ -0,0 +1,4 @@
+吾
+哉
+汝
+矣

data/lib/twfilter/errors.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module TWFilter
+  class Error < ::StandardError
+  end
+
+  class MissingTableError < Error
+  end
+
+  class InvalidTableError < Error
+  end
+end

data/lib/twfilter/evidence.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module TWFilter
+  module Evidence
+    # 沒有 V and 有沒有 V are pan-Mandarin negation and A-not-A, not the Southern Min substrate
+    # perfective. One lookbehind covers both: in 有沒有看 the 有看 match is also preceded by 沒.
+    NEGATED = "沒"
+
+    class << self
+      def count(text) = scanners.sum { |scanner| text.scan(scanner).length }
+
+      def per_100(text)
+        han = Han.count(text)
+        han.zero? ? 0.0 : count(text) * 100.0 / han
+      end
+
+      def hits(text) = terms.zip(scanners).filter_map { |term, scanner| term if found?(text, scanner) }
+
+      def markers(text) = Tables.rows("taiwan_markers.txt").select { |term| text.include?(term) }
+
+      def reset! = @terms = @scanners = nil
+
+      def terms
+        @terms ||= (Tables.rows("taiwan_lexicon.txt") +
+          Tables.rows("taiwan_particles.txt") +
+          Tables.rows("taiwan_markers.txt") +
+          Tables.rows("taiwan_grammar.txt"))
+          .uniq
+          .freeze
+      end
+
+      # A plain string scans faster than an equivalent regexp; only 有-initial terms need one.
+      def scanners
+        @scanners ||= terms.map { |term|
+          term.start_with?("有") ? /(?<!#{NEGATED})#{Regexp.escape(term)}/ : term
+        }.freeze
+      end
+
+      private
+
+      def found?(text, scanner) = scanner.is_a?(Regexp) ? scanner.match?(text) : text.include?(scanner)
+    end
+  end
+end

data/lib/twfilter/finding.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module TWFilter
+  # Offending items quoted in a finding detail. The check fires on all of them; this only
+  # bounds log width.
+  DETAIL_SAMPLE = 4
+
+  Finding = Data.define(:check, :code, :severity, :detail) do
+    def initialize(check:, code:, severity: :reject, detail: nil)
+      super
+    end
+
+    def reject? = severity == :reject
+
+    def mark? = severity == :mark
+
+    def to_h = {check: check.to_s, code: code.to_s, severity: severity.to_s, detail: detail}.compact
+  end
+
+  Report = Data.define(:text, :findings, :tier, :han, :evidence) do
+    def ok? = findings.none?(&:reject?)
+
+    def rejects = findings.select(&:reject?)
+
+    def marks = findings.select(&:mark?)
+
+    def codes = findings.map(&:code)
+
+    def reasons = rejects.map { |finding| [finding.code, finding.detail].compact.join(": ") }
+
+    def to_h
+      {
+        text: text,
+        ok: ok?,
+        tier: tier,
+        han: han,
+        evidence: evidence,
+        findings: findings.map(&:to_h)
+      }
+    end
+  end
+end

data/lib/twfilter/han.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module TWFilter
+  module Han
+    CHAR = /\p{Han}/
+    RUN = /\p{Han}+/
+    # CJK Unified Ideographs and Extension A — the planes a Taiwanese font is expected to
+    # render. \p{Han} is wider; .basic? distinguishes the two.
+    BMP = (0x4E00..0x9FFF).freeze
+    EXTENSION_A = (0x3400..0x4DBF).freeze
+
+    module_function
+
+    def char?(char) = char.match?(CHAR)
+
+    def count(text) = text.each_char.count { |char| char.match?(CHAR) }
+
+    def runs(text) = text.scan(RUN)
+
+    def ratio(text)
+      length = text.length
+      length.zero? ? 0.0 : count(text).fdiv(length)
+    end
+
+    def basic?(char)
+      code = char.ord
+      BMP.cover?(code) || EXTENSION_A.cover?(code)
+    end
+  end
+end

data/lib/twfilter/policy.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module TWFilter
+  # 教育部 chart indices, ordered by decreasing frequency of use.
+  TIERS = {common: 0, secondary: 1, rare: 2}.freeze
+
+  Policy = Data.define(
+    :han_range,
+    :min_han_ratio,
+    :max_tier,
+    :allow_latin,
+    :orthography_rejects,
+    :foreign_topics_reject,
+    :punctuation,
+    :soft_lexicon_rejects,
+    :wenyan_min_han,
+    :wenyan_min_hits,
+    :wenyan_max_density,
+    :block_tolerance,
+    :evidence_per_100
+  ) do
+    # Defaults impose no shape constraint. Literary-Chinese density fires only above 8 Han
+    # characters and 2 particle hits, since the ratio is unstable on shorter spans; 0.05 is
+    # the observed ceiling for modern prose quoting classical text. Block tolerance 0.0
+    # rejects a window on one failing member; 1.0 evidence per 100 sentences is the floor
+    # below which a window carries no Taiwan-specific signal.
+    def initialize(
+      han_range: (1..Float::INFINITY),
+      min_han_ratio: 0.0,
+      max_tier: :rare,
+      allow_latin: true,
+      orthography_rejects: true,
+      foreign_topics_reject: false,
+      punctuation: :wide,
+      soft_lexicon_rejects: false,
+      wenyan_min_han: 8,
+      wenyan_min_hits: 2,
+      wenyan_max_density: 0.05,
+      block_tolerance: 0.0,
+      evidence_per_100: 1.0
+    )
+      super
+    end
+
+    def tier_limit = TIERS.fetch(max_tier)
+
+    class << self
+      # 6..60 Han characters spans a usable sentence; 0.65 Han excludes list and table
+      # fragments while admitting embedded Latin and numerals.
+      def corpus = new(han_range: (6..60), min_han_ratio: 0.65)
+
+      # Tighter bounds for material shown to learners: 40 Han is a readable card, 0.70
+      # suppresses code-mixed text, and the 常用 chart is the pedagogical inventory.
+      def publishable
+        new(
+          han_range: (6..40),
+          min_han_ratio: 0.70,
+          max_tier: :common,
+          soft_lexicon_rejects: true,
+          punctuation: :strict,
+          foreign_topics_reject: true
+        )
+      end
+
+      def permissive = new
+    end
+  end
+end

data/lib/twfilter/punctuation.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+module TWFilter
+  module Punctuation
+    TAIWAN = "，。、；：？！「」『』（）〔〕【】《》〈〉──……‧—".chars.to_set.freeze
+    LATIN = ".,;:?!()[]{}\"'-/%&+=*\#@_<>|$^`\\".chars.to_set.freeze
+    SPACE = " \t\u{3000}\u{00A0}".chars.to_set.freeze
+    # STRICT is the 教育部 handbook inventory; WIDE adds marks in common Taiwanese use but outside it.
+    STRICT = (TAIWAN | LATIN | SPACE | "…⋯．·─".chars.to_set).freeze
+    WIDE = (STRICT | "～~℃°※○●–→§±×÷№‼⁉".chars.to_set).freeze
+    SETS = {strict: STRICT, wide: WIDE}.freeze
+    ALLOWED = WIDE
+
+    # Bopomofo and its extended block: the Taiwanese phonetic script, admitted as text.
+    BOPOMOFO = [0x3100..0x312F, 0x31A0..0x31BF].freeze
+    TONE_MARKS = "\u{02CA}\u{02C7}\u{02CB}\u{02D9}\u{02C9}".chars.to_set.freeze
+    INVISIBLE = /[\u{200B}-\u{200F}\u{2028}\u{2029}\u{FEFF}\u{FE00}-\u{FE0F}\u{00AD}]/
+
+    WIDTH = {
+      "＂" => "\"",
+      "＃" => "#",
+      "＄" => "$",
+      "％" => "%",
+      "＆" => "&",
+      "＇" => "'",
+      "＊" => "*",
+      "＋" => "+",
+      "－" => "-",
+      "．" => ".",
+      "／" => "/",
+      "＜" => "<",
+      "＝" => "=",
+      "＞" => ">",
+      "＠" => "@",
+      "［" => "[",
+      "＼" => "\\",
+      "］" => "]",
+      "＾" => "^",
+      "＿" => "_",
+      "｀" => "`",
+      "｛" => "{",
+      "｜" => "|",
+      "｝" => "}"
+    }.freeze
+
+    RULES = [
+      [INVISIBLE, ""],
+      [/[#{Regexp.escape(WIDTH.keys.join)}]/, WIDTH],
+      # Fullwidth digits and Latin letters fold to ASCII; 0xFEE0 is the fixed offset between the blocks.
+      [/[\u{FF10}-\u{FF19}\u{FF21}-\u{FF3A}\u{FF41}-\u{FF5A}]/, -> (m) { (m.ord - 0xFEE0).chr(Encoding::UTF_8) }],
+      [/[\u{201C}\u{301D}]/, "「"],
+      [/[\u{201D}\u{301E}\u{301F}]/, "」"],
+      [/\u{2018}(?=[^\u{2019}]*\p{Han})/, "『"],
+      [/(?<=\p{Han})\u{2019}/, "』"],
+      [/\u{FE50}/, "，"],
+      [/\u{FE55}|\u{FE30}|\u{FE13}/, "："],
+      [/[\u{30FB}\u{FF65}]/, "\u{2027}"],
+      [/(?<=\p{Han})[\u{00B7}\u{2022}\u{2219}]|[\u{00B7}\u{2022}\u{2219}](?=\p{Han})/, "\u{2027}"],
+      [/(?:\u{2026}|\u{22EF}|。。|\.{3,}|・{3,}){1,}/, "\u{2026}\u{2026}"],
+      [/[\u{2014}\u{2015}\u{2500}]{1,}|(?<=\p{Han})--(?=\p{Han})/, "\u{2500}\u{2500}"],
+      ["\u{301C}", "～"],
+      [/(?<=\p{Han}),(?=\s|\p{Han}|\z)/, "，"],
+      [/(?<=\p{Han})\.(?=\s|\p{Han}|\z)/, "。"],
+      [/(?<=\p{Han})\?(?=\s|\p{Han}|\z)/, "？"],
+      [/(?<=\p{Han})!(?=\s|\p{Han}|\z)/, "！"],
+      [/(?<=\p{Han});(?=\s|\p{Han}|\z)/, "；"],
+      [/(?<=\p{Han}):(?=\s|\p{Han}|\z)/, "："],
+      [/(?<=\p{Han})\s+(?=\p{Han})/, ""],
+      [/\s+(?=[，。、；：？！」』）〕】》〉])/, ""],
+      [/(?<=[，。、；：？！「『（〔【《〈])\s+/, ""],
+      [/[ \t\u{00A0}]{2,}/, " "]
+    ].freeze
+
+    COLLAPSE = /([，。！？；：])\1+/
+
+    module_function
+
+    def normalize(text, collapse_repeats: true)
+      value = RULES.reduce(text.to_s.unicode_normalize(:nfc)) { |memo, (pattern, replacement)|
+        case replacement
+        in Proc
+          memo.gsub(pattern) { replacement.call(Regexp.last_match(0)) }
+        in Hash
+          memo.gsub(pattern, replacement)
+        in String
+          memo.gsub(pattern, replacement)
+        end
+      }
+      value = value.gsub(COLLAPSE, "\\1") if collapse_repeats
+      value.strip
+    end
+
+    def acceptable?(char, set: WIDE)
+      return true if Han.char?(char) || set.include?(char) || TONE_MARKS.include?(char)
+      return true if char.match?(/[A-Za-z0-9]/)
+
+      code = char.ord
+      BOPOMOFO.any? { |range| range.cover?(code) }
+    end
+
+    def offenders(text, punctuation: :wide)
+      set = SETS.fetch(punctuation)
+      text.each_char.reject { |char| acceptable?(char, set: set) }.uniq
+    end
+  end
+end

data/lib/twfilter/sentences.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module TWFilter
+  module Sentences
+    # A boundary is a run of terminators followed by any closing brackets; 、 is not a boundary.
+    TERMINATORS = "。！？…‼⁉"
+    CLAUSE = "；"
+    CLOSERS = "」』）〕】》〉\u{201D}\u{2019}"
+
+    module_function
+
+    PIECES = {
+      true => /[^#{TERMINATORS}#{CLAUSE}]*[#{TERMINATORS}#{CLAUSE}]+[#{CLOSERS}]*|[^#{TERMINATORS}#{CLAUSE}]+/,
+      false => /[^#{TERMINATORS}]*[#{TERMINATORS}]+[#{CLOSERS}]*|[^#{TERMINATORS}]+/
+    }.freeze
+
+    def split(text, clause: true)
+      text
+        .to_s
+        .scan(PIECES.fetch(clause))
+        .map { |piece| piece.gsub(/\s+/, " ").strip }
+        .reject(&:empty?)
+    end
+
+    def terminal?(text) = TERMINATORS.chars.any? { |mark| text.end_with?(mark) }
+
+    def shaped?(text, policy: Policy.corpus)
+      han = Han.count(text)
+      policy.han_range.cover?(han) && Han.ratio(text) >= policy.min_han_ratio
+    end
+  end
+end

data/lib/twfilter/subject.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module TWFilter
+  class Subject
+    attr_reader :text, :policy
+
+    def initialize(text, policy: Policy.corpus)
+      @text = text.to_s
+      @policy = policy
+    end
+
+    def chars = @chars ||= text.chars
+
+    def han_chars = @han_chars ||= chars.select { |char| Han.char?(char) }
+
+    def han = @han ||= han_chars.length
+
+    def ratio = @ratio ||= text.empty? ? 0.0 : han.fdiv(text.length)
+
+    def empty? = text.strip.empty?
+
+    def include?(term) = text.include?(term)
+
+    def masked(exceptions)
+      exceptions.reduce(text) { |memo, allowed| memo.gsub(allowed, "") }
+    end
+  end
+end

data/lib/twfilter/tables.rb

@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+
+require "digest"
+require "json"
+require "pathname"
+require "set"
+
+module TWFilter
+  # Reference tables. The bundled copy is the source of truth for behavior and is versioned
+  # with the gem; .dir replaces it wholesale and .overlay supplements it. Any deviation from
+  # the bundled copy changes what "Taiwan Mandarin" means, so both are reported by .provenance.
+  module Tables
+    BUNDLED = Pathname(__dir__).join("data").freeze
+    MANIFEST = "MANIFEST.json"
+    ADD = ".add"
+    REMOVE = ".remove"
+
+    # Row shape per table; unknown names default to :terms so user tables are accepted.
+    SCHEMA = {
+      "moe_common.txt" => :chars,
+      "moe_secondary.txt" => :chars,
+      "moe_rare.txt" => :chars,
+      "moe_exception.txt" => :chars,
+      "simplified_only.txt" => :chars,
+      "variants_used_in_taiwan.txt" => :chars,
+      "converted_orthography.txt" => :chars,
+      "cantonese.txt" => :chars,
+      "wenyan.txt" => :chars,
+      "mainland_hard.tsv" => :pairs,
+      "mainland_soft.tsv" => :pairs,
+      "mainland_exceptions.tsv" => :pairs,
+      "regional_hard.tsv" => :pairs
+    }.freeze
+
+    class << self
+      attr_reader :overlay
+
+      def dir = @dir ||= Pathname(ENV.fetch("TWFILTER_DATA_DIR", BUNDLED.to_s))
+
+      def dir=(path)
+        @dir = path.nil? ? nil : Pathname(path)
+        TWFilter.reset!
+      end
+
+      def overlay=(path)
+        @overlay = path.nil? ? nil : Pathname(path)
+        TWFilter.reset!
+      end
+
+      def rows(name) = cache[name] ||= resolve(name)
+
+      def set(name) = cache[:"#{name}/set"] ||= rows(name).to_set
+
+      def pairs(name) = cache[:"#{name}/pairs"] ||= rows(name).to_h { |row| row.split("\t", 2) }.freeze
+
+      def columns(name) = cache[:"#{name}/columns"] ||= rows(name).map { |row| row.split("\t") }.freeze
+
+      def groups(name) = cache[:"#{name}/groups"] ||= columns(name)
+        .to_h { |row| [row.first, row.drop(1).freeze] }
+        .freeze
+
+      def names
+        base = dir.children.map { |path| path.basename.to_s }
+        extra = overlay ? overlay.children.map { |path|
+          path.basename.to_s.sub(/#{Regexp.escape(ADD)}|#{Regexp.escape(REMOVE)}(?=\.)/, "")
+        } : []
+        (base + extra).uniq.reject { |name| name == MANIFEST }.sort
+      end
+
+      def manifest
+        path = dir.join(MANIFEST)
+        path.exist? ? JSON.parse(path.read).freeze : {}
+      end
+
+      # Fingerprint of the effective tables, for recording alongside derived measurements.
+      def provenance
+        digest = Digest::SHA256.new
+        names.each { |name| digest << name << rows(name).join("\n") }
+
+        {
+          version: VERSION,
+          dir: dir.to_s,
+          overlay: overlay&.to_s,
+          tables: names.length,
+          # 64 bits, sufficient to detect an accidental table change
+          digest: digest.hexdigest[0, 16]
+        }.freeze
+      end
+
+      def reset!
+        @cache = nil
+        ENV["TWFILTER_OVERLAY_DIR"]&.then { |path| @overlay ||= Pathname(path) }
+      end
+
+      private
+
+      def cache = @cache ||= {}
+
+      def resolve(name)
+        base = overlay&.join(name)&.exist? ? read(overlay.join(name), name) : read(dir.join(name), name)
+        return base.freeze if overlay.nil?
+
+        added = optional(overlay.join(with_suffix(name, ADD)), name)
+        removed = optional(overlay.join(with_suffix(name, REMOVE)), nil).to_set
+
+        (base + added).reject { |row| removed.include?(row) || removed.include?(row.split("\t").first) }.uniq.freeze
+      end
+
+      def with_suffix(name, suffix)
+        extension = File.extname(name)
+        "#{File.basename(name, extension)}#{suffix}#{extension}"
+      end
+
+      def optional(path, name) = path.exist? ? read(path, name) : []
+
+      def read(path, name)
+        raise MissingTableError, "missing table: #{path}" unless path.exist?
+
+        rows = path.each_line.map(&:chomp).reject(&:empty?)
+        validate(rows, SCHEMA.fetch(name, :terms), path) unless name.nil?
+        rows
+      end
+
+      def validate(rows, shape, path)
+        offender = case shape
+        in :chars
+          rows.find { |row| row.length != 1 || !Han.char?(row) }
+        in :pairs
+          rows.find { |row| row.split("\t").length < 2 || row.start_with?("\t") }
+        in :terms
+          rows.find { |row| row.include?("\t") }
+        end
+
+        return if offender.nil?
+
+        raise InvalidTableError, "#{path}: expected #{shape}, got #{offender.inspect}"
+      end
+    end
+  end
+end

data/lib/twfilter/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module TWFilter
+  VERSION = "0.1.0"
+end