twpipeline 0.1.0

data/sig/twpipeline.rbs

@@ -0,0 +1,228 @@
+module TWPipeline
+  VERSION: String
+  ROOT: Pathname
+
+  type record = ::Hash[::Symbol, untyped]
+
+  class Error < ::StandardError
+  end
+
+  def self.resources: () -> Resources
+  def self.resources=: (Resources?) -> Resources?
+  def self.work: () -> Pathname
+  def self.data: (*::String parts) -> Pathname
+  def self.warm_up: () -> singleton(TWPipeline)
+
+  class Resources
+    UNITS: ::Hash[::String, ::Integer]
+    DEFAULT_SHARE: ::Float
+
+    attr_reader cores: ::Integer
+    attr_reader memory_bytes: ::Integer
+
+    def self.detect: (?cores: untyped, ?memory: untyped) -> Resources
+    def self.ask: (?io: ::IO, ?input: ::IO) -> Resources
+    def self.parse: (untyped value) -> ::Integer?
+    def self.total_memory: () -> ::Integer
+
+    def memory: () -> ::String
+    def sort_buffer: () -> ::String
+    def memory_per_worker_bytes: () -> ::Integer
+    def to_h: () -> ::Hash[::Symbol, untyped]
+    def to_s: () -> ::String
+  end
+
+  module Parallel
+    CHUNK_CAP: ::Integer
+    CHUNK_SHARE: ::Integer
+    LINE_CAP: ::Integer
+
+    def self?.map: [T, R] (::Enumerable[T], ?workers: ::Integer) { (T) -> R } -> ::Array[R]
+    def self?.fold: [T, R] (::Enumerable[T], ?workers: ::Integer) { (::Array[T]) -> R } -> ::Array[R]
+    def self?.chunk_bytes: (?Resources) -> ::Integer
+    def self?.pipe: (untyped input, untyped output, ?workers: ::Integer, ?budget: ::Integer) { (::String) -> untyped } -> ::Hash[::Symbol, ::Integer]
+    def self?.slices: [T] (::Array[T], ::Integer workers) -> ::Array[::Array[T]]
+    def self?.spawn: [T, R] (::Array[T]) { (T) -> R } -> [::Integer, ::IO]
+    def self?.collect: (::Integer pid, ::IO reader) -> untyped
+  end
+
+  module Jsonl
+    def self?.parse: (::String line) -> record
+    def self?.dump: (record) -> ::String
+    def self?.each: (untyped io) { (record) -> void } -> void
+                  | (untyped io) -> ::Enumerator[record, void]
+    def self?.wrap: (::String text, ?source: ::String, ?index: ::Integer) -> record
+  end
+
+  module Sorting
+    def self?.program: () -> ::String
+    def self?.sort: (*::String extra, ?resources: Resources) -> ::String
+    def self?.unique_pipeline: (floor: ::Integer, resources: Resources, ?collapse: bool) -> ::String
+    def self?.tally: (untyped output_path, ?floor: ::Integer, ?resources: Resources, ?collapse: bool) { (^(::String) -> void) -> void } -> ::Hash[::Symbol, untyped]
+  end
+
+  module Bench
+    class Sample
+      attr_reader label: ::String
+      attr_reader seconds: ::Float
+      attr_reader cpu_seconds: ::Float
+      attr_reader peak_rss_bytes: ::Integer
+      attr_reader facts: ::Hash[::Symbol, untyped]
+
+      def cores_used: () -> ::Float
+      def to_h: () -> ::Hash[::Symbol, untyped]
+    end
+
+    def self?.path: () -> Pathname
+    def self?.measure: (::String label, **untyped facts) { () -> untyped } -> Sample
+    def self?.cpu_seconds: () -> ::Float
+    def self?.peak_rss: () -> ::Integer
+    def self?.record: (Sample) -> Sample
+  end
+
+  module Rusage
+    SELF: ::Integer
+    CHILDREN: ::Integer
+    MAXRSS_OFFSET: ::Integer
+    SCALE: ::Integer
+
+    def self.available?: () -> bool
+    def self.max_rss: (::Integer who) -> ::Integer
+    def self.peak: () -> ::Integer
+  end
+
+  module Frequency
+    def self?.dispersion: (::Hash[::String?, ::Integer] counts, ::Hash[::String?, ::Integer] sizes) -> ::Float
+    def self?.each_key: (untyped path) { ([::String, ::Hash[::String?, ::Integer]]) -> void } -> void
+                      | (untyped path) -> ::Enumerator[[::String, ::Hash[::String?, ::Integer]], void]
+    def self?.table: (untyped path, ::Hash[::String?, ::Integer] sizes) -> ::Array[::Hash[::Symbol, untyped]]
+  end
+
+  class Segmenter
+    MAX_WORD: ::Integer
+    HAN_RUN: ::Regexp
+
+    class Model
+      attr_reader cost: ::Hash[::String, ::Float]
+      attr_reader char_cost: ::Hash[::String, ::Float]
+      attr_reader unseen_word: ::Float
+      attr_reader unseen_char: ::Float
+      attr_reader max_word: ::Integer
+    end
+
+    attr_reader words: ::Set[::String]
+    attr_reader model: Model?
+
+    def self.load: (untyped path) -> Segmenter
+    def self.entries: (untyped value) -> ::Set[::String]
+    def self.runs: (::String text) -> ::Array[::String]
+
+    def initialize: (words: ::Set[::String], ?chars: ::Set[::String], ?model: Model?) -> void
+    def fit: (::Array[::String] runs, ?rounds: ::Integer) -> self
+    def call: (::String text) -> ::Array[::String]
+    def segment: (::String run) -> ::Array[::String]
+  end
+
+  class Stage
+    REGISTRY: ::Hash[::String, singleton(Stage)]
+
+    def self.register: (::String slug, ::Integer order) -> singleton(Stage)
+    def self.slug: () -> ::String
+    def self.order: () -> ::Integer
+    def self.ordered: () -> ::Array[singleton(Stage)]
+    def self.lookup: (::String name) -> singleton(Stage)
+
+    attr_reader policy: untyped
+    attr_reader resources: Resources
+    attr_reader options: ::Hash[::Symbol, untyped]
+
+    def initialize: (?policy: untyped, ?drop: bool, ?resources: Resources, **untyped options) -> void
+    def drop?: () -> bool
+    def call: (untyped input, untyped output) -> untyped
+
+    private
+
+    def reject: (record, ::Array[untyped] findings) -> record
+    def flag: (record, ::Array[untyped] findings) -> record
+  end
+
+  class MapStage < Stage
+    def process: (::String line) -> ::String?
+    def transform: (record) -> record?
+  end
+
+  module Stages
+    class Ingest < Stage
+    end
+
+    class Normalize < MapStage
+    end
+
+    class Segment < Stage
+      CARRIED: ::Array[::Symbol]
+    end
+
+    class Script < MapStage
+    end
+
+    class Lexicon < MapStage
+      CHECKS: ::Array[untyped]
+    end
+
+    class Evidence < Stage
+      BATCH: ::Integer
+    end
+
+    class Dedup < Stage
+      MASK: ::Integer
+    end
+
+    class Tokenize < MapStage
+      FIT_LINES: ::Integer
+    end
+
+    class Count < Stage
+      DEFAULT_ORDERS: ::Array[::Integer]
+    end
+  end
+
+  module Sources
+    class HPLT
+      EXCLUDED_TLDS: ::Set[::String]
+      TAIWAN_TLD: ::String
+      TAIWAN_LANG: ::Regexp
+      URL_HEAD: ::Integer
+      URL: ::Regexp
+      HTML_LANG: ::Regexp
+      REGISTERS: ::Array[::String]
+
+      class Decision
+        attr_reader keep: bool
+        attr_reader gate: ::Symbol
+        attr_reader host: ::String?
+      end
+
+      def self.open: (untyped path) { (::IO) -> untyped } -> untyped
+      def self.allowlist: (untyped path) -> ::Set[::String]
+
+      def initialize: (?allowlist: ::Set[::String]) -> void
+      def each_record: (untyped io) { (record) -> void } -> void
+                     | (untyped io) -> ::Enumerator[record, void]
+      def decide: (::String line) -> Decision
+      def host_of: (::String line) -> ::String?
+      def probe: (::String line, ::Integer size) -> ::String
+      def taiwan_lang?: (::String line) -> bool
+      def build: (::String line, Decision decision) -> record
+      def score: (record document) -> ::Float?
+      def register: (record document) -> ::String?
+    end
+  end
+
+  class CLI
+    COMMANDS: ::Array[::String]
+
+    def self.call: (::Array[::String] argv) -> ::Integer
+    def initialize: (::Array[::String] argv) -> void
+    def call: () -> ::Integer
+  end
+end

metadata

@@ -0,0 +1,149 @@
+--- !ruby/object:Gem::Specification
+name: twpipeline
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Den Patin
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fiddle
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: twfilter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.1'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.4'
+- !ruby/object:Gem::Dependency
+  name: rbs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.1'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.13'
+description: 'Nine JSONL stages from a raw web crawl to dispersion-corrected frequency
+  and n-gram tables: source gating, Taiwan-standard punctuation normalization, sentence
+  segmentation, character-inventory and lexical-origin filtering, block-level evidence
+  judgment, type deduplication, Viterbi word segmentation and external-sort counting.
+  Resource-bounded by declared cores and memory.'
+email: hi@dpat.in
+executables:
+- twpipeline
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- LICENSE
+- README.md
+- bin/twpipeline
+- data/hplt/README.md
+- data/hplt/taiwan_domains.candidates.txt
+- data/hplt/taiwan_domains.txt
+- lib/twpipeline.rb
+- lib/twpipeline/bench.rb
+- lib/twpipeline/cli.rb
+- lib/twpipeline/frequency.rb
+- lib/twpipeline/jsonl.rb
+- lib/twpipeline/parallel.rb
+- lib/twpipeline/resources.rb
+- lib/twpipeline/rusage.rb
+- lib/twpipeline/segmenter.rb
+- lib/twpipeline/sorting.rb
+- lib/twpipeline/sources/hplt.rb
+- lib/twpipeline/stage.rb
+- lib/twpipeline/stages/count.rb
+- lib/twpipeline/stages/dedup.rb
+- lib/twpipeline/stages/evidence.rb
+- lib/twpipeline/stages/ingest.rb
+- lib/twpipeline/stages/lexicon.rb
+- lib/twpipeline/stages/normalize.rb
+- lib/twpipeline/stages/script.rb
+- lib/twpipeline/stages/segment.rb
+- lib/twpipeline/stages/tokenize.rb
+- lib/twpipeline/version.rb
+- sig/twpipeline.rbs
+homepage: https://github.com/taiwan-corpora/twpipeline
+licenses:
+- MIT
+metadata:
+  source_code_uri: https://github.com/taiwan-corpora/twpipeline
+  changelog_uri: https://github.com/taiwan-corpora/twpipeline/blob/main/CHANGELOG.md
+  bug_tracker_uri: https://github.com/taiwan-corpora/twpipeline/issues
+  documentation_uri: https://huggingface.co/datasets/taiwan-corpora/twngrams
+  rubygems_mfa_required: 'true'
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.4'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 4.0.17
+specification_version: 4
+summary: Corpus construction pipeline for Taiwan Mandarin
+test_files: []