static_embeddings 0.1.4 → 1.5.6

data/ext/static_embeddings/se_tokenizer.c

@@ -87,16 +87,17 @@ static int push_id(se_scratch_t *sc, size_t *n_ids, uint32_t id) {
     return 1;
 }
 
-#define SE_SCRATCH_CPS_KEEP   256
-#define SE_SCRATCH_IDS_KEEP   512
-#define SE_SCRATCH_BYTES_KEEP 1024
+#define SE_SCRATCH_CPS_KEEP       256
+#define SE_SCRATCH_IDS_RESERVE    512
+#define SE_SCRATCH_IDS_RETAIN_MAX 8192
+#define SE_SCRATCH_BYTES_KEEP     1024
 
 int se_scratch_reserve(se_scratch_t *s, uint32_t dim) {
     if (!grow_u32(&s->cps, &s->cps_cap, SE_SCRATCH_CPS_KEEP))
         return 0;
     if (!grow_u32(&s->cps2, &s->cps2_cap, SE_SCRATCH_CPS_KEEP))
         return 0;
-    if (!grow_u32(&s->ids, &s->ids_cap, SE_SCRATCH_IDS_KEEP))
+    if (!grow_u32(&s->ids, &s->ids_cap, SE_SCRATCH_IDS_RESERVE))
         return 0;
     if (!grow_bytes(&s->bytes, &s->bytes_cap, SE_SCRATCH_BYTES_KEEP))
         return 0;
@@ -149,7 +150,8 @@ static int shrink_bytes(uint8_t **buf, size_t *cap, size_t keep) {
 static void se_scratch_trim(se_scratch_t *s) {
     (void)shrink_u32(&s->cps, &s->cps_cap, SE_SCRATCH_CPS_KEEP);
     (void)shrink_u32(&s->cps2, &s->cps2_cap, SE_SCRATCH_CPS_KEEP);
-    (void)shrink_u32(&s->ids, &s->ids_cap, SE_SCRATCH_IDS_KEEP);
+    if (s->ids_cap > SE_SCRATCH_IDS_RETAIN_MAX)
+        (void)shrink_u32(&s->ids, &s->ids_cap, SE_SCRATCH_IDS_RETAIN_MAX);
     (void)shrink_bytes(&s->bytes, &s->bytes_cap, SE_SCRATCH_BYTES_KEEP);
 }
 
@@ -393,6 +395,37 @@ static int cp_is_cjk_segment(const se_model_t *m, uint32_t cp) {
     return m->meta.handle_chinese_chars && cp >= 0x3400 && se_is_cjk(cp);
 }
 
+typedef struct {
+    const char *text;
+    uint32_t len;
+    uint32_t bit;
+} se_added_token_spec_t;
+
+static const se_added_token_spec_t SE_ADDED_TOKENS[] = {
+    {"[MASK]", 6u, SE_ADDED_MASK}, {"[PAD]", 5u, SE_ADDED_PAD}, {"[UNK]", 5u, SE_ADDED_UNK},
+    {"[CLS]", 5u, SE_ADDED_CLS},   {"[SEP]", 5u, SE_ADDED_SEP},
+};
+
+static int boundary_splits_added_token(const se_model_t *model, const uint8_t *input,
+                                       size_t input_len, size_t boundary) {
+    if (model->meta.added_token_mask == 0 || boundary == 0 || boundary >= input_len)
+        return 0;
+
+    for (size_t k = 0; k < SE_ARRAY_LEN(SE_ADDED_TOKENS); k++) {
+        const se_added_token_spec_t *token = &SE_ADDED_TOKENS[k];
+        if ((model->meta.added_token_mask & token->bit) == 0)
+            continue;
+        for (size_t back = 1; back < token->len && back <= boundary; back++) {
+            size_t start = boundary - back;
+            if (start + token->len > input_len || input[start] != '[')
+                continue;
+            if (memcmp(input + start, token->text, token->len) == 0)
+                return 1;
+        }
+    }
+    return 0;
+}
+
 size_t se_prefix_boundary_len(const se_model_t *model, const uint8_t *input, size_t input_len,
                               size_t target, size_t backscan) {
     if (target >= input_len)
@@ -420,14 +453,15 @@ size_t se_prefix_boundary_len(const se_model_t *model, const uint8_t *input, siz
             continue;
 
         if (cp_is_cjk_segment(model, cp)) {
-            if (after <= target)
+            if (after <= target && !boundary_splits_added_token(model, input, input_len, after))
                 return after;
-            if (pos > 0)
+            if (pos > 0 && !boundary_splits_added_token(model, input, input_len, pos))
                 return pos;
-            return 0;
+            continue;
         }
 
-        if ((is_whitespace(model, cp) || is_punct(model, cp)) && after <= target)
+        if ((is_whitespace(model, cp) || is_punct(model, cp)) && after <= target &&
+            !boundary_splits_added_token(model, input, input_len, after))
             return after;
     }
 
@@ -454,6 +488,7 @@ typedef struct {
     const se_model_t *model;
     se_scratch_t *scratch;
     uint32_t max_tokens;
+    se_token_limit_t limit_mode;
     size_t n_ids;
     size_t n_unk;
     size_t segment_len;
@@ -481,17 +516,44 @@ static int append_segment_cp(token_state_t *st, uint32_t cp) {
     return 1;
 }
 
+static size_t limited_token_count(const token_state_t *st) {
+    if (st->limit_mode == SE_TOKEN_LIMIT_USABLE && st->model->meta.unk_policy == SE_UNK_DROP)
+        return st->n_ids - st->n_unk;
+    return st->n_ids;
+}
+
 static int cap_after_append(token_state_t *st) {
-    if (st->max_tokens == 0 || st->n_ids <= (size_t)st->max_tokens)
+    if (st->max_tokens == 0 || limited_token_count(st) <= (size_t)st->max_tokens)
         return 0;
 
-    size_t dropped_unk = 0;
-    for (size_t k = st->max_tokens; k < st->n_ids; k++) {
-        if (st->scratch->ids[k] == st->model->meta.unk_id)
-            dropped_unk++;
+    if (st->limit_mode == SE_TOKEN_LIMIT_RAW || st->model->meta.unk_policy != SE_UNK_DROP) {
+        size_t dropped_unk = 0;
+        for (size_t k = st->max_tokens; k < st->n_ids; k++) {
+            if (st->scratch->ids[k] == st->model->meta.unk_id)
+                dropped_unk++;
+        }
+        st->n_unk -= dropped_unk;
+        st->n_ids = st->max_tokens;
+    } else {
+        size_t usable = 0;
+        size_t kept_unk = 0;
+        size_t keep = 0;
+        for (size_t k = 0; k < st->n_ids; k++) {
+            uint32_t id = st->scratch->ids[k];
+            if (id == st->model->meta.unk_id) {
+                kept_unk++;
+                continue;
+            }
+            usable++;
+            if (usable == (size_t)st->max_tokens) {
+                keep = k + 1;
+                break;
+            }
+        }
+        st->n_ids = keep;
+        st->n_unk = kept_unk;
     }
-    st->n_unk -= dropped_unk;
-    st->n_ids = st->max_tokens;
+
     st->stats->truncated = 1;
     return 1;
 }
@@ -602,6 +664,54 @@ static wordpiece_status_t wordpiece(const se_model_t *m, se_scratch_t *sc, const
     return 1;
 }
 
+static uint32_t added_token_id(const se_model_t *model, uint32_t bit) {
+    switch (bit) {
+    case SE_ADDED_PAD:
+        return model->meta.pad_id;
+    case SE_ADDED_UNK:
+        return model->meta.unk_id;
+    case SE_ADDED_CLS:
+        return model->meta.cls_id;
+    case SE_ADDED_SEP:
+        return model->meta.sep_id;
+    case SE_ADDED_MASK:
+        return model->meta.mask_id;
+    default:
+        return SE_SLOT_EMPTY;
+    }
+}
+
+static int match_added_token(const se_model_t *model, const uint8_t *input, size_t input_len,
+                             size_t pos, uint32_t *id_out, size_t *len_out) {
+    if (model->meta.added_token_mask == 0 || pos >= input_len || input[pos] != '[')
+        return 0;
+
+    for (size_t k = 0; k < SE_ARRAY_LEN(SE_ADDED_TOKENS); k++) {
+        const se_added_token_spec_t *token = &SE_ADDED_TOKENS[k];
+        if ((model->meta.added_token_mask & token->bit) == 0)
+            continue;
+        if (token->len > input_len - pos)
+            continue;
+        if (memcmp(input + pos, token->text, token->len) != 0)
+            continue;
+
+        *id_out = added_token_id(model, token->bit);
+        *len_out = token->len;
+        return 1;
+    }
+    return 0;
+}
+
+static se_status_t append_direct_id(token_state_t *st, uint32_t id, int *stop) {
+    if (!push_id(st->scratch, &st->n_ids, id))
+        return oom(st, "adding a special token");
+    if (id == st->model->meta.unk_id)
+        st->n_unk++;
+    if (cap_after_append(st))
+        *stop = 1;
+    return SE_OK;
+}
+
 static se_status_t append_wordpiece(token_state_t *st, const uint32_t *word, size_t word_len,
                                     int *stop) {
     wordpiece_status_t wp =
@@ -752,6 +862,31 @@ static se_status_t tokenize_ascii_run(token_state_t *st, const uint8_t *input, s
         if (b >= 0x80)
             break;
 
+        if (b == '[' && st->model->meta.added_token_mask != 0) {
+            uint32_t added_id = 0;
+            size_t added_len = 0;
+            if (match_added_token(st->model, input, input_len, i, &added_id, &added_len)) {
+                se_status_t rc = flush_segment(st, stop);
+                if (rc != SE_OK) {
+                    *ip = i;
+                    return rc;
+                }
+                if (*stop) {
+                    *ip = i;
+                    return SE_OK;
+                }
+                rc = append_direct_id(st, added_id, stop);
+                if (rc != SE_OK) {
+                    *ip = i;
+                    return rc;
+                }
+                i += added_len;
+                if (*stop)
+                    break;
+                continue;
+            }
+        }
+
         if (i >= next_cancel_check) {
             if (token_cancelled(st)) {
                 *ip = i;
@@ -811,8 +946,9 @@ static se_status_t tokenize_ascii_run(token_state_t *st, const uint8_t *input, s
 }
 
 se_status_t se_tokenize(const se_model_t *model, se_scratch_t *sc, const uint8_t *input,
-                        size_t input_len, uint32_t max_tokens, se_token_stats_t *stats,
-                        se_error_t *err, volatile sig_atomic_t *cancelled) {
+                        size_t input_len, uint32_t max_tokens, se_token_limit_t limit_mode,
+                        se_token_stats_t *stats, se_error_t *err,
+                        volatile sig_atomic_t *cancelled) {
     memset(stats, 0, sizeof(*stats));
 
     token_state_t st;
@@ -820,6 +956,7 @@ se_status_t se_tokenize(const se_model_t *model, se_scratch_t *sc, const uint8_t
     st.model = model;
     st.scratch = sc;
     st.max_tokens = max_tokens;
+    st.limit_mode = limit_mode;
     st.stats = stats;
     st.err = err;
     st.cancelled = cancelled;

data/ext/static_embeddings/se_unicode.c

@@ -115,6 +115,6 @@ const se_map_entry_t *se_map_lookup(const se_map_entry_t *entries, uint32_t coun
 int se_is_cjk(uint32_t cp) {
     return (cp >= 0x4E00 && cp <= 0x9FFF) || (cp >= 0x3400 && cp <= 0x4DBF) ||
            (cp >= 0x20000 && cp <= 0x2A6DF) || (cp >= 0x2A700 && cp <= 0x2B73F) ||
-           (cp >= 0x2B740 && cp <= 0x2B81F) || (cp >= 0x2B820 && cp <= 0x2CEAF) ||
+           (cp >= 0x2B740 && cp <= 0x2B81F) || (cp >= 0x2B920 && cp <= 0x2CEAF) ||
            (cp >= 0xF900 && cp <= 0xFAFF) || (cp >= 0x2F800 && cp <= 0x2FA1F);
 }

data/ext/static_embeddings/static_embeddings.c

@@ -80,6 +80,7 @@ static ID id_blocking_p;
 static ID id_vector;
 static ID id_token_count;
 static ID id_unk_count;
+static ID id_pooled_count;
 static ID id_truncated;
 static ID id_dim;
 static ID id_allow_unfrozen;
@@ -1030,6 +1031,10 @@ static VALUE model_embed_with_stats(int argc, VALUE *argv, VALUE self) {
     rb_hash_aset(hash, ID2SYM(id_vector), vector);
     rb_hash_aset(hash, ID2SYM(id_token_count), UINT2NUM(stats.token_count));
     rb_hash_aset(hash, ID2SYM(id_unk_count), UINT2NUM(stats.unk_count));
+    uint32_t pooled_count = get_model(self)->model.meta.unk_policy == SE_UNK_DROP
+                                ? stats.token_count - stats.unk_count
+                                : stats.token_count;
+    rb_hash_aset(hash, ID2SYM(id_pooled_count), UINT2NUM(pooled_count));
     rb_hash_aset(hash, ID2SYM(id_truncated), stats.truncated ? Qtrue : Qfalse);
     return hash;
 }
@@ -1049,7 +1054,7 @@ typedef struct {
 static VALUE tokenize_scratch_body(VALUE arg) {
     tokenize_scratch_job_t *job = (tokenize_scratch_job_t *)(uintptr_t)arg;
     job->rc = se_tokenize(job->model, job->scratch, job->input, job->input_len, job->max_tokens,
-                          &job->stats, &job->err, NULL);
+                          SE_TOKEN_LIMIT_RAW, &job->stats, &job->err, NULL);
     if (job->rc != SE_OK)
         return Qnil;
 
@@ -1237,7 +1242,27 @@ static VALUE embed_token_ids_value(VALUE self, VALUE ids_value, VALUE max_tokens
     size_t n = (size_t)n_long;
     int truncated = 0;
 
-    if (max_tokens != 0 && n > (size_t)max_tokens) {
+    if (max_tokens != 0 && w->model.meta.unk_policy == SE_UNK_DROP) {
+        size_t usable = 0;
+        size_t cutoff = n;
+        for (size_t i = 0; i < n; i++) {
+            unsigned long long value = NUM2ULL(rb_ary_entry(ids_value, (long)i));
+            if (value >= w->model.meta.vocab_size)
+                rb_raise(rb_eArgError, "token id at index %zu is out of range", i);
+
+            if ((uint32_t)value == w->model.meta.unk_id)
+                continue;
+
+            usable++;
+            if (usable == (size_t)max_tokens) {
+                cutoff = i + 1;
+            } else if (usable > (size_t)max_tokens) {
+                n = cutoff;
+                truncated = 1;
+                break;
+            }
+        }
+    } else if (max_tokens != 0 && n > (size_t)max_tokens) {
         n = (size_t)max_tokens;
         truncated = 1;
     }
@@ -1297,6 +1322,10 @@ static VALUE model_embed_token_ids_with_stats(int argc, VALUE *argv, VALUE self)
     rb_hash_aset(hash, ID2SYM(id_vector), vector);
     rb_hash_aset(hash, ID2SYM(id_token_count), UINT2NUM(stats.token_count));
     rb_hash_aset(hash, ID2SYM(id_unk_count), UINT2NUM(stats.unk_count));
+    uint32_t pooled_count = get_model(self)->model.meta.unk_policy == SE_UNK_DROP
+                                ? stats.token_count - stats.unk_count
+                                : stats.token_count;
+    rb_hash_aset(hash, ID2SYM(id_pooled_count), UINT2NUM(pooled_count));
     rb_hash_aset(hash, ID2SYM(id_truncated), stats.truncated ? Qtrue : Qfalse);
     return hash;
 }
@@ -1607,6 +1636,7 @@ RUBY_FUNC_EXPORTED void Init_static_embeddings(void) {
     id_vector = rb_intern("vector");
     id_token_count = rb_intern("token_count");
     id_unk_count = rb_intern("unk_count");
+    id_pooled_count = rb_intern("pooled_count");
     id_truncated = rb_intern("truncated");
     id_dim = rb_intern("dim");
     id_allow_unfrozen = rb_intern("allow_unfrozen");

data/lib/static_embeddings/bert_wordpiece.rb

@@ -0,0 +1,191 @@
+require "static_embeddings/errors"
+require "static_embeddings/format/constants"
+
+module StaticEmbeddings
+  module BertWordPiece
+    TOKENIZER_PROFILE = "BERT_WORDPIECE_V1"
+    ALLOWED_NORMALIZER_KEYS = %w[type clean_text handle_chinese_chars strip_accents lowercase].freeze
+    STANDARD_SPECIAL_TOKENS = {
+      "[PAD]" => Format::ADDED_PAD,
+      "[UNK]" => Format::ADDED_UNK,
+      "[CLS]" => Format::ADDED_CLS,
+      "[SEP]" => Format::ADDED_SEP,
+      "[MASK]" => Format::ADDED_MASK
+    }.freeze
+
+    module_function
+
+    def compile(tokenizer, tokenizer_config = {})
+      profile = audit(tokenizer, tokenizer_config)
+      tokens = vocabulary(tokenizer)
+      validate_added_token_ids(profile.fetch(:added_tokens), tokens)
+      [profile.freeze, tokens]
+    end
+
+    def runtime_meta(profile, tokens)
+      prefix = profile.fetch(:continuing_subword_prefix)
+      {
+        tokenizer_type: Format::TOKENIZER_BERT_WORDPIECE_V1,
+        tokenizer_profile: TOKENIZER_PROFILE,
+        do_lower_case: profile.fetch(:lowercase),
+        strip_accents: profile.fetch(:strip_accents),
+        handle_chinese_chars: profile.fetch(:handle_chinese_chars),
+        clean_text: profile.fetch(:clean_text),
+        added_token_mask: profile.fetch(:added_token_mask),
+        max_input_chars_per_word: profile.fetch(:max_input_chars_per_word),
+        max_token_chars: max_token_chars(tokens, prefix),
+        subword_prefix: prefix,
+        pad_id: token_id(tokens, "[PAD]", 0),
+        unk_id: token_id(tokens, profile.fetch(:unk_token)),
+        cls_id: token_id(tokens, "[CLS]", 0),
+        sep_id: token_id(tokens, "[SEP]", 0),
+        mask_id: token_id(tokens, "[MASK]", 0)
+      }.freeze
+    end
+
+    def audit(tokenizer, tokenizer_config)
+      unsupported!("tokenizer.json is not an object") unless tokenizer.is_a?(Hash)
+      unsupported!("tokenizer_config.json is not an object") unless tokenizer_config.is_a?(Hash)
+      model = tokenizer["model"] || unsupported!("tokenizer.json has no model section")
+      normalizer = tokenizer["normalizer"] || unsupported!("tokenizer has no normalizer")
+      pre_tokenizer = tokenizer["pre_tokenizer"]
+      unsupported!("tokenizer model section is not an object") unless model.is_a?(Hash)
+      unsupported!("tokenizer normalizer is not an object") unless normalizer.is_a?(Hash)
+
+      audit_model(model)
+      audit_normalizer(normalizer)
+      audit_pre_tokenizer(pre_tokenizer)
+      added_tokens = audit_added_tokens(tokenizer["added_tokens"] || [])
+      lowercase = flag(normalizer, "lowercase", tokenizer_config["do_lower_case"], true)
+      clean_text = flag(normalizer, "clean_text", nil, true)
+      max_input_chars = Integer(model.fetch("max_input_chars_per_word", 100))
+      unsupported!("max_input_chars_per_word must be positive") unless max_input_chars.positive?
+      unsupported!("clean_text=false is not supported by the runtime") unless clean_text
+
+      {
+        lowercase: lowercase,
+        strip_accents: strip_accents(normalizer, lowercase),
+        clean_text: clean_text,
+        handle_chinese_chars: flag(normalizer, "handle_chinese_chars",
+                                   tokenizer_config["tokenize_chinese_chars"], true),
+        continuing_subword_prefix: model.fetch("continuing_subword_prefix", "##"),
+        unk_token: model.fetch("unk_token", "[UNK]"),
+        max_input_chars_per_word: max_input_chars,
+        tokenizer_class: tokenizer_config["tokenizer_class"],
+        added_tokens: added_tokens,
+        added_token_mask: added_tokens.reduce(0) do |mask, token|
+          mask | STANDARD_SPECIAL_TOKENS.fetch(token.fetch("content"))
+        end
+      }
+    end
+
+    def audit_model(model)
+      unsupported!("tokenizer model.type is #{model['type'].inspect}, expected WordPiece") unless model["type"] == "WordPiece"
+
+      prefix = model.fetch("continuing_subword_prefix", "##")
+      unsupported!("continuing_subword_prefix #{prefix.inspect} is not supported") unless prefix == "##"
+
+      unk_token = model.fetch("unk_token", "[UNK]")
+      unsupported!("unk_token #{unk_token.inspect} is not supported") unless unk_token == "[UNK]"
+    end
+
+    def audit_normalizer(normalizer)
+      unsupported!("normalizer type #{normalizer['type'].inspect} is not BertNormalizer") unless normalizer["type"] == "BertNormalizer"
+
+      unknown = normalizer.keys - ALLOWED_NORMALIZER_KEYS
+      unsupported!("normalizer has unsupported keys #{unknown.inspect}") unless unknown.empty?
+    end
+
+    def audit_pre_tokenizer(pre_tokenizer)
+      return if pre_tokenizer.is_a?(Hash) && pre_tokenizer["type"] == "BertPreTokenizer"
+
+      unsupported!("pre_tokenizer type #{pre_tokenizer && pre_tokenizer['type'].inspect} is not BertPreTokenizer")
+    end
+
+    def audit_added_tokens(tokens)
+      unsupported!("added_tokens is not an array") unless tokens.is_a?(Array)
+      unsupported!("added_tokens contains a non-object entry") unless tokens.all? { |token| token.is_a?(Hash) }
+      bad = tokens.reject { |token| standard_special?(token) }
+      unsupported!("tokenizer declares non-standard added_tokens #{bad.map { |token| token['content'] }.inspect}") unless bad.empty?
+
+      tokens.each do |token|
+        content = token["content"].to_s
+        unsupported!("added token #{content.inspect} contains whitespace") if content.match?(/\s/)
+        unsupported!("added token #{content.inspect} uses lstrip/rstrip/single_word") if token["lstrip"] || token["rstrip"] || token["single_word"]
+        unsupported!("added token #{content.inspect} must use normalized=false") unless token["normalized"] == false
+      end
+
+      duplicate = tokens.group_by { |token| token["content"] }.find { |_, rows| rows.length > 1 }
+      unsupported!("duplicate added token #{duplicate.first.inspect}") if duplicate
+      tokens.freeze
+    end
+
+    def vocabulary(tokenizer)
+      vocab = tokenizer.dig("model", "vocab") || unsupported!("tokenizer.json has no model.vocab")
+      unsupported!("tokenizer model.vocab is not an object") unless vocab.is_a?(Hash)
+      tokens = Array.new(vocab.length)
+      vocab.each { |token, id| assign_vocab(tokens, token, id) }
+      missing = tokens.index(nil)
+      invalid!("vocab has a hole at id #{missing}") if missing
+      tokens
+    end
+
+    def assign_vocab(tokens, token, id)
+      unsupported!("vocab id #{id.inspect} is not an integer") unless id.is_a?(Integer)
+      invalid!("vocab id #{id} for #{token.inspect} is outside 0...#{tokens.length}") unless id.between?(0, tokens.length - 1)
+      invalid!("duplicate vocab id #{id}") unless tokens[id].nil?
+      tokens[id] = token
+    end
+
+    def validate_added_token_ids(added_tokens, tokens)
+      added_tokens.each do |token|
+        content = token.fetch("content")
+        id = token["id"]
+        unsupported!("added token #{content.inspect} has non-integer id #{id.inspect}") unless id.is_a?(Integer)
+        valid = id.between?(0, tokens.length - 1) && tokens[id] == content
+        unsupported!("added token #{content.inspect} id #{id} does not match model.vocab") unless valid
+      end
+    end
+
+    def flag(hash, key, fallback, default)
+      return !!hash[key] if hash.key?(key) && !hash[key].nil?
+      return !!fallback unless fallback.nil?
+
+      default
+    end
+
+    def strip_accents(normalizer, lowercase)
+      value = normalizer["strip_accents"]
+      normalizer.key?("strip_accents") && !value.nil? ? !!value : lowercase
+    end
+
+    def standard_special?(token)
+      token["special"] && STANDARD_SPECIAL_TOKENS.key?(token["content"])
+    end
+
+    def max_token_chars(tokens, prefix)
+      tokens.reduce(1) do |maximum, token|
+        body = token.start_with?(prefix) ? token[prefix.length..] : token
+        [maximum, body.each_char.count].max
+      end
+    end
+
+    def token_id(tokens, token, fallback = nil)
+      id = tokens.index(token)
+      return id unless id.nil?
+      return fallback unless fallback.nil?
+
+      invalid!("vocabulary has no #{token.inspect}")
+    end
+
+    def unsupported!(message)
+      raise UnsupportedModelError,
+            "#{message}. Supported tokenizer profile: #{TOKENIZER_PROFILE}; " \
+            "other tokenizer behaviour requires a separate runtime capability."
+    end
+
+    def invalid!(message)
+      raise ConversionError, message
+    end
+  end
+end

data/lib/static_embeddings/canonical.rb

@@ -0,0 +1,50 @@
+module StaticEmbeddings
+  module Canonical
+    Dimensions = Struct.new(:native, :output, :trained, keyword_init: true)
+    Runtime = Struct.new(:normalization, :unk_policy, :empty_policy, :max_tokens, :add_special_tokens,
+                         keyword_init: true)
+    Source = Struct.new(:family, :model, :revision, :oracle, :files_sha256, :tokenizer_class,
+                        :config_seq_length, keyword_init: true)
+    Model = Struct.new(:tokens, :matrix, :dimensions, :runtime, :tokenizer, :source, keyword_init: true)
+
+    module_function
+
+    def dimensions(native:, output:, trained: nil)
+      Dimensions.new(native: native, output: output, trained: trained&.freeze).freeze
+    end
+
+    def runtime(normalization:, unk_policy:, empty_policy:, max_tokens:, add_special_tokens: false)
+      Runtime.new(
+        normalization: normalization,
+        unk_policy: unk_policy,
+        empty_policy: empty_policy,
+        max_tokens: max_tokens,
+        add_special_tokens: add_special_tokens
+      ).freeze
+    end
+
+    def source(family:, model:, oracle:, files_sha256:, revision: nil, tokenizer_class: nil,
+               config_seq_length: nil)
+      Source.new(
+        family: family,
+        model: model,
+        revision: revision,
+        oracle: oracle,
+        files_sha256: files_sha256.freeze,
+        tokenizer_class: tokenizer_class,
+        config_seq_length: config_seq_length
+      ).freeze
+    end
+
+    def model(tokens:, matrix:, dimensions:, runtime:, tokenizer:, source:)
+      Model.new(
+        tokens: tokens.freeze,
+        matrix: matrix,
+        dimensions: dimensions,
+        runtime: runtime,
+        tokenizer: tokenizer.freeze,
+        source: source
+      ).freeze
+    end
+  end
+end