ruby-laya 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/lib/laya/lang.rb ADDED
@@ -0,0 +1,301 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Laya
4
+ # Dependency-free language and script detection, used to route between checkpoints.
5
+ #
6
+ # Routing needs one decision: is this English Latin text, or something the English checkpoint
7
+ # cannot read? On non-Latin scripts that checkpoint does not degrade, it collapses to near
8
+ # random while staying confident, so *script* is the signal that matters most, and whether
9
+ # Latin text is English is the secondary one.
10
+ #
11
+ # Script detection is exact. The Latin-script language guess is a stopword and diacritic
12
+ # heuristic, and is explicitly best-effort: pass `model:` or `lang:` when you already know.
13
+ module Lang
14
+ # Unicode blocks the English checkpoint's 50k English BPE vocabulary cannot read.
15
+ SCRIPT_RANGES = [
16
+ ["greek", [[0x0370, 0x03FF], [0x1F00, 0x1FFF]]],
17
+ ["cyrillic", [[0x0400, 0x052F], [0x2DE0, 0x2DFF], [0xA640, 0xA69F]]],
18
+ ["armenian", [[0x0530, 0x058F]]],
19
+ ["hebrew", [[0x0590, 0x05FF]]],
20
+ ["arabic", [[0x0600, 0x06FF], [0x0750, 0x077F], [0x08A0, 0x08FF], [0xFB50, 0xFDFF], [0xFE70, 0xFEFF]]],
21
+ ["devanagari", [[0x0900, 0x097F], [0xA8E0, 0xA8FF]]],
22
+ ["bengali", [[0x0980, 0x09FF]]],
23
+ ["gurmukhi", [[0x0A00, 0x0A7F]]],
24
+ ["gujarati", [[0x0A80, 0x0AFF]]],
25
+ ["oriya", [[0x0B00, 0x0B7F]]],
26
+ ["tamil", [[0x0B80, 0x0BFF]]],
27
+ ["telugu", [[0x0C00, 0x0C7F]]],
28
+ ["kannada", [[0x0C80, 0x0CFF]]],
29
+ ["malayalam", [[0x0D00, 0x0D7F]]],
30
+ ["sinhala", [[0x0D80, 0x0DFF]]],
31
+ ["thai", [[0x0E00, 0x0E7F]]],
32
+ ["lao", [[0x0E80, 0x0EFF]]],
33
+ ["tibetan", [[0x0F00, 0x0FFF]]],
34
+ ["myanmar", [[0x1000, 0x109F]]],
35
+ ["georgian", [[0x10A0, 0x10FF]]],
36
+ ["ethiopic", [[0x1200, 0x137F]]],
37
+ ["khmer", [[0x1780, 0x17FF]]],
38
+ ["hangul", [[0x1100, 0x11FF], [0x3130, 0x318F], [0xAC00, 0xD7AF]]],
39
+ ["kana", [[0x3040, 0x309F], [0x30A0, 0x30FF], [0x31F0, 0x31FF]]],
40
+ ["han", [[0x3400, 0x4DBF], [0x4E00, 0x9FFF], [0xF900, 0xFAFF]]]
41
+ ].freeze
42
+
43
+ # Function words, weighted and held to a margin before anything is called non-English,
44
+ # because Latin-script languages overlap heavily. The Romance lists carry the unaccented
45
+ # spellings too: a state whose accents were stripped in transit keeps no diacritic rate for
46
+ # the non-English signal to read, so those words are the only evidence left.
47
+ STOPWORDS = {
48
+ "en" => %w[
49
+ and are as at be but can for from has have i in is it not
50
+ of on please that the their there this to was we were what
51
+ which will with would you
52
+ ],
53
+ "fr" => %w[
54
+ alors au aux avec bien bonjour ce ces cette comment dans des deux
55
+ dois doit donc du elle elles est et fait fois il ils je jour
56
+ jours la le les ma mais merci mes mois mon nous ont ou pas
57
+ peut peux plus pour pourquoi quand que qui sa ses sont sur ta
58
+ tes ton tous tout toute trois très tu une veut veux vous être
59
+ ],
60
+ "de" => %w[
61
+ aber auch auf das dem den der die ein eine für haben ist mit
62
+ nicht oder sich sind und von werden wurde zu
63
+ ],
64
+ "es" => %w[
65
+ al algo aquí aunque como con cuando del donde dos el entre es
66
+ esa ese eso esta este esto está fue fueron gracias han hay hemos
67
+ hoy la las le les lo los mi muy más nada necesito ni nos
68
+ para pero por porque puede pueden que quiero se ser sobre son su
69
+ sus también tengo tiene tienen todo tres tu un una y ya
70
+ ],
71
+ "pt" => %w[
72
+ agora ainda alguem alguém ali antes ao aos aqui as até boa cadê
73
+ com como consigo da das depois deu do dois dos e em entao então
74
+ era esta estamos estava este estou está eu ficou fiz foi gostaria
75
+ hoje isso isto ja já mais mas meu meus minha minhas muito na
76
+ nada nao nas nenhum nenhuma ninguem ninguém noite nos nossa nosso não
77
+ o obrigada obrigado olá onde ontem os para pela pelo pode podem
78
+ por porque pra preciso quando que quero sao se ser seu sou sua
79
+ são tambem também tarde tem tenho três tudo tá um uma vc vcs
80
+ voce voces você vocês é
81
+ ],
82
+ "it" => %w[
83
+ abbiamo adesso agli alla alle anche ancora avete che ci ciao col
84
+ come con da dagli dal dalla dallo degli dei del della delle dello
85
+ deve devo devono di dove e ed era fra già gli grazie ha hai
86
+ hanno ho ieri il la le lo mai mi mia mio molto ne negli nel
87
+ nell nella non o oggi per perche più poco quando questa questo
88
+ scusa sempre si sono stata stato su sua sul sulla sulle tra tuo
89
+ un una uno voglio vorrei è
90
+ ],
91
+ "nl" => %w[
92
+ aan dat deze door een het is maar met naar niet ook op te
93
+ van voor worden wordt zijn
94
+ ],
95
+ "ro" => %w[
96
+ aceasta această acest acesta acum ale care dar din după este foarte
97
+ fost fără lui mi nu pentru până sunt să trebuie vreau vă în și
98
+ ți
99
+ ]
100
+ }.transform_values { |words| words.to_h { |word| [word, true] }.freeze }.freeze
101
+
102
+ # Function words more than one list claims: matching one says "not English" without
103
+ # saying which language, so it never names a winner on its own.
104
+ SHARED_WORDS = %w[as como con da das del dois dos e era esta este está il is la le les lo mais mi nada nos o para
105
+ por porque quando que se ser su sua tu un una].to_h do |word|
106
+ [word, true]
107
+ end.freeze
108
+
109
+ NON_EN_DIACRITICS = "ßàáâãäåæçèéêëìíîïñòóôõöøùúûüýÿāăąćčďđēęěğģīıķļłńņňőœřśşšţťūůűźżžșț".each_char.to_h do |c|
110
+ [c, true]
111
+ end.freeze
112
+
113
+ # A token whose dot or at-sign joins word characters is an identifier, not prose:
114
+ # `github.com`, `user@acme.com`, `v1.2.3`. Splitting those into pieces scored languages a
115
+ # state does not contain, since `com`, `o` and `e` are all function words somewhere.
116
+ # Ruby's `\w` is ASCII-only, so both patterns spell out the Unicode classes Python's
117
+ # `\w` covers: letters and the non-decimal numerals, never digits or underscores.
118
+ IDENTIFIER = /[\p{Word}-]*(?:[.@][\p{Word}-]+)+/
119
+ WORD = /[\p{L}\p{Nl}\p{No}]+/
120
+
121
+ # Above this rate, non-English letters are evidence enough on their own.
122
+ NON_EN_DIACRITIC_RATE = 0.02
123
+
124
+ module_function
125
+
126
+ # The string leaves of a state, so detection sees content rather than keys.
127
+ def iter_text(state, depth = 0)
128
+ return [] if depth > 6 || state.nil?
129
+
130
+ case state
131
+ when String then [state]
132
+ when Symbol then [state.to_s]
133
+ when Hash then state.values.flat_map { |value| iter_text(value, depth + 1) }
134
+ when Array then state.flat_map { |value| iter_text(value, depth + 1) }
135
+ else []
136
+ end
137
+ end
138
+
139
+ # A state flattened into the text detection reads. Keys are ignored: they are usually English.
140
+ def state_text(state, max_chars: 4000)
141
+ iter_text(state).join(" ")[0, max_chars].to_s
142
+ end
143
+
144
+ # How many alphabetic characters each script claims.
145
+ #
146
+ # A letter no listed range claims counts as "other" rather than nothing: most of Unicode's
147
+ # alphabetic codepoints are outside the list, and text in one of those scripts must not be
148
+ # reported as having no letters, which `analyse` would read as English.
149
+ def script_counts(text)
150
+ counts = { "latin" => 0 }
151
+ text.each_char do |char|
152
+ next unless char.match?(/\p{L}/)
153
+
154
+ name = script_of(char.ord)
155
+ counts[name] = counts.fetch(name, 0) + 1
156
+ end
157
+ counts
158
+ end
159
+
160
+ def script_of(codepoint)
161
+ # Latin, Latin Extended Additional, and the fullwidth forms a CJK keyboard produces.
162
+ if codepoint < 0x0250 || (0x1E00..0x1EFF).cover?(codepoint) ||
163
+ (0xFF21..0xFF3A).cover?(codepoint) || (0xFF41..0xFF5A).cover?(codepoint)
164
+ return "latin"
165
+ end
166
+
167
+ found = SCRIPT_RANGES.find do |_name, ranges|
168
+ ranges.any? { |low, high| codepoint.between?(low, high) }
169
+ end
170
+ found ? found.first : "other"
171
+ end
172
+
173
+ # The dominant script: "latin", "han", "devanagari", "other", or "unknown" without letters.
174
+ def detect_script(text)
175
+ counts = script_counts(text)
176
+ latin = counts.delete("latin")
177
+ counts["latin"] = latin # inserted last, which is what decides a tie
178
+ return "unknown" if counts.values.sum.zero?
179
+
180
+ counts.max_by { |_name, count| count }.first
181
+ end
182
+
183
+ # The share of alphabetic characters each detected script holds.
184
+ def script_profile(text)
185
+ counts = script_counts(text)
186
+ total = counts.values.sum
187
+ return {} if total.zero?
188
+
189
+ counts.reject { |_name, count| count.zero? }.transform_values { |count| count.fdiv(total) }
190
+ end
191
+
192
+ # The evidence behind the Latin-script language guess: "language" (nil when undecided),
193
+ # "english_hits", "diacritic_rate" and "looks_non_english".
194
+ #
195
+ # `analyse` needs the evidence rather than the verdict, because "undecided" and "English" are
196
+ # different answers and only one of them is safe to send to the English checkpoint.
197
+ def latin_profile(text)
198
+ words = text.gsub(IDENTIFIER, " ").scan(WORD).map(&:downcase)
199
+ lowered = text.downcase
200
+ diacritics = lowered.each_char.count { |char| NON_EN_DIACRITICS[char] }
201
+ rate = diacritics.fdiv([1, lowered.length].max)
202
+ non_english = rate >= NON_EN_DIACRITIC_RATE
203
+ if words.length < 4
204
+ return { "language" => nil, "english_hits" => 0, "diacritic_rate" => rate,
205
+ "looks_non_english" => non_english }
206
+ end
207
+
208
+ english_hits = words.count { |word| STOPWORDS["en"][word] }
209
+ language, hits = best_language(words)
210
+ { "language" => named_language(language, hits, english_hits, non_english),
211
+ "english_hits" => english_hits, "diacritic_rate" => rate,
212
+ "looks_non_english" => non_english }
213
+ end
214
+
215
+ # The non-English language with the most hits, among those that matched a word no other list
216
+ # claims. Without that condition the winner can be pure overlap: `la` and `e` in Romanian
217
+ # text named Italian, which is a guess dressed as a detection.
218
+ def best_language(words)
219
+ seen = words.uniq
220
+ candidates = STOPWORDS.filter_map do |language, stopwords|
221
+ next if language == "en"
222
+
223
+ hits = words.count { |word| stopwords[word] }
224
+ own = seen.any? { |word| stopwords[word] && !SHARED_WORDS[word] }
225
+ [language, hits] if own
226
+ end
227
+ candidates.max_by { |_language, hits| hits } || [nil, 0]
228
+ end
229
+
230
+ # A non-English language needs a clear margin over English function words. With non-English
231
+ # letters present the margin relaxes, but two hits are still required: one shared word
232
+ # ("para" in Turkish text) named Spanish on the diacritics alone. English is named only when
233
+ # no other language cleared its bar, and only when nothing looks non-English.
234
+ def named_language(language, hits, english_hits, non_english)
235
+ if language && (hits >= [2, english_hits + 2].max || (non_english && hits >= [2, english_hits].max))
236
+ language
237
+ elsif english_hits.positive? && !non_english
238
+ "en"
239
+ end
240
+ end
241
+
242
+ # A best-effort language code for Latin-script text, or nil when undecided.
243
+ #
244
+ # Short inputs return nil on purpose, as does text whose only matches are words several
245
+ # languages share.
246
+ def guess_latin_language(text)
247
+ latin_profile(text)["language"]
248
+ end
249
+
250
+ # The full detection for a state: "script", "script_profile", "language", "is_english",
251
+ # "language_undecided", "diacritic_rate" and "non_latin_fraction". Every branch reports the
252
+ # same keys, so a caller can read one without guarding.
253
+ def analyse(state)
254
+ text = state_text(state)
255
+ profile = script_profile(text)
256
+ script = detect_script(text)
257
+ non_latin = profile.empty? ? 0.0 : (1.0 - profile.fetch("latin", 0.0)).round(4)
258
+
259
+ return unknown_script(profile) if script == "unknown"
260
+ return other_script(script, profile, non_latin) if script != "latin"
261
+
262
+ latin_script(profile, latin_profile(text), non_latin)
263
+ end
264
+
265
+ def unknown_script(profile)
266
+ { "script" => "unknown", "script_profile" => profile, "language" => nil,
267
+ "is_english" => true, "language_undecided" => true, "diacritic_rate" => 0.0,
268
+ "non_latin_fraction" => 0.0 }
269
+ end
270
+
271
+ def other_script(script, profile, non_latin)
272
+ { "script" => script, "script_profile" => profile, "language" => nil,
273
+ "is_english" => false, "language_undecided" => true, "diacritic_rate" => 0.0,
274
+ "non_latin_fraction" => non_latin }
275
+ end
276
+
277
+ def latin_script(profile, latin, non_latin)
278
+ language = latin["language"]
279
+ undecided = language.nil?
280
+ # Undecided is not English. Treating it as English sent every Latin-script language with
281
+ # no stopword list to the checkpoint that cannot read it, silently.
282
+ english = language == "en" || (undecided && !latin["looks_non_english"])
283
+ { "script" => "latin", "script_profile" => profile, "language" => language,
284
+ "is_english" => english, "language_undecided" => undecided,
285
+ "diacritic_rate" => latin["diacritic_rate"].round(4), "non_latin_fraction" => non_latin }
286
+ end
287
+
288
+ def analyze(state)
289
+ analyse(state)
290
+ end
291
+
292
+ # True when the English checkpoint can be expected to read this state.
293
+ def english?(state)
294
+ analyse(state)["is_english"] ? true : false
295
+ end
296
+
297
+ def is_english(state)
298
+ english?(state)
299
+ end
300
+ end
301
+ end
@@ -0,0 +1,197 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Laya
4
+ # Ready-to-use question presets for common production decision workflows.
5
+ module Presets
6
+ module_function
7
+
8
+ # Preset questions for customer support ticket triage.
9
+ def triage_questions
10
+ {
11
+ "intent" => {
12
+ "type" => "choice",
13
+ "instructions" => "What does the customer want in `message`?",
14
+ "criteria" => {
15
+ "refund" => "money returned or a duplicate charge reversed",
16
+ "technical_help" => "a bug, outage or integration problem",
17
+ "billing_question" => "a question about an invoice, plan or payment method",
18
+ "information" => "general information, pricing or how-to",
19
+ "cancellation" => "wants to cancel or downgrade",
20
+ "other" => "none of the other options fits"
21
+ }
22
+ },
23
+ "is_urgent" => {
24
+ "type" => "noul",
25
+ "instructions" => "Does `message` communicate time pressure or a deadline?"
26
+ },
27
+ "frustration" => {
28
+ "type" => "score",
29
+ "instructions" => "How frustrated does the customer sound in `message`?",
30
+ "criteria" => [
31
+ "calm and neutral",
32
+ "concerned but civil",
33
+ "clearly annoyed",
34
+ "very angry or using strong language"
35
+ ]
36
+ },
37
+ "refund_requested" => {
38
+ "type" => "noul",
39
+ "instructions" => "Does the customer ask for money back?"
40
+ },
41
+ "churn_risk" => {
42
+ "type" => "noul",
43
+ "instructions" => "Does `message` suggest the customer may leave for a competitor or cancel?"
44
+ }
45
+ }
46
+ end
47
+
48
+ DEFAULT_EMAIL_CATEGORIES = {
49
+ "billing" => "invoices, payments, refunds",
50
+ "technical" => "bugs, outages, integrations",
51
+ "sales" => "pricing, demos, new purchases",
52
+ "security" => "phishing, scams, account compromise",
53
+ "hr" => "hiring, leave, payroll",
54
+ "other" => "none of the above"
55
+ }.freeze
56
+
57
+ # Preset questions for inbound email triage and threat filtering.
58
+ def email_questions(categories = nil)
59
+ categories = DEFAULT_EMAIL_CATEGORIES.dup if categories.nil? || categories.empty?
60
+ {
61
+ "category" => {
62
+ "type" => "choice",
63
+ "instructions" => "Which team should handle the email in `body`?",
64
+ "criteria" => categories
65
+ },
66
+ "is_spam" => {
67
+ "type" => "noul",
68
+ "instructions" => "Is this email unsolicited spam or bulk marketing?"
69
+ },
70
+ "is_phishing" => {
71
+ "type" => "noul",
72
+ "instructions" => "Is this email a phishing or scam attempt to steal money, credentials, or personal data?",
73
+ "criteria" => { "true" => "phishing, scam, or fraud", "false" => "a legitimate email" }
74
+ },
75
+ "urgency" => {
76
+ "type" => "score",
77
+ "instructions" => "How urgent is the request in `body`?",
78
+ "criteria" => ["no time pressure", "needs attention soon", "blocking issue or hard deadline"]
79
+ },
80
+ "needs_reply" => {
81
+ "type" => "noul",
82
+ "instructions" => "Does the sender expect a reply?"
83
+ }
84
+ }
85
+ end
86
+
87
+ # Preset questions for real-time LLM input guardrails.
88
+ def guard_questions
89
+ {
90
+ "jailbreak" => {
91
+ "type" => "noul",
92
+ "instructions" => "Does `prompt` try to make an AI assistant ignore its rules, policies or " \
93
+ "system instructions?"
94
+ },
95
+ "prompt_injection" => {
96
+ "type" => "noul",
97
+ "instructions" => "Does `prompt` contain instructions aimed at the AI system rather than a " \
98
+ "genuine user request?"
99
+ },
100
+ "sensitive_data" => {
101
+ "type" => "noul",
102
+ "instructions" => "Does `prompt` contain credentials, personal data or other sensitive information?"
103
+ },
104
+ "harm_severity" => {
105
+ "type" => "score",
106
+ "instructions" => "How much harm would complying with `prompt` cause?",
107
+ "criteria" => [
108
+ "none: ordinary request",
109
+ "minor: mildly inappropriate",
110
+ "serious: unsafe advice or abuse",
111
+ "severe: dangerous or illegal"
112
+ ]
113
+ },
114
+ "topic" => {
115
+ "type" => "choice",
116
+ "instructions" => "What is `prompt` about?",
117
+ "criteria" => {
118
+ "product_support" => nil,
119
+ "coding" => nil,
120
+ "general_knowledge" => nil,
121
+ "personal_advice" => nil,
122
+ "security_testing" => nil,
123
+ "other" => nil
124
+ }
125
+ }
126
+ }
127
+ end
128
+
129
+ # Preset questions for content safety and moderation.
130
+ def moderation_questions
131
+ {
132
+ "toxic" => {
133
+ "type" => "noul",
134
+ "instructions" => "Is `post` toxic: rude, disrespectful or likely to make someone leave the discussion?"
135
+ },
136
+ "harassment" => {
137
+ "type" => "noul",
138
+ "instructions" => "Does `post` target or harass a specific person?"
139
+ },
140
+ "threat" => {
141
+ "type" => "noul",
142
+ "instructions" => "Does `post` threaten violence, harm or intimidation?"
143
+ },
144
+ "spam" => {
145
+ "type" => "noul",
146
+ "instructions" => "Is `post` spam or advertising?"
147
+ },
148
+ "severity" => {
149
+ "type" => "score",
150
+ "instructions" => "How severe is any rule-breaking in `post`?",
151
+ "criteria" => [
152
+ "no rule-breaking: ordinary on-topic post",
153
+ "mild: rude tone or off-topic, no target",
154
+ "clear violation: insults, harassment or spam aimed at someone",
155
+ "severe: threats, hate speech or calls for violence"
156
+ ]
157
+ }
158
+ }
159
+ end
160
+
161
+ # Preset questions for intelligent model routing.
162
+ def router_questions
163
+ {
164
+ "difficulty" => {
165
+ "type" => "score",
166
+ "instructions" => "How hard is `request` for a language model?",
167
+ "criteria" => [
168
+ "trivial: a lookup or one-liner",
169
+ "easy: short answer, no reasoning",
170
+ "moderate: several steps",
171
+ "hard: long multi-step reasoning or specialist knowledge"
172
+ ]
173
+ },
174
+ "domain" => {
175
+ "type" => "choice",
176
+ "instructions" => "What domain does `request` belong to?",
177
+ "criteria" => {
178
+ "code" => "software engineering, programming, refactoring, architecture, debugging",
179
+ "math_or_logic" => "mathematics, logic puzzles, proofs, complex calculation",
180
+ "writing" => "creative writing, essays, emails, blog posts, copywriting",
181
+ "factual_lookup" => "facts, definitions, trivia, history",
182
+ "data_analysis" => "statistics, SQL, data manipulation, metrics",
183
+ "chitchat" => "casual conversation, greetings, small talk"
184
+ }
185
+ },
186
+ "needs_tools" => {
187
+ "type" => "noul",
188
+ "instructions" => "Does answering `request` require external tools, search or private data?"
189
+ },
190
+ "is_sensitive" => {
191
+ "type" => "noul",
192
+ "instructions" => "Does `request` involve money, legal, medical or safety consequences?"
193
+ }
194
+ }
195
+ end
196
+ end
197
+ end
@@ -0,0 +1,144 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Laya
4
+ # A JSON encoder that reproduces Python's `json.dumps` byte for byte.
5
+ #
6
+ # The checkpoints were trained on states and criteria serialised by Python (`{"a": 1, "b": 2}`,
7
+ # with a space after every comma and colon, non-ASCII kept as is). Ruby's `JSON.generate`
8
+ # emits `{"a":1,"b":2}`, which tokenises differently and would shift every model input away
9
+ # from what the weights saw. Everything Laya feeds to a tokenizer goes through this module.
10
+ module PyJSON
11
+ module_function
12
+
13
+ # Serialise `obj` the way `json.dumps(obj, ensure_ascii=ensure_ascii, default=default)` does.
14
+ #
15
+ # `default` is a callable used for objects JSON cannot represent (Python's `default=str`
16
+ # becomes `default: :to_s`); without it such objects raise TypeError.
17
+ def dumps(obj, ensure_ascii: false, default: nil)
18
+ out = +""
19
+ encode(obj, out, ensure_ascii, default)
20
+ out
21
+ end
22
+
23
+ def encode(obj, out, ensure_ascii, default)
24
+ case obj
25
+ when nil then out << "null"
26
+ when true then out << "true"
27
+ when false then out << "false"
28
+ when String then encode_string(obj, out, ensure_ascii)
29
+ when Symbol then encode_string(obj.to_s, out, ensure_ascii)
30
+ when Integer then out << obj.to_s
31
+ when Float then out << float_repr(obj)
32
+ when Hash then encode_hash(obj, out, ensure_ascii, default)
33
+ when Array then encode_array(obj, out, ensure_ascii, default)
34
+ else
35
+ raise TypeError, "Object of type #{obj.class} is not JSON serializable" if default.nil?
36
+
37
+ fallback = default.is_a?(Symbol) ? obj.public_send(default) : default.call(obj)
38
+ encode(fallback, out, ensure_ascii, nil)
39
+ end
40
+ end
41
+
42
+ def encode_hash(hash, out, ensure_ascii, default)
43
+ out << "{"
44
+ first = true
45
+ hash.each do |k, v|
46
+ out << ", " unless first
47
+ first = false
48
+ encode_string(key_string(k), out, ensure_ascii)
49
+ out << ": "
50
+ encode(v, out, ensure_ascii, default)
51
+ end
52
+ out << "}"
53
+ end
54
+
55
+ def encode_array(array, out, ensure_ascii, default)
56
+ out << "["
57
+ array.each_with_index do |v, i|
58
+ out << ", " if i > 0
59
+ encode(v, out, ensure_ascii, default)
60
+ end
61
+ out << "]"
62
+ end
63
+
64
+ # Python coerces non-string keys: True -> "true", None -> "null", numbers -> their repr.
65
+ def key_string(key)
66
+ case key
67
+ when String then key
68
+ when Symbol then key.to_s
69
+ when true then "true"
70
+ when false then "false"
71
+ when nil then "null"
72
+ when Integer then key.to_s
73
+ when Float then float_repr(key)
74
+ else raise TypeError, "keys must be str, int, float, bool or None, not #{key.class}"
75
+ end
76
+ end
77
+
78
+ ESCAPES = {
79
+ "\"" => "\\\"", "\\" => "\\\\", "\n" => "\\n", "\r" => "\\r",
80
+ "\t" => "\\t", "\b" => "\\b", "\f" => "\\f"
81
+ }.freeze
82
+ private_constant :ESCAPES
83
+
84
+ def encode_string(str, out, ensure_ascii)
85
+ out << "\""
86
+ str.each_char do |ch|
87
+ out << if (esc = ESCAPES[ch])
88
+ esc
89
+ elsif ch.ord < 0x20
90
+ format("\\u%04x", ch.ord)
91
+ elsif ensure_ascii && ch.ord > 0x7e
92
+ ascii_escape(ch.ord)
93
+ else
94
+ ch
95
+ end
96
+ end
97
+ out << "\""
98
+ end
99
+
100
+ def ascii_escape(cp)
101
+ return format("\\u%04x", cp) if cp < 0x10000
102
+
103
+ cp -= 0x10000
104
+ format("\\u%04x\\u%04x", 0xD800 | (cp >> 10), 0xDC00 | (cp & 0x3FF))
105
+ end
106
+
107
+ # Python's `repr(float)`: the shortest round-trip digits, positional notation while the
108
+ # decimal exponent is in (-4, 16] and `1e-05` / `1e+16` style outside it, with `NaN` /
109
+ # `Infinity` spelled the way `json.dumps` emits them.
110
+ def float_repr(f)
111
+ return "NaN" if f.nan?
112
+ return f.positive? ? "Infinity" : "-Infinity" if f.infinite?
113
+ return f.to_s if f.zero? # "0.0" / "-0.0"
114
+
115
+ s = f.to_s
116
+ sign = s.start_with?("-") ? "-" : ""
117
+ s = s.delete_prefix("-")
118
+ mantissa, exp = s.split("e", 2)
119
+ int_part, frac_part = mantissa.split(".", 2)
120
+ frac_part ||= ""
121
+ digits = int_part + frac_part
122
+ decpt = int_part.length + exp.to_i
123
+ while digits.start_with?("0")
124
+ digits = digits[1..]
125
+ decpt -= 1
126
+ end
127
+ digits = digits.sub(/0+\z/, "")
128
+ digits = "0" if digits.empty?
129
+ body = if decpt <= -4 || decpt > 16
130
+ head = digits[0]
131
+ tail = digits.length > 1 ? ".#{digits[1..]}" : ""
132
+ e = decpt - 1
133
+ format("%s%se%s%02d", head, tail, e.negative? ? "-" : "+", e.abs)
134
+ elsif decpt <= 0
135
+ "0.#{'0' * -decpt}#{digits}"
136
+ elsif decpt >= digits.length
137
+ "#{digits}#{'0' * (decpt - digits.length)}.0"
138
+ else
139
+ "#{digits[0, decpt]}.#{digits[decpt..]}"
140
+ end
141
+ sign + body
142
+ end
143
+ end
144
+ end