cld2-small 2.0.0 → 2.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f88610ddab27246200e533e2a863ecfa32adbf0542490bfbe42deb1f54653b7a
-  data.tar.gz: fbecb14a1eac61c643a6bf8c347005d5dabed88e83a301e12708ec056336ddd0
+  metadata.gz: 6f10fe04b1de4166a0dabfbae2294879bc24e749312461fa882fc2f0de8176a1
+  data.tar.gz: 9de3c4cecb30494f28924e98c76303192ebbee33105b778b3d606a390a4a2f69
 SHA512:
-  metadata.gz: 3dd02811b0fe15197f8640019ce5235c2531d8af0a48a97cf944fb27d30797813ca91ca4ee2e80e703b8d197e2e3d5f180f45733e6d39139a830c15ea2bc95f9
-  data.tar.gz: ff09ffa0689e7e9ee426f0a383cc82b238d5f891ed0f81d1478720b9ea24ae6f4a469bb1bab43255e610847348f07416a2f3dbc07f072c0c0861ef40fc0c16ed
+  metadata.gz: '00418d5b352b925de52c3e82c4ec7651f5089a15612b8a0a2a43c76b709efae082c219c0e4513714cd0b4de7dba83fd215a769d3d67cf0995c7e0893588a5eaf'
+  data.tar.gz: '078fd9c608432d46f03adc3592438e16f54dcd7a9118129cc6af12cd2344ff010862a7bb799262e434d475a5a7f342da34609413277c5db93433dc41fb116b0e'

data/README.md

@@ -10,6 +10,8 @@ Blazing-fast language detection for Ruby provided using Compact Language Detecto
 
 ## How to Use
 
+The `detect_language` method returns a hash with the language name, code, and reliability.
+
 ```ruby
 CLD.detect_language("Working as expected")
 # => {:name => "ENGLISH", :code => "en", :reliable => true}
@@ -18,6 +20,43 @@ CLD.detect_language("plus ça change, plus c'est la même chose")
 # => {:name => "FRENCH", :code => "fr", :reliable => true}
 ```
 
+**Options**
+
+You can pass an options hash as the second argument to `detect_language`:
+
+```ruby
+CLD.detect_language(text, options = {})
+```
+
+Available options:
+
+*   `:is_plain_text` (Boolean, default: `true`): Set to `false` if the input text is HTML. CLD2 will then try to skip HTML tags.
+*   `:best_effort` (Boolean, default: `false`): If `true`, CLD2 will give its best-effort answer, even on short or ambiguous text, instead of potentially returning "Unknown".
+*   `:tld_hint` (String, default: `nil`): A Top-Level Domain hint (e.g., `"id"`, `"us"`) to boost detection accuracy for languages associated with that TLD.
+*   `:content_language_hint` (String, default: `nil`): An HTTP Content-Language header style hint (e.g., `"en,fr"`) to boost detection for the specified languages.
+*   `:score_as_quads` (Boolean, default: `false`): Forces CLD2 to use quadgram-based scoring for certain languages that are normally detected by script alone. This can be a refinement for more meaningful text detection in those languages but depends on CLD2's internal data tables.
+
+**Examples with Options:**
+
+```ruby
+# Using best_effort for short text
+CLD.detect_language("test", best_effort: true)
+# => Might return a language like {:name => "ENGLISH", :code => "en", :reliable => false}
+#    instead of "Unknown"
+
+# Providing a TLD hint
+CLD.detect_language("Ini adalah teks dalam bahasa Indonesia", tld_hint: "id")
+# => {:name => "INDONESIAN", :code => "id", :reliable => true}
+
+# Providing a content language hint
+CLD.detect_language("Ceci est un texte en français.", content_language_hint: "fr,en")
+# => {:name => "FRENCH", :code => "fr", :reliable => true}
+
+# Using score_as_quads (example, effect depends on text and CLD2 tables)
+CLD.detect_language("Ελληνικό κείμενο", score_as_quads: true)
+# => May provide a more refined result for Greek
+```
+
 ## Installation
 
 Add this line to your application's Gemfile:

data/ext/cld/thunk.cc

@@ -13,28 +13,43 @@ typedef struct {
 } RESULT;
 
 extern "C" {
-  RESULT detectLanguageThunkInt(const char * src, bool is_plain_text) {
+  RESULT detectLanguageThunkInt(const char * src,
+                                bool is_plain_text,
+                                bool best_effort,
+                                const char* tld_hint_param,
+                                const char* content_lang_hint_param,
+                                bool score_as_quads_param) {
     bool is_reliable;
     Language plus_one = UNKNOWN_LANGUAGE;
-    const char* tld_hint = NULL;
-    int encoding_hint = UNKNOWN_ENCODING;
-    Language language_hint = UNKNOWN_LANGUAGE;
+    int encoding_hint_val = UNKNOWN_ENCODING;
+    Language language_hint_val = UNKNOWN_LANGUAGE;
+
+    CLDHints cld_hints = {content_lang_hint_param, tld_hint_param, encoding_hint_val, language_hint_val};
+
+    int flags = 0;
+    if (best_effort) {
+      flags |= kCLDFlagBestEffort;
+    }
+    if (score_as_quads_param) {
+      flags |= kCLDFlagScoreAsQuads;
+    }
 
     double normalized_score3[3];
     Language language3[3];
     int percent3[3];
     int text_bytes;
+    ResultChunkVector resultchunkvector;
 
     Language lang;
     lang = ExtDetectLanguageSummary(src,
                           strlen(src),
                           is_plain_text,
-                          tld_hint,
-                          encoding_hint,
-                          language_hint,
+                          &cld_hints,
+                          flags,
                           language3,
                           percent3,
                           normalized_score3,
+                          &resultchunkvector,
                           &text_bytes,
                           &is_reliable);
 

data/lib/cld/version.rb

@@ -1,3 +1,3 @@
 module CLD
-  VERSION = "2.0.0"
+  VERSION = "2.0.5"
 end

data/lib/cld.rb

@@ -1,4 +1,4 @@
-require "cld/version"
+require_relative "cld/version"
 require "ffi"
 
 module CLD
@@ -23,8 +23,19 @@ module CLD
 
   ffi_lib lib_path
 
-  def self.detect_language(text, is_plain_text=true)
-    result = detect_language_ext(text.to_s, is_plain_text)
+  def self.detect_language(text, options = {})
+    is_plain_text = options.fetch(:is_plain_text, true)
+    best_effort = options.fetch(:best_effort, false)
+    tld_hint = options.fetch(:tld_hint, nil)
+    content_language_hint = options.fetch(:content_language_hint, nil)
+    score_as_quads = options.fetch(:score_as_quads, false)
+
+    result = detect_language_ext(text.to_s,
+                                 is_plain_text,
+                                 best_effort,
+                                 tld_hint,
+                                 content_language_hint,
+                                 score_as_quads)
     Hash[ result.members.map {|member| [member.to_sym, result[member]]} ]
   end
 
@@ -34,5 +45,7 @@ module CLD
     layout :name, :string, :code, :string, :reliable, :bool
   end
 
-  attach_function "detect_language_ext", "detectLanguageThunkInt", [:buffer_in, :bool], ReturnValue.by_value
+  attach_function "detect_language_ext", "detectLanguageThunkInt",
+                  [:buffer_in, :bool, :bool, :string, :string, :bool],
+                  ReturnValue.by_value
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cld2-small
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.0.5
 platform: ruby
 authors:
 - Alessandro Dal Grande
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-03-10 00:00:00.000000000 Z
+date: 2025-05-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -74,51 +74,39 @@ files:
 - ext/cld/cld2/internal/cld2_dynamic_data_loader.h
 - ext/cld/cld2/internal/cld2_dynamic_data_tool.cc
 - ext/cld/cld2/internal/cld2_generated_cjk_compatible.cc
-- ext/cld/cld2/internal/cld2_generated_cjk_compatible.o
 - ext/cld/cld2/internal/cld2_generated_deltaocta0122.cc
 - ext/cld/cld2/internal/cld2_generated_deltaocta0527.cc
 - ext/cld/cld2/internal/cld2_generated_deltaoctachrome.cc
-- ext/cld/cld2/internal/cld2_generated_deltaoctachrome.o
 - ext/cld/cld2/internal/cld2_generated_distinctocta0122.cc
 - ext/cld/cld2/internal/cld2_generated_distinctocta0527.cc
 - ext/cld/cld2/internal/cld2_generated_distinctoctachrome.cc
-- ext/cld/cld2/internal/cld2_generated_distinctoctachrome.o
 - ext/cld/cld2/internal/cld2_generated_octa2_dummy.cc
 - ext/cld/cld2/internal/cld2_generated_quad0122.cc
 - ext/cld/cld2/internal/cld2_generated_quad0720.cc
 - ext/cld/cld2/internal/cld2_generated_quadchrome_16.cc
 - ext/cld/cld2/internal/cld2_generated_quadchrome_2.cc
-- ext/cld/cld2/internal/cld2_generated_quadchrome_2.o
 - ext/cld/cld2/internal/cld2_unittest.cc
 - ext/cld/cld2/internal/cld2_unittest_full.cc
 - ext/cld/cld2/internal/cld2tablesummary.h
 - ext/cld/cld2/internal/cld_generated_cjk_delta_bi_32.cc
 - ext/cld/cld2/internal/cld_generated_cjk_delta_bi_4.cc
-- ext/cld/cld2/internal/cld_generated_cjk_delta_bi_4.o
 - ext/cld/cld2/internal/cld_generated_cjk_uni_prop_80.cc
-- ext/cld/cld2/internal/cld_generated_cjk_uni_prop_80.o
 - ext/cld/cld2/internal/cld_generated_score_quad_octa_0122.cc
 - ext/cld/cld2/internal/cld_generated_score_quad_octa_0122_2.cc
 - ext/cld/cld2/internal/cld_generated_score_quad_octa_1024_256.cc
 - ext/cld/cld2/internal/cld_generated_score_quad_octa_2.cc
-- ext/cld/cld2/internal/cld_generated_score_quad_octa_2.o
 - ext/cld/cld2/internal/cldutil.cc
 - ext/cld/cld2/internal/cldutil.h
-- ext/cld/cld2/internal/cldutil.o
 - ext/cld/cld2/internal/cldutil_offline.cc
 - ext/cld/cld2/internal/cldutil_offline.h
 - ext/cld/cld2/internal/cldutil_shared.cc
 - ext/cld/cld2/internal/cldutil_shared.h
-- ext/cld/cld2/internal/cldutil_shared.o
 - ext/cld/cld2/internal/clean.sh
 - ext/cld/cld2/internal/compact_lang_det.cc
-- ext/cld/cld2/internal/compact_lang_det.o
 - ext/cld/cld2/internal/compact_lang_det_hint_code.cc
 - ext/cld/cld2/internal/compact_lang_det_hint_code.h
-- ext/cld/cld2/internal/compact_lang_det_hint_code.o
 - ext/cld/cld2/internal/compact_lang_det_impl.cc
 - ext/cld/cld2/internal/compact_lang_det_impl.h
-- ext/cld/cld2/internal/compact_lang_det_impl.o
 - ext/cld/cld2/internal/compact_lang_det_test.cc
 - ext/cld/cld2/internal/compile.sh
 - ext/cld/cld2/internal/compile_and_test_all.sh
@@ -127,42 +115,31 @@ files:
 - ext/cld/cld2/internal/compile_libs.sh
 - ext/cld/cld2/internal/debug.cc
 - ext/cld/cld2/internal/debug.h
-- ext/cld/cld2/internal/debug.o
 - ext/cld/cld2/internal/debug_empty.cc
 - ext/cld/cld2/internal/fixunicodevalue.cc
 - ext/cld/cld2/internal/fixunicodevalue.h
-- ext/cld/cld2/internal/fixunicodevalue.o
 - ext/cld/cld2/internal/generated_distinct_bi_0.cc
-- ext/cld/cld2/internal/generated_distinct_bi_0.o
 - ext/cld/cld2/internal/generated_entities.cc
-- ext/cld/cld2/internal/generated_entities.o
 - ext/cld/cld2/internal/generated_language.cc
 - ext/cld/cld2/internal/generated_language.h
-- ext/cld/cld2/internal/generated_language.o
 - ext/cld/cld2/internal/generated_ulscript.cc
 - ext/cld/cld2/internal/generated_ulscript.h
-- ext/cld/cld2/internal/generated_ulscript.o
 - ext/cld/cld2/internal/getonescriptspan.cc
 - ext/cld/cld2/internal/getonescriptspan.h
-- ext/cld/cld2/internal/getonescriptspan.o
 - ext/cld/cld2/internal/integral_types.h
 - ext/cld/cld2/internal/lang_script.cc
 - ext/cld/cld2/internal/lang_script.h
-- ext/cld/cld2/internal/lang_script.o
 - ext/cld/cld2/internal/langspan.h
 - ext/cld/cld2/internal/offsetmap.cc
 - ext/cld/cld2/internal/offsetmap.h
-- ext/cld/cld2/internal/offsetmap.o
 - ext/cld/cld2/internal/port.h
 - ext/cld/cld2/internal/scoreonescriptspan.cc
 - ext/cld/cld2/internal/scoreonescriptspan.h
-- ext/cld/cld2/internal/scoreonescriptspan.o
 - ext/cld/cld2/internal/scoreutf8text.cc
 - ext/cld/cld2/internal/stringpiece.h
 - ext/cld/cld2/internal/test_shuffle_1000_48_666.utf8.gz
 - ext/cld/cld2/internal/tote.cc
 - ext/cld/cld2/internal/tote.h
-- ext/cld/cld2/internal/tote.o
 - ext/cld/cld2/internal/unittest_data.h
 - ext/cld/cld2/internal/utf8acceptinterchange.h
 - ext/cld/cld2/internal/utf8prop_lettermarkscriptnum.h
@@ -170,7 +147,6 @@ files:
 - ext/cld/cld2/internal/utf8scannot_lettermarkspecial.h
 - ext/cld/cld2/internal/utf8statetable.cc
 - ext/cld/cld2/internal/utf8statetable.h
-- ext/cld/cld2/internal/utf8statetable.o
 - ext/cld/cld2/public/compact_lang_det.h
 - ext/cld/cld2/public/encodings.h
 - ext/cld/extconf.rb