cld 0.4.0 → 0.5.0

data/.gitignore

@@ -0,0 +1,20 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+ext/cld/*.o
+ext/cld/*.a
+ext/cld/*.so

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format documentation

data/Gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in cld.gemspec
+gemspec
+
+gem "rake"

data/README.md

@@ -0,0 +1,34 @@
+# Compact Language Detection
+
+Blazing-fast language detection for Ruby provided by
+Google Chrome's Compact Language Detector.
+
+## How to Use
+
+```ruby
+CLD.detect_language("This is a test")
+# => {:name => "ENGLISH", :code => "en", :reliable => true}
+
+CLD.detect_language("plus ça change, plus c'est la même chose")
+# => {:name => "FRENCH", :code => "fr", :reliable => true}
+```
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem "cld"
+```
+
+And then execute:
+
+```sh
+$ bundle
+```
+
+## Thanks
+
+Thanks to the Chrome authors, and to Mike McCandless for writing a Python version.
+
+Licensed the same as Chrome.  Jason Toy

data/Rakefile

@@ -1,15 +1,5 @@
-require 'rubygems'
-require 'rake'
-require 'echoe'
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
 
-Echoe.new('cld', '0.4.0') do |p|
-  p.description     = "Compact Language Detection from chrome"
-  p.url             = "http://github.com/jtoy/cld"
-  p.author          = "Jason Toy"
-  p.email           = "jtoy@jtoy.net"
-  p.ignore_pattern  = ["tmp/*", "script/*"]
-  p.runtime_dependencies = ["ffi"]
-  p.development_dependencies = []
-end
-
-#Dir["#{File.dirname(__FILE__)}/tasks/*.rake"].sort.each { |ext| load ext }
+require "rspec/core/rake_task"
+RSpec::Core::RakeTask.new("spec")

data/cld.gemspec

@@ -1,33 +1,22 @@
 # -*- encoding: utf-8 -*-
+require File.expand_path('../lib/cld/version', __FILE__)
 
-Gem::Specification.new do |s|
-  s.name = %q{cld}
-  s.version = "0.4.0"
+Gem::Specification.new do |gem|
+  gem.authors       = ["Jason Toy"]
+  gem.email         = ["jtoy@jtoy.net"]
+  gem.description   = %q{Compact Language Detection for Ruby}
+  gem.summary       = %q{Compact Language Detection for Ruby}
+  gem.homepage      = "http://github.com/jtoy/cld"
 
-  s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
-  s.authors = [%q{Jason Toy}]
-  s.date = %q{2011-11-03}
-  s.description = %q{Compact Language Detection from chrome}
-  s.email = %q{jtoy@jtoy.net}
-  s.extensions = [%q{ext/cld/extconf.rb}]
-  s.extra_rdoc_files = [%q{LICENSE}, %q{README.rdoc}, %q{ext/cld/extconf.rb}, %q{lib/cld.rb}]
-  s.files = [%q{LICENSE}, %q{README.rdoc}, %q{Rakefile}, %q{base/basictypes.h}, %q{base/build_config.h}, %q{base/casts.h}, %q{base/commandlineflags.h}, %q{base/crash.h}, %q{base/dynamic_annotations.h}, %q{base/global_strip_options.h}, %q{base/log_severity.h}, %q{base/logging.h}, %q{base/macros.h}, %q{base/port.h}, %q{base/scoped_ptr.h}, %q{base/stl_decl.h}, %q{base/stl_decl_msvc.h}, %q{base/string_util.h}, %q{base/strtoint.h}, %q{base/template_util.h}, %q{base/type_traits.h}, %q{base/vlog_is_on.h}, %q{build.sh}, %q{build.win.cmd}, %q{cld_encodings.h}, %q{encodings/compact_lang_det/#cldutil.cc#}, %q{encodings/compact_lang_det/#cldutil.h#}, %q{encodings/compact_lang_det/#compact_lang_det_impl.h#}, %q{encodings/compact_lang_det/#ext_lang_enc.cc#}, %q{encodings/compact_lang_det/#ext_lang_enc.h#}, %q{encodings/compact_lang_det/#getonescriptspan.cc#}, %q{encodings/compact_lang_det/#getonescriptspan.h#}, %q{encodings/compact_lang_det/#tote.cc#}, %q{encodings/compact_lang_det/#tote.h#}, %q{encodings/compact_lang_det/cldutil.cc}, %q{encodings/compact_lang_det/cldutil.h}, %q{encodings/compact_lang_det/cldutil_dbg.h}, %q{encodings/compact_lang_det/cldutil_dbg_empty.cc}, %q{encodings/compact_lang_det/compact_lang_det.cc}, %q{encodings/compact_lang_det/compact_lang_det.h}, %q{encodings/compact_lang_det/compact_lang_det_impl.cc}, %q{encodings/compact_lang_det/compact_lang_det_impl.h}, %q{encodings/compact_lang_det/compact_lang_det_unittest_small.cc}, %q{encodings/compact_lang_det/compile.cmd}, %q{encodings/compact_lang_det/ext_lang_enc.cc}, %q{encodings/compact_lang_det/ext_lang_enc.h}, %q{encodings/compact_lang_det/generated/cld_generated_score_deltaoctachrome_0406.cc}, %q{encodings/compact_lang_det/generated/cld_generated_score_quadchrome_0406.cc}, %q{encodings/compact_lang_det/generated/compact_lang_det_generated_cjkbis_0.cc}, %q{encodings/compact_lang_det/generated/compact_lang_det_generated_ctjkvz.cc}, %q{encodings/compact_lang_det/generated/compact_lang_det_generated_ctjkvz_0.cc}, %q{encodings/compact_lang_det/generated/compact_lang_det_generated_deltaoctachrome.cc}, %q{encodings/compact_lang_det/generated/compact_lang_det_generated_longwords8_0.cc}, %q{encodings/compact_lang_det/generated/compact_lang_det_generated_meanscore.h}, %q{encodings/compact_lang_det/generated/compact_lang_det_generated_quads_0.cc}, %q{encodings/compact_lang_det/generated/compact_lang_det_generated_quadschrome.cc}, %q{encodings/compact_lang_det/getonescriptspan.cc}, %q{encodings/compact_lang_det/getonescriptspan.h}, %q{encodings/compact_lang_det/letterscript_enum.cc}, %q{encodings/compact_lang_det/letterscript_enum.h}, %q{encodings/compact_lang_det/subsetsequence.cc}, %q{encodings/compact_lang_det/subsetsequence.h}, %q{encodings/compact_lang_det/subsetsequence_unittest.cc}, %q{encodings/compact_lang_det/tote.cc}, %q{encodings/compact_lang_det/tote.h}, %q{encodings/compact_lang_det/unittest_data.h}, %q{encodings/compact_lang_det/utf8propjustletter.h}, %q{encodings/compact_lang_det/utf8propletterscriptnum.h}, %q{encodings/compact_lang_det/utf8scannotjustletterspecial.h}, %q{encodings/compact_lang_det/win/#cld_unilib_windows.cc#}, %q{encodings/compact_lang_det/win/cld_basictypes.h}, %q{encodings/compact_lang_det/win/cld_commandlineflags.h}, %q{encodings/compact_lang_det/win/cld_google.h}, %q{encodings/compact_lang_det/win/cld_htmlutils.h}, %q{encodings/compact_lang_det/win/cld_htmlutils_google3.cc}, %q{encodings/compact_lang_det/win/cld_htmlutils_windows.cc}, %q{encodings/compact_lang_det/win/cld_logging.h}, %q{encodings/compact_lang_det/win/cld_macros.h}, %q{encodings/compact_lang_det/win/cld_strtoint.h}, %q{encodings/compact_lang_det/win/cld_unicodetext.cc}, %q{encodings/compact_lang_det/win/cld_unicodetext.h}, %q{encodings/compact_lang_det/win/cld_unilib.h}, %q{encodings/compact_lang_det/win/cld_unilib_google3.cc}, %q{encodings/compact_lang_det/win/cld_unilib_windows.cc}, %q{encodings/compact_lang_det/win/cld_utf.h}, %q{encodings/compact_lang_det/win/cld_utf8statetable.cc}, %q{encodings/compact_lang_det/win/cld_utf8statetable.h}, %q{encodings/compact_lang_det/win/cld_utf8utils.h}, %q{encodings/compact_lang_det/win/cld_utf8utils_google3.cc}, %q{encodings/compact_lang_det/win/cld_utf8utils_windows.cc}, %q{encodings/compact_lang_det/win/normalizedunicodetext.cc}, %q{encodings/compact_lang_det/win/normalizedunicodetext.h}, %q{encodings/internal/encodings.cc}, %q{encodings/lang_enc.h}, %q{encodings/proto/encodings.pb.h}, %q{encodings/public/encodings.h}, %q{ext/cld/extconf.rb}, %q{languages/internal/#languages.cc#}, %q{languages/internal/languages.cc}, %q{languages/proto/languages.pb.h}, %q{languages/public/languages.h}, %q{lib/cld.rb}, %q{test/test.rb}, %q{thunk.cc}, %q{Manifest}, %q{cld.gemspec}]
-  s.homepage = %q{http://github.com/jtoy/cld}
-  s.rdoc_options = [%q{--line-numbers}, %q{--inline-source}, %q{--title}, %q{Cld}, %q{--main}, %q{README.rdoc}]
-  s.require_paths = [%q{lib}, %q{ext}]
-  s.rubyforge_project = %q{cld}
-  s.rubygems_version = %q{1.8.6.1}
-  s.summary = %q{Compact Language Detection from chrome}
+  gem.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.files         = `git ls-files`.split("\n")
+  gem.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.extensions    = ["ext/cld/extconf.rb"]
+  gem.name          = "cld"
+  gem.require_paths = ["lib"]
+  gem.version       = CLD::VERSION
 
-  if s.respond_to? :specification_version then
-    s.specification_version = 3
+  gem.add_dependency "ffi"
 
-    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<ffi>, [">= 0"])
-    else
-      s.add_dependency(%q<ffi>, [">= 0"])
-    end
-  else
-    s.add_dependency(%q<ffi>, [">= 0"])
-  end
+  gem.add_development_dependency "rspec"
 end

data/ext/cld/Makefile

@@ -0,0 +1,31 @@
+CFLAGS=-fPIC -I. -O2 -DCLD_WINDOWS
+LDFLAGS=-L.
+CC=g++
+AR=ar
+SOURCES=encodings/compact_lang_det/cldutil.cc \
+				encodings/compact_lang_det/cldutil_dbg_empty.cc \
+				encodings/compact_lang_det/compact_lang_det.cc \
+				encodings/compact_lang_det/compact_lang_det_impl.cc \
+				encodings/compact_lang_det/ext_lang_enc.cc \
+				encodings/compact_lang_det/getonescriptspan.cc \
+				encodings/compact_lang_det/letterscript_enum.cc \
+				encodings/compact_lang_det/tote.cc \
+				encodings/compact_lang_det/generated/cld_generated_score_quadchrome_0406.cc \
+				encodings/compact_lang_det/generated/compact_lang_det_generated_cjkbis_0.cc \
+				encodings/compact_lang_det/generated/compact_lang_det_generated_ctjkvz.cc \
+				encodings/compact_lang_det/generated/compact_lang_det_generated_deltaoctachrome.cc \
+				encodings/compact_lang_det/generated/compact_lang_det_generated_quadschrome.cc \
+				encodings/compact_lang_det/win/cld_htmlutils_windows.cc \
+				encodings/compact_lang_det/win/cld_unilib_windows.cc \
+				encodings/compact_lang_det/win/cld_utf8statetable.cc \
+				encodings/compact_lang_det/win/cld_utf8utils_windows.cc \
+				encodings/internal/encodings.cc \
+				languages/internal/languages.cc \
+				thunk.cc
+
+install:
+		rm -f *.o
+		rm -f libcld.a
+		$(CC) -c $(CFLAGS) $(SOURCES)
+		$(AR) rcs libcld.a *.o
+		$(CC) -DCLD_WINDOWS -I. -L. -shared -o cld.so -lstdc++ *.o

data/ext/cld/extconf.rb

@@ -1,7 +1,2 @@
-require "rake"
-home_dir = File.expand_path(File.join(File.dirname(__FILE__), "../../"))
-puts home_dir
-cmd = "cd #{home_dir}; ./build.sh"
-sh cmd
-sh "mv #{home_dir}/cld.so #{home_dir}/ext/cld/"
-sh "echo 'install:\n\tdate' > #{home_dir}/ext/cld/Makefile"
+#require 'mkmf'
+#create_makefile('cld/cld')

data/ext/cld/thunk.cc

@@ -0,0 +1,56 @@
+#include <stdio.h>
+#include <string.h>
+#include "encodings/compact_lang_det/compact_lang_det.h"
+#include "encodings/compact_lang_det/ext_lang_enc.h"
+#include "encodings/compact_lang_det/unittest_data.h"
+#include "encodings/proto/encodings.pb.h"
+
+typedef struct {
+  const char *name;
+  const char *code;
+  bool reliable;
+} RESULT;
+
+extern "C" {
+  RESULT detectLanguageThunkInt(const char * src) {
+    bool is_plain_text = true;
+    bool do_allow_extended_languages = true;
+    bool do_pick_summary_language = false;
+    bool do_remove_weak_matches = false;
+    bool is_reliable;
+    Language plus_one = UNKNOWN_LANGUAGE;
+    const char* tld_hint = NULL;
+    int encoding_hint = UNKNOWN_ENCODING;
+    Language language_hint = UNKNOWN_LANGUAGE;
+
+    double normalized_score3[3];
+    Language language3[3];
+    int percent3[3];
+    int text_bytes;
+
+    Language lang;
+    lang = CompactLangDet::DetectLanguage(0,
+                                          src, strlen(src),
+                                          is_plain_text,
+                                          do_allow_extended_languages,
+                                          do_pick_summary_language,
+                                          do_remove_weak_matches,
+                                          tld_hint,
+                                          encoding_hint,
+                                          language_hint,
+                                          language3,
+                                          percent3,
+                                          normalized_score3,
+                                          &text_bytes,
+                                          &is_reliable);
+
+    RESULT res;
+    res.name = LanguageName(lang);
+    res.code = ExtLanguageCode(lang);
+    res.reliable = is_reliable;
+    return res;
+  }
+}
+
+int main(int argc, char **argv) {
+}

data/lib/cld.rb

@@ -1,12 +1,21 @@
-require "rubygems"
+require "cld/version"
 require "ffi"
 
 module CLD
   extend FFI::Library
-  dir = File.expand_path(File.join(File.dirname(__FILE__), "../ext/cld"))
-  ffi_lib "#{dir}/cld.so"
-  attach_function "detect_language","detectLanguageThunkInt", [:buffer_in], :int
-  def self.english?(text)
-    detect_language(text) == 0
+
+  def self.detect_language(text)
+    result = detect_language_ext(text)
+    Hash[ result.members.map {|member| [member.to_sym, result[member]]} ]
   end
+
+  private
+
+  class ReturnValue < FFI::Struct
+    layout :name, :string, :code, :string, :reliable, :bool
+  end
+
+  GEM_ROOT = File.expand_path("../../", __FILE__)
+  ffi_lib "#{GEM_ROOT}/ext/cld/cld.so"
+  attach_function "detect_language_ext","detectLanguageThunkInt", [:buffer_in], ReturnValue.by_value
 end