charlock_holmes 0.7.3 → 0.7.4

data/charlock_holmes.gemspec

@@ -1,27 +0,0 @@
-# encoding: utf-8
-
-require './lib/charlock_holmes/version' unless defined? CharlockHolmes::VERSION
-
-Gem::Specification.new do |s|
-  s.name = %q{charlock_holmes}
-  s.license = "MIT"
-  s.version = CharlockHolmes::VERSION
-  s.authors = ["Brian Lopez", "Vicent Martí"]
-  s.date = Time.now.utc.strftime("%Y-%m-%d")
-  s.email = %q{seniorlopez@gmail.com}
-  s.extensions = ["ext/charlock_holmes/extconf.rb"]
-  s.files = `git ls-files`.split("\n")
-  s.homepage = %q{https://github.com/brianmario/charlock_holmes}
-  s.rdoc_options = ["--charset=UTF-8"]
-  s.require_paths = ["lib"]
-  s.rubygems_version = %q{1.4.2}
-  s.summary = %q{Character encoding detection, brought to you by ICU}
-  s.description = "charlock_holmes provides binary and text detection as well as text transcoding using libicu"
-  s.test_files = `git ls-files spec`.split("\n")
-
-  # tests
-  s.add_development_dependency 'rake-compiler', ">= 0.7.5"
-  s.add_development_dependency 'minitest'
-  # benchmarks
-  s.add_development_dependency 'chardet'
-end

data/test/converter_test.rb

@@ -1,48 +0,0 @@
-# encoding: utf-8
-require File.expand_path("../helper", __FILE__)
-
-class ConverterTest < MiniTest::Test
-  def test_convert_ascii_from_iso859_1_to_utf16_and_back
-    input = 'test'
-
-    output = CharlockHolmes::Converter.convert input, 'ISO-8859-1', 'UTF-16'
-    assert input.bytesize < output.bytesize
-    assert input != output
-
-    output = CharlockHolmes::Converter.convert output, 'UTF-16', 'ISO-8859-1'
-    assert input.bytesize == output.bytesize
-    assert input == output
-  end
-
-  def test_convert_utf8_to_utf16_and_back
-    input = 'λ, λ, λ'
-
-    output = CharlockHolmes::Converter.convert input, 'UTF-8', 'UTF-16'
-    assert input.bytesize < output.bytesize
-    assert input != output
-
-    output = CharlockHolmes::Converter.convert output, 'UTF-16', 'UTF-8'
-    assert input.bytesize == output.bytesize
-    assert input == output
-  end
-
-  def test_params_must_be_strings
-    assert_raises TypeError do
-      CharlockHolmes::Converter.convert nil, 'UTF-8', 'UTF-16'
-    end
-
-    assert_raises TypeError do
-      CharlockHolmes::Converter.convert 'lol', nil, 'UTF-16'
-    end
-
-    assert_raises TypeError do
-      CharlockHolmes::Converter.convert 'lol', 'UTF-8', nil
-    end
-
-    begin
-      CharlockHolmes::Converter.convert 'lol', 'UTF-8', 'UTF-16'
-    rescue Exception => e
-      assert_nil e, "#{e.class.name} raised, expected nothing"
-    end
-  end
-end

data/test/encoding_detector_test.rb

@@ -1,145 +0,0 @@
-# encoding: utf-8
-require File.expand_path("../helper", __FILE__)
-
-class EncodingDetectorTest < MiniTest::Test
-  def setup
-    @detector = CharlockHolmes::EncodingDetector.new
-  end
-
-  def test_has_class_level_detect_method
-    CharlockHolmes::EncodingDetector.respond_to? :detect
-    detected = CharlockHolmes::EncodingDetector.detect 'test'
-    assert_equal 'ISO-8859-1', detected[:encoding]
-  end
-
-  def test_class_level_detect_accepts_encoding_hint
-    CharlockHolmes::EncodingDetector.respond_to? :detect
-    detected = CharlockHolmes::EncodingDetector.detect 'test', 'UTF-8'
-    assert_equal 'ISO-8859-1', detected[:encoding]
-  end
-
-  def test_has_class_level_detect_all_method
-    CharlockHolmes::EncodingDetector.respond_to? :detect_all
-    detected_list = CharlockHolmes::EncodingDetector.detect_all 'test'
-    assert detected_list.is_a? Array
-
-    encoding_list = detected_list.map {|d| d[:encoding]}.sort
-    assert_equal ['ISO-8859-1', 'ISO-8859-2', 'UTF-8'], encoding_list
-  end
-
-  def test_class_level_detect_all_method_accepts_encoding_hint
-    CharlockHolmes::EncodingDetector.respond_to? :detect_all
-    detected_list = CharlockHolmes::EncodingDetector.detect_all 'test', 'UTF-8'
-    assert detected_list.is_a? Array
-
-    encoding_list = detected_list.map {|d| d[:encoding]}.sort
-    assert_equal ['ISO-8859-1', 'ISO-8859-2', 'UTF-8'], encoding_list
-  end
-
-  def test_has_detect_method
-    @detector.respond_to? :detect
-    detected = @detector.detect 'test'
-    assert_equal 'ISO-8859-1', detected[:encoding]
-  end
-
-  def test_detect_accepts_encoding_hint
-    @detector.respond_to? :detect
-    detected = @detector.detect 'test', 'UTF-8'
-    assert_equal 'ISO-8859-1', detected[:encoding]
-  end
-
-  def test_has_detect_all_method
-    @detector.respond_to? :detect_all
-    detected_list = @detector.detect_all 'test'
-    assert detected_list.is_a? Array
-
-    encoding_list = detected_list.map {|d| d[:encoding]}.sort
-    assert_equal ['ISO-8859-1', 'ISO-8859-2', 'UTF-8'], encoding_list
-  end
-
-  def test_detect_all_accepts_encoding_hint
-    @detector.respond_to? :detect_all
-    detected_list = @detector.detect_all 'test', 'UTF-8'
-    assert detected_list.is_a? Array
-
-    encoding_list = detected_list.map {|d| d[:encoding]}.sort
-    assert_equal ['ISO-8859-1', 'ISO-8859-2', 'UTF-8'], encoding_list
-  end
-
-  def test_strip_tags_flag
-    detector = CharlockHolmes::EncodingDetector.new
-    detector.strip_tags = true
-    assert detector.strip_tags
-
-    detection = detector.detect "<div ascii_attribute='some more ascii'>λ, λ, λ</div>"
-    assert_equal 'UTF-8', detection[:encoding]
-
-    detector.strip_tags = false
-    assert !detector.strip_tags
-
-    detection = detector.detect "<div ascii_attribute='some more ascii'>λ, λ, λ</div>"
-    assert_equal 'UTF-8', detection[:encoding]
-  end
-
-  def test_has_list_of_supported_encodings
-    CharlockHolmes::EncodingDetector.respond_to? :supported_encodings
-    supported_encodings = CharlockHolmes::EncodingDetector.supported_encodings
-
-    assert supported_encodings.is_a?(Array)
-    assert supported_encodings.include? 'UTF-8'
-    assert supported_encodings.include? 'windows-1250'
-    assert supported_encodings.include? 'windows-1252'
-    assert supported_encodings.include? 'windows-1253'
-    assert supported_encodings.include? 'windows-1254'
-    assert supported_encodings.include? 'windows-1255'
-  end
-
-  def test_returns_a_ruby_compatible_encoding_name
-    detected = @detector.detect 'test'
-    assert_equal 'ISO-8859-1', detected[:encoding]
-    assert_equal 'ISO-8859-1', detected[:ruby_encoding]
-
-    not_compat_txt = fixture("ISO-2022-KR.txt").read
-    detected = @detector.detect not_compat_txt
-    assert_equal 'ISO-2022-KR', detected[:encoding]
-    assert_equal 'binary', detected[:ruby_encoding]
-  end
-
-  MAPPING = [
-    ['repl2.cljs',                'ISO-8859-1', :text],
-    ['cl-messagepack.lisp',       'ISO-8859-1', :text],
-    ['sierpinski.ps',             'ISO-8859-1', :text],
-    ['core.rkt',                  'UTF-8',      :text],
-    ['TwigExtensionsDate.es.yml', 'UTF-8',      :text],
-    ['laholator.py',              'UTF-8',      :text],
-    ['vimrc',                     'UTF-8',      :text],
-    ['AnsiGraph.psm1',            'UTF-16LE',   :text],
-    ['utf16be.html',              'UTF-16BE',   :text],
-    ['utf32le.html',              'UTF-32LE',   :text],
-    ['utf32be.html',              'UTF-32BE',   :text],
-    ['hello_world',               nil,          :binary],
-    ['octocat.png',               nil,          :binary],
-    ['octocat.jpg',               nil,          :binary],
-    ['octocat.psd',               nil,          :binary],
-    ['octocat.gif',               nil,          :binary],
-    ['octocat.ai',                nil,          :binary],
-    ['foo.pdf',                   nil,          :binary],
-  ]
-
-  def test_detection_works_as_expected
-    MAPPING.each do |mapping|
-      file, encoding, type = mapping
-
-      content = fixture(file).read
-      guessed = @detector.detect content
-
-      assert_equal encoding, guessed[:encoding]
-      assert_equal type, guessed[:type]
-
-      if content.respond_to?(:force_encoding) && guessed[:type] == :text
-        content.force_encoding guessed[:encoding]
-        assert content.valid_encoding?
-      end
-    end
-  end
-end

data/test/fixtures/ISO-2022-KR.txt

@@ -1,43 +0,0 @@
-$)C#
-# Out-AnsiGraph.psm1
-# Author:       xcud
-# History:
-#       v0.1 September 21, 2009 initial version
-#
-# PS Example> ps | select -first 5 | sort -property VM | 
-#             Out-AnsiGraph ProcessName, VM
-#                 AEADISRV  14508032
-#                  audiodg  50757632
-#                  conhost  73740288
-# AppleMobileDeviceService  92061696
-#                    btdna  126443520
-#
-function Out-AnsiGraph($Parameter1=$null) {
-	BEGIN {
-		$q = new-object Collections.queue
-		$max = 0; $namewidth = 0;
-	}
-
-	PROCESS {
-		if($_) {
-			$name = $_.($Parameter1[0]);
-			$val = $_.($Parameter1[1])
-			if($max -lt $val) { $max = $val}		 
-			if($namewidth -lt $name.length) { 
-				$namewidth = $name.length }
-			$q.enqueue(@($name, $val))			
-		}
-	}
-
-	END {
-		$q | %{
-			$graph = ""; 0..($_[1]/$max*20) | 
-				%{ $graph += "" }
-			$name = "{0,$namewidth}" -f $_[0]
-			"$name $graph " + $_[1]
-		}
-
-	}
-}
-
-Export-ModuleMember Out-AnsiGraph

data/test/fixtures/TwigExtensionsDate.es.yml

@@ -1,8 +0,0 @@
-date.year: '%year% año|%year% años'
-date.month: '%month% mes|%month% meses'
-date.day: '%day% día|%day% días'
-date.hour: '%hour% hora|%hour% horas'
-date.minute: '%minute% minuto|%minute% minutos'
-date.second: '%second% segundo|%second% segundos'
-date.new: 'menos de un minuto'
-date.and: ' y '

data/test/fixtures/cl-messagepack.lisp

@@ -1,264 +0,0 @@
-;;;; cl-messagepack.lisp
-
-(in-package #:messagepack)
-
-(declaim (optimize (debug 3)))
-
-(eval-when (:compile-toplevel :load-toplevel :execute)
-  (defun mkstr (&rest args)
-    (format nil "~{~a~}" args))
-  (defun mksymb (&rest args)
-    (intern (apply #'mkstr args))))
-
-(defmacro signed-unsigned-convertors (size)
-  (let ((speed (if (< size 32) 3 0)))
-    `(progn
-       (defun ,(mksymb 'sb size '-> 'ub size) (sb)
-         (declare (optimize (debug 0) (safety 0) (speed ,speed))
-                  (type (integer ,(- (expt 2 (1- size))) ,(1- (expt 2 (1- size)))) sb))
-         (if (< sb 0)
-             (ldb (byte ,size 0) sb)
-             sb))
-       (defun ,(mksymb 'ub size '-> 'sb size) (sb)
-         (declare (optimize (debug 0) (safety 0) (speed ,speed))
-                  (type (mod ,(expt 2 size)) sb))
-         (if (logbitp (1- ,size) sb)
-             (- (1+ (logxor (1- (expt 2 ,size)) sb)))
-             sb)))))
-
-(signed-unsigned-convertors 8)
-(signed-unsigned-convertors 16)
-(signed-unsigned-convertors 32)
-(signed-unsigned-convertors 64)
-
-(defun write-hex (data)
-  (let (line)
-    (loop
-       for i from 0 to (1- (length data))
-       do (push (elt data i) line)
-       when (= (length line) 16)
-       do
-         (format t "~{~2,'0x ~}~%" (nreverse line))
-         (setf line nil))
-    (when line
-      (format t "~{~2,'0x ~}~%" (nreverse line)))))
-
-(defun encode (data)
-  (flexi-streams:with-output-to-sequence (stream)
-    (encode-stream data stream)))
-
-(defun make-hash (data)
-  (let ((result (make-hash-table)))
-    (dolist (kv data)
-      (cond ((consp (cdr kv))
-             (setf (gethash (first kv) result) (second kv)))
-            (t
-             (setf (gethash (car kv) result) (cdr kv)))))
-    result))
-
-(defun is-byte-array (data-type)
-  (and (vectorp data-type)
-       (equal '(unsigned-byte 8) (array-element-type data-type))))
-
-(defun encode-stream (data stream)
-  (cond ((floatp data) (encode-float data stream))
-        ((numberp data) (encode-integer data stream))
-        ((null data) (write-byte #xc0 stream))
-        ((eq data t) (write-byte #xc3 stream))
-        ((stringp data)
-         (encode-string data stream))
-        ((is-byte-array data)
-         (encode-raw-bytes data stream))
-        ((or (consp data) (vectorp data))
-         (encode-array data stream))
-        ((hash-table-p data)
-         (encode-hash data stream))
-        ((symbolp data)
-         (encode-string (symbol-name data) stream))
-        (t (error "Cannot encode data."))))
-
-(defun encode-string (data stream)
-  (encode-raw-bytes (babel:string-to-octets data) stream))
-
-#+sbcl (defun sbcl-encode-float (data stream)
-         (cond ((equal (type-of data) 'single-float)
-                (write-byte #xca stream)
-                (store-big-endian (sb-kernel:single-float-bits data) stream 4))
-               ((equal (type-of data) 'double-float)
-                (write-byte #xcb stream)
-                (store-big-endian (sb-kernel:double-float-high-bits data) stream 4)
-                (store-big-endian (sb-kernel:double-float-low-bits data) stream 4)))
-         t)
-
-(defun encode-float (data stream)
-  (or #+sbcl (sbcl-encode-float data stream)
-      #-(or sbcl) (error "No floating point support yet.")))
-
-(defun encode-each (data stream &optional (encoder #'encode-stream))
-  (cond ((hash-table-p data)
-         (maphash (lambda (key value)
-                    (funcall encoder key stream)
-                    (funcall encoder value stream))
-                  data))
-        ((or (vectorp data) (consp data))
-         (mapc (lambda (subdata)
-                 (funcall encoder subdata stream))
-               (coerce data 'list)))
-        (t (error "Not sequence or hash table."))))
-
-(defun encode-sequence (data stream
-                        short-prefix short-length
-                        typecode-16 typecode-32
-                        &optional (encoder #'encode-stream))
-  (let ((len (if (hash-table-p data)
-                 (hash-table-count data)
-                 (length data))))
-    (cond ((<= 0 len short-length)
-           (write-byte (+ short-prefix len) stream)
-           (encode-each data stream encoder))
-          ((<= 0 len 65535)
-           (write-byte typecode-16 stream)
-           (store-big-endian len stream 2)
-           (encode-each data stream encoder))
-          ((<= 0 len (1- (expt 2 32)))
-           (write-byte typecode-32 stream)
-           (store-big-endian len stream 4)
-           (encode-each data stream encoder)))))
-
-(defun encode-hash (data stream)
-  (encode-sequence data stream #x80 15 #xdc #xdd))
-
-(defun encode-array (data stream)
-  (encode-sequence data stream #x90 15 #xdc #xdd))
-
-(defun encode-raw-bytes (data stream)
-  (encode-sequence data stream #xa0 31 #xda #xdb #'write-byte))
-
-(defun encode-integer (data stream)
-  (cond ((<= 0 data 127) (write-byte data stream))
-        ((<= -32 data -1) (write-byte (sb8->ub8 data) stream))
-        ((<= 0 data 255)
-         (write-byte #xcc stream)
-         (write-byte data stream))
-        ((<= 0 data 65535)
-         (write-byte #xcd stream)
-         (store-big-endian data stream 2))
-        ((<= 0 data (1- (expt 2 32)))
-         (write-byte #xce stream)
-         (store-big-endian data stream 4))
-        ((<= 0 data (1- (expt 2 64)))
-         (write-byte #xcf stream)
-         (store-big-endian data stream 8))
-        ((<= -128 data 127)
-         (write-byte #xd0 stream)
-         (write-byte (sb8->ub8 data) stream))
-        ((<= -32768 data 32767)
-         (write-byte #xd1 stream)
-         (write-byte (sb16->ub16 data) stream))
-        ((<= (- (expt 2 31)) data (1- (expt 2 31)))
-         (write-byte #xd2 stream)
-         (write-byte (sb32->ub32 data) stream))
-        ((<= (- (expt 2 63)) data (1- (expt 2 63)))
-         (write-byte #xd3 stream)
-         (write-byte (sb64->ub64 data) stream))
-        (t (error "Integer too large or too small."))))
-
-(defun store-big-endian (number stream byte-count)
-  (let (byte-list)
-    (loop
-       while (> number 0)
-       do
-         (push (rem number 256)
-               byte-list)
-         (setf number (ash number -8)))
-    (loop
-       while (< (length byte-list) byte-count)
-       do (push 0 byte-list))
-    (when (> (length byte-list) byte-count)
-      (error "Number too large."))
-    (write-sequence byte-list stream)))
-
-(defun decode (byte-array)
-  (flexi-streams:with-input-from-sequence (stream byte-array)
-    (decode-stream stream)))
-
-(defun decode-stream (stream)
-  (let ((byte (read-byte stream)))
-    (cond ((= 0 (ldb (byte 1 7) byte))
-           byte)
-          ((= 7 (ldb (byte 3 5) byte))
-           (ub8->sb8 byte))
-          ((= #xcc byte)
-           (read-byte stream))
-          ((= #xcd byte)
-           (load-big-endian stream 2))
-          ((= #xce byte)
-           (load-big-endian stream 4))
-          ((= #xcf byte)
-           (load-big-endian stream 8))
-          ((= #xd0 byte)
-           (ub8->sb8 (read-byte stream)))
-          ((= #xd1 byte)
-           (ub16->sb16 (load-big-endian stream 2)))
-          ((= #xd2 byte)
-           (ub32->sb32 (load-big-endian stream 4)))
-          ((= #xd3 byte)
-           (ub64->sb64 (load-big-endian stream 8)))
-          ((= #xc0 byte)
-           nil)
-          ((= #xc3 byte)
-           t)
-          ((= #xc2 byte)
-           nil)
-          ((= #xca byte)
-           (or #+sbcl (sb-kernel:make-single-float (load-big-endian stream 4))
-               #-(or sbcl) (error "No floating point support yet.")))
-          ((= #xcb byte)
-           (or #+sbcl (sb-kernel:make-double-float (load-big-endian stream 4)
-                                                   (load-big-endian stream 4))
-               #-(or sbcl) (error "No floating point support yet.")))
-          ((= 5 (ldb (byte 3 5) byte))
-           (decode-raw-sequence (ldb (byte 5 0) byte) stream))
-          ((= #xda byte)
-           (decode-raw-sequence (load-big-endian stream 2) stream))
-          ((= #xdb byte)
-           (decode-raw-sequence (load-big-endian stream 4) stream))
-          ((= 9 (ldb (byte 4 4) byte))
-           (decode-array (- byte #x90) stream))
-          ((= #xdc byte)
-           (decode-array (load-big-endian stream 2) stream))
-          ((= #xdd byte)
-           (decode-array (load-big-endian stream 4) stream))
-          ((= 8 (ldb (byte 4 4) byte))
-           (decode-map (- byte #x80) stream))
-          ((= #xde byte)
-           (decode-map (load-big-endian stream 2) stream))
-          ((= #xdf byte)
-           (decode-map (load-big-endian stream 4) stream)))))
-
-(defun decode-map (length stream)
-  (let ((hash-table (make-hash-table :test #'equal)))
-    (loop repeat length
-       do (let ((key (decode-stream stream))
-                (value (decode-stream stream)))
-            (setf (gethash key hash-table) value)))
-    hash-table))
-
-(defun decode-array (length stream)
-  (let ((array (make-array length)))
-    (dotimes (i length)
-      (setf (aref array i) (decode-stream stream)))
-    array))
-
-(defun decode-raw-sequence (length stream)
-  (let ((seq (make-array length :element-type '(mod 256))))
-    (read-sequence seq stream)
-    (babel:octets-to-string seq)))
-
-(defun load-big-endian (stream byte-count)
-  (let ((result 0))
-    (loop
-       repeat byte-count
-       do (setf result (+ (ash result 8)
-                          (read-byte stream))))
-    result))