cld3 3.4.4 → 3.5.1

data/ext/cld3/feature_extractor.proto

@@ -1,50 +0,0 @@
-/* Copyright 2016 Google Inc. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-// Protocol buffers for feature extractor.
-
-syntax = "proto2";
-option optimize_for = LITE_RUNTIME;
-
-package chrome_lang_id;
-
-message Parameter {
-  optional string name = 1;
-  optional string value = 2;
-}
-
-// Descriptor for feature function.
-message FeatureFunctionDescriptor {
-  // Feature function type.
-  required string type = 1;
-
-  // Feature function name.
-  optional string name = 2;
-
-  // Default argument for feature function.
-  optional int32 argument = 3 [default = 0];
-
-  // Named parameters for feature descriptor.
-  repeated Parameter parameter = 4;
-
-  // Nested sub-feature function descriptors.
-  repeated FeatureFunctionDescriptor feature = 7;
-};
-
-// Descriptor for feature extractor.
-message FeatureExtractorDescriptor {
-  // Top-level feature function for extractor.
-  repeated FeatureFunctionDescriptor feature = 1;
-};

data/ext/cld3/libcld3.def

@@ -1,8 +0,0 @@
-EXPORTS
-  NNetLanguageIdentifier_find_language
-  NNetLanguageIdentifier_find_top_n_most_freq_langs
-  delete_NNetLanguageIdentifier
-  delete_result
-  delete_results
-  new_NNetLanguageIdentifier
-  refer_to_nth_result

data/ext/cld3/mkmf.log

@@ -1,37 +0,0 @@
-"pkg-config --exists protobuf"
-| pkg-config --libs protobuf
-=> "-lprotobuf -lpthread \n"
-"gcc -o conftest -I/usr/include -I/usr/include/ruby/backward -I/usr/include -I.    -O2  -fexceptions -g -grecord-gcc-switches -pipe -Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -fstack-protector-strong -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1  -mbranch-protection=standard -fasynchronous-unwind-tables -fstack-clash-protection -fPIC conftest.c  -L. -L/usr/lib64 -L. -Wl,-z,relro -Wl,--as-needed  -Wl,-z,now -specs=/usr/lib/rpm/redhat/redhat-hardened-ld  -fstack-protector-strong -rdynamic -Wl,-export-dynamic     -lruby  -lm   -lc"
-checked program was:
-/* begin */
-1: #include "ruby.h"
-2: 
-3: int main(int argc, char **argv)
-4: {
-5:   return !!argv[argc];
-6: }
-/* end */
-
-"gcc -o conftest -I/usr/include -I/usr/include/ruby/backward -I/usr/include -I.    -O2  -fexceptions -g -grecord-gcc-switches -pipe -Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -fstack-protector-strong -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1  -mbranch-protection=standard -fasynchronous-unwind-tables -fstack-clash-protection -fPIC conftest.c  -L. -L/usr/lib64 -L. -Wl,-z,relro -Wl,--as-needed  -Wl,-z,now -specs=/usr/lib/rpm/redhat/redhat-hardened-ld  -fstack-protector-strong -rdynamic -Wl,-export-dynamic     -lruby -lprotobuf -lpthread -lm   -lc"
-checked program was:
-/* begin */
-1: #include "ruby.h"
-2: 
-3: int main(int argc, char **argv)
-4: {
-5:   return !!argv[argc];
-6: }
-/* end */
-
-| pkg-config --cflags-only-I protobuf
-=> "\n"
-| pkg-config --cflags-only-other protobuf
-=> "\n"
-| pkg-config --libs-only-l protobuf
-=> "-lprotobuf -lpthread \n"
-package configuration for protobuf
-incflags: 
-cflags: 
-ldflags: 
-libs: -lprotobuf -lpthread
-

data/ext/cld3/sentence.proto

@@ -1,77 +0,0 @@
-/* Copyright 2016 Google Inc. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-// Protocol buffer specification for sentence analysis.
-
-syntax = "proto2";
-option optimize_for = LITE_RUNTIME;
-
-package chrome_lang_id;
-
-// A Sentence contains the raw text contents of a sentence, as well as an
-// analysis.
-message Sentence {
-  // Identifier for sentence.
-  optional string id = 1;
-
-  // Raw text contents of the sentence.
-  optional string text = 2;
-
-  // Tokenization of the sentence.
-  repeated Token token = 3;
-
-  extensions 1000 to max;
-}
-
-// A sentence token marks a span of bytes in the sentence text as a token
-// or word.
-message Token {
-  // Token word form.
-  required string word = 1;
-
-  // Start position of token in text.
-  required int32 start = 2;
-
-  // End position of token in text. Gives index of last byte, not one past
-  // the last byte. If token came from lexer, excludes any trailing HTML tags.
-  required int32 end = 3;
-
-  // Head of this token in the dependency tree: the id of the token which has an
-  // arc going to this one. If it is the root token of a sentence, then it is
-  // set to -1.
-  optional int32 head = 4 [default = -1];
-
-  // Part-of-speech tag for token.
-  optional string tag = 5;
-
-  // Coarse-grained word category for token.
-  optional string category = 6;
-
-  // Label for dependency relation between this token and its head.
-  optional string label = 7;
-
-  // Break level for tokens that indicates how it was separated from the
-  // previous token in the text.
-  enum BreakLevel {
-    NO_BREAK = 0;         // No separation between tokens.
-    SPACE_BREAK = 1;      // Tokens separated by space.
-    LINE_BREAK = 2;       // Tokens separated by line break.
-    SENTENCE_BREAK = 3;   // Tokens separated by sentence break.
-  }
-
-  optional BreakLevel break_level = 8 [default = SPACE_BREAK];
-
-  extensions 1000 to max;
-}

data/ext/cld3/task_spec.proto

@@ -1,98 +0,0 @@
-/* Copyright 2016 Google Inc. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-// LINT: ALLOW_GROUPS
-// Protocol buffer specifications for task configuration.
-
-syntax = "proto2";
-option optimize_for = LITE_RUNTIME;
-
-package chrome_lang_id;
-
-// Task input descriptor.
-message TaskInput {
-  // Name of input resource.
-  required string name = 1;
-
-  // Name of stage responsible of creating this resource.
-  optional string creator = 2;
-
-  // File format for resource.
-  repeated string file_format = 3;
-
-  // Record format for resource.
-  repeated string record_format = 4;
-
-  // Is this resource multi-file?
-  optional bool multi_file = 5 [default = false];
-
-  // An input can consist of multiple file sets.
-  repeated group Part = 6 {
-    // File pattern for file set.
-    optional string file_pattern = 7;
-
-    // File format for file set.
-    optional string file_format = 8;
-
-    // Record format for file set.
-    optional string record_format = 9;
-  }
-}
-
-// Task output descriptor.
-message TaskOutput {
-  // Name of output resource.
-  required string name = 1;
-
-  // File format for output resource.
-  optional string file_format = 2;
-
-  // Record format for output resource.
-  optional string record_format = 3;
-
-  // Number of shards in output. If it is different from zero this output is
-  // sharded. If the number of shards is set to -1 this means that the output is
-  // sharded, but the number of shard is unknown. The files are then named
-  // 'base-*-of-*'.
-  optional int32 shards = 4 [default = 0];
-
-  // Base file name for output resource. If this is not set by the task
-  // component it is set to a default value by the workflow engine.
-  optional string file_base = 5;
-
-  // Optional extension added to the file name.
-  optional string file_extension = 6;
-}
-
-// A task specification is used for describing executing parameters.
-message TaskSpec {
-  // Name of task.
-  optional string task_name = 1;
-
-  // Workflow task type.
-  optional string task_type = 2;
-
-  // Task parameters.
-  repeated group Parameter = 3 {
-    required string name = 4;
-    optional string value = 5;
-  }
-
-  // Task inputs.
-  repeated TaskInput input = 6;
-
-  // Task outputs.
-  repeated TaskOutput output = 7;
-}

data/lib/a.rb

@@ -1,24 +0,0 @@
-require "cld3"
-
-# Kafka text as an example + the word Velcro
-text = "Πολυαγαπημένε πατέρα πρόσφατα Velcro με ρώτησες κάποια φορά γιατί ισχυρίζομαι πως σε φοβάμαι. Εγώ δεν ήξερα, ως συνήθως, τι να σου απαντήσω, εν μέρει ακριβώς λόγω του φόβου που νιώθω για σένα, εν μέρει επειδή στην αιτιολόγηση του φόβου αυτού συγκαταλέγονται πάρα πολλές λεπτομέρειες, που εν τη ρύμη του λόγου εγώ ούτε κατά το ήμισυ δεν θα μπορούσα να τις συγκρατήσω. Κι αν εδώ προσπαθώ να σου απαντήσω γραπτώς, μόνο ανολοκλήρωτο κατά πολύ θα αποβεί και τούτο, επειδή και κατά τη γραφή ο φόβος και οι συνέπειές του με κωλύουν έναντί σου κι επειδή το μέγεθος του υλικού εν γένει υπερβαίνει κατά πολύ τη μνήμη μου και το λογικό μου. Για σένα το ζήτημα αποδεικνυόταν πάντοτε πολύ απλό, τουλάχιστον στον βαθμό που μιλούσες εσύ γι’ αυτό ενώπιόν μου και, αδιακρίτως, ενώπιον πολλών άλλων. Εσένα σου φαινόταν να είναι κάπως έτσι: Εσύ εργαζόσουν σκληρά σ’ όλη σου τη ζωή, τα πάντα για τα παιδιά σου, προ πάντων για εμένα τα θυσίαζες, εγώ έκαμνα συνεπώς «ζωή χαρισάμενη», είχα πλήρη ελευθερία να μάθω ό,τι ήθελα, κανέναν λόγο δεν είχα να έχω έγνοιες για την καθημερινή διατροφή, να έχω έγνοιες συνεπώς εν γένει• εσύ αντ’ αυτών καμμίαν ευγνωμοσύνη δεν αξίωνες, γνωρίζεις «την ευγνωμοσύνη των παιδιών, αλλά εν τούτοις τουλάχιστον μια "
-pp text.bytesize
-
-200.times { |i|
-  max_bytes = 500 + i * 10
-  cld3 = CLD3::NNetLanguageIdentifier.new("foo", max_bytes)
-
-  lang = cld3.find_language(text)
-  lang2 = cld3.find_top_n_most_freq_langs(text, 1)
-
-  puts "When max_bytes is #{max_bytes} probability is less than 0.999: #{lang.probability}" if lang.probability < 0.999
-
-  if lang.language != :el
-    puts "When max_bytes is #{max_bytes} then cld3::find_language returns #{lang.language},
-      find_top_n_most_freq_langs returns #{lang2.first.language}"
-    #pp lang
-    #pp lang2
-  end
-}
-
-puts "Size: #{text.length} - Bytesize: #{text.encode(Encoding::UTF_8).bytesize}"

data/lib/cld3/unstable.rb

@@ -1,58 +0,0 @@
-
-# Copyright 2021 Akihiko Odaki <akihiko.odaki@gmail.com>
-# All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-module CLD3
-  module Unstable
-    extend FFI::Library
-
-    ffi_lib File.join(__dir__, "..", "..", "ext", "cld3", "libcld3." + RbConfig::CONFIG["DLEXT"])
-
-    module NNetLanguageIdentifier
-      class Pointer < FFI::AutoPointer
-        def self.release(pointer)
-          Unstable.delete_NNetLanguageIdentifier(pointer)
-        end
-      end
-
-      class SpanInfo < FFI::Struct
-        layout :start_index, :int, :end_index, :int, :probability, :float
-      end
-
-      class Result < FFI::Struct
-        layout :language_data, :pointer, :language_size, :size_t, :byte_ranges_data, :pointer, :byte_ranges_size, :size_t, :probability, :float, :proportion, :float, :reliable?, :bool
-      end
-    end
-
-    attach_function :delete_NNetLanguageIdentifier, [ :pointer ], :void
-
-    attach_function :delete_result, [ :pointer ], :void
-
-    attach_function :delete_results, [ :pointer ], :void
-
-    attach_function :new_NNetLanguageIdentifier, [ :int, :int ], :pointer
-
-    attach_function :refer_to_nth_result, [ :pointer, :size_t ], NNetLanguageIdentifier::Result.by_value
-
-    attach_function :NNetLanguageIdentifier_find_language,
-        [ :pointer, :buffer_in, :size_t ], :pointer
-
-    attach_function :NNetLanguageIdentifier_find_top_n_most_freq_langs,
-        [ :pointer, :buffer_in, :size_t, :int ], :pointer
-  end
-
-  private_constant :Unstable
-end