cld3 3.4.2 → 3.5.0

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cld3
 version: !ruby/object:Gem::Version
-  version: 3.4.2
+  version: 3.5.0
 platform: ruby
 authors:
 - Akihiko Odaki
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-04-17 00:00:00.000000000 Z
+date: 2022-07-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -30,26 +30,66 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: 1.16.0
+- !ruby/object:Gem::Dependency
+  name: rbs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.6.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 2.7.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.6.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 2.7.0
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: 3.11.0
     - - "<"
+      - !ruby/object:Gem::Version
+        version: 3.12.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
       - !ruby/object:Gem::Version
         version: 3.11.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 3.12.0
+- !ruby/object:Gem::Dependency
+  name: steep
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 1.1.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: 1.0.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: 3.11.0
+        version: 1.1.0
 description: Compact Language Detector v3 (CLD3) is a neural network model for language
   identification.
 email: akihiko.odaki@gmail.com
@@ -68,6 +108,9 @@ files:
 - ext/cld3/base.h
 - ext/cld3/base.o
 - ext/cld3/casts.h
+- ext/cld3/cld_3/protos/feature_extractor.pb.h
+- ext/cld3/cld_3/protos/sentence.pb.h
+- ext/cld3/cld_3/protos/task_spec.pb.h
 - ext/cld3/embedding_feature_extractor.cc
 - ext/cld3/embedding_feature_extractor.h
 - ext/cld3/embedding_feature_extractor.o
@@ -79,8 +122,6 @@ files:
 - ext/cld3/feature_extractor.cc
 - ext/cld3/feature_extractor.h
 - ext/cld3/feature_extractor.o
-- ext/cld3/feature_extractor.pb.o
-- ext/cld3/feature_extractor.proto
 - ext/cld3/feature_types.cc
 - ext/cld3/feature_types.h
 - ext/cld3/feature_types.o
@@ -108,7 +149,6 @@ files:
 - ext/cld3/language_identifier_features.o
 - ext/cld3/libcld3.def
 - ext/cld3/libcld3.so
-- ext/cld3/mkmf.log
 - ext/cld3/nnet_language_identifier.cc
 - ext/cld3/nnet_language_identifier.h
 - ext/cld3/nnet_language_identifier.o
@@ -125,8 +165,19 @@ files:
 - ext/cld3/relevant_script_feature.h
 - ext/cld3/relevant_script_feature.o
 - ext/cld3/script_detector.h
-- ext/cld3/sentence.pb.o
-- ext/cld3/sentence.proto
+- ext/cld3/script_span/fixunicodevalue.h
+- ext/cld3/script_span/generated_ulscript.h
+- ext/cld3/script_span/getonescriptspan.h
+- ext/cld3/script_span/integral_types.h
+- ext/cld3/script_span/offsetmap.h
+- ext/cld3/script_span/port.h
+- ext/cld3/script_span/stringpiece.h
+- ext/cld3/script_span/text_processing.h
+- ext/cld3/script_span/utf8acceptinterchange.h
+- ext/cld3/script_span/utf8prop_lettermarkscriptnum.h
+- ext/cld3/script_span/utf8repl_lettermarklower.h
+- ext/cld3/script_span/utf8scannot_lettermarkspecial.h
+- ext/cld3/script_span/utf8statetable.h
 - ext/cld3/sentence_features.cc
 - ext/cld3/sentence_features.h
 - ext/cld3/sentence_features.o
@@ -138,8 +189,6 @@ files:
 - ext/cld3/task_context_params.cc
 - ext/cld3/task_context_params.h
 - ext/cld3/task_context_params.o
-- ext/cld3/task_spec.pb.o
-- ext/cld3/task_spec.proto
 - ext/cld3/text_processing.cc
 - ext/cld3/text_processing.h
 - ext/cld3/text_processing.o
@@ -160,6 +209,8 @@ files:
 - ext/cld3/workspace.h
 - ext/cld3/workspace.o
 - lib/cld3.rb
+- lib/cld3/unstable.rb
+- sig/cld3.rbs
 homepage: https://github.com/akihikodaki/cld3-ruby
 licenses:
 - Apache-2.0
@@ -172,17 +223,17 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.6.0
+      version: 2.7.0
   - - "<"
     - !ruby/object:Gem::Version
-      version: 3.1.0
+      version: 3.3.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.3.7
 signing_key:
 specification_version: 4
 summary: Compact Language Detector v3 (CLD3)

data/ext/cld3/feature_extractor.proto

@@ -1,50 +0,0 @@
-/* Copyright 2016 Google Inc. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-// Protocol buffers for feature extractor.
-
-syntax = "proto2";
-option optimize_for = LITE_RUNTIME;
-
-package chrome_lang_id;
-
-message Parameter {
-  optional string name = 1;
-  optional string value = 2;
-}
-
-// Descriptor for feature function.
-message FeatureFunctionDescriptor {
-  // Feature function type.
-  required string type = 1;
-
-  // Feature function name.
-  optional string name = 2;
-
-  // Default argument for feature function.
-  optional int32 argument = 3 [default = 0];
-
-  // Named parameters for feature descriptor.
-  repeated Parameter parameter = 4;
-
-  // Nested sub-feature function descriptors.
-  repeated FeatureFunctionDescriptor feature = 7;
-};
-
-// Descriptor for feature extractor.
-message FeatureExtractorDescriptor {
-  // Top-level feature function for extractor.
-  repeated FeatureFunctionDescriptor feature = 1;
-};

data/ext/cld3/mkmf.log

@@ -1,37 +0,0 @@
-"pkg-config --exists protobuf"
-| pkg-config --libs protobuf
-=> "-lprotobuf -lpthread \n"
-"gcc -o conftest -I/usr/include -I/usr/include/ruby/backward -I/usr/include -I.    -O2  -fexceptions -g -grecord-gcc-switches -pipe -Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -fstack-protector-strong -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1  -mbranch-protection=standard -fasynchronous-unwind-tables -fstack-clash-protection -fPIC conftest.c  -L. -L/usr/lib64 -L. -Wl,-z,relro -Wl,--as-needed  -Wl,-z,now -specs=/usr/lib/rpm/redhat/redhat-hardened-ld  -fstack-protector-strong -rdynamic -Wl,-export-dynamic     -lruby  -lm   -lc"
-checked program was:
-/* begin */
-1: #include "ruby.h"
-2: 
-3: int main(int argc, char **argv)
-4: {
-5:   return !!argv[argc];
-6: }
-/* end */
-
-"gcc -o conftest -I/usr/include -I/usr/include/ruby/backward -I/usr/include -I.    -O2  -fexceptions -g -grecord-gcc-switches -pipe -Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -fstack-protector-strong -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1  -mbranch-protection=standard -fasynchronous-unwind-tables -fstack-clash-protection -fPIC conftest.c  -L. -L/usr/lib64 -L. -Wl,-z,relro -Wl,--as-needed  -Wl,-z,now -specs=/usr/lib/rpm/redhat/redhat-hardened-ld  -fstack-protector-strong -rdynamic -Wl,-export-dynamic     -lruby -lprotobuf -lpthread -lm   -lc"
-checked program was:
-/* begin */
-1: #include "ruby.h"
-2: 
-3: int main(int argc, char **argv)
-4: {
-5:   return !!argv[argc];
-6: }
-/* end */
-
-| pkg-config --cflags-only-I protobuf
-=> "\n"
-| pkg-config --cflags-only-other protobuf
-=> "\n"
-| pkg-config --libs-only-l protobuf
-=> "-lprotobuf -lpthread \n"
-package configuration for protobuf
-incflags: 
-cflags: 
-ldflags: 
-libs: -lprotobuf -lpthread
-

data/ext/cld3/sentence.proto

@@ -1,77 +0,0 @@
-/* Copyright 2016 Google Inc. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-// Protocol buffer specification for sentence analysis.
-
-syntax = "proto2";
-option optimize_for = LITE_RUNTIME;
-
-package chrome_lang_id;
-
-// A Sentence contains the raw text contents of a sentence, as well as an
-// analysis.
-message Sentence {
-  // Identifier for sentence.
-  optional string id = 1;
-
-  // Raw text contents of the sentence.
-  optional string text = 2;
-
-  // Tokenization of the sentence.
-  repeated Token token = 3;
-
-  extensions 1000 to max;
-}
-
-// A sentence token marks a span of bytes in the sentence text as a token
-// or word.
-message Token {
-  // Token word form.
-  required string word = 1;
-
-  // Start position of token in text.
-  required int32 start = 2;
-
-  // End position of token in text. Gives index of last byte, not one past
-  // the last byte. If token came from lexer, excludes any trailing HTML tags.
-  required int32 end = 3;
-
-  // Head of this token in the dependency tree: the id of the token which has an
-  // arc going to this one. If it is the root token of a sentence, then it is
-  // set to -1.
-  optional int32 head = 4 [default = -1];
-
-  // Part-of-speech tag for token.
-  optional string tag = 5;
-
-  // Coarse-grained word category for token.
-  optional string category = 6;
-
-  // Label for dependency relation between this token and its head.
-  optional string label = 7;
-
-  // Break level for tokens that indicates how it was separated from the
-  // previous token in the text.
-  enum BreakLevel {
-    NO_BREAK = 0;         // No separation between tokens.
-    SPACE_BREAK = 1;      // Tokens separated by space.
-    LINE_BREAK = 2;       // Tokens separated by line break.
-    SENTENCE_BREAK = 3;   // Tokens separated by sentence break.
-  }
-
-  optional BreakLevel break_level = 8 [default = SPACE_BREAK];
-
-  extensions 1000 to max;
-}

data/ext/cld3/task_spec.proto

@@ -1,98 +0,0 @@
-/* Copyright 2016 Google Inc. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-// LINT: ALLOW_GROUPS
-// Protocol buffer specifications for task configuration.
-
-syntax = "proto2";
-option optimize_for = LITE_RUNTIME;
-
-package chrome_lang_id;
-
-// Task input descriptor.
-message TaskInput {
-  // Name of input resource.
-  required string name = 1;
-
-  // Name of stage responsible of creating this resource.
-  optional string creator = 2;
-
-  // File format for resource.
-  repeated string file_format = 3;
-
-  // Record format for resource.
-  repeated string record_format = 4;
-
-  // Is this resource multi-file?
-  optional bool multi_file = 5 [default = false];
-
-  // An input can consist of multiple file sets.
-  repeated group Part = 6 {
-    // File pattern for file set.
-    optional string file_pattern = 7;
-
-    // File format for file set.
-    optional string file_format = 8;
-
-    // Record format for file set.
-    optional string record_format = 9;
-  }
-}
-
-// Task output descriptor.
-message TaskOutput {
-  // Name of output resource.
-  required string name = 1;
-
-  // File format for output resource.
-  optional string file_format = 2;
-
-  // Record format for output resource.
-  optional string record_format = 3;
-
-  // Number of shards in output. If it is different from zero this output is
-  // sharded. If the number of shards is set to -1 this means that the output is
-  // sharded, but the number of shard is unknown. The files are then named
-  // 'base-*-of-*'.
-  optional int32 shards = 4 [default = 0];
-
-  // Base file name for output resource. If this is not set by the task
-  // component it is set to a default value by the workflow engine.
-  optional string file_base = 5;
-
-  // Optional extension added to the file name.
-  optional string file_extension = 6;
-}
-
-// A task specification is used for describing executing parameters.
-message TaskSpec {
-  // Name of task.
-  optional string task_name = 1;
-
-  // Workflow task type.
-  optional string task_type = 2;
-
-  // Task parameters.
-  repeated group Parameter = 3 {
-    required string name = 4;
-    optional string value = 5;
-  }
-
-  // Task inputs.
-  repeated TaskInput input = 6;
-
-  // Task outputs.
-  repeated TaskOutput output = 7;
-}