connectors_utility 8.6.0.4.pre.20221116T024609Z → 8.7.0.0.pre.20221117T004939Z
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/utility/bulk_queue.rb +3 -1
- data/lib/utility/constants.rb +5 -0
- metadata +7 -7
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 5d2972f8e6974a79b6088ce6c03453c327132ce19ffb09dbf30f349eae4c2108
|
|
4
|
+
data.tar.gz: 4fd458de07be07923e0675dc0f341b8211ba3daeec8ac27bfb4f9eb9aff2334a
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 9db02a3003d5645cbb5d57d4ca1bdb1acb65234f5a931afd9ccb06e2fbbe25be2394c65a72a9ae36038c8c127c35a0d937c83558ede0e4960fc688b073db052a
|
|
7
|
+
data.tar.gz: d02681e0d4009420b949ec649c9eac52bd533eee493c181ea1ffb15d939b561e81f234bac4885eefd0ea82c8564d7d330c0635c535b0616cfb9280e2e38512df
|
data/lib/utility/bulk_queue.rb
CHANGED
|
@@ -6,12 +6,14 @@
|
|
|
6
6
|
|
|
7
7
|
require 'json'
|
|
8
8
|
|
|
9
|
+
require 'utility/constants'
|
|
10
|
+
|
|
9
11
|
module Utility
|
|
10
12
|
class BulkQueue
|
|
11
13
|
class QueueOverflowError < StandardError; end
|
|
12
14
|
|
|
13
15
|
# 500 items or 5MB
|
|
14
|
-
def initialize(operation_count_threshold =
|
|
16
|
+
def initialize(operation_count_threshold = Utility::Constants::DEFAULT_MAX_INGESTION_QUEUE_LENGTH, size_threshold = Utility::Constants::DEFAULT_MAX_INGESTION_QUEUE_BYTES)
|
|
15
17
|
@operation_count_threshold = operation_count_threshold.freeze
|
|
16
18
|
@size_threshold = size_threshold.freeze
|
|
17
19
|
|
data/lib/utility/constants.rb
CHANGED
|
@@ -18,5 +18,10 @@ module Utility
|
|
|
18
18
|
CRAWLER_SERVICE_TYPE = 'elastic-crawler'
|
|
19
19
|
FILTERING_RULES_FEATURE = 'filtering_rules'
|
|
20
20
|
FILTERING_ADVANCED_FEATURE = 'filtering_advanced_config'
|
|
21
|
+
|
|
22
|
+
# Maximum number of operations in BULK Elasticsearch operation that will ingest the data
|
|
23
|
+
DEFAULT_MAX_INGESTION_QUEUE_SIZE = 500
|
|
24
|
+
# Maximum size of either whole BULK Elasticsearch operation or one document in it
|
|
25
|
+
DEFAULT_MAX_INGESTION_QUEUE_BYTES = 5 * 1024 * 1024
|
|
21
26
|
end
|
|
22
27
|
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: connectors_utility
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 8.
|
|
4
|
+
version: 8.7.0.0.pre.20221117T004939Z
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Elastic
|
|
8
|
-
autorequire:
|
|
8
|
+
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2022-11-
|
|
11
|
+
date: 2022-11-17 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: activesupport
|
|
@@ -130,9 +130,9 @@ homepage: https://github.com/elastic/connectors-ruby
|
|
|
130
130
|
licenses:
|
|
131
131
|
- Elastic-2.0
|
|
132
132
|
metadata:
|
|
133
|
-
revision:
|
|
134
|
-
repository:
|
|
135
|
-
post_install_message:
|
|
133
|
+
revision: 294214a26b0fe9a4347763b01de681c336e8daae
|
|
134
|
+
repository: https://github.com/elastic/connectors-ruby.git
|
|
135
|
+
post_install_message:
|
|
136
136
|
rdoc_options: []
|
|
137
137
|
require_paths:
|
|
138
138
|
- lib
|
|
@@ -148,7 +148,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
148
148
|
version: 1.3.1
|
|
149
149
|
requirements: []
|
|
150
150
|
rubygems_version: 3.0.3.1
|
|
151
|
-
signing_key:
|
|
151
|
+
signing_key:
|
|
152
152
|
specification_version: 4
|
|
153
153
|
summary: Gem containing shared Connector Services libraries
|
|
154
154
|
test_files: []
|