connectors_service 8.7.0.0.pre.20221117T010623Z → 8.11.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/config/connectors.yml +10 -8
- data/lib/app/config.rb +6 -1
- data/lib/app/console_app.rb +1 -1
- data/lib/app/dispatcher.rb +18 -3
- data/lib/connectors/base/connector.rb +39 -22
- data/lib/connectors/crawler/scheduler.rb +36 -0
- data/lib/connectors/example/connector.rb +2 -2
- data/lib/connectors/example/example_advanced_snippet_validator.rb +4 -3
- data/lib/connectors/gitlab/connector.rb +4 -4
- data/lib/connectors/gitlab/gitlab_advanced_snippet_validator.rb +8 -10
- data/lib/{connectors_app/// → connectors/job_trigger_method.rb} +6 -5
- data/lib/connectors/mongodb/connector.rb +66 -56
- data/lib/connectors/mongodb/mongo_advanced_snippet_against_schema_validator.rb +2 -2
- data/lib/connectors/mongodb/mongo_advanced_snippet_schema.rb +3 -2
- data/lib/connectors/mongodb/mongo_advanced_snippet_snake_case_transformer.rb +49 -0
- data/lib/connectors/registry.rb +1 -1
- data/lib/connectors/tolerable_error_helper.rb +5 -1
- data/lib/connectors_utility.rb +6 -3
- data/lib/core/configuration.rb +13 -1
- data/lib/core/connector_job.rb +48 -7
- data/lib/core/connector_settings.rb +52 -20
- data/lib/core/elastic_connector_actions.rb +54 -38
- data/lib/core/filtering/advanced_snippet/advanced_snippet_against_schema_validator.rb +32 -0
- data/lib/core/filtering/advanced_snippet/advanced_snippet_validator.rb +27 -0
- data/lib/core/filtering/filter_validator.rb +103 -0
- data/lib/{connectors/base/advanced_snippet_against_schema_validator.rb → core/filtering/hash_against_schema_validator.rb} +58 -44
- data/lib/core/filtering/post_process_engine.rb +2 -2
- data/lib/core/filtering/processing_stage.rb +20 -0
- data/lib/core/filtering/{simple_rule.rb → simple_rules/simple_rule.rb} +34 -1
- data/lib/core/filtering/simple_rules/simple_rules_parser.rb +44 -0
- data/lib/core/filtering/simple_rules/validation/no_conflicting_policies_rules_validator.rb +47 -0
- data/lib/core/filtering/simple_rules/validation/simple_rules_schema.rb +68 -0
- data/lib/core/filtering/simple_rules/validation/simple_rules_validator.rb +25 -0
- data/lib/core/filtering/simple_rules/validation/single_rule_against_schema_validator.rb +37 -0
- data/lib/core/filtering/transform/filter_transformer.rb +26 -0
- data/lib/core/filtering/transform/filter_transformer_facade.rb +61 -0
- data/lib/core/filtering/transform/transformation_target.rb +10 -0
- data/lib/core/filtering/validation_job_runner.rb +1 -3
- data/lib/core/filtering.rb +5 -3
- data/lib/core/job_cleanup.rb +66 -0
- data/lib/core/jobs/consumer.rb +62 -64
- data/lib/core/jobs/producer.rb +3 -0
- data/lib/core/scheduler.rb +67 -52
- data/lib/core/sync_job_runner.rb +170 -83
- data/lib/core.rb +1 -0
- data/lib/utility/bulk_queue.rb +1 -1
- data/lib/utility/constants.rb +0 -2
- data/lib/utility/error_monitor.rb +26 -5
- data/lib/utility/es_client.rb +4 -0
- data/lib/utility/filtering.rb +4 -0
- metadata +32 -21
- data/lib/connectors/base/advanced_snippet_validator.rb +0 -34
- data/lib/connectors/base/simple_rules_parser.rb +0 -42
- data/lib/connectors/mongodb/mongo_rules_parser.rb +0 -81
@@ -1,42 +0,0 @@
|
|
1
|
-
#
|
2
|
-
# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
3
|
-
# or more contributor license agreements. Licensed under the Elastic License;
|
4
|
-
# you may not use this file except in compliance with the Elastic License.
|
5
|
-
#
|
6
|
-
# frozen_string_literal: true
|
7
|
-
|
8
|
-
require 'active_support/core_ext/hash/indifferent_access'
|
9
|
-
require 'active_support/core_ext/object/blank'
|
10
|
-
require 'core/filtering/simple_rule'
|
11
|
-
|
12
|
-
module Connectors
|
13
|
-
module Base
|
14
|
-
class SimpleRulesParser
|
15
|
-
def initialize(rules)
|
16
|
-
@rules = (rules || []).map(&:with_indifferent_access).filter { |r| r[:id] != 'DEFAULT' }.sort_by { |r| r[:order] }
|
17
|
-
end
|
18
|
-
|
19
|
-
def parse
|
20
|
-
merge_rules(@rules.map do |rule_hash|
|
21
|
-
rule = Core::Filtering::SimpleRule.new(rule_hash)
|
22
|
-
unless rule.is_include? || rule.is_exclude?
|
23
|
-
raise "Unknown policy: #{rule.policy}"
|
24
|
-
end
|
25
|
-
parse_rule(rule)
|
26
|
-
end)
|
27
|
-
end
|
28
|
-
|
29
|
-
private
|
30
|
-
|
31
|
-
# merge all rules into a filter object or array
|
32
|
-
# in a base case, does no transformations
|
33
|
-
def merge_rules(rules)
|
34
|
-
rules || []
|
35
|
-
end
|
36
|
-
|
37
|
-
def parse_rule(_rule)
|
38
|
-
raise 'Not implemented'
|
39
|
-
end
|
40
|
-
end
|
41
|
-
end
|
42
|
-
end
|
@@ -1,81 +0,0 @@
|
|
1
|
-
#
|
2
|
-
# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
3
|
-
# or more contributor license agreements. Licensed under the Elastic License;
|
4
|
-
# you may not use this file except in compliance with the Elastic License.
|
5
|
-
#
|
6
|
-
|
7
|
-
# frozen_string_literal: true
|
8
|
-
|
9
|
-
require 'active_support/core_ext/object'
|
10
|
-
require 'connectors/base/simple_rules_parser'
|
11
|
-
require 'core/filtering/simple_rule'
|
12
|
-
|
13
|
-
module Connectors
|
14
|
-
module MongoDB
|
15
|
-
class MongoRulesParser < Connectors::Base::SimpleRulesParser
|
16
|
-
def parse_rule(rule)
|
17
|
-
field = rule.field
|
18
|
-
value = rule.value
|
19
|
-
unless value.present?
|
20
|
-
raise "value is required for field: #{field}"
|
21
|
-
end
|
22
|
-
unless field.present?
|
23
|
-
raise "field is required for rule: #{rule}"
|
24
|
-
end
|
25
|
-
op = rule.rule
|
26
|
-
case op
|
27
|
-
when Core::Filtering::SimpleRule::Rule::EQUALS
|
28
|
-
parse_equals(rule)
|
29
|
-
when Core::Filtering::SimpleRule::Rule::GREATER_THAN
|
30
|
-
parse_greater_than(rule)
|
31
|
-
when Core::Filtering::SimpleRule::Rule::LESS_THAN
|
32
|
-
parse_less_than(rule)
|
33
|
-
when Core::Filtering::SimpleRule::Rule::REGEX
|
34
|
-
parse_regex(rule)
|
35
|
-
else
|
36
|
-
raise "Unknown operator: #{op}"
|
37
|
-
end
|
38
|
-
end
|
39
|
-
|
40
|
-
def merge_rules(rules)
|
41
|
-
return {} if rules.empty?
|
42
|
-
return rules[0] if rules.size == 1
|
43
|
-
{ '$and' => rules }
|
44
|
-
end
|
45
|
-
|
46
|
-
private
|
47
|
-
|
48
|
-
def parse_equals(rule)
|
49
|
-
if rule.is_include?
|
50
|
-
{ rule.field => rule.value }
|
51
|
-
else
|
52
|
-
{ rule.field => { '$ne' => rule.value } }
|
53
|
-
end
|
54
|
-
end
|
55
|
-
|
56
|
-
def parse_greater_than(rule)
|
57
|
-
if rule.is_include?
|
58
|
-
{ rule.field => { '$gt' => rule.value } }
|
59
|
-
else
|
60
|
-
{ rule.field => { '$lte' => rule.value } }
|
61
|
-
end
|
62
|
-
end
|
63
|
-
|
64
|
-
def parse_less_than(rule)
|
65
|
-
if rule.is_include?
|
66
|
-
{ rule.field => { '$lt' => rule.value } }
|
67
|
-
else
|
68
|
-
{ rule.field => { '$gte' => rule.value } }
|
69
|
-
end
|
70
|
-
end
|
71
|
-
|
72
|
-
def parse_regex(rule)
|
73
|
-
if rule.is_include?
|
74
|
-
{ rule.field => /#{rule.value}/ }
|
75
|
-
else
|
76
|
-
{ rule.field => { '$not' => /#{rule.value}/ } }
|
77
|
-
end
|
78
|
-
end
|
79
|
-
end
|
80
|
-
end
|
81
|
-
end
|