connectors_service 8.7.0.0.pre.20221117T010623Z → 8.11.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (55) hide show
  1. checksums.yaml +4 -4
  2. data/config/connectors.yml +10 -8
  3. data/lib/app/config.rb +6 -1
  4. data/lib/app/console_app.rb +1 -1
  5. data/lib/app/dispatcher.rb +18 -3
  6. data/lib/connectors/base/connector.rb +39 -22
  7. data/lib/connectors/crawler/scheduler.rb +36 -0
  8. data/lib/connectors/example/connector.rb +2 -2
  9. data/lib/connectors/example/example_advanced_snippet_validator.rb +4 -3
  10. data/lib/connectors/gitlab/connector.rb +4 -4
  11. data/lib/connectors/gitlab/gitlab_advanced_snippet_validator.rb +8 -10
  12. data/lib/{connectors_app/// → connectors/job_trigger_method.rb} +6 -5
  13. data/lib/connectors/mongodb/connector.rb +66 -56
  14. data/lib/connectors/mongodb/mongo_advanced_snippet_against_schema_validator.rb +2 -2
  15. data/lib/connectors/mongodb/mongo_advanced_snippet_schema.rb +3 -2
  16. data/lib/connectors/mongodb/mongo_advanced_snippet_snake_case_transformer.rb +49 -0
  17. data/lib/connectors/registry.rb +1 -1
  18. data/lib/connectors/tolerable_error_helper.rb +5 -1
  19. data/lib/connectors_utility.rb +6 -3
  20. data/lib/core/configuration.rb +13 -1
  21. data/lib/core/connector_job.rb +48 -7
  22. data/lib/core/connector_settings.rb +52 -20
  23. data/lib/core/elastic_connector_actions.rb +54 -38
  24. data/lib/core/filtering/advanced_snippet/advanced_snippet_against_schema_validator.rb +32 -0
  25. data/lib/core/filtering/advanced_snippet/advanced_snippet_validator.rb +27 -0
  26. data/lib/core/filtering/filter_validator.rb +103 -0
  27. data/lib/{connectors/base/advanced_snippet_against_schema_validator.rb → core/filtering/hash_against_schema_validator.rb} +58 -44
  28. data/lib/core/filtering/post_process_engine.rb +2 -2
  29. data/lib/core/filtering/processing_stage.rb +20 -0
  30. data/lib/core/filtering/{simple_rule.rb → simple_rules/simple_rule.rb} +34 -1
  31. data/lib/core/filtering/simple_rules/simple_rules_parser.rb +44 -0
  32. data/lib/core/filtering/simple_rules/validation/no_conflicting_policies_rules_validator.rb +47 -0
  33. data/lib/core/filtering/simple_rules/validation/simple_rules_schema.rb +68 -0
  34. data/lib/core/filtering/simple_rules/validation/simple_rules_validator.rb +25 -0
  35. data/lib/core/filtering/simple_rules/validation/single_rule_against_schema_validator.rb +37 -0
  36. data/lib/core/filtering/transform/filter_transformer.rb +26 -0
  37. data/lib/core/filtering/transform/filter_transformer_facade.rb +61 -0
  38. data/lib/core/filtering/transform/transformation_target.rb +10 -0
  39. data/lib/core/filtering/validation_job_runner.rb +1 -3
  40. data/lib/core/filtering.rb +5 -3
  41. data/lib/core/job_cleanup.rb +66 -0
  42. data/lib/core/jobs/consumer.rb +62 -64
  43. data/lib/core/jobs/producer.rb +3 -0
  44. data/lib/core/scheduler.rb +67 -52
  45. data/lib/core/sync_job_runner.rb +170 -83
  46. data/lib/core.rb +1 -0
  47. data/lib/utility/bulk_queue.rb +1 -1
  48. data/lib/utility/constants.rb +0 -2
  49. data/lib/utility/error_monitor.rb +26 -5
  50. data/lib/utility/es_client.rb +4 -0
  51. data/lib/utility/filtering.rb +4 -0
  52. metadata +32 -21
  53. data/lib/connectors/base/advanced_snippet_validator.rb +0 -34
  54. data/lib/connectors/base/simple_rules_parser.rb +0 -42
  55. data/lib/connectors/mongodb/mongo_rules_parser.rb +0 -81
@@ -1,42 +0,0 @@
1
- #
2
- # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3
- # or more contributor license agreements. Licensed under the Elastic License;
4
- # you may not use this file except in compliance with the Elastic License.
5
- #
6
- # frozen_string_literal: true
7
-
8
- require 'active_support/core_ext/hash/indifferent_access'
9
- require 'active_support/core_ext/object/blank'
10
- require 'core/filtering/simple_rule'
11
-
12
- module Connectors
13
- module Base
14
- class SimpleRulesParser
15
- def initialize(rules)
16
- @rules = (rules || []).map(&:with_indifferent_access).filter { |r| r[:id] != 'DEFAULT' }.sort_by { |r| r[:order] }
17
- end
18
-
19
- def parse
20
- merge_rules(@rules.map do |rule_hash|
21
- rule = Core::Filtering::SimpleRule.new(rule_hash)
22
- unless rule.is_include? || rule.is_exclude?
23
- raise "Unknown policy: #{rule.policy}"
24
- end
25
- parse_rule(rule)
26
- end)
27
- end
28
-
29
- private
30
-
31
- # merge all rules into a filter object or array
32
- # in a base case, does no transformations
33
- def merge_rules(rules)
34
- rules || []
35
- end
36
-
37
- def parse_rule(_rule)
38
- raise 'Not implemented'
39
- end
40
- end
41
- end
42
- end
@@ -1,81 +0,0 @@
1
- #
2
- # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3
- # or more contributor license agreements. Licensed under the Elastic License;
4
- # you may not use this file except in compliance with the Elastic License.
5
- #
6
-
7
- # frozen_string_literal: true
8
-
9
- require 'active_support/core_ext/object'
10
- require 'connectors/base/simple_rules_parser'
11
- require 'core/filtering/simple_rule'
12
-
13
- module Connectors
14
- module MongoDB
15
- class MongoRulesParser < Connectors::Base::SimpleRulesParser
16
- def parse_rule(rule)
17
- field = rule.field
18
- value = rule.value
19
- unless value.present?
20
- raise "value is required for field: #{field}"
21
- end
22
- unless field.present?
23
- raise "field is required for rule: #{rule}"
24
- end
25
- op = rule.rule
26
- case op
27
- when Core::Filtering::SimpleRule::Rule::EQUALS
28
- parse_equals(rule)
29
- when Core::Filtering::SimpleRule::Rule::GREATER_THAN
30
- parse_greater_than(rule)
31
- when Core::Filtering::SimpleRule::Rule::LESS_THAN
32
- parse_less_than(rule)
33
- when Core::Filtering::SimpleRule::Rule::REGEX
34
- parse_regex(rule)
35
- else
36
- raise "Unknown operator: #{op}"
37
- end
38
- end
39
-
40
- def merge_rules(rules)
41
- return {} if rules.empty?
42
- return rules[0] if rules.size == 1
43
- { '$and' => rules }
44
- end
45
-
46
- private
47
-
48
- def parse_equals(rule)
49
- if rule.is_include?
50
- { rule.field => rule.value }
51
- else
52
- { rule.field => { '$ne' => rule.value } }
53
- end
54
- end
55
-
56
- def parse_greater_than(rule)
57
- if rule.is_include?
58
- { rule.field => { '$gt' => rule.value } }
59
- else
60
- { rule.field => { '$lte' => rule.value } }
61
- end
62
- end
63
-
64
- def parse_less_than(rule)
65
- if rule.is_include?
66
- { rule.field => { '$lt' => rule.value } }
67
- else
68
- { rule.field => { '$gte' => rule.value } }
69
- end
70
- end
71
-
72
- def parse_regex(rule)
73
- if rule.is_include?
74
- { rule.field => /#{rule.value}/ }
75
- else
76
- { rule.field => { '$not' => /#{rule.value}/ } }
77
- end
78
- end
79
- end
80
- end
81
- end