csdl 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +9 -0
- data/.ruby-version +1 -0
- data/.travis.yml +4 -0
- data/Gemfile +4 -0
- data/README.md +153 -0
- data/Rakefile +10 -0
- data/csdl.gemspec +24 -0
- data/lib/csdl.rb +13 -0
- data/lib/csdl/builder.rb +73 -0
- data/lib/csdl/error.rb +14 -0
- data/lib/csdl/interaction_filter_processor.rb +65 -0
- data/lib/csdl/operators.rb +52 -0
- data/lib/csdl/processor.rb +78 -0
- data/lib/csdl/query_filter_processor.rb +12 -0
- data/lib/csdl/targets.rb +96 -0
- data/lib/csdl/version.rb +3 -0
- metadata +115 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 5f8bac9da29e72abe7b9ed3d735c9e43dd0299f8
|
4
|
+
data.tar.gz: fbddc9f5fe3edd6c023acfad56e2dac01e3baff6
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: bbe665e31c04cacc5922e9f66a6e09835b63516cf900b785df53ade0939aa78e170f29b37723ebe88b3fca8d905467a438a26eff9ab604dc6d899376928c711e
|
7
|
+
data.tar.gz: cdff5b7766ac63008c2c0421f84294a9046bc6d4657149f15675e2768c94fdf24014c8b4daffab0f98f246602979bb7f01bba0c19cf9feff7656e6f5b70326b1
|
data/.gitignore
ADDED
data/.ruby-version
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
2.2.2
|
data/.travis.yml
ADDED
data/Gemfile
ADDED
data/README.md
ADDED
@@ -0,0 +1,153 @@
|
|
1
|
+
# CSDL
|
2
|
+
|
3
|
+
CSDL is a gem for producing Abstract Syntax Trees for the [DataSift CSDL Filter Language](http://dev.datasift.com/docs/csdl).
|
4
|
+
Working with an AST instead of raw strings provides a simpler way to test and validate any given CSDL filter.
|
5
|
+
|
6
|
+
[![Build Status](https://travis-ci.org/localshred/csdl.svg)](https://travis-ci.org/localshred/csdl)
|
7
|
+
|
8
|
+
## Installation
|
9
|
+
|
10
|
+
Add this line to your application's Gemfile:
|
11
|
+
|
12
|
+
```ruby
|
13
|
+
gem "csdl"
|
14
|
+
```
|
15
|
+
|
16
|
+
And then execute:
|
17
|
+
|
18
|
+
$ bundle
|
19
|
+
|
20
|
+
Or install it yourself as:
|
21
|
+
|
22
|
+
$ gem install csdl
|
23
|
+
|
24
|
+
## Usage
|
25
|
+
|
26
|
+
Use the DSL provided by `CSDL::Builder` to produce an AST representation of your query, and use `CSDL::Processor` to turn your AST into a raw CSDL string.
|
27
|
+
|
28
|
+
Valid builder methods are `closure`, `filter`, `_and`, `_or`, and `_not`. The last three are prefixed to avoid keyword collision.
|
29
|
+
|
30
|
+
```ruby
|
31
|
+
builder = ::CSDL::Builder.new._or do
|
32
|
+
[
|
33
|
+
closure {
|
34
|
+
_and {
|
35
|
+
[
|
36
|
+
closure {
|
37
|
+
_or {
|
38
|
+
[
|
39
|
+
filter("fb.content", :contains_any, "ebola"),
|
40
|
+
filter("fb.parent.content", :contains_any, "ebola")
|
41
|
+
]
|
42
|
+
}
|
43
|
+
},
|
44
|
+
_not("fb.content", :contains_any, "government,politics"),
|
45
|
+
filter("fb.author.country_code", :in, "GB")
|
46
|
+
]
|
47
|
+
}
|
48
|
+
},
|
49
|
+
closure {
|
50
|
+
_and {
|
51
|
+
[
|
52
|
+
closure {
|
53
|
+
_or {
|
54
|
+
[
|
55
|
+
filter("fb.content", :contains_any, "malta,malta island,#malta"),
|
56
|
+
filter("fb.parent.content", :contains_any, "malta,malta island,#malta")
|
57
|
+
]
|
58
|
+
}
|
59
|
+
},
|
60
|
+
_not("fb.content", :contains_any, "vacation,poker awards")
|
61
|
+
]
|
62
|
+
}
|
63
|
+
}
|
64
|
+
]
|
65
|
+
end
|
66
|
+
|
67
|
+
puts
|
68
|
+
puts "Builder..."
|
69
|
+
puts builder.to_sexp
|
70
|
+
|
71
|
+
puts
|
72
|
+
puts "Processing AST..."
|
73
|
+
puts ::CSDL::Processor.new.process(builder)
|
74
|
+
```
|
75
|
+
|
76
|
+
The previous script produces the following output:
|
77
|
+
|
78
|
+
```
|
79
|
+
Builder...
|
80
|
+
(or
|
81
|
+
(closure
|
82
|
+
(and
|
83
|
+
(closure
|
84
|
+
(or
|
85
|
+
(filter
|
86
|
+
(target "fb.content")
|
87
|
+
(operator :contains_any)
|
88
|
+
(argument
|
89
|
+
(string "ebola")))
|
90
|
+
(filter
|
91
|
+
(target "fb.parent.content")
|
92
|
+
(operator :contains_any)
|
93
|
+
(argument
|
94
|
+
(string "ebola")))))
|
95
|
+
(not
|
96
|
+
(target "fb.content")
|
97
|
+
(operator :contains_any)
|
98
|
+
(argument
|
99
|
+
(string "government,politics")))
|
100
|
+
(filter
|
101
|
+
(target "fb.author.country_code")
|
102
|
+
(operator :in)
|
103
|
+
(argument
|
104
|
+
(string "GB")))))
|
105
|
+
(closure
|
106
|
+
(and
|
107
|
+
(closure
|
108
|
+
(or
|
109
|
+
(filter
|
110
|
+
(target "fb.content")
|
111
|
+
(operator :contains_any)
|
112
|
+
(argument
|
113
|
+
(string "malta,malta island,#malta")))
|
114
|
+
(filter
|
115
|
+
(target "fb.parent.content")
|
116
|
+
(operator :contains_any)
|
117
|
+
(argument
|
118
|
+
(string "malta,malta island,#malta")))))
|
119
|
+
(not
|
120
|
+
(target "fb.content")
|
121
|
+
(operator :contains_any)
|
122
|
+
(argument
|
123
|
+
(string "vacation,poker awards"))))))
|
124
|
+
|
125
|
+
Processing AST...
|
126
|
+
((fb.content contains_any "ebola" OR fb.parent.content contains_any "ebola") AND NOT fb.content contains_any "government,politics" AND fb.author.country_code in "GB") OR ((fb.content contains_any "malta,malta island,#malta" OR fb.parent.content contains_any "malta,malta island,#malta") AND NOT fb.content contains_any "vacation,poker awards")
|
127
|
+
```
|
128
|
+
|
129
|
+
The processed AST looks like this (manually expanded):
|
130
|
+
|
131
|
+
```
|
132
|
+
(
|
133
|
+
(
|
134
|
+
fb.content contains_any "ebola"
|
135
|
+
OR fb.parent.content contains_any "ebola"
|
136
|
+
)
|
137
|
+
AND NOT fb.content contains_any "government,politics"
|
138
|
+
AND fb.author.country_code in "GB"
|
139
|
+
)
|
140
|
+
OR
|
141
|
+
(
|
142
|
+
(
|
143
|
+
fb.content contains_any "malta,malta island,#malta"
|
144
|
+
OR fb.parent.content contains_any "malta,malta island,#malta"
|
145
|
+
)
|
146
|
+
AND NOT fb.content contains_any "vacation,poker awards"
|
147
|
+
)
|
148
|
+
```
|
149
|
+
|
150
|
+
## Contributing
|
151
|
+
|
152
|
+
Bug reports and pull requests are welcome on GitHub at [https://github.com/localshred/csdl](https://github.com/localshred/csdl).
|
153
|
+
|
data/Rakefile
ADDED
data/csdl.gemspec
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'csdl/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "csdl"
|
8
|
+
spec.version = Csdl::VERSION
|
9
|
+
spec.authors = ["BJ Neilsen"]
|
10
|
+
spec.email = ["bj.neilsen@gmail.com"]
|
11
|
+
|
12
|
+
spec.summary = %q{AST Processor and Query Builder for DataSift's CSDL language}
|
13
|
+
spec.description = spec.summary
|
14
|
+
spec.homepage = "https://localshred.github.io"
|
15
|
+
|
16
|
+
spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
17
|
+
spec.require_paths = ["lib"]
|
18
|
+
|
19
|
+
spec.add_dependency "ast", "~> 2.0"
|
20
|
+
|
21
|
+
spec.add_development_dependency "bundler", "~> 1.10"
|
22
|
+
spec.add_development_dependency "rake", "~> 10.0"
|
23
|
+
spec.add_development_dependency "minitest"
|
24
|
+
end
|
data/lib/csdl.rb
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
require "ast"
|
2
|
+
require "csdl/version"
|
3
|
+
|
4
|
+
module CSDL
|
5
|
+
end
|
6
|
+
|
7
|
+
require "csdl/error"
|
8
|
+
require "csdl/targets"
|
9
|
+
require "csdl/operators"
|
10
|
+
require "csdl/builder"
|
11
|
+
require "csdl/processor"
|
12
|
+
require "csdl/interaction_filter_processor"
|
13
|
+
require "csdl/query_filter_processor"
|
data/lib/csdl/builder.rb
ADDED
@@ -0,0 +1,73 @@
|
|
1
|
+
module CSDL
|
2
|
+
class Builder
|
3
|
+
include ::AST::Sexp
|
4
|
+
|
5
|
+
def __one_or_more_child_nodes(&block)
|
6
|
+
children = instance_eval(&block)
|
7
|
+
[ children ].flatten
|
8
|
+
end
|
9
|
+
|
10
|
+
def _or(&block)
|
11
|
+
s(:or, *__one_or_more_child_nodes(&block))
|
12
|
+
end
|
13
|
+
|
14
|
+
def _and(&block)
|
15
|
+
s(:and, *__one_or_more_child_nodes(&block))
|
16
|
+
end
|
17
|
+
|
18
|
+
def _not(target, operator, argument = nil)
|
19
|
+
node = filter(target, operator, argument)
|
20
|
+
node.updated(:not)
|
21
|
+
end
|
22
|
+
|
23
|
+
def _return(&block)
|
24
|
+
s(:return, statement_scope(&block))
|
25
|
+
end
|
26
|
+
|
27
|
+
def filter(target, operator, argument = nil)
|
28
|
+
target_node = s(:target, target)
|
29
|
+
operator_node = s(:operator, operator)
|
30
|
+
argument_node = nil
|
31
|
+
|
32
|
+
unless argument.nil?
|
33
|
+
argument_node_type = argument.class.name.to_s.downcase.to_sym
|
34
|
+
child_argument_node = s(argument_node_type, argument)
|
35
|
+
argument_node = s(:argument, child_argument_node)
|
36
|
+
end
|
37
|
+
|
38
|
+
s(:filter, *[target_node, operator_node, argument_node].compact)
|
39
|
+
end
|
40
|
+
|
41
|
+
def logical_group(logical_operator = nil, &block)
|
42
|
+
if logical_operator.nil?
|
43
|
+
s(:logical_group, *__one_or_more_child_nodes(&block))
|
44
|
+
else
|
45
|
+
s(:logical_group, s(logical_operator, *__one_or_more_child_nodes(&block)))
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
def statement_scope(&block)
|
50
|
+
s(:statement_scope, *__one_or_more_child_nodes(&block))
|
51
|
+
end
|
52
|
+
|
53
|
+
def tag(tag_class, &block)
|
54
|
+
s(:tag,
|
55
|
+
s(:tag_class,
|
56
|
+
s(:string, tag_class)),
|
57
|
+
statement_scope(&block))
|
58
|
+
end
|
59
|
+
|
60
|
+
def tag_tree(tag_nodes, tag_class, &block)
|
61
|
+
tag_node_nodes = tag_nodes.map do |tag_node|
|
62
|
+
s(:tag_node, tag_node)
|
63
|
+
end
|
64
|
+
|
65
|
+
s(:tag,
|
66
|
+
s(:tag_nodes,
|
67
|
+
*tag_node_nodes),
|
68
|
+
s(:tag_class,
|
69
|
+
s(:string, tag_class)),
|
70
|
+
statement_scope(&block))
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
data/lib/csdl/error.rb
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
module CSDL
|
2
|
+
class Error < ::StandardError; end
|
3
|
+
|
4
|
+
class InvalidInteractionTargetError < Error; end
|
5
|
+
class InvalidQueryTargetError < Error; end
|
6
|
+
class MissingChildNodesError < Error; end
|
7
|
+
class MissingReturnStatementScopeError < Error; end
|
8
|
+
class MissingTagClassError < Error; end
|
9
|
+
class MissingTagNodesError < Error; end
|
10
|
+
class MissingTagStatementScopeError < Error; end
|
11
|
+
class UnknownOperatorError < Error; end
|
12
|
+
class UnknownTargetError < Error; end
|
13
|
+
end
|
14
|
+
|
@@ -0,0 +1,65 @@
|
|
1
|
+
module CSDL
|
2
|
+
class InteractionFilterProcessor < ::CSDL::Processor
|
3
|
+
|
4
|
+
def on_return(node)
|
5
|
+
statement_scope = node.children.find { |child| child.type == :statement_scope }
|
6
|
+
|
7
|
+
if statement_scope.nil?
|
8
|
+
fail ::CSDL::MissingReturnStatementScopeError, "Invalid CSDL AST: return statment scope is missing"
|
9
|
+
end
|
10
|
+
|
11
|
+
"return #{process(statement_scope)}"
|
12
|
+
end
|
13
|
+
|
14
|
+
def on_statement_scope(node)
|
15
|
+
"{" + process_all(node.children).join(" ") + "}"
|
16
|
+
end
|
17
|
+
|
18
|
+
def on_tag(node)
|
19
|
+
tag_nodes = node.children.find { |child| child.type == :tag_nodes }
|
20
|
+
tag_class = node.children.find { |child| child.type == :tag_class }
|
21
|
+
statement_scope = node.children.find { |child| child.type == :statement_scope }
|
22
|
+
|
23
|
+
if tag_class.nil?
|
24
|
+
fail ::CSDL::MissingTagClassError, "Invalid CSDL AST: :tag node must have a :tag_class child node"
|
25
|
+
end
|
26
|
+
|
27
|
+
if statement_scope.nil?
|
28
|
+
fail ::CSDL::MissingTagStatementScopeError, "Invalid CSDL AST: :tag node must have a :statement_scope child node"
|
29
|
+
end
|
30
|
+
|
31
|
+
tag_namespace = "tag"
|
32
|
+
unless tag_nodes.nil?
|
33
|
+
tag_namespace += process(tag_nodes)
|
34
|
+
end
|
35
|
+
|
36
|
+
children = [tag_namespace] + process_all([ tag_class, statement_scope ])
|
37
|
+
children.join(" ")
|
38
|
+
end
|
39
|
+
|
40
|
+
def on_tag_class(node)
|
41
|
+
process(node.children.first)
|
42
|
+
end
|
43
|
+
|
44
|
+
def on_tag_node(node)
|
45
|
+
node.children.first.to_s
|
46
|
+
end
|
47
|
+
|
48
|
+
def on_tag_nodes(node)
|
49
|
+
child_tag_nodes = node.children.select { |child| child.type == :tag_node }
|
50
|
+
|
51
|
+
if child_tag_nodes.empty?
|
52
|
+
fail ::CSDL::MissingTagNodesError, "Invalid CSDL AST: A :tag_nodes node must have at least one :tag_node child"
|
53
|
+
end
|
54
|
+
|
55
|
+
"." + process_all(child_tag_nodes).join(".")
|
56
|
+
end
|
57
|
+
|
58
|
+
def validate_target!(target_key)
|
59
|
+
unless ::CSDL.interaction_target?(target_key)
|
60
|
+
fail ::CSDL::InvalidInteractionTargetError, "Interaction filters cannot use target '#{target_key}'"
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
end
|
65
|
+
end
|
@@ -0,0 +1,52 @@
|
|
1
|
+
module CSDL
|
2
|
+
|
3
|
+
Operator = Struct.new(:name, :argument_types)
|
4
|
+
|
5
|
+
raw_operators = [
|
6
|
+
[ "contains" , [ :string ] ],
|
7
|
+
[ "cs contains" , [ :string ] ],
|
8
|
+
[ "substr" , [ :string ] ],
|
9
|
+
[ "cs substr" , [ :string ] ],
|
10
|
+
[ "contains_any" , [ :string ] ],
|
11
|
+
[ "cs contains_any" , [ :string ] ],
|
12
|
+
[ "any" , [ :string ] ],
|
13
|
+
[ "cs any" , [ :string ] ],
|
14
|
+
[ "wildcard" , [ :string ] ],
|
15
|
+
[ "cs wildcard" , [ :string ] ],
|
16
|
+
[ "wild" , [ :string ] ],
|
17
|
+
[ "cs wild" , [ :string ] ],
|
18
|
+
[ "contains_all" , [ :string ] ],
|
19
|
+
[ "cs contains_all" , [ :string ] ],
|
20
|
+
[ "all" , [ :string ] ],
|
21
|
+
[ "cs all" , [ :string ] ],
|
22
|
+
[ "contains_near" , [ :string ] ],
|
23
|
+
[ "cs contains_near" , [ :string ] ],
|
24
|
+
[ "exists" , [ :string ] ],
|
25
|
+
[ "in" , [ :string ] ],
|
26
|
+
[ "url_in" , [ :string ] ],
|
27
|
+
[ "==" , [ :string ] ],
|
28
|
+
[ "!=" , [ :string ] ],
|
29
|
+
[ "cs ==" , [ :string ] ],
|
30
|
+
[ "cs !=" , [ :string ] ],
|
31
|
+
[ ">" , [ :string ] ],
|
32
|
+
[ ">=" , [ :string ] ],
|
33
|
+
[ "<" , [ :string ] ],
|
34
|
+
[ "<=" , [ :string ] ],
|
35
|
+
[ "regex_partial" , [ :string ] ],
|
36
|
+
[ "regex_exact" , [ :string ] ],
|
37
|
+
[ "geo_box" , [ :string ] ],
|
38
|
+
[ "geo_radius" , [ :string ] ],
|
39
|
+
[ "geo_polygon" , [ :string ] ]
|
40
|
+
]
|
41
|
+
|
42
|
+
OPERATORS = raw_operators.reduce({}) do |accumulator, (operator_name, argument_types)|
|
43
|
+
accumulator[operator_name] = Operator.new(operator_name, argument_types)
|
44
|
+
accumulator
|
45
|
+
end.freeze
|
46
|
+
|
47
|
+
def self.operator?(operator)
|
48
|
+
OPERATORS.key?(operator)
|
49
|
+
end
|
50
|
+
|
51
|
+
end
|
52
|
+
|
@@ -0,0 +1,78 @@
|
|
1
|
+
module CSDL
|
2
|
+
class Processor < ::AST::Processor
|
3
|
+
|
4
|
+
def on_and(node)
|
5
|
+
initial = process(node.children.first)
|
6
|
+
rest = node.children.drop(1)
|
7
|
+
|
8
|
+
rest.reduce(initial) do |csdl, child|
|
9
|
+
csdl += " AND " + process(child)
|
10
|
+
csdl
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
def on_argument(node)
|
15
|
+
process(node.children.first)
|
16
|
+
end
|
17
|
+
|
18
|
+
def on_logical_group(node)
|
19
|
+
"(" + process_all(node.children).join(" ") + ")"
|
20
|
+
end
|
21
|
+
|
22
|
+
def on_not(node)
|
23
|
+
"NOT " + process(node.updated(:filter))
|
24
|
+
end
|
25
|
+
|
26
|
+
def on_operator(node)
|
27
|
+
operator = node.children.first.to_s
|
28
|
+
unless ::CSDL.operator?(operator)
|
29
|
+
fail ::CSDL::UnknownOperatorError, "Operator #{operator} is unknown"
|
30
|
+
end
|
31
|
+
operator
|
32
|
+
end
|
33
|
+
|
34
|
+
def on_or(node)
|
35
|
+
if node.children.empty?
|
36
|
+
fail ::CSDL::MissingChildNodesError, "Invalid CSDL AST: 'or' nodes must contain at least two child nodes. Expected >= 2, got 0"
|
37
|
+
end
|
38
|
+
|
39
|
+
initial = process(node.children.first)
|
40
|
+
rest = node.children.drop(1)
|
41
|
+
|
42
|
+
if rest.empty?
|
43
|
+
fail ::CSDL::MissingChildNodesError, "Invalid CSDL AST: 'or' nodes must contain at least two child nodes. Expected >= 2, got #{node.children.size}"
|
44
|
+
end
|
45
|
+
|
46
|
+
rest.reduce(initial) do |csdl, child|
|
47
|
+
csdl += " OR " + process(child)
|
48
|
+
csdl
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
def on_string(node)
|
53
|
+
'"' + node.children.first.to_s.gsub(/"/, '\"') + '"'
|
54
|
+
end
|
55
|
+
|
56
|
+
def on_target(node)
|
57
|
+
target = node.children.first.to_s
|
58
|
+
validate_target!(target)
|
59
|
+
target
|
60
|
+
end
|
61
|
+
|
62
|
+
def on_filter(node)
|
63
|
+
target = node.children.find { |child| child.type == :target }
|
64
|
+
operator = node.children.find { |child| child.type == :operator }
|
65
|
+
argument = node.children.find { |child| child.type == :argument }
|
66
|
+
process_all([ target, operator, argument ].compact).join(" ")
|
67
|
+
end
|
68
|
+
|
69
|
+
def validate_target!(target_key)
|
70
|
+
unless ::CSDL.target?(target_key)
|
71
|
+
fail ::CSDL::InvalidQueryTargetError, "Query filters cannot use target '#{target_key}'"
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
|
@@ -0,0 +1,12 @@
|
|
1
|
+
module CSDL
|
2
|
+
class QueryFilterProcessor < ::CSDL::Processor
|
3
|
+
|
4
|
+
def validate_target!(target_key)
|
5
|
+
unless ::CSDL.query_target?(target_key)
|
6
|
+
fail ::CSDL::InvalidQueryTargetError, "Query filters cannot use target '#{target_key}'"
|
7
|
+
end
|
8
|
+
end
|
9
|
+
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
data/lib/csdl/targets.rb
ADDED
@@ -0,0 +1,96 @@
|
|
1
|
+
module CSDL
|
2
|
+
|
3
|
+
Target = Struct.new(:name, :interaction?, :analysis?, :query?)
|
4
|
+
|
5
|
+
raw_targets = [
|
6
|
+
|
7
|
+
[ "fb.author.age" , true , true , true ] ,
|
8
|
+
[ "fb.author.country" , true , true , true ] ,
|
9
|
+
[ "fb.author.country_code" , true , true , true ] ,
|
10
|
+
[ "fb.author.gender" , true , true , true ] ,
|
11
|
+
[ "fb.author.region" , true , true , true ] ,
|
12
|
+
[ "fb.author.type" , true , true , true ] ,
|
13
|
+
[ "fb.content" , true , false , true ] ,
|
14
|
+
[ "fb.hashtags" , true , true , true ] ,
|
15
|
+
[ "fb.language" , true , true , true ] ,
|
16
|
+
[ "fb.link" , true , true , true ] ,
|
17
|
+
[ "fb.media_type" , true , true , true ] ,
|
18
|
+
[ "fb.parent.author.age" , true , true , true ] ,
|
19
|
+
[ "fb.parent.author.country" , true , true , true ] ,
|
20
|
+
[ "fb.parent.author.country_code" , true , true , true ] ,
|
21
|
+
[ "fb.parent.author.gender" , true , true , true ] ,
|
22
|
+
[ "fb.parent.author.type" , true , true , true ] ,
|
23
|
+
[ "fb.parent.content" , true , false , true ] ,
|
24
|
+
[ "fb.parent.hashtags" , true , true , true ] ,
|
25
|
+
[ "fb.parent.interface" , true , true , true ] ,
|
26
|
+
[ "fb.parent.language" , true , true , true ] ,
|
27
|
+
[ "fb.parent.link" , true , true , true ] ,
|
28
|
+
[ "fb.parent.media_type" , true , true , true ] ,
|
29
|
+
[ "fb.parent.sentiment" , true , true , true ] ,
|
30
|
+
[ "fb.parent.topics.about" , true , false , false ] ,
|
31
|
+
[ "fb.parent.topics.category" , true , true , true ] ,
|
32
|
+
[ "fb.parent.topics.company_overview" , true , false , false ] ,
|
33
|
+
[ "fb.parent.topics.location_city" , true , false , false ] ,
|
34
|
+
[ "fb.parent.topics.location_street" , true , false , false ] ,
|
35
|
+
[ "fb.parent.topics.mission" , true , false , false ] ,
|
36
|
+
[ "fb.parent.topics.name" , true , true , true ] ,
|
37
|
+
[ "fb.parent.topics.products" , true , false , false ] ,
|
38
|
+
[ "fb.parent.topics.release_date" , true , false , false ] ,
|
39
|
+
[ "fb.parent.topics.username" , true , false , false ] ,
|
40
|
+
[ "fb.parent.topics.website" , true , false , false ] ,
|
41
|
+
[ "fb.parent.topic_ids" , true , true , true ] ,
|
42
|
+
[ "fb.sentiment" , true , true , true ] ,
|
43
|
+
[ "fb.topics.about" , true , false , false ] ,
|
44
|
+
[ "fb.topics.category" , true , true , true ] ,
|
45
|
+
[ "fb.topics.company_overview" , true , false , false ] ,
|
46
|
+
[ "fb.topics.location_city" , true , false , false ] ,
|
47
|
+
[ "fb.topics.location_street" , true , false , false ] ,
|
48
|
+
[ "fb.topics.mission" , true , false , false ] ,
|
49
|
+
[ "fb.topics.name" , true , true , true ] ,
|
50
|
+
[ "fb.topics.products" , true , false , false ] ,
|
51
|
+
[ "fb.topics.release_date" , true , false , false ] ,
|
52
|
+
[ "fb.topics.username" , true , false , false ] ,
|
53
|
+
[ "fb.topics.website" , true , false , false ] ,
|
54
|
+
[ "fb.topic_ids" , true , true , true ] ,
|
55
|
+
[ "fb.type" , true , true , true ] ,
|
56
|
+
[ "interaction.content" , true , false , true ] ,
|
57
|
+
[ "interaction.hashtags" , true , true , true ] ,
|
58
|
+
[ "interaction.media_type" , true , true , true ] ,
|
59
|
+
[ "interaction.ml.categories" , false , true , true ] ,
|
60
|
+
[ "interaction.raw_content" , true , false , true ] ,
|
61
|
+
[ "interaction.subtype" , true , true , true ] ,
|
62
|
+
[ "interaction.tag_tree" , false , true , false ] ,
|
63
|
+
[ "interaction.tags" , false , true , true ] ,
|
64
|
+
[ "links.code" , true , true , true ] ,
|
65
|
+
[ "links.domain" , true , true , true ] ,
|
66
|
+
[ "links.normalized_url" , true , true , true ] ,
|
67
|
+
[ "links.url" , true , true , true ]
|
68
|
+
]
|
69
|
+
|
70
|
+
TARGETS = raw_targets.reduce({}) do |accumulator, (target_name, interaction, analysis, query)|
|
71
|
+
accumulator[target_name] = Target.new(target_name, interaction, analysis, query)
|
72
|
+
accumulator
|
73
|
+
end.freeze
|
74
|
+
|
75
|
+
INTERACTION_TARGETS = TARGETS.select { |_, target| target.interaction? }
|
76
|
+
ANALYSIS_TARGETS = TARGETS.select { |_, target| target.analysis? }
|
77
|
+
QUERY_TARGETS = TARGETS.select { |_, target| target.query? }
|
78
|
+
|
79
|
+
def self.target?(target_name)
|
80
|
+
TARGETS.key?(target_name)
|
81
|
+
end
|
82
|
+
|
83
|
+
def self.interaction_target?(target_name)
|
84
|
+
INTERACTION_TARGETS.key?(target_name)
|
85
|
+
end
|
86
|
+
|
87
|
+
def self.analysis_target?(target_name)
|
88
|
+
ANALYSIS_TARGETS.key?(target_name)
|
89
|
+
end
|
90
|
+
|
91
|
+
def self.query_target?(target_name)
|
92
|
+
QUERY_TARGETS.key?(target_name)
|
93
|
+
end
|
94
|
+
|
95
|
+
end
|
96
|
+
|
data/lib/csdl/version.rb
ADDED
metadata
ADDED
@@ -0,0 +1,115 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: csdl
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- BJ Neilsen
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2015-06-16 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: ast
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '2.0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '2.0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: bundler
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '1.10'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '1.10'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rake
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '10.0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '10.0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: minitest
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
69
|
+
description: AST Processor and Query Builder for DataSift's CSDL language
|
70
|
+
email:
|
71
|
+
- bj.neilsen@gmail.com
|
72
|
+
executables: []
|
73
|
+
extensions: []
|
74
|
+
extra_rdoc_files: []
|
75
|
+
files:
|
76
|
+
- ".gitignore"
|
77
|
+
- ".ruby-version"
|
78
|
+
- ".travis.yml"
|
79
|
+
- Gemfile
|
80
|
+
- README.md
|
81
|
+
- Rakefile
|
82
|
+
- csdl.gemspec
|
83
|
+
- lib/csdl.rb
|
84
|
+
- lib/csdl/builder.rb
|
85
|
+
- lib/csdl/error.rb
|
86
|
+
- lib/csdl/interaction_filter_processor.rb
|
87
|
+
- lib/csdl/operators.rb
|
88
|
+
- lib/csdl/processor.rb
|
89
|
+
- lib/csdl/query_filter_processor.rb
|
90
|
+
- lib/csdl/targets.rb
|
91
|
+
- lib/csdl/version.rb
|
92
|
+
homepage: https://localshred.github.io
|
93
|
+
licenses: []
|
94
|
+
metadata: {}
|
95
|
+
post_install_message:
|
96
|
+
rdoc_options: []
|
97
|
+
require_paths:
|
98
|
+
- lib
|
99
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ">="
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
105
|
+
requirements:
|
106
|
+
- - ">="
|
107
|
+
- !ruby/object:Gem::Version
|
108
|
+
version: '0'
|
109
|
+
requirements: []
|
110
|
+
rubyforge_project:
|
111
|
+
rubygems_version: 2.4.5
|
112
|
+
signing_key:
|
113
|
+
specification_version: 4
|
114
|
+
summary: AST Processor and Query Builder for DataSift's CSDL language
|
115
|
+
test_files: []
|