embulk-input-yelp 0.2.2 → 0.2.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/build.gradle +1 -1
- data/src/main/java/org/embulk/input/yelp/YelpInputPluginDelegate.java +98 -50
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: da55630ab13c030342e8881b74c8458571491a5e
|
4
|
+
data.tar.gz: cf839bf3fd2e2b9d7bfaffaba165db64361f9cb2
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0b083c91738b745bebe2fb4fca27fead52f05910c8fb74b751656136f0d79ef594643ce9afa8c6ba76173b63b0bf08800e3bf819ed761ea4530bf01c276a16aa
|
7
|
+
data.tar.gz: debcf29a45b83ae256c9b7048223476fe62ae215d23c51640597e1432fd6d1a8bf11027236c7be42738c27376c9100a7ded27f1514db9e0bc765eb21238a8342
|
data/build.gradle
CHANGED
@@ -1,6 +1,5 @@
|
|
1
1
|
package org.embulk.input.yelp;
|
2
2
|
|
3
|
-
import java.util.HashMap;
|
4
3
|
import java.util.List;
|
5
4
|
import java.util.Locale;
|
6
5
|
import java.util.Map;
|
@@ -10,6 +9,7 @@ import javax.ws.rs.client.WebTarget;
|
|
10
9
|
|
11
10
|
import com.fasterxml.jackson.databind.JsonNode;
|
12
11
|
import com.fasterxml.jackson.databind.node.ArrayNode;
|
12
|
+
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
|
13
13
|
import com.fasterxml.jackson.databind.node.ObjectNode;
|
14
14
|
|
15
15
|
import com.google.common.base.Optional;
|
@@ -25,6 +25,7 @@ import org.embulk.spi.DataException;
|
|
25
25
|
import org.embulk.spi.Exec;
|
26
26
|
import org.embulk.spi.PageBuilder;
|
27
27
|
import org.embulk.spi.Schema;
|
28
|
+
import org.embulk.spi.type.Type;
|
28
29
|
import org.embulk.spi.type.Types;
|
29
30
|
|
30
31
|
import org.embulk.base.restclient.DefaultServiceDataSplitter;
|
@@ -35,6 +36,7 @@ import org.embulk.base.restclient.jackson.JacksonJsonPointerValueLocator;
|
|
35
36
|
import org.embulk.base.restclient.jackson.JacksonServiceRecord;
|
36
37
|
import org.embulk.base.restclient.jackson.JacksonServiceResponseMapper;
|
37
38
|
import org.embulk.base.restclient.jackson.JacksonTopLevelValueLocator;
|
39
|
+
import org.embulk.base.restclient.jackson.JacksonValueLocator;
|
38
40
|
import org.embulk.base.restclient.jackson.StringJsonParser;
|
39
41
|
import org.embulk.base.restclient.record.RecordImporter;
|
40
42
|
|
@@ -107,39 +109,27 @@ public class YelpInputPluginDelegate
|
|
107
109
|
@Override // Overridden from |ServiceResponseMapperBuildable|
|
108
110
|
public JacksonServiceResponseMapper buildServiceResponseMapper(PluginTask task)
|
109
111
|
{
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
Types.STRING)
|
132
|
-
.add(new JacksonJsonPointerValueLocator("/coordinates/longitude"),
|
133
|
-
columnMap.get("longitude"),
|
134
|
-
Types.STRING)
|
135
|
-
.add(new JacksonTopLevelValueLocator("image_url"), columnMap.get("image_url"), Types.STRING)
|
136
|
-
.add(new JacksonJsonPointerValueLocator("/location/city"),
|
137
|
-
columnMap.get("location_city"),
|
138
|
-
Types.STRING)
|
139
|
-
.add(new JacksonJsonPointerValueLocator("/location/country"),
|
140
|
-
columnMap.get("location_country"),
|
141
|
-
Types.STRING)
|
142
|
-
.build();
|
112
|
+
JacksonServiceResponseMapper.Builder builder = JacksonServiceResponseMapper.builder();
|
113
|
+
addColumn(builder, new JacksonYelpCategoriesValueLocator(), "categories",
|
114
|
+
Types.JSON, task.getColumns());
|
115
|
+
addColumn(builder, new JacksonJsonPointerValueLocator("/coordinates/latitude"), "latitude",
|
116
|
+
Types.STRING, task.getColumns());
|
117
|
+
addColumn(builder, new JacksonJsonPointerValueLocator("/coordinates/longitude"), "longitude",
|
118
|
+
Types.STRING, task.getColumns());
|
119
|
+
addColumn(builder, "id", Types.STRING, task.getColumns());
|
120
|
+
addColumn(builder, "image_url", Types.STRING, task.getColumns());
|
121
|
+
addColumn(builder, "is_closed", Types.BOOLEAN, task.getColumns());
|
122
|
+
addColumn(builder, new JacksonJsonPointerValueLocator("/location/city"), "city",
|
123
|
+
Types.STRING, task.getColumns());
|
124
|
+
addColumn(builder, new JacksonJsonPointerValueLocator("/location/country"), "country",
|
125
|
+
Types.STRING, task.getColumns());
|
126
|
+
addColumn(builder, "name", Types.STRING, task.getColumns());
|
127
|
+
addColumn(builder, "phone", Types.STRING, task.getColumns());
|
128
|
+
addColumn(builder, "price", Types.STRING, task.getColumns());
|
129
|
+
addColumn(builder, "rating", Types.STRING, task.getColumns());
|
130
|
+
addColumn(builder, "review_count", Types.LONG, task.getColumns());
|
131
|
+
addColumn(builder, "url", Types.STRING, task.getColumns());
|
132
|
+
return builder.build();
|
143
133
|
}
|
144
134
|
|
145
135
|
@Override // Overridden from |ConfigDiffBuildable|
|
@@ -208,6 +198,56 @@ public class YelpInputPluginDelegate
|
|
208
198
|
return report;
|
209
199
|
}
|
210
200
|
|
201
|
+
private static class JacksonYelpCategoriesValueLocator
|
202
|
+
extends JacksonValueLocator
|
203
|
+
{
|
204
|
+
public JacksonYelpCategoriesValueLocator(String name)
|
205
|
+
{
|
206
|
+
this.name = name;
|
207
|
+
}
|
208
|
+
|
209
|
+
public JacksonYelpCategoriesValueLocator()
|
210
|
+
{
|
211
|
+
this("categories");
|
212
|
+
}
|
213
|
+
|
214
|
+
@Override
|
215
|
+
public JsonNode seekValue(ObjectNode record)
|
216
|
+
{
|
217
|
+
JsonNode categoriesNode = record.get(this.name);
|
218
|
+
if (categoriesNode == null || categoriesNode.isNull()) {
|
219
|
+
return JsonNodeFactory.instance.arrayNode();
|
220
|
+
}
|
221
|
+
if (!categoriesNode.isArray()) {
|
222
|
+
throw new RuntimeException("categories must be an array");
|
223
|
+
}
|
224
|
+
ArrayNode categoriesArray = (ArrayNode) categoriesNode;
|
225
|
+
|
226
|
+
ArrayNode categoryAliases = JsonNodeFactory.instance.arrayNode();
|
227
|
+
for (JsonNode categoryNode : categoriesArray) {
|
228
|
+
if (!categoryNode.isObject()) {
|
229
|
+
throw new RuntimeException("a category must be an object");
|
230
|
+
}
|
231
|
+
ObjectNode categoryObject = (ObjectNode) categoryNode;
|
232
|
+
JsonNode aliasNode = categoryObject.get("alias");
|
233
|
+
if (aliasNode == null || !aliasNode.isTextual()) {
|
234
|
+
throw new RuntimeException("a category alias must be textual");
|
235
|
+
}
|
236
|
+
categoryAliases.add(aliasNode.asText());
|
237
|
+
}
|
238
|
+
|
239
|
+
return categoryAliases;
|
240
|
+
}
|
241
|
+
|
242
|
+
@Override
|
243
|
+
public void placeValue(ObjectNode record, JsonNode value)
|
244
|
+
{
|
245
|
+
throw new RuntimeException("Not implemented.");
|
246
|
+
}
|
247
|
+
|
248
|
+
private final String name;
|
249
|
+
}
|
250
|
+
|
211
251
|
private ArrayNode extractArrayField(String content)
|
212
252
|
{
|
213
253
|
ObjectNode object = jsonParser.parseJsonObject(content);
|
@@ -270,22 +310,30 @@ public class YelpInputPluginDelegate
|
|
270
310
|
});
|
271
311
|
}
|
272
312
|
|
273
|
-
private
|
274
|
-
|
275
|
-
|
276
|
-
|
277
|
-
|
278
|
-
|
279
|
-
|
280
|
-
|
281
|
-
|
282
|
-
|
283
|
-
|
284
|
-
|
285
|
-
|
286
|
-
|
287
|
-
|
288
|
-
}
|
313
|
+
private void addColumn(JacksonServiceResponseMapper.Builder builder,
|
314
|
+
JacksonValueLocator locator,
|
315
|
+
String name,
|
316
|
+
Type type,
|
317
|
+
Map<String, String> alternatives)
|
318
|
+
{
|
319
|
+
if (alternatives.containsKey(name)) {
|
320
|
+
String alternativeName = alternatives.get(name);
|
321
|
+
if (alternativeName != null && !alternativeName.isEmpty()) {
|
322
|
+
builder.add(locator, alternativeName, type);
|
323
|
+
}
|
324
|
+
}
|
325
|
+
else {
|
326
|
+
builder.add(locator, name, type);
|
327
|
+
}
|
328
|
+
}
|
329
|
+
|
330
|
+
private void addColumn(JacksonServiceResponseMapper.Builder builder,
|
331
|
+
String name,
|
332
|
+
Type type,
|
333
|
+
Map<String, String> alternatives)
|
334
|
+
{
|
335
|
+
addColumn(builder, new JacksonTopLevelValueLocator(name), name, type, alternatives);
|
336
|
+
}
|
289
337
|
|
290
338
|
private final Logger logger = Exec.getLogger(YelpInputPluginDelegate.class);
|
291
339
|
}
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-input-yelp
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Dai MIKURUBE
|
@@ -61,7 +61,7 @@ files:
|
|
61
61
|
- src/test/java/org/embulk/input/yelp/TestYelpInputPlugin.java
|
62
62
|
- classpath/aopalliance-repackaged-2.5.0-b32.jar
|
63
63
|
- classpath/embulk-base-restclient-0.5.0.jar
|
64
|
-
- classpath/embulk-input-yelp-0.2.
|
64
|
+
- classpath/embulk-input-yelp-0.2.3.jar
|
65
65
|
- classpath/embulk-util-retryhelper-jaxrs-0.5.0.jar
|
66
66
|
- classpath/hk2-api-2.5.0-b32.jar
|
67
67
|
- classpath/hk2-locator-2.5.0-b32.jar
|