embulk-input-yelp 0.2.2 → 0.2.3

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 783d1729496c8fcbff44ca42358f5692a30f7b29
4
- data.tar.gz: 36caf4c514c2ada8ddb51829743330a9d6064632
3
+ metadata.gz: da55630ab13c030342e8881b74c8458571491a5e
4
+ data.tar.gz: cf839bf3fd2e2b9d7bfaffaba165db64361f9cb2
5
5
  SHA512:
6
- metadata.gz: 4e8b86b970ab51d00e4883b0d29fe5642d726e1566a64cb944990f7d3f3cb29de76a3002083324812cc36a17ed20441bc8928a21ad4474d9fdf6c7057ea3ad2d
7
- data.tar.gz: 4f63f3a4ad043e27a5b0237c9d839ad8d5dc09dde4ee5bef4e0260b81eee4791b921db47dfffde87dd0435f459a2b611c27df733ca6ba713453c81b72ff4160f
6
+ metadata.gz: 0b083c91738b745bebe2fb4fca27fead52f05910c8fb74b751656136f0d79ef594643ce9afa8c6ba76173b63b0bf08800e3bf819ed761ea4530bf01c276a16aa
7
+ data.tar.gz: debcf29a45b83ae256c9b7048223476fe62ae215d23c51640597e1432fd6d1a8bf11027236c7be42738c27376c9100a7ded27f1514db9e0bc765eb21238a8342
data/build.gradle CHANGED
@@ -17,7 +17,7 @@ configurations {
17
17
  provided
18
18
  }
19
19
 
20
- version = "0.2.2"
20
+ version = "0.2.3"
21
21
 
22
22
  sourceCompatibility = 1.7
23
23
  targetCompatibility = 1.7
@@ -1,6 +1,5 @@
1
1
  package org.embulk.input.yelp;
2
2
 
3
- import java.util.HashMap;
4
3
  import java.util.List;
5
4
  import java.util.Locale;
6
5
  import java.util.Map;
@@ -10,6 +9,7 @@ import javax.ws.rs.client.WebTarget;
10
9
 
11
10
  import com.fasterxml.jackson.databind.JsonNode;
12
11
  import com.fasterxml.jackson.databind.node.ArrayNode;
12
+ import com.fasterxml.jackson.databind.node.JsonNodeFactory;
13
13
  import com.fasterxml.jackson.databind.node.ObjectNode;
14
14
 
15
15
  import com.google.common.base.Optional;
@@ -25,6 +25,7 @@ import org.embulk.spi.DataException;
25
25
  import org.embulk.spi.Exec;
26
26
  import org.embulk.spi.PageBuilder;
27
27
  import org.embulk.spi.Schema;
28
+ import org.embulk.spi.type.Type;
28
29
  import org.embulk.spi.type.Types;
29
30
 
30
31
  import org.embulk.base.restclient.DefaultServiceDataSplitter;
@@ -35,6 +36,7 @@ import org.embulk.base.restclient.jackson.JacksonJsonPointerValueLocator;
35
36
  import org.embulk.base.restclient.jackson.JacksonServiceRecord;
36
37
  import org.embulk.base.restclient.jackson.JacksonServiceResponseMapper;
37
38
  import org.embulk.base.restclient.jackson.JacksonTopLevelValueLocator;
39
+ import org.embulk.base.restclient.jackson.JacksonValueLocator;
38
40
  import org.embulk.base.restclient.jackson.StringJsonParser;
39
41
  import org.embulk.base.restclient.record.RecordImporter;
40
42
 
@@ -107,39 +109,27 @@ public class YelpInputPluginDelegate
107
109
  @Override // Overridden from |ServiceResponseMapperBuildable|
108
110
  public JacksonServiceResponseMapper buildServiceResponseMapper(PluginTask task)
109
111
  {
110
- HashMap<String, String> columnMap = new HashMap<String, String>();
111
- for (String defaultColumn : DEFAULT_COLUMNS) {
112
- if (task.getColumns().containsKey(defaultColumn)) {
113
- columnMap.put(defaultColumn, task.getColumns().get(defaultColumn));
114
- }
115
- else {
116
- columnMap.put(defaultColumn, defaultColumn);
117
- }
118
- }
119
- return JacksonServiceResponseMapper.builder()
120
- .add(new JacksonTopLevelValueLocator("rating"), columnMap.get("rating"), Types.LONG)
121
- .add(new JacksonTopLevelValueLocator("price"), columnMap.get("price"), Types.STRING)
122
- .add(new JacksonTopLevelValueLocator("phone"), columnMap.get("phone"), Types.STRING)
123
- .add(new JacksonTopLevelValueLocator("id"), columnMap.get("id"), Types.STRING)
124
- .add(new JacksonTopLevelValueLocator("is_closed"), columnMap.get("is_closed"), Types.BOOLEAN)
125
- .add(new JacksonTopLevelValueLocator("categories"), columnMap.get("categories"), Types.JSON)
126
- .add(new JacksonTopLevelValueLocator("review_count"), columnMap.get("review_count"), Types.LONG)
127
- .add(new JacksonTopLevelValueLocator("name"), columnMap.get("name"), Types.STRING)
128
- .add(new JacksonTopLevelValueLocator("url"), columnMap.get("url"), Types.STRING)
129
- .add(new JacksonJsonPointerValueLocator("/coordinates/latitude"),
130
- columnMap.get("latitude"),
131
- Types.STRING)
132
- .add(new JacksonJsonPointerValueLocator("/coordinates/longitude"),
133
- columnMap.get("longitude"),
134
- Types.STRING)
135
- .add(new JacksonTopLevelValueLocator("image_url"), columnMap.get("image_url"), Types.STRING)
136
- .add(new JacksonJsonPointerValueLocator("/location/city"),
137
- columnMap.get("location_city"),
138
- Types.STRING)
139
- .add(new JacksonJsonPointerValueLocator("/location/country"),
140
- columnMap.get("location_country"),
141
- Types.STRING)
142
- .build();
112
+ JacksonServiceResponseMapper.Builder builder = JacksonServiceResponseMapper.builder();
113
+ addColumn(builder, new JacksonYelpCategoriesValueLocator(), "categories",
114
+ Types.JSON, task.getColumns());
115
+ addColumn(builder, new JacksonJsonPointerValueLocator("/coordinates/latitude"), "latitude",
116
+ Types.STRING, task.getColumns());
117
+ addColumn(builder, new JacksonJsonPointerValueLocator("/coordinates/longitude"), "longitude",
118
+ Types.STRING, task.getColumns());
119
+ addColumn(builder, "id", Types.STRING, task.getColumns());
120
+ addColumn(builder, "image_url", Types.STRING, task.getColumns());
121
+ addColumn(builder, "is_closed", Types.BOOLEAN, task.getColumns());
122
+ addColumn(builder, new JacksonJsonPointerValueLocator("/location/city"), "city",
123
+ Types.STRING, task.getColumns());
124
+ addColumn(builder, new JacksonJsonPointerValueLocator("/location/country"), "country",
125
+ Types.STRING, task.getColumns());
126
+ addColumn(builder, "name", Types.STRING, task.getColumns());
127
+ addColumn(builder, "phone", Types.STRING, task.getColumns());
128
+ addColumn(builder, "price", Types.STRING, task.getColumns());
129
+ addColumn(builder, "rating", Types.STRING, task.getColumns());
130
+ addColumn(builder, "review_count", Types.LONG, task.getColumns());
131
+ addColumn(builder, "url", Types.STRING, task.getColumns());
132
+ return builder.build();
143
133
  }
144
134
 
145
135
  @Override // Overridden from |ConfigDiffBuildable|
@@ -208,6 +198,56 @@ public class YelpInputPluginDelegate
208
198
  return report;
209
199
  }
210
200
 
201
+ private static class JacksonYelpCategoriesValueLocator
202
+ extends JacksonValueLocator
203
+ {
204
+ public JacksonYelpCategoriesValueLocator(String name)
205
+ {
206
+ this.name = name;
207
+ }
208
+
209
+ public JacksonYelpCategoriesValueLocator()
210
+ {
211
+ this("categories");
212
+ }
213
+
214
+ @Override
215
+ public JsonNode seekValue(ObjectNode record)
216
+ {
217
+ JsonNode categoriesNode = record.get(this.name);
218
+ if (categoriesNode == null || categoriesNode.isNull()) {
219
+ return JsonNodeFactory.instance.arrayNode();
220
+ }
221
+ if (!categoriesNode.isArray()) {
222
+ throw new RuntimeException("categories must be an array");
223
+ }
224
+ ArrayNode categoriesArray = (ArrayNode) categoriesNode;
225
+
226
+ ArrayNode categoryAliases = JsonNodeFactory.instance.arrayNode();
227
+ for (JsonNode categoryNode : categoriesArray) {
228
+ if (!categoryNode.isObject()) {
229
+ throw new RuntimeException("a category must be an object");
230
+ }
231
+ ObjectNode categoryObject = (ObjectNode) categoryNode;
232
+ JsonNode aliasNode = categoryObject.get("alias");
233
+ if (aliasNode == null || !aliasNode.isTextual()) {
234
+ throw new RuntimeException("a category alias must be textual");
235
+ }
236
+ categoryAliases.add(aliasNode.asText());
237
+ }
238
+
239
+ return categoryAliases;
240
+ }
241
+
242
+ @Override
243
+ public void placeValue(ObjectNode record, JsonNode value)
244
+ {
245
+ throw new RuntimeException("Not implemented.");
246
+ }
247
+
248
+ private final String name;
249
+ }
250
+
211
251
  private ArrayNode extractArrayField(String content)
212
252
  {
213
253
  ObjectNode object = jsonParser.parseJsonObject(content);
@@ -270,22 +310,30 @@ public class YelpInputPluginDelegate
270
310
  });
271
311
  }
272
312
 
273
- private static final String[] DEFAULT_COLUMNS = {
274
- "rating",
275
- "price",
276
- "phone",
277
- "id",
278
- "is_closed",
279
- "categories",
280
- "review_count",
281
- "name",
282
- "url",
283
- "latitude",
284
- "longitude",
285
- "image_url",
286
- "location_city",
287
- "location_country"
288
- };
313
+ private void addColumn(JacksonServiceResponseMapper.Builder builder,
314
+ JacksonValueLocator locator,
315
+ String name,
316
+ Type type,
317
+ Map<String, String> alternatives)
318
+ {
319
+ if (alternatives.containsKey(name)) {
320
+ String alternativeName = alternatives.get(name);
321
+ if (alternativeName != null && !alternativeName.isEmpty()) {
322
+ builder.add(locator, alternativeName, type);
323
+ }
324
+ }
325
+ else {
326
+ builder.add(locator, name, type);
327
+ }
328
+ }
329
+
330
+ private void addColumn(JacksonServiceResponseMapper.Builder builder,
331
+ String name,
332
+ Type type,
333
+ Map<String, String> alternatives)
334
+ {
335
+ addColumn(builder, new JacksonTopLevelValueLocator(name), name, type, alternatives);
336
+ }
289
337
 
290
338
  private final Logger logger = Exec.getLogger(YelpInputPluginDelegate.class);
291
339
  }
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-input-yelp
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.2
4
+ version: 0.2.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Dai MIKURUBE
@@ -61,7 +61,7 @@ files:
61
61
  - src/test/java/org/embulk/input/yelp/TestYelpInputPlugin.java
62
62
  - classpath/aopalliance-repackaged-2.5.0-b32.jar
63
63
  - classpath/embulk-base-restclient-0.5.0.jar
64
- - classpath/embulk-input-yelp-0.2.2.jar
64
+ - classpath/embulk-input-yelp-0.2.3.jar
65
65
  - classpath/embulk-util-retryhelper-jaxrs-0.5.0.jar
66
66
  - classpath/hk2-api-2.5.0-b32.jar
67
67
  - classpath/hk2-locator-2.5.0-b32.jar