embulk 0.7.11-java → 0.8.0-java

Sign up to get free protection for your applications and to get access to all the features.
Files changed (79) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile.lock +3 -3
  3. data/README.md +1 -1
  4. data/build.gradle +2 -2
  5. data/embulk-core/build.gradle +2 -0
  6. data/embulk-core/src/main/java/org/embulk/config/ConfigLoader.java +11 -3
  7. data/embulk-core/src/main/java/org/embulk/config/YamlTagResolver.java +53 -0
  8. data/embulk-core/src/main/java/org/embulk/exec/ExecModule.java +0 -1
  9. data/embulk-core/src/main/java/org/embulk/exec/LocalExecutorPlugin.java +479 -69
  10. data/embulk-core/src/main/java/org/embulk/spi/Column.java +3 -0
  11. data/embulk-core/src/main/java/org/embulk/spi/ColumnVisitor.java +2 -0
  12. data/embulk-core/src/main/java/org/embulk/spi/ExecSession.java +12 -5
  13. data/embulk-core/src/main/java/org/embulk/spi/Page.java +19 -0
  14. data/embulk-core/src/main/java/org/embulk/spi/PageBuilder.java +26 -5
  15. data/embulk-core/src/main/java/org/embulk/spi/PageReader.java +13 -0
  16. data/embulk-core/src/main/java/org/embulk/spi/json/JsonParseException.java +17 -0
  17. data/embulk-core/src/main/java/org/embulk/spi/json/JsonParser.java +125 -0
  18. data/embulk-core/src/main/java/org/embulk/spi/json/RubyValueApi.java +55 -0
  19. data/embulk-core/src/main/java/org/embulk/spi/type/JsonType.java +14 -0
  20. data/embulk-core/src/main/java/org/embulk/spi/type/TypeDeserializer.java +1 -0
  21. data/embulk-core/src/main/java/org/embulk/spi/type/Types.java +2 -0
  22. data/embulk-core/src/main/java/org/embulk/spi/util/DynamicColumnSetterFactory.java +6 -0
  23. data/embulk-core/src/main/java/org/embulk/spi/util/PagePrinter.java +5 -0
  24. data/embulk-core/src/main/java/org/embulk/spi/util/Pages.java +10 -0
  25. data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/AbstractDynamicColumnSetter.java +3 -0
  26. data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/BooleanColumnSetter.java +7 -0
  27. data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/DefaultValueSetter.java +2 -0
  28. data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/DoubleColumnSetter.java +7 -0
  29. data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/JsonColumnSetter.java +73 -0
  30. data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/LongColumnSetter.java +11 -2
  31. data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/NullDefaultValueSetter.java +5 -0
  32. data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/SkipColumnSetter.java +5 -0
  33. data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/StringColumnSetter.java +7 -0
  34. data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/TimestampColumnSetter.java +9 -1
  35. data/embulk-core/src/test/java/org/embulk/spi/MockFormatterPlugin.java +7 -0
  36. data/embulk-docs/src/built-in.rst +40 -3
  37. data/embulk-docs/src/conf.py +2 -2
  38. data/embulk-docs/src/release.rst +1 -1
  39. data/embulk-docs/src/release/release-0.8.0.rst +68 -0
  40. data/embulk-standards/src/main/java/org/embulk/standards/CsvFormatterPlugin.java +12 -1
  41. data/embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java +18 -0
  42. data/embulk-standards/src/main/java/org/embulk/standards/CsvTokenizer.java +1 -1
  43. data/embulk.gemspec +1 -1
  44. data/gradle/wrapper/gradle-wrapper.jar +0 -0
  45. data/gradle/wrapper/gradle-wrapper.properties +2 -2
  46. data/gradlew +3 -7
  47. data/lib/embulk/column.rb +2 -0
  48. data/lib/embulk/command/embulk_migrate_plugin.rb +76 -10
  49. data/lib/embulk/command/embulk_new_plugin.rb +2 -0
  50. data/lib/embulk/command/embulk_run.rb +17 -10
  51. data/lib/embulk/data/bundle/.ruby-version +1 -1
  52. data/lib/embulk/data/new/java/build.gradle.erb +21 -0
  53. data/lib/embulk/data/new/java/config/checkstyle/checkstyle.xml +128 -0
  54. data/lib/embulk/data/new/java/config/checkstyle/default.xml +108 -0
  55. data/lib/embulk/data/new/java/gradle/wrapper/gradle-wrapper.jar +0 -0
  56. data/lib/embulk/data/new/java/gradle/wrapper/gradle-wrapper.properties +2 -2
  57. data/lib/embulk/data/new/java/gradlew +3 -7
  58. data/lib/embulk/data/new/ruby/.ruby-version +1 -1
  59. data/lib/embulk/guess/csv.rb +1 -1
  60. data/lib/embulk/guess/schema_guess.rb +6 -0
  61. data/lib/embulk/guess_plugin.rb +1 -1
  62. data/lib/embulk/java/imports.rb +4 -0
  63. data/lib/embulk/plugin_registry.rb +8 -12
  64. data/lib/embulk/schema.rb +6 -0
  65. data/lib/embulk/version.rb +1 -1
  66. data/test/guess/test_csv_guess.rb +170 -0
  67. data/test/helper.rb +2 -0
  68. metadata +17 -15
  69. data/embulk-core/src/main/java/org/embulk/exec/LocalThreadExecutor.java +0 -34
  70. data/embulk-core/src/main/java/org/embulk/guice/Bootstrap.java +0 -157
  71. data/embulk-core/src/main/java/org/embulk/guice/CloseableInjector.java +0 -22
  72. data/embulk-core/src/main/java/org/embulk/guice/InjectorProxy.java +0 -145
  73. data/embulk-core/src/main/java/org/embulk/guice/LifeCycleInjector.java +0 -26
  74. data/embulk-core/src/main/java/org/embulk/guice/LifeCycleInjectorProxy.java +0 -61
  75. data/embulk-core/src/main/java/org/embulk/guice/LifeCycleManager.java +0 -187
  76. data/embulk-core/src/main/java/org/embulk/guice/LifeCycleMethods.java +0 -89
  77. data/embulk-core/src/main/java/org/embulk/guice/LifeCycleMethodsMap.java +0 -38
  78. data/embulk-core/src/main/java/org/embulk/guice/LifeCycleModule.java +0 -97
  79. data/embulk-docs/src/release/release-0.7.11.rst +0 -13
@@ -1,26 +0,0 @@
1
- /*
2
- * Copyright 2015 Sadayuki Furuhashi
3
- *
4
- * Licensed under the Apache License, Version 2.0 (the "License");
5
- * you may not use this file except in compliance with the License.
6
- * You may obtain a copy of the License at
7
- *
8
- * http://www.apache.org/licenses/LICENSE-2.0
9
- *
10
- * Unless required by applicable law or agreed to in writing, software
11
- * distributed under the License is distributed on an "AS IS" BASIS,
12
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
- * See the License for the specific language governing permissions and
14
- * limitations under the License.
15
- */
16
- package org.embulk.guice;
17
-
18
- import com.google.inject.Injector;
19
-
20
- public interface LifeCycleInjector
21
- extends Injector
22
- {
23
- boolean isDestroyed();
24
-
25
- void destroy() throws Exception;
26
- }
@@ -1,61 +0,0 @@
1
- /*
2
- * Copyright 2015 Sadayuki Furuhashi
3
- *
4
- * Licensed under the Apache License, Version 2.0 (the "License");
5
- * you may not use this file except in compliance with the License.
6
- * You may obtain a copy of the License at
7
- *
8
- * http://www.apache.org/licenses/LICENSE-2.0
9
- *
10
- * Unless required by applicable law or agreed to in writing, software
11
- * distributed under the License is distributed on an "AS IS" BASIS,
12
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
- * See the License for the specific language governing permissions and
14
- * limitations under the License.
15
- */
16
- package org.embulk.guice;
17
-
18
- import com.google.inject.Injector;
19
-
20
- class LifeCycleInjectorProxy
21
- extends InjectorProxy
22
- implements LifeCycleInjector, CloseableInjector
23
- {
24
- private final Injector injector;
25
- private final LifeCycleManager lifeCycleManager;
26
-
27
- public LifeCycleInjectorProxy(Injector injector, LifeCycleManager lifeCycleManager)
28
- {
29
- this.injector = injector;
30
- this.lifeCycleManager = lifeCycleManager;
31
- }
32
-
33
- @Override
34
- protected synchronized Injector injector()
35
- {
36
- if (isDestroyed()) {
37
- throw new IllegalStateException("Injector already destroyed");
38
- }
39
- return injector;
40
- }
41
-
42
- @Override
43
- public synchronized boolean isDestroyed()
44
- {
45
- return lifeCycleManager.isDestroyed();
46
- }
47
-
48
- @Override
49
- public synchronized void destroy()
50
- throws Exception
51
- {
52
- lifeCycleManager.destroy(); // reentrant
53
- }
54
-
55
- @Override
56
- public void close()
57
- throws Exception
58
- {
59
- destroy();
60
- }
61
- }
@@ -1,187 +0,0 @@
1
- /*
2
- * Copyright 2010 Proofpoint, Inc.
3
- *
4
- * Licensed under the Apache License, Version 2.0 (the "License");
5
- * you may not use this file except in compliance with the License.
6
- * You may obtain a copy of the License at
7
- *
8
- * http://www.apache.org/licenses/LICENSE-2.0
9
- *
10
- * Unless required by applicable law or agreed to in writing, software
11
- * distributed under the License is distributed on an "AS IS" BASIS,
12
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
- * See the License for the specific language governing permissions and
14
- * limitations under the License.
15
- */
16
- /*
17
- * Copyright 2015 Sadayuki Furuhashi
18
- */
19
- package org.embulk.guice;
20
-
21
- import com.google.common.collect.Lists;
22
-
23
- import javax.annotation.PostConstruct;
24
- import javax.annotation.PreDestroy;
25
- import java.lang.reflect.InvocationTargetException;
26
- import java.lang.reflect.Method;
27
- import java.util.Collections;
28
- import java.util.List;
29
- import java.util.Queue;
30
- import java.util.concurrent.ConcurrentLinkedQueue;
31
- import java.util.concurrent.atomic.AtomicReference;
32
-
33
- /**
34
- * Manages PostConstruct and PreDestroy life cycles
35
- */
36
- public final class LifeCycleManager
37
- {
38
- private final AtomicReference<State> state = new AtomicReference<State>(State.LATENT);
39
- private final Queue<Object> managedInstances = new ConcurrentLinkedQueue<Object>();
40
- private final LifeCycleMethodsMap methodsMap;
41
-
42
- private enum State
43
- {
44
- LATENT,
45
- STARTING,
46
- STARTED,
47
- STOPPING,
48
- STOPPED
49
- }
50
-
51
- /**
52
- * @param managedInstances list of objects that have life cycle annotations
53
- * @param methodsMap existing or new methods map
54
- * @throws Exception exceptions starting instances (depending on mode)
55
- */
56
- public LifeCycleManager(List<Object> managedInstances, LifeCycleMethodsMap methodsMap)
57
- throws Exception
58
- {
59
- this.methodsMap = (methodsMap != null) ? methodsMap : new LifeCycleMethodsMap();
60
- for (Object instance : managedInstances) {
61
- addInstance(instance);
62
- }
63
- }
64
-
65
- /**
66
- * Returns the number of managed instances
67
- *
68
- * @return qty
69
- */
70
- public int size()
71
- {
72
- return managedInstances.size();
73
- }
74
-
75
- /**
76
- * Start the life cycle - all instances will have their {@link javax.annotation.PostConstruct} method(s) called
77
- */
78
- public void start()
79
- {
80
- if (!state.compareAndSet(State.LATENT, State.STARTING)) {
81
- throw new IllegalStateException("System already starting");
82
- }
83
- //log.info("Life cycle starting...");
84
-
85
- for (Object obj : managedInstances) {
86
- LifeCycleMethods methods = methodsMap.get(obj.getClass());
87
- if (!methods.hasFor(PreDestroy.class)) {
88
- managedInstances.remove(obj); // remove reference to instances that aren't needed anymore
89
- }
90
- }
91
-
92
- state.set(State.STARTED);
93
- //log.info("Life cycle startup complete. System ready.");
94
- }
95
-
96
- /**
97
- * Add a shutdown hook that calls {@link destroy} method
98
- */
99
- public void destroyOnShutdownHook()
100
- {
101
- Runtime.getRuntime().addShutdownHook(new Thread()
102
- {
103
- @Override
104
- public void run()
105
- {
106
- try {
107
- LifeCycleManager.this.destroy();
108
- }
109
- catch (Exception e) {
110
- //log.error(e, "Trying to shut down");
111
- }
112
- }
113
- });
114
- }
115
-
116
- /**
117
- * Stop the life cycle - all instances will have their {@link javax.annotation.PreDestroy} method(s) called
118
- *
119
- * @throws Exception errors
120
- */
121
- public void destroy()
122
- throws Exception
123
- {
124
- if (!state.compareAndSet(State.STARTED, State.STOPPING)) {
125
- return;
126
- }
127
-
128
- //log.info("Life cycle stopping...");
129
-
130
- List<Object> reversedInstances = Lists.newArrayList(managedInstances);
131
- Collections.reverse(reversedInstances);
132
-
133
- for (Object obj : reversedInstances) {
134
- //log.debug("Stopping %s", obj.getClass().getName());
135
- LifeCycleMethods methods = methodsMap.get(obj.getClass());
136
- for (Method preDestroy : methods.methodsFor(PreDestroy.class)) {
137
- //log.debug("\t%s()", preDestroy.getName());
138
- preDestroy.invoke(obj);
139
- }
140
- }
141
-
142
- state.set(State.STOPPED);
143
- //log.info("Life cycle stopped.");
144
- }
145
-
146
- /**
147
- * Return true if {@link destroy} is called
148
- *
149
- * @return true if already destroyed
150
- */
151
- public boolean isDestroyed()
152
- {
153
- State currentState = state.get();
154
- return currentState == State.STOPPING || currentState == State.STOPPED;
155
- }
156
-
157
- /**
158
- * Add an additional managed instance
159
- *
160
- * @param instance instance to add
161
- * @throws Exception errors
162
- */
163
- public void addInstance(Object instance)
164
- throws Exception
165
- {
166
- if (isDestroyed()) {
167
- throw new IllegalStateException("System already stopped");
168
- }
169
- else {
170
- startInstance(instance);
171
- if (methodsMap.get(instance.getClass()).hasFor(PreDestroy.class)) {
172
- managedInstances.add(instance);
173
- }
174
- }
175
- }
176
-
177
- private void startInstance(Object obj)
178
- throws IllegalAccessException, InvocationTargetException
179
- {
180
- //log.debug("Starting %s", obj.getClass().getName());
181
- LifeCycleMethods methods = methodsMap.get(obj.getClass());
182
- for (Method postConstruct : methods.methodsFor(PostConstruct.class)) {
183
- //log.debug("\t%s()", postConstruct.getName());
184
- postConstruct.invoke(obj);
185
- }
186
- }
187
- }
@@ -1,89 +0,0 @@
1
- /*
2
- * Copyright 2010 Proofpoint, Inc.
3
- *
4
- * Licensed under the Apache License, Version 2.0 (the "License");
5
- * you may not use this file except in compliance with the License.
6
- * You may obtain a copy of the License at
7
- *
8
- * http://www.apache.org/licenses/LICENSE-2.0
9
- *
10
- * Unless required by applicable law or agreed to in writing, software
11
- * distributed under the License is distributed on an "AS IS" BASIS,
12
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
- * See the License for the specific language governing permissions and
14
- * limitations under the License.
15
- */
16
- /*
17
- * Copyright 2015 Sadayuki Furuhashi
18
- */
19
- package org.embulk.guice;
20
-
21
- import com.google.common.collect.ArrayListMultimap;
22
- import com.google.common.collect.Lists;
23
- import com.google.common.collect.Multimap;
24
-
25
- import javax.annotation.PostConstruct;
26
- import javax.annotation.PreDestroy;
27
- import java.lang.annotation.Annotation;
28
- import java.lang.reflect.Method;
29
- import java.util.Collection;
30
- import java.util.HashSet;
31
- import java.util.Set;
32
-
33
- class LifeCycleMethods
34
- {
35
- private final Multimap<Class<? extends Annotation>, Method> methodMap = ArrayListMultimap.create();
36
-
37
- LifeCycleMethods(Class<?> clazz)
38
- {
39
- addLifeCycleMethods(clazz, new HashSet<String>(), new HashSet<String>());
40
- }
41
-
42
- boolean hasFor(Class<? extends Annotation> annotation)
43
- {
44
- Collection<Method> methods = methodMap.get(annotation);
45
- return (methods != null) && (methods.size() > 0);
46
- }
47
-
48
- Collection<Method> methodsFor(Class<? extends Annotation> annotation)
49
- {
50
- Collection<Method> methods = methodMap.get(annotation);
51
- return (methods != null) ? methods : Lists.<Method>newArrayList();
52
- }
53
-
54
- private void addLifeCycleMethods(Class<?> clazz, Set<String> usedConstructNames, Set<String> usedDestroyNames)
55
- {
56
- if (clazz == null) {
57
- return;
58
- }
59
-
60
- for (Method method : clazz.getDeclaredMethods()) {
61
- if (method.isSynthetic() || method.isBridge()) {
62
- continue;
63
- }
64
-
65
- processMethod(method, PostConstruct.class, usedConstructNames);
66
- processMethod(method, PreDestroy.class, usedDestroyNames);
67
- }
68
-
69
- addLifeCycleMethods(clazz.getSuperclass(), usedConstructNames, usedDestroyNames);
70
- for (Class<?> face : clazz.getInterfaces()) {
71
- addLifeCycleMethods(face, usedConstructNames, usedDestroyNames);
72
- }
73
- }
74
-
75
- private void processMethod(Method method, Class<? extends Annotation> annotationClass, Set<String> usedSet)
76
- {
77
- if (method.isAnnotationPresent(annotationClass)) {
78
- if (!usedSet.contains(method.getName())) {
79
- if (method.getParameterTypes().length != 0) {
80
- throw new UnsupportedOperationException(String.format("@PostConstruct/@PreDestroy methods cannot have arguments: %s", method.getDeclaringClass().getName() + "." + method.getName() + "(...)"));
81
- }
82
-
83
- method.setAccessible(true);
84
- usedSet.add(method.getName());
85
- methodMap.put(annotationClass, method);
86
- }
87
- }
88
- }
89
- }
@@ -1,38 +0,0 @@
1
- /*
2
- * Copyright 2010 Proofpoint, Inc.
3
- *
4
- * Licensed under the Apache License, Version 2.0 (the "License");
5
- * you may not use this file except in compliance with the License.
6
- * You may obtain a copy of the License at
7
- *
8
- * http://www.apache.org/licenses/LICENSE-2.0
9
- *
10
- * Unless required by applicable law or agreed to in writing, software
11
- * distributed under the License is distributed on an "AS IS" BASIS,
12
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
- * See the License for the specific language governing permissions and
14
- * limitations under the License.
15
- */
16
- /*
17
- * Copyright 2015 Sadayuki Furuhashi
18
- */
19
- package org.embulk.guice;
20
-
21
- import com.google.common.collect.Maps;
22
-
23
- import java.util.Map;
24
-
25
- class LifeCycleMethodsMap
26
- {
27
- private final Map<Class<?>, LifeCycleMethods> map = Maps.newHashMap();
28
-
29
- synchronized LifeCycleMethods get(Class<?> clazz)
30
- {
31
- LifeCycleMethods methods = map.get(clazz);
32
- if (methods == null) {
33
- methods = new LifeCycleMethods(clazz);
34
- map.put(clazz, methods);
35
- }
36
- return methods;
37
- }
38
- }
@@ -1,97 +0,0 @@
1
- /*
2
- * Copyright 2010 Proofpoint, Inc.
3
- *
4
- * Licensed under the Apache License, Version 2.0 (the "License");
5
- * you may not use this file except in compliance with the License.
6
- * You may obtain a copy of the License at
7
- *
8
- * http://www.apache.org/licenses/LICENSE-2.0
9
- *
10
- * Unless required by applicable law or agreed to in writing, software
11
- * distributed under the License is distributed on an "AS IS" BASIS,
12
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
- * See the License for the specific language governing permissions and
14
- * limitations under the License.
15
- */
16
- /*
17
- * Copyright 2015 Sadayuki Furuhashi
18
- */
19
- package org.embulk.guice;
20
-
21
- import com.google.common.collect.Lists;
22
- import com.google.inject.Binder;
23
- import com.google.inject.Module;
24
- import com.google.inject.Provides;
25
- import com.google.inject.Singleton;
26
- import com.google.inject.TypeLiteral;
27
- import com.google.inject.spi.InjectionListener;
28
- import com.google.inject.spi.TypeEncounter;
29
- import com.google.inject.spi.TypeListener;
30
-
31
- import javax.annotation.PostConstruct;
32
- import javax.annotation.PreDestroy;
33
- import java.util.List;
34
- import java.util.concurrent.atomic.AtomicReference;
35
-
36
- import static com.google.inject.matcher.Matchers.any;
37
-
38
- /**
39
- * Guice module for binding the LifeCycle manager
40
- */
41
- public class LifeCycleModule implements Module
42
- {
43
- private final List<Object> injectedInstances = Lists.newArrayList();
44
- private final LifeCycleMethodsMap lifeCycleMethodsMap = new LifeCycleMethodsMap();
45
- private final AtomicReference<LifeCycleManager> lifeCycleManagerRef = new AtomicReference<LifeCycleManager>(null);
46
-
47
- @Override
48
- public void configure(Binder binder)
49
- {
50
- binder.disableCircularProxies();
51
-
52
- binder.bindListener(any(), new TypeListener()
53
- {
54
- @Override
55
- public <T> void hear(TypeLiteral<T> type, TypeEncounter<T> encounter)
56
- {
57
- encounter.register(new InjectionListener<T>()
58
- {
59
- @Override
60
- public void afterInjection(T obj)
61
- {
62
- if (isLifeCycleClass(obj.getClass())) {
63
- LifeCycleManager lifeCycleManager = lifeCycleManagerRef.get();
64
- if (lifeCycleManager != null) {
65
- try {
66
- lifeCycleManager.addInstance(obj);
67
- }
68
- catch (Exception e) {
69
- throw new Error(e);
70
- }
71
- }
72
- else {
73
- injectedInstances.add(obj);
74
- }
75
- }
76
- }
77
- });
78
- }
79
- });
80
- }
81
-
82
- @Provides
83
- @Singleton
84
- public LifeCycleManager getServerManager()
85
- throws Exception
86
- {
87
- LifeCycleManager lifeCycleManager = new LifeCycleManager(injectedInstances, lifeCycleMethodsMap);
88
- lifeCycleManagerRef.set(lifeCycleManager);
89
- return lifeCycleManager;
90
- }
91
-
92
- private boolean isLifeCycleClass(Class<?> clazz)
93
- {
94
- LifeCycleMethods methods = lifeCycleMethodsMap.get(clazz);
95
- return methods.hasFor(PostConstruct.class) || methods.hasFor(PreDestroy.class);
96
- }
97
- }