embulk 0.7.11-java → 0.8.0-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile.lock +3 -3
  3. data/README.md +1 -1
  4. data/build.gradle +2 -2
  5. data/embulk-core/build.gradle +2 -0
  6. data/embulk-core/src/main/java/org/embulk/config/ConfigLoader.java +11 -3
  7. data/embulk-core/src/main/java/org/embulk/config/YamlTagResolver.java +53 -0
  8. data/embulk-core/src/main/java/org/embulk/exec/ExecModule.java +0 -1
  9. data/embulk-core/src/main/java/org/embulk/exec/LocalExecutorPlugin.java +479 -69
  10. data/embulk-core/src/main/java/org/embulk/spi/Column.java +3 -0
  11. data/embulk-core/src/main/java/org/embulk/spi/ColumnVisitor.java +2 -0
  12. data/embulk-core/src/main/java/org/embulk/spi/ExecSession.java +12 -5
  13. data/embulk-core/src/main/java/org/embulk/spi/Page.java +19 -0
  14. data/embulk-core/src/main/java/org/embulk/spi/PageBuilder.java +26 -5
  15. data/embulk-core/src/main/java/org/embulk/spi/PageReader.java +13 -0
  16. data/embulk-core/src/main/java/org/embulk/spi/json/JsonParseException.java +17 -0
  17. data/embulk-core/src/main/java/org/embulk/spi/json/JsonParser.java +125 -0
  18. data/embulk-core/src/main/java/org/embulk/spi/json/RubyValueApi.java +55 -0
  19. data/embulk-core/src/main/java/org/embulk/spi/type/JsonType.java +14 -0
  20. data/embulk-core/src/main/java/org/embulk/spi/type/TypeDeserializer.java +1 -0
  21. data/embulk-core/src/main/java/org/embulk/spi/type/Types.java +2 -0
  22. data/embulk-core/src/main/java/org/embulk/spi/util/DynamicColumnSetterFactory.java +6 -0
  23. data/embulk-core/src/main/java/org/embulk/spi/util/PagePrinter.java +5 -0
  24. data/embulk-core/src/main/java/org/embulk/spi/util/Pages.java +10 -0
  25. data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/AbstractDynamicColumnSetter.java +3 -0
  26. data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/BooleanColumnSetter.java +7 -0
  27. data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/DefaultValueSetter.java +2 -0
  28. data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/DoubleColumnSetter.java +7 -0
  29. data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/JsonColumnSetter.java +73 -0
  30. data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/LongColumnSetter.java +11 -2
  31. data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/NullDefaultValueSetter.java +5 -0
  32. data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/SkipColumnSetter.java +5 -0
  33. data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/StringColumnSetter.java +7 -0
  34. data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/TimestampColumnSetter.java +9 -1
  35. data/embulk-core/src/test/java/org/embulk/spi/MockFormatterPlugin.java +7 -0
  36. data/embulk-docs/src/built-in.rst +40 -3
  37. data/embulk-docs/src/conf.py +2 -2
  38. data/embulk-docs/src/release.rst +1 -1
  39. data/embulk-docs/src/release/release-0.8.0.rst +68 -0
  40. data/embulk-standards/src/main/java/org/embulk/standards/CsvFormatterPlugin.java +12 -1
  41. data/embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java +18 -0
  42. data/embulk-standards/src/main/java/org/embulk/standards/CsvTokenizer.java +1 -1
  43. data/embulk.gemspec +1 -1
  44. data/gradle/wrapper/gradle-wrapper.jar +0 -0
  45. data/gradle/wrapper/gradle-wrapper.properties +2 -2
  46. data/gradlew +3 -7
  47. data/lib/embulk/column.rb +2 -0
  48. data/lib/embulk/command/embulk_migrate_plugin.rb +76 -10
  49. data/lib/embulk/command/embulk_new_plugin.rb +2 -0
  50. data/lib/embulk/command/embulk_run.rb +17 -10
  51. data/lib/embulk/data/bundle/.ruby-version +1 -1
  52. data/lib/embulk/data/new/java/build.gradle.erb +21 -0
  53. data/lib/embulk/data/new/java/config/checkstyle/checkstyle.xml +128 -0
  54. data/lib/embulk/data/new/java/config/checkstyle/default.xml +108 -0
  55. data/lib/embulk/data/new/java/gradle/wrapper/gradle-wrapper.jar +0 -0
  56. data/lib/embulk/data/new/java/gradle/wrapper/gradle-wrapper.properties +2 -2
  57. data/lib/embulk/data/new/java/gradlew +3 -7
  58. data/lib/embulk/data/new/ruby/.ruby-version +1 -1
  59. data/lib/embulk/guess/csv.rb +1 -1
  60. data/lib/embulk/guess/schema_guess.rb +6 -0
  61. data/lib/embulk/guess_plugin.rb +1 -1
  62. data/lib/embulk/java/imports.rb +4 -0
  63. data/lib/embulk/plugin_registry.rb +8 -12
  64. data/lib/embulk/schema.rb +6 -0
  65. data/lib/embulk/version.rb +1 -1
  66. data/test/guess/test_csv_guess.rb +170 -0
  67. data/test/helper.rb +2 -0
  68. metadata +17 -15
  69. data/embulk-core/src/main/java/org/embulk/exec/LocalThreadExecutor.java +0 -34
  70. data/embulk-core/src/main/java/org/embulk/guice/Bootstrap.java +0 -157
  71. data/embulk-core/src/main/java/org/embulk/guice/CloseableInjector.java +0 -22
  72. data/embulk-core/src/main/java/org/embulk/guice/InjectorProxy.java +0 -145
  73. data/embulk-core/src/main/java/org/embulk/guice/LifeCycleInjector.java +0 -26
  74. data/embulk-core/src/main/java/org/embulk/guice/LifeCycleInjectorProxy.java +0 -61
  75. data/embulk-core/src/main/java/org/embulk/guice/LifeCycleManager.java +0 -187
  76. data/embulk-core/src/main/java/org/embulk/guice/LifeCycleMethods.java +0 -89
  77. data/embulk-core/src/main/java/org/embulk/guice/LifeCycleMethodsMap.java +0 -38
  78. data/embulk-core/src/main/java/org/embulk/guice/LifeCycleModule.java +0 -97
  79. data/embulk-docs/src/release/release-0.7.11.rst +0 -13
@@ -1,26 +0,0 @@
1
- /*
2
- * Copyright 2015 Sadayuki Furuhashi
3
- *
4
- * Licensed under the Apache License, Version 2.0 (the "License");
5
- * you may not use this file except in compliance with the License.
6
- * You may obtain a copy of the License at
7
- *
8
- * http://www.apache.org/licenses/LICENSE-2.0
9
- *
10
- * Unless required by applicable law or agreed to in writing, software
11
- * distributed under the License is distributed on an "AS IS" BASIS,
12
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
- * See the License for the specific language governing permissions and
14
- * limitations under the License.
15
- */
16
- package org.embulk.guice;
17
-
18
- import com.google.inject.Injector;
19
-
20
- public interface LifeCycleInjector
21
- extends Injector
22
- {
23
- boolean isDestroyed();
24
-
25
- void destroy() throws Exception;
26
- }
@@ -1,61 +0,0 @@
1
- /*
2
- * Copyright 2015 Sadayuki Furuhashi
3
- *
4
- * Licensed under the Apache License, Version 2.0 (the "License");
5
- * you may not use this file except in compliance with the License.
6
- * You may obtain a copy of the License at
7
- *
8
- * http://www.apache.org/licenses/LICENSE-2.0
9
- *
10
- * Unless required by applicable law or agreed to in writing, software
11
- * distributed under the License is distributed on an "AS IS" BASIS,
12
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
- * See the License for the specific language governing permissions and
14
- * limitations under the License.
15
- */
16
- package org.embulk.guice;
17
-
18
- import com.google.inject.Injector;
19
-
20
- class LifeCycleInjectorProxy
21
- extends InjectorProxy
22
- implements LifeCycleInjector, CloseableInjector
23
- {
24
- private final Injector injector;
25
- private final LifeCycleManager lifeCycleManager;
26
-
27
- public LifeCycleInjectorProxy(Injector injector, LifeCycleManager lifeCycleManager)
28
- {
29
- this.injector = injector;
30
- this.lifeCycleManager = lifeCycleManager;
31
- }
32
-
33
- @Override
34
- protected synchronized Injector injector()
35
- {
36
- if (isDestroyed()) {
37
- throw new IllegalStateException("Injector already destroyed");
38
- }
39
- return injector;
40
- }
41
-
42
- @Override
43
- public synchronized boolean isDestroyed()
44
- {
45
- return lifeCycleManager.isDestroyed();
46
- }
47
-
48
- @Override
49
- public synchronized void destroy()
50
- throws Exception
51
- {
52
- lifeCycleManager.destroy(); // reentrant
53
- }
54
-
55
- @Override
56
- public void close()
57
- throws Exception
58
- {
59
- destroy();
60
- }
61
- }
@@ -1,187 +0,0 @@
1
- /*
2
- * Copyright 2010 Proofpoint, Inc.
3
- *
4
- * Licensed under the Apache License, Version 2.0 (the "License");
5
- * you may not use this file except in compliance with the License.
6
- * You may obtain a copy of the License at
7
- *
8
- * http://www.apache.org/licenses/LICENSE-2.0
9
- *
10
- * Unless required by applicable law or agreed to in writing, software
11
- * distributed under the License is distributed on an "AS IS" BASIS,
12
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
- * See the License for the specific language governing permissions and
14
- * limitations under the License.
15
- */
16
- /*
17
- * Copyright 2015 Sadayuki Furuhashi
18
- */
19
- package org.embulk.guice;
20
-
21
- import com.google.common.collect.Lists;
22
-
23
- import javax.annotation.PostConstruct;
24
- import javax.annotation.PreDestroy;
25
- import java.lang.reflect.InvocationTargetException;
26
- import java.lang.reflect.Method;
27
- import java.util.Collections;
28
- import java.util.List;
29
- import java.util.Queue;
30
- import java.util.concurrent.ConcurrentLinkedQueue;
31
- import java.util.concurrent.atomic.AtomicReference;
32
-
33
- /**
34
- * Manages PostConstruct and PreDestroy life cycles
35
- */
36
- public final class LifeCycleManager
37
- {
38
- private final AtomicReference<State> state = new AtomicReference<State>(State.LATENT);
39
- private final Queue<Object> managedInstances = new ConcurrentLinkedQueue<Object>();
40
- private final LifeCycleMethodsMap methodsMap;
41
-
42
- private enum State
43
- {
44
- LATENT,
45
- STARTING,
46
- STARTED,
47
- STOPPING,
48
- STOPPED
49
- }
50
-
51
- /**
52
- * @param managedInstances list of objects that have life cycle annotations
53
- * @param methodsMap existing or new methods map
54
- * @throws Exception exceptions starting instances (depending on mode)
55
- */
56
- public LifeCycleManager(List<Object> managedInstances, LifeCycleMethodsMap methodsMap)
57
- throws Exception
58
- {
59
- this.methodsMap = (methodsMap != null) ? methodsMap : new LifeCycleMethodsMap();
60
- for (Object instance : managedInstances) {
61
- addInstance(instance);
62
- }
63
- }
64
-
65
- /**
66
- * Returns the number of managed instances
67
- *
68
- * @return qty
69
- */
70
- public int size()
71
- {
72
- return managedInstances.size();
73
- }
74
-
75
- /**
76
- * Start the life cycle - all instances will have their {@link javax.annotation.PostConstruct} method(s) called
77
- */
78
- public void start()
79
- {
80
- if (!state.compareAndSet(State.LATENT, State.STARTING)) {
81
- throw new IllegalStateException("System already starting");
82
- }
83
- //log.info("Life cycle starting...");
84
-
85
- for (Object obj : managedInstances) {
86
- LifeCycleMethods methods = methodsMap.get(obj.getClass());
87
- if (!methods.hasFor(PreDestroy.class)) {
88
- managedInstances.remove(obj); // remove reference to instances that aren't needed anymore
89
- }
90
- }
91
-
92
- state.set(State.STARTED);
93
- //log.info("Life cycle startup complete. System ready.");
94
- }
95
-
96
- /**
97
- * Add a shutdown hook that calls {@link destroy} method
98
- */
99
- public void destroyOnShutdownHook()
100
- {
101
- Runtime.getRuntime().addShutdownHook(new Thread()
102
- {
103
- @Override
104
- public void run()
105
- {
106
- try {
107
- LifeCycleManager.this.destroy();
108
- }
109
- catch (Exception e) {
110
- //log.error(e, "Trying to shut down");
111
- }
112
- }
113
- });
114
- }
115
-
116
- /**
117
- * Stop the life cycle - all instances will have their {@link javax.annotation.PreDestroy} method(s) called
118
- *
119
- * @throws Exception errors
120
- */
121
- public void destroy()
122
- throws Exception
123
- {
124
- if (!state.compareAndSet(State.STARTED, State.STOPPING)) {
125
- return;
126
- }
127
-
128
- //log.info("Life cycle stopping...");
129
-
130
- List<Object> reversedInstances = Lists.newArrayList(managedInstances);
131
- Collections.reverse(reversedInstances);
132
-
133
- for (Object obj : reversedInstances) {
134
- //log.debug("Stopping %s", obj.getClass().getName());
135
- LifeCycleMethods methods = methodsMap.get(obj.getClass());
136
- for (Method preDestroy : methods.methodsFor(PreDestroy.class)) {
137
- //log.debug("\t%s()", preDestroy.getName());
138
- preDestroy.invoke(obj);
139
- }
140
- }
141
-
142
- state.set(State.STOPPED);
143
- //log.info("Life cycle stopped.");
144
- }
145
-
146
- /**
147
- * Return true if {@link destroy} is called
148
- *
149
- * @return true if already destroyed
150
- */
151
- public boolean isDestroyed()
152
- {
153
- State currentState = state.get();
154
- return currentState == State.STOPPING || currentState == State.STOPPED;
155
- }
156
-
157
- /**
158
- * Add an additional managed instance
159
- *
160
- * @param instance instance to add
161
- * @throws Exception errors
162
- */
163
- public void addInstance(Object instance)
164
- throws Exception
165
- {
166
- if (isDestroyed()) {
167
- throw new IllegalStateException("System already stopped");
168
- }
169
- else {
170
- startInstance(instance);
171
- if (methodsMap.get(instance.getClass()).hasFor(PreDestroy.class)) {
172
- managedInstances.add(instance);
173
- }
174
- }
175
- }
176
-
177
- private void startInstance(Object obj)
178
- throws IllegalAccessException, InvocationTargetException
179
- {
180
- //log.debug("Starting %s", obj.getClass().getName());
181
- LifeCycleMethods methods = methodsMap.get(obj.getClass());
182
- for (Method postConstruct : methods.methodsFor(PostConstruct.class)) {
183
- //log.debug("\t%s()", postConstruct.getName());
184
- postConstruct.invoke(obj);
185
- }
186
- }
187
- }
@@ -1,89 +0,0 @@
1
- /*
2
- * Copyright 2010 Proofpoint, Inc.
3
- *
4
- * Licensed under the Apache License, Version 2.0 (the "License");
5
- * you may not use this file except in compliance with the License.
6
- * You may obtain a copy of the License at
7
- *
8
- * http://www.apache.org/licenses/LICENSE-2.0
9
- *
10
- * Unless required by applicable law or agreed to in writing, software
11
- * distributed under the License is distributed on an "AS IS" BASIS,
12
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
- * See the License for the specific language governing permissions and
14
- * limitations under the License.
15
- */
16
- /*
17
- * Copyright 2015 Sadayuki Furuhashi
18
- */
19
- package org.embulk.guice;
20
-
21
- import com.google.common.collect.ArrayListMultimap;
22
- import com.google.common.collect.Lists;
23
- import com.google.common.collect.Multimap;
24
-
25
- import javax.annotation.PostConstruct;
26
- import javax.annotation.PreDestroy;
27
- import java.lang.annotation.Annotation;
28
- import java.lang.reflect.Method;
29
- import java.util.Collection;
30
- import java.util.HashSet;
31
- import java.util.Set;
32
-
33
- class LifeCycleMethods
34
- {
35
- private final Multimap<Class<? extends Annotation>, Method> methodMap = ArrayListMultimap.create();
36
-
37
- LifeCycleMethods(Class<?> clazz)
38
- {
39
- addLifeCycleMethods(clazz, new HashSet<String>(), new HashSet<String>());
40
- }
41
-
42
- boolean hasFor(Class<? extends Annotation> annotation)
43
- {
44
- Collection<Method> methods = methodMap.get(annotation);
45
- return (methods != null) && (methods.size() > 0);
46
- }
47
-
48
- Collection<Method> methodsFor(Class<? extends Annotation> annotation)
49
- {
50
- Collection<Method> methods = methodMap.get(annotation);
51
- return (methods != null) ? methods : Lists.<Method>newArrayList();
52
- }
53
-
54
- private void addLifeCycleMethods(Class<?> clazz, Set<String> usedConstructNames, Set<String> usedDestroyNames)
55
- {
56
- if (clazz == null) {
57
- return;
58
- }
59
-
60
- for (Method method : clazz.getDeclaredMethods()) {
61
- if (method.isSynthetic() || method.isBridge()) {
62
- continue;
63
- }
64
-
65
- processMethod(method, PostConstruct.class, usedConstructNames);
66
- processMethod(method, PreDestroy.class, usedDestroyNames);
67
- }
68
-
69
- addLifeCycleMethods(clazz.getSuperclass(), usedConstructNames, usedDestroyNames);
70
- for (Class<?> face : clazz.getInterfaces()) {
71
- addLifeCycleMethods(face, usedConstructNames, usedDestroyNames);
72
- }
73
- }
74
-
75
- private void processMethod(Method method, Class<? extends Annotation> annotationClass, Set<String> usedSet)
76
- {
77
- if (method.isAnnotationPresent(annotationClass)) {
78
- if (!usedSet.contains(method.getName())) {
79
- if (method.getParameterTypes().length != 0) {
80
- throw new UnsupportedOperationException(String.format("@PostConstruct/@PreDestroy methods cannot have arguments: %s", method.getDeclaringClass().getName() + "." + method.getName() + "(...)"));
81
- }
82
-
83
- method.setAccessible(true);
84
- usedSet.add(method.getName());
85
- methodMap.put(annotationClass, method);
86
- }
87
- }
88
- }
89
- }
@@ -1,38 +0,0 @@
1
- /*
2
- * Copyright 2010 Proofpoint, Inc.
3
- *
4
- * Licensed under the Apache License, Version 2.0 (the "License");
5
- * you may not use this file except in compliance with the License.
6
- * You may obtain a copy of the License at
7
- *
8
- * http://www.apache.org/licenses/LICENSE-2.0
9
- *
10
- * Unless required by applicable law or agreed to in writing, software
11
- * distributed under the License is distributed on an "AS IS" BASIS,
12
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
- * See the License for the specific language governing permissions and
14
- * limitations under the License.
15
- */
16
- /*
17
- * Copyright 2015 Sadayuki Furuhashi
18
- */
19
- package org.embulk.guice;
20
-
21
- import com.google.common.collect.Maps;
22
-
23
- import java.util.Map;
24
-
25
- class LifeCycleMethodsMap
26
- {
27
- private final Map<Class<?>, LifeCycleMethods> map = Maps.newHashMap();
28
-
29
- synchronized LifeCycleMethods get(Class<?> clazz)
30
- {
31
- LifeCycleMethods methods = map.get(clazz);
32
- if (methods == null) {
33
- methods = new LifeCycleMethods(clazz);
34
- map.put(clazz, methods);
35
- }
36
- return methods;
37
- }
38
- }
@@ -1,97 +0,0 @@
1
- /*
2
- * Copyright 2010 Proofpoint, Inc.
3
- *
4
- * Licensed under the Apache License, Version 2.0 (the "License");
5
- * you may not use this file except in compliance with the License.
6
- * You may obtain a copy of the License at
7
- *
8
- * http://www.apache.org/licenses/LICENSE-2.0
9
- *
10
- * Unless required by applicable law or agreed to in writing, software
11
- * distributed under the License is distributed on an "AS IS" BASIS,
12
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
- * See the License for the specific language governing permissions and
14
- * limitations under the License.
15
- */
16
- /*
17
- * Copyright 2015 Sadayuki Furuhashi
18
- */
19
- package org.embulk.guice;
20
-
21
- import com.google.common.collect.Lists;
22
- import com.google.inject.Binder;
23
- import com.google.inject.Module;
24
- import com.google.inject.Provides;
25
- import com.google.inject.Singleton;
26
- import com.google.inject.TypeLiteral;
27
- import com.google.inject.spi.InjectionListener;
28
- import com.google.inject.spi.TypeEncounter;
29
- import com.google.inject.spi.TypeListener;
30
-
31
- import javax.annotation.PostConstruct;
32
- import javax.annotation.PreDestroy;
33
- import java.util.List;
34
- import java.util.concurrent.atomic.AtomicReference;
35
-
36
- import static com.google.inject.matcher.Matchers.any;
37
-
38
- /**
39
- * Guice module for binding the LifeCycle manager
40
- */
41
- public class LifeCycleModule implements Module
42
- {
43
- private final List<Object> injectedInstances = Lists.newArrayList();
44
- private final LifeCycleMethodsMap lifeCycleMethodsMap = new LifeCycleMethodsMap();
45
- private final AtomicReference<LifeCycleManager> lifeCycleManagerRef = new AtomicReference<LifeCycleManager>(null);
46
-
47
- @Override
48
- public void configure(Binder binder)
49
- {
50
- binder.disableCircularProxies();
51
-
52
- binder.bindListener(any(), new TypeListener()
53
- {
54
- @Override
55
- public <T> void hear(TypeLiteral<T> type, TypeEncounter<T> encounter)
56
- {
57
- encounter.register(new InjectionListener<T>()
58
- {
59
- @Override
60
- public void afterInjection(T obj)
61
- {
62
- if (isLifeCycleClass(obj.getClass())) {
63
- LifeCycleManager lifeCycleManager = lifeCycleManagerRef.get();
64
- if (lifeCycleManager != null) {
65
- try {
66
- lifeCycleManager.addInstance(obj);
67
- }
68
- catch (Exception e) {
69
- throw new Error(e);
70
- }
71
- }
72
- else {
73
- injectedInstances.add(obj);
74
- }
75
- }
76
- }
77
- });
78
- }
79
- });
80
- }
81
-
82
- @Provides
83
- @Singleton
84
- public LifeCycleManager getServerManager()
85
- throws Exception
86
- {
87
- LifeCycleManager lifeCycleManager = new LifeCycleManager(injectedInstances, lifeCycleMethodsMap);
88
- lifeCycleManagerRef.set(lifeCycleManager);
89
- return lifeCycleManager;
90
- }
91
-
92
- private boolean isLifeCycleClass(Class<?> clazz)
93
- {
94
- LifeCycleMethods methods = lifeCycleMethodsMap.get(clazz);
95
- return methods.hasFor(PostConstruct.class) || methods.hasFor(PreDestroy.class);
96
- }
97
- }