embulk 0.8.31 → 0.8.32

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 6ff6c6b94f1515912bd831a3b7110fa284bfb6bd
4
- data.tar.gz: 2ae51b3c9418fe73fe20b494a120d7163d15a6d8
3
+ metadata.gz: 97436b4c67ee17c776dfe126aa2d145346195823
4
+ data.tar.gz: 5be788162adb62bb4b6e6d1cc82f7eb8ccb4e5e6
5
5
  SHA512:
6
- metadata.gz: 9aba47a88b711b9ae862f8f391c1b8ea439e781d5b77c189309a871b05d538e707c8b8aca9b7609a497de81eadad430c4ab1b6dbcc9edaefd93526dd09fd49f5
7
- data.tar.gz: 5a615308623290f1e2bc2a90eca9b841ffee8ad3428eabd59d289839d42af7037ab5a46e62a69e92967034ebc932d1200407eecc7d4c048f89825845b3e1fb60
6
+ metadata.gz: 2b1c113fbc81ee13a6bc1c26ed7a84388c4228d148d61d451387eb5c9a90aaad21b9ac94de2f0ebbac52d854693973beac89b0c2e388db8aa4542def00867110
7
+ data.tar.gz: 0e144e78fabeece178474301c9c1f27f801723475946e385dbcb9ba0f51de49e90b305be80cbc79f0f44576ff7a33c54dfea2b2c0d750880893bbbd0c12ca911
data/build.gradle CHANGED
@@ -1,3 +1,11 @@
1
+ // TODO: Remove this block once rubygems.lasagna.io is back, or jruby-gradle-jar-plugin is upgraded to 1.5.0.
2
+ // See also: https://github.com/jruby-gradle/jruby-gradle-plugin/issues/297
3
+ // jruby-gradle-jar-plugin is not upgraded yet because its 1.5.0 depends on Java 8.
4
+ repositories {
5
+ mavenLocal()
6
+ maven { url 'http://rubygems-proxy.torquebox.org/releases' }
7
+ }
8
+
1
9
  buildscript {
2
10
  repositories { jcenter() }
3
11
  dependencies {
@@ -16,7 +24,7 @@ def release_projects = [project(":embulk-core"), project(":embulk-standards"), p
16
24
 
17
25
  allprojects {
18
26
  group = 'org.embulk'
19
- version = '0.8.31'
27
+ version = '0.8.32'
20
28
 
21
29
  ext {
22
30
  jrubyVersion = '9.1.5.0'
@@ -1,5 +1,6 @@
1
1
  package org.embulk.cli;
2
2
 
3
+ import java.nio.file.Paths;
3
4
  import java.util.ArrayList;
4
5
  import java.util.Arrays;
5
6
  import java.util.Collections;
@@ -12,27 +13,19 @@ public class EmbulkCommandLine
12
13
  private EmbulkCommandLine(
13
14
  final List<String> arguments,
14
15
  final Map<String, Object> systemConfig,
15
- final String bundle,
16
16
  final String bundlePath,
17
- final List<String> classpath,
18
17
  final String configDiff,
19
18
  final boolean force,
20
19
  final String format,
21
- final List<String> load,
22
- final List<String> loadPath,
23
20
  final String output,
24
21
  final String resumeState)
25
22
  {
26
23
  this.arguments = Collections.unmodifiableList(arguments);
27
24
  this.systemConfig = Collections.unmodifiableMap(systemConfig);
28
- this.bundle = bundle;
29
25
  this.bundlePath = bundlePath;
30
- this.classpath = Collections.unmodifiableList(classpath);
31
26
  this.configDiff = configDiff;
32
27
  this.force = force;
33
28
  this.format = format;
34
- this.load = Collections.unmodifiableList(load);
35
- this.loadPath = Collections.unmodifiableList(loadPath);
36
29
  this.output = output;
37
30
  this.resumeState = resumeState;
38
31
  }
@@ -43,9 +36,7 @@ public class EmbulkCommandLine
43
36
  {
44
37
  this.arguments = new ArrayList<String>();
45
38
  this.systemConfig = new HashMap<String, Object>();
46
- this.bundle = null;
47
39
  this.bundlePath = null;
48
- this.classpath = new ArrayList<String>();
49
40
  this.configDiff = null;
50
41
  this.force = false;
51
42
  this.format = null;
@@ -57,17 +48,27 @@ public class EmbulkCommandLine
57
48
 
58
49
  public EmbulkCommandLine build()
59
50
  {
51
+ final HashMap<String, Object> systemConfigJRubyLoadPath = new HashMap<String, Object>(this.systemConfig);
52
+ // first $LOAD_PATH has highet priority. later load_paths should have highest priority.
53
+ for (final String oneJRubyLoadPath : this.loadPath) { // "-L"
54
+ addInHashMap(systemConfigJRubyLoadPath, "jruby_load_path", oneJRubyLoadPath);
55
+ }
56
+ // Gem::StubSpecification is an internal API that seems chainging often.
57
+ // Gem::Specification.add_spec is deprecated also. Therefore, here makes
58
+ // -L <path> option alias of -I <path>/lib by assuming that *.gemspec file
59
+ // always has require_paths = ["lib"].
60
+ for (final String oneJRubyLoadPathToAddLib : this.load) { // "-I"
61
+ addInHashMap(systemConfigJRubyLoadPath,
62
+ "jruby_load_path",
63
+ Paths.get(oneJRubyLoadPathToAddLib).resolve("lib").toString());
64
+ }
60
65
  return new EmbulkCommandLine(
61
66
  this.arguments,
62
- this.systemConfig,
63
- this.bundle,
67
+ systemConfigJRubyLoadPath,
64
68
  this.bundlePath,
65
- this.classpath,
66
69
  this.configDiff,
67
70
  this.force,
68
71
  this.format,
69
- this.load,
70
- this.loadPath,
71
72
  this.output,
72
73
  this.resumeState);
73
74
  }
@@ -86,25 +87,7 @@ public class EmbulkCommandLine
86
87
 
87
88
  public Builder addSystemConfig(final String key, final String value)
88
89
  {
89
- final Object existingValue = this.systemConfig.get(key);
90
- if (existingValue != null && existingValue instanceof String) {
91
- this.systemConfig.put(key, Arrays.asList((String) existingValue, value));
92
- }
93
- else if (existingValue != null && existingValue instanceof List) {
94
- @SuppressWarnings("unchecked")
95
- final ArrayList<String> newList = new ArrayList<String>((List<String>) existingValue);
96
- newList.add(value);
97
- this.systemConfig.put(key, Collections.unmodifiableList(newList));
98
- }
99
- else {
100
- this.systemConfig.put(key, Arrays.asList(value));
101
- }
102
- return this;
103
- }
104
-
105
- public Builder setBundle(final String bundle)
106
- {
107
- this.bundle = bundle;
90
+ addInHashMap(this.systemConfig, key, value);
108
91
  return this;
109
92
  }
110
93
 
@@ -114,12 +97,6 @@ public class EmbulkCommandLine
114
97
  return this;
115
98
  }
116
99
 
117
- public Builder addClasspath(final String classpath)
118
- {
119
- this.classpath.add(classpath);
120
- return this;
121
- }
122
-
123
100
  public Builder setConfigDiff(final String configDiff)
124
101
  {
125
102
  this.configDiff = configDiff;
@@ -162,11 +139,26 @@ public class EmbulkCommandLine
162
139
  return this;
163
140
  }
164
141
 
142
+ private static void addInHashMap(final HashMap<String, Object> map, final String key, final String value)
143
+ {
144
+ final Object existingValue = map.get(key);
145
+ if (existingValue != null && existingValue instanceof String) {
146
+ map.put(key, Arrays.asList((String) existingValue, value));
147
+ }
148
+ else if (existingValue != null && existingValue instanceof List) {
149
+ @SuppressWarnings("unchecked")
150
+ final ArrayList<String> newList = new ArrayList<String>((List<String>) existingValue);
151
+ newList.add(value);
152
+ map.put(key, Collections.unmodifiableList(newList));
153
+ }
154
+ else {
155
+ map.put(key, Arrays.asList(value));
156
+ }
157
+ }
158
+
165
159
  private ArrayList<String> arguments;
166
160
  private HashMap<String, Object> systemConfig;
167
- private String bundle;
168
161
  private String bundlePath;
169
- private ArrayList<String> classpath;
170
162
  private String configDiff;
171
163
  private boolean force;
172
164
  private String format;
@@ -191,21 +183,11 @@ public class EmbulkCommandLine
191
183
  return this.systemConfig;
192
184
  }
193
185
 
194
- public final String getBundle()
195
- {
196
- return this.bundle;
197
- }
198
-
199
186
  public final String getBundlePath()
200
187
  {
201
188
  return this.bundlePath;
202
189
  }
203
190
 
204
- public final List<String> getClasspath()
205
- {
206
- return this.classpath;
207
- }
208
-
209
191
  public final String getConfigDiff()
210
192
  {
211
193
  return this.configDiff;
@@ -221,16 +203,6 @@ public class EmbulkCommandLine
221
203
  return this.format;
222
204
  }
223
205
 
224
- public final List<String> getLoad()
225
- {
226
- return this.load;
227
- }
228
-
229
- public final List<String> getLoadPath()
230
- {
231
- return this.loadPath;
232
- }
233
-
234
206
  public final String getOutput()
235
207
  {
236
208
  return this.output;
@@ -243,14 +215,10 @@ public class EmbulkCommandLine
243
215
 
244
216
  private final List<String> arguments;
245
217
  private final Map<String, Object> systemConfig;
246
- private final String bundle;
247
218
  private final String bundlePath;
248
- private final List<String> classpath;
249
219
  private final String configDiff;
250
220
  private final boolean force;
251
221
  private final String format;
252
- private final List<String> load;
253
- private final List<String> loadPath;
254
222
  private final String output;
255
223
  private final String resumeState;
256
224
  }
@@ -13,6 +13,7 @@ import java.nio.file.SimpleFileVisitor;
13
13
  import java.nio.file.StandardCopyOption;
14
14
  import java.nio.file.StandardOpenOption;
15
15
  import java.nio.file.attribute.BasicFileAttributes;
16
+ import java.nio.file.attribute.PosixFilePermission;
16
17
  import java.util.ArrayList;
17
18
  import java.util.Arrays;
18
19
  import java.util.List;
@@ -94,7 +95,9 @@ public class EmbulkMigrate
94
95
 
95
96
  // upgrade gradle version
96
97
  if (migrator.match("gradle/wrapper/gradle-wrapper.properties", GRADLE_VERSION_IN_WRAPPER)) {
97
- // gradle < 3.2.1
98
+ // gradle < 4.1
99
+ migrator.copy("embulk/data/new/java/gradlew", "gradlew");
100
+ migrator.setExecutable("gradlew");
98
101
  migrator.copy("embulk/data/new/java/gradle/wrapper/gradle-wrapper.properties",
99
102
  "gradle/wrapper/gradle-wrapper.properties");
100
103
  migrator.copy("embulk/data/new/java/gradle/wrapper/gradle-wrapper.jar",
@@ -410,6 +413,18 @@ public class EmbulkMigrate
410
413
  }
411
414
  }
412
415
 
416
+ private void setExecutable(String targetFileName)
417
+ throws IOException
418
+ {
419
+ final Path targetPath = this.basePath.resolve(targetFileName);
420
+ final Set<PosixFilePermission> permissions =
421
+ new HashSet<PosixFilePermission>(Files.getPosixFilePermissions(targetPath));
422
+ permissions.add(PosixFilePermission.OWNER_EXECUTE);
423
+ permissions.add(PosixFilePermission.GROUP_EXECUTE);
424
+ permissions.add(PosixFilePermission.OTHERS_EXECUTE);
425
+ Files.setPosixFilePermissions(targetPath, permissions);
426
+ }
427
+
413
428
  private final Path basePath;
414
429
  private final Set<Path> modifiedFiles;
415
430
  }
@@ -84,7 +84,7 @@ public class EmbulkRun
84
84
  final EmbulkCommandLine commandLine;
85
85
  try {
86
86
  commandLine = parser.parse(
87
- subcommandArguments, new PrintWriter(System.out), new PrintWriter(System.err));
87
+ subcommandArguments, jrubyOptions, new PrintWriter(System.out), new PrintWriter(System.err));
88
88
  }
89
89
  catch (EmbulkCommandLineParseException ex) {
90
90
  parser.printHelp(System.err);
@@ -446,13 +446,7 @@ public class EmbulkRun
446
446
  // embulk classes
447
447
  // NOTE: |EmbulkSetup.setup| returns |EmbulkEmbed| while it stores Ruby |Embulk::EmbulkRunner(EmbulkEmbed)|
448
448
  // into Ruby |Embulk::Runner|.
449
- final EmbulkRunner runner = EmbulkSetup.setup(
450
- jrubyOptions,
451
- commandLine.getSystemConfig(),
452
- commandLine.getLoadPath(),
453
- commandLine.getLoad(),
454
- commandLine.getClasspath(),
455
- commandLine.getBundle());
449
+ final EmbulkRunner runner = EmbulkSetup.setup(commandLine.getSystemConfig());
456
450
 
457
451
  final Path configDiffPath =
458
452
  (commandLine.getConfigDiff() == null ? null : Paths.get(commandLine.getConfigDiff()));
@@ -628,7 +622,7 @@ public class EmbulkRun
628
622
  {
629
623
  final String[] classpaths = argument.split("\\" + java.io.File.pathSeparator);
630
624
  for (final String classpath : classpaths) {
631
- commandLineBuilder.addClasspath(classpath);
625
+ commandLineBuilder.addSystemConfig("jruby_classpath", classpath);
632
626
  }
633
627
  }
634
628
  }));
@@ -641,7 +635,7 @@ public class EmbulkRun
641
635
  {
642
636
  public void behave(final EmbulkCommandLine.Builder commandLineBuilder, final String argument)
643
637
  {
644
- commandLineBuilder.setBundle(argument);
638
+ commandLineBuilder.setSystemConfig("jruby_global_bundler_plugin_source_directory", argument);
645
639
  }
646
640
  }));
647
641
  }
@@ -113,6 +113,7 @@ public class EmbulkCommandLineParser
113
113
  }
114
114
 
115
115
  public EmbulkCommandLine parse(final List<String> argsEmbulk,
116
+ final List<String> jrubyOptions,
116
117
  final PrintWriter helpPrintWriter,
117
118
  final PrintWriter errorPrintWriter)
118
119
  throws EmbulkCommandLineParseException, EmbulkCommandLineHelpRequired
@@ -144,6 +145,9 @@ public class EmbulkCommandLineParser
144
145
  }
145
146
 
146
147
  final EmbulkCommandLine.Builder commandLineBuilder = EmbulkCommandLine.builder();
148
+ for (final String jrubyOption : jrubyOptions) {
149
+ commandLineBuilder.addSystemConfig("jruby_command_line_options", jrubyOption);
150
+ }
147
151
  commandLineBuilder.addArguments(arguments);
148
152
  for (final Option cliOptionSpecified : cliCommandLine.getOptions()) {
149
153
  final OptionDefinition optionDefinitionSpecified =
@@ -1,3 +1,11 @@
1
+ // TODO: Remove this block once rubygems.lasagna.io is back, or jruby-gradle-jar-plugin is upgraded to 1.5.0.
2
+ // See also: https://github.com/jruby-gradle/jruby-gradle-plugin/issues/297
3
+ // jruby-gradle-jar-plugin is not upgraded yet because its 1.5.0 depends on Java 8.
4
+ repositories {
5
+ mavenLocal()
6
+ maven { url 'http://rubygems-proxy.torquebox.org/releases' }
7
+ }
8
+
1
9
  apply plugin: "com.github.jruby-gradle.jar"
2
10
 
3
11
  // include ruby scripts to jar. don't use sourceSets.main.resources.srcDirs
@@ -39,19 +39,9 @@ public class EmbulkRunner
39
39
  {
40
40
  // |EmbulkSetup.setup| initializes:
41
41
  // new EmbulkRunner(embed)
42
- public EmbulkRunner(final EmbulkEmbed embed,
43
- final List<String> jrubyOptions,
44
- final List<String> loadPaths,
45
- final List<String> pluginPaths,
46
- final List<String> classpaths,
47
- final String bundlePath)
42
+ public EmbulkRunner(final EmbulkEmbed embed)
48
43
  {
49
44
  this.embed = embed; // org.embulk.EmbulkEmbed
50
- this.jrubyOptions = jrubyOptions;
51
- this.loadPaths = loadPaths;
52
- this.pluginPaths = pluginPaths;
53
- this.classpaths = classpaths;
54
- this.bundlePath = bundlePath;
55
45
  }
56
46
 
57
47
  /**
@@ -228,8 +218,7 @@ public class EmbulkRunner
228
218
  private void guessInternal(final ConfigSource configSource, final Path outputPath)
229
219
  throws IOException
230
220
  {
231
- initializeGlobalJRubyScriptingContainer(
232
- this.jrubyOptions, this.loadPaths, this.pluginPaths, this.classpaths, this.bundlePath);
221
+ initializeGlobalJRubyScriptingContainer();
233
222
 
234
223
  try {
235
224
  checkFileWritable(outputPath);
@@ -253,8 +242,7 @@ public class EmbulkRunner
253
242
  private void previewInternal(final ConfigSource configSource, final String format)
254
243
  throws IOException
255
244
  {
256
- initializeGlobalJRubyScriptingContainer(
257
- this.jrubyOptions, this.loadPaths, this.pluginPaths, this.classpaths, this.bundlePath);
245
+ initializeGlobalJRubyScriptingContainer();
258
246
 
259
247
  final PreviewResult previewResult = this.embed.preview(configSource);
260
248
  final ModelManager modelManager = this.embed.getModelManager();
@@ -283,8 +271,7 @@ public class EmbulkRunner
283
271
  final Path resumeStatePath)
284
272
  throws IOException
285
273
  {
286
- initializeGlobalJRubyScriptingContainer(
287
- this.jrubyOptions, this.loadPaths, this.pluginPaths, this.classpaths, this.bundlePath);
274
+ initializeGlobalJRubyScriptingContainer();
288
275
 
289
276
  try {
290
277
  checkFileWritable(outputPath);
@@ -541,46 +528,48 @@ public class EmbulkRunner
541
528
  // end
542
529
 
543
530
  // TODO: Check if it is required to process JRuby options.
544
- private void initializeGlobalJRubyScriptingContainer(
545
- final List<String> jrubyOptions,
546
- final List<String> loadPaths,
547
- final List<String> pluginPaths,
548
- final List<String> classpaths,
549
- final String bundlePath)
531
+ private void initializeGlobalJRubyScriptingContainer()
550
532
  {
551
533
  final ScriptingContainer globalJRubyContainer =
552
- EmbulkGlobalJRubyScriptingContainer.setup(jrubyOptions, bundlePath, System.err);
553
-
554
- // first $LOAD_PATH has highet priority. later load_paths should have highest priority.
555
- for (final String loadPath : loadPaths) {
556
- // ruby script directory (use unshift to make it highest priority)
557
- globalJRubyContainer.put("__internal_load_path__", loadPath);
558
- globalJRubyContainer.runScriptlet("$LOAD_PATH.unshift File.expand_path(__internal_load_path__)");
559
- globalJRubyContainer.remove("__internal_load_path__");
560
- }
561
-
562
- // # Gem::StubSpecification is an internal API that seems chainging often.
563
- // # Gem::Specification.add_spec is deprecated also. Therefore, here makes
564
- // # -L <path> option alias of -I <path>/lib by assuming that *.gemspec file
565
- // # always has require_paths = ["lib"].
566
- for (final String pluginPath : pluginPaths) {
567
- globalJRubyContainer.put("__internal_plugin_path__", pluginPath);
568
- globalJRubyContainer.runScriptlet("$LOAD_PATH.unshift File.expand_path(File.join(__internal_plugin_path__, 'lib'))");
569
- globalJRubyContainer.remove("__internal_plugin_path__");
570
- }
571
-
572
- for (final String classpath : classpaths) {
573
- globalJRubyContainer.put("__internal_classpath__", classpath);
574
- // $CLASSPATH object doesn't have concat method
575
- globalJRubyContainer.runScriptlet("$CLASSPATH << __internal_classpath__");
576
- globalJRubyContainer.remove("__internal_classpath__");
577
- }
578
-
579
- // see also embulk/java/bootstrap.rb loaded by JRubyScriptingModule
580
- globalJRubyContainer.runScriptlet("module Embulk; end");
581
- globalJRubyContainer.put("__internal_embulk_runner_java__", this);
582
- globalJRubyContainer.runScriptlet("Embulk.const_set :Runner, Embulk::EmbulkRunner.new(__internal_embulk_runner_java__)");
583
- globalJRubyContainer.remove("__internal_embulk_runner_java__");
534
+ new ScriptingContainer(LocalContextScope.SINGLETON, LocalVariableBehavior.PERSISTENT);
535
+
536
+ // TODO: Remove the Embulk::Runner definition after confirming nobody uses Embulk::Runner from Java.
537
+ globalJRubyContainer.put("__internal_runner_java__", this);
538
+ globalJRubyContainer.runScriptlet(
539
+ "class DummyEmbulkRunner\n" +
540
+ " def initialize(runner_orig)\n" +
541
+ " @runner_orig = runner_orig\n" +
542
+ " end\n" +
543
+ " def guess(config, options={})\n" +
544
+ " STDERR.puts '################################################################################'\n" +
545
+ " STDERR.puts '[WARN] Embulk::Runner will be no longer defined when Embulk runs from Java.'\n" +
546
+ " STDERR.puts '[WARN] Comment at https://github.com/embulk/embulk/issues/766 if you see this.'\n" +
547
+ " STDERR.puts '################################################################################'\n" +
548
+ " STDERR.puts ''\n" +
549
+ " @runner_orig.guess(config, options)\n" +
550
+ " end\n" +
551
+ " def preview(config, options={})\n" +
552
+ " STDERR.puts '################################################################################'\n" +
553
+ " STDERR.puts '[WARN] Embulk::Runner will be no longer defined when Embulk runs from Java.'\n" +
554
+ " STDERR.puts '[WARN] Comment at https://github.com/embulk/embulk/issues/766 if you see this.'\n" +
555
+ " STDERR.puts '################################################################################'\n" +
556
+ " STDERR.puts ''\n" +
557
+ " @runner_orig.preview(config, options)\n" +
558
+ " end\n" +
559
+ " def run(config, options={})\n" +
560
+ " STDERR.puts '################################################################################'\n" +
561
+ " STDERR.puts '[WARN] Embulk::Runner will be no longer defined when Embulk runs from Java.'\n" +
562
+ " STDERR.puts '[WARN] Comment at https://github.com/embulk/embulk/issues/766 if you see this.'\n" +
563
+ " STDERR.puts '################################################################################'\n" +
564
+ " STDERR.puts ''\n" +
565
+ " @runner_orig.run(config, options)\n" +
566
+ " end\n" +
567
+ "end\n" +
568
+ "\n" +
569
+ "unless Embulk.const_defined?(:Runner)\n" +
570
+ " Embulk.const_set :Runner, DummyEmbulkRunner.new(Embulk::EmbulkRunner.new(__internal_runner_java__))\n" +
571
+ "end\n");
572
+ globalJRubyContainer.remove("__internal_runner_java__");
584
573
  }
585
574
 
586
575
  // NOTE: The root logger directly from |LoggerFactory|, not from |Exec.getLogger| as it's outside of |Exec.doWith|.
@@ -590,9 +579,4 @@ public class EmbulkRunner
590
579
  private final Pattern EXT_YAML_LIQUID = Pattern.compile(".*\\.ya?ml\\.liquid$");
591
580
 
592
581
  private final EmbulkEmbed embed;
593
- private final List<String> jrubyOptions;
594
- private final List<String> loadPaths;
595
- private final List<String> pluginPaths;
596
- private final List<String> classpaths;
597
- private final String bundlePath;
598
582
  }