embulk 0.8.35 → 0.8.36

Sign up to get free protection for your applications and to get access to all the features.
Files changed (36) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +4 -0
  3. data/build.gradle +1 -1
  4. data/embulk-cli/src/main/java/org/embulk/cli/EmbulkExample.java +5 -1
  5. data/embulk-cli/src/main/java/org/embulk/cli/EmbulkRun.java +12 -0
  6. data/embulk-core/src/main/java/org/embulk/EmbulkRunner.java +2 -2
  7. data/embulk-core/src/main/java/org/embulk/plugin/PluginClassLoader.java +802 -17
  8. data/embulk-core/src/main/java/org/embulk/plugin/PluginClassLoaderFactory.java +8 -1
  9. data/embulk-core/src/main/java/org/embulk/plugin/PluginClassLoaderModule.java +33 -2
  10. data/embulk-core/src/main/java/org/embulk/plugin/jar/JarPluginLoader.java +32 -5
  11. data/embulk-core/src/main/java/org/embulk/spi/ExecSession.java +1 -6
  12. data/embulk-core/src/main/java/org/embulk/spi/json/RubyValueApi.java +39 -1
  13. data/embulk-core/src/main/java/org/embulk/spi/time/Timestamp.java +21 -0
  14. data/embulk-core/src/main/java/org/embulk/spi/time/TimestampFormat.java +21 -0
  15. data/embulk-core/src/main/java/org/embulk/spi/util/DynamicColumnSetterFactory.java +43 -9
  16. data/embulk-core/src/main/java/org/embulk/spi/util/DynamicPageBuilder.java +46 -8
  17. data/embulk-core/src/main/java/org/embulk/spi/util/PagePrinter.java +19 -1
  18. data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/AbstractDynamicColumnSetter.java +11 -0
  19. data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/SkipColumnSetter.java +12 -1
  20. data/embulk-core/src/main/resources/embulk/parent_first_packages.properties +1 -0
  21. data/embulk-docs/build.gradle +8 -0
  22. data/embulk-docs/src/built-in.rst +47 -35
  23. data/embulk-docs/src/index.rst +9 -1
  24. data/embulk-docs/src/release.rst +1 -0
  25. data/embulk-docs/src/release/release-0.8.36.rst +32 -0
  26. data/embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java +22 -0
  27. data/embulk-standards/src/main/java/org/embulk/standards/CsvTokenizer.java +34 -1
  28. data/embulk-standards/src/main/java/org/embulk/standards/StdoutOutputPlugin.java +8 -2
  29. data/embulk-standards/src/test/java/org/embulk/standards/TestCsvTokenizer.java +76 -0
  30. data/lib/embulk/guess/schema_guess.rb +1 -1
  31. data/lib/embulk/input_plugin.rb +8 -1
  32. data/lib/embulk/page_builder.rb +38 -5
  33. data/lib/embulk/schema.rb +5 -6
  34. data/lib/embulk/version.rb +1 -1
  35. data/test/guess/test_schema_guess.rb +18 -0
  36. metadata +7 -6
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 6c1436731c89b017de39d0cfb4772b18246ea2c2
4
- data.tar.gz: b38e40ecb3a4966f36f882d38bed1cc640f837fd
3
+ metadata.gz: e53c57904a7874b17528ab6137cf1dbc74a807a5
4
+ data.tar.gz: 371ca4d97fbaaf8870167962d0ca196c282598b9
5
5
  SHA512:
6
- metadata.gz: e874d88576e38f489fbddc34bed57209250c2483eb585455fdd8e5a6a7c3492f52b17ab27f1b5adb7bc50b97a30f30e3507908390d57b673d988693c35d80ef0
7
- data.tar.gz: 7cd545fef7815c8d1cc334adca872b0763842d662032b9e1d5bcc11075d432bb5b40f16aefa8900d26fc91442c8f70a9da4f688b2dde5678ac2f6f27d7ff8efe
6
+ metadata.gz: b45b886e769a4b199cf5b67ec0be33e6bf087a94daecaba16367a94629a471d3fd299cf68e0016444a2362bee669841bd61c8a8dcf8a35a6d0ad25e07f2e2ced
7
+ data.tar.gz: 2044b9df066673c2deaa4f687ac03bbd0cf03ce3d85ca24f10480537657f357862b4599f5852fa516be71ea5cbded64667e66ec1b32b196bf503c1990d3141bc
data/README.md CHANGED
@@ -11,6 +11,10 @@ Embulk is a parallel bulk data loader that **helps data transfer between various
11
11
 
12
12
  Embulk documents: http://www.embulk.org/docs/
13
13
 
14
+ # Mailing list
15
+
16
+ * [Embulk-announce](https://groups.google.com/forum/#!forum/embulk-announce): Embulk core members post important updates such as **key releases**, **compatibility information**, and **feedback requests to users**.
17
+
14
18
  ## Quick Start
15
19
 
16
20
  ### Linux & Mac & BSD
data/build.gradle CHANGED
@@ -24,7 +24,7 @@ def release_projects = [project(":embulk-core"), project(":embulk-standards"), p
24
24
 
25
25
  allprojects {
26
26
  group = 'org.embulk'
27
- version = '0.8.35'
27
+ version = '0.8.36'
28
28
 
29
29
  ext {
30
30
  jrubyVersion = '9.1.13.0'
@@ -69,7 +69,11 @@ public class EmbulkExample
69
69
  StringBuilder ymlBuilder = new StringBuilder();
70
70
  ymlBuilder.append("in:\n");
71
71
  ymlBuilder.append(" type: file\n");
72
- ymlBuilder.append(String.format(" path_prefix: \"%s\"\n",
72
+
73
+ // Use single-quotes to quote path strings in YAML for Windows.
74
+ // Ref YAML spec: Single-Quoted Style
75
+ // http://yaml.org/spec/1.2/spec.html#id2788097
76
+ ymlBuilder.append(String.format(" path_prefix: \'%s\'\n",
73
77
  csvPath.toAbsolutePath().resolve("sample_").toString()));
74
78
  ymlBuilder.append("out:\n");
75
79
  ymlBuilder.append(" type: stdout\n");
@@ -72,6 +72,7 @@ public class EmbulkRun
72
72
  }
73
73
 
74
74
  printEmbulkVersionHeader(System.out);
75
+ printEmbulkGeneralNotifications(System.out);
75
76
 
76
77
  switch (subcommand) {
77
78
  case BUNDLE:
@@ -725,6 +726,17 @@ public class EmbulkRun
725
726
  out.println(now + ": Embulk v" + this.embulkVersion);
726
727
  }
727
728
 
729
+ private void printEmbulkGeneralNotifications(final PrintStream out)
730
+ {
731
+ out.println("");
732
+ out.println("********************************** INFORMATION **********************************");
733
+ out.println(" Join us! Embulk-announce mailing list is up for IMPORTANT annoucement such as");
734
+ out.println(" compatibility-breaking changes and key feature updates.");
735
+ out.println(" https://groups.google.com/forum/#!forum/embulk-announce");
736
+ out.println("*********************************************************************************");
737
+ out.println("");
738
+ }
739
+
728
740
  // TODO: Check if it is required to process JRuby options.
729
741
  private ScriptingContainer createLocalJRubyScriptingContainer()
730
742
  {
@@ -480,7 +480,7 @@ public class EmbulkRunner
480
480
  {
481
481
  final String yamlString = dumpDataSourceInYaml(modelObject);
482
482
  if (path != null) {
483
- Files.write(path, yamlString.getBytes(), StandardOpenOption.CREATE, StandardOpenOption.WRITE);
483
+ Files.write(path, yamlString.getBytes(), StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING);
484
484
  }
485
485
  return yamlString;
486
486
  }
@@ -490,7 +490,7 @@ public class EmbulkRunner
490
490
  {
491
491
  final String yamlString = dumpResumeStateInYaml(modelObject);
492
492
  if (path != null) {
493
- Files.write(path, yamlString.getBytes(), StandardOpenOption.CREATE, StandardOpenOption.WRITE);
493
+ Files.write(path, yamlString.getBytes(), StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING);
494
494
  }
495
495
  return yamlString;
496
496
  }
@@ -1,31 +1,71 @@
1
1
  package org.embulk.plugin;
2
2
 
3
- import java.util.List;
4
- import java.util.Collection;
5
- import java.util.Iterator;
6
- import java.util.ArrayList;
7
- import java.util.Enumeration;
8
- import java.io.IOException;
9
- import java.nio.file.Path;
10
- import java.net.URL;
11
- import java.net.URLClassLoader;
12
- import java.net.MalformedURLException;
13
3
  import com.google.common.base.Function;
14
4
  import com.google.common.collect.ImmutableList;
15
5
  import com.google.common.collect.Iterables;
16
6
  import com.google.common.collect.Iterators;
7
+ import java.io.ByteArrayOutputStream;
8
+ import java.io.InputStream;
9
+ import java.io.IOException;
10
+ import java.net.JarURLConnection;
11
+ import java.net.MalformedURLException;
12
+ import java.net.URL;
13
+ import java.net.URLClassLoader;
14
+ import java.net.URLConnection;
15
+ import java.net.URLStreamHandler;
16
+ import java.nio.file.Path;
17
+ import java.security.AccessControlContext;
18
+ import java.security.AccessController;
19
+ import java.security.CodeSigner;
20
+ import java.security.CodeSource;
21
+ import java.security.PrivilegedActionException;
22
+ import java.security.PrivilegedExceptionAction;
23
+ import java.util.ArrayList;
24
+ import java.util.Collection;
25
+ import java.util.Collections;
26
+ import java.util.Enumeration;
27
+ import java.util.Iterator;
28
+ import java.util.List;
29
+ import java.util.Vector;
30
+ import java.util.jar.Attributes;
31
+ import java.util.jar.JarEntry;
32
+ import java.util.jar.JarInputStream;
33
+ import java.util.jar.Manifest;
17
34
 
18
35
  public class PluginClassLoader
19
36
  extends URLClassLoader
20
37
  {
21
- private final List<String> parentFirstPackagePrefixes;
22
- private final List<String> parentFirstResourcePrefixes;
23
-
24
- public PluginClassLoader(Collection<URL> urls, ClassLoader parent,
25
- Collection<String> parentFirstPackages,
26
- Collection<String> parentFirstResources)
38
+ private PluginClassLoader(
39
+ final ClassLoader parentClassLoader,
40
+ final URL oneNestedJarFileUrl,
41
+ final Collection<String> embeddedJarPathsInNestedJar,
42
+ final Collection<URL> flatJarUrls,
43
+ final Collection<String> parentFirstPackages,
44
+ final Collection<String> parentFirstResources)
27
45
  {
28
- super(urls.toArray(new URL[urls.size()]), parent);
46
+ super(combineUrlsToArray(oneNestedJarFileUrl, flatJarUrls == null ? Collections.<URL>emptyList() : flatJarUrls),
47
+ parentClassLoader);
48
+
49
+ // Given |oneNestedJarFileUrl| should be "file:...". |this.oneNestedJarUrlBase| should be "jar:file:...".
50
+ URL oneNestedJarUrlBaseBuilt = null;
51
+ if (oneNestedJarFileUrl != null) {
52
+ try {
53
+ oneNestedJarUrlBaseBuilt = new URL("jar", "", -1, oneNestedJarFileUrl + "!/");
54
+ }
55
+ catch (MalformedURLException ex) {
56
+ // TODO: Notify this to reporters as far as possible.
57
+ System.err.println("FATAL: Invalid JAR file URL: " + oneNestedJarFileUrl.toString());
58
+ ex.printStackTrace();
59
+ }
60
+ }
61
+ this.oneNestedJarUrlBase = oneNestedJarUrlBaseBuilt;
62
+
63
+ if (embeddedJarPathsInNestedJar == null) {
64
+ this.embeddedJarPathsInNestedJar = Collections.<String>emptyList();
65
+ }
66
+ else {
67
+ this.embeddedJarPathsInNestedJar = Collections.unmodifiableCollection(embeddedJarPathsInNestedJar);
68
+ }
29
69
  this.parentFirstPackagePrefixes = ImmutableList.copyOf(
30
70
  Iterables.transform(parentFirstPackages, new Function<String, String>() {
31
71
  public String apply(String pkg)
@@ -40,6 +80,60 @@ public class PluginClassLoader
40
80
  return pkg + "/";
41
81
  }
42
82
  }));
83
+ this.accessControlContext = AccessController.getContext();
84
+ }
85
+
86
+ @Deprecated // Constructing directly with the constructor is deprecated (no warnings). Use static creator methods.
87
+ public PluginClassLoader(
88
+ final Collection<URL> flatJarUrls,
89
+ final ClassLoader parentClassLoader,
90
+ final Collection<String> parentFirstPackages,
91
+ final Collection<String> parentFirstResources)
92
+ {
93
+ this(parentClassLoader, null, null, flatJarUrls, parentFirstPackages, parentFirstResources);
94
+ }
95
+
96
+ /**
97
+ * Creates PluginClassLoader for plugins with dependency JARs flat on the file system, like Gem-based plugins.
98
+ */
99
+ public static PluginClassLoader createForFlatJars(
100
+ final ClassLoader parentClassLoader,
101
+ final Collection<URL> flatJarUrls,
102
+ final Collection<String> parentFirstPackages,
103
+ final Collection<String> parentFirstResources)
104
+ {
105
+ return new PluginClassLoader(
106
+ parentClassLoader,
107
+ null,
108
+ null,
109
+ flatJarUrls,
110
+ parentFirstPackages,
111
+ parentFirstResources);
112
+ }
113
+
114
+ /**
115
+ * Creates PluginClassLoader for plugins with dependency JARs embedded in the plugin JAR itself.
116
+ *
117
+ * @param parentClassLoader the parent ClassLoader of this PluginClassLoader instance
118
+ * @param oneNestedJarFileUrl "file:" URL of the plugin JAR file
119
+ * @param embeddedJarPathsInNestedJar collection of resource names of embedded dependency JARs in the plugin JAR
120
+ * @param parentFirstPackages collection of package names that are to be loaded first before the plugin's
121
+ * @param parentFirstResources collection of resource names that are to be loaded first before the plugin's
122
+ */
123
+ public static PluginClassLoader createForNestedJar(
124
+ final ClassLoader parentClassLoader,
125
+ final URL oneNestedJarFileUrl,
126
+ final Collection<String> embeddedJarPathsInNestedJar,
127
+ final Collection<String> parentFirstPackages,
128
+ final Collection<String> parentFirstResources)
129
+ {
130
+ return new PluginClassLoader(
131
+ parentClassLoader,
132
+ oneNestedJarFileUrl,
133
+ embeddedJarPathsInNestedJar,
134
+ null,
135
+ parentFirstPackages,
136
+ parentFirstResources);
43
137
  }
44
138
 
45
139
  /**
@@ -65,6 +159,62 @@ public class PluginClassLoader
65
159
  super.addURL(url);
66
160
  }
67
161
 
162
+ /**
163
+ * Finds a class defined by the given name from given JARs and JARs in the given JAR.
164
+ *
165
+ * Classes directly inthe given JARs are always prioritized. Only if no such a class is found
166
+ * directly in the given JAR, it tries to find the class in JARs in the given JAR.
167
+ */
168
+ @Override
169
+ protected Class<?> findClass(final String className)
170
+ throws ClassNotFoundException
171
+ {
172
+ if (this.oneNestedJarUrlBase == null || this.embeddedJarPathsInNestedJar.isEmpty()) {
173
+ // Multiple flat JARs -- Gem-based plugins, or Single JAR (JAR-based plugins) without any embedded JAR
174
+ return super.findClass(className);
175
+ }
176
+ else {
177
+ // Single nested JAR -- JAR-based plugins
178
+ try {
179
+ // Classes directly in the plugin JAR are always prioritized.
180
+ return super.findClass(className);
181
+ }
182
+ catch (ClassNotFoundException directClassNotFoundException) {
183
+ try {
184
+ return AccessController.doPrivileged(
185
+ new PrivilegedExceptionAction<Class<?>>() {
186
+ public Class<?> run()
187
+ throws ClassNotFoundException
188
+ {
189
+ try {
190
+ return defineClassFromEmbeddedJars(className);
191
+ }
192
+ catch (ClassNotFoundException | LinkageError | ClassCastException ex) {
193
+ throw ex;
194
+ }
195
+ catch (Throwable ex) {
196
+ // Resource found from JARs in the JAR, but failed to load it as a class.
197
+ throw new ClassNotFoundException(className, ex);
198
+ }
199
+ }
200
+ }, this.accessControlContext);
201
+ } catch (PrivilegedActionException ex) {
202
+ final Throwable internalException = ex.getException();
203
+ if (internalException instanceof ClassNotFoundException) {
204
+ throw (ClassNotFoundException) internalException;
205
+ }
206
+ if (internalException instanceof LinkageError) {
207
+ throw (LinkageError) internalException;
208
+ }
209
+ if (internalException instanceof ClassCastException) {
210
+ throw (ClassCastException) internalException;
211
+ }
212
+ throw new ClassNotFoundException(className, ex);
213
+ }
214
+ }
215
+ }
216
+ }
217
+
68
218
  /**
69
219
  * Loads the class with the specified binary name prioritized by the "parent-first" condition.
70
220
  *
@@ -135,6 +285,87 @@ public class PluginClassLoader
135
285
  return clazz;
136
286
  }
137
287
 
288
+ /**
289
+ * Finds a resource recognized as the given name from given JARs and JARs in the given JAR.
290
+ *
291
+ * Resources directly in the given JARs are always prioritized. Only if no such a resource is found
292
+ * directly in the given JAR, it tries to find the resource in JARs in the given JAR.
293
+ *
294
+ * Note that URLClassLoader#findResource is public while ClassLoader#findResource is protected.
295
+ */
296
+ @Override
297
+ public URL findResource(final String resourceName)
298
+ {
299
+ if (this.oneNestedJarUrlBase == null || this.embeddedJarPathsInNestedJar.isEmpty()) {
300
+ // Multiple flat JARs -- Gem-based plugins, or Single JAR (JAR-based plugins) without any embedded JAR
301
+ return super.findResource(resourceName);
302
+ }
303
+ else {
304
+ // Single nested JAR -- JAR-based plugins
305
+ // Classes directly in the plugin JAR are always prioritized.
306
+ final URL rootUrl = super.findResource(resourceName);
307
+ if (rootUrl != null) {
308
+ return rootUrl;
309
+ }
310
+
311
+ try {
312
+ return AccessController.doPrivileged(
313
+ new PrivilegedExceptionAction<URL>() {
314
+ public URL run()
315
+ {
316
+ return findResourceFromEmbeddedJars(resourceName);
317
+ }
318
+ }, this.accessControlContext);
319
+ } catch (PrivilegedActionException ignored) {
320
+ }
321
+
322
+ return null;
323
+ }
324
+ }
325
+
326
+ /**
327
+ * Finds resources recognized as the given name from given JARs and JARs in the given JAR.
328
+ *
329
+ * Resources directly in the given JARs precede. Resources in JARs in the given JAR follow resources
330
+ * directly in the given JARs.
331
+ *
332
+ * Note that URLClassLoader#findResources is public while ClassLoader#findResources is protected.
333
+ */
334
+ @Override
335
+ public Enumeration<URL> findResources(final String resourceName)
336
+ throws IOException
337
+ {
338
+ if (this.oneNestedJarUrlBase == null || this.embeddedJarPathsInNestedJar.isEmpty()) {
339
+ // Multiple flat JARs -- Gem-based plugins, or Single JAR (JAR-based plugins) without any embedded JAR
340
+ return super.findResources(resourceName);
341
+ }
342
+ else {
343
+ // Single nested JAR -- JAR-based plugins
344
+ final Vector<URL> urls = new Vector<URL>();
345
+
346
+ // Classes directly in the plugin JAR are always prioritized.
347
+ // Note that |super.findResources| may throw IOException.
348
+ for (final Enumeration<URL> rootUrls = super.findResources(resourceName); rootUrls.hasMoreElements(); ) {
349
+ urls.add(rootUrls.nextElement());
350
+ }
351
+
352
+ try {
353
+ final List<URL> childUrls = AccessController.doPrivileged(
354
+ new PrivilegedExceptionAction<List<URL>>() {
355
+ public List<URL> run()
356
+ throws IOException
357
+ {
358
+ return findResourcesFromEmbeddedJars(resourceName);
359
+ }
360
+ }, this.accessControlContext);
361
+ urls.addAll(childUrls);
362
+ } catch (PrivilegedActionException ignored) {
363
+ }
364
+
365
+ return urls.elements();
366
+ }
367
+ }
368
+
138
369
  @Override
139
370
  public URL getResource(String name)
140
371
  {
@@ -162,6 +393,33 @@ public class PluginClassLoader
162
393
  return null;
163
394
  }
164
395
 
396
+ @Override
397
+ public InputStream getResourceAsStream(final String resourceName)
398
+ {
399
+ final boolean childFirst = isParentFirstPath(resourceName);
400
+
401
+ if (childFirst) {
402
+ final InputStream childInputStream = getResourceAsStreamFromChild(resourceName);
403
+ if (childInputStream != null) {
404
+ return childInputStream;
405
+ }
406
+ }
407
+
408
+ final InputStream parentInputStream = getParent().getResourceAsStream(resourceName);
409
+ if (parentInputStream != null) {
410
+ return parentInputStream;
411
+ }
412
+
413
+ if (!childFirst) {
414
+ final InputStream childInputStream = getResourceAsStreamFromChild(resourceName);
415
+ if (childInputStream != null) {
416
+ return childInputStream;
417
+ }
418
+ }
419
+
420
+ return null;
421
+ }
422
+
165
423
  @Override
166
424
  public Enumeration<URL> getResources(String name)
167
425
  throws IOException
@@ -186,6 +444,527 @@ public class PluginClassLoader
186
444
  return Iterators.asEnumeration(Iterators.concat(resources.iterator()));
187
445
  }
188
446
 
447
+ /**
448
+ * URLStreamHandler to handle resources in embedded JARs in the plugin JAR.
449
+ */
450
+ private static class PluginClassURLStreamHandler
451
+ extends URLStreamHandler
452
+ {
453
+ public PluginClassURLStreamHandler(final String protocol)
454
+ {
455
+ this.protocol = protocol;
456
+ }
457
+
458
+ @Override
459
+ protected URLConnection openConnection(final URL url)
460
+ throws IOException
461
+ {
462
+ // Note that declaring variables here may cause unexpected behaviors.
463
+ // https://stackoverflow.com/questions/9952815/s3-java-client-fails-a-lot-with-premature-end-of-content-length-delimited-messa
464
+ return new URLConnection(url) {
465
+ @Override
466
+ public void connect()
467
+ {
468
+ }
469
+
470
+ @Override
471
+ public InputStream getInputStream()
472
+ throws IOException
473
+ {
474
+ final URL embulkPluginJarUrl = getURL();
475
+ if (!embulkPluginJarUrl.getProtocol().equals(protocol)) {
476
+ return null;
477
+ }
478
+ final String[] embulkPluginJarUrlSeparate = embulkPluginJarUrl.getPath().split("!!/");
479
+ if (embulkPluginJarUrlSeparate.length != 2) {
480
+ return null;
481
+ }
482
+ final URL embeddedJarUrl = new URL(embulkPluginJarUrlSeparate[0]);
483
+ final String embeddedResourceName = embulkPluginJarUrlSeparate[1];
484
+
485
+ final JarURLConnection embeddedJarURLConnection;
486
+ try {
487
+ final URLConnection urlConnection = embeddedJarUrl.openConnection();
488
+ embeddedJarURLConnection = (JarURLConnection) urlConnection;
489
+ }
490
+ catch (ClassCastException ex) {
491
+ return null;
492
+ }
493
+
494
+ final JarInputStream embeddedJarInputStream =
495
+ new JarInputStream(embeddedJarURLConnection.getInputStream());
496
+
497
+ // Note that |JarInputStream.getNextJarEntry| may throw IOException.
498
+ JarEntry jarEntry = embeddedJarInputStream.getNextJarEntry();
499
+ while (jarEntry != null) {
500
+ if (jarEntry.getName().equals(embeddedResourceName)) {
501
+ return embeddedJarInputStream; // The InputStream points the specific "JAR entry".
502
+ }
503
+ // Note that |JarInputStream.getNextJarEntry| may throw IOException.
504
+ jarEntry = embeddedJarInputStream.getNextJarEntry();
505
+ }
506
+ return null;
507
+ }
508
+ };
509
+ }
510
+
511
+ public final String protocol;
512
+ }
513
+
514
+ private static URL[] combineUrlsToArray(final URL oneNestedJarFileUrl, final Collection<URL> flatJarUrls)
515
+ {
516
+ final int offset;
517
+ final URL[] allDirectJarUrls;
518
+ if (oneNestedJarFileUrl == null) {
519
+ offset = 0;
520
+ allDirectJarUrls = new URL[flatJarUrls.size()];
521
+ }
522
+ else {
523
+ offset = 1;
524
+ allDirectJarUrls = new URL[flatJarUrls.size() + 1];
525
+ allDirectJarUrls[0] = oneNestedJarFileUrl;
526
+ }
527
+ int i = 0;
528
+ for (final URL flatJarUrl : flatJarUrls) {
529
+ allDirectJarUrls[i + offset] = flatJarUrl;
530
+ ++i;
531
+ }
532
+ return allDirectJarUrls;
533
+ }
534
+
535
+ /**
536
+ * Defines a class with given class name from JARs embedded in the plugin JAR.
537
+ *
538
+ * It tries to continue even if Exceptions are throws in one of embedded JARs so that
539
+ * it can find the target class without affected from other unrelated JARs.
540
+ */
541
+ private Class<?> defineClassFromEmbeddedJars(final String className)
542
+ throws ClassNotFoundException
543
+ {
544
+ final String classResourceName = className.replace('.', '/').concat(".class");
545
+
546
+ Throwable lastException = null;
547
+ // TODO: Speed up class loading by caching?
548
+ for (final String embeddedJarPath : this.embeddedJarPathsInNestedJar) {
549
+ final URL embeddedJarUrl;
550
+ final JarURLConnection embeddedJarUrlConnection;
551
+ final JarInputStream embeddedJarInputStream;
552
+ try {
553
+ embeddedJarUrl = getEmbeddedJarUrl(embeddedJarPath);
554
+ embeddedJarUrlConnection = getEmbeddedJarURLConnection(embeddedJarUrl);
555
+ embeddedJarInputStream = getEmbeddedJarInputStream(embeddedJarUrlConnection);
556
+ }
557
+ catch (IOException ex) {
558
+ lastException = ex;
559
+ continue;
560
+ }
561
+
562
+ final Manifest manifest;
563
+ try {
564
+ manifest = embeddedJarUrlConnection.getManifest();
565
+ }
566
+ catch (IOException ex) {
567
+ // TODO: Notify this to reporters as far as possible.
568
+ lastException = ex;
569
+ System.err.println("Failed to load manifest in embedded JAR: " + embeddedJarPath);
570
+ ex.printStackTrace();
571
+ continue;
572
+ }
573
+
574
+ JarEntry jarEntry;
575
+ try {
576
+ jarEntry = embeddedJarInputStream.getNextJarEntry();
577
+ }
578
+ catch (IOException ex) {
579
+ // TODO: Notify this to reporters as far as possible.
580
+ lastException = ex;
581
+ System.err.println("Failed to load entry in embedded JAR: " + embeddedJarPath);
582
+ ex.printStackTrace();
583
+ continue;
584
+ }
585
+ jarEntries: while (jarEntry != null) {
586
+ if (jarEntry.getName().equals(classResourceName)) {
587
+ final int lastDotIndexInClassName = className.lastIndexOf('.');
588
+ final String packageName;
589
+ if (lastDotIndexInClassName != -1) {
590
+ packageName = className.substring(0, lastDotIndexInClassName);
591
+ }
592
+ else {
593
+ packageName = null;
594
+ }
595
+
596
+ final URL codeSourceUrl = embeddedJarUrl;
597
+
598
+ // Define the package if the package is not loaded / defined yet.
599
+ if (packageName != null) {
600
+ // TODO: Consider package sealing.
601
+ // https://docs.oracle.com/javase/tutorial/deployment/jar/sealman.html
602
+ if (this.getPackage(packageName) == null) {
603
+ try {
604
+ final Attributes fileAttributes;
605
+ final Attributes mainAttributes;
606
+ if (manifest != null) {
607
+ fileAttributes = manifest.getAttributes(classResourceName);
608
+ mainAttributes = manifest.getMainAttributes();
609
+ }
610
+ else {
611
+ fileAttributes = null;
612
+ mainAttributes = null;
613
+ }
614
+
615
+ this.definePackage(
616
+ packageName,
617
+ getAttributeFromAttributes(mainAttributes, fileAttributes, classResourceName,
618
+ Attributes.Name.SPECIFICATION_TITLE),
619
+ getAttributeFromAttributes(mainAttributes, fileAttributes, classResourceName,
620
+ Attributes.Name.SPECIFICATION_VERSION),
621
+ getAttributeFromAttributes(mainAttributes, fileAttributes, classResourceName,
622
+ Attributes.Name.SPECIFICATION_VENDOR),
623
+ getAttributeFromAttributes(mainAttributes, fileAttributes, classResourceName,
624
+ Attributes.Name.IMPLEMENTATION_TITLE),
625
+ getAttributeFromAttributes(mainAttributes, fileAttributes, classResourceName,
626
+ Attributes.Name.IMPLEMENTATION_VERSION),
627
+ getAttributeFromAttributes(mainAttributes, fileAttributes, classResourceName,
628
+ Attributes.Name.IMPLEMENTATION_VENDOR),
629
+ null);
630
+ }
631
+ catch (IllegalArgumentException ex) {
632
+ // The package duplicates -- in parallel cases
633
+ if (getPackage(packageName) == null) {
634
+ // TODO: Notify this to reporters as far as possible.
635
+ lastException = ex;
636
+ System.err.println("FATAL: Should not happen. Package duplicated: " + packageName);
637
+ ex.printStackTrace();
638
+ continue;
639
+ }
640
+ }
641
+ }
642
+ }
643
+
644
+ final long classResourceSize = jarEntry.getSize();
645
+ final byte[] classResourceBytes;
646
+ final long actualSize;
647
+
648
+ if (classResourceSize > -1) { // JAR entry size available
649
+ classResourceBytes = new byte[(int) classResourceSize];
650
+ try {
651
+ actualSize = embeddedJarInputStream.read(classResourceBytes, 0, (int) classResourceSize);
652
+ }
653
+ catch (IOException ex) {
654
+ // TODO: Notify this to reporters as far as possible.
655
+ lastException = ex;
656
+ System.err.println("Failed to load entry in embedded JAR: " + classResourceName);
657
+ ex.printStackTrace();
658
+ break jarEntries; // Breaking from loading since this JAR looks broken.
659
+ }
660
+ if (actualSize != classResourceSize) {
661
+ // TODO: Notify this to reporters as far as possible.
662
+ System.err.println("Broken entry in embedded JAR: " + classResourceName);
663
+ break jarEntries; // Breaking from loading since this JAR looks broken.
664
+ }
665
+ }
666
+ else { // JAR entry size unavailable
667
+ final ByteArrayOutputStream bytesStream = new ByteArrayOutputStream();
668
+ final byte[] buffer = new byte[1024];
669
+ long accumulatedSize = 0;
670
+ while (true) {
671
+ final long readSize;
672
+ try {
673
+ readSize = embeddedJarInputStream.read(buffer, 0, 1024);
674
+ }
675
+ catch (IOException ex) {
676
+ // TODO: Notify this to reporters as far as possible.
677
+ lastException = ex;
678
+ System.err.println("Failed to load entry in embedded JAR: " + classResourceName);
679
+ ex.printStackTrace();
680
+ break jarEntries; // Breaking from loading since this JAR looks broken.
681
+ }
682
+
683
+ if (readSize < 0) {
684
+ break;
685
+ }
686
+
687
+ bytesStream.write(buffer, 0, (int) readSize);
688
+ accumulatedSize += readSize;
689
+ }
690
+ actualSize = accumulatedSize;
691
+ classResourceBytes = bytesStream.toByteArray();
692
+ }
693
+
694
+ final CodeSource codeSource = new CodeSource(codeSourceUrl, (CodeSigner[]) null);
695
+ final Class<?> definedClass;
696
+ try {
697
+ definedClass = defineClass(className, classResourceBytes, 0, (int) actualSize, codeSource);
698
+ }
699
+ catch (Throwable ex) {
700
+ // TODO: Notify this to reporters as far as possible.
701
+ lastException = ex;
702
+ System.err.println("Failed to load entry in embedded JAR: " + classResourceName);
703
+ ex.printStackTrace();
704
+ continue;
705
+ }
706
+ return definedClass;
707
+ }
708
+ try {
709
+ jarEntry = embeddedJarInputStream.getNextJarEntry();
710
+ }
711
+ catch (IOException ex) {
712
+ // TODO: Notify this to reporters as far as possible.
713
+ lastException = ex;
714
+ System.err.println("Failed to load entry in embedded JAR: " + classResourceName);
715
+ ex.printStackTrace();
716
+ break jarEntries; // Breaking from loading since this JAR looks broken.
717
+ }
718
+ }
719
+ }
720
+ if (lastException != null) {
721
+ throw new ClassNotFoundException(className, lastException);
722
+ }
723
+ else {
724
+ throw new ClassNotFoundException(className);
725
+ }
726
+ }
727
+
728
+ private InputStream getResourceAsStreamFromChild(final String resourceName)
729
+ {
730
+ if (this.oneNestedJarUrlBase == null || this.embeddedJarPathsInNestedJar.isEmpty()) {
731
+ // Multiple flat JARs -- Gem-based plugins, or Single JAR (JAR-based plugins) without any embedded JAR
732
+ return super.getResourceAsStream(resourceName);
733
+ }
734
+ else {
735
+ // Single nested JAR -- JAR-based plugins
736
+ // Resources directly in the plugin JAR are prioritized.
737
+ final InputStream inputStream = super.getResourceAsStream(resourceName);
738
+ if (inputStream == null) {
739
+ try {
740
+ final InputStream childInputStream = AccessController.doPrivileged(
741
+ new PrivilegedExceptionAction<InputStream>() {
742
+ public InputStream run()
743
+ {
744
+ return getResourceAsStreamFromEmbeddedJars(resourceName);
745
+ }
746
+ }, this.accessControlContext);
747
+ if (childInputStream != null) {
748
+ return childInputStream;
749
+ }
750
+ } catch (PrivilegedActionException ignored) {
751
+ }
752
+ }
753
+ }
754
+ return null;
755
+ }
756
+
757
+ private InputStream getResourceAsStreamFromEmbeddedJars(final String resourceName)
758
+ {
759
+ for (final String embeddedJarPath : this.embeddedJarPathsInNestedJar) {
760
+ final JarInputStream embeddedJarInputStream;
761
+ try {
762
+ embeddedJarInputStream = getEmbeddedJarInputStream(embeddedJarPath);
763
+ }
764
+ catch (IOException ex) {
765
+ // TODO: Notify this to reporters as far as possible.
766
+ System.err.println("Failed to load entry in embedded JAR: " + resourceName + " / " + embeddedJarPath);
767
+ ex.printStackTrace();
768
+ continue;
769
+ }
770
+
771
+ JarEntry jarEntry;
772
+ try {
773
+ jarEntry = embeddedJarInputStream.getNextJarEntry();
774
+ }
775
+ catch (IOException ex) {
776
+ // TODO: Notify this to reporters as far as possible.
777
+ System.err.println("Failed to load entry in embedded JAR: " + resourceName + " / " + embeddedJarPath);
778
+ ex.printStackTrace();
779
+ continue;
780
+ }
781
+ while (jarEntry != null) {
782
+ if (jarEntry.getName().equals(resourceName)) {
783
+ return embeddedJarInputStream; // Pointing the specific "JAR entry"
784
+ }
785
+ }
786
+ }
787
+ return null;
788
+ }
789
+
790
+ private URL findResourceFromEmbeddedJars(final String resourceName)
791
+ {
792
+ for (final String embeddedJarPath : this.embeddedJarPathsInNestedJar) {
793
+ final URL embeddedJarUrl;
794
+ final JarURLConnection embeddedJarUrlConnection;
795
+ final JarInputStream embeddedJarInputStream;
796
+ try {
797
+ embeddedJarUrl = getEmbeddedJarUrl(embeddedJarPath);
798
+ embeddedJarUrlConnection = getEmbeddedJarURLConnection(embeddedJarUrl);
799
+ embeddedJarInputStream = getEmbeddedJarInputStream(embeddedJarUrlConnection);
800
+ }
801
+ catch (IOException ex) {
802
+ // TODO: Notify this to reporters as far as possible.
803
+ System.err.println("Failed to load entry in embedded JAR: " +
804
+ resourceName + " / " + embeddedJarPath);
805
+ ex.printStackTrace();
806
+ continue;
807
+ }
808
+
809
+ JarEntry jarEntry;
810
+ try {
811
+ jarEntry = embeddedJarInputStream.getNextJarEntry();
812
+ }
813
+ catch (IOException ex) {
814
+ // TODO: Notify this to reporters as far as possible.
815
+ System.err.println("Failed to load entry in embedded JAR: " +
816
+ resourceName + " / " + embeddedJarPath);
817
+ ex.printStackTrace();
818
+ continue;
819
+ }
820
+ jarEntries: while (jarEntry != null) {
821
+ if (jarEntry.getName().equals(resourceName)) {
822
+ // For resources (not classes) in nested JARs, the schema and the URL should be like:
823
+ // "embulk-plugin-jar:jar:file://.../plugin.jar!/classpath/library.jar!!/org.library/resource.txt"
824
+ //
825
+ // The "embulk-plugin-jar" URL is processed with |PluginClassURLStreamHandler|.
826
+ // See also: https://www.ibm.com/developerworks/library/j-onejar/index.html
827
+ //
828
+ // The URL lives only in the JVM execution.
829
+ try {
830
+ // The URL lives only in the JVM execution.
831
+ return new URL("embulk-plugin-jar", "", -1, embeddedJarUrl + "!!/" + resourceName,
832
+ new PluginClassURLStreamHandler("embulk-plugin-jar"));
833
+ }
834
+ catch (MalformedURLException ex) {
835
+ // TODO: Notify this to reporters as far as possible.
836
+ System.err.println("Failed to load entry in embedded JAR: " +
837
+ resourceName + " / " + embeddedJarPath);
838
+ ex.printStackTrace();
839
+ break jarEntries;
840
+ }
841
+ }
842
+ try {
843
+ jarEntry = embeddedJarInputStream.getNextJarEntry();
844
+ }
845
+ catch (IOException ex) {
846
+ // TODO: Notify this to reporters as far as possible.
847
+ System.err.println("Failed to load entry in embedded JAR: " +
848
+ resourceName + " / " + embeddedJarPath);
849
+ ex.printStackTrace();
850
+ break jarEntries;
851
+ }
852
+ }
853
+ }
854
+ return null;
855
+ }
856
+
857
+ private List<URL> findResourcesFromEmbeddedJars(final String resourceName)
858
+ throws IOException
859
+ {
860
+ final ArrayList<URL> resourceUrls = new ArrayList<URL>();
861
+ for (final String embeddedJarPath : this.embeddedJarPathsInNestedJar) {
862
+ final URL embeddedJarUrl = getEmbeddedJarUrl(embeddedJarPath);
863
+ final JarURLConnection embeddedJarUrlConnection = getEmbeddedJarURLConnection(embeddedJarUrl);
864
+ final JarInputStream embeddedJarInputStream = getEmbeddedJarInputStream(embeddedJarUrlConnection);
865
+
866
+ // Note that |JarInputStream.getNextJarEntry| may throw IOException.
867
+ JarEntry jarEntry = embeddedJarInputStream.getNextJarEntry();
868
+ while (jarEntry != null) {
869
+ if (jarEntry.getName().equals(resourceName)) {
870
+ // For resources (not classes) in nested JARs, the schema and the URL should be like:
871
+ // "embulk-plugin-jar:jar:file://.../plugin.jar!/classpath/library.jar!/org.library/resource.txt"
872
+ //
873
+ // The "embulk-plugin-jar" URL is processed with |PluginClassURLStreamHandler|.
874
+ // See also: https://www.ibm.com/developerworks/library/j-onejar/index.html
875
+ //
876
+ // The URL lives only in the JVM execution.
877
+ //
878
+ // Note that |new URL| may throw MalformedURLException (extending IOException).
879
+ resourceUrls.add(new URL("embulk-plugin-jar", "", -1, embeddedJarUrl + "!!/" + resourceName,
880
+ new PluginClassURLStreamHandler("embulk-plugin-jar")));
881
+ }
882
+ // Note that |JarInputStream.getNextJarEntry| may throw IOException.
883
+ jarEntry = embeddedJarInputStream.getNextJarEntry();
884
+ }
885
+ }
886
+ return resourceUrls;
887
+ }
888
+
889
+ private URL getEmbeddedJarUrl(final String embeddedJarPath)
890
+ throws MalformedURLException
891
+ {
892
+ final URL embeddedJarUrl;
893
+ try {
894
+ embeddedJarUrl = new URL(this.oneNestedJarUrlBase, embeddedJarPath);
895
+ }
896
+ catch (MalformedURLException ex) {
897
+ // TODO: Notify this to reporters as far as possible.
898
+ System.err.println("Failed to load entry in embedded JAR: " + embeddedJarPath);
899
+ ex.printStackTrace();
900
+ throw ex;
901
+ }
902
+ return embeddedJarUrl;
903
+ }
904
+
905
+ private JarURLConnection getEmbeddedJarURLConnection(final URL embeddedJarUrl)
906
+ throws IOException
907
+ {
908
+ final JarURLConnection embeddedJarURLConnection;
909
+ try {
910
+ final URLConnection urlConnection = embeddedJarUrl.openConnection();
911
+ embeddedJarURLConnection = (JarURLConnection) urlConnection;
912
+ }
913
+ catch (IOException ex) {
914
+ // TODO: Notify this to reporters as far as possible.
915
+ System.err.println("Failed to load entry in embedded JAR: " + embeddedJarUrl.toString());
916
+ ex.printStackTrace();
917
+ throw ex;
918
+ }
919
+ return embeddedJarURLConnection;
920
+ }
921
+
922
+ private JarInputStream getEmbeddedJarInputStream(final JarURLConnection embeddedJarURLConnection)
923
+ throws IOException
924
+ {
925
+ final JarInputStream embeddedJarInputStream;
926
+ try {
927
+ final InputStream inputStream = embeddedJarURLConnection.getInputStream();
928
+ embeddedJarInputStream = new JarInputStream(inputStream);
929
+ }
930
+ catch (IOException ex) {
931
+ // TODO: Notify this to reporters as far as possible.
932
+ System.err.println("Failed to load entry in embedded JAR: " + embeddedJarURLConnection.toString());
933
+ ex.printStackTrace();
934
+ throw ex;
935
+ }
936
+ return embeddedJarInputStream;
937
+ }
938
+
939
+ private JarInputStream getEmbeddedJarInputStream(final String embeddedJarPath)
940
+ throws IOException
941
+ {
942
+ final URL embeddedJarUrl = getEmbeddedJarUrl(embeddedJarPath);
943
+ final JarURLConnection embeddedJarUrlConnection = getEmbeddedJarURLConnection(embeddedJarUrl);
944
+ return getEmbeddedJarInputStream(embeddedJarUrlConnection);
945
+ }
946
+
947
+ private static String getAttributeFromAttributes(
948
+ final Attributes mainAttributes,
949
+ final Attributes fileAttributes,
950
+ final String classResourceName,
951
+ final Attributes.Name attributeName)
952
+ {
953
+ if (fileAttributes != null) {
954
+ final String value = fileAttributes.getValue(attributeName);
955
+ if (value != null) {
956
+ return value;
957
+ }
958
+ }
959
+ if (mainAttributes != null) {
960
+ final String value = mainAttributes.getValue(attributeName);
961
+ if (value != null) {
962
+ return value;
963
+ }
964
+ }
965
+ return null;
966
+ }
967
+
189
968
  private boolean isParentFirstPackage(String name)
190
969
  {
191
970
  for (String pkg : parentFirstPackagePrefixes) {
@@ -205,4 +984,10 @@ public class PluginClassLoader
205
984
  }
206
985
  return false;
207
986
  }
987
+
988
+ private final URL oneNestedJarUrlBase;
989
+ private final Collection<String> embeddedJarPathsInNestedJar;
990
+ private final List<String> parentFirstPackagePrefixes;
991
+ private final List<String> parentFirstResourcePrefixes;
992
+ private final AccessControlContext accessControlContext;
208
993
  }