embulk-output-oracle 0.2.1 → 0.2.2

data/src/main/java/org/embulk/output/oracle/oci/TableDefinition.java

@@ -0,0 +1,24 @@
+package org.embulk.output.oracle.oci;
+
+import java.util.List;
+
+public class TableDefinition
+{
+
+    public final String tableName;
+    public final short charsetId;
+    public final ColumnDefinition[] columns;
+
+
+    public TableDefinition(String tableName, short charsetId, ColumnDefinition... columns)
+    {
+        this.tableName = tableName;
+        this.charsetId = charsetId;
+        this.columns = columns;
+    }
+
+    public TableDefinition(String tableName, short charsetId, List<ColumnDefinition> columns)
+    {
+        this(tableName, charsetId, columns.toArray(new ColumnDefinition[columns.size()]));
+    }
+}

data/src/main/java/org/embulk/output/oracle/setter/OracleColumnSetterFactory.java

@@ -12,20 +12,20 @@ import org.embulk.spi.time.TimestampFormatter;
 
 public class OracleColumnSetterFactory extends ColumnSetterFactory
 {
-	public OracleColumnSetterFactory(BatchInsert batch, PageReader pageReader,
-			TimestampFormatter timestampFormatter)
-	{
-		super(batch, pageReader, timestampFormatter);
-	}
-	
-	@Override
-	public ColumnSetter newColumnSetter(JdbcColumn column)
+    public OracleColumnSetterFactory(BatchInsert batch, PageReader pageReader,
+            TimestampFormatter timestampFormatter)
     {
-		switch (column.getSqlType()) {
-			case Types.DECIMAL:
-	            return new StringColumnSetter(batch, pageReader, column, timestampFormatter);
+        super(batch, pageReader, timestampFormatter);
+    }
+    
+    @Override
+    public ColumnSetter newColumnSetter(JdbcColumn column)
+    {
+        switch (column.getSqlType()) {
+            case Types.DECIMAL:
+                return new StringColumnSetter(batch, pageReader, column, timestampFormatter);
             default:
-        		return super.newColumnSetter(column);
-		}
-	}
+                return super.newColumnSetter(column);
+        }
+    }
 }

data/src/test/cpp/common/embulk-output-oracle-test.cpp

@@ -0,0 +1,69 @@
+#include <stdio.h>
+#include <string.h>
+#include <dir-path-load.h>
+
+
+static int test(EMBULK_OUTPUT_ORACLE_OCI_CONTEXT *context, const char *db, const char *user, const char *pass, const char *csvFileName)
+{
+	if (embulk_output_oracle_prepareDirPathCtx(context, db, user, pass)) {
+		return OCI_ERROR;
+	}
+
+	EMBULK_OUTPUT_ORACLE_OCI_COL_DEF colDefs[] = {
+		{"ID", SQLT_INT, 4},
+		//{"ID", SQLT_CHR, 8},
+		{"NUM", SQLT_INT, 4},
+		//{"NUM", SQLT_CHR, 12},
+		{"VALUE1", SQLT_CHR, 60},
+		{"VALUE2", SQLT_CHR, 60},
+		{"VALUE3", SQLT_CHR, 60},
+		{"VALUE4", SQLT_CHR, 60},
+		{"VALUE5", SQLT_CHR, 60},
+		{"VALUE6", SQLT_CHR, 60},
+		{"VALUE7", SQLT_CHR, 60},
+		{"VALUE8", SQLT_CHR, 60},
+		{"VALUE9", SQLT_CHR, 60},
+		{"VALUE10", SQLT_CHR, 60},
+		{NULL, 0, 0}
+	};
+	if (embulk_output_oracle_prepareDirPathStream(context, "EXAMPLE", 832, colDefs)) {
+		return OCI_ERROR;
+	}
+
+	if (embulk_output_oracle_loadCSV(context, colDefs, csvFileName)) {
+		return OCI_ERROR;
+	}
+
+	if (embulk_output_oracle_commitDirPath(context)) {
+		return OCI_ERROR;
+	}
+
+	return OCI_SUCCESS;
+}
+
+
+int main(int argc, char* argv[])
+{
+	sword major_version;
+    sword minor_version; 
+    sword update_num;
+    sword patch_num;
+    sword port_update_num;
+	OCIClientVersion(&major_version, &minor_version, &update_num, &patch_num, &port_update_num);
+	printf("OCI client version = %d.%d.%d.%d.%d\r\n", major_version, minor_version, update_num, patch_num, port_update_num);
+
+	if (argc < 5) {
+		printf("embulk-output-oracle-test <db> <user> <password> <csv file name>\r\n");
+		return OCI_ERROR;
+	}
+
+	EMBULK_OUTPUT_ORACLE_OCI_CONTEXT context;
+	memset(&context, 0, sizeof(EMBULK_OUTPUT_ORACLE_OCI_CONTEXT));
+	int result = test(&context, argv[1], argv[2], argv[3], argv[4]);
+	if (result == OCI_ERROR) {
+		printf("%s\r\n", context.message);
+	}
+	embulk_output_oracle_freeDirPathHandles(&context);
+	return result;
+}
+

data/src/test/cpp/linux/build.sh

@@ -0,0 +1,19 @@
+# gcc, g++ and Oracle Instant Client SDK are requred.
+#
+# ln libocci.so.x.x libocci.so
+# ln libclntsh.so.x.x libclntsh.so
+#
+
+if [ "$JAVA_HOME" = "" ]
+then
+    echo "You should set the environment variable 'JAVA_HOME'."
+    exit 1
+fi
+
+if [ "$OCI_PATH" = "" ]
+then
+    echo "You should set the environment variable 'OCI_PATH'."
+    exit 1
+fi
+
+gcc -I. -I"$JAVA_HOME/include" -I"$OCI_PATH/sdk/include" -I../../../main/cpp/common -L"$OCI_PATH" ../common/embulk-output-oracle-test.cpp ../../../main/cpp/common/embulk-output-oracle.cpp ../../../main/cpp/common/dir-path-load.cpp -locci -lclntsh -lstdc++ -o embulk-output-oracle-test

data/src/test/cpp/win/build.bat

@@ -0,0 +1,29 @@
+@ECHO OFF
+
+REM You should set the environment variable 'PATH' to CL.exe(x86_amd64) of Visual Studio.
+
+IF "%JAVA_HOME%" == "" (
+    ECHO "You should set the environment variable 'JAVA_HOME'."
+    EXIT /B 1
+)
+
+
+IF "%OCI_SDK_PATH%" == "" (
+    ECHO "You should set the environment variable 'OCI_SDK_PATH'."
+    EXIT /B 1
+)
+
+IF "%MSVC_PATH%" == "" (
+    ECHO "You should set the environment variable 'MSVC_PATH'."
+    ECHO "For example : SET MSVC_PATH=C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC"
+    EXIT /B 1
+)
+
+IF "%MSSDK_PATH%" == "" (
+    ECHO "You should set the environment variable 'MSSDK_PATH'."
+    ECHO "For example : SET MSSDK_PATH=C:\Program Files (x86)\Microsoft SDKs\Windows\v7.0A"
+    EXIT /B 1
+)
+
+
+CL /I"..\..\..\main\cpp\common" /I"%MSSDK_PATH%\Include" /I"%MSVC_PATH%\include" /I"%OCI_SDK_PATH%\include" /I"%JAVA_HOME%\include" /I"%JAVA_HOME%\include\win32"  /Zi /nologo /W3 /WX- /O2 /Oi /GL /D "WIN32" /D "NDEBUG" /D "_WINDOWS" /D "_USRDLL" /D "EMBULKOUTPUTORACLE_EXPORTS" /D "_WINDLL" /D "_UNICODE" /D "UNICODE" /Gm- /EHsc /GS /Gy /fp:precise /Zc:wchar_t /Zc:forScope /Gd /errorReport:queue ..\common\embulk-output-oracle-test.cpp ..\..\..\main\cpp\common\dir-path-load.cpp /link /LIBPATH:"%MSVC_PATH%\lib\amd64" /LIBPATH:"%MSSDK_PATH%\Lib\x64" /LIBPATH:"%OCI_SDK_PATH%\lib\msvc"   /INCREMENTAL:NO /NOLOGO /LIBPATH:"C:\Oracle\instantclient_12_1\x64\sdk\lib\msvc" "oci.lib" "kernel32.lib" "user32.lib" "gdi32.lib" "winspool.lib" "comdlg32.lib" "advapi32.lib" "shell32.lib" "ole32.lib" "oleaut32.lib" "uuid.lib" "odbc32.lib" "odbccp32.lib" /ALLOWISOLATION /SUBSYSTEM:CONSOLE /OPT:REF /OPT:ICF /LTCG /TLBID:1 /DYNAMICBASE /NXCOMPAT /MACHINE:X64 /ERRORREPORT:QUEUE 

data/src/test/cpp/win/embulk-output-oracle-test.vcxproj

@@ -0,0 +1,155 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|Win32">
+      <Configuration>Debug</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|Win32">
+      <Configuration>Release</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <ItemGroup>
+    <ClInclude Include="..\..\..\main\cpp\common\dir-path-load.h" />
+  </ItemGroup>
+  <ItemGroup>
+    <ClCompile Include="..\..\..\main\cpp\common\dir-path-load.cpp" />
+    <ClCompile Include="..\common\embulk-output-oracle-test.cpp" />
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{C14B433B-122A-4BDB-BF50-0673A57E5716}</ProjectGuid>
+    <Keyword>Win32Proj</Keyword>
+    <RootNamespace>embulkoutputoracletest</RootNamespace>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <CharacterSet>Unicode</CharacterSet>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <CharacterSet>Unicode</CharacterSet>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <LinkIncremental>true</LinkIncremental>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <LinkIncremental>true</LinkIncremental>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <LinkIncremental>false</LinkIncremental>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <LinkIncremental>false</LinkIncremental>
+  </PropertyGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <ClCompile>
+      <PrecompiledHeader>NotUsing</PrecompiledHeader>
+      <WarningLevel>Level3</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>$(OCI_SDK_PATH)\include;..\..\..\main\cpp\common</AdditionalIncludeDirectories>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <AdditionalLibraryDirectories>$(OCI_SDK_PATH)\lib\msvc</AdditionalLibraryDirectories>
+      <AdditionalDependencies>oci.lib;%(AdditionalDependencies)</AdditionalDependencies>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <ClCompile>
+      <PrecompiledHeader>NotUsing</PrecompiledHeader>
+      <WarningLevel>Level3</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>$(OCI_SDK_PATH)\include;..\..\..\main\cpp\common</AdditionalIncludeDirectories>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <AdditionalLibraryDirectories>$(OCI_SDK_PATH)\lib\msvc</AdditionalLibraryDirectories>
+      <AdditionalDependencies>oci.lib;%(AdditionalDependencies)</AdditionalDependencies>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <PrecompiledHeader>NotUsing</PrecompiledHeader>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>$(OCI_SDK_PATH)\include;..\..\..\main\cpp\common</AdditionalIncludeDirectories>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+      <AdditionalLibraryDirectories>$(OCI_SDK_PATH)\lib\msvc</AdditionalLibraryDirectories>
+      <AdditionalDependencies>oci.lib;%(AdditionalDependencies)</AdditionalDependencies>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <PrecompiledHeader>NotUsing</PrecompiledHeader>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>$(OCI_SDK_PATH)\include;..\..\..\main\cpp\common</AdditionalIncludeDirectories>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+      <AdditionalLibraryDirectories>$(OCI_SDK_PATH)\lib\msvc</AdditionalLibraryDirectories>
+      <AdditionalDependencies>oci.lib;%(AdditionalDependencies)</AdditionalDependencies>
+    </Link>
+  </ItemDefinitionGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+  </ImportGroup>
+</Project>

data/src/test/java/org/embulk/input/filesplit/LocalFileSplitInputPlugin.java

@@ -0,0 +1,187 @@
+package org.embulk.input.filesplit;
+
+import java.io.BufferedInputStream;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.SequenceInputStream;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.embulk.config.CommitReport;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigInject;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.Task;
+import org.embulk.config.TaskSource;
+import org.embulk.spi.BufferAllocator;
+import org.embulk.spi.Exec;
+import org.embulk.spi.FileInputPlugin;
+import org.embulk.spi.TransactionalFileInput;
+import org.embulk.spi.util.InputStreamFileInput;
+
+import com.google.common.base.Optional;
+
+
+public class LocalFileSplitInputPlugin
+        implements FileInputPlugin
+{
+	public interface PluginTask
+            extends Task
+    {
+        @Config("path")
+        public String getPath();
+        
+        @Config("tasks")
+        @ConfigDefault("null")
+        public Optional<Integer> getTasks();
+        
+        @Config("header_line")
+        @ConfigDefault("false")
+        public boolean getHeaderLine();
+
+        public List<PartialFile> getFiles();
+        public void setFiles(List<PartialFile> files);
+
+        @ConfigInject
+        public BufferAllocator getBufferAllocator();
+    }
+
+    @Override
+    public ConfigDiff transaction(ConfigSource config, FileInputPlugin.Control control)
+    {
+        PluginTask task = config.loadConfig(PluginTask.class);
+
+        int tasks;
+        if (task.getTasks().isPresent()) {
+        	tasks = task.getTasks().get();
+        	if (tasks <= 0) {
+        		throw new IllegalArgumentException(String.format("'tasks' is %d but must be greater than 0", tasks));
+        	}
+        } else {
+        	tasks = Runtime.getRuntime().availableProcessors() * 2;
+        }
+
+        long size = new File(task.getPath()).length();
+        List<PartialFile> files = new ArrayList<PartialFile>();
+        for (int i = 0; i < tasks; i++) {
+        	long start = size * i / tasks;
+        	long end = size * (i + 1) / tasks;
+        	if (start < end) {
+        		files.add(new PartialFile(task.getPath(), start, end));
+        	}
+        }
+        
+        task.setFiles(files);
+
+        return resume(task.dump(), task.getFiles().size(), control);
+    }
+
+    @Override
+    public ConfigDiff resume(TaskSource taskSource,
+            int taskCount,
+            FileInputPlugin.Control control)
+    {
+        control.run(taskSource, taskCount);
+
+        return Exec.newConfigDiff();
+    }
+
+    @Override
+    public void cleanup(TaskSource taskSource,
+            int taskCount,
+            List<CommitReport> successCommitReports)
+    { }
+
+    @Override
+    public TransactionalFileInput open(TaskSource taskSource, int taskIndex)
+    {
+        PluginTask task = taskSource.loadTask(PluginTask.class);
+        return new LocalFileSplitInput(task, taskIndex);
+    }
+
+    public static class LocalFileSplitInput
+            extends InputStreamFileInput
+            implements TransactionalFileInput
+    {
+        public static class FileSplitProvider
+                implements InputStreamFileInput.Provider
+        {
+            private final PartialFile file;
+            private final boolean hasHeader;
+            private boolean opened = false;
+
+            public FileSplitProvider(PartialFile file, boolean hasHeader)
+            {
+                this.file = file;
+                this.hasHeader = hasHeader;
+            }
+
+            @Override
+            public InputStream openNext() throws IOException
+            {
+                if (opened) {
+                    return null;
+                }
+                opened = true;
+                
+                InputStream in = new PartialFileInputStream(new FileInputStream(file.getPath()), file.getStart(), file.getEnd());
+                if (file.getStart() > 0 && hasHeader) {
+                	in = new SequenceInputStream(openHeader(file.getPath()), in);
+                }
+                return in;
+            }
+
+            @Override
+            public void close() { }
+            
+            private InputStream openHeader(String path) throws IOException
+            {
+            	ByteArrayOutputStream header = new ByteArrayOutputStream();
+            	try (BufferedInputStream in = new BufferedInputStream(new FileInputStream(path))) {
+            		while (true) {
+            			int c = in.read();
+            			if (c < 0) {
+            				break;
+            			}
+            			
+        				header.write(c);
+        				
+            			if (c == '\n') {
+            				break;
+            			}
+            			
+            			if (c == '\r') {
+            				int c2 = in.read();
+            				if (c2 == '\n') {
+            					header.write(c2);
+            				}
+            				break;
+            			}
+            		}
+            	}
+            	header.close();
+            	return new ByteArrayInputStream(header.toByteArray());
+            }
+        }
+
+        public LocalFileSplitInput(PluginTask task, int taskIndex)
+        {
+            super(task.getBufferAllocator(), new FileSplitProvider(task.getFiles().get(taskIndex), task.getHeaderLine()));
+        }
+
+        @Override
+        public void abort() { }
+
+        @Override
+        public CommitReport commit()
+        {
+            return Exec.newCommitReport();
+        }
+    }
+}