embulk-output-postgres-json 0.2.0

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    YWFmMWQ2YjM0ZTRiMzRmY2IxMjdlYmU5ZWQyOTFlOGZkNjEwNDY5Mg==
+  data.tar.gz: !binary |-
+    MzliNjcwNjk5NTZkM2UyYzgzMzZlOTc1ZjY4ZWIwYmQ4NjQzOTM1NQ==
+SHA512:
+  metadata.gz: !binary |-
+    ZjU2MjE4YmE4ZmQ4Mzg0ZTc0YzY5ZGNlYTc4MTI2NTBiMDIwMzZhNTkyMzBh
+    OTkzMjUwMGJkOTQ3ZDlhZDQ2NmY0MmNiYTcyYjNlNjgyMWU4ZWI1Y2I5ZDE0
+    MWZkOWViMDdkZmIzYTgzMmFhZmUwOTE0OTBjNzM0ZDdmMGExYWQ=
+  data.tar.gz: !binary |-
+    ODU2ZmExMTIwODYyMmE2OGZmYWM2MjRiYjI3M2Q2YzNmNTY5MGM1M2Q5YmNl
+    ZDRlNzIzOWMyYTcwOGI2YzY2YzcxOTI0MzQwYmIxMjQ3ZmEzM2Q0YzZiODg4
+    ZGE4MjA3NjI5MDY5ZDE2Y2E0NjgwZDU5YzJmN2Y0NjU5ZmFjZmE=

data/ChangeLog

@@ -0,0 +1,5 @@
+
+2015-01-26 version 0.1.0:
+
+* The first release
+

data/Gemfile

@@ -0,0 +1,2 @@
+source "http://rubygems.org"
+gemspec

data/Gemfile.lock

@@ -0,0 +1,19 @@
+PATH
+  remote: .
+  specs:
+    embulk-plugin-postgres-json (0.1.0)
+      jdbc-postgres (>= 3.2.0)
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    jdbc-postgres (9.3.1102)
+    rake (10.4.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.0)
+  embulk-plugin-postgres-json!
+  rake (>= 0.9.2)

data/README.md

@@ -0,0 +1,33 @@
+# Embulk output plugin for PostgreSQL json column
+
+This [Embulk](https://github.com/embulk/embulk) output plugin writes records to a json column of a table.
+
+This works as following:
+
+1. transaction begin: creates a temporary table
+2. run: insert records to the temporary table
+3. transaction commit: copy the records to the actual table and drop the temporary table
+
+## Configuration
+
+- **host** host name of the PostgreSQL server (string, required)
+- **port** port of the PostgreSQL server (integer, default: 5432)
+- **username** login user name (string, required)
+- **password** login password (string, default: "")
+- **database** destination database name (string, required)
+- **table** destination table name (string, required)
+- **column** destination column name (string, default: "json")
+- **column_type** json or jsonb (string, default: 'json')
+
+### Example
+
+```yaml
+out:
+  type: postgres_json
+  host: localhost
+  port: 5432
+  username: pg
+  database: embulk_test
+  table: load01
+```
+

data/Rakefile

@@ -0,0 +1,4 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+task :default => [:build]

data/embulk-plugin-postgres-json.gemspec

@@ -0,0 +1,21 @@
+
+Gem::Specification.new do |gem|
+  gem.name          = "embulk-output-postgres-json"
+  gem.version       = "0.2.0"
+
+  gem.summary       = %q{Embulk output for PostgreSQL json and jsonb column}
+  gem.description   = gem.summary
+  gem.authors       = ["Sadayuki Furuhashi"]
+  gem.email         = ["frsyuki@gmail.com"]
+  gem.license       = "Apache 2.0"
+  gem.homepage      = "https://github.com/frsyuki/embulk-output-postgres-json"
+
+  gem.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+  gem.has_rdoc      = false
+
+  gem.add_dependency 'jdbc-postgres', ['>= 3.2.0']
+  gem.add_development_dependency 'bundler', ['~> 1.0']
+  gem.add_development_dependency 'rake', ['>= 0.9.2']
+end

data/lib/embulk/output/postgres_json.rb

@@ -0,0 +1,109 @@
+module Embulk
+  require 'jdbc/postgres'
+  Jdbc::Postgres.load_driver
+
+  class PostgresJsonOutputPlugin < OutputPlugin
+    Plugin.register_output('postgres_json', self)
+
+    def self.transaction(config, schema, processor_count, &control)
+      task = {
+        'host' => config.param('host', :string),
+        'port' => config.param('port', :integer, default: 5432),
+        'username' => config.param('username', :string),
+        'password' => config.param('password', :string, default: ''),
+        'database' => config.param('database', :string),
+        'table' => config.param('table', :string),
+        'column' => config.param('column', :string, default: 'json'),
+        'column_type' => config.param('column_type', :string, default: 'json'),
+      }
+
+      now = Time.now
+      unique_name = "%08x%08x" % [now.tv_sec, now.tv_nsec]  # TODO add org.embulk.spi.ExecSession.getTransactionUniqueName() method
+      task['temp_table'] = "#{task['table']}_LOAD_TEMP_#{unique_name}"
+
+      connect(task) do |pg|
+        # drop table if exists "DEST"
+        # create table if exists "TEMP" ("COL" json)
+        execute_sql(pg, %[drop table if exists "#{task['temp_table']}"])
+        execute_sql(pg, %[create table "#{task['temp_table']}" ("#{task['column']}" #{task['column_type']})])
+      end
+
+      begin
+        yield(task)
+
+        connect(task) do |pg|
+          # create table if not exists "DEST" ("COL" json)
+          # insert into "DEST" ("COL") select "COL" from "TEMP"
+          execute_sql(pg, %[create table if not exists "#{task['table']}" ("#{task['column']}" #{task['column_type']})])
+          execute_sql(pg, %[insert into "#{task['table']}" ("#{task['column']}") select "#{task['column']}" from "#{task['temp_table']}"])
+        end
+
+      ensure
+        connect(task) do |pg|
+          # drop table if exists TEMP
+          execute_sql(pg, %[drop table if exists "#{task['temp_table']}"])
+        end
+      end
+
+      return {}
+    end
+
+    def self.connect(task)
+      url = "jdbc:postgresql://#{task['host']}:#{task['port']}/#{task['database']}"
+      props = java.util.Properties.new
+      props.put("user", task['username'])
+      props.put("password", task['password'])
+
+      pg = org.postgresql.Driver.new.connect(url, props)
+      if block_given?
+        begin
+          yield pg
+        ensure
+          pg.close
+        end
+      end
+      pg
+    end
+
+    def self.execute_sql(pg, sql, *args)
+      stmt = pg.createStatement
+      begin
+        stmt.execute(sql)
+      ensure
+        stmt.close
+      end
+    end
+
+    def initialize(task, schema, index)
+      super
+      @pg = self.class.connect(task)
+    end
+
+    def close
+      @pg.close
+    end
+
+    def add(page)
+      prep = @pg.prepareStatement(%[insert into "#{@task['temp_table']}" (#{@task['column']}) values (?::#{@task['column_type']})])
+      begin
+        page.each do |record|
+          prep.setString(1, record.to_json)
+          prep.execute
+        end
+      ensure
+        prep.close
+      end
+    end
+
+    def finish
+    end
+
+    def abort
+    end
+
+    def commit
+      {}
+    end
+  end
+
+end

metadata

@@ -0,0 +1,93 @@
+--- !ruby/object:Gem::Specification
+name: embulk-output-postgres-json
+version: !ruby/object:Gem::Version
+  version: 0.2.0
+platform: ruby
+authors:
+- Sadayuki Furuhashi
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-02-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: jdbc-postgres
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 3.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 3.2.0
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+description: Embulk output for PostgreSQL json and jsonb column
+email:
+- frsyuki@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ChangeLog
+- Gemfile
+- Gemfile.lock
+- README.md
+- Rakefile
+- embulk-plugin-postgres-json.gemspec
+- lib/embulk/output/postgres_json.rb
+homepage: https://github.com/frsyuki/embulk-output-postgres-json
+licenses:
+- Apache 2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Embulk output for PostgreSQL json and jsonb column
+test_files: []