samidare 0.0.9 → 0.0.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ec15b952226e1fe811ab80393aa6e23d078cbf0d
-  data.tar.gz: ea9f9a470b9c13b05891bc7c5672b36c1d1abf68
+  metadata.gz: a515fd2312967aab172fba7b511223f40bc85f93
+  data.tar.gz: 344ef5f724464bb45cd6a858e99958978d615922
 SHA512:
-  metadata.gz: e1cb9d8ffacf52b9673ca9f6f192650c7202e8593aca20b2474687867326f9dcc6e8e417ad6733a43cd0c34a155dd26874eba9da0e48ba251fcb5aa3bee0807c
-  data.tar.gz: e4c0e30a4e533f51d0a54cb8fc41f332b16a0dbf4868470d897967fb64a39e0cf93a698fc93ad64669880cc3953e05ccff5abff6ef6131b73ba58ac01f13d9f3
+  metadata.gz: b90e3598957f6d64a1a8e24ff3bc8a207267dd116a062c0d84b6b555934b7ca77357d4854c6fe40abb6a2ce60405aef2e34db482bc2a007d8585047994db29ed
+  data.tar.gz: 29aba7ab5b5c45d81807e6fc1d56f0ee69bb7a394684162bbc846d84d9970a5448a1cd19698eb2d1fffa9c4efe712e13271c89c30943562c70022db3b0bac600

data/README.md

@@ -19,7 +19,7 @@ Or install it yourself as:
     $ gem install samidare
 
 ## Usage
-Require `database.yml` and `table.yml`.  
+Require `database.yml` and `table.yml`.
 Below is a sample config file.
 
 #### database.yml
@@ -27,19 +27,21 @@ Below is a sample config file.
 db01:
   host: localhost
   username: root
-  password:
+  password: pswd
   database: production
   bq_dataset: mysql_db01
 
 db02:
   host: localhost
   username: root
-  password:
+  password: pswd
   database: production
   bq_dataset: mysql_db02
 
 ```
 
+**Caution: Embulk doesn't allow no password for MySQL**
+
 #### table.yml
 ```yml
 db01:
@@ -54,11 +56,11 @@ db02:
     - name: configs
 ```
 
-Samidare requires BigQuery parameters.
+Samidare requires BigQuery parameters like below.
 ```ruby
+[sample.rb]
 config = {
  'project_id' => 'BIGQUERY_PROJECT_ID',
- 'project_name' => 'BIGQUERY_PROJECT_NAME',
  'service_email' => 'SERVICE_ACCOUNT_EMAIL',
  'key' => '/etc/embulk/bigquery.p12',
  'schema_dir' => '/var/tmp/embulk/schema',
@@ -71,6 +73,41 @@ client.generate_config(config)
 client.run(config)
 ```
 
+```bash
+bundle exec ruby sample.rb
+```
+
+## Features
+#### daily snapshot
+BigQuery supports table wildcard expression of a specific set of daily tables, for example, `sales20150701` .  
+If you need daily snapshot of a table for BigQuery, use `daily_snapshot` option to `database.yml` or `table.yml` like below.  
+`daily_snapshot` option effects all tables in case of  `database.yml` .  
+On the other hand, only target table in `table.yml` .  
+**Daily part is determined by execute date.**
+
+```yml
+[database.yml]
+production:
+  host: localhost
+  username: root
+  password: pswd
+  database: production
+  bq_dataset: mysql
+  daily_snapshot: true
+```
+
+```yml
+[table.yml]
+production:
+  tables:
+    - name: users
+    - name: events
+      daily_snapshot: true
+    - name: hobbies
+
+Only `events` is renamed to `eventsYYYYMMDD` for BigQuery.
+```
+
 ## Contributing
 
 1. Fork it ( https://github.com/[my-github-username]/samidare/fork )

data/lib/samidare/bigquery_utility.rb

@@ -44,10 +44,12 @@ module Samidare
       "[\n" + json_body + "\n]\n"
     end
 
-    def self.generate_sql(table_name, column_infos)
+    def self.generate_sql(table_info, column_infos)
       columns = column_infos.map { |column_info| column_info.converted_value }
       sql = "SELECT " + columns.join(",")
-      sql << " FROM #{table_name}\n"
+      sql << " FROM #{table_info.name}\n"
+      sql << "WHERE #{table_info.condition}\n" if table_info.condition
+      sql
     end
 
     def generate_embulk_config(db_name, db_info, table_info, column_infos)
@@ -55,7 +57,7 @@ module Samidare
       user = db_info['username']
       password = db_info['password']
       database = db_info['database']
-      query = Samidare::BigQueryUtility.generate_sql(table_info.name, column_infos)
+      query = Samidare::BigQueryUtility.generate_sql(table_info, column_infos)
       project = @config['project_id']
       p12_keyfile_path = @config['key']
       service_account_email = @config['service_email']

data/lib/samidare/embulk_utility.rb

@@ -102,6 +102,10 @@ module Samidare
       def daily_snapshot
         @config['daily_snapshot'] || false
       end
+
+      def condition
+        @config['condition']
+      end
     end
 
     class ColumnInfo
@@ -137,18 +141,23 @@ module Samidare
       def converted_value
         if bigquery_data_type == 'timestamp'
           # time zone translate to UTC
-          "UNIX_TIMESTAMP(#{@column_name}) AS #{@column_name}"
+          "UNIX_TIMESTAMP(#{escaped_column_name}) AS #{escaped_column_name}"
         elsif data_type == 'tinyint'
           # for MySQL tinyint(1) problem
-          "CAST(#{@column_name} AS signed) AS #{@column_name}"
+          "CAST(#{escaped_column_name} AS signed) AS #{escaped_column_name}"
         else
-          @column_name
+          escaped_column_name
         end
       end
 
       def to_json(*a)
         { "name" => @column_name, "type" => bigquery_data_type }.to_json(*a)
       end
+
+      private
+      def escaped_column_name
+        "`#{@column_name}`"
+      end
     end
   end
 end

data/lib/samidare/version.rb

@@ -1,3 +1,3 @@
 module Samidare
-  VERSION = "0.0.9"
+  VERSION = "0.0.10"
 end

data/spec/samidare/bigquery_utility_spec.rb

@@ -24,16 +24,44 @@ describe Samidare::BigQueryUtility do
   end
 
   describe '.generate_sql' do
-    subject { Samidare::BigQueryUtility.generate_sql(table_name, column_infos) }
+    subject { Samidare::BigQueryUtility.generate_sql(table_info, column_infos) }
 
-    let(:table_name) { 'simple' }
     let(:column_infos) { [
       Samidare::EmbulkUtility::ColumnInfo.new('id', 'int'),
       Samidare::EmbulkUtility::ColumnInfo.new('name', 'varchar'),
       Samidare::EmbulkUtility::ColumnInfo.new('created_at', 'datetime')
       ] }
-    let(:sql) { "SELECT id,name,UNIX_TIMESTAMP(created_at) AS created_at FROM simple\n" }
-    it { expect(subject).to eq sql }
+
+    context 'no condition' do
+      let(:table_info) { Samidare::EmbulkUtility::TableInfo.new({ 'name' => 'simple' }) }
+      let(:sql) { "SELECT `id`,`name`,UNIX_TIMESTAMP(`created_at`) AS `created_at` FROM simple\n" }
+      it { expect(subject).to eq sql }
+    end
+
+    context 'has condition' do
+      let(:table_info) { Samidare::EmbulkUtility::TableInfo.new({ 'name' => 'simple', 'condition' => 'created_at >= CURRENT_DATE() - INTERVAL 3 MONTH' }) }
+      let(:sql) { "SELECT `id`,`name`,UNIX_TIMESTAMP(`created_at`) AS `created_at` FROM simple\nWHERE created_at >= CURRENT_DATE() - INTERVAL 3 MONTH\n" }
+      it { expect(subject).to eq sql }
+    end
+  end
+
+  describe '#actual_table_name' do
+    before { Timecop.freeze(Time.now) }
+
+    after { Timecop.return }
+
+    subject { Samidare::BigQueryUtility.new({}).actual_table_name(table_name, daily_snapshot) }
+    let(:table_name) { 'users' }
+    let(:daily_snapshot) { false }
+
+    context 'do not use daily snapshot' do
+      it { expect(subject).to eq table_name }
+    end
+
+    context 'use daily snapshot' do
+      let(:daily_snapshot) { true }
+      it { expect(subject).to eq table_name + Time.now.strftime('%Y%m%d') }
+    end
   end
 
   describe '#actual_table_name' do

data/spec/samidare/embulk_utility_spec.rb

@@ -93,19 +93,19 @@ describe Samidare::EmbulkUtility::ColumnInfo do
     context 'datetime' do
       let(:column_name) { 'create_at' }
       let(:data_type) { 'datetime' }
-      it { expect(subject).to eq 'UNIX_TIMESTAMP(create_at) AS create_at' }
+      it { expect(subject).to eq 'UNIX_TIMESTAMP(`create_at`) AS `create_at`' }
     end
 
     context 'int' do
       let(:column_name) { 'id' }
       let(:data_type) { 'int' }
-      it { expect(subject).to eq 'id' }
+      it { expect(subject).to eq '`id`' }
     end
 
     context 'varchar' do
       let(:column_name) { 'explanation' }
       let(:data_type) { 'varchar' }
-      it { expect(subject).to eq 'explanation' }
+      it { expect(subject).to eq '`explanation`' }
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: samidare
 version: !ruby/object:Gem::Version
-  version: 0.0.9
+  version: 0.0.10
 platform: ruby
 authors:
 - Ryoji Kobori
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-07-30 00:00:00.000000000 Z
+date: 2015-11-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -215,4 +215,3 @@ test_files:
 - spec/samidare/embulk_utility_spec.rb
 - spec/samidare_spec.rb
 - spec/spec_helper.rb
-has_rdoc: