flydata 0.5.12 → 0.5.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 82270f47e1280076e3dc0b40a86cc548e0da6636
-  data.tar.gz: bed0f9307f53602e2956369b5419e3173410d8bd
+  metadata.gz: d4b03db81cb7465b262dce0c4c8e33a14b5e290b
+  data.tar.gz: 42c08437e1b5f0d8452f2f2504666d6ce83006a7
 SHA512:
-  metadata.gz: e765cd79b44de76a883867a81718e9c7b4ab4834c0ae97d59faa57728bea4e1888bc97d8c993edd6103ea6c5da1786ca33b58f18f532c5eb04c3b6e518f95a2f
-  data.tar.gz: e62f777a303bae1fee4810bc1a1133ef2ca3d12b212ab4932666605e822d57f3ebcaf0ced16433c1b9894e43a286bd824edf30a210f24ab0a764a706420429d4
+  metadata.gz: 6de4243e12d2ec8d0c0c62a68a21fd9f1bef9f5a40caebe494687656ab5714d1d88e56ff7aa091796c52a57321e74e6c287ddab02e3968a0029e0203ff3ea47a
+  data.tar.gz: 910e524b426859dae7e5996bf25148aedf2eebea36decbc602965b483d8cf010ff224e998707c6d35bac7f80036ff0f609ef6dfd88410bf562700d967ad30b34

data/Gemfile

@@ -29,4 +29,5 @@ group :development do
   gem 'activerecord', '~> 4.0', '>= 4.0.0'
   gem 'protected_attributes', '~> 1.0', '>= 1.0.8'
   gem 'pry'
+  gem 'rake-compiler', '~> 0.9', '>= 0.9.5'
 end

data/Gemfile.lock

@@ -88,6 +88,8 @@ GEM
       slop (~> 3.4)
     rack (1.5.2)
     rake (10.3.2)
+    rake-compiler (0.9.5)
+      rake
     rdoc (4.1.1)
       json (~> 1.4)
     rest-client (1.7.2)
@@ -139,6 +141,7 @@ DEPENDENCIES
   mysql2 (~> 0.3, >= 0.3.17)
   protected_attributes (~> 1.0, >= 1.0.8)
   pry
+  rake-compiler (~> 0.9, >= 0.9.5)
   rest-client (~> 1.6, >= 1.6.7)
   rspec (~> 3.0)
   ruby-prof (~> 0.15, >= 0.15.1)

data/Rakefile

@@ -32,3 +32,15 @@ RSpec::Core::RakeTask.new(:spec) do |t|
 end
 
 task :default => :spec
+
+require 'rake/extensiontask'
+
+Rake::ExtensionTask.new "dump_parser_ext" do |ext|
+  ext.ext_dir = "ext/flydata/parser/mysql"
+  ext.lib_dir = "lib/flydata/parser/mysql"
+end
+
+Rake::ExtensionTask.new "json_ext" do |ext|
+  ext.ext_dir = "ext/flydata/json"
+  ext.lib_dir = "lib/flydata/json"
+end

data/VERSION

@@ -1 +1 @@
-0.5.12
+0.5.13

data/ext/flydata/flydata.h

@@ -0,0 +1,13 @@
+#ifndef __FLYDATA_H
+#define __FLYDATA_H
+
+#define __F(f) (reinterpret_cast<VALUE (*)(...)>(f))
+
+#ifdef DEBUG
+#define debug_print(fmt, ...) \
+            do { if (DEBUG) fprintf(stderr, fmt, __VA_ARGS__); } while (0)
+#else
+#define debug_print
+#endif
+
+#endif // __FLYDATA_H

data/ext/flydata/json/extconf.rb

@@ -0,0 +1,3 @@
+require "mkmf"
+
+create_makefile "flydata/json/json_ext"

data/ext/flydata/json/json_ext.cpp

@@ -0,0 +1,137 @@
+#include <sstream>
+#include <ruby.h>
+#include <ruby/encoding.h>
+#include "../flydata.h"
+
+extern "C" {
+#ifdef _WIN32
+_declspec(dllexport)
+#endif
+void Init_json_ext();
+}
+
+inline void add_json_value(const char* buf, long buf_len, bool quote, std::stringstream& ss)
+{
+  if (quote)
+  {
+    ss.put('\"');
+  }
+  const char* ptr = 0;
+  long len = 0;
+  const char* esc_str = 0;
+  for (long i = 0; i < buf_len; i++) {
+    if (!ptr) { ptr = buf + i; len = 0; }
+    switch (buf[i])
+    {
+      case '\"':
+        esc_str = "\\\"";
+      case '\\':
+        esc_str = "\\\\";
+      case '/':
+        esc_str = "\\/";
+      case '\b':
+        esc_str = "\\b";
+      case '\f':
+        esc_str = "\\f";
+      case '\n':
+        esc_str = "\\n";
+      case '\r':
+        esc_str = "\\r";
+      case '\t':
+        esc_str = "\\t";
+  
+        if (len > 0)
+        {
+          ss.write(ptr, len);
+        }
+        ss.write(esc_str, 2);
+        ptr = 0; len = 0;
+        break;
+      default:
+        len++;
+        break;
+    }
+  }
+  if (len > 0) {
+    ss.write(ptr, len);
+  }
+
+  if (quote)
+  {
+    ss.put('\"');
+  }
+}
+
+inline void add_json_value(VALUE value, std::stringstream& ss)
+{
+  switch(TYPE(value))
+  {
+    case T_STRING:
+      add_json_value(RSTRING_PTR(value), RSTRING_LEN(value), true, ss);
+      break;
+    case T_NIL:
+      add_json_value("null", 4, false, ss);
+      break;
+    case T_TRUE:
+      add_json_value("true", 4, false, ss);
+      break;
+    case T_FALSE:
+      add_json_value("false", 5, false, ss);
+      break;
+    default:
+      VALUE v = rb_funcall(value, rb_intern("to_s"), 0);
+      add_json_value(RSTRING_PTR(v), RSTRING_LEN(v), true, ss);
+      break;
+  }
+}
+
+inline void add_kv_pair(VALUE key, VALUE value, std::stringstream& ss)
+{
+  if (TYPE(key) != T_STRING)
+  {
+    key = rb_funcall(key, rb_intern("to_s"), 0);
+  }
+  add_json_value(key, ss);
+  ss.put(':');
+  add_json_value(value, ss);
+}
+
+static VALUE rb_json_generate_kv_pairs(VALUE self, VALUE key_ary, VALUE value_ary)
+{
+  long key_len = RARRAY_LEN(key_ary);
+  long value_len = RARRAY_LEN(value_ary);
+
+  if (key_len != value_len)
+  {
+    rb_raise(rb_eArgError, "length of arrays must match");
+  }
+
+  std::stringstream ss;
+
+  ss << "{";
+
+  for (long i = 0; i < key_len; i++)
+  {
+    if (i > 0) { ss << ","; }
+
+    VALUE key = rb_ary_entry(key_ary, i);
+    VALUE value = rb_ary_entry(value_ary, i);
+
+    add_kv_pair(key, value, ss);
+  }
+  ss << "}";
+
+  std::string str = ss.str();
+  VALUE json = rb_str_new(str.data(), str.size());
+  int enc = rb_enc_find_index("UTF-8");
+  rb_enc_associate_index(json, enc);
+
+  return json;
+}
+
+void Init_json_ext()
+{
+  VALUE mJSON  = rb_define_module("JSON");
+
+  rb_define_singleton_method(mJSON, "generate_kv_pairs", __F(&rb_json_generate_kv_pairs), 2);
+}

data/ext/flydata/parser/mysql/.gitignore

@@ -0,0 +1 @@
+Makefile

data/ext/flydata/parser/mysql/dump_parser_ext.cpp

@@ -0,0 +1,106 @@
+//#define DEBUG 1
+
+#include <iostream>
+#include <ruby.h>
+#include <ruby/encoding.h>
+#include "sql_parser.h"
+#include "../../flydata.h"
+
+extern "C" {
+#ifdef _WIN32
+_declspec(dllexport)
+#endif
+void Init_dump_parser_ext();
+}
+
+class InsertParserCallbackHandler : public ParserCallbackHandler
+{
+  VALUE m_rows;
+  VALUE m_row;
+  VALUE m_value;
+
+public:
+
+  InsertParserCallbackHandler() : ParserCallbackHandler(), m_rows(rb_ary_new()), m_row(rb_ary_new()), m_value(Qnil)
+  {
+  }
+
+  void value_callback(const char* str, long len, bool end_value)
+  {
+    if (m_value == Qnil)
+    {
+      if (str || len > 0)
+      {
+        m_value = rb_str_new(str, len);
+        int enc = rb_enc_find_index("UTF-8");
+        rb_enc_associate_index(m_value, enc);
+        debug_print("new value:%s\n", StringValueCStr(m_value));
+      }
+      else
+      {
+        m_value = Qnil;
+        debug_print("new value:%s\n", "nil");
+      }
+    }
+    else
+    {
+      rb_str_cat(m_value, str, len);
+      debug_print("value appended:%s\n", StringValueCStr(m_value));
+    }
+
+    if (end_value)
+    {
+      rb_ary_push(m_row, m_value);
+      debug_print("value committed:%s\n",
+                    m_value == Qnil ? "nil" : StringValueCStr(m_value));
+      m_value = Qnil;
+    }
+  }
+
+  void row_end_callback()
+  {
+    if (m_row)
+    {
+      rb_ary_push(m_rows, m_row);
+      debug_print("row pushed\n", 0);
+      m_row = rb_ary_new();
+    }
+  }
+
+  VALUE rows()
+  {
+    return m_rows;
+  }
+};
+
+static VALUE rb_insert_parser_parse2(VALUE self, VALUE sql)
+{
+  InsertParserCallbackHandler ch;
+
+  long error_at = -1;
+  if (sql != Qnil)
+  {
+    StringValue(sql);
+    error_at = parse_insert_query(RSTRING_PTR(sql), RSTRING_LEN(sql), ch);
+  }
+
+  if (error_at > -1) {
+    VALUE mFlydata = rb_define_module("Flydata");
+    VALUE eAgentError = rb_define_class_under(mFlydata, "AgentError", rb_eStandardError);
+    VALUE eDumpParseError = rb_define_class_under(mFlydata, "DumpParseError", eAgentError);
+    rb_raise(eDumpParseError, "Parse error at %ld", error_at);
+  }
+
+  return ch.rows();
+}
+
+void Init_dump_parser_ext()
+{
+  VALUE mFlydata = rb_define_module("Flydata");
+  VALUE mParser  = rb_define_module_under(mFlydata, "Parser");
+  VALUE mMysql   = rb_define_module_under(mParser, "Mysql");
+  VALUE cMysqlDumpParser = rb_define_class_under(mMysql, "MysqlDumpParser", rb_cObject);
+  VALUE cInsertParser = rb_define_class_under(cMysqlDumpParser, "InsertParser", rb_cObject);
+
+  rb_define_private_method(cInsertParser, "_parse2", __F(&rb_insert_parser_parse2), 1);
+}

data/ext/flydata/parser/mysql/extconf.rb

@@ -0,0 +1,3 @@
+require "mkmf"
+
+create_makefile "flydata/parser/mysql/dump_parser_ext"

data/ext/flydata/parser/mysql/parser.txt

@@ -0,0 +1,121 @@
+start:
+  "(" => :value
+  else => :start
+
+value:
+  " ", "\t", "\n", "," => :value
+  "'" => :first_char
+  "0" => (mark), :first_zero
+  [1-9-] => (mark), :number
+  "N" => :null_n
+  ")" => (row_end_callback), :start
+  else => :error
+
+first_zero:
+  "0" => (mark), :leading_zero
+  [1-9] => (mark), :number
+  "." => :number
+  [xX] => :hex_blob
+  "," => (value_callback, :end_value)(mark reset), :value
+  ")" => (value_callback, :end_value)(mark reset)(row_end_callback), :start
+  else => :error
+
+leading_zero:
+  "0" => (mark), :leading_zero
+  [1-9] => (mark), :number
+  "." => :number
+  "," => (value_callback, :end_value)(mark reset), :value
+  ")" => (value_callback, :end_value)(mark reset)(row_end_callback), :start
+  else => :error
+
+number:
+  [0-9.] => :number
+  "," => (value_callback, :end_value)(mark reset), :value
+  ")" => (value_callback, :end_value)(mark reset)(row_end_callback), :start
+  else => :error
+
+null_n:
+  "U" => :null_u
+  else => :error
+
+null_u:
+  "L" => :null_l1
+  else => :error
+
+null_l1:
+  "L" => :null_l2
+  else => :error
+
+null_l2:
+  "," => (value_callback 0, :end_value)(mark_reset), :value  # 0 is NULL pointer
+  ")" => (value_callback 0, :end_value)(mark_reset)(row_end_callback), :start  # 0 is NULL pointer
+  else => :error
+
+hex_blob:
+  [0-9a-fA-F] => :hex_blob_number
+  else => :error
+
+hex_blob_number:
+  [0-9a-fA-F] => :hex_blob_number
+  "," => (value_callback, :end_value)(mark_reset), :value
+  ")" => (value_callback, :end_value)(mark_reset)(row_end_callback), :start
+  else => :error
+
+first_char:
+  "\\" => (mark), :escape
+  "'" => (mark)(value_callback, :end_value)(mark_reset), :value
+  mask 110xxxxx => (mark)(temp_mark), :utf8_2_firstbyte
+  mask 1110xxxx => (mark)(temp_mark), :utf8_3_firstbyte
+  mask 11110xxx => (mark)(temp_mark), :utf8_4_firstbyte
+  else => (mark), :following_char
+
+utf8_2_firstbyte:
+  mask 10xxxxxx => (temp_mark reset), :following_char
+  else (value_callback mark...temp_mark)(mark reset)(value_callback "\xc2\xbf\xc2\xbf")(temp_mark reset), :first_char
+
+utf8_3_firstbyte:
+  mask 10xxxxxx => :utf8_3_secondbyte
+  else => :utf8_3_error_secondbyte
+
+utf8_3_secondbyte:
+  mask 10xxxxxx => (temp_mark reset), :following_char
+  else (value_callback mark...temp_mark)(mark reset)(value_callback "\xc2\xbf\xc2\xbf\xc2\xbf")(temp_mark reset), :first_char
+
+utf8_3_error_secondbyte:
+  any char => (value_callback mark...temp_mark)(mark reset)(value_callback "\xc2\xbf\xc2\xbf\xc2\xbf")(temp_mark reset), :first_char
+
+utf8_4_firstbyte:
+  mask 10xxxxxx => :utf8_4_secondbyte
+  else => :utf8_4_error_secondbyte
+
+utf8_4_secondbyte:
+  mask 10xxxxxx => :utf_4_thirdbyte
+  else => :utf8_4_error_thirdbyte
+
+utf8_4_error_secondbyte:
+  any_char => :utf8_4_error_thirdbyte
+
+utf8_4_thirdbyte:
+  mask 10xxxxxx => (temp_mark reset), :following_char
+  else (value_callback mark...temp_mark)(mark reset)(value_callback "\xc2\xbf\xc2\xbf\xc2\xbf\xc2\xbf")(temp_mark reset), :first_char
+
+utf8_4_error_thirdbyte:
+  any char => (value_callback mark...temp_mark)(mark reset)(value_callback "\xc2\xbf\xc2\xbf\xc2\xbf\xc2\xbf")(temp_mark reset), :first_char
+
+following_char:
+  "\\" => (value_callback)(mark), :escape
+  "'" => (value_callback, :end_value)(mark_reset), :value
+  mask 110xxxxx => (temp_mark), :utf8_2_firstbyte
+  mask 1110xxxx => (temp_mark), :utf8_3_firstbyte
+  mask 11110xxx => (temp_mark), :utf8_4_firstbyte
+  else => :following_char
+
+escape:
+  "\\" => (value_callback "\\")(mark_reset), :first_char
+  "'" =>  (value_callback "'")(mark_reset), :first_char
+  '"' =>  (value_callback '"')(mark_reset), :first_char
+  "n" =>  (value_callback "\n")(mark_reset), :first_char
+  "r" =>  (value_callback "\r")(mark_reset), :first_char
+  "0" =>  (value_callback "\0")(mark_reset), :first_char
+  "Z" =>  (value_callback "\Z")(mark_reset), :first_char
+  else => :following_char