prestogres 0.4.1 → 0.4.2

data/ChangeLog

@@ -1,4 +1,13 @@
 
+2014-03-19 version 0.4.2:
+
+* Send queries to PostgreSQL if they most likely cause parse erorr because
+  Presto doesn't return appropriate error code (42601 SYNTAX ERROR).
+* Cast Presto query results to PostgreSQL types explicitly when it inserts
+  themn to a temporary table to not cause "value too long for type character
+  varying(255)" exceptions
+
+
 2014-03-05 version 0.4.1:
 
 * Send queries to Presto if they include Presto's SQL syntax extension

data/VERSION

@@ -1 +1 @@
-0.4.1
+0.4.2

data/pgpool2/pool_query_context.c

@@ -508,40 +508,68 @@ static void run_and_rewrite_system_catalog_query(POOL_SESSION_CONTEXT* session_c
 	do_replace_query(query_context, rewrite_query_string_buffer);
 }
 
-/*
- * /\A\s*select\s*\*\s*from\s+(("[^\\"]*([\\"][^\\"]*)*")|[a-zA-Z_][a-zA-Z0-9_]*)(\.(("[^\\"]*([\\"][^\\"]*)*")|[a-zA-Z_][a-zA-Z0-9_]*))?\s*(;|\z)/.to_s
- */
-#define AUTO_LIMIT_QUERY_PATTERN "\\A\\s*select\\s*\\*\\s*from\\s+((\"[^\\\\\"]*([\\\\\"][^\\\\\"]*)*\")|[a-zA-Z_][a-zA-Z0-9_]*)(\\.((\"[^\\\\\"]*([\\\\\"][^\\\\\"]*)*\")|[a-zA-Z_][a-zA-Z0-9_]*))?\\s*(;|\\z)"
-
-static bool match_auto_limit_pattern(const char* query)
-{
+typedef struct {
 	const char* errptr;
 	int erroffset;
 	pcre* pattern;
+} regexp_context;
+
+static bool regexp_match(const char* regexp, regexp_context* context, const char* string)
+{
 	int ret;
 	int ovec[10];
 
-	pattern = pcre_compile(AUTO_LIMIT_QUERY_PATTERN,
-			PCRE_CASELESS | PCRE_NO_AUTO_CAPTURE | PCRE_UTF8, &errptr, &erroffset, NULL);
-	if (pattern == NULL) {
-		// TODO pcre pattern should be precompiled. see also pcre_study.
-		pool_error("match_auto_limit_pattern: invalid regexp %s at %d", errptr, erroffset);
-		pcre_free(pattern);
+	if (context->errptr != NULL) {
 		return false;
 	}
 
-	ret = pcre_exec(pattern, NULL, query, strlen(query), 0, 0, &ovec, sizeof(ovec));
+	if (context->pattern == NULL) {
+		pcre* pattern;
+		pattern = pcre_compile(regexp, PCRE_CASELESS | PCRE_NO_AUTO_CAPTURE | PCRE_UTF8,
+				&context->errptr, &context->erroffset, NULL);
+		if (pattern == NULL) {
+			pool_error("regexp_match: invalid regexp %s at %d", context->errptr, context->erroffset);
+			return false;
+		}
+		context->pattern = pattern;
+		context->errptr = NULL;
+
+		// TODO pcre_study?
+	}
+
+	ret = pcre_exec(context->pattern, NULL, string, strlen(string), 0, 0, ovec, sizeof(ovec));
 	if (ret < 0) {
 		// error. pattern didn't match in most of cases
-		pcre_free(pattern);
 		return false;
 	}
 
-	pcre_free(pattern);
-
 	return true;
 }
 
+/*
+ * /\A(?!.*select).*\z/i
+ */
+#define LIKELY_PARSE_ERROR "\\A(?!.*select).*\\z"
+
+static regexp_context LIKELY_PARSE_ERROR_REGEXP = {0};
+
+static bool match_likely_parse_error(const char* query)
+{
+	return regexp_match(LIKELY_PARSE_ERROR, &LIKELY_PARSE_ERROR_REGEXP, query);
+}
+
+/*
+ * /\A\s*select\s*\*\s*from\s+(("[^\\"]*([\\"][^\\"]*)*")|[a-zA-Z_][a-zA-Z0-9_]*)(\.(("[^\\"]*([\\"][^\\"]*)*")|[a-zA-Z_][a-zA-Z0-9_]*))?\s*(;|\z)/i
+ */
+#define AUTO_LIMIT_QUERY_PATTERN "\\A\\s*select\\s*\\*\\s*from\\s+((\"[^\\\\\"]*([\\\\\"][^\\\\\"]*)*\")|[a-zA-Z_][a-zA-Z0-9_]*)(\\.((\"[^\\\\\"]*([\\\\\"][^\\\\\"]*)*\")|[a-zA-Z_][a-zA-Z0-9_]*))?\\s*(;|\\z)"
+
+static regexp_context AUTO_LIMIT_REGEXP = {0};
+
+static bool match_auto_limit_pattern(const char* query)
+{
+	return regexp_match(AUTO_LIMIT_QUERY_PATTERN, &AUTO_LIMIT_REGEXP, query);
+}
+
 static void run_and_rewrite_presto_query(POOL_SESSION_CONTEXT* session_context, POOL_QUERY_CONTEXT* query_context)
 {
 	char *buffer, *bufend;
@@ -721,7 +749,7 @@ void pool_where_to_send(POOL_QUERY_CONTEXT *query_context, char *query, Node *no
 		 * If failed to parse the query, run it on Presto because
 		 * it may include Presto's SQL syntax extensions.
 		 */
-		if (query_context->is_parse_error)
+		if (query_context->is_parse_error && !match_likely_parse_error(query_context->original_query))
 		{
 			pool_debug("prestogres: send_to_where: parse-error");
 			pool_set_node_to_be_sent(query_context,

data/pgsql/prestogres.py

@@ -31,58 +31,72 @@ def _pg_table_type(presto_type):
 
 # build CREATE TEMPORARY TABLE statement
 def _build_create_temp_table_sql(table_name, column_names, column_types):
-    create_sql = "create temporary table %s (\n  " % plpy.quote_ident(table_name)
+    create_sql = ["create temporary table %s (\n  " % plpy.quote_ident(table_name)]
 
     first = True
     for column_name, column_type in zip(column_names, column_types):
         if first:
             first = False
         else:
-            create_sql += ",\n  "
+            create_sql.append(",\n  ")
 
-        create_sql += plpy.quote_ident(column_name)
-        create_sql += " "
-        create_sql += column_type
+        create_sql.append(plpy.quote_ident(column_name))
+        create_sql.append(" ")
+        create_sql.append(column_type)
 
-    create_sql += "\n)"
-    return create_sql
+    create_sql.append("\n)")
+    return ''.join(create_sql)
 
 # build CREATE TABLE statement
 def _build_alter_table_holder_sql(schema_name, table_name, column_names, column_types, not_nulls):
-    alter_sql = "alter table %s.%s \n  " % (plpy.quote_ident(schema_name), plpy.quote_ident(table_name))
+    alter_sql = ["alter table %s.%s \n  " % (plpy.quote_ident(schema_name), plpy.quote_ident(table_name))]
 
     first = True
     for column_name, column_type, not_null in zip(column_names, column_types, not_nulls):
         if first:
             first = False
         else:
-            alter_sql += ",\n  "
+            alter_sql.append(",\n  ")
 
-        alter_sql += "add %s %s" % (plpy.quote_ident(column_name), column_type)
+        alter_sql.append("add %s %s" % (plpy.quote_ident(column_name), column_type))
 
         if not_null:
-            alter_sql += " not null"
+            alter_sql.append(" not null")
 
-    return alter_sql
+    return ''.join(alter_sql)
 
 # build INSERT INTO statement and string format to build VALUES (..), ...
-def _build_insert_into_sql(table_name, column_names):
-    insert_sql = "insert into %s (\n  " % plpy.quote_ident(table_name)
+def _build_insert_into_sql(table_name, column_names, column_types):
+    # INSERT INTO table_name (column_name, column_name, ...)
+    insert_sql = ["insert into %s (\n  " % plpy.quote_ident(table_name)]
 
     first = True
     for column_name in column_names:
         if first:
             first = False
         else:
-            insert_sql += ",\n  "
+            insert_sql.append(",\n  ")
 
-        insert_sql += plpy.quote_ident(column_name)
+        insert_sql.append(plpy.quote_ident(column_name))
 
-    insert_sql += "\n) values\n"
+    insert_sql.append("\n) values\n")
 
-    values_sql_format = "(%s)" % (", ".join(["${}"] * len(column_names)))
+    # VALUES (${}::column_type, ${}::column_type, ...)
+    values_sql_format = ["("]
 
-    return (insert_sql, values_sql_format)
+    first = True
+    for column_type in column_types:
+        if first:
+            first = False
+        else:
+            values_sql_format.append(",  ")
+
+        values_sql_format.append("${}::")
+        values_sql_format.append(column_type)
+
+    values_sql_format.append(")")
+
+    return (''.join(insert_sql), ''.join(values_sql_format))
 
 # create a prepared statement for batch INSERT
 def _plan_batch(insert_sql, values_sql_format, column_types, batch_size):
@@ -173,7 +187,7 @@ def run_presto_as_temp_table(server, user, catalog, schema, result_table, query)
 
             # build SQL
             create_sql = _build_create_temp_table_sql(result_table, column_names, column_types)
-            insert_sql, values_sql_format = _build_insert_into_sql(result_table, column_names)
+            insert_sql, values_sql_format = _build_insert_into_sql(result_table, column_names, column_types)
 
             # run CREATE TABLE
             plpy.execute("drop table if exists " + plpy.quote_ident(result_table))
@@ -326,7 +340,7 @@ def run_system_catalog_as_temp_table(server, user, catalog, schema, result_table
                 subxact.exit("rollback subtransaction", None, None)
 
         create_sql = _build_create_temp_table_sql(result_table, column_names, column_types)
-        insert_sql, values_sql_format = _build_insert_into_sql(result_table, column_names)
+        insert_sql, values_sql_format = _build_insert_into_sql(result_table, column_names, column_types)
 
         # run CREATE TABLE and INSERT
         plpy.execute("drop table if exists " + plpy.quote_ident(result_table))

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: prestogres
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.4.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-03-05 00:00:00.000000000 Z
+date: 2014-03-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler