duckdb 0.3.3-dev149.0 → 0.3.3-dev1506.0

package/src/parquet-amalgamation.hpp

@@ -7133,7 +7133,7 @@ public:
 			throw std::runtime_error("Decode bit width too large");
 		}
 		byte_encoded_len = ((bit_width_ + 7) / 8);
-		max_val = (1 << bit_width_) - 1;
+		max_val = (uint64_t(1) << bit_width_) - 1;
 	}
 
 	template <typename T>
@@ -7185,12 +7185,12 @@ private:
 	ByteBuffer buffer_;
 
 	/// Number of bits needed to encode the value. Must be between 0 and 64.
-	int bit_width_;
+	uint32_t bit_width_;
 	uint64_t current_value_;
 	uint32_t repeat_count_;
 	uint32_t literal_count_;
 	uint8_t byte_encoded_len;
-	uint32_t max_val;
+	uint64_t max_val;
 
 	uint8_t bitpack_pos = 0;
 
@@ -7410,12 +7410,16 @@ public:
 
 	virtual void Skip(idx_t num_values);
 
-	const LogicalType &Type();
-	const SchemaElement &Schema();
+	ParquetReader &Reader();
+	const LogicalType &Type() const;
+	const SchemaElement &Schema() const;
+	idx_t FileIdx() const;
+	idx_t MaxDefine() const;
+	idx_t MaxRepeat() const;
 
 	virtual idx_t GroupRowsAvailable();
 
-	unique_ptr<BaseStatistics> Stats(const std::vector<ColumnChunk> &columns);
+	virtual unique_ptr<BaseStatistics> Stats(const std::vector<ColumnChunk> &columns);
 
 protected:
 	// readers that use the default Read() need to implement those
@@ -7574,10 +7578,17 @@ public:
 	    : ParquetReader(allocator, move(file_handle_p), vector<LogicalType>(), string()) {
 	}
 
-	ParquetReader(ClientContext &context, string file_name, const vector<LogicalType> &expected_types_p,
+	ParquetReader(ClientContext &context, string file_name, const vector<string> &names,
+	              const vector<LogicalType> &expected_types_p, const vector<column_t> &column_ids,
 	              ParquetOptions parquet_options, const string &initial_filename = string());
 	ParquetReader(ClientContext &context, string file_name, ParquetOptions parquet_options)
-	    : ParquetReader(context, move(file_name), vector<LogicalType>(), parquet_options, string()) {
+	    : ParquetReader(context, move(file_name), vector<string>(), vector<LogicalType>(), vector<column_t>(),
+	                    parquet_options, string()) {
+	}
+	ParquetReader(ClientContext &context, string file_name, const vector<LogicalType> &expected_types_p,
+	              ParquetOptions parquet_options)
+	    : ParquetReader(context, move(file_name), vector<string>(), expected_types_p, vector<column_t>(),
+	                    parquet_options, string()) {
 	}
 	~ParquetReader();
 
@@ -7604,7 +7615,8 @@ public:
 	static LogicalType DeriveLogicalType(const SchemaElement &s_ele, bool binary_as_string);
 
 private:
-	void InitializeSchema(const vector<LogicalType> &expected_types_p, const string &initial_filename_p);
+	void InitializeSchema(const vector<string> &names, const vector<LogicalType> &expected_types_p,
+	                      const vector<column_t> &column_ids, const string &initial_filename_p);
 	bool ScanInternal(ParquetReaderScanState &state, DataChunk &output);
 	unique_ptr<ColumnReader> CreateReader(const duckdb_parquet::format::FileMetaData *file_meta_data);
 
@@ -7623,6 +7635,12 @@ private:
 
 private:
 	unique_ptr<FileHandle> file_handle;
+	//! column-id map, used when reading multiple parquet files since separate parquet files might have columns at
+	//! different positions e.g. the first file might have column "a" at position 0, the second at position 1, etc
+	vector<column_t> column_id_map;
+	//! Map of column_id -> cast, used when reading multiple parquet files when parquet files have diverging types
+	//! for the same column
+	unordered_map<column_t, LogicalType> cast_map;
 };
 
 } // namespace duckdb

package/test/extension.test.js

@@ -0,0 +1,86 @@
+var duckdb = require('..');
+var fs = require('fs');
+var assert = require('assert');
+var path = require('path');
+
+const extension_base_path = "../../../build/release/extension";
+
+// Look for extensions that we can load and test
+let extension_paths = [];
+const extension_full_path = path.resolve(__dirname, extension_base_path);
+if (fs.existsSync(extension_full_path)) {
+    extension_paths = extension_paths.concat(fs.readdirSync(extension_full_path).map(function (file) {
+        if (!fs.statSync(extension_full_path+'/'+file).isDirectory())
+            return undefined;
+        const potential_extension_path = extension_full_path+`/${file}/${file}.duckdb_extension`;
+        if (fs.existsSync(potential_extension_path)) {
+            return potential_extension_path;
+        }
+    }).filter(a=>a));
+}
+
+// Note: test will pass on http request failing due to connection issues.
+const test_httpfs = async function (db, done) {
+    db.all("SELECT id, first_name, last_name FROM PARQUET_SCAN('https://raw.githubusercontent.com/cwida/duckdb/master/data/parquet-testing/userdata1.parquet') LIMIT 3;", function(err, rows) {
+        if (err) {
+            if (err.message.startsWith("Unable to connect to URL")) {
+                console.warn("Warning: HTTP request failed in extension.test.js");
+                done();
+            } else {
+                throw err;
+            }
+        } else {
+            assert.deepEqual(rows, [
+                { id: 1, first_name: 'Amanda', last_name: 'Jordan'},
+                { id: 2, first_name: 'Albert', last_name: 'Freeman'},
+                { id: 3, first_name: 'Evelyn', last_name: 'Morgan'},
+            ]);
+            done();
+        }
+    });
+};
+
+const test_tpch = async function (db, done) {
+    db.all("CALL DBGEN(sf=0.01);", function(err) {
+        if (err) {
+            throw err;
+        }
+        done();
+    });
+};
+
+const test_extension = function(extension_name, db, done) {
+    switch(extension_name) {
+        case 'httpfs.duckdb_extension':
+            test_httpfs(db, done);
+            break;
+        case 'tpch.duckdb_extension':
+            test_tpch(db, done);
+            break;
+        default:
+            done();
+            break;
+    }
+};
+
+describe('Extension loading', function() {
+    var db;
+
+    before(function(done) {
+        db = new duckdb.Database(':memory:', done);
+    });
+
+    for (ext of extension_paths) {
+        const extension_path = ext;
+        const extension_name = ext.replace(/^.*[\\\/]/, '');
+
+        it(extension_name, function(done) {
+            db.run(`LOAD '${extension_path}';`, function(err) {
+                if (err) {
+                    throw err;
+                }
+                test_extension(extension_name, db, done);
+            });
+        });
+    }
+});

package/test/udf.test.js

@@ -0,0 +1,149 @@
+var duckdb = require('..');
+var assert = require('assert');
+
+describe('UDFs', function() {
+    var db;
+
+    before(function(done) {
+        db = new duckdb.Database(':memory:', done);
+    });
+
+    it('0ary int', function(done) {
+        db.register("udf", "integer", () => 42);
+        db.all("select udf() v", function(err, rows) {
+            if (err) throw err;
+            assert.equal(rows[0].v, 42);
+        });
+        db.unregister("udf", done);
+    });
+
+    it('0ary double', function(done) {
+        db.register("udf", "double", () => 4.2);
+        db.all("select udf() v", function(err, rows) {
+            if (err) throw err;
+            assert.equal(rows[0].v, 4.2);
+        });
+        db.unregister("udf", done);
+    });
+
+    it('0ary string', function(done) {
+        db.register("udf", "varchar", () => 'hello');
+        db.all("select udf() v", function(err, rows) {
+            if (err) throw err;
+            assert.equal(rows[0].v, 'hello');
+        });
+        db.unregister("udf", done);
+    });
+
+    it('0ary int null', function(done) {
+        db.register("udf", "integer", () => undefined);
+        db.all("select udf() v", function(err, rows) {
+            if (err) throw err;
+            assert.equal(rows[0].v, undefined);
+        });
+        db.unregister("udf", done);
+    });
+
+
+    it('0ary string null', function(done) {
+        db.register("udf", "varchar", () => undefined);
+        db.all("select udf() v", function(err, rows) {
+            if (err) throw err;
+            assert.equal(rows[0].v, undefined);
+        });
+        db.unregister("udf", done);
+    });
+
+
+    it('unary int', function(done) {
+        db.register("udf", "integer", (x) => x+1);
+        db.all("select udf(42) v", function(err, rows) {
+            if (err) throw err;
+            assert.equal(rows[0].v, 43);
+        });
+        db.unregister("udf", done);
+    });
+
+    it('unary double', function(done) {
+        db.register("udf", "double", (x) => x);
+        db.all("select udf(4.2::double) v", function(err, rows) {
+            if (err) throw err;
+            assert.equal(rows[0].v, 4.2);
+        });
+        db.unregister("udf", done);
+    });
+
+    it('unary int null', function(done) {
+        db.register("udf", "integer", (x) => undefined);
+        db.all("select udf(42) v", function(err, rows) {
+            if (err) throw err;
+            assert.equal(rows[0].v, undefined);
+        });
+        db.unregister("udf", done);
+    });
+
+
+    it('unary double null', function(done) {
+        db.register("udf", "double", (x) => undefined);
+        db.all("select udf(4.2::double) v", function(err, rows) {
+            if (err) throw err;
+            assert.equal(rows[0].v, undefined);
+        });
+        db.unregister("udf", done);
+    });
+
+
+    it('unary string', function(done) {
+        db.register("udf", "varchar", (x) => 'hello ' + x);
+        db.all("select udf('world') v", function(err, rows) {
+            if (err) throw err;
+            assert.equal(rows[0].v, 'hello world');
+        });
+        db.unregister("udf", done);
+    });
+
+    it('unary string null', function(done) {
+        db.register("udf", "varchar", (x) => undefined);
+        db.all("select udf('world') v", function(err, rows) {
+            if (err) throw err;
+            assert.equal(rows[0].v, undefined);
+        });
+        db.unregister("udf", done);
+    });
+
+    it('binary int', function(done) {
+        db.register("udf", "integer", (x, y) => x + y);
+        db.all("select udf(40, 2) v", function(err, rows) {
+            if (err) throw err;
+            assert.equal(rows[0].v, 42);
+        });
+        db.unregister("udf", done);
+    });
+
+    it('binary string', function(done) {
+        db.register("udf", "varchar", (x, y) => x + ' ' + y);
+        db.all("select udf('hello', 'world') v", function(err, rows) {
+            if (err) throw err;
+            assert.equal(rows[0].v, 'hello world');
+        });
+        db.unregister("udf", done);
+    });
+
+    it('ternary int', function(done) {
+        db.register("udf", "integer", (x, y, z) => x + y + z);
+        db.all("select udf(21, 20, 1) v", function(err, rows) {
+            if (err) throw err;
+            assert.equal(rows[0].v, 42);
+        });
+        db.unregister("udf", done);
+    });
+
+    it('unary larger series', function(done) {
+        db.register("udf", "integer", (x) => 1);
+        db.all("select sum(udf(range::double)) v from range(10000)", function(err, rows) {
+            if (err) throw err;
+            assert.equal(rows[0].v, 10000);
+        });
+        db.unregister("udf", done);
+    });
+});