duckdb 0.6.2-dev960.0 → 0.6.2-dev964.0

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.6.2-dev960.0",
+  "version": "0.6.2-dev964.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,8 +1,8 @@
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "0.6.2-dev960"
+#define DUCKDB_VERSION "0.6.2-dev964"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "84b9e770f3"
+#define DUCKDB_SOURCE_ID "29c0bbc275"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"

package/src/duckdb/third_party/fsst/libfsst.cpp

@@ -454,21 +454,24 @@ static inline size_t compressBulk(SymbolTable &symbolTable, size_t nlines, size_
 #define FSST_SAMPLELINE ((size_t) 512)
 
 // quickly select a uniformly random set of lines such that we have between [FSST_SAMPLETARGET,FSST_SAMPLEMAXSZ) string bytes
-vector<u8*> makeSample(u8* sampleBuf, u8* strIn[], size_t **lenRef, size_t nlines) {
-	size_t totSize = 0, *lenIn = *lenRef;
+vector<u8*> makeSample(u8* sampleBuf, u8* strIn[], size_t *lenIn, size_t nlines,
+                                                    unique_ptr<vector<size_t>>& sample_len_out) {
+	size_t totSize = 0;
 	vector<u8*> sample;
 
 	for(size_t i=0; i<nlines; i++)
 		totSize += lenIn[i];
-
 	if (totSize < FSST_SAMPLETARGET) {
 		for(size_t i=0; i<nlines; i++)
 			sample.push_back(strIn[i]);
 	} else {
 		size_t sampleRnd = FSST_HASH(4637947);
 		u8* sampleLim = sampleBuf + FSST_SAMPLETARGET;
-		size_t *sampleLen = *lenRef = new size_t[nlines + FSST_SAMPLEMAXSZ/FSST_SAMPLELINE];
 
+		sample_len_out = unique_ptr<vector<size_t>>(new vector<size_t>());
+		sample_len_out->reserve(nlines + FSST_SAMPLEMAXSZ/FSST_SAMPLELINE);
+
+		// This fails if we have a lot of small strings and a few big ones?
 		while(sampleBuf < sampleLim) {
 			// choose a non-empty line
 			sampleRnd = FSST_HASH(sampleRnd);
@@ -485,7 +488,9 @@ vector<u8*> makeSample(u8* sampleBuf, u8* strIn[], size_t **lenRef, size_t nline
 			size_t len = min(lenIn[linenr]-chunk,FSST_SAMPLELINE);
 			memcpy(sampleBuf, strIn[linenr]+chunk, len);
 			sample.push_back(sampleBuf);
-			sampleBuf += *sampleLen++ = len;
+
+			sample_len_out->push_back(len);
+			sampleBuf += len;
 		}
 	}
 	return sample;
@@ -493,11 +498,11 @@ vector<u8*> makeSample(u8* sampleBuf, u8* strIn[], size_t **lenRef, size_t nline
 
 extern "C" duckdb_fsst_encoder_t* duckdb_fsst_create(size_t n, size_t lenIn[], u8 *strIn[], int zeroTerminated) {
 	u8* sampleBuf = new u8[FSST_SAMPLEMAXSZ];
-	size_t *sampleLen = lenIn;
-	vector<u8*> sample = makeSample(sampleBuf, strIn, &sampleLen, n?n:1); // careful handling of input to get a right-size and representative sample
+	unique_ptr<vector<size_t>> sample_sizes;
+	vector<u8*> sample = makeSample(sampleBuf, strIn, lenIn, n?n:1, sample_sizes); // careful handling of input to get a right-size and representative sample
 	Encoder *encoder = new Encoder();
+	size_t* sampleLen = sample_sizes ? sample_sizes->data() : &lenIn[0];
 	encoder->symbolTable = shared_ptr<SymbolTable>(buildSymbolTable(encoder->counters, sample, sampleLen, zeroTerminated));
-	if (sampleLen != lenIn) delete[] sampleLen;
 	delete[] sampleBuf;
 	return (duckdb_fsst_encoder_t*) encoder;
 }