@owox/connectors 0.12.0-next-20251106105920 → 0.12.0-next-20251106142331
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +5 -5
- package/dist/index.js +5 -5
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -1861,7 +1861,7 @@ const GoogleBigQuery = (function() {
|
|
|
1861
1861
|
//----------------------------------------------------------------
|
|
1862
1862
|
//---- loads Google BigQuery Table Schema ---------------------------
|
|
1863
1863
|
async loadTableSchema() {
|
|
1864
|
-
this.existingColumns = this.getAListOfExistingColumns() || {};
|
|
1864
|
+
this.existingColumns = await this.getAListOfExistingColumns() || {};
|
|
1865
1865
|
if (Object.keys(this.existingColumns).length == 0) {
|
|
1866
1866
|
await this.createDatasetIfItDoesntExist();
|
|
1867
1867
|
this.existingColumns = await this.createTableIfItDoesntExist();
|
|
@@ -1880,7 +1880,7 @@ const GoogleBigQuery = (function() {
|
|
|
1880
1880
|
* @return columns (object)
|
|
1881
1881
|
*
|
|
1882
1882
|
*/
|
|
1883
|
-
getAListOfExistingColumns() {
|
|
1883
|
+
async getAListOfExistingColumns() {
|
|
1884
1884
|
let query = "----- Getting a list of existing columns ------\n";
|
|
1885
1885
|
query += `DECLARE dataset_exists BOOL;
|
|
1886
1886
|
SET dataset_exists = EXISTS (
|
|
@@ -1893,7 +1893,7 @@ const GoogleBigQuery = (function() {
|
|
|
1893
1893
|
FROM \`${this.config.DestinationDatasetID.value}.INFORMATION_SCHEMA.COLUMNS\`
|
|
1894
1894
|
WHERE table_name = '${this.config.DestinationTableName.value}';
|
|
1895
1895
|
END IF`;
|
|
1896
|
-
let queryResults = this.executeQuery(query);
|
|
1896
|
+
let queryResults = await this.executeQuery(query);
|
|
1897
1897
|
let columns = {};
|
|
1898
1898
|
if (queryResults.rows) {
|
|
1899
1899
|
queryResults.rows.map((row) => {
|
|
@@ -2075,12 +2075,12 @@ OPTIONS(description="${this.description}")`;
|
|
|
2075
2075
|
this.totalRecordsProcessed += currentBatch.length;
|
|
2076
2076
|
console.log(`BigQuery MERGE completed successfully for ${currentBatch.length} records (Total processed: ${this.totalRecordsProcessed})`);
|
|
2077
2077
|
if (remainingRecords.length > 0) {
|
|
2078
|
-
this.executeMergeQueryRecursively(remainingRecords, batchSize);
|
|
2078
|
+
await this.executeMergeQueryRecursively(remainingRecords, batchSize);
|
|
2079
2079
|
}
|
|
2080
2080
|
} catch (error) {
|
|
2081
2081
|
if (error.message && error.message.includes("query is too large")) {
|
|
2082
2082
|
console.log(`Query execution failed due to size. Reducing batch size from ${batchSize} to ${Math.floor(batchSize / 2)}`);
|
|
2083
|
-
this.executeMergeQueryRecursively(recordKeys, Math.floor(batchSize / 2));
|
|
2083
|
+
await this.executeMergeQueryRecursively(recordKeys, Math.floor(batchSize / 2));
|
|
2084
2084
|
} else {
|
|
2085
2085
|
throw error;
|
|
2086
2086
|
}
|
package/dist/index.js
CHANGED
|
@@ -1866,7 +1866,7 @@ var require_index = __commonJS({
|
|
|
1866
1866
|
//----------------------------------------------------------------
|
|
1867
1867
|
//---- loads Google BigQuery Table Schema ---------------------------
|
|
1868
1868
|
async loadTableSchema() {
|
|
1869
|
-
this.existingColumns = this.getAListOfExistingColumns() || {};
|
|
1869
|
+
this.existingColumns = await this.getAListOfExistingColumns() || {};
|
|
1870
1870
|
if (Object.keys(this.existingColumns).length == 0) {
|
|
1871
1871
|
await this.createDatasetIfItDoesntExist();
|
|
1872
1872
|
this.existingColumns = await this.createTableIfItDoesntExist();
|
|
@@ -1885,7 +1885,7 @@ var require_index = __commonJS({
|
|
|
1885
1885
|
* @return columns (object)
|
|
1886
1886
|
*
|
|
1887
1887
|
*/
|
|
1888
|
-
getAListOfExistingColumns() {
|
|
1888
|
+
async getAListOfExistingColumns() {
|
|
1889
1889
|
let query = "----- Getting a list of existing columns ------\n";
|
|
1890
1890
|
query += `DECLARE dataset_exists BOOL;
|
|
1891
1891
|
SET dataset_exists = EXISTS (
|
|
@@ -1898,7 +1898,7 @@ var require_index = __commonJS({
|
|
|
1898
1898
|
FROM \`${this.config.DestinationDatasetID.value}.INFORMATION_SCHEMA.COLUMNS\`
|
|
1899
1899
|
WHERE table_name = '${this.config.DestinationTableName.value}';
|
|
1900
1900
|
END IF`;
|
|
1901
|
-
let queryResults = this.executeQuery(query);
|
|
1901
|
+
let queryResults = await this.executeQuery(query);
|
|
1902
1902
|
let columns = {};
|
|
1903
1903
|
if (queryResults.rows) {
|
|
1904
1904
|
queryResults.rows.map((row) => {
|
|
@@ -2080,12 +2080,12 @@ OPTIONS(description="${this.description}")`;
|
|
|
2080
2080
|
this.totalRecordsProcessed += currentBatch.length;
|
|
2081
2081
|
console.log(`BigQuery MERGE completed successfully for ${currentBatch.length} records (Total processed: ${this.totalRecordsProcessed})`);
|
|
2082
2082
|
if (remainingRecords.length > 0) {
|
|
2083
|
-
this.executeMergeQueryRecursively(remainingRecords, batchSize);
|
|
2083
|
+
await this.executeMergeQueryRecursively(remainingRecords, batchSize);
|
|
2084
2084
|
}
|
|
2085
2085
|
} catch (error) {
|
|
2086
2086
|
if (error.message && error.message.includes("query is too large")) {
|
|
2087
2087
|
console.log(`Query execution failed due to size. Reducing batch size from ${batchSize} to ${Math.floor(batchSize / 2)}`);
|
|
2088
|
-
this.executeMergeQueryRecursively(recordKeys, Math.floor(batchSize / 2));
|
|
2088
|
+
await this.executeMergeQueryRecursively(recordKeys, Math.floor(batchSize / 2));
|
|
2089
2089
|
} else {
|
|
2090
2090
|
throw error;
|
|
2091
2091
|
}
|