@8ms/helpers 3.2.8 → 3.2.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adverity/server/getJobs.cjs +1 -1
- package/dist/adverity/server/getJobs.mjs +1 -1
- package/dist/api/_shared/index.cjs +1 -1
- package/dist/api/_shared/index.mjs +1 -1
- package/dist/api/client/index.cjs +1 -1
- package/dist/api/client/index.mjs +1 -1
- package/dist/api/server/index.cjs +1 -1
- package/dist/api/server/index.mjs +1 -1
- package/dist/aws/s3/server/AwsS3.cjs +1 -1
- package/dist/aws/s3/server/AwsS3.mjs +1 -1
- package/dist/googleCloud/bigquery/_shared/types.d.cts +5 -19
- package/dist/googleCloud/bigquery/_shared/types.d.mts +5 -19
- package/dist/googleCloud/bigquery/server/GoogleCloudBigquery.cjs +27 -40
- package/dist/googleCloud/bigquery/server/GoogleCloudBigquery.d.cts +1 -1
- package/dist/googleCloud/bigquery/server/GoogleCloudBigquery.d.mts +1 -1
- package/dist/googleCloud/bigquery/server/GoogleCloudBigquery.mjs +27 -40
- package/dist/lodash/_shared/index.cjs +4 -4
- package/dist/lodash/_shared/index.mjs +4 -4
- package/dist/lodash/client/index.cjs +4 -4
- package/dist/lodash/client/index.mjs +4 -4
- package/dist/lodash/server/index.cjs +4 -4
- package/dist/lodash/server/index.mjs +4 -4
- package/dist/xlsx/_shared/createXlsx.cjs +1 -1
- package/dist/xlsx/_shared/createXlsx.mjs +1 -1
- package/package.json +1 -1
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
+
const require_api__shared_get = require("../../api/_shared/get.cjs");
|
|
2
3
|
const require_date__shared_calculation = require("../../date/_shared/calculation.cjs");
|
|
3
4
|
const require_date__shared_format = require("../../date/_shared/format.cjs");
|
|
4
|
-
const require_api__shared_get = require("../../api/_shared/get.cjs");
|
|
5
5
|
//#region src/adverity/server/getJobs.ts
|
|
6
6
|
/**
|
|
7
7
|
* Fetch all the jobs that ran yesterday by default.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
+
import { get } from "../../api/_shared/get.mjs";
|
|
1
2
|
import { getLuxonDate, getToday, getYesterday } from "../../date/_shared/calculation.mjs";
|
|
2
3
|
import { getYmdString } from "../../date/_shared/format.mjs";
|
|
3
|
-
import { get } from "../../api/_shared/get.mjs";
|
|
4
4
|
//#region src/adverity/server/getJobs.ts
|
|
5
5
|
/**
|
|
6
6
|
* Fetch all the jobs that ran yesterday by default.
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
2
|
const require_api__shared_api = require("./api.cjs");
|
|
3
3
|
const require_api__shared_ApiResponseClass = require("./ApiResponseClass.cjs");
|
|
4
|
-
const require_api__shared_get = require("./get.cjs");
|
|
5
4
|
const require_api__shared_deleteRequest = require("./deleteRequest.cjs");
|
|
6
5
|
const require_api__shared_functions = require("./functions.cjs");
|
|
6
|
+
const require_api__shared_get = require("./get.cjs");
|
|
7
7
|
const require_api__shared_httpMethod = require("./httpMethod.cjs");
|
|
8
8
|
const require_api__shared_post = require("./post.cjs");
|
|
9
9
|
exports.ApiResponseClass = require_api__shared_ApiResponseClass.ApiResponseClass;
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { apiState, defaultResponse, unexpectedError } from "./api.mjs";
|
|
2
2
|
import { ApiResponseClass } from "./ApiResponseClass.mjs";
|
|
3
|
-
import { get } from "./get.mjs";
|
|
4
3
|
import { deleteRequest } from "./deleteRequest.mjs";
|
|
5
4
|
import { isCancelled, isError, isIdle, isPending, isSuccess, isValidationError } from "./functions.mjs";
|
|
5
|
+
import { get } from "./get.mjs";
|
|
6
6
|
import { httpMethod } from "./httpMethod.mjs";
|
|
7
7
|
import { post } from "./post.mjs";
|
|
8
8
|
export { ApiResponseClass, apiState, defaultResponse, deleteRequest, get, httpMethod, isCancelled, isError, isIdle, isPending, isSuccess, isValidationError, post, unexpectedError };
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
2
|
const require_api__shared_api = require("../_shared/api.cjs");
|
|
3
3
|
const require_api__shared_ApiResponseClass = require("../_shared/ApiResponseClass.cjs");
|
|
4
|
-
const require_api__shared_get = require("../_shared/get.cjs");
|
|
5
4
|
const require_api__shared_deleteRequest = require("../_shared/deleteRequest.cjs");
|
|
6
5
|
const require_api__shared_functions = require("../_shared/functions.cjs");
|
|
6
|
+
const require_api__shared_get = require("../_shared/get.cjs");
|
|
7
7
|
const require_api__shared_httpMethod = require("../_shared/httpMethod.cjs");
|
|
8
8
|
const require_api__shared_post = require("../_shared/post.cjs");
|
|
9
9
|
require("../_shared/index.cjs");
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { apiState, defaultResponse, unexpectedError } from "../_shared/api.mjs";
|
|
2
2
|
import { ApiResponseClass } from "../_shared/ApiResponseClass.mjs";
|
|
3
|
-
import { get } from "../_shared/get.mjs";
|
|
4
3
|
import { deleteRequest } from "../_shared/deleteRequest.mjs";
|
|
5
4
|
import { isCancelled, isError, isIdle, isPending, isSuccess, isValidationError } from "../_shared/functions.mjs";
|
|
5
|
+
import { get } from "../_shared/get.mjs";
|
|
6
6
|
import { httpMethod } from "../_shared/httpMethod.mjs";
|
|
7
7
|
import { post } from "../_shared/post.mjs";
|
|
8
8
|
import "../_shared/index.mjs";
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
2
|
const require_api__shared_api = require("../_shared/api.cjs");
|
|
3
3
|
const require_api__shared_ApiResponseClass = require("../_shared/ApiResponseClass.cjs");
|
|
4
|
-
const require_api__shared_get = require("../_shared/get.cjs");
|
|
5
4
|
const require_api__shared_deleteRequest = require("../_shared/deleteRequest.cjs");
|
|
6
5
|
const require_api__shared_functions = require("../_shared/functions.cjs");
|
|
6
|
+
const require_api__shared_get = require("../_shared/get.cjs");
|
|
7
7
|
const require_api__shared_httpMethod = require("../_shared/httpMethod.cjs");
|
|
8
8
|
const require_api__shared_post = require("../_shared/post.cjs");
|
|
9
9
|
require("../_shared/index.cjs");
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { apiState, defaultResponse, unexpectedError } from "../_shared/api.mjs";
|
|
2
2
|
import { ApiResponseClass } from "../_shared/ApiResponseClass.mjs";
|
|
3
|
-
import { get } from "../_shared/get.mjs";
|
|
4
3
|
import { deleteRequest } from "../_shared/deleteRequest.mjs";
|
|
5
4
|
import { isCancelled, isError, isIdle, isPending, isSuccess, isValidationError } from "../_shared/functions.mjs";
|
|
5
|
+
import { get } from "../_shared/get.mjs";
|
|
6
6
|
import { httpMethod } from "../_shared/httpMethod.mjs";
|
|
7
7
|
import { post } from "../_shared/post.mjs";
|
|
8
8
|
import "../_shared/index.mjs";
|
|
@@ -3,9 +3,9 @@ const require_runtime = require("../../../_virtual/_rolldown/runtime.cjs");
|
|
|
3
3
|
const require__class__shared_BaseNamespace = require("../../../_class/_shared/BaseNamespace.cjs");
|
|
4
4
|
const require_date__shared_calculation = require("../../../date/_shared/calculation.cjs");
|
|
5
5
|
const require_date__shared_format = require("../../../date/_shared/format.cjs");
|
|
6
|
+
const require_aws_server_isResponse200 = require("../../server/isResponse200.cjs");
|
|
6
7
|
const require_date__shared_date = require("../../../date/_shared/date.cjs");
|
|
7
8
|
require("../../../date/_shared/index.cjs");
|
|
8
|
-
const require_aws_server_isResponse200 = require("../../server/isResponse200.cjs");
|
|
9
9
|
const require_stream_server_json_read = require("../../../stream/server/json/read.cjs");
|
|
10
10
|
const require_stream_server_json_write = require("../../../stream/server/json/write.cjs");
|
|
11
11
|
require("../../../stream/server/index.cjs");
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { BaseNamespace } from "../../../_class/_shared/BaseNamespace.mjs";
|
|
2
2
|
import { getToday } from "../../../date/_shared/calculation.mjs";
|
|
3
3
|
import { getYmdHisString } from "../../../date/_shared/format.mjs";
|
|
4
|
+
import { isResponse200 } from "../../server/isResponse200.mjs";
|
|
4
5
|
import { defaultDateTime } from "../../../date/_shared/date.mjs";
|
|
5
6
|
import "../../../date/_shared/index.mjs";
|
|
6
|
-
import { isResponse200 } from "../../server/isResponse200.mjs";
|
|
7
7
|
import { readJsonGzipStream, readJsonStream } from "../../../stream/server/json/read.mjs";
|
|
8
8
|
import { writeJsonGzipStream, writeJsonStream } from "../../../stream/server/json/write.mjs";
|
|
9
9
|
import "../../../stream/server/index.mjs";
|
|
@@ -10,7 +10,7 @@ export type BigqueryLoadFormat = typeof fileType.CSV | typeof fileType.JSON_NEWL
|
|
|
10
10
|
export type TableDefinition = {
|
|
11
11
|
projectId?: string;
|
|
12
12
|
datasetId: string;
|
|
13
|
-
tableId
|
|
13
|
+
tableId?: string;
|
|
14
14
|
options?: {
|
|
15
15
|
location?: string;
|
|
16
16
|
schema: BQField[];
|
|
@@ -45,28 +45,14 @@ export type LoadData<TData> = {
|
|
|
45
45
|
sourceFormat?: BigqueryLoadFormat;
|
|
46
46
|
region?: string;
|
|
47
47
|
};
|
|
48
|
-
table: {
|
|
49
|
-
|
|
50
|
-
tableId?: string;
|
|
51
|
-
region?: string;
|
|
52
|
-
definition: TableDefinition;
|
|
53
|
-
options?: TableDefinition["options"];
|
|
48
|
+
table: TableDefinition & {
|
|
49
|
+
tableId: string;
|
|
54
50
|
};
|
|
55
51
|
};
|
|
56
52
|
export type LoadTableProps = LoadData<any[]>;
|
|
57
53
|
export type LoadStreamTableProps = LoadData<RowSource<any>>;
|
|
58
|
-
export type MergeTableProps = {
|
|
59
|
-
|
|
60
|
-
file: {
|
|
61
|
-
bucket: string;
|
|
62
|
-
key: string;
|
|
63
|
-
data: any[];
|
|
64
|
-
};
|
|
65
|
-
table: {
|
|
66
|
-
datasetId: string;
|
|
67
|
-
tableId?: string;
|
|
68
|
-
region?: string;
|
|
69
|
-
definition: TableDefinition;
|
|
54
|
+
export type MergeTableProps = Omit<LoadData<any[]>, "table"> & {
|
|
55
|
+
table: LoadData<any[]>["table"] & {
|
|
70
56
|
mergeKey: string;
|
|
71
57
|
updateColumns: string[];
|
|
72
58
|
};
|
|
@@ -10,7 +10,7 @@ export type BigqueryLoadFormat = typeof fileType.CSV | typeof fileType.JSON_NEWL
|
|
|
10
10
|
export type TableDefinition = {
|
|
11
11
|
projectId?: string;
|
|
12
12
|
datasetId: string;
|
|
13
|
-
tableId
|
|
13
|
+
tableId?: string;
|
|
14
14
|
options?: {
|
|
15
15
|
location?: string;
|
|
16
16
|
schema: BQField[];
|
|
@@ -45,28 +45,14 @@ export type LoadData<TData> = {
|
|
|
45
45
|
sourceFormat?: BigqueryLoadFormat;
|
|
46
46
|
region?: string;
|
|
47
47
|
};
|
|
48
|
-
table: {
|
|
49
|
-
|
|
50
|
-
tableId?: string;
|
|
51
|
-
region?: string;
|
|
52
|
-
definition: TableDefinition;
|
|
53
|
-
options?: TableDefinition["options"];
|
|
48
|
+
table: TableDefinition & {
|
|
49
|
+
tableId: string;
|
|
54
50
|
};
|
|
55
51
|
};
|
|
56
52
|
export type LoadTableProps = LoadData<any[]>;
|
|
57
53
|
export type LoadStreamTableProps = LoadData<RowSource<any>>;
|
|
58
|
-
export type MergeTableProps = {
|
|
59
|
-
|
|
60
|
-
file: {
|
|
61
|
-
bucket: string;
|
|
62
|
-
key: string;
|
|
63
|
-
data: any[];
|
|
64
|
-
};
|
|
65
|
-
table: {
|
|
66
|
-
datasetId: string;
|
|
67
|
-
tableId?: string;
|
|
68
|
-
region?: string;
|
|
69
|
-
definition: TableDefinition;
|
|
54
|
+
export type MergeTableProps = Omit<LoadData<any[]>, "table"> & {
|
|
55
|
+
table: LoadData<any[]>["table"] & {
|
|
70
56
|
mergeKey: string;
|
|
71
57
|
updateColumns: string[];
|
|
72
58
|
};
|
|
@@ -147,13 +147,13 @@ var GoogleCloudBigquery = class extends require__class__shared_BaseNamespace.Bas
|
|
|
147
147
|
* pre-checking existence and silently skipping the load if it isn't there.
|
|
148
148
|
*/
|
|
149
149
|
loadData = async (props) => {
|
|
150
|
-
const
|
|
151
|
-
await this.createTable(props.table.datasetId, {},
|
|
152
|
-
await this.client.dataset(props.table.datasetId, { location:
|
|
150
|
+
const finalRegion = props.table.options?.location ?? this.region;
|
|
151
|
+
await this.createTable(props.table.datasetId, {}, props.table.tableId, props.table.options);
|
|
152
|
+
await this.client.dataset(props.table.datasetId, { location: finalRegion }).table(props.table.tableId).load(props.googleCloudStorage.client.bucket(props.file.bucket).file(props.file.key), {
|
|
153
153
|
autodetect: true,
|
|
154
154
|
sourceFormat: bigQuerySourceFormat[props.file.sourceFormat ?? require_file__shared_fileType.fileType.JSON_NEWLINE],
|
|
155
155
|
schemaUpdateOptions: ["ALLOW_FIELD_ADDITION"],
|
|
156
|
-
location:
|
|
156
|
+
location: finalRegion
|
|
157
157
|
});
|
|
158
158
|
};
|
|
159
159
|
/**
|
|
@@ -163,17 +163,16 @@ var GoogleCloudBigquery = class extends require__class__shared_BaseNamespace.Bas
|
|
|
163
163
|
* this library has no Parquet encoder.
|
|
164
164
|
*/
|
|
165
165
|
loadTable = async (props) => {
|
|
166
|
-
const
|
|
167
|
-
|
|
168
|
-
|
|
166
|
+
const finalSourceFormat = props.file?.sourceFormat ?? require_file__shared_fileType.fileType.JSON_NEWLINE;
|
|
167
|
+
let finalData;
|
|
168
|
+
switch (finalSourceFormat) {
|
|
169
|
+
case require_file__shared_fileType.fileType.JSON_NEWLINE:
|
|
170
|
+
finalData = require_json__shared_getJsonNewline.getJsonNewline(props.file.data);
|
|
171
|
+
break;
|
|
172
|
+
default: finalData = props.file.data;
|
|
173
|
+
}
|
|
169
174
|
await props.googleCloudStorage.writeFile(props.file.bucket, props.file.key, finalData);
|
|
170
|
-
await this.loadData(
|
|
171
|
-
...props,
|
|
172
|
-
table: {
|
|
173
|
-
...props.table,
|
|
174
|
-
tableId: finalTableId
|
|
175
|
-
}
|
|
176
|
-
});
|
|
175
|
+
await this.loadData(props);
|
|
177
176
|
};
|
|
178
177
|
/**
|
|
179
178
|
* Same as loadTable, but streams a row source into Google Cloud Storage instead of buffering the
|
|
@@ -184,7 +183,6 @@ var GoogleCloudBigquery = class extends require__class__shared_BaseNamespace.Bas
|
|
|
184
183
|
* byte-level Readable of encoded Parquet content, piped straight through.
|
|
185
184
|
*/
|
|
186
185
|
loadStreamTable = async (props) => {
|
|
187
|
-
const finalTableId = props.table.tableId ?? props.table.definition.tableId;
|
|
188
186
|
const fileSourceFormat = props.file.sourceFormat ?? require_file__shared_fileType.fileType.JSON_NEWLINE;
|
|
189
187
|
const destination = props.googleCloudStorage.client.bucket(props.file.bucket).file(props.file.key).createWriteStream({
|
|
190
188
|
contentType: require_file__shared_fileMimeType.fileMimeType[fileSourceFormat],
|
|
@@ -197,25 +195,19 @@ var GoogleCloudBigquery = class extends require__class__shared_BaseNamespace.Bas
|
|
|
197
195
|
destination
|
|
198
196
|
]);
|
|
199
197
|
else await (0, node_stream_promises.pipeline)([source, destination]);
|
|
200
|
-
await this.loadData(
|
|
201
|
-
...props,
|
|
202
|
-
table: {
|
|
203
|
-
...props.table,
|
|
204
|
-
tableId: finalTableId
|
|
205
|
-
}
|
|
206
|
-
});
|
|
198
|
+
await this.loadData(props);
|
|
207
199
|
};
|
|
208
200
|
/**
|
|
209
201
|
* Brings an existing table's clustering in line with `definition.options.clustering`, if it differs.
|
|
210
202
|
* Partitioning is immutable on an existing BigQuery table - changing it requires recreating the
|
|
211
203
|
* table and backfilling data, which this method deliberately does not attempt.
|
|
212
204
|
*/
|
|
213
|
-
|
|
214
|
-
const [metadata] = await this.client.dataset(datasetId).table(tableId).getMetadata();
|
|
205
|
+
updateTableClustering = async (table) => {
|
|
206
|
+
const [metadata] = await this.client.dataset(table.datasetId).table(table.tableId).getMetadata();
|
|
215
207
|
const currentFields = metadata?.clustering?.fields ?? [];
|
|
216
|
-
const desiredFields = clustering?.fields ?? [];
|
|
208
|
+
const desiredFields = table.options?.clustering?.fields ?? [];
|
|
217
209
|
if (currentFields.length === desiredFields.length && currentFields.every((field, index) => field === desiredFields[index])) return;
|
|
218
|
-
await this.client.dataset(datasetId).table(tableId).setMetadata(desiredFields.length > 0 ? { clustering: { fields: desiredFields } } : { clustering: void 0 });
|
|
210
|
+
await this.client.dataset(table.datasetId).table(table.tableId).setMetadata(desiredFields.length > 0 ? { clustering: { fields: desiredFields } } : { clustering: void 0 });
|
|
219
211
|
};
|
|
220
212
|
/**
|
|
221
213
|
* Upsert rows into a table via a staging table + a single MERGE statement.
|
|
@@ -226,16 +218,16 @@ var GoogleCloudBigquery = class extends require__class__shared_BaseNamespace.Bas
|
|
|
226
218
|
*/
|
|
227
219
|
mergeTable = async (props) => {
|
|
228
220
|
if (0 === props.file.data.length) return;
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
const stagingTableId = `${finalTableId}_staging_${require_date__shared_format.getYmdHisNumber()}`;
|
|
221
|
+
if (await this.tableExists(props.table.datasetId, props.table.tableId)) await this.updateTableClustering(props.table);
|
|
222
|
+
const stagingTableId = `${props.table.tableId}_staging_${require_date__shared_format.getYmdHisNumber()}`;
|
|
232
223
|
const stagingTableRef = `\`${props.table.datasetId}.${stagingTableId}\``;
|
|
233
224
|
try {
|
|
225
|
+
const { mergeKey: _mergeKey, updateColumns: _updateColumns, ...tableDefinition } = props.table;
|
|
234
226
|
const stagingDefinition = {
|
|
235
|
-
...
|
|
227
|
+
...tableDefinition,
|
|
236
228
|
tableId: stagingTableId,
|
|
237
|
-
options: props.table.
|
|
238
|
-
...props.table.
|
|
229
|
+
options: props.table.options ? {
|
|
230
|
+
...props.table.options,
|
|
239
231
|
timePartitioning: void 0,
|
|
240
232
|
clustering: void 0
|
|
241
233
|
} : void 0
|
|
@@ -243,16 +235,11 @@ var GoogleCloudBigquery = class extends require__class__shared_BaseNamespace.Bas
|
|
|
243
235
|
await this.loadTable({
|
|
244
236
|
googleCloudStorage: props.googleCloudStorage,
|
|
245
237
|
file: props.file,
|
|
246
|
-
table:
|
|
247
|
-
datasetId: props.table.datasetId,
|
|
248
|
-
definition: stagingDefinition,
|
|
249
|
-
tableId: stagingTableId,
|
|
250
|
-
region: props.table.region
|
|
251
|
-
}
|
|
238
|
+
table: stagingDefinition
|
|
252
239
|
});
|
|
253
|
-
const mainTableRef = `\`${props.table.datasetId}.${
|
|
240
|
+
const mainTableRef = `\`${props.table.datasetId}.${props.table.tableId}\``;
|
|
254
241
|
const updateSet = props.table.updateColumns.map((column) => `${column} = S.${column}`).join(", ");
|
|
255
|
-
const allColumns = props.table.
|
|
242
|
+
const allColumns = props.table.options?.schema.map((field) => field.name) ?? [];
|
|
256
243
|
const mergeQuery = `
|
|
257
244
|
MERGE ${mainTableRef} AS T
|
|
258
245
|
USING ${stagingTableRef} AS S
|
|
@@ -79,7 +79,7 @@ export declare class GoogleCloudBigquery extends BaseNamespace {
|
|
|
79
79
|
* Partitioning is immutable on an existing BigQuery table - changing it requires recreating the
|
|
80
80
|
* table and backfilling data, which this method deliberately does not attempt.
|
|
81
81
|
*/
|
|
82
|
-
private
|
|
82
|
+
private updateTableClustering;
|
|
83
83
|
/**
|
|
84
84
|
* Upsert rows into a table via a staging table + a single MERGE statement.
|
|
85
85
|
*
|
|
@@ -79,7 +79,7 @@ export declare class GoogleCloudBigquery extends BaseNamespace {
|
|
|
79
79
|
* Partitioning is immutable on an existing BigQuery table - changing it requires recreating the
|
|
80
80
|
* table and backfilling data, which this method deliberately does not attempt.
|
|
81
81
|
*/
|
|
82
|
-
private
|
|
82
|
+
private updateTableClustering;
|
|
83
83
|
/**
|
|
84
84
|
* Upsert rows into a table via a staging table + a single MERGE statement.
|
|
85
85
|
*
|
|
@@ -146,13 +146,13 @@ var GoogleCloudBigquery = class extends BaseNamespace {
|
|
|
146
146
|
* pre-checking existence and silently skipping the load if it isn't there.
|
|
147
147
|
*/
|
|
148
148
|
loadData = async (props) => {
|
|
149
|
-
const
|
|
150
|
-
await this.createTable(props.table.datasetId, {},
|
|
151
|
-
await this.client.dataset(props.table.datasetId, { location:
|
|
149
|
+
const finalRegion = props.table.options?.location ?? this.region;
|
|
150
|
+
await this.createTable(props.table.datasetId, {}, props.table.tableId, props.table.options);
|
|
151
|
+
await this.client.dataset(props.table.datasetId, { location: finalRegion }).table(props.table.tableId).load(props.googleCloudStorage.client.bucket(props.file.bucket).file(props.file.key), {
|
|
152
152
|
autodetect: true,
|
|
153
153
|
sourceFormat: bigQuerySourceFormat[props.file.sourceFormat ?? fileType.JSON_NEWLINE],
|
|
154
154
|
schemaUpdateOptions: ["ALLOW_FIELD_ADDITION"],
|
|
155
|
-
location:
|
|
155
|
+
location: finalRegion
|
|
156
156
|
});
|
|
157
157
|
};
|
|
158
158
|
/**
|
|
@@ -162,17 +162,16 @@ var GoogleCloudBigquery = class extends BaseNamespace {
|
|
|
162
162
|
* this library has no Parquet encoder.
|
|
163
163
|
*/
|
|
164
164
|
loadTable = async (props) => {
|
|
165
|
-
const
|
|
166
|
-
|
|
167
|
-
|
|
165
|
+
const finalSourceFormat = props.file?.sourceFormat ?? fileType.JSON_NEWLINE;
|
|
166
|
+
let finalData;
|
|
167
|
+
switch (finalSourceFormat) {
|
|
168
|
+
case fileType.JSON_NEWLINE:
|
|
169
|
+
finalData = getJsonNewline(props.file.data);
|
|
170
|
+
break;
|
|
171
|
+
default: finalData = props.file.data;
|
|
172
|
+
}
|
|
168
173
|
await props.googleCloudStorage.writeFile(props.file.bucket, props.file.key, finalData);
|
|
169
|
-
await this.loadData(
|
|
170
|
-
...props,
|
|
171
|
-
table: {
|
|
172
|
-
...props.table,
|
|
173
|
-
tableId: finalTableId
|
|
174
|
-
}
|
|
175
|
-
});
|
|
174
|
+
await this.loadData(props);
|
|
176
175
|
};
|
|
177
176
|
/**
|
|
178
177
|
* Same as loadTable, but streams a row source into Google Cloud Storage instead of buffering the
|
|
@@ -183,7 +182,6 @@ var GoogleCloudBigquery = class extends BaseNamespace {
|
|
|
183
182
|
* byte-level Readable of encoded Parquet content, piped straight through.
|
|
184
183
|
*/
|
|
185
184
|
loadStreamTable = async (props) => {
|
|
186
|
-
const finalTableId = props.table.tableId ?? props.table.definition.tableId;
|
|
187
185
|
const fileSourceFormat = props.file.sourceFormat ?? fileType.JSON_NEWLINE;
|
|
188
186
|
const destination = props.googleCloudStorage.client.bucket(props.file.bucket).file(props.file.key).createWriteStream({
|
|
189
187
|
contentType: fileMimeType[fileSourceFormat],
|
|
@@ -196,25 +194,19 @@ var GoogleCloudBigquery = class extends BaseNamespace {
|
|
|
196
194
|
destination
|
|
197
195
|
]);
|
|
198
196
|
else await pipeline([source, destination]);
|
|
199
|
-
await this.loadData(
|
|
200
|
-
...props,
|
|
201
|
-
table: {
|
|
202
|
-
...props.table,
|
|
203
|
-
tableId: finalTableId
|
|
204
|
-
}
|
|
205
|
-
});
|
|
197
|
+
await this.loadData(props);
|
|
206
198
|
};
|
|
207
199
|
/**
|
|
208
200
|
* Brings an existing table's clustering in line with `definition.options.clustering`, if it differs.
|
|
209
201
|
* Partitioning is immutable on an existing BigQuery table - changing it requires recreating the
|
|
210
202
|
* table and backfilling data, which this method deliberately does not attempt.
|
|
211
203
|
*/
|
|
212
|
-
|
|
213
|
-
const [metadata] = await this.client.dataset(datasetId).table(tableId).getMetadata();
|
|
204
|
+
updateTableClustering = async (table) => {
|
|
205
|
+
const [metadata] = await this.client.dataset(table.datasetId).table(table.tableId).getMetadata();
|
|
214
206
|
const currentFields = metadata?.clustering?.fields ?? [];
|
|
215
|
-
const desiredFields = clustering?.fields ?? [];
|
|
207
|
+
const desiredFields = table.options?.clustering?.fields ?? [];
|
|
216
208
|
if (currentFields.length === desiredFields.length && currentFields.every((field, index) => field === desiredFields[index])) return;
|
|
217
|
-
await this.client.dataset(datasetId).table(tableId).setMetadata(desiredFields.length > 0 ? { clustering: { fields: desiredFields } } : { clustering: void 0 });
|
|
209
|
+
await this.client.dataset(table.datasetId).table(table.tableId).setMetadata(desiredFields.length > 0 ? { clustering: { fields: desiredFields } } : { clustering: void 0 });
|
|
218
210
|
};
|
|
219
211
|
/**
|
|
220
212
|
* Upsert rows into a table via a staging table + a single MERGE statement.
|
|
@@ -225,16 +217,16 @@ var GoogleCloudBigquery = class extends BaseNamespace {
|
|
|
225
217
|
*/
|
|
226
218
|
mergeTable = async (props) => {
|
|
227
219
|
if (0 === props.file.data.length) return;
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
const stagingTableId = `${finalTableId}_staging_${getYmdHisNumber()}`;
|
|
220
|
+
if (await this.tableExists(props.table.datasetId, props.table.tableId)) await this.updateTableClustering(props.table);
|
|
221
|
+
const stagingTableId = `${props.table.tableId}_staging_${getYmdHisNumber()}`;
|
|
231
222
|
const stagingTableRef = `\`${props.table.datasetId}.${stagingTableId}\``;
|
|
232
223
|
try {
|
|
224
|
+
const { mergeKey: _mergeKey, updateColumns: _updateColumns, ...tableDefinition } = props.table;
|
|
233
225
|
const stagingDefinition = {
|
|
234
|
-
...
|
|
226
|
+
...tableDefinition,
|
|
235
227
|
tableId: stagingTableId,
|
|
236
|
-
options: props.table.
|
|
237
|
-
...props.table.
|
|
228
|
+
options: props.table.options ? {
|
|
229
|
+
...props.table.options,
|
|
238
230
|
timePartitioning: void 0,
|
|
239
231
|
clustering: void 0
|
|
240
232
|
} : void 0
|
|
@@ -242,16 +234,11 @@ var GoogleCloudBigquery = class extends BaseNamespace {
|
|
|
242
234
|
await this.loadTable({
|
|
243
235
|
googleCloudStorage: props.googleCloudStorage,
|
|
244
236
|
file: props.file,
|
|
245
|
-
table:
|
|
246
|
-
datasetId: props.table.datasetId,
|
|
247
|
-
definition: stagingDefinition,
|
|
248
|
-
tableId: stagingTableId,
|
|
249
|
-
region: props.table.region
|
|
250
|
-
}
|
|
237
|
+
table: stagingDefinition
|
|
251
238
|
});
|
|
252
|
-
const mainTableRef = `\`${props.table.datasetId}.${
|
|
239
|
+
const mainTableRef = `\`${props.table.datasetId}.${props.table.tableId}\``;
|
|
253
240
|
const updateSet = props.table.updateColumns.map((column) => `${column} = S.${column}`).join(", ");
|
|
254
|
-
const allColumns = props.table.
|
|
241
|
+
const allColumns = props.table.options?.schema.map((field) => field.name) ?? [];
|
|
255
242
|
const mergeQuery = `
|
|
256
243
|
MERGE ${mainTableRef} AS T
|
|
257
244
|
USING ${stagingTableRef} AS S
|
|
@@ -2,16 +2,16 @@ Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
|
2
2
|
const require_lodash__shared_set = require("./set.cjs");
|
|
3
3
|
const require_lodash__shared_isObject = require("./isObject.cjs");
|
|
4
4
|
const require_lodash__shared_isDate = require("./isDate.cjs");
|
|
5
|
+
const require_lodash__shared_merge = require("./merge.cjs");
|
|
6
|
+
const require_lodash__shared_unescape = require("./unescape.cjs");
|
|
7
|
+
const require_lodash__shared_trim = require("./trim.cjs");
|
|
5
8
|
const require_lodash__shared_chunk = require("./chunk.cjs");
|
|
6
|
-
const require_lodash__shared_findKey = require("./findKey.cjs");
|
|
7
9
|
const require_lodash__shared_flatten = require("./flatten.cjs");
|
|
8
|
-
const
|
|
10
|
+
const require_lodash__shared_findKey = require("./findKey.cjs");
|
|
9
11
|
const require_lodash__shared_isEmpty = require("./isEmpty.cjs");
|
|
10
12
|
const require_lodash__shared_isEqual = require("./isEqual.cjs");
|
|
11
13
|
const require_lodash__shared_isFunction = require("./isFunction.cjs");
|
|
12
|
-
const require_lodash__shared_merge = require("./merge.cjs");
|
|
13
14
|
const require_lodash__shared_orderBy = require("./orderBy.cjs");
|
|
14
|
-
const require_lodash__shared_unescape = require("./unescape.cjs");
|
|
15
15
|
const require_lodash__shared_uniq = require("./uniq.cjs");
|
|
16
16
|
const require_lodash__shared_uniqBy = require("./uniqBy.cjs");
|
|
17
17
|
exports.chunk = require_lodash__shared_chunk.chunk;
|
|
@@ -1,16 +1,16 @@
|
|
|
1
1
|
import { set } from "./set.mjs";
|
|
2
2
|
import { isObject } from "./isObject.mjs";
|
|
3
3
|
import { isDate } from "./isDate.mjs";
|
|
4
|
+
import { merge } from "./merge.mjs";
|
|
5
|
+
import { unescape } from "./unescape.mjs";
|
|
6
|
+
import { trim } from "./trim.mjs";
|
|
4
7
|
import { chunk } from "./chunk.mjs";
|
|
5
|
-
import { findKey } from "./findKey.mjs";
|
|
6
8
|
import { flatten } from "./flatten.mjs";
|
|
7
|
-
import {
|
|
9
|
+
import { findKey } from "./findKey.mjs";
|
|
8
10
|
import { isEmpty } from "./isEmpty.mjs";
|
|
9
11
|
import { isEqual } from "./isEqual.mjs";
|
|
10
12
|
import { isFunction } from "./isFunction.mjs";
|
|
11
|
-
import { merge } from "./merge.mjs";
|
|
12
13
|
import { orderBy } from "./orderBy.mjs";
|
|
13
|
-
import { unescape } from "./unescape.mjs";
|
|
14
14
|
import { uniq } from "./uniq.mjs";
|
|
15
15
|
import { uniqBy } from "./uniqBy.mjs";
|
|
16
16
|
export { chunk, findKey, flatten, isDate, isEmpty, isEqual, isFunction, isObject, merge, orderBy, set, trim, unescape, uniq, uniqBy };
|
|
@@ -2,16 +2,16 @@ Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
|
2
2
|
const require_lodash__shared_set = require("../_shared/set.cjs");
|
|
3
3
|
const require_lodash__shared_isObject = require("../_shared/isObject.cjs");
|
|
4
4
|
const require_lodash__shared_isDate = require("../_shared/isDate.cjs");
|
|
5
|
+
const require_lodash__shared_merge = require("../_shared/merge.cjs");
|
|
6
|
+
const require_lodash__shared_unescape = require("../_shared/unescape.cjs");
|
|
7
|
+
const require_lodash__shared_trim = require("../_shared/trim.cjs");
|
|
5
8
|
const require_lodash__shared_chunk = require("../_shared/chunk.cjs");
|
|
6
|
-
const require_lodash__shared_findKey = require("../_shared/findKey.cjs");
|
|
7
9
|
const require_lodash__shared_flatten = require("../_shared/flatten.cjs");
|
|
8
|
-
const
|
|
10
|
+
const require_lodash__shared_findKey = require("../_shared/findKey.cjs");
|
|
9
11
|
const require_lodash__shared_isEmpty = require("../_shared/isEmpty.cjs");
|
|
10
12
|
const require_lodash__shared_isEqual = require("../_shared/isEqual.cjs");
|
|
11
13
|
const require_lodash__shared_isFunction = require("../_shared/isFunction.cjs");
|
|
12
|
-
const require_lodash__shared_merge = require("../_shared/merge.cjs");
|
|
13
14
|
const require_lodash__shared_orderBy = require("../_shared/orderBy.cjs");
|
|
14
|
-
const require_lodash__shared_unescape = require("../_shared/unescape.cjs");
|
|
15
15
|
const require_lodash__shared_uniq = require("../_shared/uniq.cjs");
|
|
16
16
|
const require_lodash__shared_uniqBy = require("../_shared/uniqBy.cjs");
|
|
17
17
|
exports.chunk = require_lodash__shared_chunk.chunk;
|
|
@@ -1,16 +1,16 @@
|
|
|
1
1
|
import { set } from "../_shared/set.mjs";
|
|
2
2
|
import { isObject } from "../_shared/isObject.mjs";
|
|
3
3
|
import { isDate } from "../_shared/isDate.mjs";
|
|
4
|
+
import { merge } from "../_shared/merge.mjs";
|
|
5
|
+
import { unescape } from "../_shared/unescape.mjs";
|
|
6
|
+
import { trim } from "../_shared/trim.mjs";
|
|
4
7
|
import { chunk } from "../_shared/chunk.mjs";
|
|
5
|
-
import { findKey } from "../_shared/findKey.mjs";
|
|
6
8
|
import { flatten } from "../_shared/flatten.mjs";
|
|
7
|
-
import {
|
|
9
|
+
import { findKey } from "../_shared/findKey.mjs";
|
|
8
10
|
import { isEmpty } from "../_shared/isEmpty.mjs";
|
|
9
11
|
import { isEqual } from "../_shared/isEqual.mjs";
|
|
10
12
|
import { isFunction } from "../_shared/isFunction.mjs";
|
|
11
|
-
import { merge } from "../_shared/merge.mjs";
|
|
12
13
|
import { orderBy } from "../_shared/orderBy.mjs";
|
|
13
|
-
import { unescape } from "../_shared/unescape.mjs";
|
|
14
14
|
import { uniq } from "../_shared/uniq.mjs";
|
|
15
15
|
import { uniqBy } from "../_shared/uniqBy.mjs";
|
|
16
16
|
export { chunk, findKey, flatten, isDate, isEmpty, isEqual, isFunction, isObject, merge, orderBy, set, trim, unescape, uniq, uniqBy };
|
|
@@ -2,16 +2,16 @@ Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
|
2
2
|
const require_lodash__shared_set = require("../_shared/set.cjs");
|
|
3
3
|
const require_lodash__shared_isObject = require("../_shared/isObject.cjs");
|
|
4
4
|
const require_lodash__shared_isDate = require("../_shared/isDate.cjs");
|
|
5
|
+
const require_lodash__shared_merge = require("../_shared/merge.cjs");
|
|
6
|
+
const require_lodash__shared_unescape = require("../_shared/unescape.cjs");
|
|
7
|
+
const require_lodash__shared_trim = require("../_shared/trim.cjs");
|
|
5
8
|
const require_lodash__shared_chunk = require("../_shared/chunk.cjs");
|
|
6
|
-
const require_lodash__shared_findKey = require("../_shared/findKey.cjs");
|
|
7
9
|
const require_lodash__shared_flatten = require("../_shared/flatten.cjs");
|
|
8
|
-
const
|
|
10
|
+
const require_lodash__shared_findKey = require("../_shared/findKey.cjs");
|
|
9
11
|
const require_lodash__shared_isEmpty = require("../_shared/isEmpty.cjs");
|
|
10
12
|
const require_lodash__shared_isEqual = require("../_shared/isEqual.cjs");
|
|
11
13
|
const require_lodash__shared_isFunction = require("../_shared/isFunction.cjs");
|
|
12
|
-
const require_lodash__shared_merge = require("../_shared/merge.cjs");
|
|
13
14
|
const require_lodash__shared_orderBy = require("../_shared/orderBy.cjs");
|
|
14
|
-
const require_lodash__shared_unescape = require("../_shared/unescape.cjs");
|
|
15
15
|
const require_lodash__shared_uniq = require("../_shared/uniq.cjs");
|
|
16
16
|
const require_lodash__shared_uniqBy = require("../_shared/uniqBy.cjs");
|
|
17
17
|
exports.chunk = require_lodash__shared_chunk.chunk;
|
|
@@ -1,16 +1,16 @@
|
|
|
1
1
|
import { set } from "../_shared/set.mjs";
|
|
2
2
|
import { isObject } from "../_shared/isObject.mjs";
|
|
3
3
|
import { isDate } from "../_shared/isDate.mjs";
|
|
4
|
+
import { merge } from "../_shared/merge.mjs";
|
|
5
|
+
import { unescape } from "../_shared/unescape.mjs";
|
|
6
|
+
import { trim } from "../_shared/trim.mjs";
|
|
4
7
|
import { chunk } from "../_shared/chunk.mjs";
|
|
5
|
-
import { findKey } from "../_shared/findKey.mjs";
|
|
6
8
|
import { flatten } from "../_shared/flatten.mjs";
|
|
7
|
-
import {
|
|
9
|
+
import { findKey } from "../_shared/findKey.mjs";
|
|
8
10
|
import { isEmpty } from "../_shared/isEmpty.mjs";
|
|
9
11
|
import { isEqual } from "../_shared/isEqual.mjs";
|
|
10
12
|
import { isFunction } from "../_shared/isFunction.mjs";
|
|
11
|
-
import { merge } from "../_shared/merge.mjs";
|
|
12
13
|
import { orderBy } from "../_shared/orderBy.mjs";
|
|
13
|
-
import { unescape } from "../_shared/unescape.mjs";
|
|
14
14
|
import { uniq } from "../_shared/uniq.mjs";
|
|
15
15
|
import { uniqBy } from "../_shared/uniqBy.mjs";
|
|
16
16
|
export { chunk, findKey, flatten, isDate, isEmpty, isEqual, isFunction, isObject, merge, orderBy, set, trim, unescape, uniq, uniqBy };
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
2
|
const require_xlsx__shared_column = require("./column.cjs");
|
|
3
3
|
const require_xlsx__shared_cell = require("./cell.cjs");
|
|
4
|
-
const require_xlsx__shared_sheet = require("./sheet.cjs");
|
|
5
4
|
const require_zip__shared_ZipArchive = require("../../zip/_shared/ZipArchive.cjs");
|
|
5
|
+
const require_xlsx__shared_sheet = require("./sheet.cjs");
|
|
6
6
|
//#region src/xlsx/_shared/createXlsx.ts
|
|
7
7
|
/**
|
|
8
8
|
* Builds a complete .xlsx workbook from scratch, using hand-written OOXML rather
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { getColumLetter, getColumnWidth } from "./column.mjs";
|
|
2
2
|
import { buildCellXml, getCellValue, getEscapedXml } from "./cell.mjs";
|
|
3
|
-
import { getUniqueSheetNames } from "./sheet.mjs";
|
|
4
3
|
import { ZipArchive } from "../../zip/_shared/ZipArchive.mjs";
|
|
4
|
+
import { getUniqueSheetNames } from "./sheet.mjs";
|
|
5
5
|
//#region src/xlsx/_shared/createXlsx.ts
|
|
6
6
|
/**
|
|
7
7
|
* Builds a complete .xlsx workbook from scratch, using hand-written OOXML rather
|