@8ms/helpers 3.2.8 → 3.2.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/aws/s3/server/AwsS3.cjs +1 -1
- package/dist/aws/s3/server/AwsS3.mjs +1 -1
- package/dist/googleCloud/bigquery/_shared/types.d.cts +3 -14
- package/dist/googleCloud/bigquery/_shared/types.d.mts +3 -14
- package/dist/googleCloud/bigquery/server/GoogleCloudBigquery.cjs +14 -24
- package/dist/googleCloud/bigquery/server/GoogleCloudBigquery.mjs +14 -24
- package/dist/xlsx/_shared/createXlsx.cjs +1 -1
- package/dist/xlsx/_shared/createXlsx.mjs +1 -1
- package/package.json +1 -1
|
@@ -3,9 +3,9 @@ const require_runtime = require("../../../_virtual/_rolldown/runtime.cjs");
|
|
|
3
3
|
const require__class__shared_BaseNamespace = require("../../../_class/_shared/BaseNamespace.cjs");
|
|
4
4
|
const require_date__shared_calculation = require("../../../date/_shared/calculation.cjs");
|
|
5
5
|
const require_date__shared_format = require("../../../date/_shared/format.cjs");
|
|
6
|
+
const require_aws_server_isResponse200 = require("../../server/isResponse200.cjs");
|
|
6
7
|
const require_date__shared_date = require("../../../date/_shared/date.cjs");
|
|
7
8
|
require("../../../date/_shared/index.cjs");
|
|
8
|
-
const require_aws_server_isResponse200 = require("../../server/isResponse200.cjs");
|
|
9
9
|
const require_stream_server_json_read = require("../../../stream/server/json/read.cjs");
|
|
10
10
|
const require_stream_server_json_write = require("../../../stream/server/json/write.cjs");
|
|
11
11
|
require("../../../stream/server/index.cjs");
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { BaseNamespace } from "../../../_class/_shared/BaseNamespace.mjs";
|
|
2
2
|
import { getToday } from "../../../date/_shared/calculation.mjs";
|
|
3
3
|
import { getYmdHisString } from "../../../date/_shared/format.mjs";
|
|
4
|
+
import { isResponse200 } from "../../server/isResponse200.mjs";
|
|
4
5
|
import { defaultDateTime } from "../../../date/_shared/date.mjs";
|
|
5
6
|
import "../../../date/_shared/index.mjs";
|
|
6
|
-
import { isResponse200 } from "../../server/isResponse200.mjs";
|
|
7
7
|
import { readJsonGzipStream, readJsonStream } from "../../../stream/server/json/read.mjs";
|
|
8
8
|
import { writeJsonGzipStream, writeJsonStream } from "../../../stream/server/json/write.mjs";
|
|
9
9
|
import "../../../stream/server/index.mjs";
|
|
@@ -49,24 +49,13 @@ export type LoadData<TData> = {
|
|
|
49
49
|
datasetId: string;
|
|
50
50
|
tableId?: string;
|
|
51
51
|
region?: string;
|
|
52
|
-
definition
|
|
53
|
-
options?: TableDefinition["options"];
|
|
52
|
+
definition?: TableDefinition;
|
|
54
53
|
};
|
|
55
54
|
};
|
|
56
55
|
export type LoadTableProps = LoadData<any[]>;
|
|
57
56
|
export type LoadStreamTableProps = LoadData<RowSource<any>>;
|
|
58
|
-
export type MergeTableProps = {
|
|
59
|
-
|
|
60
|
-
file: {
|
|
61
|
-
bucket: string;
|
|
62
|
-
key: string;
|
|
63
|
-
data: any[];
|
|
64
|
-
};
|
|
65
|
-
table: {
|
|
66
|
-
datasetId: string;
|
|
67
|
-
tableId?: string;
|
|
68
|
-
region?: string;
|
|
69
|
-
definition: TableDefinition;
|
|
57
|
+
export type MergeTableProps = Omit<LoadData<any[]>, "table"> & {
|
|
58
|
+
table: LoadData<any[]>["table"] & {
|
|
70
59
|
mergeKey: string;
|
|
71
60
|
updateColumns: string[];
|
|
72
61
|
};
|
|
@@ -49,24 +49,13 @@ export type LoadData<TData> = {
|
|
|
49
49
|
datasetId: string;
|
|
50
50
|
tableId?: string;
|
|
51
51
|
region?: string;
|
|
52
|
-
definition
|
|
53
|
-
options?: TableDefinition["options"];
|
|
52
|
+
definition?: TableDefinition;
|
|
54
53
|
};
|
|
55
54
|
};
|
|
56
55
|
export type LoadTableProps = LoadData<any[]>;
|
|
57
56
|
export type LoadStreamTableProps = LoadData<RowSource<any>>;
|
|
58
|
-
export type MergeTableProps = {
|
|
59
|
-
|
|
60
|
-
file: {
|
|
61
|
-
bucket: string;
|
|
62
|
-
key: string;
|
|
63
|
-
data: any[];
|
|
64
|
-
};
|
|
65
|
-
table: {
|
|
66
|
-
datasetId: string;
|
|
67
|
-
tableId?: string;
|
|
68
|
-
region?: string;
|
|
69
|
-
definition: TableDefinition;
|
|
57
|
+
export type MergeTableProps = Omit<LoadData<any[]>, "table"> & {
|
|
58
|
+
table: LoadData<any[]>["table"] & {
|
|
70
59
|
mergeKey: string;
|
|
71
60
|
updateColumns: string[];
|
|
72
61
|
};
|
|
@@ -147,13 +147,15 @@ var GoogleCloudBigquery = class extends require__class__shared_BaseNamespace.Bas
|
|
|
147
147
|
* pre-checking existence and silently skipping the load if it isn't there.
|
|
148
148
|
*/
|
|
149
149
|
loadData = async (props) => {
|
|
150
|
-
const finalTableId = props.table.tableId;
|
|
151
|
-
|
|
152
|
-
|
|
150
|
+
const finalTableId = props.table.tableId ?? props.table.definition?.tableId;
|
|
151
|
+
if (!finalTableId) throw new Error("loadData requires either table.tableId or table.definition.tableId");
|
|
152
|
+
const finalRegion = props.table.region ?? this.region;
|
|
153
|
+
await this.createTable(props.table.datasetId, {}, finalTableId, props.table.definition?.options);
|
|
154
|
+
await this.client.dataset(props.table.datasetId, { location: finalRegion }).table(finalTableId).load(props.googleCloudStorage.client.bucket(props.file.bucket).file(props.file.key), {
|
|
153
155
|
autodetect: true,
|
|
154
156
|
sourceFormat: bigQuerySourceFormat[props.file.sourceFormat ?? require_file__shared_fileType.fileType.JSON_NEWLINE],
|
|
155
157
|
schemaUpdateOptions: ["ALLOW_FIELD_ADDITION"],
|
|
156
|
-
location:
|
|
158
|
+
location: finalRegion
|
|
157
159
|
});
|
|
158
160
|
};
|
|
159
161
|
/**
|
|
@@ -163,17 +165,10 @@ var GoogleCloudBigquery = class extends require__class__shared_BaseNamespace.Bas
|
|
|
163
165
|
* this library has no Parquet encoder.
|
|
164
166
|
*/
|
|
165
167
|
loadTable = async (props) => {
|
|
166
|
-
const finalTableId = props.table.tableId ?? props.table.definition.tableId;
|
|
167
168
|
const fileSourceFormat = props.file.sourceFormat ?? require_file__shared_fileType.fileType.JSON_NEWLINE;
|
|
168
169
|
const finalData = require_file__shared_fileType.fileType.JSON_NEWLINE === fileSourceFormat ? require_json__shared_getJsonNewline.getJsonNewline(props.file.data) : props.file.data;
|
|
169
170
|
await props.googleCloudStorage.writeFile(props.file.bucket, props.file.key, finalData);
|
|
170
|
-
await this.loadData(
|
|
171
|
-
...props,
|
|
172
|
-
table: {
|
|
173
|
-
...props.table,
|
|
174
|
-
tableId: finalTableId
|
|
175
|
-
}
|
|
176
|
-
});
|
|
171
|
+
await this.loadData(props);
|
|
177
172
|
};
|
|
178
173
|
/**
|
|
179
174
|
* Same as loadTable, but streams a row source into Google Cloud Storage instead of buffering the
|
|
@@ -184,7 +179,6 @@ var GoogleCloudBigquery = class extends require__class__shared_BaseNamespace.Bas
|
|
|
184
179
|
* byte-level Readable of encoded Parquet content, piped straight through.
|
|
185
180
|
*/
|
|
186
181
|
loadStreamTable = async (props) => {
|
|
187
|
-
const finalTableId = props.table.tableId ?? props.table.definition.tableId;
|
|
188
182
|
const fileSourceFormat = props.file.sourceFormat ?? require_file__shared_fileType.fileType.JSON_NEWLINE;
|
|
189
183
|
const destination = props.googleCloudStorage.client.bucket(props.file.bucket).file(props.file.key).createWriteStream({
|
|
190
184
|
contentType: require_file__shared_fileMimeType.fileMimeType[fileSourceFormat],
|
|
@@ -197,13 +191,7 @@ var GoogleCloudBigquery = class extends require__class__shared_BaseNamespace.Bas
|
|
|
197
191
|
destination
|
|
198
192
|
]);
|
|
199
193
|
else await (0, node_stream_promises.pipeline)([source, destination]);
|
|
200
|
-
await this.loadData(
|
|
201
|
-
...props,
|
|
202
|
-
table: {
|
|
203
|
-
...props.table,
|
|
204
|
-
tableId: finalTableId
|
|
205
|
-
}
|
|
206
|
-
});
|
|
194
|
+
await this.loadData(props);
|
|
207
195
|
};
|
|
208
196
|
/**
|
|
209
197
|
* Brings an existing table's clustering in line with `definition.options.clustering`, if it differs.
|
|
@@ -226,15 +214,17 @@ var GoogleCloudBigquery = class extends require__class__shared_BaseNamespace.Bas
|
|
|
226
214
|
*/
|
|
227
215
|
mergeTable = async (props) => {
|
|
228
216
|
if (0 === props.file.data.length) return;
|
|
229
|
-
const finalTableId = props.table.tableId ?? props.table.definition
|
|
230
|
-
if (
|
|
217
|
+
const finalTableId = props.table.tableId ?? props.table.definition?.tableId;
|
|
218
|
+
if (!finalTableId) throw new Error("mergeTable requires either table.tableId or table.definition.tableId");
|
|
219
|
+
if (await this.tableExists(props.table.datasetId, finalTableId)) await this.syncTableClustering(props.table.datasetId, finalTableId, props.table.definition?.options?.clustering);
|
|
231
220
|
const stagingTableId = `${finalTableId}_staging_${require_date__shared_format.getYmdHisNumber()}`;
|
|
232
221
|
const stagingTableRef = `\`${props.table.datasetId}.${stagingTableId}\``;
|
|
233
222
|
try {
|
|
234
223
|
const stagingDefinition = {
|
|
235
224
|
...props.table.definition,
|
|
225
|
+
datasetId: props.table.datasetId,
|
|
236
226
|
tableId: stagingTableId,
|
|
237
|
-
options: props.table.definition
|
|
227
|
+
options: props.table.definition?.options ? {
|
|
238
228
|
...props.table.definition.options,
|
|
239
229
|
timePartitioning: void 0,
|
|
240
230
|
clustering: void 0
|
|
@@ -252,7 +242,7 @@ var GoogleCloudBigquery = class extends require__class__shared_BaseNamespace.Bas
|
|
|
252
242
|
});
|
|
253
243
|
const mainTableRef = `\`${props.table.datasetId}.${finalTableId}\``;
|
|
254
244
|
const updateSet = props.table.updateColumns.map((column) => `${column} = S.${column}`).join(", ");
|
|
255
|
-
const allColumns = props.table.definition
|
|
245
|
+
const allColumns = props.table.definition?.options?.schema.map((field) => field.name) ?? [];
|
|
256
246
|
const mergeQuery = `
|
|
257
247
|
MERGE ${mainTableRef} AS T
|
|
258
248
|
USING ${stagingTableRef} AS S
|
|
@@ -146,13 +146,15 @@ var GoogleCloudBigquery = class extends BaseNamespace {
|
|
|
146
146
|
* pre-checking existence and silently skipping the load if it isn't there.
|
|
147
147
|
*/
|
|
148
148
|
loadData = async (props) => {
|
|
149
|
-
const finalTableId = props.table.tableId;
|
|
150
|
-
|
|
151
|
-
|
|
149
|
+
const finalTableId = props.table.tableId ?? props.table.definition?.tableId;
|
|
150
|
+
if (!finalTableId) throw new Error("loadData requires either table.tableId or table.definition.tableId");
|
|
151
|
+
const finalRegion = props.table.region ?? this.region;
|
|
152
|
+
await this.createTable(props.table.datasetId, {}, finalTableId, props.table.definition?.options);
|
|
153
|
+
await this.client.dataset(props.table.datasetId, { location: finalRegion }).table(finalTableId).load(props.googleCloudStorage.client.bucket(props.file.bucket).file(props.file.key), {
|
|
152
154
|
autodetect: true,
|
|
153
155
|
sourceFormat: bigQuerySourceFormat[props.file.sourceFormat ?? fileType.JSON_NEWLINE],
|
|
154
156
|
schemaUpdateOptions: ["ALLOW_FIELD_ADDITION"],
|
|
155
|
-
location:
|
|
157
|
+
location: finalRegion
|
|
156
158
|
});
|
|
157
159
|
};
|
|
158
160
|
/**
|
|
@@ -162,17 +164,10 @@ var GoogleCloudBigquery = class extends BaseNamespace {
|
|
|
162
164
|
* this library has no Parquet encoder.
|
|
163
165
|
*/
|
|
164
166
|
loadTable = async (props) => {
|
|
165
|
-
const finalTableId = props.table.tableId ?? props.table.definition.tableId;
|
|
166
167
|
const fileSourceFormat = props.file.sourceFormat ?? fileType.JSON_NEWLINE;
|
|
167
168
|
const finalData = fileType.JSON_NEWLINE === fileSourceFormat ? getJsonNewline(props.file.data) : props.file.data;
|
|
168
169
|
await props.googleCloudStorage.writeFile(props.file.bucket, props.file.key, finalData);
|
|
169
|
-
await this.loadData(
|
|
170
|
-
...props,
|
|
171
|
-
table: {
|
|
172
|
-
...props.table,
|
|
173
|
-
tableId: finalTableId
|
|
174
|
-
}
|
|
175
|
-
});
|
|
170
|
+
await this.loadData(props);
|
|
176
171
|
};
|
|
177
172
|
/**
|
|
178
173
|
* Same as loadTable, but streams a row source into Google Cloud Storage instead of buffering the
|
|
@@ -183,7 +178,6 @@ var GoogleCloudBigquery = class extends BaseNamespace {
|
|
|
183
178
|
* byte-level Readable of encoded Parquet content, piped straight through.
|
|
184
179
|
*/
|
|
185
180
|
loadStreamTable = async (props) => {
|
|
186
|
-
const finalTableId = props.table.tableId ?? props.table.definition.tableId;
|
|
187
181
|
const fileSourceFormat = props.file.sourceFormat ?? fileType.JSON_NEWLINE;
|
|
188
182
|
const destination = props.googleCloudStorage.client.bucket(props.file.bucket).file(props.file.key).createWriteStream({
|
|
189
183
|
contentType: fileMimeType[fileSourceFormat],
|
|
@@ -196,13 +190,7 @@ var GoogleCloudBigquery = class extends BaseNamespace {
|
|
|
196
190
|
destination
|
|
197
191
|
]);
|
|
198
192
|
else await pipeline([source, destination]);
|
|
199
|
-
await this.loadData(
|
|
200
|
-
...props,
|
|
201
|
-
table: {
|
|
202
|
-
...props.table,
|
|
203
|
-
tableId: finalTableId
|
|
204
|
-
}
|
|
205
|
-
});
|
|
193
|
+
await this.loadData(props);
|
|
206
194
|
};
|
|
207
195
|
/**
|
|
208
196
|
* Brings an existing table's clustering in line with `definition.options.clustering`, if it differs.
|
|
@@ -225,15 +213,17 @@ var GoogleCloudBigquery = class extends BaseNamespace {
|
|
|
225
213
|
*/
|
|
226
214
|
mergeTable = async (props) => {
|
|
227
215
|
if (0 === props.file.data.length) return;
|
|
228
|
-
const finalTableId = props.table.tableId ?? props.table.definition
|
|
229
|
-
if (
|
|
216
|
+
const finalTableId = props.table.tableId ?? props.table.definition?.tableId;
|
|
217
|
+
if (!finalTableId) throw new Error("mergeTable requires either table.tableId or table.definition.tableId");
|
|
218
|
+
if (await this.tableExists(props.table.datasetId, finalTableId)) await this.syncTableClustering(props.table.datasetId, finalTableId, props.table.definition?.options?.clustering);
|
|
230
219
|
const stagingTableId = `${finalTableId}_staging_${getYmdHisNumber()}`;
|
|
231
220
|
const stagingTableRef = `\`${props.table.datasetId}.${stagingTableId}\``;
|
|
232
221
|
try {
|
|
233
222
|
const stagingDefinition = {
|
|
234
223
|
...props.table.definition,
|
|
224
|
+
datasetId: props.table.datasetId,
|
|
235
225
|
tableId: stagingTableId,
|
|
236
|
-
options: props.table.definition
|
|
226
|
+
options: props.table.definition?.options ? {
|
|
237
227
|
...props.table.definition.options,
|
|
238
228
|
timePartitioning: void 0,
|
|
239
229
|
clustering: void 0
|
|
@@ -251,7 +241,7 @@ var GoogleCloudBigquery = class extends BaseNamespace {
|
|
|
251
241
|
});
|
|
252
242
|
const mainTableRef = `\`${props.table.datasetId}.${finalTableId}\``;
|
|
253
243
|
const updateSet = props.table.updateColumns.map((column) => `${column} = S.${column}`).join(", ");
|
|
254
|
-
const allColumns = props.table.definition
|
|
244
|
+
const allColumns = props.table.definition?.options?.schema.map((field) => field.name) ?? [];
|
|
255
245
|
const mergeQuery = `
|
|
256
246
|
MERGE ${mainTableRef} AS T
|
|
257
247
|
USING ${stagingTableRef} AS S
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
2
|
const require_xlsx__shared_column = require("./column.cjs");
|
|
3
3
|
const require_xlsx__shared_cell = require("./cell.cjs");
|
|
4
|
-
const require_xlsx__shared_sheet = require("./sheet.cjs");
|
|
5
4
|
const require_zip__shared_ZipArchive = require("../../zip/_shared/ZipArchive.cjs");
|
|
5
|
+
const require_xlsx__shared_sheet = require("./sheet.cjs");
|
|
6
6
|
//#region src/xlsx/_shared/createXlsx.ts
|
|
7
7
|
/**
|
|
8
8
|
* Builds a complete .xlsx workbook from scratch, using hand-written OOXML rather
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { getColumLetter, getColumnWidth } from "./column.mjs";
|
|
2
2
|
import { buildCellXml, getCellValue, getEscapedXml } from "./cell.mjs";
|
|
3
|
-
import { getUniqueSheetNames } from "./sheet.mjs";
|
|
4
3
|
import { ZipArchive } from "../../zip/_shared/ZipArchive.mjs";
|
|
4
|
+
import { getUniqueSheetNames } from "./sheet.mjs";
|
|
5
5
|
//#region src/xlsx/_shared/createXlsx.ts
|
|
6
6
|
/**
|
|
7
7
|
* Builds a complete .xlsx workbook from scratch, using hand-written OOXML rather
|