@tailglow/cli 0.8.1 → 0.9.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -8
- package/package.json +2 -2
- package/src/commands.generated.js +69 -134
- package/src/config.js +8 -1
- package/src/docs.generated.js +18 -18
- package/src/import-command.js +2 -13
- package/src/import-options.js +0 -1
- package/src/import-worker.generated.js +23 -41
- package/src/import.generated.d.ts +0 -5
- package/src/import.generated.js +60 -86
- package/src/index.js +14 -0
- package/src/request.js +12 -4
- package/src/update-check.js +137 -0
package/src/import-command.js
CHANGED
|
@@ -23,7 +23,6 @@ const HELP_NOTES = [
|
|
|
23
23
|
"Upload previews a bounded sample, then validates records while sending bounded batches.",
|
|
24
24
|
"--validate-first copies a private snapshot and checks it fully before upload; it needs temporary disk space approximately equal to the input size.",
|
|
25
25
|
"Default upload reads one open file and stops if it detects changes. Do not edit it during import.",
|
|
26
|
-
"Selected timestamps must exist and be unambiguous. Distinct times may produce small batches.",
|
|
27
26
|
"Only HTTP 202 acknowledges a batch. Delivery uncertainty stops upload without retrying.",
|
|
28
27
|
"Ctrl-C reports accepted rows and any uncertain batch. Retrying a partial import may duplicate rows."
|
|
29
28
|
];
|
|
@@ -35,7 +34,6 @@ const HELP = [
|
|
|
35
34
|
"",
|
|
36
35
|
"--format auto|xml|jsonl|json|csv|tsv",
|
|
37
36
|
"--record-path /root/item Optional XML filter; imports all top-level entries by default.",
|
|
38
|
-
"--timestamp-field FIELD JSON pointer or exact top-level name; never selected automatically.",
|
|
39
37
|
"--workers auto|0|N XML worker threads for full reads; auto uses 2 from 32 MiB.",
|
|
40
38
|
"--validate-first Check a private snapshot completely before uploading it.",
|
|
41
39
|
"--ingest-url URL Explicit ingest URL, including its key. Keep this credential private.",
|
|
@@ -256,7 +254,6 @@ async function resolveDestination(flags, signal, request, read_config) {
|
|
|
256
254
|
}
|
|
257
255
|
}
|
|
258
256
|
url.searchParams.set("collection", String(flags.collection));
|
|
259
|
-
url.searchParams.delete("ts");
|
|
260
257
|
return { url, project_id };
|
|
261
258
|
}
|
|
262
259
|
|
|
@@ -474,9 +471,6 @@ export async function runImportCommand(argv, context = {}) {
|
|
|
474
471
|
const options = Object.freeze({
|
|
475
472
|
format: detection.format,
|
|
476
473
|
...(typeof flags["record-path"] === "string" ? { record_path: flags["record-path"] } : {}),
|
|
477
|
-
...(typeof flags["timestamp-field"] === "string"
|
|
478
|
-
? { timestamp_field: flags["timestamp-field"] }
|
|
479
|
-
: {}),
|
|
480
474
|
workers,
|
|
481
475
|
create_worker: () => createNodeImportWorker(IMPORT_WORKER_URL),
|
|
482
476
|
onworkers,
|
|
@@ -502,7 +496,6 @@ export async function runImportCommand(argv, context = {}) {
|
|
|
502
496
|
throw new Error("The full file check did not finish.");
|
|
503
497
|
total_rows = report.is_complete && !report.is_sampled ? report.row_count : null;
|
|
504
498
|
outcome = "destination_failed";
|
|
505
|
-
const batch_options = Object.freeze({ timestamp_field: options.timestamp_field });
|
|
506
499
|
const destination = await resolveDestination(
|
|
507
500
|
flags,
|
|
508
501
|
signal,
|
|
@@ -513,14 +506,10 @@ export async function runImportCommand(argv, context = {}) {
|
|
|
513
506
|
const readiness = await getJson(destination.url, {}, signal, request);
|
|
514
507
|
if (readiness?.status !== "ready" || readiness.project_id !== destination.project_id)
|
|
515
508
|
throw new Error("The ingest endpoint is not ready for the expected project.");
|
|
516
|
-
if (options.timestamp_field)
|
|
517
|
-
stderr.write("Preserving selected timestamps; distinct values may produce small batches.\n");
|
|
518
509
|
outcome = "validation_failed";
|
|
519
|
-
for await (const batch of createImportBatches(readImportRows(source, options)
|
|
510
|
+
for await (const batch of createImportBatches(readImportRows(source, options))) {
|
|
520
511
|
checkAbort(signal);
|
|
521
512
|
await input.assertUnchanged();
|
|
522
|
-
const target = new URL(destination.url);
|
|
523
|
-
if (batch.timestamp) target.searchParams.set("ts", batch.timestamp);
|
|
524
513
|
affected_batch = {
|
|
525
514
|
index: batch.index,
|
|
526
515
|
first_row: batch.rows[0].index,
|
|
@@ -530,7 +519,7 @@ export async function runImportCommand(argv, context = {}) {
|
|
|
530
519
|
outcome = "uncertain_delivery";
|
|
531
520
|
const linked = requestSignal(signal, 30_000);
|
|
532
521
|
try {
|
|
533
|
-
const response = await request(
|
|
522
|
+
const response = await request(destination.url, {
|
|
534
523
|
method: "POST",
|
|
535
524
|
headers: { "Content-Type": "application/x-ndjson" },
|
|
536
525
|
body: batch.body,
|
package/src/import-options.js
CHANGED
|
@@ -1254,6 +1254,27 @@ class ImportRowError extends Error {
|
|
|
1254
1254
|
}
|
|
1255
1255
|
}
|
|
1256
1256
|
|
|
1257
|
+
// packages/import/src/batch.js
|
|
1258
|
+
var encoder2 = new TextEncoder;
|
|
1259
|
+
function serializeImportRow(row) {
|
|
1260
|
+
const line = JSON.stringify(row.record);
|
|
1261
|
+
if (line === undefined || line.length + 1 > IMPORT_BATCH_BYTES)
|
|
1262
|
+
throw new ImportRowError(row.index, " exceeds the 1 MiB record limit.");
|
|
1263
|
+
const bytes = encoder2.encode(line).byteLength + 1;
|
|
1264
|
+
if (bytes > IMPORT_BATCH_BYTES)
|
|
1265
|
+
throw new ImportRowError(row.index, " exceeds the 1 MiB record limit.");
|
|
1266
|
+
return { row, line, bytes };
|
|
1267
|
+
}
|
|
1268
|
+
|
|
1269
|
+
// packages/import/src/row.js
|
|
1270
|
+
function createImportRow(record, index) {
|
|
1271
|
+
if (!record || typeof record !== "object" || Array.isArray(record))
|
|
1272
|
+
throw new ImportRowError(index, " must be a JSON object.");
|
|
1273
|
+
const row = { index, record };
|
|
1274
|
+
serializeImportRow(row);
|
|
1275
|
+
return row;
|
|
1276
|
+
}
|
|
1277
|
+
|
|
1257
1278
|
// packages/import/src/timestamp.js
|
|
1258
1279
|
function getImportField(record, path) {
|
|
1259
1280
|
if (!path.startsWith("/"))
|
|
@@ -1325,41 +1346,6 @@ function parseImportTimestamp(value) {
|
|
|
1325
1346
|
function isAmbiguousTimestamp(value) {
|
|
1326
1347
|
return typeof value === "string" && (/^\d{4}-\d{2}-\d{2}(?:[T ]\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?)?$/.test(value) || /-00:?00$/.test(value));
|
|
1327
1348
|
}
|
|
1328
|
-
function getRowTimestamp(row, timestamp_field) {
|
|
1329
|
-
if (!timestamp_field)
|
|
1330
|
-
return null;
|
|
1331
|
-
const value = getImportField(row.record, timestamp_field);
|
|
1332
|
-
const timestamp = parseImportTimestamp(value);
|
|
1333
|
-
if (!timestamp)
|
|
1334
|
-
throw new ImportRowError(row.index, `: timestamp field ${JSON.stringify(timestamp_field)} is missing, invalid or timezone-ambiguous. Choose an explicit timezone or a different field.`);
|
|
1335
|
-
const milliseconds = Date.parse(timestamp);
|
|
1336
|
-
if (milliseconds < 0)
|
|
1337
|
-
throw new ImportRowError(row.index, `: timestamp field ${JSON.stringify(timestamp_field)} is before 1970-01-01T00:00:00Z and cannot be used as an import timestamp.`);
|
|
1338
|
-
if (milliseconds > Date.now())
|
|
1339
|
-
throw new ImportRowError(row.index, `: timestamp field ${JSON.stringify(timestamp_field)} is in the future and cannot be used as an import timestamp.`);
|
|
1340
|
-
return timestamp;
|
|
1341
|
-
}
|
|
1342
|
-
|
|
1343
|
-
// packages/import/src/batch.js
|
|
1344
|
-
var encoder2 = new TextEncoder;
|
|
1345
|
-
function serializeImportRow(row) {
|
|
1346
|
-
const line = JSON.stringify(row.record);
|
|
1347
|
-
if (line === undefined || line.length + 1 > IMPORT_BATCH_BYTES)
|
|
1348
|
-
throw new ImportRowError(row.index, " exceeds the 1 MiB record limit.");
|
|
1349
|
-
const bytes = encoder2.encode(line).byteLength + 1;
|
|
1350
|
-
if (bytes > IMPORT_BATCH_BYTES)
|
|
1351
|
-
throw new ImportRowError(row.index, " exceeds the 1 MiB record limit.");
|
|
1352
|
-
return { row, line, bytes };
|
|
1353
|
-
}
|
|
1354
|
-
|
|
1355
|
-
// packages/import/src/row.js
|
|
1356
|
-
function createImportRow(record, index) {
|
|
1357
|
-
if (!record || typeof record !== "object" || Array.isArray(record))
|
|
1358
|
-
throw new ImportRowError(index, " must be a JSON object.");
|
|
1359
|
-
const row = { index, record };
|
|
1360
|
-
serializeImportRow(row);
|
|
1361
|
-
return row;
|
|
1362
|
-
}
|
|
1363
1349
|
|
|
1364
1350
|
// packages/import/src/scan-profile.js
|
|
1365
1351
|
var IMPORT_PROFILE_FIELDS = 256;
|
|
@@ -1367,7 +1353,7 @@ var MAX_DEPTH = 16;
|
|
|
1367
1353
|
var MAX_PATH_CHARS = 4096;
|
|
1368
1354
|
var MAX_VISITED_FIELDS = 2048;
|
|
1369
1355
|
var PREVIEW_ROWS = 12;
|
|
1370
|
-
function createScanProfile({
|
|
1356
|
+
function createScanProfile({ max_fields = IMPORT_PROFILE_FIELDS } = {}) {
|
|
1371
1357
|
const fields = new Map;
|
|
1372
1358
|
const preview = [];
|
|
1373
1359
|
let rows = 0;
|
|
@@ -1470,7 +1456,6 @@ function createScanProfile({ timestamp_field, max_fields = IMPORT_PROFILE_FIELDS
|
|
|
1470
1456
|
},
|
|
1471
1457
|
add(row) {
|
|
1472
1458
|
const entry = serializeImportRow(row);
|
|
1473
|
-
getRowTimestamp(row, timestamp_field);
|
|
1474
1459
|
rows++;
|
|
1475
1460
|
max_row_bytes = Math.max(max_row_bytes, entry.bytes);
|
|
1476
1461
|
if (preview.length < PREVIEW_ROWS)
|
|
@@ -1575,10 +1560,7 @@ var IMPORT_XML_TASK = {
|
|
|
1575
1560
|
}
|
|
1576
1561
|
};
|
|
1577
1562
|
}
|
|
1578
|
-
const createProfile = () => createScanProfile({
|
|
1579
|
-
timestamp_field: options.timestamp_field,
|
|
1580
|
-
max_fields: PARTIAL_PROFILE_FIELDS
|
|
1581
|
-
});
|
|
1563
|
+
const createProfile = () => createScanProfile({ max_fields: PARTIAL_PROFILE_FIELDS });
|
|
1582
1564
|
let profile = createProfile();
|
|
1583
1565
|
return {
|
|
1584
1566
|
add(record) {
|
|
@@ -104,12 +104,8 @@ export interface ImportReport {
|
|
|
104
104
|
}
|
|
105
105
|
export interface ImportScanOptions extends ImportOptions {
|
|
106
106
|
is_full?: boolean;
|
|
107
|
-
/** Selected timestamps must be valid, explicit-zone values between the Unix epoch and now. */
|
|
108
|
-
timestamp_field?: string;
|
|
109
107
|
}
|
|
110
108
|
export interface ImportBatchOptions {
|
|
111
|
-
/** Selected timestamps must be valid, explicit-zone values between the Unix epoch and now. */
|
|
112
|
-
timestamp_field?: string;
|
|
113
109
|
max_bytes?: number;
|
|
114
110
|
max_rows?: number;
|
|
115
111
|
}
|
|
@@ -118,7 +114,6 @@ export interface ImportBatch {
|
|
|
118
114
|
body: string;
|
|
119
115
|
bytes: number;
|
|
120
116
|
rows: ImportRow[];
|
|
121
|
-
timestamp: string | null;
|
|
122
117
|
}
|
|
123
118
|
export const IMPORT_BATCH_BYTES: number;
|
|
124
119
|
export const IMPORT_BATCH_ROWS: number;
|
package/src/import.generated.js
CHANGED
|
@@ -3913,6 +3913,61 @@ class ImportRowError extends Error {
|
|
|
3913
3913
|
}
|
|
3914
3914
|
}
|
|
3915
3915
|
|
|
3916
|
+
// packages/import/src/batch.js
|
|
3917
|
+
var encoder2 = new TextEncoder;
|
|
3918
|
+
function serializeImportRow(row) {
|
|
3919
|
+
const line = JSON.stringify(row.record);
|
|
3920
|
+
if (line === undefined || line.length + 1 > IMPORT_BATCH_BYTES)
|
|
3921
|
+
throw new ImportRowError(row.index, " exceeds the 1 MiB record limit.");
|
|
3922
|
+
const bytes = encoder2.encode(line).byteLength + 1;
|
|
3923
|
+
if (bytes > IMPORT_BATCH_BYTES)
|
|
3924
|
+
throw new ImportRowError(row.index, " exceeds the 1 MiB record limit.");
|
|
3925
|
+
return { row, line, bytes };
|
|
3926
|
+
}
|
|
3927
|
+
async function* createImportBatches(rows, options = {}) {
|
|
3928
|
+
const max_bytes = options.max_bytes ?? IMPORT_BATCH_BYTES;
|
|
3929
|
+
const max_rows = options.max_rows ?? IMPORT_BATCH_ROWS;
|
|
3930
|
+
if (!Number.isSafeInteger(max_bytes) || max_bytes < 1 || max_bytes > IMPORT_BATCH_BYTES)
|
|
3931
|
+
throw new RangeError("max_bytes must be between 1 and 1048576.");
|
|
3932
|
+
if (!Number.isSafeInteger(max_rows) || max_rows < 1 || max_rows > IMPORT_BATCH_ROWS)
|
|
3933
|
+
throw new RangeError("max_rows must be between 1 and 500.");
|
|
3934
|
+
let batch_rows = [];
|
|
3935
|
+
let body = "";
|
|
3936
|
+
let bytes = 0;
|
|
3937
|
+
let index = 0;
|
|
3938
|
+
function finish() {
|
|
3939
|
+
const batch = { index: ++index, body, bytes, rows: batch_rows };
|
|
3940
|
+
batch_rows = [];
|
|
3941
|
+
body = "";
|
|
3942
|
+
bytes = 0;
|
|
3943
|
+
return batch;
|
|
3944
|
+
}
|
|
3945
|
+
for await (const row of rows) {
|
|
3946
|
+
const entry = serializeImportRow(row);
|
|
3947
|
+
if (entry.bytes > max_bytes)
|
|
3948
|
+
throw new Error(`Row ${row.index} exceeds max_bytes (${max_bytes}).`);
|
|
3949
|
+
if (batch_rows.length && bytes + entry.bytes > max_bytes)
|
|
3950
|
+
yield finish();
|
|
3951
|
+
batch_rows.push(row);
|
|
3952
|
+
body += entry.line + `
|
|
3953
|
+
`;
|
|
3954
|
+
bytes += entry.bytes;
|
|
3955
|
+
if (batch_rows.length === max_rows || bytes === max_bytes)
|
|
3956
|
+
yield finish();
|
|
3957
|
+
}
|
|
3958
|
+
if (batch_rows.length)
|
|
3959
|
+
yield finish();
|
|
3960
|
+
}
|
|
3961
|
+
|
|
3962
|
+
// packages/import/src/row.js
|
|
3963
|
+
function createImportRow(record, index) {
|
|
3964
|
+
if (!record || typeof record !== "object" || Array.isArray(record))
|
|
3965
|
+
throw new ImportRowError(index, " must be a JSON object.");
|
|
3966
|
+
const row = { index, record };
|
|
3967
|
+
serializeImportRow(row);
|
|
3968
|
+
return row;
|
|
3969
|
+
}
|
|
3970
|
+
|
|
3916
3971
|
// packages/import/src/timestamp.js
|
|
3917
3972
|
function getImportField(record, path) {
|
|
3918
3973
|
if (!path.startsWith("/"))
|
|
@@ -3984,78 +4039,6 @@ function parseImportTimestamp(value) {
|
|
|
3984
4039
|
function isAmbiguousTimestamp(value) {
|
|
3985
4040
|
return typeof value === "string" && (/^\d{4}-\d{2}-\d{2}(?:[T ]\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?)?$/.test(value) || /-00:?00$/.test(value));
|
|
3986
4041
|
}
|
|
3987
|
-
function getRowTimestamp(row, timestamp_field) {
|
|
3988
|
-
if (!timestamp_field)
|
|
3989
|
-
return null;
|
|
3990
|
-
const value = getImportField(row.record, timestamp_field);
|
|
3991
|
-
const timestamp = parseImportTimestamp(value);
|
|
3992
|
-
if (!timestamp)
|
|
3993
|
-
throw new ImportRowError(row.index, `: timestamp field ${JSON.stringify(timestamp_field)} is missing, invalid or timezone-ambiguous. Choose an explicit timezone or a different field.`);
|
|
3994
|
-
const milliseconds = Date.parse(timestamp);
|
|
3995
|
-
if (milliseconds < 0)
|
|
3996
|
-
throw new ImportRowError(row.index, `: timestamp field ${JSON.stringify(timestamp_field)} is before 1970-01-01T00:00:00Z and cannot be used as an import timestamp.`);
|
|
3997
|
-
if (milliseconds > Date.now())
|
|
3998
|
-
throw new ImportRowError(row.index, `: timestamp field ${JSON.stringify(timestamp_field)} is in the future and cannot be used as an import timestamp.`);
|
|
3999
|
-
return timestamp;
|
|
4000
|
-
}
|
|
4001
|
-
|
|
4002
|
-
// packages/import/src/batch.js
|
|
4003
|
-
var encoder2 = new TextEncoder;
|
|
4004
|
-
function serializeImportRow(row) {
|
|
4005
|
-
const line = JSON.stringify(row.record);
|
|
4006
|
-
if (line === undefined || line.length + 1 > IMPORT_BATCH_BYTES)
|
|
4007
|
-
throw new ImportRowError(row.index, " exceeds the 1 MiB record limit.");
|
|
4008
|
-
const bytes = encoder2.encode(line).byteLength + 1;
|
|
4009
|
-
if (bytes > IMPORT_BATCH_BYTES)
|
|
4010
|
-
throw new ImportRowError(row.index, " exceeds the 1 MiB record limit.");
|
|
4011
|
-
return { row, line, bytes };
|
|
4012
|
-
}
|
|
4013
|
-
async function* createImportBatches(rows, options = {}) {
|
|
4014
|
-
const max_bytes = options.max_bytes ?? IMPORT_BATCH_BYTES;
|
|
4015
|
-
const max_rows = options.max_rows ?? IMPORT_BATCH_ROWS;
|
|
4016
|
-
if (!Number.isSafeInteger(max_bytes) || max_bytes < 1 || max_bytes > IMPORT_BATCH_BYTES)
|
|
4017
|
-
throw new RangeError("max_bytes must be between 1 and 1048576.");
|
|
4018
|
-
if (!Number.isSafeInteger(max_rows) || max_rows < 1 || max_rows > IMPORT_BATCH_ROWS)
|
|
4019
|
-
throw new RangeError("max_rows must be between 1 and 500.");
|
|
4020
|
-
let batch_rows = [];
|
|
4021
|
-
let body = "";
|
|
4022
|
-
let bytes = 0;
|
|
4023
|
-
let index = 0;
|
|
4024
|
-
let timestamp = null;
|
|
4025
|
-
function finish() {
|
|
4026
|
-
const batch = { index: ++index, body, bytes, rows: batch_rows, timestamp };
|
|
4027
|
-
batch_rows = [];
|
|
4028
|
-
body = "";
|
|
4029
|
-
bytes = 0;
|
|
4030
|
-
return batch;
|
|
4031
|
-
}
|
|
4032
|
-
for await (const row of rows) {
|
|
4033
|
-
const entry = serializeImportRow(row);
|
|
4034
|
-
const next_timestamp = getRowTimestamp(row, options.timestamp_field);
|
|
4035
|
-
if (entry.bytes > max_bytes)
|
|
4036
|
-
throw new Error(`Row ${row.index} exceeds max_bytes (${max_bytes}).`);
|
|
4037
|
-
if (batch_rows.length && (bytes + entry.bytes > max_bytes || timestamp !== next_timestamp))
|
|
4038
|
-
yield finish();
|
|
4039
|
-
timestamp = next_timestamp;
|
|
4040
|
-
batch_rows.push(row);
|
|
4041
|
-
body += entry.line + `
|
|
4042
|
-
`;
|
|
4043
|
-
bytes += entry.bytes;
|
|
4044
|
-
if (batch_rows.length === max_rows || bytes === max_bytes)
|
|
4045
|
-
yield finish();
|
|
4046
|
-
}
|
|
4047
|
-
if (batch_rows.length)
|
|
4048
|
-
yield finish();
|
|
4049
|
-
}
|
|
4050
|
-
|
|
4051
|
-
// packages/import/src/row.js
|
|
4052
|
-
function createImportRow(record, index) {
|
|
4053
|
-
if (!record || typeof record !== "object" || Array.isArray(record))
|
|
4054
|
-
throw new ImportRowError(index, " must be a JSON object.");
|
|
4055
|
-
const row = { index, record };
|
|
4056
|
-
serializeImportRow(row);
|
|
4057
|
-
return row;
|
|
4058
|
-
}
|
|
4059
4042
|
|
|
4060
4043
|
// packages/import/src/scan-profile.js
|
|
4061
4044
|
var IMPORT_PROFILE_FIELDS = 256;
|
|
@@ -4063,7 +4046,7 @@ var MAX_DEPTH = 16;
|
|
|
4063
4046
|
var MAX_PATH_CHARS = 4096;
|
|
4064
4047
|
var MAX_VISITED_FIELDS = 2048;
|
|
4065
4048
|
var PREVIEW_ROWS = 12;
|
|
4066
|
-
function createScanProfile({
|
|
4049
|
+
function createScanProfile({ max_fields = IMPORT_PROFILE_FIELDS } = {}) {
|
|
4067
4050
|
const fields = new Map;
|
|
4068
4051
|
const preview = [];
|
|
4069
4052
|
let rows = 0;
|
|
@@ -4166,7 +4149,6 @@ function createScanProfile({ timestamp_field, max_fields = IMPORT_PROFILE_FIELDS
|
|
|
4166
4149
|
},
|
|
4167
4150
|
add(row) {
|
|
4168
4151
|
const entry = serializeImportRow(row);
|
|
4169
|
-
getRowTimestamp(row, timestamp_field);
|
|
4170
4152
|
rows++;
|
|
4171
4153
|
max_row_bytes = Math.max(max_row_bytes, entry.bytes);
|
|
4172
4154
|
if (preview.length < PREVIEW_ROWS)
|
|
@@ -4286,10 +4268,7 @@ var IMPORT_XML_TASK = {
|
|
|
4286
4268
|
}
|
|
4287
4269
|
};
|
|
4288
4270
|
}
|
|
4289
|
-
const createProfile = () => createScanProfile({
|
|
4290
|
-
timestamp_field: options.timestamp_field,
|
|
4291
|
-
max_fields: PARTIAL_PROFILE_FIELDS
|
|
4292
|
-
});
|
|
4271
|
+
const createProfile = () => createScanProfile({ max_fields: PARTIAL_PROFILE_FIELDS });
|
|
4293
4272
|
let profile = createProfile();
|
|
4294
4273
|
return {
|
|
4295
4274
|
add(record) {
|
|
@@ -4405,11 +4384,8 @@ async function* readXmlRowsWithWorkers(source, options, requested) {
|
|
|
4405
4384
|
}
|
|
4406
4385
|
async function scanXmlWithWorkers(source, options, requested) {
|
|
4407
4386
|
const state = { status: { requested, actual: 0 } };
|
|
4408
|
-
const profile = createScanProfile(
|
|
4409
|
-
for await (const event of readImportXmlSegments(source, options, requested, {
|
|
4410
|
-
kind: "scan",
|
|
4411
|
-
timestamp_field: options.timestamp_field
|
|
4412
|
-
})) {
|
|
4387
|
+
const profile = createScanProfile();
|
|
4388
|
+
for await (const event of readImportXmlSegments(source, options, requested, { kind: "scan" })) {
|
|
4413
4389
|
if (!reportWorkerEvent(event, state, options))
|
|
4414
4390
|
return null;
|
|
4415
4391
|
if (event.type !== "records")
|
|
@@ -4657,8 +4633,6 @@ async function scanImport(source, options = {}) {
|
|
|
4657
4633
|
report.warnings.push("Schema discovery was truncated (256 fields, 16 levels, 4096 characters per path or 2048 visited fields per row). Counts cover scanned rows only.");
|
|
4658
4634
|
if (report.is_sampled)
|
|
4659
4635
|
report.warnings.push("This bounded sample does not certify full file validity or schema coverage. Records must be validated as they are imported.");
|
|
4660
|
-
if (options.timestamp_field)
|
|
4661
|
-
report.warnings.push("The selected timestamp is sent at request level. Changes between rows split batches and may produce many small requests.");
|
|
4662
4636
|
options.onprogress?.({
|
|
4663
4637
|
bytes_read: report.bytes_read,
|
|
4664
4638
|
total_bytes: source.size,
|
|
@@ -4684,7 +4658,7 @@ async function scanSequentially(source, options, format, report) {
|
|
|
4684
4658
|
}
|
|
4685
4659
|
}
|
|
4686
4660
|
};
|
|
4687
|
-
const profile = createScanProfile(
|
|
4661
|
+
const profile = createScanProfile();
|
|
4688
4662
|
report.bytes_read = 0;
|
|
4689
4663
|
try {
|
|
4690
4664
|
for await (const row of readImportRows(scanned_source, {
|
package/src/index.js
CHANGED
|
@@ -36,6 +36,7 @@ import {
|
|
|
36
36
|
writeCommandResult
|
|
37
37
|
} from "./output.js";
|
|
38
38
|
import { requestAllPages, requestApi } from "./request.js";
|
|
39
|
+
import { readUpdateNotice } from "./update-check.js";
|
|
39
40
|
|
|
40
41
|
/** Path parameters people refer to by the shorter name they actually type. */
|
|
41
42
|
const PATH_ALIASES = { project_id: "project", team_id: "team" };
|
|
@@ -420,6 +421,9 @@ function isEntrypoint() {
|
|
|
420
421
|
}
|
|
421
422
|
|
|
422
423
|
if (isEntrypoint()) {
|
|
424
|
+
// Started first so it runs alongside the command, and printed to stderr after the result.
|
|
425
|
+
const update_notice = readUpdateNotice();
|
|
426
|
+
|
|
423
427
|
// `process.exit` discards whatever is still buffered for a pipe, and a pipe's buffer is 64 KB.
|
|
424
428
|
// `tglow records list --all --json | jq` would receive a JSON document cut off mid-value, which
|
|
425
429
|
// is exactly the invariant this CLI sells. Set the code and let the write drain instead.
|
|
@@ -447,4 +451,14 @@ if (isEntrypoint()) {
|
|
|
447
451
|
{ json }
|
|
448
452
|
);
|
|
449
453
|
}
|
|
454
|
+
|
|
455
|
+
// A failed command in JSON mode answers with one JSON document on stderr, so it gets no notice.
|
|
456
|
+
// Nothing about the notice may fail a finished command, a closed stderr included.
|
|
457
|
+
const notice = await update_notice;
|
|
458
|
+
if (notice && !(json && process.exitCode !== 0)) {
|
|
459
|
+
process.stderr.on("error", () => {});
|
|
460
|
+
try {
|
|
461
|
+
process.stderr.write(`${notice}\n`);
|
|
462
|
+
} catch {}
|
|
463
|
+
}
|
|
450
464
|
}
|
package/src/request.js
CHANGED
|
@@ -25,17 +25,25 @@ const LOGIN_REFUSAL_ADVICE = {
|
|
|
25
25
|
|
|
26
26
|
/**
|
|
27
27
|
* Headers every request carries. The team header picks which of a person's teams a CLI session acts
|
|
28
|
-
* on; an API key belongs to one team and ignores it.
|
|
28
|
+
* on; an API key belongs to one team and ignores it. The terminal and platform name the device in
|
|
29
|
+
* the team's authentication log, since the CLI has no browser user agent to read it from.
|
|
29
30
|
*
|
|
30
|
-
* @param {{ secret: string | null; team_id: string | null }} options
|
|
31
|
+
* @param {{ secret: string | null; team_id: string | null; env?: NodeJS.ProcessEnv }} options
|
|
31
32
|
*/
|
|
32
|
-
export async function buildRequestHeaders({ secret, team_id }) {
|
|
33
|
+
export async function buildRequestHeaders({ secret, team_id, env = process.env }) {
|
|
33
34
|
/** @type {Record<string, string>} */
|
|
34
35
|
const headers = {
|
|
35
36
|
"Content-Type": "application/json",
|
|
36
37
|
"x-tailglow-client": "cli",
|
|
37
|
-
"x-tailglow-client-version": await readCliVersion()
|
|
38
|
+
"x-tailglow-client-version": await readCliVersion(),
|
|
39
|
+
"x-tailglow-client-platform": process.platform
|
|
38
40
|
};
|
|
41
|
+
|
|
42
|
+
// fetch throws on some header characters, such as a line break, so only a plain name is sent.
|
|
43
|
+
const terminal = env.TERM_PROGRAM?.trim();
|
|
44
|
+
if (terminal && /^[\w .+-]{1,32}$/.test(terminal)) {
|
|
45
|
+
headers["x-tailglow-client-terminal"] = terminal;
|
|
46
|
+
}
|
|
39
47
|
if (secret) headers.Authorization = `Bearer ${secret}`;
|
|
40
48
|
if (team_id) headers["x-tailglow-team-id"] = team_id;
|
|
41
49
|
return headers;
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
import { constants } from "node:fs";
|
|
2
|
+
import fs from "node:fs/promises";
|
|
3
|
+
import https from "node:https";
|
|
4
|
+
import { CLI_CONFIGS, readCliVersion, saveConfig } from "./config.js";
|
|
5
|
+
|
|
6
|
+
/** @typedef {{ checked_at: number; latest_version: string | null }} UpdateCheck */
|
|
7
|
+
|
|
8
|
+
/** @param {string} version */
|
|
9
|
+
function parseVersion(version) {
|
|
10
|
+
return /^(\d+)\.(\d+)\.(\d+)$/.exec(version)?.slice(1).map(Number) ?? null;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Whether `candidate` is a later release than `installed`. Only plain `x.y.z` versions compare;
|
|
15
|
+
* anything else never counts as newer.
|
|
16
|
+
*
|
|
17
|
+
* @param {string} candidate
|
|
18
|
+
* @param {string} installed
|
|
19
|
+
*/
|
|
20
|
+
export function isNewerVersion(candidate, installed) {
|
|
21
|
+
const next = parseVersion(candidate);
|
|
22
|
+
const current = parseVersion(installed);
|
|
23
|
+
if (!next || !current) return false;
|
|
24
|
+
for (let index = 0; index < 3; index += 1) {
|
|
25
|
+
if (next[index] !== current[index]) return next[index] > current[index];
|
|
26
|
+
}
|
|
27
|
+
return false;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* The stored check, when the path holds a small regular file. Opened without blocking, so a pipe
|
|
32
|
+
* left at the path cannot hang the command.
|
|
33
|
+
*
|
|
34
|
+
* @param {string} path
|
|
35
|
+
* @returns {Promise<UpdateCheck | null>}
|
|
36
|
+
*/
|
|
37
|
+
async function readUpdateCheck(path) {
|
|
38
|
+
/** @type {import("node:fs/promises").FileHandle | null} */
|
|
39
|
+
let handle = null;
|
|
40
|
+
try {
|
|
41
|
+
handle = await fs.open(path, constants.O_RDONLY | constants.O_NONBLOCK);
|
|
42
|
+
const stats = await handle.stat();
|
|
43
|
+
if (!stats.isFile() || stats.size > CLI_CONFIGS.UPDATE_CHECK_MAX_BYTES) return null;
|
|
44
|
+
const stored = JSON.parse(await handle.readFile("utf-8"));
|
|
45
|
+
if (typeof stored?.checked_at !== "number") return null;
|
|
46
|
+
const latest_version = typeof stored.latest_version === "string" ? stored.latest_version : null;
|
|
47
|
+
return { checked_at: stored.checked_at, latest_version };
|
|
48
|
+
} catch {
|
|
49
|
+
return null;
|
|
50
|
+
} finally {
|
|
51
|
+
await handle?.close().catch(() => {});
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** @param {string} body */
|
|
56
|
+
function readManifestVersion(body) {
|
|
57
|
+
try {
|
|
58
|
+
const manifest = JSON.parse(body);
|
|
59
|
+
return typeof manifest?.version === "string" ? manifest.version : null;
|
|
60
|
+
} catch {
|
|
61
|
+
return null;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* The version npm tags `latest`, or null. `node:https` rather than fetch: an aborted Node fetch
|
|
67
|
+
* keeps a stalled connection, and with it the process, alive for about ten more seconds, while an
|
|
68
|
+
* aborted request here closes its socket.
|
|
69
|
+
*
|
|
70
|
+
* @returns {Promise<string | null>}
|
|
71
|
+
*/
|
|
72
|
+
function fetchLatestVersion() {
|
|
73
|
+
return new Promise((resolve) => {
|
|
74
|
+
const request = https.get(
|
|
75
|
+
CLI_CONFIGS.UPDATE_CHECK_URL,
|
|
76
|
+
{ signal: AbortSignal.timeout(CLI_CONFIGS.UPDATE_CHECK_TIMEOUT_MS) },
|
|
77
|
+
(response) => {
|
|
78
|
+
let body = "";
|
|
79
|
+
response.setEncoding("utf-8");
|
|
80
|
+
response.on("data", (chunk) => {
|
|
81
|
+
body += chunk;
|
|
82
|
+
if (body.length > CLI_CONFIGS.UPDATE_CHECK_MAX_BYTES) request.destroy();
|
|
83
|
+
});
|
|
84
|
+
response.on("end", () =>
|
|
85
|
+
resolve(response.statusCode === 200 ? readManifestVersion(body) : null)
|
|
86
|
+
);
|
|
87
|
+
// Settles an aborted or destroyed response; after `end` the promise has already resolved.
|
|
88
|
+
response.on("close", () => resolve(null));
|
|
89
|
+
}
|
|
90
|
+
);
|
|
91
|
+
request.on("error", () => resolve(null));
|
|
92
|
+
// A request can also end with no response and no error, such as an upgrade nothing accepts.
|
|
93
|
+
request.on("close", () => resolve(null));
|
|
94
|
+
});
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* The line to print when a newer release is published, or null. npm is asked at most once a day,
|
|
99
|
+
* failed attempts included, so an offline machine is not slowed down on every command. Never
|
|
100
|
+
* throws: a missing answer only means no notice.
|
|
101
|
+
*
|
|
102
|
+
* @param {{
|
|
103
|
+
* env?: NodeJS.ProcessEnv;
|
|
104
|
+
* fetch_latest_version?: () => Promise<string | null>;
|
|
105
|
+
* now?: number;
|
|
106
|
+
* path?: string;
|
|
107
|
+
* }} [options]
|
|
108
|
+
* @returns {Promise<string | null>}
|
|
109
|
+
*/
|
|
110
|
+
export async function readUpdateNotice({
|
|
111
|
+
env = process.env,
|
|
112
|
+
fetch_latest_version = fetchLatestVersion,
|
|
113
|
+
now = Date.now(),
|
|
114
|
+
path = CLI_CONFIGS.UPDATE_CHECK_PATH
|
|
115
|
+
} = {}) {
|
|
116
|
+
if (env.CI || env.TAILGLOW_NO_UPDATE_CHECK) return null;
|
|
117
|
+
try {
|
|
118
|
+
let check = await readUpdateCheck(path);
|
|
119
|
+
if (!check || now - check.checked_at >= CLI_CONFIGS.UPDATE_CHECK_INTERVAL_MS) {
|
|
120
|
+
const previous = check?.latest_version ?? null;
|
|
121
|
+
// Recorded before npm is asked: an attempt that cannot be recorded is not made, so a machine
|
|
122
|
+
// that cannot write the file is never slowed down on every command.
|
|
123
|
+
await saveConfig({ checked_at: now, latest_version: previous }, path);
|
|
124
|
+
check = { checked_at: now, latest_version: (await fetch_latest_version()) ?? previous };
|
|
125
|
+
if (check.latest_version !== previous) await saveConfig(check, path);
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
const installed = await readCliVersion();
|
|
129
|
+
if (!check.latest_version || !isNewerVersion(check.latest_version, installed)) return null;
|
|
130
|
+
return (
|
|
131
|
+
`tglow ${check.latest_version} is available (you have ${installed}). ` +
|
|
132
|
+
"Update with `npm install -g @tailglow/cli@latest`, or run `npx @tailglow/cli@latest`."
|
|
133
|
+
);
|
|
134
|
+
} catch {
|
|
135
|
+
return null;
|
|
136
|
+
}
|
|
137
|
+
}
|