@tailglow/cli 0.6.5 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +88 -7
- package/package.json +2 -2
- package/src/commands.generated.js +593 -262
- package/src/docs.generated.js +32 -32
- package/src/help.js +7 -1
- package/src/import-command.js +647 -0
- package/src/import-file.js +181 -0
- package/src/import-options.js +24 -0
- package/src/import-worker.generated.js +1616 -0
- package/src/import.generated.d.ts +155 -0
- package/src/import.generated.js +4731 -0
- package/src/index.js +14 -2
package/src/help.js
CHANGED
|
@@ -85,7 +85,7 @@ export const GLOBAL_FLAGS = [
|
|
|
85
85
|
*/
|
|
86
86
|
export const USAGE_NOTES = [
|
|
87
87
|
"In a terminal, `tglow` offers sign-in when no credential is configured. `tglow help` always shows help.",
|
|
88
|
-
"Flags are the field name in kebab-case: a field named `
|
|
88
|
+
"Flags are the field name in kebab-case: a field named `repeat_window_minutes` is `--repeat-window-minutes`.",
|
|
89
89
|
"Path parameters are flags too, never positional: `--project-id prj_x`, not `tglow ... prj_x`.",
|
|
90
90
|
"A true/false flag is passed alone for true, or written attached for false: `--is-public` or `--is-public=false`.",
|
|
91
91
|
'A field typed `object`, `object[]` or `Scopes` takes JSON as a single argument: --joins \'[{"lookup_view_id":"view_b","alias":"b","base_field":"user_id","lookup_field":"id"}]\'. Its keys are listed under it in the command\'s help.',
|
|
@@ -229,6 +229,11 @@ export function describeForMachines({ resource, command } = {}) {
|
|
|
229
229
|
description:
|
|
230
230
|
"Every guide and API reference page as markdown, shipped with this CLI. Start with `tglow docs guides/agents`."
|
|
231
231
|
},
|
|
232
|
+
import: {
|
|
233
|
+
command: "tglow import scan|validate FILE; tglow import FILE --collection NAME",
|
|
234
|
+
description:
|
|
235
|
+
"Preview XML, JSONL, JSON, CSV and TSV, then validate records while importing. Full checking is optional. Run tglow import --help."
|
|
236
|
+
},
|
|
232
237
|
account: describeLoginForMachines(),
|
|
233
238
|
global_flags: globals,
|
|
234
239
|
usage_notes: USAGE_NOTES
|
|
@@ -250,6 +255,7 @@ export function renderRootHelp() {
|
|
|
250
255
|
...resources.map((resource) => ` ${resource}`),
|
|
251
256
|
" config (stored defaults and credentials)",
|
|
252
257
|
" docs (every guide and API reference page, as markdown)",
|
|
258
|
+
" import (scan, validate and upload XML, JSONL, JSON, CSV or TSV files)",
|
|
253
259
|
"",
|
|
254
260
|
"Account:",
|
|
255
261
|
" tglow login Sign in through the browser. `tglow login --help` for agents and scripts.",
|
|
@@ -0,0 +1,647 @@
|
|
|
1
|
+
import { parseArgs } from "node:util";
|
|
2
|
+
import { loadConfig, resolveApiUrl, resolveCredential, resolveSetting } from "./config.js";
|
|
3
|
+
import { openImportFile, snapshotImportFile } from "./import-file.js";
|
|
4
|
+
import { IMPORT_PARSER_OPTIONS } from "./import-options.js";
|
|
5
|
+
import {
|
|
6
|
+
createImportBatches,
|
|
7
|
+
createNodeImportWorker,
|
|
8
|
+
detectImport,
|
|
9
|
+
IMPORT_MAX_WORKERS,
|
|
10
|
+
IMPORT_REJECTED_STATUSES,
|
|
11
|
+
readImportRows,
|
|
12
|
+
scanImport
|
|
13
|
+
} from "./import.generated.js";
|
|
14
|
+
import { sanitizeText, shouldEmitJson } from "./output.js";
|
|
15
|
+
import { buildRequestHeaders } from "./request.js";
|
|
16
|
+
|
|
17
|
+
const FORMATS = ["auto", "xml", "jsonl", "json", "csv", "tsv"];
|
|
18
|
+
// Bundled from packages/import/src/worker.node.js by scripts/build/import.build.js.
|
|
19
|
+
const IMPORT_WORKER_URL = new URL("./import-worker.generated.js", import.meta.url);
|
|
20
|
+
const LOCAL_HOSTS = new Set(["localhost", "127.0.0.1", "[::1]"]);
|
|
21
|
+
const HELP_NOTES = [
|
|
22
|
+
"Scan and validate run offline. Scan samples unless --full is given; a sample is not validation.",
|
|
23
|
+
"Upload previews a bounded sample, then validates records while sending bounded batches.",
|
|
24
|
+
"--validate-first copies a private snapshot and checks it fully before upload; it needs temporary disk space approximately equal to the input size.",
|
|
25
|
+
"Default upload reads one open file and stops if it detects changes. Do not edit it during import.",
|
|
26
|
+
"Selected timestamps must exist and be unambiguous. Distinct times may produce small batches.",
|
|
27
|
+
"Only HTTP 202 acknowledges a batch. Delivery uncertainty stops upload without retrying.",
|
|
28
|
+
"Ctrl-C reports accepted rows and any uncertain batch. Retrying a partial import may duplicate rows."
|
|
29
|
+
];
|
|
30
|
+
|
|
31
|
+
const HELP = [
|
|
32
|
+
"tglow import scan FILE [--full] [options]",
|
|
33
|
+
"tglow import validate FILE [options]",
|
|
34
|
+
"tglow import FILE --collection NAME [destination] [options]",
|
|
35
|
+
"",
|
|
36
|
+
"--format auto|xml|jsonl|json|csv|tsv",
|
|
37
|
+
"--record-path /root/item Optional XML filter; imports all top-level entries by default.",
|
|
38
|
+
"--timestamp-field FIELD JSON pointer or exact top-level name; never selected automatically.",
|
|
39
|
+
"--workers auto|0|N XML worker threads for full reads; auto uses 2 from 32 MiB.",
|
|
40
|
+
"--validate-first Check a private snapshot completely before uploading it.",
|
|
41
|
+
"--ingest-url URL Explicit ingest URL, including its key. Keep this credential private.",
|
|
42
|
+
"--project ID Expected project, also accepts --project-id; defaults from config.",
|
|
43
|
+
"--source-id ID --ingest-key-id ID Resolve this exact key through the authenticated API.",
|
|
44
|
+
"--collection NAME Required upload destination collection.",
|
|
45
|
+
"--json Machine JSON; implied when stdout is piped.",
|
|
46
|
+
"",
|
|
47
|
+
...HELP_NOTES
|
|
48
|
+
].join("\n");
|
|
49
|
+
|
|
50
|
+
/** @returns {Record<string, unknown>} */
|
|
51
|
+
export function describeImportForMachines() {
|
|
52
|
+
return {
|
|
53
|
+
usage: ["tglow import scan FILE [--full]", "tglow import validate FILE", "tglow import FILE"],
|
|
54
|
+
formats: FORMATS,
|
|
55
|
+
flags: Object.entries(IMPORT_PARSER_OPTIONS).map(([name, option]) => ({
|
|
56
|
+
flag: "--" + name,
|
|
57
|
+
type: option.type
|
|
58
|
+
})),
|
|
59
|
+
notes: HELP_NOTES
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/** @param {unknown} error @param {Record<string, unknown>} flags */
|
|
64
|
+
function safeError(error, flags) {
|
|
65
|
+
let message = error instanceof Error ? error.message : "Import failed.";
|
|
66
|
+
for (const name of ["api-key", "ingest-url"]) {
|
|
67
|
+
const value = flags[name];
|
|
68
|
+
if (typeof value === "string" && value) message = message.split(value).join("[redacted]");
|
|
69
|
+
}
|
|
70
|
+
message = message.replace(/https?:\/\/[^\s"'<>]+/gi, "[redacted URL]");
|
|
71
|
+
return sanitizeText(message).slice(0, 1200);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/** @param {unknown} value @returns {"auto" | number} */
|
|
75
|
+
function parseWorkers(value) {
|
|
76
|
+
if (value === undefined || value === "auto") return "auto";
|
|
77
|
+
if (typeof value === "string" && /^\d{1,2}$/.test(value) && Number(value) <= IMPORT_MAX_WORKERS)
|
|
78
|
+
return Number(value);
|
|
79
|
+
throw new Error(`--workers takes auto or a whole number from 0 to ${IMPORT_MAX_WORKERS}.`);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/** @param {AbortSignal} signal */
|
|
83
|
+
function checkAbort(signal) {
|
|
84
|
+
if (signal.aborted) throw new Error("Import cancelled.");
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** @param {string} address @param {boolean} [is_ingest] */
|
|
88
|
+
function safeUrl(address, is_ingest = false) {
|
|
89
|
+
let url;
|
|
90
|
+
try {
|
|
91
|
+
url = new URL(address);
|
|
92
|
+
} catch {
|
|
93
|
+
throw new Error("Invalid destination address.");
|
|
94
|
+
}
|
|
95
|
+
if (
|
|
96
|
+
url.username ||
|
|
97
|
+
url.password ||
|
|
98
|
+
url.hash ||
|
|
99
|
+
(url.protocol !== "https:" && !(url.protocol === "http:" && LOCAL_HOSTS.has(url.hostname)))
|
|
100
|
+
) {
|
|
101
|
+
throw new Error(
|
|
102
|
+
"Use an HTTPS address, or HTTP on localhost, without URL credentials or fragments."
|
|
103
|
+
);
|
|
104
|
+
}
|
|
105
|
+
if (is_ingest && !url.searchParams.get("key"))
|
|
106
|
+
throw new Error("The ingest address must contain its key.");
|
|
107
|
+
return url;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* A manually linked signal works on the oldest supported Node release.
|
|
112
|
+
*
|
|
113
|
+
* @param {AbortSignal} parent @param {number} milliseconds
|
|
114
|
+
*/
|
|
115
|
+
function requestSignal(parent, milliseconds) {
|
|
116
|
+
const controller = new AbortController();
|
|
117
|
+
const abort = () => controller.abort();
|
|
118
|
+
if (parent.aborted) abort();
|
|
119
|
+
else parent.addEventListener("abort", abort, { once: true });
|
|
120
|
+
const timer = setTimeout(abort, milliseconds);
|
|
121
|
+
return {
|
|
122
|
+
signal: controller.signal,
|
|
123
|
+
dispose() {
|
|
124
|
+
clearTimeout(timer);
|
|
125
|
+
parent.removeEventListener("abort", abort);
|
|
126
|
+
}
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/** @param {Response} response */
|
|
131
|
+
async function smallJson(response) {
|
|
132
|
+
if (!response.body) throw new Error("Missing response.");
|
|
133
|
+
const reader = response.body.getReader();
|
|
134
|
+
const decoder = new TextDecoder("utf-8", { fatal: true });
|
|
135
|
+
let text = "";
|
|
136
|
+
let bytes = 0;
|
|
137
|
+
try {
|
|
138
|
+
for (;;) {
|
|
139
|
+
const { value, done } = await reader.read();
|
|
140
|
+
if (done) break;
|
|
141
|
+
bytes += value.byteLength;
|
|
142
|
+
if (bytes > 64 * 1024) throw new Error("Oversized response.");
|
|
143
|
+
text += decoder.decode(value, { stream: true });
|
|
144
|
+
}
|
|
145
|
+
return JSON.parse(text + decoder.decode());
|
|
146
|
+
} finally {
|
|
147
|
+
// Stream cleanup must not hold up the result or cancellation of the command.
|
|
148
|
+
void reader.cancel().catch(() => {});
|
|
149
|
+
reader.releaseLock();
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
/**
|
|
154
|
+
* @param {URL} url @param {RequestInit} options @param {AbortSignal} signal @param {typeof fetch}
|
|
155
|
+
* request
|
|
156
|
+
*/
|
|
157
|
+
async function getJson(url, options, signal, request) {
|
|
158
|
+
const linked = requestSignal(signal, 8000);
|
|
159
|
+
try {
|
|
160
|
+
const response = await request(url, {
|
|
161
|
+
...options,
|
|
162
|
+
method: "GET",
|
|
163
|
+
signal: linked.signal,
|
|
164
|
+
redirect: "error",
|
|
165
|
+
cache: "no-store",
|
|
166
|
+
credentials: "omit"
|
|
167
|
+
});
|
|
168
|
+
if (!response.ok) {
|
|
169
|
+
void response.body?.cancel().catch(() => {});
|
|
170
|
+
throw new Error("Refused response.");
|
|
171
|
+
}
|
|
172
|
+
return await smallJson(response);
|
|
173
|
+
} catch {
|
|
174
|
+
checkAbort(signal);
|
|
175
|
+
throw new Error(
|
|
176
|
+
"Could not verify the destination. Check credentials, IDs, and endpoint availability."
|
|
177
|
+
);
|
|
178
|
+
} finally {
|
|
179
|
+
linked.dispose();
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
/**
|
|
184
|
+
* @param {Record<string, unknown>} flags
|
|
185
|
+
* @param {AbortSignal} signal
|
|
186
|
+
* @param {typeof fetch} request
|
|
187
|
+
* @param {typeof loadConfig} read_config
|
|
188
|
+
*/
|
|
189
|
+
async function resolveDestination(flags, signal, request, read_config) {
|
|
190
|
+
const config = await read_config();
|
|
191
|
+
const project_id =
|
|
192
|
+
typeof flags["project-id"] === "string"
|
|
193
|
+
? flags["project-id"]
|
|
194
|
+
: resolveSetting("project", { flags, config });
|
|
195
|
+
if (flags.project && flags["project-id"] && flags.project !== flags["project-id"]) {
|
|
196
|
+
throw new Error("--project and --project-id must name the same project.");
|
|
197
|
+
}
|
|
198
|
+
if (!project_id || project_id === "." || project_id === "..")
|
|
199
|
+
throw new Error("Upload requires --project (or a configured project).");
|
|
200
|
+
let url;
|
|
201
|
+
if (typeof flags["ingest-url"] === "string") {
|
|
202
|
+
if (flags["source-id"] || flags["ingest-key-id"])
|
|
203
|
+
throw new Error("Choose --ingest-url or explicit --source-id/--ingest-key-id, not both.");
|
|
204
|
+
url = safeUrl(flags["ingest-url"], true);
|
|
205
|
+
} else {
|
|
206
|
+
const source_id = flags["source-id"];
|
|
207
|
+
const ingest_key_id = flags["ingest-key-id"];
|
|
208
|
+
if (typeof source_id !== "string" || typeof ingest_key_id !== "string") {
|
|
209
|
+
throw new Error("Upload requires --ingest-url, or both --source-id and --ingest-key-id.");
|
|
210
|
+
}
|
|
211
|
+
if ([source_id, ingest_key_id].some((id) => id === "." || id === ".."))
|
|
212
|
+
throw new Error("Destination IDs cannot be dot segments.");
|
|
213
|
+
const api_url = safeUrl(resolveApiUrl({ flags, config }));
|
|
214
|
+
let credential;
|
|
215
|
+
try {
|
|
216
|
+
credential = resolveCredential({ flags, config });
|
|
217
|
+
} catch {
|
|
218
|
+
throw new Error(
|
|
219
|
+
"Sign in to the configured API or supply an API key to resolve this ingest key."
|
|
220
|
+
);
|
|
221
|
+
}
|
|
222
|
+
const team_id = resolveSetting("team", { flags, config }) ?? credential.login?.team_id ?? null;
|
|
223
|
+
const path =
|
|
224
|
+
"/v1/projects/" +
|
|
225
|
+
encodeURIComponent(project_id) +
|
|
226
|
+
"/sources/" +
|
|
227
|
+
encodeURIComponent(source_id) +
|
|
228
|
+
"/ingest_keys/" +
|
|
229
|
+
encodeURIComponent(ingest_key_id);
|
|
230
|
+
|
|
231
|
+
const payload = await getJson(
|
|
232
|
+
new URL(path, api_url),
|
|
233
|
+
{
|
|
234
|
+
headers: await buildRequestHeaders({ secret: credential.secret, team_id })
|
|
235
|
+
},
|
|
236
|
+
signal,
|
|
237
|
+
request
|
|
238
|
+
);
|
|
239
|
+
|
|
240
|
+
const key = payload?.data;
|
|
241
|
+
if (
|
|
242
|
+
key?.id !== ingest_key_id ||
|
|
243
|
+
key.source_id !== source_id ||
|
|
244
|
+
key.status !== "active" ||
|
|
245
|
+
typeof key.url !== "string"
|
|
246
|
+
) {
|
|
247
|
+
throw new Error(
|
|
248
|
+
"The selected ingest key is unavailable, inactive, or belongs to another source."
|
|
249
|
+
);
|
|
250
|
+
}
|
|
251
|
+
url = safeUrl(key.url, true);
|
|
252
|
+
if (LOCAL_HOSTS.has(api_url.hostname) && !LOCAL_HOSTS.has(url.hostname)) {
|
|
253
|
+
throw new Error(
|
|
254
|
+
"A local API returned a remote ingest address. Pass an explicit --ingest-url for the intended destination."
|
|
255
|
+
);
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
url.searchParams.set("collection", String(flags.collection));
|
|
259
|
+
url.searchParams.delete("ts");
|
|
260
|
+
return { url, project_id };
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
/** @param {Record<string, unknown>} report */
|
|
264
|
+
function renderHumanReport(report) {
|
|
265
|
+
if (!report.error && (report.action === "scan" || report.action === "validate")) {
|
|
266
|
+
const lines = [
|
|
267
|
+
String(report.format).toUpperCase() + ": " + report.row_count + " selected rows.",
|
|
268
|
+
report.is_complete
|
|
269
|
+
? "Full validation complete."
|
|
270
|
+
: "Sample or discovery only; upload validation is still required."
|
|
271
|
+
];
|
|
272
|
+
lines.push(
|
|
273
|
+
"File size: " + report.total_bytes + " bytes; scanned " + report.bytes_read + " bytes."
|
|
274
|
+
);
|
|
275
|
+
const xml = /** @type {any} */ (report.xml);
|
|
276
|
+
if (xml?.record_paths?.length)
|
|
277
|
+
lines.push(
|
|
278
|
+
"XML record paths: " +
|
|
279
|
+
xml.record_paths
|
|
280
|
+
.map((/** @type {{ path: string }} */ entry) => sanitizeText(entry.path))
|
|
281
|
+
.join(", ")
|
|
282
|
+
);
|
|
283
|
+
const fields = /** @type {import("./import.generated.js").FieldProfile[]} */ (
|
|
284
|
+
report.fields ?? []
|
|
285
|
+
);
|
|
286
|
+
if (fields.length) {
|
|
287
|
+
lines.push("Fields:");
|
|
288
|
+
for (const field of fields.slice(0, 20))
|
|
289
|
+
lines.push(
|
|
290
|
+
" " +
|
|
291
|
+
sanitizeText(field.path) +
|
|
292
|
+
": " +
|
|
293
|
+
field.types.join(" | ") +
|
|
294
|
+
"; present " +
|
|
295
|
+
field.present +
|
|
296
|
+
", missing " +
|
|
297
|
+
field.missing
|
|
298
|
+
);
|
|
299
|
+
if (fields.length > 20) lines.push(" More fields are available with --json.");
|
|
300
|
+
}
|
|
301
|
+
const candidates = /** @type {import("./import.generated.js").TimestampCandidate[]} */ (
|
|
302
|
+
report.timestamp_candidates ?? []
|
|
303
|
+
);
|
|
304
|
+
if (candidates.length)
|
|
305
|
+
lines.push(
|
|
306
|
+
"Timestamp candidates: " + candidates.map((entry) => sanitizeText(entry.path)).join(", ")
|
|
307
|
+
);
|
|
308
|
+
for (const warning of /** @type {string[]} */ (report.warnings ?? []))
|
|
309
|
+
lines.push(sanitizeText(warning));
|
|
310
|
+
return lines.join("\n");
|
|
311
|
+
}
|
|
312
|
+
const affected = /** @type {any} */ (report.affected_batch);
|
|
313
|
+
return [
|
|
314
|
+
report.status === "complete"
|
|
315
|
+
? "Import complete."
|
|
316
|
+
: sanitizeText(String(report.error ?? "Import stopped.")),
|
|
317
|
+
"Accepted rows: " + report.accepted_count + ".",
|
|
318
|
+
...(report.status !== "complete" && Number(report.accepted_count) > 0
|
|
319
|
+
? ["Accepted rows remain in the collection. Retrying this file may duplicate them."]
|
|
320
|
+
: []),
|
|
321
|
+
...(affected
|
|
322
|
+
? [
|
|
323
|
+
"Batch rows " +
|
|
324
|
+
affected.first_row +
|
|
325
|
+
"-" +
|
|
326
|
+
affected.last_row +
|
|
327
|
+
": " +
|
|
328
|
+
affected.status +
|
|
329
|
+
". No retry was attempted."
|
|
330
|
+
]
|
|
331
|
+
: [])
|
|
332
|
+
].join("\n");
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
/**
|
|
336
|
+
* @typedef {Object} ImportCommandContext
|
|
337
|
+
* @property {typeof fetch} [request]
|
|
338
|
+
* @property {typeof loadConfig} [read_config]
|
|
339
|
+
* @property {AbortSignal} [signal]
|
|
340
|
+
* @property {{ write(text: string): unknown }} [stderr]
|
|
341
|
+
* @property {string} [temporary_root]
|
|
342
|
+
*/
|
|
343
|
+
|
|
344
|
+
/**
|
|
345
|
+
* Dedicated import parsing keeps file formats and local flags separate from the API registry.
|
|
346
|
+
*
|
|
347
|
+
* @param {string[]} argv Complete argv, including the import command.
|
|
348
|
+
* @param {ImportCommandContext} [context]
|
|
349
|
+
* @returns {Promise<{ output: string; code: number }>}
|
|
350
|
+
*/
|
|
351
|
+
export async function runImportCommand(argv, context = {}) {
|
|
352
|
+
/** @type {Record<string, unknown>} */
|
|
353
|
+
let flags = {};
|
|
354
|
+
let action = "import";
|
|
355
|
+
let accepted_count = 0;
|
|
356
|
+
/** @type {number | null} */
|
|
357
|
+
let total_rows = null;
|
|
358
|
+
let outcome = "invalid_input";
|
|
359
|
+
/**
|
|
360
|
+
* @type {{
|
|
361
|
+
* index: number;
|
|
362
|
+
* first_row: number;
|
|
363
|
+
* last_row: number;
|
|
364
|
+
* status: string;
|
|
365
|
+
* http_status?: number;
|
|
366
|
+
* } | null}
|
|
367
|
+
*/
|
|
368
|
+
let affected_batch = null;
|
|
369
|
+
/** @type {Awaited<ReturnType<typeof openImportFile>> | undefined} */
|
|
370
|
+
let input;
|
|
371
|
+
const controller = new AbortController();
|
|
372
|
+
const signal = controller.signal;
|
|
373
|
+
const abort = () => controller.abort();
|
|
374
|
+
const external_signal = context.signal;
|
|
375
|
+
if (external_signal?.aborted) abort();
|
|
376
|
+
else external_signal?.addEventListener("abort", abort, { once: true });
|
|
377
|
+
process.on("SIGINT", abort);
|
|
378
|
+
const stderr = context.stderr ?? process.stderr;
|
|
379
|
+
const request = context.request ?? fetch;
|
|
380
|
+
let result;
|
|
381
|
+
try {
|
|
382
|
+
let positionals;
|
|
383
|
+
try {
|
|
384
|
+
({ values: flags, positionals } = parseArgs({
|
|
385
|
+
args: argv,
|
|
386
|
+
options: IMPORT_PARSER_OPTIONS,
|
|
387
|
+
allowPositionals: true,
|
|
388
|
+
strict: false
|
|
389
|
+
}));
|
|
390
|
+
} catch {
|
|
391
|
+
throw new Error("Invalid import options. Run tglow import --help.");
|
|
392
|
+
}
|
|
393
|
+
for (const [name, value] of Object.entries(flags)) {
|
|
394
|
+
const option = IMPORT_PARSER_OPTIONS[name];
|
|
395
|
+
if (!option) throw new Error("Unknown import option. Run tglow import --help.");
|
|
396
|
+
if (option.type === "boolean") {
|
|
397
|
+
if (value === "true" || value === "false") flags[name] = value === "true";
|
|
398
|
+
else if (typeof value !== "boolean")
|
|
399
|
+
throw new Error("--" + name + " takes no value, or =true / =false.");
|
|
400
|
+
} else if (typeof value !== "string" || !value.length) {
|
|
401
|
+
throw new Error("--" + name + " requires a value.");
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
if (flags.help || positionals.length === 1) {
|
|
405
|
+
return {
|
|
406
|
+
output: shouldEmitJson(flags) ? JSON.stringify(describeImportForMachines(), null, 2) : HELP,
|
|
407
|
+
code: 0
|
|
408
|
+
};
|
|
409
|
+
}
|
|
410
|
+
if (positionals[0] !== "import") throw new Error("Expected the import command.");
|
|
411
|
+
const words = positionals.slice(1);
|
|
412
|
+
if (words[0] === "scan" || words[0] === "validate") {
|
|
413
|
+
action = words[0];
|
|
414
|
+
words.shift();
|
|
415
|
+
}
|
|
416
|
+
if (words.length !== 1)
|
|
417
|
+
throw new Error("Specify exactly one input file. Run tglow import --help.");
|
|
418
|
+
if (action !== "scan" && flags.full !== undefined)
|
|
419
|
+
throw new Error("--full only applies to import scan.");
|
|
420
|
+
if (action !== "import" && flags["validate-first"] !== undefined)
|
|
421
|
+
throw new Error("--validate-first only applies to uploading.");
|
|
422
|
+
if (!FORMATS.includes(String(flags.format ?? "auto")))
|
|
423
|
+
throw new Error("Unsupported --format. Use auto, xml, jsonl, json, csv, or tsv.");
|
|
424
|
+
const workers = parseWorkers(flags.workers);
|
|
425
|
+
if (
|
|
426
|
+
action !== "import" &&
|
|
427
|
+
["ingest-url", "collection", "source-id", "ingest-key-id"].some(
|
|
428
|
+
(name) => flags[name] !== undefined
|
|
429
|
+
)
|
|
430
|
+
) {
|
|
431
|
+
throw new Error("Destination options apply only to uploading, not scan or validate.");
|
|
432
|
+
}
|
|
433
|
+
if (action === "import" && (typeof flags.collection !== "string" || !flags.collection.trim()))
|
|
434
|
+
throw new Error("Upload requires an explicit --collection.");
|
|
435
|
+
checkAbort(signal);
|
|
436
|
+
let last_progress = 0;
|
|
437
|
+
/** @param {import("./import.generated.js").ImportProgress} progress */
|
|
438
|
+
const onprogress = ({ bytes_read, total_bytes, rows }) => {
|
|
439
|
+
if (Date.now() - last_progress < 1000) return;
|
|
440
|
+
last_progress = Date.now();
|
|
441
|
+
stderr.write("Reading: " + bytes_read + "/" + total_bytes + " bytes, " + rows + " rows.\n");
|
|
442
|
+
};
|
|
443
|
+
|
|
444
|
+
// Each scan and upload pass starts its own workers and reports what actually ran. Automatic
|
|
445
|
+
// sequential reads stay quiet; explicit counts always explain the outcome.
|
|
446
|
+
/** @param {import("./import.generated.js").ImportWorkerStatus} status */
|
|
447
|
+
const onworkers = (status) => {
|
|
448
|
+
let line;
|
|
449
|
+
if (status.fallback)
|
|
450
|
+
line = `XML workers stopped (${status.fallback}); continuing sequentially.`;
|
|
451
|
+
else if (status.actual)
|
|
452
|
+
line = `XML workers: ${status.actual}${status.reason ? ` (${status.reason})` : ""}.`;
|
|
453
|
+
else if (status.requested || (workers !== "auto" && workers > 0))
|
|
454
|
+
line = `XML workers: none, reading sequentially (${status.reason}).`;
|
|
455
|
+
else return;
|
|
456
|
+
stderr.write(sanitizeText(line) + "\n");
|
|
457
|
+
};
|
|
458
|
+
|
|
459
|
+
outcome = "file_error";
|
|
460
|
+
if (action === "import" && flags["validate-first"] === true) {
|
|
461
|
+
stderr.write("Making a private snapshot for a full check before upload.\n");
|
|
462
|
+
input = await snapshotImportFile(words[0], {
|
|
463
|
+
signal,
|
|
464
|
+
temporary_root: context.temporary_root
|
|
465
|
+
});
|
|
466
|
+
} else input = await openImportFile(words[0], { signal });
|
|
467
|
+
const source = input.source;
|
|
468
|
+
outcome = "validation_failed";
|
|
469
|
+
const detection = await detectImport(source, {
|
|
470
|
+
format: /** @type {any} */ (flags.format ?? "auto"),
|
|
471
|
+
signal
|
|
472
|
+
});
|
|
473
|
+
|
|
474
|
+
const options = Object.freeze({
|
|
475
|
+
format: detection.format,
|
|
476
|
+
...(typeof flags["record-path"] === "string" ? { record_path: flags["record-path"] } : {}),
|
|
477
|
+
...(typeof flags["timestamp-field"] === "string"
|
|
478
|
+
? { timestamp_field: flags["timestamp-field"] }
|
|
479
|
+
: {}),
|
|
480
|
+
workers,
|
|
481
|
+
create_worker: () => createNodeImportWorker(IMPORT_WORKER_URL),
|
|
482
|
+
onworkers,
|
|
483
|
+
signal,
|
|
484
|
+
onprogress
|
|
485
|
+
});
|
|
486
|
+
|
|
487
|
+
const report = await scanImport(source, {
|
|
488
|
+
...options,
|
|
489
|
+
is_full: action === "validate" || flags.full === true || flags["validate-first"] === true
|
|
490
|
+
});
|
|
491
|
+
if (action !== "import") {
|
|
492
|
+
if (action === "validate" && (!report.is_complete || report.is_sampled))
|
|
493
|
+
throw new Error("Validation did not cover the complete selected input.");
|
|
494
|
+
return {
|
|
495
|
+
output: shouldEmitJson(flags)
|
|
496
|
+
? JSON.stringify({ action, ...report }, null, 2)
|
|
497
|
+
: renderHumanReport({ action, ...report }),
|
|
498
|
+
code: 0
|
|
499
|
+
};
|
|
500
|
+
}
|
|
501
|
+
if (flags["validate-first"] === true && (!report.is_complete || report.is_sampled))
|
|
502
|
+
throw new Error("The full file check did not finish.");
|
|
503
|
+
total_rows = report.is_complete && !report.is_sampled ? report.row_count : null;
|
|
504
|
+
outcome = "destination_failed";
|
|
505
|
+
const batch_options = Object.freeze({ timestamp_field: options.timestamp_field });
|
|
506
|
+
const destination = await resolveDestination(
|
|
507
|
+
flags,
|
|
508
|
+
signal,
|
|
509
|
+
request,
|
|
510
|
+
context.read_config ?? loadConfig
|
|
511
|
+
);
|
|
512
|
+
|
|
513
|
+
const readiness = await getJson(destination.url, {}, signal, request);
|
|
514
|
+
if (readiness?.status !== "ready" || readiness.project_id !== destination.project_id)
|
|
515
|
+
throw new Error("The ingest endpoint is not ready for the expected project.");
|
|
516
|
+
if (options.timestamp_field)
|
|
517
|
+
stderr.write("Preserving selected timestamps; distinct values may produce small batches.\n");
|
|
518
|
+
outcome = "validation_failed";
|
|
519
|
+
for await (const batch of createImportBatches(readImportRows(source, options), batch_options)) {
|
|
520
|
+
checkAbort(signal);
|
|
521
|
+
await input.assertUnchanged();
|
|
522
|
+
const target = new URL(destination.url);
|
|
523
|
+
if (batch.timestamp) target.searchParams.set("ts", batch.timestamp);
|
|
524
|
+
affected_batch = {
|
|
525
|
+
index: batch.index,
|
|
526
|
+
first_row: batch.rows[0].index,
|
|
527
|
+
last_row: batch.rows[batch.rows.length - 1].index,
|
|
528
|
+
status: "uncertain"
|
|
529
|
+
};
|
|
530
|
+
outcome = "uncertain_delivery";
|
|
531
|
+
const linked = requestSignal(signal, 30_000);
|
|
532
|
+
try {
|
|
533
|
+
const response = await request(target, {
|
|
534
|
+
method: "POST",
|
|
535
|
+
headers: { "Content-Type": "application/x-ndjson" },
|
|
536
|
+
body: batch.body,
|
|
537
|
+
signal: linked.signal,
|
|
538
|
+
redirect: "error",
|
|
539
|
+
credentials: "omit"
|
|
540
|
+
});
|
|
541
|
+
if (response.status !== 202) {
|
|
542
|
+
affected_batch.status = IMPORT_REJECTED_STATUSES.includes(response.status)
|
|
543
|
+
? "rejected"
|
|
544
|
+
: "uncertain";
|
|
545
|
+
outcome = affected_batch.status === "rejected" ? "rejected" : "uncertain_delivery";
|
|
546
|
+
affected_batch.http_status = response.status;
|
|
547
|
+
void response.body?.cancel().catch(() => {});
|
|
548
|
+
throw new Error(
|
|
549
|
+
"The server did not acknowledge the batch. Upload stopped without retrying."
|
|
550
|
+
);
|
|
551
|
+
}
|
|
552
|
+
accepted_count += batch.rows.length;
|
|
553
|
+
affected_batch = null;
|
|
554
|
+
void response.body?.cancel().catch(() => {});
|
|
555
|
+
if (Date.now() - last_progress >= 1000 || accepted_count === total_rows) {
|
|
556
|
+
last_progress = Date.now();
|
|
557
|
+
stderr.write(
|
|
558
|
+
"Accepted " +
|
|
559
|
+
accepted_count +
|
|
560
|
+
(total_rows === null ? "" : "/" + total_rows) +
|
|
561
|
+
" rows.\n"
|
|
562
|
+
);
|
|
563
|
+
}
|
|
564
|
+
} catch {
|
|
565
|
+
if (affected_batch?.status === "rejected")
|
|
566
|
+
throw new Error("The server rejected the batch. Upload stopped without retrying.");
|
|
567
|
+
throw new Error(
|
|
568
|
+
"No acknowledgement from the server. The affected batch may have arrived; upload stopped without retrying."
|
|
569
|
+
);
|
|
570
|
+
} finally {
|
|
571
|
+
linked.dispose();
|
|
572
|
+
}
|
|
573
|
+
outcome = "validation_failed";
|
|
574
|
+
}
|
|
575
|
+
checkAbort(signal);
|
|
576
|
+
await input.assertUnchanged();
|
|
577
|
+
if (!accepted_count)
|
|
578
|
+
throw new Error(
|
|
579
|
+
"This selection has no data rows. Check the XML record path or file contents."
|
|
580
|
+
);
|
|
581
|
+
if (total_rows !== null && accepted_count !== total_rows)
|
|
582
|
+
throw new Error("The imported row count differed from the full file check.");
|
|
583
|
+
total_rows = accepted_count;
|
|
584
|
+
result = {
|
|
585
|
+
action,
|
|
586
|
+
status: "complete",
|
|
587
|
+
outcome: "complete",
|
|
588
|
+
accepted_count,
|
|
589
|
+
total_rows,
|
|
590
|
+
affected_batch: null
|
|
591
|
+
};
|
|
592
|
+
} catch (error) {
|
|
593
|
+
if (input && !signal.aborted && !affected_batch) {
|
|
594
|
+
try {
|
|
595
|
+
await input.assertUnchanged();
|
|
596
|
+
} catch (source_error) {
|
|
597
|
+
if (
|
|
598
|
+
source_error &&
|
|
599
|
+
typeof source_error === "object" &&
|
|
600
|
+
"code" in source_error &&
|
|
601
|
+
source_error.code === "IMPORT_SOURCE_CHANGED"
|
|
602
|
+
)
|
|
603
|
+
error = source_error;
|
|
604
|
+
}
|
|
605
|
+
}
|
|
606
|
+
result = {
|
|
607
|
+
action,
|
|
608
|
+
status: signal.aborted ? "cancelled" : "failed",
|
|
609
|
+
outcome:
|
|
610
|
+
error &&
|
|
611
|
+
typeof error === "object" &&
|
|
612
|
+
"code" in error &&
|
|
613
|
+
error.code === "IMPORT_SOURCE_CHANGED"
|
|
614
|
+
? "file_changed"
|
|
615
|
+
: signal.aborted && !affected_batch
|
|
616
|
+
? "cancelled"
|
|
617
|
+
: outcome,
|
|
618
|
+
...(error &&
|
|
619
|
+
typeof error === "object" &&
|
|
620
|
+
"index" in error &&
|
|
621
|
+
Number.isSafeInteger(error.index)
|
|
622
|
+
? { failed_row: error.index }
|
|
623
|
+
: {}),
|
|
624
|
+
error: safeError(error, flags),
|
|
625
|
+
accepted_count,
|
|
626
|
+
total_rows,
|
|
627
|
+
affected_batch
|
|
628
|
+
};
|
|
629
|
+
} finally {
|
|
630
|
+
if (input) {
|
|
631
|
+
try {
|
|
632
|
+
await input.cleanup();
|
|
633
|
+
} catch {
|
|
634
|
+
const cleanup_error =
|
|
635
|
+
"Could not close the input or remove its private temporary snapshot. Check the tailglow-import directory in your temporary folder.";
|
|
636
|
+
result = { ...result, cleanup_error };
|
|
637
|
+
stderr.write("Warning: " + cleanup_error + "\n");
|
|
638
|
+
}
|
|
639
|
+
}
|
|
640
|
+
external_signal?.removeEventListener("abort", abort);
|
|
641
|
+
process.removeListener("SIGINT", abort);
|
|
642
|
+
}
|
|
643
|
+
return {
|
|
644
|
+
output: shouldEmitJson(flags) ? JSON.stringify(result, null, 2) : renderHumanReport(result),
|
|
645
|
+
code: result.status === "complete" ? 0 : signal.aborted ? 130 : 1
|
|
646
|
+
};
|
|
647
|
+
}
|