@tailglow/cli 0.7.1 → 0.9.4

package/src/help.js

@@ -229,6 +229,11 @@ export function describeForMachines({ resource, command } = {}) {
       description:
         "Every guide and API reference page as markdown, shipped with this CLI. Start with `tglow docs guides/agents`."
     },
+    import: {
+      command: "tglow import scan|validate FILE; tglow import FILE --collection NAME",
+      description:
+        "Preview XML, JSONL, JSON, CSV and TSV, then validate records while importing. Full checking is optional. Run tglow import --help."
+    },
     account: describeLoginForMachines(),
     global_flags: globals,
     usage_notes: USAGE_NOTES
@@ -250,6 +255,7 @@ export function renderRootHelp() {
     ...resources.map((resource) => `  ${resource}`),
     "  config  (stored defaults and credentials)",
     "  docs    (every guide and API reference page, as markdown)",
+    "  import  (scan, validate and upload XML, JSONL, JSON, CSV or TSV files)",
     "",
     "Account:",
     "  tglow login   Sign in through the browser. `tglow login --help` for agents and scripts.",

package/src/import-command.js

@@ -0,0 +1,636 @@
+import { parseArgs } from "node:util";
+import { loadConfig, resolveApiUrl, resolveCredential, resolveSetting } from "./config.js";
+import { openImportFile, snapshotImportFile } from "./import-file.js";
+import { IMPORT_PARSER_OPTIONS } from "./import-options.js";
+import {
+  createImportBatches,
+  createNodeImportWorker,
+  detectImport,
+  IMPORT_MAX_WORKERS,
+  IMPORT_REJECTED_STATUSES,
+  readImportRows,
+  scanImport
+} from "./import.generated.js";
+import { sanitizeText, shouldEmitJson } from "./output.js";
+import { buildRequestHeaders } from "./request.js";
+
+const FORMATS = ["auto", "xml", "jsonl", "json", "csv", "tsv"];
+// Bundled from packages/import/src/worker.node.js by scripts/build/import.build.js.
+const IMPORT_WORKER_URL = new URL("./import-worker.generated.js", import.meta.url);
+const LOCAL_HOSTS = new Set(["localhost", "127.0.0.1", "[::1]"]);
+const HELP_NOTES = [
+  "Scan and validate run offline. Scan samples unless --full is given; a sample is not validation.",
+  "Upload previews a bounded sample, then validates records while sending bounded batches.",
+  "--validate-first copies a private snapshot and checks it fully before upload; it needs temporary disk space approximately equal to the input size.",
+  "Default upload reads one open file and stops if it detects changes. Do not edit it during import.",
+  "Only HTTP 202 acknowledges a batch. Delivery uncertainty stops upload without retrying.",
+  "Ctrl-C reports accepted rows and any uncertain batch. Retrying a partial import may duplicate rows."
+];
+
+const HELP = [
+  "tglow import scan FILE [--full] [options]",
+  "tglow import validate FILE [options]",
+  "tglow import FILE --collection NAME [destination] [options]",
+  "",
+  "--format auto|xml|jsonl|json|csv|tsv",
+  "--record-path /root/item   Optional XML filter; imports all top-level entries by default.",
+  "--workers auto|0|N       XML worker threads for full reads; auto uses 2 from 32 MiB.",
+  "--validate-first         Check a private snapshot completely before uploading it.",
+  "--ingest-url URL         Explicit ingest URL, including its key. Keep this credential private.",
+  "--project ID             Expected project, also accepts --project-id; defaults from config.",
+  "--source-id ID --ingest-key-id ID  Resolve this exact key through the authenticated API.",
+  "--collection NAME        Required upload destination collection.",
+  "--json                   Machine JSON; implied when stdout is piped.",
+  "",
+  ...HELP_NOTES
+].join("\n");
+
+/** @returns {Record<string, unknown>} */
+export function describeImportForMachines() {
+  return {
+    usage: ["tglow import scan FILE [--full]", "tglow import validate FILE", "tglow import FILE"],
+    formats: FORMATS,
+    flags: Object.entries(IMPORT_PARSER_OPTIONS).map(([name, option]) => ({
+      flag: "--" + name,
+      type: option.type
+    })),
+    notes: HELP_NOTES
+  };
+}
+
+/** @param {unknown} error @param {Record<string, unknown>} flags */
+function safeError(error, flags) {
+  let message = error instanceof Error ? error.message : "Import failed.";
+  for (const name of ["api-key", "ingest-url"]) {
+    const value = flags[name];
+    if (typeof value === "string" && value) message = message.split(value).join("[redacted]");
+  }
+  message = message.replace(/https?:\/\/[^\s"'<>]+/gi, "[redacted URL]");
+  return sanitizeText(message).slice(0, 1200);
+}
+
+/** @param {unknown} value @returns {"auto" | number} */
+function parseWorkers(value) {
+  if (value === undefined || value === "auto") return "auto";
+  if (typeof value === "string" && /^\d{1,2}$/.test(value) && Number(value) <= IMPORT_MAX_WORKERS)
+    return Number(value);
+  throw new Error(`--workers takes auto or a whole number from 0 to ${IMPORT_MAX_WORKERS}.`);
+}
+
+/** @param {AbortSignal} signal */
+function checkAbort(signal) {
+  if (signal.aborted) throw new Error("Import cancelled.");
+}
+
+/** @param {string} address @param {boolean} [is_ingest] */
+function safeUrl(address, is_ingest = false) {
+  let url;
+  try {
+    url = new URL(address);
+  } catch {
+    throw new Error("Invalid destination address.");
+  }
+  if (
+    url.username ||
+    url.password ||
+    url.hash ||
+    (url.protocol !== "https:" && !(url.protocol === "http:" && LOCAL_HOSTS.has(url.hostname)))
+  ) {
+    throw new Error(
+      "Use an HTTPS address, or HTTP on localhost, without URL credentials or fragments."
+    );
+  }
+  if (is_ingest && !url.searchParams.get("key"))
+    throw new Error("The ingest address must contain its key.");
+  return url;
+}
+
+/**
+ * A manually linked signal works on the oldest supported Node release.
+ *
+ * @param {AbortSignal} parent @param {number} milliseconds
+ */
+function requestSignal(parent, milliseconds) {
+  const controller = new AbortController();
+  const abort = () => controller.abort();
+  if (parent.aborted) abort();
+  else parent.addEventListener("abort", abort, { once: true });
+  const timer = setTimeout(abort, milliseconds);
+  return {
+    signal: controller.signal,
+    dispose() {
+      clearTimeout(timer);
+      parent.removeEventListener("abort", abort);
+    }
+  };
+}
+
+/** @param {Response} response */
+async function smallJson(response) {
+  if (!response.body) throw new Error("Missing response.");
+  const reader = response.body.getReader();
+  const decoder = new TextDecoder("utf-8", { fatal: true });
+  let text = "";
+  let bytes = 0;
+  try {
+    for (;;) {
+      const { value, done } = await reader.read();
+      if (done) break;
+      bytes += value.byteLength;
+      if (bytes > 64 * 1024) throw new Error("Oversized response.");
+      text += decoder.decode(value, { stream: true });
+    }
+    return JSON.parse(text + decoder.decode());
+  } finally {
+    // Stream cleanup must not hold up the result or cancellation of the command.
+    void reader.cancel().catch(() => {});
+    reader.releaseLock();
+  }
+}
+
+/**
+ * @param {URL} url @param {RequestInit} options @param {AbortSignal} signal @param {typeof fetch}
+ *   request
+ */
+async function getJson(url, options, signal, request) {
+  const linked = requestSignal(signal, 8000);
+  try {
+    const response = await request(url, {
+      ...options,
+      method: "GET",
+      signal: linked.signal,
+      redirect: "error",
+      cache: "no-store",
+      credentials: "omit"
+    });
+    if (!response.ok) {
+      void response.body?.cancel().catch(() => {});
+      throw new Error("Refused response.");
+    }
+    return await smallJson(response);
+  } catch {
+    checkAbort(signal);
+    throw new Error(
+      "Could not verify the destination. Check credentials, IDs, and endpoint availability."
+    );
+  } finally {
+    linked.dispose();
+  }
+}
+
+/**
+ * @param {Record<string, unknown>} flags
+ * @param {AbortSignal} signal
+ * @param {typeof fetch} request
+ * @param {typeof loadConfig} read_config
+ */
+async function resolveDestination(flags, signal, request, read_config) {
+  const config = await read_config();
+  const project_id =
+    typeof flags["project-id"] === "string"
+      ? flags["project-id"]
+      : resolveSetting("project", { flags, config });
+  if (flags.project && flags["project-id"] && flags.project !== flags["project-id"]) {
+    throw new Error("--project and --project-id must name the same project.");
+  }
+  if (!project_id || project_id === "." || project_id === "..")
+    throw new Error("Upload requires --project (or a configured project).");
+  let url;
+  if (typeof flags["ingest-url"] === "string") {
+    if (flags["source-id"] || flags["ingest-key-id"])
+      throw new Error("Choose --ingest-url or explicit --source-id/--ingest-key-id, not both.");
+    url = safeUrl(flags["ingest-url"], true);
+  } else {
+    const source_id = flags["source-id"];
+    const ingest_key_id = flags["ingest-key-id"];
+    if (typeof source_id !== "string" || typeof ingest_key_id !== "string") {
+      throw new Error("Upload requires --ingest-url, or both --source-id and --ingest-key-id.");
+    }
+    if ([source_id, ingest_key_id].some((id) => id === "." || id === ".."))
+      throw new Error("Destination IDs cannot be dot segments.");
+    const api_url = safeUrl(resolveApiUrl({ flags, config }));
+    let credential;
+    try {
+      credential = resolveCredential({ flags, config });
+    } catch {
+      throw new Error(
+        "Sign in to the configured API or supply an API key to resolve this ingest key."
+      );
+    }
+    const team_id = resolveSetting("team", { flags, config }) ?? credential.login?.team_id ?? null;
+    const path =
+      "/v1/projects/" +
+      encodeURIComponent(project_id) +
+      "/sources/" +
+      encodeURIComponent(source_id) +
+      "/ingest_keys/" +
+      encodeURIComponent(ingest_key_id);
+
+    const payload = await getJson(
+      new URL(path, api_url),
+      {
+        headers: await buildRequestHeaders({ secret: credential.secret, team_id })
+      },
+      signal,
+      request
+    );
+
+    const key = payload?.data;
+    if (
+      key?.id !== ingest_key_id ||
+      key.source_id !== source_id ||
+      key.status !== "active" ||
+      typeof key.url !== "string"
+    ) {
+      throw new Error(
+        "The selected ingest key is unavailable, inactive, or belongs to another source."
+      );
+    }
+    url = safeUrl(key.url, true);
+    if (LOCAL_HOSTS.has(api_url.hostname) && !LOCAL_HOSTS.has(url.hostname)) {
+      throw new Error(
+        "A local API returned a remote ingest address. Pass an explicit --ingest-url for the intended destination."
+      );
+    }
+  }
+  url.searchParams.set("collection", String(flags.collection));
+  return { url, project_id };
+}
+
+/** @param {Record<string, unknown>} report */
+function renderHumanReport(report) {
+  if (!report.error && (report.action === "scan" || report.action === "validate")) {
+    const lines = [
+      String(report.format).toUpperCase() + ": " + report.row_count + " selected rows.",
+      report.is_complete
+        ? "Full validation complete."
+        : "Sample or discovery only; upload validation is still required."
+    ];
+    lines.push(
+      "File size: " + report.total_bytes + " bytes; scanned " + report.bytes_read + " bytes."
+    );
+    const xml = /** @type {any} */ (report.xml);
+    if (xml?.record_paths?.length)
+      lines.push(
+        "XML record paths: " +
+          xml.record_paths
+            .map((/** @type {{ path: string }} */ entry) => sanitizeText(entry.path))
+            .join(", ")
+      );
+    const fields = /** @type {import("./import.generated.js").FieldProfile[]} */ (
+      report.fields ?? []
+    );
+    if (fields.length) {
+      lines.push("Fields:");
+      for (const field of fields.slice(0, 20))
+        lines.push(
+          "  " +
+            sanitizeText(field.path) +
+            ": " +
+            field.types.join(" | ") +
+            "; present " +
+            field.present +
+            ", missing " +
+            field.missing
+        );
+      if (fields.length > 20) lines.push("  More fields are available with --json.");
+    }
+    const candidates = /** @type {import("./import.generated.js").TimestampCandidate[]} */ (
+      report.timestamp_candidates ?? []
+    );
+    if (candidates.length)
+      lines.push(
+        "Timestamp candidates: " + candidates.map((entry) => sanitizeText(entry.path)).join(", ")
+      );
+    for (const warning of /** @type {string[]} */ (report.warnings ?? []))
+      lines.push(sanitizeText(warning));
+    return lines.join("\n");
+  }
+  const affected = /** @type {any} */ (report.affected_batch);
+  return [
+    report.status === "complete"
+      ? "Import complete."
+      : sanitizeText(String(report.error ?? "Import stopped.")),
+    "Accepted rows: " + report.accepted_count + ".",
+    ...(report.status !== "complete" && Number(report.accepted_count) > 0
+      ? ["Accepted rows remain in the collection. Retrying this file may duplicate them."]
+      : []),
+    ...(affected
+      ? [
+          "Batch rows " +
+            affected.first_row +
+            "-" +
+            affected.last_row +
+            ": " +
+            affected.status +
+            ". No retry was attempted."
+        ]
+      : [])
+  ].join("\n");
+}
+
+/**
+ * @typedef {Object} ImportCommandContext
+ * @property {typeof fetch} [request]
+ * @property {typeof loadConfig} [read_config]
+ * @property {AbortSignal} [signal]
+ * @property {{ write(text: string): unknown }} [stderr]
+ * @property {string} [temporary_root]
+ */
+
+/**
+ * Dedicated import parsing keeps file formats and local flags separate from the API registry.
+ *
+ * @param {string[]} argv Complete argv, including the import command.
+ * @param {ImportCommandContext} [context]
+ * @returns {Promise<{ output: string; code: number }>}
+ */
+export async function runImportCommand(argv, context = {}) {
+  /** @type {Record<string, unknown>} */
+  let flags = {};
+  let action = "import";
+  let accepted_count = 0;
+  /** @type {number | null} */
+  let total_rows = null;
+  let outcome = "invalid_input";
+  /**
+   * @type {{
+   *   index: number;
+   *   first_row: number;
+   *   last_row: number;
+   *   status: string;
+   *   http_status?: number;
+   * } | null}
+   */
+  let affected_batch = null;
+  /** @type {Awaited<ReturnType<typeof openImportFile>> | undefined} */
+  let input;
+  const controller = new AbortController();
+  const signal = controller.signal;
+  const abort = () => controller.abort();
+  const external_signal = context.signal;
+  if (external_signal?.aborted) abort();
+  else external_signal?.addEventListener("abort", abort, { once: true });
+  process.on("SIGINT", abort);
+  const stderr = context.stderr ?? process.stderr;
+  const request = context.request ?? fetch;
+  let result;
+  try {
+    let positionals;
+    try {
+      ({ values: flags, positionals } = parseArgs({
+        args: argv,
+        options: IMPORT_PARSER_OPTIONS,
+        allowPositionals: true,
+        strict: false
+      }));
+    } catch {
+      throw new Error("Invalid import options. Run tglow import --help.");
+    }
+    for (const [name, value] of Object.entries(flags)) {
+      const option = IMPORT_PARSER_OPTIONS[name];
+      if (!option) throw new Error("Unknown import option. Run tglow import --help.");
+      if (option.type === "boolean") {
+        if (value === "true" || value === "false") flags[name] = value === "true";
+        else if (typeof value !== "boolean")
+          throw new Error("--" + name + " takes no value, or =true / =false.");
+      } else if (typeof value !== "string" || !value.length) {
+        throw new Error("--" + name + " requires a value.");
+      }
+    }
+    if (flags.help || positionals.length === 1) {
+      return {
+        output: shouldEmitJson(flags) ? JSON.stringify(describeImportForMachines(), null, 2) : HELP,
+        code: 0
+      };
+    }
+    if (positionals[0] !== "import") throw new Error("Expected the import command.");
+    const words = positionals.slice(1);
+    if (words[0] === "scan" || words[0] === "validate") {
+      action = words[0];
+      words.shift();
+    }
+    if (words.length !== 1)
+      throw new Error("Specify exactly one input file. Run tglow import --help.");
+    if (action !== "scan" && flags.full !== undefined)
+      throw new Error("--full only applies to import scan.");
+    if (action !== "import" && flags["validate-first"] !== undefined)
+      throw new Error("--validate-first only applies to uploading.");
+    if (!FORMATS.includes(String(flags.format ?? "auto")))
+      throw new Error("Unsupported --format. Use auto, xml, jsonl, json, csv, or tsv.");
+    const workers = parseWorkers(flags.workers);
+    if (
+      action !== "import" &&
+      ["ingest-url", "collection", "source-id", "ingest-key-id"].some(
+        (name) => flags[name] !== undefined
+      )
+    ) {
+      throw new Error("Destination options apply only to uploading, not scan or validate.");
+    }
+    if (action === "import" && (typeof flags.collection !== "string" || !flags.collection.trim()))
+      throw new Error("Upload requires an explicit --collection.");
+    checkAbort(signal);
+    let last_progress = 0;
+    /** @param {import("./import.generated.js").ImportProgress} progress */
+    const onprogress = ({ bytes_read, total_bytes, rows }) => {
+      if (Date.now() - last_progress < 1000) return;
+      last_progress = Date.now();
+      stderr.write("Reading: " + bytes_read + "/" + total_bytes + " bytes, " + rows + " rows.\n");
+    };
+
+    // Each scan and upload pass starts its own workers and reports what actually ran. Automatic
+    // sequential reads stay quiet; explicit counts always explain the outcome.
+    /** @param {import("./import.generated.js").ImportWorkerStatus} status */
+    const onworkers = (status) => {
+      let line;
+      if (status.fallback)
+        line = `XML workers stopped (${status.fallback}); continuing sequentially.`;
+      else if (status.actual)
+        line = `XML workers: ${status.actual}${status.reason ? ` (${status.reason})` : ""}.`;
+      else if (status.requested || (workers !== "auto" && workers > 0))
+        line = `XML workers: none, reading sequentially (${status.reason}).`;
+      else return;
+      stderr.write(sanitizeText(line) + "\n");
+    };
+
+    outcome = "file_error";
+    if (action === "import" && flags["validate-first"] === true) {
+      stderr.write("Making a private snapshot for a full check before upload.\n");
+      input = await snapshotImportFile(words[0], {
+        signal,
+        temporary_root: context.temporary_root
+      });
+    } else input = await openImportFile(words[0], { signal });
+    const source = input.source;
+    outcome = "validation_failed";
+    const detection = await detectImport(source, {
+      format: /** @type {any} */ (flags.format ?? "auto"),
+      signal
+    });
+
+    const options = Object.freeze({
+      format: detection.format,
+      ...(typeof flags["record-path"] === "string" ? { record_path: flags["record-path"] } : {}),
+      workers,
+      create_worker: () => createNodeImportWorker(IMPORT_WORKER_URL),
+      onworkers,
+      signal,
+      onprogress
+    });
+
+    const report = await scanImport(source, {
+      ...options,
+      is_full: action === "validate" || flags.full === true || flags["validate-first"] === true
+    });
+    if (action !== "import") {
+      if (action === "validate" && (!report.is_complete || report.is_sampled))
+        throw new Error("Validation did not cover the complete selected input.");
+      return {
+        output: shouldEmitJson(flags)
+          ? JSON.stringify({ action, ...report }, null, 2)
+          : renderHumanReport({ action, ...report }),
+        code: 0
+      };
+    }
+    if (flags["validate-first"] === true && (!report.is_complete || report.is_sampled))
+      throw new Error("The full file check did not finish.");
+    total_rows = report.is_complete && !report.is_sampled ? report.row_count : null;
+    outcome = "destination_failed";
+    const destination = await resolveDestination(
+      flags,
+      signal,
+      request,
+      context.read_config ?? loadConfig
+    );
+
+    const readiness = await getJson(destination.url, {}, signal, request);
+    if (readiness?.status !== "ready" || readiness.project_id !== destination.project_id)
+      throw new Error("The ingest endpoint is not ready for the expected project.");
+    outcome = "validation_failed";
+    for await (const batch of createImportBatches(readImportRows(source, options))) {
+      checkAbort(signal);
+      await input.assertUnchanged();
+      affected_batch = {
+        index: batch.index,
+        first_row: batch.rows[0].index,
+        last_row: batch.rows[batch.rows.length - 1].index,
+        status: "uncertain"
+      };
+      outcome = "uncertain_delivery";
+      const linked = requestSignal(signal, 30_000);
+      try {
+        const response = await request(destination.url, {
+          method: "POST",
+          headers: { "Content-Type": "application/x-ndjson" },
+          body: batch.body,
+          signal: linked.signal,
+          redirect: "error",
+          credentials: "omit"
+        });
+        if (response.status !== 202) {
+          affected_batch.status = IMPORT_REJECTED_STATUSES.includes(response.status)
+            ? "rejected"
+            : "uncertain";
+          outcome = affected_batch.status === "rejected" ? "rejected" : "uncertain_delivery";
+          affected_batch.http_status = response.status;
+          void response.body?.cancel().catch(() => {});
+          throw new Error(
+            "The server did not acknowledge the batch. Upload stopped without retrying."
+          );
+        }
+        accepted_count += batch.rows.length;
+        affected_batch = null;
+        void response.body?.cancel().catch(() => {});
+        if (Date.now() - last_progress >= 1000 || accepted_count === total_rows) {
+          last_progress = Date.now();
+          stderr.write(
+            "Accepted " +
+              accepted_count +
+              (total_rows === null ? "" : "/" + total_rows) +
+              " rows.\n"
+          );
+        }
+      } catch {
+        if (affected_batch?.status === "rejected")
+          throw new Error("The server rejected the batch. Upload stopped without retrying.");
+        throw new Error(
+          "No acknowledgement from the server. The affected batch may have arrived; upload stopped without retrying."
+        );
+      } finally {
+        linked.dispose();
+      }
+      outcome = "validation_failed";
+    }
+    checkAbort(signal);
+    await input.assertUnchanged();
+    if (!accepted_count)
+      throw new Error(
+        "This selection has no data rows. Check the XML record path or file contents."
+      );
+    if (total_rows !== null && accepted_count !== total_rows)
+      throw new Error("The imported row count differed from the full file check.");
+    total_rows = accepted_count;
+    result = {
+      action,
+      status: "complete",
+      outcome: "complete",
+      accepted_count,
+      total_rows,
+      affected_batch: null
+    };
+  } catch (error) {
+    if (input && !signal.aborted && !affected_batch) {
+      try {
+        await input.assertUnchanged();
+      } catch (source_error) {
+        if (
+          source_error &&
+          typeof source_error === "object" &&
+          "code" in source_error &&
+          source_error.code === "IMPORT_SOURCE_CHANGED"
+        )
+          error = source_error;
+      }
+    }
+    result = {
+      action,
+      status: signal.aborted ? "cancelled" : "failed",
+      outcome:
+        error &&
+        typeof error === "object" &&
+        "code" in error &&
+        error.code === "IMPORT_SOURCE_CHANGED"
+          ? "file_changed"
+          : signal.aborted && !affected_batch
+            ? "cancelled"
+            : outcome,
+      ...(error &&
+      typeof error === "object" &&
+      "index" in error &&
+      Number.isSafeInteger(error.index)
+        ? { failed_row: error.index }
+        : {}),
+      error: safeError(error, flags),
+      accepted_count,
+      total_rows,
+      affected_batch
+    };
+  } finally {
+    if (input) {
+      try {
+        await input.cleanup();
+      } catch {
+        const cleanup_error =
+          "Could not close the input or remove its private temporary snapshot. Check the tailglow-import directory in your temporary folder.";
+        result = { ...result, cleanup_error };
+        stderr.write("Warning: " + cleanup_error + "\n");
+      }
+    }
+    external_signal?.removeEventListener("abort", abort);
+    process.removeListener("SIGINT", abort);
+  }
+  return {
+    output: shouldEmitJson(flags) ? JSON.stringify(result, null, 2) : renderHumanReport(result),
+    code: result.status === "complete" ? 0 : signal.aborted ? 130 : 1
+  };
+}