@tailglow/cli 0.7.1 → 0.9.4

package/src/import-file.js

@@ -0,0 +1,181 @@
+import { constants } from "node:fs";
+import { chmod, mkdtemp, open, rm, stat } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { basename, join } from "node:path";
+
+/** @param {AbortSignal | undefined} signal */
+function checkAbort(signal) {
+  if (signal?.aborted) throw new Error("Import cancelled.");
+}
+
+/** @param {string} [message] */
+function changedInput(message = "The input file changed while it was being read.") {
+  const error = new Error(message);
+  return Object.assign(error, { code: "IMPORT_SOURCE_CHANGED" });
+}
+
+/**
+ * All passes read one opened descriptor at explicit offsets. The owner closes it after every
+ * consumer settles; ending a sampled read never closes or advances a shared stream position.
+ *
+ * @param {string} file_path
+ * @param {{ signal?: AbortSignal }} [options]
+ */
+export async function openImportFile(file_path, { signal } = {}) {
+  checkAbort(signal);
+  const handle = await open(file_path, constants.O_RDONLY | constants.O_NONBLOCK);
+  try {
+    const initial = await handle.stat({ bigint: true });
+    if (!initial.isFile()) throw new Error("Import input must be a regular file.");
+    const size = Number(initial.size);
+    if (!Number.isSafeInteger(size))
+      throw new Error("The input file is too large to address safely.");
+    let is_closed = false;
+    async function assertUnchanged() {
+      checkAbort(signal);
+      if (is_closed) throw new Error("The import file reader is closed.");
+      const current = await stat(file_path, { bigint: true }).catch(() => null);
+      const opened = await handle.stat({ bigint: true });
+      if (!current || !sameFile(initial, current) || !sameFile(initial, opened))
+        throw changedInput();
+    }
+    /** @param {number} start @param {number} end */
+    async function readRange(start, end) {
+      checkAbort(signal);
+      if (
+        !Number.isSafeInteger(start) ||
+        !Number.isSafeInteger(end) ||
+        start < 0 ||
+        end < start ||
+        end > size
+      )
+        throw new RangeError("The requested file range is outside the input.");
+      const bytes = new Uint8Array(end - start);
+      for (let offset = 0; offset < bytes.length; ) {
+        checkAbort(signal);
+        const { bytesRead: count } = await handle.read(
+          bytes,
+          offset,
+          bytes.length - offset,
+          start + offset
+        );
+        if (!count) throw changedInput();
+        offset += count;
+      }
+      return bytes;
+    }
+    /** @satisfies {import("./import.generated.js").ImportSource} */
+    const source = {
+      name: basename(file_path),
+      size,
+      async *open() {
+        await assertUnchanged();
+        for (let offset = 0; offset < size; ) {
+          const end = Math.min(size, offset + 64 * 1024);
+          yield await readRange(offset, end);
+          offset = end;
+        }
+        // Run before the row iterator returns, so its final partial batch cannot precede this check.
+        await assertUnchanged();
+      },
+      async read(start, end) {
+        await assertUnchanged();
+        const bytes = await readRange(start, end);
+        await assertUnchanged();
+        return bytes;
+      }
+    };
+    await assertUnchanged();
+    return {
+      source: Object.freeze(source),
+      assertUnchanged,
+      async cleanup() {
+        if (is_closed) return;
+        is_closed = true;
+        await handle.close();
+      }
+    };
+  } catch (error) {
+    await handle.close().catch(() => {});
+    throw error;
+  }
+}
+
+/** @param {import("node:fs").BigIntStats} first @param {import("node:fs").BigIntStats} second */
+function sameFile(first, second) {
+  return (
+    first.dev === second.dev &&
+    first.ino === second.ino &&
+    first.size === second.size &&
+    first.mtimeNs === second.mtimeNs &&
+    first.ctimeNs === second.ctimeNs
+  );
+}
+
+/**
+ * Copy through the opened descriptor and reject replacement or ordinary in-place mutation during
+ * copying. The private copy is the sole input to both subsequent passes.
+ *
+ * @param {string} file_path
+ * @param {{ signal?: AbortSignal; temporary_root?: string }} [options]
+ */
+export async function snapshotImportFile(file_path, { signal, temporary_root = tmpdir() } = {}) {
+  checkAbort(signal);
+  const directory = await mkdtemp(join(temporary_root, "tailglow-import-"));
+  let original;
+  let snapshot;
+  let is_complete = false;
+  try {
+    await chmod(directory, 0o700);
+    original = await open(file_path, constants.O_RDONLY | constants.O_NONBLOCK);
+    const before = await original.stat({ bigint: true });
+    if (!before.isFile()) throw new Error("Import input must be a regular file.");
+    const snapshot_path = join(directory, "input");
+    snapshot = await open(snapshot_path, "wx", 0o600);
+    await snapshot.chmod(0o600);
+    const buffer = Buffer.allocUnsafe(64 * 1024);
+    let copied_bytes = 0;
+    for (;;) {
+      checkAbort(signal);
+      const { bytesRead: bytes_read } = await original.read(buffer, 0, buffer.length, null);
+      if (!bytes_read) break;
+      copied_bytes += bytes_read;
+      if (BigInt(copied_bytes) > before.size)
+        throw changedInput("Input changed while making its private snapshot.");
+      await snapshot.writeFile(buffer.subarray(0, bytes_read));
+    }
+    checkAbort(signal);
+    const after = await original.stat({ bigint: true });
+    const current = await stat(file_path, { bigint: true });
+    if (
+      !sameFile(before, after) ||
+      !sameFile(before, current) ||
+      BigInt(copied_bytes) !== before.size
+    ) {
+      throw changedInput("Input changed while making its private snapshot.");
+    }
+    if ((await snapshot.stat({ bigint: true })).size !== before.size)
+      throw new Error("The private snapshot size differs from the input file.");
+    await snapshot.close();
+    snapshot = undefined;
+    await original.close();
+    original = undefined;
+    const input = await openImportFile(snapshot_path, { signal });
+    is_complete = true;
+    return {
+      source: Object.freeze({ ...input.source, name: basename(file_path) }),
+      assertUnchanged: input.assertUnchanged,
+      async cleanup() {
+        try {
+          await input.cleanup();
+        } finally {
+          await rm(directory, { recursive: true, force: true });
+        }
+      }
+    };
+  } finally {
+    await snapshot?.close().catch(() => {});
+    await original?.close().catch(() => {});
+    if (!is_complete) await rm(directory, { recursive: true, force: true });
+  }
+}

package/src/import-options.js

@@ -0,0 +1,23 @@
+/**
+ * Import flags are separate from generated API fields, but must consume their values at dispatch.
+ *
+ * @type {Record<string, { type: "string" | "boolean" }>}
+ */
+export const IMPORT_PARSER_OPTIONS = {
+  format: { type: "string" },
+  "record-path": { type: "string" },
+  "ingest-url": { type: "string" },
+  collection: { type: "string" },
+  "project-id": { type: "string" },
+  project: { type: "string" },
+  "source-id": { type: "string" },
+  "ingest-key-id": { type: "string" },
+  "api-key": { type: "string" },
+  "api-url": { type: "string" },
+  team: { type: "string" },
+  workers: { type: "string" },
+  full: { type: "boolean" },
+  "validate-first": { type: "boolean" },
+  json: { type: "boolean" },
+  help: { type: "boolean" }
+};