@substrio/cli 0.0.0-stage → 0.1.1

package/src/commands/import.mjs

@@ -0,0 +1,506 @@
+/**
+ * Substrio CLI: `substrio data import COLLECTION --file F`.
+ *
+ *   substrio data import COLLECTION --file FILE [--format ndjson|json|csv]
+ *       NDJSON (.ndjson, .jsonl), a JSON array, {"documents": [...]} or CSV
+ *       (- is stdin; --format is needed for stdin, and overrides the
+ *       extension for a file)
+ *   substrio data import COLLECTION --data TEXT     the same, given inline
+ *
+ *   --types a:string,b:number,c:date,d:boolean,e:json   CSV columns' types (default: string)
+ *   --empty-as null             an empty CSV field becomes null (default: the field is absent)
+ *   --batch-bytes N, --batch-docs N   a batch's size cap (bytes of JSON text, document count)
+ *   --concurrency N             batches in flight at once (default 4; 1 is today's order)
+ *   --checkpoint FILE           where progress is saved (default INPUT.substrio-import); only
+ *                               written with --upsert or --on-conflict skip (see --resume)
+ *   --resume                    continue from the checkpoint (needs a file, not stdin, and
+ *                               --upsert or --on-conflict skip: without an _id, a batch the
+ *                               server stored just before a crash would be sent again)
+ *   --upsert                    a document with _id already stored is replaced, not refused
+ *                               (data.import's own on_conflict=replace: one call per batch)
+ *   --on-conflict skip          a document whose _id is already stored is skipped, not refused
+ *                               (data.import's own on_conflict=skip: one call per batch)
+ *   --allow-failures            exit 0 even when some documents were refused
+ *
+ * The documents are read (src/documents.mjs, src/readers.mjs) a chunk at a
+ * time, as their own bytes, never parsed or re-serialized: a file of any
+ * size is never loaded whole, and no document pays a JSON.parse or
+ * JSON.stringify to pass through. They are cut into batches (by byte size
+ * and document count) and sent through the registry's data.import, as
+ * NDJSON or a JSON array (its own `format` query option) with `on_conflict`
+ * when --upsert or --on-conflict skip is given, several batches in flight at
+ * once; a 429 or 503 backs off and retries, never dropping a batch (each
+ * retry, and the wait it is taking, is reported as it happens); any other
+ * failure stops the run. A document is refused alone and the others are
+ * written: every refused one, a line, element or row that is not usable
+ * included, is reported with its 1-based position, its code and its reason
+ * (a schema or CSV-type refusal also each rule it broke, as `violations`),
+ * and the run exits non-zero with ExitCode.PARTIAL (8) unless
+ * --allow-failures, apart from every failure of the whole run, so a script
+ * can tell "some rows were refused, the rest stored" from a run that
+ * failed. A document skipped or replaced under --on-conflict is not a
+ * failure. A call that fails as a whole (the server refuses the batch, or is
+ * gone) stops the run: what was read up to there is reported, the documents
+ * after it were not read, and the run exits with the server's typed exit
+ * code.
+ *
+ * Under --json, one JSON line is written per progress tick (at most one a
+ * second), one per refused document and one per retry, as they happen, and
+ * exactly one final line (`done` or `stopped`): nothing else reaches stdout.
+ */
+
+import { chmodSync, readFileSync, renameSync, statSync, writeFileSync } from 'node:fs';
+
+import { CliError, ErrorCode, ExitCode, emitLine, fail, globals } from '../cli.mjs';
+import { errorText } from '../api.mjs';
+import { DocumentForm, formOfFile, readDocuments } from '../documents.mjs';
+import { findOperation } from '../ops.mjs';
+import { CsvType } from '../readers.mjs';
+import { FlagKind, parseFlags, wholeNumber } from '../local/args.mjs';
+import { OpError, callOperation, loadOperations } from './op.mjs';
+
+const C = {
+    reset: '\x1b[0m', bold: '\x1b[1m', dim: '\x1b[2m', red: '\x1b[91m', green: '\x1b[92m', yellow: '\x1b[93m',
+};
+
+/** The documents of one call: enough for a request of ordinary size, the common case. */
+export const BATCH_DOCUMENTS = 1000;
+/**
+ * The JSON text of one call, reached before its document count: large
+ * documents go in fewer per call. A single larger document goes alone. Kept
+ * below the server's own request-body ceiling (substrio_server.h:63, 16 MiB).
+ */
+export const BATCH_BYTES = 4 * 1024 * 1024;
+/** Batches sent at once by default; 1 restores the strict, in-order behaviour of a single connection. */
+export const CONCURRENCY = 4;
+/** The first retry's delay (SUBSTRIO_IMPORT_RETRY_MS sets another); it doubles up to RETRY_MAX_MS, each with up to 50% jitter. */
+export const RETRY_BASE_MS = wholeNumberOr(process.env.SUBSTRIO_IMPORT_RETRY_MS, 100);
+export const RETRY_MAX_MS = 10_000;
+
+const USAGE = 'substrio data import COLLECTION --file FILE [--format ndjson|json|csv] (- is stdin) | --data TEXT';
+
+export const IMPORT_FLAGS = Object.freeze({
+    '--file': FlagKind.VALUE, '--data': FlagKind.VALUE, '--format': FlagKind.VALUE,
+    '--types': FlagKind.VALUE, '--empty-as': FlagKind.VALUE,
+    '--batch-bytes': FlagKind.VALUE, '--batch-docs': FlagKind.VALUE, '--concurrency': FlagKind.VALUE,
+    '--checkpoint': FlagKind.VALUE, '--resume': FlagKind.SWITCH,
+    '--upsert': FlagKind.SWITCH, '--on-conflict': FlagKind.VALUE, '--allow-failures': FlagKind.SWITCH,
+});
+
+/** The codes this CLI gives a document it refused itself. */
+export const RejectCode = Object.freeze({
+    INVALID_DOCUMENT: 'INVALID_DOCUMENT', // a line, element or row that is not usable
+    UNACCOUNTED: 'UNACCOUNTED',           // the server answered for fewer documents than the batch held
+});
+
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+
+// The wire-batch separators (Buffer.from once, never re-allocated per document or per batch).
+const NEWLINE = Buffer.from('\n');
+const COMMA = Buffer.from(',');
+const OPEN_BRACKET = Buffer.from('[');
+const CLOSE_BRACKET = Buffer.from(']');
+
+function wholeNumberOr(text, fallback) {
+    const n = Number(text);
+    return text !== undefined && text !== '' && Number.isInteger(n) && n >= 0 ? n : fallback;
+}
+
+// A delay for the (attempt)th retry of a 429/503: doubling from RETRY_BASE_MS, capped at RETRY_MAX_MS, with jitter.
+function backoffMs(attempt) {
+    const base = Math.min(RETRY_MAX_MS, RETRY_BASE_MS * 2 ** (attempt - 1));
+    return base * (0.5 + Math.random() * 0.5);
+}
+
+// An answer that does not say whether the batch was stored: no answer at all (the connection
+// dropped), or a gateway between the CLI and the server that gave up on it (502, 504, and a
+// proxy's own 520 to 524, such as a CDN in front of the server).
+const UNKNOWN_FATE = new Set([502, 504, 520, 521, 522, 523, 524]);
+/** Sends of one batch whose fate stays unknown before the import stops (a server that is gone stays gone). */
+export const RETRY_UNKNOWN_ATTEMPTS = 8;
+
+/**
+ * Sends `items` through `send(items, again)`, retrying a 429 or 503 answer with backoff
+ * (never giving up: the batch is never dropped), the server's own
+ * Retry-After wait (`result.retryAfterMs`, callOperation's) when it sent
+ * one, else `backoffMs`. A batch whose fate is unknown (UNKNOWN_FATE, or a
+ * NETWORK_ERROR thrown by `send`) is sent again up to RETRY_UNKNOWN_ATTEMPTS
+ * times, with `again` true from then on: the caller makes a plain insert
+ * on_conflict skip for it, so what landed the first time is skipped, not
+ * refused (the stored count, not the report, is what a caller checks).
+ * `onRetry({status, attempt, wait_ms})` fires before each wait (status 0:
+ * no answer), so a retry is never silent.
+ */
+async function sendWithRetry(items, send, onRetry) {
+    let again = false;
+    for (let attempt = 1; ; attempt++) {
+        let result;
+        try {
+            result = await send(items, again);
+        } catch (err) {
+            if (!(err instanceof CliError) || err.code !== ErrorCode.NETWORK_ERROR || attempt >= RETRY_UNKNOWN_ATTEMPTS) throw err;
+            result = { ok: false, status: 0 };
+        }
+        if (result.ok) return result;
+        const unknown = result.status === 0 || UNKNOWN_FATE.has(result.status);
+        if (unknown ? attempt >= RETRY_UNKNOWN_ATTEMPTS : (result.status !== 429 && result.status !== 503)) return result;
+        again ||= unknown;
+        const waitMs = result.retryAfterMs ?? backoffMs(attempt);
+        onRetry?.({ status: result.status, attempt, wait_ms: Math.round(waitMs) });
+        await sleep(waitMs);
+    }
+}
+
+/**
+ * Sends `documents` (an async iterable of {position, raw, endOffset} or
+ * {position, error, endOffset, violations?}; `raw` is a Buffer of that
+ * document's own JSON text) in batches through `send(items)` (the items of
+ * the batch, callOperation's {ok, status, data, request, retryAfterMs}
+ * shape back), up to `concurrency` at once. Returns the report: {total,
+ * inserted, skipped, replaced, rejectedCount, rejected?, stopped: null |
+ * {position, error}}. `rejected` ({position, code, message, violations?})
+ * is kept only when `onReject` is not given (a bounded caller, library use
+ * or a test): with it, every rejection is handed to `onReject(entry, report)`
+ * the moment it is known (read time or batch time) and none are kept, so a
+ * run with no bound on failures does not grow one unbounded array. `onBatch
+ * (endOffset)` fires, in input order, once every batch up to and including
+ * it is done and written (a batch the server refused as a whole never fires
+ * it, so the checkpoint never passes data that was not stored). `onRetry`
+ * is sendWithRetry's.
+ */
+export async function importDocuments(documents, send, {
+    batchDocuments = BATCH_DOCUMENTS, batchBytes = BATCH_BYTES, concurrency = 1, onReject = null, onBatch = null, onRetry = null,
+} = {}) {
+    const report = { total: 0, inserted: 0, skipped: 0, replaced: 0, rejectedCount: 0, stopped: null };
+    if (onReject === null) report.rejected = [];
+    const reject = (entry) => {
+        report.rejectedCount++;
+        if (onReject === null) report.rejected.push(entry);
+        else onReject(entry, report);
+    };
+
+    let batch = [];
+    let bytes = 0;
+    let seq = 0;
+    let nextSeq = 0;
+    const finished = new Map(); // seq -> {endOffset, ok}
+    const inFlight = new Set();
+    let fatal = null;
+
+    const advance = () => {
+        while (finished.has(nextSeq)) {
+            const entry = finished.get(nextSeq);
+            finished.delete(nextSeq);
+            nextSeq++;
+            if (!entry.ok) break; // a batch the server refused as a whole: nothing past it is acknowledged
+            onBatch?.(entry.endOffset, report);
+        }
+    };
+
+    const runBatch = async (mySeq, items) => {
+        const result = await sendWithRetry(items, send, onRetry);
+        if (fatal === null) {
+            if (!result.ok) {
+                fatal = { position: items[0].position, error: CliError.fromResponse(`${result.request.method} ${result.request.path}: ${errorText(result.data, result.status)}`, result.data, result.status) };
+            } else {
+                const { inserted = 0, skipped = 0, replaced = 0, errors = [] } = result.data ?? {};
+                report.inserted += inserted;
+                report.skipped += skipped;
+                report.replaced += replaced;
+                for (const e of errors) {
+                    const refused = { position: items[e.index]?.position ?? null, code: e.code, message: e.message };
+                    if (Array.isArray(e.violations)) refused.violations = e.violations;
+                    reject(refused);
+                }
+                if (inserted + skipped + replaced + errors.length !== items.length) {
+                    reject({
+                        position: items[0].position, code: RejectCode.UNACCOUNTED,
+                        message: `the server answered for ${inserted + skipped + replaced + errors.length} of the ${items.length} documents of the batch starting here`,
+                    });
+                }
+            }
+        }
+        finished.set(mySeq, { endOffset: items[items.length - 1].endOffset, ok: result.ok });
+        advance();
+    };
+
+    const dispatch = (items) => {
+        const mySeq = seq++;
+        const p = runBatch(mySeq, items).finally(() => inFlight.delete(p));
+        inFlight.add(p);
+    };
+
+    let unread = null;
+    const iterator = documents[Symbol.asyncIterator]();
+    try {
+        while (fatal === null) {
+            const { value: item, done } = await iterator.next();
+            if (done) break;
+            report.total++;
+            if (item.error !== undefined) {
+                reject({ position: item.position, code: RejectCode.INVALID_DOCUMENT, message: item.error, ...(item.violations ? { violations: item.violations } : {}) });
+                continue;
+            }
+            batch.push(item);
+            bytes += item.raw.length;
+            if (batch.length >= batchDocuments || bytes >= batchBytes) {
+                dispatch(batch);
+                batch = [];
+                bytes = 0;
+                if (inFlight.size >= concurrency) await Promise.race(inFlight);
+            }
+        }
+    } catch (err) {
+        if (!(err instanceof CliError)) throw err;
+        unread = { position: report.total + 1, error: err };
+    }
+    if (fatal === null && batch.length) dispatch(batch);
+    await Promise.all(inFlight);
+    report.stopped = fatal ?? unread;
+    return report;
+}
+
+// One rejection, as the lines printed for it (human mode): the moment it is known (onReject),
+// never kept to be listed at the end (report.rejectedCount is the bounded count that is kept).
+function violationLines(entry) {
+    const lines = [`  ${C.red}✗${C.reset} document ${entry.position ?? '?'}: ${entry.code ? `${entry.code}: ` : ''}${entry.message}`];
+    for (const v of entry.violations ?? []) lines.push(`      ${C.dim}${v.field}: ${v.message}${C.reset}`);
+    return lines;
+}
+
+// The summary line and, when the run stopped, why: never a list of rejections (streamed live instead, as
+// they happen, by onReject; report.rejectedCount is the count, not kept as an array under --json either).
+function describe(report, collection) {
+    const lines = [];
+    const ok = report.rejectedCount === 0 && report.stopped === null;
+    const extras = [
+        report.rejectedCount ? `${report.rejectedCount} rejected` : '',
+        report.skipped ? `${report.skipped} skipped` : '',
+        report.replaced ? `${report.replaced} replaced` : '',
+    ].filter(Boolean).join(', ');
+    lines.push(`${ok ? C.green : C.yellow}${ok ? '✓' : '⚠'}${C.reset} Imported ${C.bold}${report.inserted}${C.reset} of ${report.total} documents into ${C.bold}${collection}${C.reset}` +
+        `${extras ? `; ${extras}` : ''}`);
+    if (report.stopped !== null) {
+        lines.push(`  ${C.red}✗${C.reset} stopped at document ${report.stopped.position}: ${report.stopped.error.message}`);
+        lines.push(`  ${C.dim}document ${report.stopped.position} and everything after it were not imported${C.reset}`);
+    }
+    return lines;
+}
+
+// --types a:string,b:number,...: a Map field -> CsvType, or a typed usage error.
+function parseTypes(text) {
+    const allowed = new Set(Object.values(CsvType));
+    const map = new Map();
+    for (const pair of text.split(',')) {
+        const at = pair.indexOf(':');
+        if (at <= 0) throw new CliError(`--types: "${pair}" must be field:type`, { code: ErrorCode.INVALID_ARGUMENT });
+        const type = pair.slice(at + 1).trim();
+        if (!allowed.has(type)) throw new CliError(`--types: "${type}" is not a type (string, number, date, boolean or json)`, { code: ErrorCode.INVALID_ARGUMENT });
+        map.set(pair.slice(0, at).trim(), type);
+    }
+    return map;
+}
+
+// The checkpoint file beside `path` (unless --checkpoint names one): never for stdin unless given explicitly.
+function defaultCheckpointPath(path) {
+    return path === '-' ? null : `${path}.substrio-import`;
+}
+
+// What must stay the same between a run and the --resume of it.
+function checkpointOptionsOf({ format, typesText, upsert, onConflict, collection }) {
+    return { format, types: typesText ?? null, upsert, onConflict, collection };
+}
+
+function readCheckpoint(path) {
+    try {
+        return JSON.parse(readFileSync(path, 'utf8'));
+    } catch (err) {
+        throw new CliError(`--resume: ${path}: ${err.code === 'ENOENT' ? 'no checkpoint there' : err.message}`, { code: ErrorCode.FILE_NOT_FOUND });
+    }
+}
+
+// Written to a temp file beside `path` then renamed, so it is never torn; owner-only (0600).
+function writeCheckpoint(path, data) {
+    const tmp = `${path}.tmp-${process.pid}`;
+    writeFileSync(tmp, JSON.stringify(data), { mode: 0o600 });
+    chmodSync(tmp, 0o600);
+    renameSync(tmp, path);
+}
+
+/** `substrio data import`: `resolved` is resolveGroup's {positional, rest}. */
+export async function cmdImport(api, { positional, rest }) {
+    let flags;
+    let words;
+    try {
+        ({ flags, positional: words } = parseFlags(rest, IMPORT_FLAGS));
+    } catch (err) {
+        return fail(err, USAGE);
+    }
+    const [collection, ...extra] = positional;
+    if (collection === undefined || extra.length || words.length) fail('data import takes one collection and --file or --data', USAGE);
+    if (flags.has('--file') === flags.has('--data')) fail('give the documents as --file FILE or --data TEXT, one of them', USAGE);
+    const source = flags.has('--file') ? { path: flags.get('--file') } : { text: flags.get('--data') };
+    const hasFile = source.path !== undefined; // --data (inline text) has no file: not seekable, no default checkpoint
+    const isStdin = source.path === '-';
+
+    let format;
+    try {
+        if (flags.has('--format')) {
+            const word = flags.get('--format');
+            if (!Object.values(DocumentForm).includes(word) || word === DocumentForm.AUTO) {
+                throw new CliError(`--format must be ndjson, json or csv`, { code: ErrorCode.INVALID_ARGUMENT });
+            }
+            format = word;
+        } else if (hasFile && !isStdin) {
+            format = formOfFile(source.path) === DocumentForm.AUTO ? undefined : formOfFile(source.path);
+        }
+        const typesText = flags.get('--types');
+        const types = typesText !== undefined ? parseTypes(typesText) : null;
+        const emptyAsText = flags.get('--empty-as');
+        if (emptyAsText !== undefined && emptyAsText !== 'null') throw new CliError('--empty-as takes "null" (the only value it accepts today)', { code: ErrorCode.INVALID_ARGUMENT });
+        const emptyAs = emptyAsText === 'null' ? null : undefined;
+
+        const batchDocuments = wholeNumber(flags, '--batch-docs', 1, 1_000_000, BATCH_DOCUMENTS);
+        const batchBytes = wholeNumber(flags, '--batch-bytes', 1, 16 * 1024 * 1024, BATCH_BYTES);
+        const concurrency = wholeNumber(flags, '--concurrency', 1, 64, CONCURRENCY);
+        const upsert = flags.has('--upsert');
+        const onConflict = flags.get('--on-conflict') ?? null;
+        if (onConflict !== null && onConflict !== 'skip') throw new CliError('--on-conflict takes "skip" (the only value it accepts today)', { code: ErrorCode.INVALID_ARGUMENT });
+        const resume = flags.has('--resume');
+        const allowFailures = flags.has('--allow-failures');
+
+        // A batch the server stored just before a crash, and before the checkpoint past it was
+        // written, is sent again on --resume: only an id (--upsert or --on-conflict skip) makes
+        // that harmless, so without one no checkpoint is written at all, and --resume is refused.
+        const canResume = upsert || onConflict === 'skip';
+        if (resume && !hasFile) throw new CliError('--resume needs a file (--file FILE), not inline --data or stdin: it cannot be sought back to', { code: ErrorCode.INVALID_ARGUMENT });
+        if (resume && isStdin) throw new CliError('--resume needs a file (--file), not stdin: it cannot be sought back to', { code: ErrorCode.INVALID_ARGUMENT });
+        if (resume && !canResume) {
+            throw new CliError('--resume needs --upsert or --on-conflict skip: a batch the server stored just before a crash is sent again, and only an id makes that harmless', { code: ErrorCode.INVALID_ARGUMENT });
+        }
+        const checkpointPath = canResume ? (flags.get('--checkpoint') ?? (hasFile && !isStdin ? defaultCheckpointPath(source.path) : null)) : null;
+
+        let resumeOffset = 0;
+        let stat = null;
+        if (hasFile && !isStdin) {
+            try { stat = statSync(source.path); } catch (err) { throw new CliError(`${source.path}: ${err.message}`, { code: ErrorCode.FILE_NOT_FOUND }); }
+        }
+        if (resume) {
+            if (checkpointPath === null) throw new CliError('--resume needs --checkpoint (or a file input, for its default)', { code: ErrorCode.INVALID_ARGUMENT });
+            const saved = readCheckpoint(checkpointPath);
+            if (saved.input.size !== stat.size || saved.input.mtimeMs !== stat.mtimeMs) {
+                throw new CliError(`${source.path} changed since the checkpoint was made (size or modified time differs): --resume refused`, { code: ErrorCode.INVALID_ARGUMENT });
+            }
+            const wanted = checkpointOptionsOf({ format: format ?? null, typesText, upsert, onConflict, collection });
+            if (JSON.stringify(wanted) !== JSON.stringify(saved.options)) {
+                throw new CliError('--resume: --format, --types, --upsert or --on-conflict differs from the checkpoint; run with the same options as before', { code: ErrorCode.INVALID_ARGUMENT });
+            }
+            resumeOffset = saved.offset;
+        }
+
+        let ops;
+        try {
+            ops = await loadOperations(api);
+        } catch (err) {
+            return fail(err);
+        }
+        let routes;
+        try {
+            routes = findOperation(ops, 'data.import');
+        } catch (err) {
+            return fail(err);
+        }
+        const supportsQuery = (name) => routes.some((r) => r.query.includes(name));
+        if (!supportsQuery('format')) {
+            return fail(new CliError("this server's data.import has no format option (needed to send the input as its own bytes, unparsed): the CLI needs a newer server", { code: ErrorCode.INVALID_ARGUMENT }));
+        }
+        const conflictMode = upsert ? 'replace' : (onConflict === 'skip' ? 'skip' : null);
+        if (conflictMode !== null && !supportsQuery('on_conflict')) {
+            return fail(new CliError(`--${upsert ? 'upsert' : 'on-conflict skip'}: this server's data.import has no on_conflict option`, { code: ErrorCode.INVALID_ARGUMENT }));
+        }
+
+        // The wire format: a JSON array (bracket/comma-joined) only for --format json (the
+        // jsonArrayDocuments reader); every other source (NDJSON, CSV, the AUTO fallback) hands
+        // each document its own JSON text already, newline-joined, and the server reads it the
+        // same way NDJSON always is. One Buffer.concat per batch, never per document.
+        const wireFormat = format === DocumentForm.JSON ? 'json' : 'ndjson';
+        const contentType = wireFormat === 'json' ? 'application/json' : 'application/x-ndjson';
+        const joinBatch = wireFormat === 'json'
+            ? (items) => { const parts = [OPEN_BRACKET]; for (let k = 0; k < items.length; k++) { if (k > 0) parts.push(COMMA); parts.push(items[k].raw); } parts.push(CLOSE_BRACKET); return Buffer.concat(parts); }
+            : (items) => { const parts = []; for (let k = 0; k < items.length; k++) { if (k > 0) parts.push(NEWLINE); parts.push(items[k].raw); } return Buffer.concat(parts); };
+        // `again`: a resend of a batch whose fate was unknown (sendWithRetry); a plain insert
+        // skips what landed the first time instead of refusing it.
+        const canSkip = supportsQuery('on_conflict');
+        const send = async (items, again = false) => {
+            const body = joinBatch(items);
+            const mode = conflictMode ?? (again && canSkip ? 'skip' : null);
+            try {
+                return await callOperation(api, ops, 'data.import', {
+                    collection, format: wireFormat, ...(mode !== null ? { on_conflict: mode } : {}),
+                }, null, { body, headers: { 'Content-Type': contentType } });
+            } catch (err) {
+                if (err instanceof OpError) throw new CliError(err.message, { code: ErrorCode.INVALID_ARGUMENT });
+                throw err;
+            }
+        };
+
+        const totalBytes = stat?.size ?? null;
+        let lastTick = 0;
+        const tick = (live, extra2 = {}) => {
+            const now = Date.now();
+            if (now - lastTick < 1000) return;
+            lastTick = now;
+            const elapsedS = Math.max(0.001, (now - started) / 1000);
+            emitLine({
+                event: 'progress', imported: live.inserted, skipped: live.skipped, replaced: live.replaced, failed: live.rejectedCount,
+                offset: lastOffset, total_bytes: totalBytes, docs_per_s: Math.round((live.inserted / elapsedS) * 10) / 10, ...extra2,
+            }, () => { if (process.stdout.isTTY) process.stdout.write(`\rImported ${live.inserted}, ${live.rejectedCount} failed${totalBytes ? ` (${Math.round((lastOffset / totalBytes) * 100)}%)` : ''}...`); });
+        };
+
+        const started = Date.now();
+        let lastOffset = resumeOffset;
+        let report;
+        const nextCommand = () => `substrio data import ${collection} --file ${source.path} --checkpoint ${checkpointPath} --resume${upsert ? ' --upsert' : ''}${onConflict ? ` --on-conflict ${onConflict}` : ''}`;
+        try {
+            report = await importDocuments(readDocuments(source, { form: format, types, emptyAs, resumeOffset }), send, {
+                batchDocuments, batchBytes, concurrency,
+                onReject: (entry, live) => { emitLine({ event: 'violation', ...entry }, () => { for (const line of violationLines(entry)) console.log(line); }); tick(live); },
+                onRetry: ({ status, attempt, wait_ms }) => {
+                    emitLine({ event: 'retry', status, attempt, wait_ms }, () => console.log(`  ${C.dim}retry ${attempt}: HTTP ${status}, waiting ${wait_ms}ms${C.reset}`));
+                },
+                onBatch: (endOffset, live) => {
+                    lastOffset = endOffset;
+                    if (checkpointPath !== null) {
+                        writeCheckpoint(checkpointPath, {
+                            offset: endOffset, imported: live.inserted, skipped: live.skipped, replaced: live.replaced, failed: live.rejectedCount,
+                            input: { size: stat.size, mtimeMs: stat.mtimeMs }, collection,
+                            options: checkpointOptionsOf({ format: format ?? null, typesText, upsert, onConflict, collection }),
+                        });
+                    }
+                    tick(live);
+                },
+            });
+        } catch (err) {
+            return fail(err);
+        }
+
+        const elapsedMs = Date.now() - started;
+        if (report.stopped !== null) {
+            if (globals.json) {
+                emitLine({ event: 'stopped', error: { ...report.stopped.error.toJSON(), next: nextCommand() } }, () => {});
+            } else {
+                for (const line of describe(report, collection)) console.log(line);
+            }
+            return fail(report.stopped.error);
+        }
+        if (globals.json) {
+            emitLine({ event: 'done', imported: report.inserted, skipped: report.skipped, replaced: report.replaced, failed: report.rejectedCount, elapsed_ms: elapsedMs, checkpoint: checkpointPath }, () => {});
+        } else {
+            if (process.stdout.isTTY) process.stdout.write('\r\n');
+            for (const line of describe(report, collection)) console.log(line);
+        }
+        if (report.rejectedCount > 0 && !allowFailures) process.exitCode = ExitCode.PARTIAL;
+    } catch (err) {
+        return fail(err);
+    }
+}