@getopencrawl/cli 0.0.0-stage → 0.1.0

package/src/uploads.mjs

@@ -0,0 +1,141 @@
+import fs from 'node:fs/promises';
+import { createReadStream, openAsBlob } from 'node:fs';
+import path from 'node:path';
+import crypto from 'node:crypto';
+import { allowedOptions, identifier, requestFailure, waitForJob, validateJob, withInterrupt, waitOptions, renderJob } from './jobs.mjs';
+import { displayLine } from './display.mjs';
+
+export const UPLOADS_HELP = `OpenCrawl · recover an upload receipt
+
+  opencrawl uploads show RECEIPT_ID [--json]
+  opencrawl uploads lookup REQUEST_KEY [--json]
+  opencrawl ingest FILE [--wait] [--timeout 300] [--request-key UUID]
+  opencrawl import FILE [--provider P] [--wait] [--timeout 300] [--request-key UUID]
+
+Every queued upload prints its request key before sending. Keep that key if the
+connection drops. Lookup is read-only and never resends a file. To explicitly
+retry a prepared/retryable receipt, rerun the original command with the same
+file, source, visibility, provider, and --request-key. Changed details are rejected.
+Accepted receipts return their existing job; receiving receipts are only checked.
+Expired receipts require an explicit new upload without the old request key.
+--wait follows acceptance and job completion for at most 1–3600 seconds (default
+300). Ctrl+C stops local waiting. No automatic retry, reupload, or cancellation.
+Upload options: --source ID --visibility private|workspace (default private)
+Global options: --account PROFILE_ID --server URL --config-dir DIRECTORY --json
+`;
+export const PROVIDERS = ['auto', 'chatgpt', 'claude', 'claude-code', 'codex', 'generic'];
+const STATES = new Set(['prepared', 'receiving', 'retryable', 'accepted', 'expired']);
+export function requestKey(value) {
+  if (typeof value !== 'string' || !/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(value)) throw new Error('--request-key must be the UUID printed before the original upload.');
+  return value.toLowerCase();
+}
+export function validateReceipt(value, { key, id } = {}) {
+  if (!value || typeof value.id !== 'string' || !/^[A-Za-z0-9_-]{1,200}$/.test(value.id) || !STATES.has(value.state) || id && value.id !== id || key && value.request_key !== key || !['upload', 'chat_import'].includes(value.operation)) throw new Error('Server returned an invalid upload receipt. No file was resent.');
+  requestKey(value.request_key);
+  if (value.state === 'accepted') validateJob(value.job);
+  return value;
+}
+export async function lookupReceipt(client, key, options = {}) {
+  key = requestKey(key);
+  return validateReceipt(await client.request(`/api/v1/uploads/receipts/by-key/${key}`, { maxResponseBytes: 512 * 1024, ...options }), { key });
+}
+export async function fileDescriptor(file, { chat = false, provider = 'auto', source, visibility = 'private' } = {}) {
+  if (!['private', 'workspace'].includes(visibility)) throw new Error('--visibility must be private or workspace.');
+  if (!PROVIDERS.includes(provider)) throw new Error('Choose a supported --provider from import --help.');
+  if (source !== undefined) identifier(source, 'source');
+  const before = await fs.stat(file); const filename = path.basename(file);
+  if (!before.isFile() || !before.size || before.size > 200 * 1024 * 1024) throw new Error('Upload a nonempty regular file of at most 200 MB.');
+  if (/[\\\u0000-\u001f\u007f]/u.test(filename)) throw new Error('Use a filename without backslashes or control characters.');
+  if (Array.from(filename).length > 200) throw new Error('The upload filename must be at most 200 characters.');
+  const hash = crypto.createHash('sha256');
+  let size = 0;
+  for await (const chunk of createReadStream(file)) {
+    size += chunk.length;
+    if (size > before.size) throw new Error('The file grew while its checksum was being read. Wait for it to finish writing, then explicitly import it again.');
+    hash.update(chunk);
+  }
+  const after = await fs.stat(file);
+  if (['dev', 'ino', 'size', 'mtimeMs', 'ctimeMs'].some(key => before[key] !== after[key])) throw new Error('The file changed while its checksum was being read. Wait for it to finish writing, then explicitly import it again.');
+  return { operation: chat ? 'chat_import' : 'upload', filename, size_bytes: before.size, sha256: hash.digest('hex'), source_id: source || null, visibility, ...(chat ? { provider } : {}) };
+}
+const snapshot = (key, receipt, error) => ({ request_key: key, receipt_id: receipt?.id || null, receipt: receipt || null,
+  job_id: receipt?.job?.id || null, job: receipt?.job || null, outcome: receipt?.state || 'unconfirmed', ...(error ? { error } : {}) });
+
+/** Sends file bytes at most once. A failed response is reconciled only by GET. */
+export async function submitUpload(file, client, options = {}, { onIntent = async () => {}, onReceipt = async () => {}, stderr = text => process.stderr.write(text), signal } = {}) {
+  const descriptor = await fileDescriptor(file, options);
+  const key = options['request-key'] === undefined ? crypto.randomUUID() : requestKey(options['request-key']);
+  await onIntent(key, descriptor);
+  stderr(`Upload request key: ${key}\n`);
+  let receipt;
+  const recover = async error => {
+    let confirmed = false;
+    try { receipt = await lookupReceipt(client, key, { timeout: 10000, signal }); confirmed = true; await onReceipt(receipt); }
+    catch { /* Acceptance remains unknown; keep the key rather than retransmitting. */ }
+    return { ...snapshot(key, receipt, requestFailure(error)), ...(!confirmed && receipt?.state !== 'accepted' ? { outcome: 'unconfirmed' } : {}) };
+  };
+  try {
+    // Idempotent prepare validates the entire descriptor even for an existing key.
+    receipt = validateReceipt(await client.request('/api/v1/uploads/prepare', { method: 'POST', body: { request_key: key, ...descriptor }, timeout: 30000, signal, maxResponseBytes: 512 * 1024 }), { key });
+    await onReceipt(receipt);
+  } catch (error) {
+    // A definite descriptor conflict must not be disguised as successful reuse.
+    if (Number.isInteger(error.status) && error.status >= 400 && error.status < 500) return snapshot(key, null, requestFailure(error));
+    return recover(error);
+  }
+  if (receipt.state === 'accepted' || receipt.state === 'receiving' || receipt.state === 'expired') return snapshot(key, receipt);
+  const route = `/api/v1/uploads/receipts/${encodeURIComponent(receipt.id)}/content`;
+  if (receipt.upload_url !== route) return snapshot(key, receipt, { code: 'invalid_upload_route', message: 'The receipt returned an unexpected upload route. No file was sent.' });
+  try {
+    const form = new FormData();
+    const type = /\.zip$/i.test(file) ? 'application/zip' : /\.json$/i.test(file) ? 'application/json' : /\.jsonl$/i.test(file) ? 'application/x-ndjson' : 'application/octet-stream';
+    form.set('file', await openAsBlob(file, { type }), descriptor.filename);
+    receipt = validateReceipt(await client.request(route, { method: 'POST', body: form, timeout: 300000, signal, maxResponseBytes: 512 * 1024 }), { key, id: receipt.id });
+    await onReceipt(receipt);
+    return snapshot(key, receipt);
+  } catch (error) { return recover(error); }
+}
+
+export async function followUpload(result, client, { timeout = 300, signal, now = () => performance.now(), sleep = ms => new Promise(resolve => setTimeout(resolve, ms)) } = {}) {
+  const deadline = now() + timeout * 1000;
+  while (!result.job_id && result.receipt?.state === 'receiving') {
+    if (signal?.aborted || now() >= deadline) return { ...result, wait: { outcome: signal?.aborted ? 'interrupted' : 'timeout' } };
+    const delay = Number.isInteger(result.receipt.retry_after_seconds) ? Math.max(1, Math.min(10, result.receipt.retry_after_seconds)) * 1000 : 2000;
+    const next = Math.min(deadline, now() + delay);
+    while (!signal?.aborted && now() < next) await sleep(Math.min(100, next - now()));
+    if (signal?.aborted || now() >= deadline) continue;
+    try { result = snapshot(result.request_key, await lookupReceipt(client, result.request_key, { timeout: Math.max(1, Math.min(30000, Math.ceil(deadline - now()))), signal })); }
+    catch (error) { return { ...result, wait: { outcome: signal?.aborted ? 'interrupted' : now() >= deadline ? 'timeout' : 'unavailable', error: requestFailure(error) } }; }
+  }
+  if (!result.job_id) return { ...result, wait: { outcome: signal?.aborted ? 'interrupted' : 'not_accepted' } };
+  const waited = await waitForJob(client, result.job_id, { timeout, signal, now, sleep, deadline, initial: result.job });
+  return { ...result, ...waited };
+}
+export function renderUpload(result) {
+  let text = `Upload request key: ${displayLine(result.request_key)}\n`;
+  if (result.receipt_id) text += `Receipt: ${displayLine(result.receipt_id)}\n`;
+  text += `Upload: ${displayLine(result.outcome)}\n`;
+  if (result.job) text += renderJob(result.job);
+  if (result.wait) text += `Wait: ${displayLine(result.wait.outcome)}. No job was cancelled or resubmitted.\n`;
+  if (result.receipt?.error?.message) text += `Receipt issue: ${displayLine(result.receipt.error.message)}\n`;
+  if (result.error || result.wait?.error) text += displayLine((result.wait?.error || result.error).message) + '\n';
+  if (result.outcome !== 'accepted') text += `Inspect with: opencrawl uploads lookup ${displayLine(result.request_key)}\n`;
+  if (['prepared', 'retryable'].includes(result.outcome)) text += 'Retry only explicitly with the original file and scope plus --request-key.\n';
+  if (result.outcome === 'receiving') text += 'The server is still receiving or reconciling this request. Lookup only; do not upload it again.\n';
+  if (result.outcome === 'expired') text += 'This receipt expired. Start a new upload explicitly if it is still needed.\n';
+  return text;
+}
+export function uploadExitCode(result, wait) { return wait ? result.wait?.outcome === 'completed' ? 0 : 1 : result.outcome === 'accepted' ? 0 : 1; }
+export async function finishUpload(result, options, client, { stdout = text => process.stdout.write(text) } = {}) {
+  if (options.wait) result = await withInterrupt(signal => followUpload(result, client, { timeout: waitOptions(options), signal }));
+  stdout(options.json ? JSON.stringify(result, null, 2) + '\n' : renderUpload(result));
+  return uploadExitCode(result, options.wait);
+}
+export async function uploadsCommand(rest, options, client, { stdout = text => process.stdout.write(text) } = {}) {
+  const [command, id, ...extra] = rest;
+  allowedOptions(options, [], 'uploads');
+  if (!['show', 'lookup'].includes(command) || extra.length) throw new Error('Choose uploads show RECEIPT_ID or uploads lookup REQUEST_KEY.');
+  const receipt = command === 'lookup' ? await lookupReceipt(client, id) : validateReceipt(await client.request(`/api/v1/uploads/receipts/${encodeURIComponent(identifier(id, 'receipt'))}`, { maxResponseBytes: 512 * 1024 }), { id });
+  stdout(options.json ? JSON.stringify(receipt, null, 2) + '\n' : renderUpload(snapshot(receipt.request_key, receipt)));
+  return 0;
+}