@getopencrawl/cli 0.1.0 → 0.1.1

package/src/upload-commands.test.mjs

@@ -0,0 +1,140 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import http from 'node:http';
+import fs from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+import crypto from 'node:crypto';
+import { spawn } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
+
+const CLI = fileURLToPath(new URL('../bin/opencrawl.mjs', import.meta.url));
+const jobId = 'job_' + 'a'.repeat(32);
+async function fixture(t) {
+  const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'opencrawl-commands-'));
+  const receipts = new Map(), calls = [], state = { jobStatus: 'completed', losePrepare: false, loseContent: false, failLookup: 0, holdJob: false };
+  const server = http.createServer(async (req, res) => {
+    const chunks = []; for await (const chunk of req) chunks.push(chunk);
+    const raw = Buffer.concat(chunks), body = req.headers['content-type']?.startsWith('application/json') ? JSON.parse(raw) : raw;
+    calls.push({ path: req.url, method: req.method, body });
+    const send = (status, value) => { res.writeHead(status, { 'Content-Type': 'application/json' }); res.end(JSON.stringify(value)); };
+    const job = () => ({ id: jobId, kind: 'upload', status: state.jobStatus, terminal: ['completed','failed','cancelled','discarded'].includes(state.jobStatus), indexed_documents: state.jobStatus === 'completed' ? 1 : 0, filename: 'test\u001b[2J.pdf', documents: [] });
+    if (req.url === '/api/auth/me') return send(200, { user: { id: 'usr_test' }, workspace: { id: 'ws_test' } });
+    if (req.url === '/api/v1/uploads/prepare') {
+      let receipt = receipts.get(body.request_key);
+      if (receipt && JSON.stringify(receipt.descriptor) !== JSON.stringify(body)) return send(409, { detail: { code: 'descriptor_mismatch', message: 'raw body must not escape' } });
+      if (!receipt) { receipt = { id: 'receipt_' + body.request_key.replaceAll('-', ''), request_key: body.request_key, state: 'prepared', operation: body.operation, job: null, descriptor: body }; receipt.upload_url = `/api/v1/uploads/receipts/${receipt.id}/content`; receipts.set(body.request_key, receipt); }
+      if (receipt.state === 'accepted') receipt.job = job();
+      if (state.losePrepare) { state.losePrepare = false; res.destroy(); return; }
+      return send(200, receipt);
+    }
+    if (req.url.endsWith('/content')) {
+      const receipt = [...receipts.values()].find(item => item.upload_url === req.url);
+      assert(receipt); assert(raw.includes(Buffer.from('synthetic fixture'))); receipt.state = 'accepted'; receipt.job = job();
+      if (state.loseContent) { state.loseContent = false; res.destroy(); return; }
+      return send(200, receipt);
+    }
+    if (req.url.startsWith('/api/v1/uploads/receipts/by-key/')) {
+      if (state.failLookup-- > 0) { res.destroy(); return; }
+      const receipt = receipts.get(req.url.split('/').at(-1)); if (!receipt) return send(404, { detail: 'Unavailable' });
+      if (receipt.state === 'accepted') receipt.job = job(); return send(200, receipt);
+    }
+    if (req.url === `/api/v1/jobs/${jobId}`) { if (state.holdJob) return; return send(200, job()); }
+    if (req.url === '/api/v1/search') return send(200, { results: [{ title: 'bad\u001b[2J', snippet: 'keep\nline\u001b]52;x\u0007', url: 'a\rb' }] });
+    if (req.url === '/api/v1/documents') return send(200, { id: 'doc_' + 'c'.repeat(32), title: 'synchronous' });
+    return send(404, { detail: 'Unknown route' });
+  });
+  await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
+  t.after(async () => { server.closeAllConnections(); await new Promise(resolve => server.close(resolve)); await fs.rm(dir, { recursive: true, force: true }); });
+  const env = { PATH: process.env.PATH, HOME: dir, OPENCRAWL_CONFIG_DIR: path.join(dir, 'config'), OPENCRAWL_API_KEY: 'oc_synthetic_test_only', OPENCRAWL_URL: `http://127.0.0.1:${server.address().port}` };
+  const run = (args, onSpawn) => new Promise((resolve, reject) => {
+    const child = spawn(process.execPath, [CLI, ...args], { env, cwd: dir }); let stdout = '', stderr = '';
+    child.stdout.on('data', data => { stdout += data; }); child.stderr.on('data', data => { stderr += data; });
+    const timer = setTimeout(() => { child.kill('SIGKILL'); reject(Error('CLI test timeout')); }, 10000);
+    child.on('error', reject); child.on('exit', (code, signal) => { clearTimeout(timer); resolve({ code, signal, stdout, stderr }); });
+    onSpawn?.(child);
+  });
+  const file = path.join(dir, 'fixture.pdf'); await fs.writeFile(file, 'synthetic fixture');
+  return { dir, file, run, calls, state, receipts, env };
+}
+test('real CLI waits finitely, preserves job receipt on timeout/failure, and can later inspect completion', async t => {
+  const f = await fixture(t); f.state.jobStatus = 'queued';
+  const timed = await f.run(['ingest', f.file, '--wait', '--timeout', '1', '--json']);
+  assert.equal(timed.code, 1); const receipt = JSON.parse(timed.stdout); assert.equal(receipt.job_id, jobId); assert.equal(receipt.wait.outcome, 'timeout'); assert(timed.stderr.includes(receipt.request_key));
+  f.state.jobStatus = 'completed';
+  const completed = await f.run(['jobs', 'wait', jobId, '--timeout', '2', '--json']);
+  assert.equal(completed.code, 0); assert.equal(JSON.parse(completed.stdout).wait.outcome, 'completed');
+  f.state.jobStatus = 'failed'; const failed = await f.run(['jobs', 'wait', jobId, '--json']);
+  assert.equal(failed.code, 1); assert.equal(JSON.parse(failed.stdout).job_id, jobId); assert.equal(JSON.parse(failed.stdout).wait.outcome, 'failed');
+  assert.equal(f.calls.filter(call => call.path.endsWith('/content')).length, 1);
+});
+test('real dropped acceptance is looked up, explicit reuse never uploads twice, changed scope conflicts', async t => {
+  const f = await fixture(t); f.state.loseContent = true;
+  const first = await f.run(['ingest', f.file, '--json']); assert.equal(first.code, 0); const result = JSON.parse(first.stdout); assert.equal(result.job_id, jobId);
+  const repeated = await f.run(['ingest', f.file, '--request-key', result.request_key, '--json']); assert.equal(repeated.code, 0); assert.equal(JSON.parse(repeated.stdout).job_id, jobId);
+  const conflict = await f.run(['ingest', f.file, '--request-key', result.request_key, '--visibility', 'workspace', '--json']);
+  assert.equal(conflict.code, 1); assert.equal(JSON.parse(conflict.stdout).error.status, 409); assert.equal(f.calls.filter(call => call.path.endsWith('/content')).length, 1);
+});
+test('real lost prepare response requires explicit same-key retry and never auto-transfers', async t => {
+  const f = await fixture(t); f.state.losePrepare = true;
+  const first = await f.run(['ingest', f.file, '--json']); assert.equal(first.code, 1); const result = JSON.parse(first.stdout); assert.equal(result.outcome, 'prepared'); assert.equal(f.calls.filter(call => call.path.endsWith('/content')).length, 0);
+  const retried = await f.run(['ingest', f.file, '--request-key', result.request_key, '--wait', '--json']); assert.equal(retried.code, 0); assert.equal(JSON.parse(retried.stdout).wait.outcome, 'completed');
+});
+test('watch persists unknown acceptance across process restart and key rotation without resending pending bytes', async t => {
+  const f = await fixture(t); const folder = path.join(f.dir, 'exports'); await fs.mkdir(folder); await fs.writeFile(path.join(folder, 'chat.json'), JSON.stringify({ messages: [{ content: 'synthetic fixture' }] }));
+  f.state.loseContent = true; f.state.failLookup = 1;
+  const first = await f.run(['watch', folder, '--provider', 'generic', '--once', '--json']); assert.equal(first.code, 1); assert.equal(JSON.parse(first.stdout).outcomes[0].outcome, 'unconfirmed');
+  const stateFile = (await fs.readdir(path.join(f.dir, 'config'))).find(name => /^watch-.*\.json$/.test(name));
+  const saved = JSON.parse(await fs.readFile(path.join(f.dir, 'config', stateFile), 'utf8')); assert(Object.values(saved.entries)[0].pending.request_key);
+  f.env.OPENCRAWL_API_KEY = 'oc_synthetic_rotated_same_account';
+  const second = await f.run(['watch', folder, '--provider', 'generic', '--once', '--json']); assert.equal(second.code, 0); assert.equal(JSON.parse(second.stdout).outcomes[0].outcome, 'accepted');
+  const third = await f.run(['watch', folder, '--provider', 'generic', '--once', '--json']); assert.equal(third.code, 0); assert.equal(JSON.parse(third.stdout).outcomes.length, 0);
+  assert.equal(f.calls.filter(call => call.path.endsWith('/content')).length, 1); assert.equal(f.calls.filter(call => call.path.endsWith('/prepare')).length, 1);
+});
+test('interrupt aborts an in-flight job request locally and keeps exact ID', async t => {
+  const f = await fixture(t); f.state.holdJob = true;
+  const result = await f.run(['jobs', 'wait', jobId, '--json'], child => {
+    const poll = setInterval(() => { if (f.calls.some(call => call.path === `/api/v1/jobs/${jobId}`)) { clearInterval(poll); child.kill('SIGINT'); } }, 10);
+    child.on('exit', () => clearInterval(poll));
+  });
+  assert.equal(result.code, 1); assert.equal(result.signal, null); assert.equal(JSON.parse(result.stdout).job_id, jobId); assert.equal(JSON.parse(result.stdout).wait.outcome, 'interrupted'); assert.equal(f.calls.length, 1);
+});
+test('strict flags fail with usable JSON and no network; human search escapes controls while JSON preserves bytes', async t => {
+  const f = await fixture(t);
+  for (const args of [['ingest', f.file, '--timeout', '1'], ['ingest', f.file, '--wait=false'], ['ingest', f.file, '--wait', '--wait'], ['jobs', 'wait', jobId, '--retry'], ['ingest', f.file, '--surprise', 'x']]) {
+    const result = await f.run([...args, '--json']); assert.equal(result.code, 1); assert(JSON.parse(result.stdout).error);
+  }
+  assert.equal(f.calls.length, 0);
+  const human = await f.run(['search', 'test']); assert.equal(human.code, 0); assert(!human.stdout.includes('\u001b')); assert(human.stdout.includes('\\u001b')); assert(human.stdout.includes('keep\nline'));
+  const json = await f.run(['search', 'test', '--json']); assert.equal(JSON.parse(json.stdout).results[0].title, 'bad\u001b[2J');
+  const text = path.join(f.dir, 'note.txt'); await fs.writeFile(text, 'synchronous note');
+  const inline = await f.run(['import', text, '--wait', '--json']); assert.equal(inline.code, 0); assert.equal(JSON.parse(inline.stdout).id, 'doc_' + 'c'.repeat(32)); assert(!JSON.parse(inline.stdout).job_id);
+});
+
+
+test('watch treats changed explicit source or visibility as a separate intent and preserves old pending scope', async t => {
+  const f = await fixture(t); const folder = path.join(f.dir, 'exports'); await fs.mkdir(folder); await fs.writeFile(path.join(folder, 'chat.json'), '{"content":"synthetic fixture"}');
+  f.state.losePrepare = true;
+  const first = await f.run(['watch', folder, '--provider', 'generic', '--source', 'src_private', '--once', '--json']);
+  assert.equal(first.code, 1);
+  const next = await f.run(['watch', folder, '--provider', 'generic', '--source', 'src_shared', '--visibility', 'workspace', '--once', '--json']);
+  assert.equal(next.code, 0);
+  const states = await Promise.all((await fs.readdir(path.join(f.dir, 'config'))).filter(name => name.endsWith('.json')).map(name => fs.readFile(path.join(f.dir, 'config', name), 'utf8').then(JSON.parse)));
+  assert.equal(states.length, 2);
+  assert(Object.values(states.find(value => value.scope.source_id === 'src_private').entries)[0].pending);
+  assert(Object.values(states.find(value => value.scope.source_id === 'src_shared').entries)[0].accepted);
+  assert.equal(f.calls.filter(call => call.path.endsWith('/prepare')).length, 2);
+  assert.equal(f.calls.filter(call => call.path.endsWith('/content')).length, 1);
+});
+
+test('unscoped legacy watcher state is preserved and blocks automatic migration or new uploads', async t => {
+  const f = await fixture(t); const folder = path.join(f.dir, 'exports'); await fs.mkdir(folder); const file = path.join(folder, 'chat.json'); await fs.writeFile(file, '{"content":"synthetic fixture"}');
+  const root = await fs.realpath(folder), identity = crypto.createHash('sha256').update(f.env.OPENCRAWL_API_KEY).digest('hex').slice(0,16);
+  const name = crypto.createHash('sha256').update(`${f.env.OPENCRAWL_URL}:${identity}:${root}:generic`).digest('hex').slice(0,24);
+  const config = path.join(f.dir, 'config'); await fs.mkdir(config);
+  const legacy = path.join(config, `watch-${name}.json`), original = JSON.stringify({ [file]: '25:1' }); await fs.writeFile(legacy, original);
+  const result = await f.run(['watch', folder, '--provider', 'generic', '--once', '--json']);
+  assert.equal(result.code, 1); assert(JSON.parse(result.stdout).error.message.includes('Older watcher state'));
+  assert.equal(await fs.readFile(legacy, 'utf8'), original);
+  assert(!f.calls.some(call => call.method === 'POST')); assert.equal((await fs.readdir(config)).length, 1);
+});

package/src/uploads.test.mjs

@@ -0,0 +1,64 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import fs from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+import crypto from 'node:crypto';
+import { submitUpload, fileDescriptor, followUpload, uploadExitCode, requestKey } from './uploads.mjs';
+
+const key = 'abcd1234-1234-4234-8234-abcd12345678', id = 'receipt_' + 'b'.repeat(32), jobId = 'job_' + 'a'.repeat(32);
+const receipt = (state, overrides = {}) => ({ id, request_key: key, state, operation: 'upload', upload_url: `/api/v1/uploads/receipts/${id}/content`, job: state === 'accepted' ? { id: jobId, kind: 'upload', status: 'queued' } : null, ...overrides });
+async function fixture(t) { const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'opencrawl-upload-')); t.after(() => fs.rm(dir, { recursive: true, force: true })); const file = path.join(dir, 'binary.pdf'); await fs.writeFile(file, Buffer.from([1,2,0,255,127])); return file; }
+const quiet = { stderr: () => {} };
+test('upload hashes exact bytes then sends one multipart file and no duplicated scope fields', async t => {
+  const file = await fixture(t); const events = [];
+  const client = { request: async (route, options) => {
+    events.push(route);
+    if (route.endsWith('/prepare')) { assert.equal(options.body.sha256, crypto.createHash('sha256').update(await fs.readFile(file)).digest('hex')); assert.equal(options.body.visibility, 'workspace'); assert.equal(options.body.source_id, 'src_test'); return receipt('prepared'); }
+    assert.deepEqual([...options.body.keys()], ['file']); assert.deepEqual(Buffer.from(await options.body.get('file').arrayBuffer()), await fs.readFile(file)); return receipt('accepted');
+  } };
+  const result = await submitUpload(file, client, { 'request-key': key, source: 'src_test', visibility: 'workspace' }, { ...quiet, onIntent: async () => { events.push('persist-intent'); } });
+  assert.equal(events[0], 'persist-intent'); assert.equal(events.length, 3); assert.equal(result.job_id, jobId); assert.equal(uploadExitCode(result, false), 0);
+});
+test('accepted, receiving, and expired prepare receipts never resend file bytes', async t => {
+  const file = await fixture(t);
+  for (const state of ['accepted','receiving','expired']) {
+    const calls = [];
+    const result = await submitUpload(file, { request: async route => { calls.push(route); return receipt(state); } }, { 'request-key': key }, quiet);
+    assert.deepEqual(calls, ['/api/v1/uploads/prepare']); assert.equal(result.outcome, state);
+  }
+});
+test('lost prepare response only looks up, even when lookup says prepared', async t => {
+  const file = await fixture(t); const calls = [];
+  const result = await submitUpload(file, { request: async route => { calls.push(route); if (calls.length === 1) throw Error('dropped'); return receipt('prepared'); } }, { 'request-key': key }, quiet);
+  assert.deepEqual(calls, ['/api/v1/uploads/prepare', `/api/v1/uploads/receipts/by-key/${key}`]); assert.equal(result.outcome, 'prepared'); assert.equal(uploadExitCode(result, false), 1);
+});
+test('lost acceptance resolves to original job by lookup without resending', async t => {
+  const file = await fixture(t); const calls = [];
+  const result = await submitUpload(file, { request: async route => { calls.push(route); if (route.endsWith('/prepare')) return receipt('prepared'); if (route.endsWith('/content')) throw Error('dropped acceptance'); return receipt('accepted'); } }, { 'request-key': key }, quiet);
+  assert.equal(calls.length, 3); assert.equal(result.job_id, jobId); assert.equal(result.outcome, 'accepted'); assert.equal(calls.filter(route => route.endsWith('/content')).length, 1);
+});
+test('descriptor conflicts do not treat the old accepted job as a new successful upload', async t => {
+  const file = await fixture(t); let calls = 0;
+  const result = await submitUpload(file, { request: async () => { calls++; const error = Error('private server body'); error.status = 409; throw error; } }, { 'request-key': key }, quiet);
+  assert.equal(calls, 1); assert.equal(result.request_key, key); assert.equal(result.job_id, null); assert.equal(result.error.status, 409); assert(!JSON.stringify(result).includes('private server'));
+});
+test('unknown outcomes retain request key and receipt ID; foreign upload routes never receive bytes', async t => {
+  const file = await fixture(t); const result = await submitUpload(file, { request: async route => { if (route.endsWith('/prepare')) return receipt('prepared'); throw Error('network'); } }, { 'request-key': key }, quiet);
+  assert.equal(result.request_key, key); assert.equal(result.receipt_id, id); assert.equal(result.job_id, null); assert.equal(result.outcome, 'unconfirmed'); assert.equal(result.receipt.state, 'prepared');
+  let calls = 0; const foreign = await submitUpload(file, { request: async () => { calls++; return receipt('prepared', { upload_url: 'https://other.example/upload' }); } }, { 'request-key': key }, quiet);
+  assert.equal(calls, 1); assert.equal(foreign.error.code, 'invalid_upload_route');
+});
+test('receiving wait checks receipt then existing job with one bounded deadline', async () => {
+  let time = 0; const calls = [];
+  const result = await followUpload({ request_key: key, receipt_id: id, receipt: receipt('receiving'), job_id: null, outcome: 'receiving' }, { request: async route => { calls.push(route); return route.includes('/receipts/') ? receipt('accepted') : { id: jobId, status: 'completed', kind: 'upload' }; } }, { timeout: 3, now: () => time, sleep: async ms => { time += ms; } });
+  assert.equal(result.wait.outcome, 'completed'); assert.deepEqual(calls, [`/api/v1/uploads/receipts/by-key/${key}`, `/api/v1/jobs/${jobId}`]);
+  const timeout = await followUpload({ request_key: key, receipt: receipt('receiving'), outcome: 'receiving' }, { request: async () => { throw Error('not reached'); } }, { timeout: 1, now: () => time, sleep: async ms => { time += ms; } });
+  assert.equal(timeout.wait.outcome, 'timeout'); assert.equal(timeout.request_key, key);
+});
+test('file validation and failed durable intent precede all network activity', async t => {
+  const file = await fixture(t); let calls = 0; const client = { request: async () => { calls++; } };
+  await assert.rejects(submitUpload(file, client, {}, { ...quiet, onIntent: async () => { throw Error('disk full'); } }), /disk full/);
+  await assert.rejects(fileDescriptor(file, { visibility: 'public' }), /visibility/);
+  assert.throws(() => requestKey('invalid'), /UUID/); await fs.writeFile(file, ''); await assert.rejects(fileDescriptor(file), /nonempty/); assert.equal(calls, 0);
+});