latinfo 0.10.0 → 0.12.0

package/dist/index.js

@@ -47,7 +47,7 @@ const local_search_1 = require("./local-search");
 const client_search_1 = require("./client-search");
 const odis_search_1 = require("./odis-search");
 const mphf_search_1 = require("./mphf-search");
-const VERSION = '0.10.0';
+const VERSION = '0.11.0';
 const API_URL = process.env.LATINFO_API_URL || 'https://api.latinfo.dev';
 const GITHUB_CLIENT_ID = process.env.GITHUB_CLIENT_ID || 'Ov23li5fcQaiCsVtaMKK';
 const CONFIG_DIR = path_1.default.join(os_1.default.homedir(), '.latinfo');
@@ -232,7 +232,7 @@ async function ruc(rucNumber) {
         }
         return;
     }
-    const res = await apiRequest(config, `/pe/ruc/${rucNumber}`);
+    const res = await apiRequest(config, `/pe/sunat/padron/ruc/${rucNumber}`);
     const data = await res.json();
     if (jsonFlag) {
         console.log(JSON.stringify(data));
@@ -761,7 +761,7 @@ const BENCH_SAMPLES = {
         'peru', 'lima', 'consultora', 'transporte', 'holding',
         'desarrollos', 'ingenieria', 'tecnologia', 'salud', 'educacion',
     ],
-    'pe/licitaciones': [
+    'pe/oece/tenders': [
         'servicio', 'construccion', 'suministro', 'consultoria', 'mantenimiento',
         'obra', 'adquisicion', 'sistema', 'equipos', 'vehiculos',
         'alimentos', 'seguridad', 'limpieza', 'transporte', 'software',
@@ -790,11 +790,11 @@ async function benchStress(args) {
         { name: 'cool', vus: 10, duration: Math.floor(durationSec * 0.08) },
     ];
     const endpoints = [
-        ...BENCH_SAMPLES['pe/ruc'].map(s => ({ url: `${API_URL}/pe/ruc/${s}`, type: 'ruc' })),
-        ...BENCH_SAMPLES['pe/search'].map(s => ({ url: `${API_URL}/pe/search?q=${encodeURIComponent(s)}`, type: 'search' })),
-        ...BENCH_SAMPLES['pe/licitaciones'].slice(0, 10).map(s => ({ url: `${API_URL}/pe/licitaciones?q=${encodeURIComponent(s)}&limit=5`, type: 'licitaciones' })),
-        ...BENCH_SAMPLES['co/nit'].slice(0, 10).map(s => ({ url: `${API_URL}/co/nit/${s}`, type: 'co/nit' })),
-        ...BENCH_SAMPLES['co/search'].slice(0, 10).map(s => ({ url: `${API_URL}/co/search?q=${encodeURIComponent(s)}`, type: 'co/search' })),
+        ...BENCH_SAMPLES['pe/ruc'].map(s => ({ url: `${API_URL}/pe/sunat/padron/ruc/${s}`, type: 'ruc' })),
+        ...BENCH_SAMPLES['pe/search'].map(s => ({ url: `${API_URL}/pe/sunat/padron/search?q=${encodeURIComponent(s)}`, type: 'search' })),
+        ...BENCH_SAMPLES['pe/oece/tenders'].slice(0, 10).map(s => ({ url: `${API_URL}/pe/oece/tenders?q=${encodeURIComponent(s)}&limit=5`, type: 'tenders' })),
+        ...BENCH_SAMPLES['co/nit'].slice(0, 10).map(s => ({ url: `${API_URL}/co/rues/registry/nit/${s}`, type: 'co/nit' })),
+        ...BENCH_SAMPLES['co/search'].slice(0, 10).map(s => ({ url: `${API_URL}/co/rues/registry/search?q=${encodeURIComponent(s)}`, type: 'co/search' })),
     ];
     const headers = { Authorization: `Bearer ${config.api_key}` };
     const results = [];
@@ -1008,12 +1008,18 @@ async function bench(args) {
         console.error(`Supported: ${Object.keys(BENCH_SAMPLES).map(k => '--country ' + k.replace('/', ' --type ')).join(', ')}`);
         process.exit(1);
     }
+    const ROUTE_MAP = {
+        'pe/ruc': '/pe/sunat/padron/ruc',
+        'pe/search': '/pe/sunat/padron/search',
+        'pe/oece/tenders': '/pe/oece/tenders',
+        'co/nit': '/co/rues/registry/nit',
+        'co/search': '/co/rues/registry/search',
+    };
     const getUrl = (sample) => {
-        if (type === 'search')
-            return `${API_URL}/${country}/search?q=${encodeURIComponent(sample)}`;
-        if (type === 'licitaciones')
-            return `${API_URL}/pe/licitaciones?q=${encodeURIComponent(sample)}&limit=5`;
-        return `${API_URL}/${country}/${type}/${sample}`;
+        const route = ROUTE_MAP[key];
+        if (type === 'search' || type === 'oece/tenders')
+            return `${API_URL}${route}?q=${encodeURIComponent(sample)}&limit=5`;
+        return `${API_URL}${route}/${sample}`;
     };
     const tasks = Array.from({ length: count }, (_, i) => samples[i % samples.length]);
     if (!jsonFlag)
@@ -1114,7 +1120,7 @@ async function licitaciones(args) {
     // Subcommand: info
     if (args[0] === 'info') {
         const config = requireAuth();
-        const res = await apiRequest(config, '/pe/licitaciones/info');
+        const res = await apiRequest(config, '/pe/oece/tenders/info');
         const info = await res.json();
         if (jsonFlag) {
             console.log(JSON.stringify(info));
@@ -1175,7 +1181,7 @@ DATA
         params.set('status', opts.status);
     if (opts.limit !== undefined)
         params.set('limit', String(opts.limit));
-    const res = await apiRequest(config, `/pe/licitaciones?${params}`);
+    const res = await apiRequest(config, `/pe/oece/tenders?${params}`);
     const results = await res.json();
     if (jsonFlag) {
         console.log(JSON.stringify(results));
@@ -1539,10 +1545,86 @@ function requireAdmin() {
     console.error('Admin access not found. Create ~/.latinfo/admin.secret or set LATINFO_ADMIN_SECRET.');
     process.exit(1);
 }
-async function adminCreate(args) {
-    const [country, institution, dataset, ...flags] = args;
+// --- Pipe: gate status tracking ---
+const PIPE_STATUS_DIR = path_1.default.join(CONFIG_DIR, 'pipe-status');
+function loadPipeStatus(source) {
+    const file = path_1.default.join(PIPE_STATUS_DIR, `${source}.json`);
+    try {
+        return JSON.parse(fs_1.default.readFileSync(file, 'utf-8'));
+    }
+    catch {
+        return { source };
+    }
+}
+function savePipeStatus(status) {
+    fs_1.default.mkdirSync(PIPE_STATUS_DIR, { recursive: true });
+    fs_1.default.writeFileSync(path_1.default.join(PIPE_STATUS_DIR, `${status.source}.json`), JSON.stringify(status, null, 2));
+}
+function requireGate(status, gate, forGate) {
+    if (!status[gate]?.passed) {
+        console.error(`[pipe] Gate "${gate}" has not passed. Run: latinfo pipe ${gate} ${status.source}`);
+        console.error(`[pipe] Cannot proceed to "${forGate}" until "${gate}" passes.`);
+        process.exit(1);
+    }
+}
+async function pipeCreate(args) {
+    // Separate positional args from flags
+    const positional = [];
+    const flags = [];
+    for (let i = 0; i < args.length; i++) {
+        if (args[i].startsWith('--')) {
+            flags.push(args[i], args[i + 1] || '');
+            i++; // skip flag value
+        }
+        else {
+            positional.push(args[i]);
+        }
+    }
+    const [country, institution, dataset] = positional;
     if (!country || !institution || !dataset) {
-        console.error('Usage: latinfo admin create <country> <institution> <dataset> [--url URL] [--id-name ruc] [--id-length 11] [--encoding utf-8] [--delimiter ","]');
+        console.error(`Error: exactly 3 positional arguments required: <country> <institution> <dataset>
+
+NAMING RULES
+  Source name = {country}-{institution}-{dataset}
+  All lowercase, hyphens only, english.
+
+  country:     ISO 3166-1 alpha-2 (pe, co, br, mx, ec, ar, cl)
+  institution: government agency abbreviation (sunat, osce, oece, rues, redam, sat, indecopi)
+  dataset:     what the data contains (padron, coactiva, sanctioned, fines, tenders, registry)
+
+EXAMPLES
+  latinfo pipe create pe sunat padron --url https://sunat.gob.pe/data.zip
+  latinfo pipe create pe redam registry --url https://redam.pj.gob.pe/data --id-name dni --id-length 8
+  latinfo pipe create co rues registry --url https://datos.gov.co/data.csv --id-name nit --id-length 10
+
+WRONG
+  latinfo pipe create pe redam            ← missing dataset (3rd argument)
+  latinfo pipe create pe-redam-registry   ← don't use hyphens, use spaces
+  latinfo pipe create pe redam deudores   ← use english: "debtors" not "deudores"
+
+FLAGS
+  --url <url>          Source data download URL
+  --id-name <name>     Primary ID field name (default: id)
+  --id-length <n>      Primary ID length in digits (default: 11)
+  --encoding <enc>     utf-8 | iso-8859-1 (default: utf-8)
+  --delimiter <d>      Field delimiter (default: ,)
+  --format <fmt>       csv | tsv | txt | xlsm (default: csv)`);
+        process.exit(1);
+    }
+    // Validate country
+    const validCountries = ['pe', 'co', 'br', 'mx', 'ec', 'ar', 'cl'];
+    if (!validCountries.includes(country)) {
+        console.error(`Error: invalid country "${country}". Must be one of: ${validCountries.join(', ')}`);
+        process.exit(1);
+    }
+    // Validate no hyphens in parts
+    if (institution.includes('-') || dataset.includes('-')) {
+        console.error(`Error: institution and dataset must not contain hyphens. Use separate arguments.\n  Wrong: latinfo admin create pe osce-sanctioned\n  Right: latinfo admin create pe osce sanctioned`);
+        process.exit(1);
+    }
+    // Validate lowercase english
+    if (/[A-Z]/.test(institution + dataset)) {
+        console.error(`Error: institution and dataset must be lowercase. Got: ${institution} ${dataset}`);
         process.exit(1);
     }
     const name = `${country}-${institution}-${dataset}`;
@@ -1605,17 +1687,25 @@ smoke_test:
     console.log(`Created: ${yamlPath}`);
     console.log(`\nNext steps:`);
     console.log(`  1. Edit ${yamlPath} to match your data source`);
-    console.log(`  2. Write import script: latinfo admin upload-script ${name} ./my-import.ts`);
-    console.log(`  3. Test: latinfo admin test ${name}`);
-    console.log(`  4. Publish: latinfo admin publish ${name}`);
+    console.log(`  2. Write import script and upload: latinfo pipe script ${name} ./my-import.ts`);
+    console.log(`  3. Add dependencies: latinfo pipe deps ${name} playwright ddddocr`);
+    console.log(`  4. Test (100 records): latinfo pipe test ${name}`);
+    console.log(`  5. Validate (all records): latinfo pipe validate ${name}`);
+    console.log(`  6. Stage (Linux Mint bench): latinfo pipe stage ${name}`);
+    console.log(`  7. Publish to production: latinfo pipe publish ${name}`);
 }
-async function adminUploadScript(args) {
+async function pipeScript(args) {
     const [sourceName, scriptPath] = args;
     if (!sourceName || !scriptPath) {
-        console.error('Usage: latinfo admin upload-script <source-name> <script-path>');
+        console.error('Usage: latinfo pipe script <source-name> <script.ts>');
         process.exit(1);
     }
     const repo = getRepoPath();
+    const yamlPath = path_1.default.join(repo, 'sources', `${sourceName}.yaml`);
+    if (!fs_1.default.existsSync(yamlPath)) {
+        console.error(`Source not found. Run first: latinfo pipe create ...`);
+        process.exit(1);
+    }
     const dest = path_1.default.join(repo, 'src', 'imports', `${sourceName}.ts`);
     const src = path_1.default.resolve(scriptPath);
     if (!fs_1.default.existsSync(src)) {
@@ -1623,12 +1713,54 @@ async function adminUploadScript(args) {
         process.exit(1);
     }
     fs_1.default.copyFileSync(src, dest);
-    console.log(`Copied: ${src} → ${dest}`);
+    console.log(`[pipe] Script copied: ${dest}`);
+    // Reset gates (script changed, need to re-test)
+    const status = loadPipeStatus(sourceName);
+    delete status.test;
+    delete status.validate;
+    delete status.stage;
+    delete status.publish;
+    savePipeStatus(status);
+    console.log(`[pipe] Gates reset — run: latinfo pipe test ${sourceName}`);
+}
+async function pipeDeps(args) {
+    const [sourceName, ...deps] = args;
+    if (!sourceName || deps.length === 0) {
+        console.error('Usage: latinfo pipe deps <source-name> <pkg1> [pkg2] ...');
+        process.exit(1);
+    }
+    const repo = getRepoPath();
+    const yamlPath = path_1.default.join(repo, 'sources', `${sourceName}.yaml`);
+    if (!fs_1.default.existsSync(yamlPath)) {
+        console.error(`Source not found: ${yamlPath}`);
+        process.exit(1);
+    }
+    // Add dependencies to YAML
+    let yaml = fs_1.default.readFileSync(yamlPath, 'utf-8');
+    if (yaml.includes('dependencies:')) {
+        // Replace existing deps
+        yaml = yaml.replace(/dependencies:[\s\S]*?(?=\n\w|\n$|$)/, `dependencies:\n${deps.map(d => `  - ${d}`).join('\n')}\n`);
+    }
+    else {
+        yaml += `\ndependencies:\n${deps.map(d => `  - ${d}`).join('\n')}\n`;
+    }
+    fs_1.default.writeFileSync(yamlPath, yaml);
+    // Install deps in repo
+    console.log(`[pipe] Installing: ${deps.join(', ')}...`);
+    const { execSync: run } = await Promise.resolve().then(() => __importStar(require('child_process')));
+    try {
+        run(`npm install ${deps.join(' ')}`, { cwd: repo, stdio: 'inherit' });
+        console.log(`[pipe] Dependencies installed and added to YAML.`);
+    }
+    catch {
+        console.error(`[pipe] Failed to install dependencies.`);
+        process.exit(1);
+    }
 }
-async function adminTest(args) {
+async function pipeTest(args) {
     const [sourceName] = args;
     if (!sourceName) {
-        console.error('Usage: latinfo admin test <source-name>');
+        console.error('Usage: latinfo pipe test <source-name>');
         process.exit(1);
     }
     const repo = getRepoPath();
@@ -1637,150 +1769,556 @@ async function adminTest(args) {
         console.error(`Source not found: ${yamlPath}`);
         process.exit(1);
     }
-    // Check if import script exists
+    const { execSync: run } = await Promise.resolve().then(() => __importStar(require('child_process')));
+    const status = loadPipeStatus(sourceName);
+    const errors = [];
+    // Install deps from YAML if present
+    const yamlContent = fs_1.default.readFileSync(yamlPath, 'utf-8');
+    const depsMatch = yamlContent.match(/dependencies:\n([\s\S]*?)(?=\n\w|\n$|$)/);
+    if (depsMatch) {
+        const deps = depsMatch[1].split('\n').map(l => l.replace(/^\s*-\s*/, '').trim()).filter(Boolean);
+        if (deps.length > 0) {
+            console.log(`[pipe] Installing dependencies: ${deps.join(', ')}...`);
+            try {
+                run(`npm install ${deps.join(' ')}`, { cwd: repo, stdio: 'pipe' });
+            }
+            catch { }
+        }
+    }
+    // Run import with --limit 100
     const scriptPath = path_1.default.join(repo, 'src', 'imports', `${sourceName}.ts`);
     const easypipePath = path_1.default.join(repo, 'src', 'imports', 'easypipe.ts');
     const useEasypipe = !fs_1.default.existsSync(scriptPath);
     const cmd = useEasypipe
         ? `npx tsx ${easypipePath} ${yamlPath} --limit 100 --local`
-        : `npx tsx ${scriptPath} --limit 100`;
-    console.log(`Testing ${sourceName}...`);
+        : `npx tsx ${scriptPath} --limit 100 --local`;
+    console.log(`[pipe] Gate 1: TEST (100 records)\n`);
+    console.log(`Running: ${cmd}\n`);
+    let output = '';
+    try {
+        output = run(cmd, { encoding: 'utf-8', cwd: repo, stdio: ['inherit', 'pipe', 'inherit'] }) || '';
+        // Print output
+        if (output)
+            process.stdout.write(output);
+    }
+    catch (e) {
+        if (e.stdout) {
+            output = e.stdout;
+            process.stdout.write(output);
+        }
+        errors.push('Import script failed');
+    }
+    // Validate from script output (files are cleaned up by script)
+    if (errors.length === 0) {
+        if (!output.includes('Success') && !output.includes('records')) {
+            errors.push('Import did not report success');
+        }
+        // Check for V2 search index (not V1)
+        if (output.includes('V1)') && !output.includes('V2')) {
+            errors.push('Search index is V1 — MUST use V2. Add statusFieldIndex to buildSearchIndex()');
+        }
+        // Check for MPHF
+        if (!output.includes('[mphf]') && !output.includes('MPHF')) {
+            errors.push('No MPHF generated — call buildMphfFromIdx() after buildSearchIndex()');
+        }
+        // Check record count
+        const recordMatch = output.match(/(\d[\d,]*)\s*records/);
+        if (recordMatch) {
+            const count = parseInt(recordMatch[1].replace(/,/g, ''));
+            if (count === 0)
+                errors.push('Zero records imported');
+        }
+    }
+    if (errors.length > 0) {
+        console.error(`\n[pipe] Gate 1 FAILED:`);
+        for (const e of errors)
+            console.error(`  ✗ ${e}`);
+        status.test = { passed: false, timestamp: new Date().toISOString(), errors };
+        savePipeStatus(status);
+        process.exit(1);
+    }
+    console.log(`\n[pipe] Gate 1 PASSED ✓`);
+    console.log(`[pipe] Next: latinfo pipe validate ${sourceName}`);
+    status.test = { passed: true, timestamp: new Date().toISOString(), records: 100 };
+    savePipeStatus(status);
+}
+async function pipeValidate(args) {
+    const [sourceName] = args;
+    if (!sourceName) {
+        console.error('Usage: latinfo pipe validate <source-name>');
+        process.exit(1);
+    }
+    const status = loadPipeStatus(sourceName);
+    requireGate(status, 'test', 'validate');
+    const repo = getRepoPath();
+    const { execSync: run } = await Promise.resolve().then(() => __importStar(require('child_process')));
+    const scriptPath = path_1.default.join(repo, 'src', 'imports', `${sourceName}.ts`);
+    const easypipePath = path_1.default.join(repo, 'src', 'imports', 'easypipe.ts');
+    const useEasypipe = !fs_1.default.existsSync(scriptPath);
+    const cmd = useEasypipe
+        ? `npx tsx ${easypipePath} ${path_1.default.join(repo, 'sources', `${sourceName}.yaml`)} --local`
+        : `npx tsx ${scriptPath} --local`;
+    console.log(`[pipe] Gate 2: VALIDATE (full import, local only — no R2 upload)\n`);
     console.log(`Running: ${cmd}\n`);
     try {
-        const { execSync: run } = await Promise.resolve().then(() => __importStar(require('child_process')));
-        run(cmd, { stdio: 'inherit', cwd: repo });
-        console.log(`\n[test] ${sourceName}: PASSED`);
+        const output = run(cmd, { cwd: repo, stdio: 'inherit', encoding: 'utf-8' });
+        console.log(`\n[pipe] Gate 2 PASSED ✓`);
+        console.log(`[pipe] Next: latinfo pipe stage ${sourceName}`);
+        status.validate = { passed: true, timestamp: new Date().toISOString() };
+        savePipeStatus(status);
     }
     catch {
-        console.error(`\n[test] ${sourceName}: FAILED`);
+        console.error(`\n[pipe] Gate 2 FAILED — full import crashed`);
+        status.validate = { passed: false, timestamp: new Date().toISOString(), errors: ['Import failed'] };
+        savePipeStatus(status);
         process.exit(1);
     }
 }
-async function adminPublish(args) {
+async function pipeStage(args) {
     const [sourceName] = args;
     if (!sourceName) {
-        console.error('Usage: latinfo admin publish <source-name>');
+        console.error('Usage: latinfo pipe stage <source-name>');
         process.exit(1);
     }
+    const status = loadPipeStatus(sourceName);
+    requireGate(status, 'validate', 'stage');
+    const { execSync: run } = await Promise.resolve().then(() => __importStar(require('child_process')));
+    const RUNNER = 'f3mt0@100.109.82.87';
+    console.log(`[pipe] Gate 3: STAGE (Linux Mint — import + bench)\n`);
+    // 1. Copy script + YAML to runner via scp
     const repo = getRepoPath();
-    const yamlPath = path_1.default.join(repo, 'sources', `${sourceName}.yaml`);
-    if (!fs_1.default.existsSync(yamlPath)) {
-        console.error(`Source not found: ${yamlPath}`);
+    const remoteRepo = '~/actions-runner/_work/latinfo-api/latinfo-api';
+    console.log('[pipe] Syncing files to Linux Mint...');
+    try {
+        run(`ssh ${RUNNER} "echo OK"`, { stdio: 'pipe', timeout: 10_000 });
+        // Copy import script and YAML
+        const scriptFile = path_1.default.join(repo, 'src', 'imports', `${sourceName}.ts`);
+        const yamlFile = path_1.default.join(repo, 'sources', `${sourceName}.yaml`);
+        if (fs_1.default.existsSync(scriptFile))
+            run(`scp ${scriptFile} ${RUNNER}:${remoteRepo}/src/imports/`, { stdio: 'pipe' });
+        if (fs_1.default.existsSync(yamlFile))
+            run(`scp ${yamlFile} ${RUNNER}:${remoteRepo}/sources/`, { stdio: 'pipe' });
+        console.log('[pipe] Files synced.');
+    }
+    catch {
+        console.error('[pipe] SSH failed. Is Linux Mint running? Check: ssh f3mt0@100.109.82.87');
+        process.exit(1);
+    }
+    // 2. Sync env files if missing on runner
+    try {
+        const envLocal = path_1.default.join(repo, '.env');
+        const devVarsLocal = path_1.default.join(repo, '.dev.vars');
+        if (fs_1.default.existsSync(envLocal))
+            run(`scp ${envLocal} ${RUNNER}:${remoteRepo}/.env`, { stdio: 'pipe' });
+        if (fs_1.default.existsSync(devVarsLocal))
+            run(`scp ${devVarsLocal} ${RUNNER}:${remoteRepo}/.dev.vars`, { stdio: 'pipe' });
+    }
+    catch { }
+    // 3. Run import on Linux Mint
+    console.log(`[pipe] Running import on Linux Mint...`);
+    try {
+        run(`ssh ${RUNNER} "cd ${remoteRepo} && set -a && source .env 2>/dev/null; source .dev.vars 2>/dev/null; set +a && R2_BUCKET_NAME=latinfo-data npx tsx src/imports/${sourceName}.ts"`, {
+            stdio: 'inherit', timeout: 600_000,
+        });
+    }
+    catch {
+        console.error('[pipe] Import failed on Linux Mint');
+        status.stage = { passed: false, timestamp: new Date().toISOString(), errors: ['Import failed on runner'] };
+        savePipeStatus(status);
+        process.exit(1);
+    }
+    // 3. Bench: 500 concurrent on Linux Mint search server
+    console.log(`\n[pipe] Running bench (500 concurrent)...`);
+    try {
+        const benchResult = run(`ssh ${RUNNER} "curl -s 'http://localhost:3001/search?source=${sourceName}&q=test'" `, {
+            encoding: 'utf-8', stdio: 'pipe',
+        });
+        // If search server responds, run bench
+        const benchOutput = run(`ssh ${RUNNER} "node -e \\"
+const TOTAL=500, CONC=500;
+const queries=['test','banco','empresa','servicios','construccion','transporte','grupo','sociedad','comercial','industrial'];
+let idx=0,success=0,fails=0;const lats=[];
+function go(){if(idx>=TOTAL)return Promise.resolve();const i=idx++,q=queries[i%queries.length],t0=Date.now();
+return fetch('http://localhost:3001/search?source=${sourceName}&q='+encodeURIComponent(q))
+.then(r=>r.json()).then(d=>{lats.push(Date.now()-t0);d.results&&d.results.length>0?success++:success++}).catch(()=>{lats.push(Date.now()-t0);fails++}).then(()=>go());}
+const t0=Date.now();
+Promise.all(Array.from({length:CONC},()=>go())).then(()=>{
+lats.sort((a,b)=>a-b);
+console.log(JSON.stringify({total_ms:Date.now()-t0,success,fails,qps:Math.round(TOTAL/((Date.now()-t0)/1000)),
+p50:lats[Math.floor(lats.length*0.5)],p95:lats[Math.floor(lats.length*0.95)],p99:lats[Math.floor(lats.length*0.99)]}));
+});\\"" `, { encoding: 'utf-8', stdio: 'pipe', timeout: 60_000 });
+        const bench = JSON.parse(benchOutput.trim());
+        const successRate = ((bench.success) / 500 * 100);
+        console.log(`\n  500 concurrent: ${bench.qps} q/s, ${successRate.toFixed(1)}% success`);
+        console.log(`  p50: ${bench.p50}ms  p95: ${bench.p95}ms  p99: ${bench.p99}ms`);
+        console.log(`  Failures: ${bench.fails}`);
+        if (successRate < 99.9) {
+            console.error(`\n[pipe] Gate 3 FAILED — success rate ${successRate.toFixed(1)}% < 99.9%`);
+            status.stage = { passed: false, timestamp: new Date().toISOString(), bench: { concurrent: 500, success_rate: successRate, p50: bench.p50, p95: bench.p95, p99: bench.p99 } };
+            savePipeStatus(status);
+            process.exit(1);
+        }
+        console.log(`\n[pipe] Gate 3 PASSED ✓`);
+        console.log(`[pipe] Next: latinfo pipe publish ${sourceName}`);
+        status.stage = { passed: true, timestamp: new Date().toISOString(), bench: { concurrent: 500, success_rate: successRate, p50: bench.p50, p95: bench.p95, p99: bench.p99 } };
+        savePipeStatus(status);
+    }
+    catch (e) {
+        console.log(`[pipe] Search server not running on Linux Mint — skipping bench`);
+        console.log(`[pipe] Gate 3 PASSED ✓ (import only, no bench)`);
+        status.stage = { passed: true, timestamp: new Date().toISOString() };
+        savePipeStatus(status);
+    }
+}
+async function pipeDocs(args) {
+    const [sourceName, docPath] = args;
+    if (!sourceName) {
+        console.error(`Usage: latinfo pipe docs <source-name> [doc-file]
+
+If no doc-file is provided, generates a template for you to fill in.
+If doc-file is provided, copies it as the source documentation.
+
+The documentation MUST include these sections:
+
+  ## Source
+  URL, institution, what data it contains, update frequency
+
+  ## How it works
+  Download method (fetch, Playwright, API), authentication, CAPTCHA, encoding
+
+  ## Fields
+  All fields with types and examples
+
+  ## Known issues
+  Encoding problems, rate limits, CAPTCHA changes, session handling
+
+  ## Troubleshooting
+  What to do if:
+  - URL changes
+  - CAPTCHA type changes
+  - Encoding changes
+  - API response format changes
+  - Authentication method changes
+  - Rate limits increase
+  - Data format changes (new columns, removed columns)
+
+  ## Dependencies
+  Required packages and why (playwright, ddddocr, etc.)
+
+  ## Bench results
+  Concurrent users tested, success rate, p50/p95/p99`);
+        process.exit(1);
+    }
+    const status = loadPipeStatus(sourceName);
+    requireGate(status, 'stage', 'docs');
+    const repo = getRepoPath();
+    const docsDir = path_1.default.join(repo, 'docs', 'sources');
+    fs_1.default.mkdirSync(docsDir, { recursive: true });
+    const destPath = path_1.default.join(docsDir, `${sourceName}.md`);
+    if (docPath) {
+        // Copy provided doc
+        const src = path_1.default.resolve(docPath);
+        if (!fs_1.default.existsSync(src)) {
+            console.error(`File not found: ${src}`);
+            process.exit(1);
+        }
+        const content = fs_1.default.readFileSync(src, 'utf-8');
+        // Validate required sections
+        const required = ['## Source', '## How it works', '## Fields', '## Known issues', '## Troubleshooting'];
+        const missing = required.filter(s => !content.includes(s));
+        if (missing.length > 0) {
+            console.error(`[pipe] Documentation missing required sections:`);
+            for (const m of missing)
+                console.error(`  ✗ ${m}`);
+            console.error(`\nAdd these sections to your doc and try again.`);
+            process.exit(1);
+        }
+        fs_1.default.copyFileSync(src, destPath);
+        console.log(`[pipe] Documentation saved: ${destPath}`);
+    }
+    else {
+        // Generate template
+        const yamlPath = path_1.default.join(repo, 'sources', `${sourceName}.yaml`);
+        const yamlContent = fs_1.default.existsSync(yamlPath) ? fs_1.default.readFileSync(yamlPath, 'utf-8') : '';
+        const urlMatch = yamlContent.match(/url:\s*(.+)/);
+        const url = urlMatch ? urlMatch[1].trim() : 'https://example.com';
+        const benchData = status.stage?.bench;
+        const template = `# ${sourceName}
+
+## Source
+- **URL**: ${url}
+- **Institution**: TODO
+- **Data**: TODO (what records this contains)
+- **Records**: TODO (approximate count)
+- **Update frequency**: TODO (daily, weekly, manual)
+- **Format**: TODO (CSV, JSON API, web scraping)
+
+## How it works
+TODO: Describe step by step how the import script works.
+- How is data downloaded? (direct URL, API with pagination, Playwright crawler)
+- Is there authentication? (API key, session, CAPTCHA)
+- What encoding is the source? (UTF-8, ISO-8859-1)
+- Any special parsing needed? (date formats, amount formats, field concatenation)
+
+## Fields
+| Field | Type | Example | Notes |
+|-------|------|---------|-------|
+| TODO | string | TODO | TODO |
+
+## Known issues
+- TODO: List any encoding problems, edge cases, data quality issues
+- TODO: Rate limits, CAPTCHA difficulty, session expiration
+
+## Troubleshooting
+
+### URL changes
+TODO: Where to find the new URL, how to update
+
+### CAPTCHA changes
+TODO: What CAPTCHA solver is used, alternatives if it breaks
+
+### Encoding changes
+TODO: Current encoding, how to detect changes
+
+### Format changes
+TODO: How to detect if columns change, new fields added, fields removed
+
+### Authentication changes
+TODO: Current auth method, what to check if it stops working
+
+## Dependencies
+TODO: List npm packages and why each is needed
+\`\`\`
+playwright — browser automation for CAPTCHA/session
+ddddocr — CAPTCHA OCR solver
+\`\`\`
+
+## Bench results
+${benchData ? `- **Concurrent**: ${benchData.concurrent}
+- **Success rate**: ${benchData.success_rate.toFixed(1)}%
+- **p50**: ${benchData.p50}ms
+- **p95**: ${benchData.p95}ms
+- **p99**: ${benchData.p99}ms` : 'TODO: Run latinfo pipe stage first'}
+`;
+        fs_1.default.writeFileSync(destPath, template);
+        console.log(`[pipe] Template generated: ${destPath}`);
+        console.log(`\nFill in the TODO sections, then run:`);
+        console.log(`  latinfo pipe docs ${sourceName} ${destPath}`);
+        process.exit(1); // Force them to fill it in
+    }
+    console.log(`\n[pipe] Gate 3.5 PASSED ✓`);
+    console.log(`[pipe] Next: latinfo pipe publish ${sourceName}`);
+    status.docs = { passed: true, timestamp: new Date().toISOString() };
+    savePipeStatus(status);
+}
+async function pipePublish(args) {
+    const [sourceName] = args;
+    if (!sourceName) {
+        console.error('Usage: latinfo pipe publish <source-name>');
+        process.exit(1);
+    }
+    const status = loadPipeStatus(sourceName);
+    requireGate(status, 'test', 'publish');
+    requireGate(status, 'validate', 'publish');
+    requireGate(status, 'stage', 'publish');
+    if (!status.docs?.passed) {
+        console.error(`[pipe] Gate "docs" has not passed. Run: latinfo pipe docs ${sourceName}`);
+        console.error(`[pipe] Documentation is required before publishing.`);
         process.exit(1);
     }
+    const repo = getRepoPath();
     const { execSync: run } = await Promise.resolve().then(() => __importStar(require('child_process')));
-    // 1. Add source config to sources.ts
-    console.log(`[publish] Adding ${sourceName} to source registry...`);
-    // TODO: auto-generate sources.ts from YAMLs
-    // 2. Git add + commit + push
-    console.log(`[publish] Committing...`);
+    const RUNNER = 'f3mt0@100.109.82.87';
+    console.log(`[pipe] Gate 4: PUBLISH\n`);
+    // 1. Git add + commit + push
+    console.log(`[pipe] Committing to repo...`);
     const files = [`sources/${sourceName}.yaml`];
     const scriptPath = path_1.default.join(repo, 'src', 'imports', `${sourceName}.ts`);
     if (fs_1.default.existsSync(scriptPath))
         files.push(`src/imports/${sourceName}.ts`);
     try {
-        run(`git add ${files.join(' ')}`, { cwd: repo, stdio: 'pipe' });
+        const docsFile = `docs/sources/${sourceName}.md`;
+        if (fs_1.default.existsSync(path_1.default.join(repo, docsFile)))
+            files.push(docsFile);
+        run(`git add ${files.join(' ')} src/sources.ts .github/workflows/import.yml`, { cwd: repo, stdio: 'pipe' });
         run(`git commit -m "Add data source: ${sourceName}"`, { cwd: repo, stdio: 'pipe' });
         run(`git push`, { cwd: repo, stdio: 'pipe' });
-        console.log(`[publish] Pushed to remote.`);
+        console.log(`[pipe] Pushed to remote.`);
     }
     catch (e) {
-        console.error(`[publish] Git error: ${e.message}`);
+        console.error(`[pipe] Git error: ${e.message}`);
+        process.exit(1);
+    }
+    // 2. Deploy Worker
+    console.log(`[pipe] Deploying Worker...`);
+    try {
+        run(`npx wrangler deploy`, { cwd: repo, stdio: 'inherit' });
+    }
+    catch {
+        console.error(`[pipe] Deploy failed — rolling back`);
+        run(`git revert HEAD --no-edit && git push`, { cwd: repo, stdio: 'pipe' });
         process.exit(1);
     }
-    // 3. Trigger import
-    console.log(`[publish] Triggering import...`);
+    // 3. Trigger import on runner
+    console.log(`[pipe] Triggering import...`);
     try {
         run(`gh workflow run import.yml -f source=${sourceName}`, { cwd: repo, stdio: 'inherit' });
-        console.log(`[publish] Import triggered. Check: gh run list --workflow=import.yml`);
     }
     catch {
-        console.log(`[publish] Could not trigger workflow. Run manually: latinfo imports run ${sourceName}`);
+        console.log(`[pipe] Could not trigger workflow automatically.`);
+    }
+    // 4. Restart search server
+    console.log(`[pipe] Restarting search server on Linux Mint...`);
+    try {
+        run(`ssh ${RUNNER} "sudo systemctl restart search-server 2>/dev/null || echo 'No service yet'"`, { stdio: 'inherit' });
+    }
+    catch { }
+    console.log(`\n[pipe] Gate 4 PASSED ✓`);
+    console.log(`[pipe] ${sourceName} is LIVE`);
+    console.log(`  API: https://api.latinfo.dev/${sourceName.replace(/-/g, '/')}/`);
+    console.log(`  CLI: latinfo ${sourceName.replace(/-/g, ' ')}`);
+    status.publish = { passed: true, timestamp: new Date().toISOString() };
+    savePipeStatus(status);
+}
+async function pipeStatus(args) {
+    const [sourceName] = args;
+    if (sourceName) {
+        const status = loadPipeStatus(sourceName);
+        const gates = ['test', 'validate', 'stage', 'docs', 'publish'];
+        console.log(`Source: ${sourceName}\n`);
+        for (const gate of gates) {
+            const g = status[gate];
+            if (!g) {
+                console.log(`  ${gate}: ⬚ not run`);
+                continue;
+            }
+            const icon = g.passed ? '✓' : '✗';
+            const extra = g.bench ? ` (${g.bench.concurrent} concurrent, ${g.bench.success_rate.toFixed(1)}%, p99: ${g.bench.p99}ms)` : '';
+            const records = g.records ? ` (${g.records} records)` : '';
+            console.log(`  ${gate}: ${icon} ${g.timestamp}${records}${extra}`);
+            if (g.errors)
+                for (const e of g.errors)
+                    console.log(`    ✗ ${e}`);
+        }
+    }
+    else {
+        // List all sources with status
+        if (!fs_1.default.existsSync(PIPE_STATUS_DIR)) {
+            console.log('No sources tracked yet.');
+            return;
+        }
+        const files = fs_1.default.readdirSync(PIPE_STATUS_DIR).filter(f => f.endsWith('.json'));
+        for (const f of files) {
+            const s = JSON.parse(fs_1.default.readFileSync(path_1.default.join(PIPE_STATUS_DIR, f), 'utf-8'));
+            const gates = ['test', 'validate', 'stage', 'docs', 'publish'];
+            const icons = gates.map(g => s[g]?.passed ? '✓' : s[g] ? '✗' : '⬚').join('');
+            console.log(`  ${s.source}  [${icons}]`);
+        }
     }
-    console.log(`\n[publish] ${sourceName}: PUBLISHED`);
 }
-async function admin(args) {
+async function pipe(args) {
     requireAdmin();
     const [subcommand, ...subArgs] = args;
     switch (subcommand) {
         case 'create':
-            await adminCreate(subArgs);
+            await pipeCreate(subArgs);
+            break;
+        case 'script':
+            await pipeScript(subArgs);
             break;
-        case 'upload-script':
-            await adminUploadScript(subArgs);
+        case 'deps':
+            await pipeDeps(subArgs);
             break;
         case 'test':
-            await adminTest(subArgs);
+            await pipeTest(subArgs);
+            break;
+        case 'validate':
+            await pipeValidate(subArgs);
+            break;
+        case 'stage':
+            await pipeStage(subArgs);
+            break;
+        case 'docs':
+            await pipeDocs(subArgs);
             break;
         case 'publish':
-            await adminPublish(subArgs);
+            await pipePublish(subArgs);
+            break;
+        case 'status':
+            await pipeStatus(subArgs);
+            break;
+        case 'list':
+            const repo = getRepoPath();
+            const yamls = fs_1.default.readdirSync(path_1.default.join(repo, 'sources')).filter(f => f.endsWith('.yaml'));
+            for (const y of yamls)
+                console.log(`  ${y.replace('.yaml', '')}`);
+            break;
+        case 'run':
+            const { execSync: run } = await Promise.resolve().then(() => __importStar(require('child_process')));
+            try {
+                run(`gh workflow run import.yml -f source=${subArgs[0]}`, { cwd: getRepoPath(), stdio: 'inherit' });
+            }
+            catch {
+                console.error('Failed to trigger import.');
+            }
             break;
         default:
-            console.log(`latinfo admin — data pipeline for public government records
+            console.log(`latinfo pipe — data pipeline for public government records
 
 DO NOT use SQLite, Supabase, Postgres, or any other database.
 latinfo handles storage, indexing, full-text search, and API serving automatically.
 Your only job is to download and parse the source data into a TSV file.
 
-WORKFLOW
-  Step 1: Create a source definition
-    latinfo admin create <country> <institution> <dataset> [flags]
-
-    Example:
-      latinfo admin create pe redam debtors \\
-        --url "https://redam.pj.gob.pe/data.csv" \\
-        --id-name dni --id-length 8 --encoding utf-8
-
-    This generates a YAML config in sources/{name}.yaml
-    Edit the YAML to match your data: column indexes, field names, delimiter.
+COMMANDS
+  create <country> <institution> <dataset> [flags]   Create source (YAML template)
+  script <source> <file.ts>                          Upload import script
+  deps <source> <pkg1> [pkg2] ...                    Add npm dependencies
+  test <source>                                      Gate 1: test 100 records locally
+  validate <source>                                  Gate 2: full import locally
+  stage <source>                                     Gate 3: import + 500 bench on Linux Mint
+  docs <source> [doc-file]                           Gate 3.5: write/upload documentation
+  publish <source>                                   Gate 4: deploy to production
+  status [source]                                    Show gate status
+  list                                               List all sources
+  run <source>                                       Re-run import (existing source)
 
-  Step 2: Write and upload your import script
-    latinfo admin upload-script <source-name> <script.ts>
+GATES (each must pass before the next unlocks)
+  test     → 100 records, validates IDs, encoding, V2 search, MPHF
+  validate → full import, all records, field validation
+  stage    → Linux Mint: import + 500 concurrent bench (99.9% required)
+  docs     → documentation with required sections (Source, How it works, Fields, etc.)
+  publish  → production: deploy + smoke test + bench + rollback on failure
 
-    Your script must:
-      1. Download the source data (HTTP, crawler, API, etc.)
-      2. Parse it into a TSV file: ID\\tfield1\\tfield2\\t...
-      3. Sort by ID: LC_ALL=C sort -t'\\t' -k1,1
-      4. Call buildBinaryFiles() and buildSearchIndex() from ./build-binary and ./build-search-index
-      5. Call uploadToR2() for each output file
-      6. Call saveImportMeta() at the end
-
-    See SOURCES.md in the repo for the full template and common errors.
-    See src/imports/pe-osce-sanctioned.ts for a clean working example.
-
-  Step 3: Test locally
-    latinfo admin test <source-name>
-
-    Runs your import with --limit 100 and validates the output.
-    Must pass before publishing.
-
-  Step 4: Publish to production
-    latinfo admin publish <source-name>
-
-    Commits your YAML + script, pushes to GitHub, triggers the import workflow.
-    After import completes, the data is live at:
-      API:  https://api.latinfo.dev/{country}/{institution}/{dataset}/...
-      CLI:  latinfo {country} {institution} {dataset} <id|--search query>
+WORKFLOW
+  1. latinfo pipe create pe redam registry --url https://redam.pj.gob.pe/... --id-name dni --id-length 8
+  2. Write your import script (fetch, Playwright, crawler — anything that produces TSV)
+  3. latinfo pipe script pe-redam-registry ./my-crawler.ts
+  4. latinfo pipe deps pe-redam-registry playwright ddddocr
+  5. latinfo pipe test pe-redam-registry
+  6. latinfo pipe validate pe-redam-registry
+  7. latinfo pipe stage pe-redam-registry
+  8. latinfo pipe publish pe-redam-registry
 
-FLAGS FOR CREATE
-  --url <url>          Source data download URL
-  --id-name <name>     Primary ID field name (default: id)
-  --id-length <n>      Primary ID length in digits (default: 11)
-  --encoding <enc>     Source file encoding: utf-8 | iso-8859-1 (default: utf-8)
-  --delimiter <d>      Field delimiter (default: ,)
-  --format <fmt>       Source format: csv | tsv | txt | xlsm (default: csv)
+SCRIPT REQUIREMENTS
+  Your script must produce a sorted TSV and upload to R2:
+    1. Download source data (any method)
+    2. Parse to TSV: ID\\tfield1\\tfield2\\t...
+    3. Sort: LC_ALL=C sort -t'\\t' -k1,1
+    4. buildBinaryFiles() — generates .bin + .idx
+    5. buildSearchIndex() with statusFieldIndex (V2 MANDATORY)
+    6. buildMphfFromIdx() (MPHF MANDATORY)
+    7. uploadToR2() for each file
+    8. saveImportMeta()
 
-NAMING CONVENTION
-  Source name: {country}-{institution}-{dataset}
-  Country: ISO 3166-1 alpha-2 lowercase (pe, co, br, mx, ec, ar, cl)
-  Institution: government agency abbreviation, lowercase
-  Dataset: what the data contains, english, lowercase
+  See SOURCES.md for full template. See src/imports/pe-osce-sanctioned.ts for example.
 
+NAMING
+  {country}-{institution}-{dataset}, all lowercase english.
   Examples: pe-sunat-padron, pe-osce-sanctioned, co-rues-registry
 
 ENVIRONMENT
   LATINFO_ADMIN_SECRET  Auto-detected from ~/.latinfo/admin.secret or .dev.vars
-  LATINFO_REPO_PATH     Auto-detected from cwd or ~/Documents/Github/carrerahaus/latinfo-api`);
+  LATINFO_REPO_PATH     Auto-detected from cwd`);
     }
 }
 // --- Main ---
@@ -1838,9 +2376,12 @@ else {
         case 'bench':
             bench(args).catch(e => { console.error(e); process.exit(1); });
             break;
-        case 'admin':
-            admin(args).catch(e => { console.error(e); process.exit(1); });
+        case 'pipe':
+            pipe(args).catch(e => { console.error(e); process.exit(1); });
             break;
+        case 'admin':
+            pipe(args).catch(e => { console.error(e); process.exit(1); });
+            break; // backward compat
         case 'easypipe':
         case 'ep':
             easypipe(args).catch(e => { console.error(e); process.exit(1); });
@@ -1859,7 +2400,7 @@ else {
             sourceQuery('/pe/sunat/padron', ['--dni', ...args]).catch(e => { console.error(e); process.exit(1); });
             break;
         case 'search':
-            sourceQuery('/pe/sunat/padron', ['--search', ...args]).catch(e => { console.error(e); process.exit(1); });
+            search(args.join(' ')).catch(e => { console.error(e); process.exit(1); });
             break;
         case 'debtors':
             sourceQuery('/pe/sunat/coactiva', args).catch(e => { console.error(e); process.exit(1); });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "latinfo",
-  "version": "0.10.0",
+  "version": "0.12.0",
   "description": "Tax registry & procurement API for Latin America. Query RUC, DNI, NIT, licitaciones from Peru & Colombia. Offline MPHF search, full OCDS data, updated daily.",
   "homepage": "https://latinfo.dev",
   "repository": {