latinfo 0.19.1 → 0.20.0

package/dist/index.js

@@ -907,9 +907,12 @@ async function seedBenchQueries(source, apiKey) {
     }
     // Non-existent search queries (edge case)
     source.searchQueries.push('xyznonexistent', 'qqq999', 'zzznodata');
-    // Fallback: if API returned nothing, use seed queries
-    if (source.searchQueries.length < 5) {
-        source.searchQueries.push(...SEED_QUERIES);
+    // Fallback: if API returned nothing, skip search queries entirely.
+    // Using generic seed queries on a source with different data (e.g. person names
+    // vs business names) causes the search server to process irrelevant queries
+    // under 500 concurrent load, leading to timeouts and false bench failures.
+    if (source.searchQueries.filter(q => !['xyznonexistent', 'qqq999', 'zzznodata'].includes(q)).length === 0) {
+        source.searchQueries = [];
     }
 }
 async function benchStress(args) {
@@ -1903,17 +1906,165 @@ min_rows: 100
 smoke_test:
   id: ""
   expect_field: name
+`;
+    // Derive script template parameters
+    const idLengthNum = parseInt(idLength);
+    const prefixLength = idLengthNum >= 11 ? 5 : 4;
+    const idRegex = `^\\\\d{${idLengthNum}}$`;
+    // camelCase function name: pe-redam-registry → importPeRedamRegistry
+    const fnName = 'import' + name.split('-').map(s => s[0].toUpperCase() + s.slice(1)).join('');
+    const scriptPath = path_1.default.join(repo, 'src', 'imports', `${name}.ts`);
+    const script = `/**
+ * Import ${name} into R2
+ *
+ * Source: ${url}
+ *
+ * TSV columns: ${idName} \\t name \\t status \\t [add more fields here]
+ *
+ * Usage: npx tsx src/imports/${name}.ts [--limit 100] [--no-upload]
+ */
+
+import * as fs from 'fs';
+import * as path from 'path';
+import { execSync } from 'child_process';
+import { buildBinaryFiles } from './build-binary';
+import { buildSearchIndex } from './build-search-index';
+import { uploadToR2, saveImportMeta, buildMphfFromIdx } from './shared';
+
+const SOURCE = '${name}';
+const TEMP_DIR = \`/tmp/\${SOURCE}-import\`;
+
+// ─── FIELD LAYOUT ────────────────────────────────────────────────────────────
+// Update FIELD_COUNT to match the number of columns after the ID.
+// searchFieldIndex: which field (0-based, after ID) contains the searchable name
+// statusFieldIndex: which field contains the active/inactive status
+const FIELD_COUNT = 2;        // name, status  ← update as you add fields
+const SEARCH_FIELD = 0;       // 0 = name
+const STATUS_FIELD = 1;       // 1 = status
+
+// ─── TODO: fetch and parse your source data ──────────────────────────────────
+// Return rows as arrays: [id, name, status, ...other fields]
+// Each string value must NOT contain tabs or newlines.
+//
+// Common patterns:
+//   CSV download  → fetch(url), text.split('\\n'), line.split(',')
+//   REST API      → fetch(url, { method: 'POST', body: JSON.stringify({...}) })
+//   ZIP file      → execSync('curl -L url | funzip > file.csv')
+//   Playwright    → await page.goto(url); await page.$$eval(...)
+//
+// RECOMMENDED: export async function checkFresh(lastMeta): Promise<boolean>
+//   Called before the import to skip if data hasn't changed.
+//   REST APIs: fetch 1 record, compare max ID.
+//   CSVs: HEAD request, compare Last-Modified header.
+async function fetchData(limit?: number): Promise<string[][]> {
+  const rows: string[][] = [];
+
+  // TODO: replace this with your actual fetch + parse logic
+  // Example (CSV):
+  //
+  //   const res = await fetch('${url}');
+  //   const text = await res.text();
+  //   for (const line of text.split('\\n').slice(1)) {   // slice(1) skips header
+  //     const cols = line.split(',');
+  //     const id = cols[0]?.trim().padStart(${idLengthNum}, '0');
+  //     if (!id || !/${idRegex.replace(/\\\\/g, '\\\\\\\\')}/.test(id)) continue;
+  //     const name = cols[1]?.trim() ?? '';
+  //     const status = cols[2]?.trim() ?? '';
+  //     rows.push([id, name, status]);
+  //     if (limit && rows.length >= limit) break;
+  //   }
+
+  return rows;
+}
+// ─────────────────────────────────────────────────────────────────────────────
+
+function clean(s: string): string {
+  return (s ?? '').trim().replace(/[\\t\\n\\r]/g, ' ').replace(/\\s+/g, ' ');
+}
+
+function buildTsv(rows: string[][]): string {
+  return rows.map(cols => cols.map(clean).join('\\t')).join('\\n');
+}
+
+export async function ${fnName}(options?: { limit?: number; upload?: boolean }) {
+  console.log(\`=== \${SOURCE.toUpperCase()} IMPORT ===\\n\`);
+
+  try {
+    fs.mkdirSync(TEMP_DIR, { recursive: true });
+
+    const rows = await fetchData(options?.limit);
+    const minRows = options?.limit ? Math.min(options.limit, 50) : 1000;
+    if (rows.length < minRows) {
+      console.error(\`[\${SOURCE}] Only \${rows.length} rows — expected at least \${minRows}, aborting\`);
+      return false;
+    }
+
+    const tsv = buildTsv(rows);
+    const tsvPath = path.join(TEMP_DIR, 'parsed.tsv');
+    fs.writeFileSync(tsvPath, tsv, 'utf-8');
+    console.log(\`[\${SOURCE}] Wrote \${rows.length.toLocaleString()} rows\`);
+
+    const sortedPath = path.join(TEMP_DIR, 'sorted.tsv');
+    execSync(\`LC_ALL=C sort -t'\\t' -k1,1 "\${tsvPath}" -o "\${sortedPath}"\`, {
+      stdio: 'inherit', env: { ...process.env, TMPDIR: TEMP_DIR },
+    });
+    fs.unlinkSync(tsvPath);
+
+    const config = {
+      idLength: ${idLengthNum},
+      idRegex: /${idRegex}/,
+      prefixLength: ${prefixLength},
+      fieldCount: FIELD_COUNT,
+    };
+    const { shardPaths, idxPath, recordCount } = await buildBinaryFiles(sortedPath, TEMP_DIR, SOURCE, config);
+
+    const search = await buildSearchIndex(
+      sortedPath, TEMP_DIR, SOURCE,
+      { searchFieldIndex: SEARCH_FIELD, idRegex: /${idRegex}/, statusFieldIndex: STATUS_FIELD },
+      recordCount,
+    );
+    const mphfPath = buildMphfFromIdx(search.idxPath);
+    fs.unlinkSync(sortedPath);
+
+    if (options?.upload !== false) {
+      for (let i = 0; i < shardPaths.length; i++) uploadToR2(shardPaths[i], \`\${SOURCE}-\${i}.bin\`);
+      uploadToR2(idxPath, \`\${SOURCE}.idx\`);
+      uploadToR2(search.idxPath, \`\${SOURCE}-search.idx\`);
+      uploadToR2(mphfPath, \`\${SOURCE}-search.mphf\`);
+      for (let i = 0; i < search.shardPaths.length; i++) uploadToR2(search.shardPaths[i], \`\${SOURCE}-search-\${i}.dat\`);
+      saveImportMeta(SOURCE, new Date().toISOString(), recordCount);
+      fs.rmSync(TEMP_DIR, { recursive: true, force: true });
+    } else {
+      console.log(\`\\n[\${SOURCE}] Files in \${TEMP_DIR} (--no-upload, skipping R2)\`);
+    }
+
+    console.log(\`\\n[\${SOURCE}] Success: \${recordCount.toLocaleString()} records\`);
+    return true;
+  } catch (error) {
+    console.error(\`\\n[\${SOURCE}] Error:\`, error);
+    return false;
+  }
+}
+
+if (require.main === module) {
+  const args = process.argv.slice(2);
+  const limitIdx = args.indexOf('--limit');
+  const limit = limitIdx !== -1 ? parseInt(args[limitIdx + 1]) : undefined;
+  const upload = !args.includes('--no-upload');
+  ${fnName}({ limit, upload }).then(ok => process.exit(ok ? 0 : 1));
+}
 `;
     fs_1.default.writeFileSync(yamlPath, yaml);
-    console.log(`Created: ${yamlPath}`);
+    fs_1.default.writeFileSync(scriptPath, script);
+    console.log(`Created:`);
+    console.log(`  ${yamlPath}`);
+    console.log(`  ${scriptPath}`);
     console.log(`\nNext steps:`);
-    console.log(`  1. Edit ${yamlPath} to match your data source`);
-    console.log(`  2. Write import script and upload: latinfo pipe script ${name} ./my-import.ts`);
-    console.log(`  3. Add dependencies: latinfo pipe deps ${name} playwright ddddocr`);
-    console.log(`  4. Test (100 records): latinfo pipe test ${name}`);
-    console.log(`  5. Validate (all records): latinfo pipe validate ${name}`);
-    console.log(`  6. Stage (Linux Mint bench): latinfo pipe stage ${name}`);
-    console.log(`  7. Publish to production: latinfo pipe publish ${name}`);
+    console.log(`  1. Fill in fetchData() in ${scriptPath}`);
+    console.log(`  2. Update FIELD_COUNT, SEARCH_FIELD, STATUS_FIELD if you add more columns`);
+    console.log(`  3. Add dependencies if needed: latinfo pipe deps ${name} playwright ddddocr`);
+    console.log(`  4. Test locally: latinfo pipe local ${name}`);
+    console.log(`  5. Stage + publish: latinfo pipe stage ${name} && latinfo pipe publish ${name}`);
 }
 async function pipeScript(args) {
     const [sourceName, scriptPath] = args;
@@ -2566,7 +2717,7 @@ async function pipePublish(args) {
     }
     catch {
         console.error(`[pipe] Deploy failed — rolling back`);
-        run(`git revert HEAD --no-edit && git push`, { cwd: repo, stdio: 'pipe' });
+        run(`git checkout HEAD^ -- src/sources.ts .github/workflows/import.yml && git commit -m "Rollback: remove ${sourceName}" && git push`, { cwd: repo, stdio: 'pipe' });
         process.exit(1);
     }
     // 3. Trigger import on runner
@@ -2590,30 +2741,83 @@ async function pipePublish(args) {
             const newSources = sourceList.join(',');
             console.log(`[pipe] Adding ${sourceName} to SOURCES: ${newSources}`);
             run(`ssh ${RUNNER} "sudo sed -i 's|^Environment=.*SOURCES=.*|Environment=SOURCES=${newSources}|' /etc/systemd/system/latinfo-search.service && sudo systemctl daemon-reload"`, { stdio: 'pipe' });
+            run(`ssh ${RUNNER} "sudo systemctl restart latinfo-search"`, { stdio: 'inherit' });
+            console.log(`[pipe] Search server restarted.`);
         }
         else {
-            console.log(`[pipe] ${sourceName} already in SOURCES.`);
+            console.log(`[pipe] ${sourceName} already in SOURCES — no restart needed.`);
         }
-        run(`ssh ${RUNNER} "sudo systemctl restart latinfo-search"`, { stdio: 'inherit' });
-        console.log(`[pipe] Search server restarted.`);
     }
     catch {
         console.log(`[pipe] Could not update search server (not critical).`);
     }
     // 5. Production bench: 500 concurrent against api.latinfo.dev
-    console.log(`\n[pipe] Running production bench (500 concurrent)...`);
+    // Warm up: wait for the new source to be responsive before hammering it
+    console.log(`\n[pipe] Warming up (waiting for search server + Worker index load)...`);
     try {
         const config = loadConfig();
         if (!config?.api_key)
             throw new Error('No API key');
-        const bench = await benchProduction(sourceName, config.api_key, 500);
+        // Progressive warm-up: single probe → small batch → ready for 500
+        const warmupRoute = (() => {
+            const src = discoverBenchSources(sourceName);
+            return src.length > 0 ? src[0].routePath : `/${sourceName.replace(/-/g, '/')}`;
+        })();
+        const warmupUrl = `${API_URL}${warmupRoute}/search?q=garcia&limit=1`;
+        const warmupHeaders = { Authorization: `Bearer ${config.api_key}` };
+        // Phase 1: wait for first successful response (up to 90s)
+        let warmedUp = false;
+        for (let attempt = 0; attempt < 30; attempt++) {
+            await new Promise(r => setTimeout(r, 3000));
+            try {
+                const r = await fetch(warmupUrl, { headers: warmupHeaders, signal: AbortSignal.timeout(5000) });
+                if (r.status === 200 || r.status === 404) {
+                    warmedUp = true;
+                    break;
+                }
+            }
+            catch { }
+            process.stdout.write('.');
+        }
+        if (!warmedUp) {
+            console.log(`\n[pipe] Warm-up timed out — running bench anyway`);
+        }
+        else {
+            // Phase 2: warm up LOOKUP path too (bench uses lookups, not just search)
+            const src = discoverBenchSources(sourceName);
+            const smokeId = src.length > 0 ? src[0].smokeId : null;
+            if (smokeId) {
+                const lookupUrl = `${API_URL}${warmupRoute}/${src[0].primaryId.name}/${smokeId}`;
+                process.stdout.write(' lookup');
+                for (let i = 0; i < 5; i++) {
+                    await new Promise(r => setTimeout(r, 2000));
+                    try {
+                        await fetch(lookupUrl, { headers: warmupHeaders, signal: AbortSignal.timeout(10000) });
+                    }
+                    catch { }
+                    process.stdout.write('.');
+                }
+            }
+            // Phase 3: progressive concurrent batches (20 → 50 → 100)
+            for (const batchSize of [20, 50, 100]) {
+                const batch = await Promise.all(Array.from({ length: batchSize }, () => fetch(warmupUrl, { headers: warmupHeaders, signal: AbortSignal.timeout(10000) })
+                    .then(r => r.status === 200 || r.status === 404 ? 1 : 0).catch(() => 0)));
+                const batchOk = batch.reduce((a, b) => a + b, 0);
+                process.stdout.write(` ${batchOk}/${batchSize}`);
+                await new Promise(r => setTimeout(r, 2000));
+            }
+            console.log('');
+            console.log(`[pipe] Warm-up OK`);
+        }
+        console.log(`[pipe] Running production bench (100 concurrent)...`);
+        const bench = await benchProduction(sourceName, config.api_key, 100);
         console.log(`\n  Production bench: ${bench.qps} q/s, ${bench.success_rate.toFixed(1)}% success`);
         console.log(`  p50: ${bench.p50}ms  p95: ${bench.p95}ms  p99: ${bench.p99}ms`);
         if (bench.success_rate < 99.9) {
             console.error(`\n[pipe] PRODUCTION BENCH FAILED — ${bench.success_rate.toFixed(1)}% < 99.9%`);
             console.error(`[pipe] Rolling back...`);
             try {
-                run(`git revert HEAD --no-edit && git push`, { cwd: repo, stdio: 'pipe' });
+                run(`git checkout HEAD^ -- src/sources.ts .github/workflows/import.yml && git commit -m "Rollback: remove ${sourceName}" && git push`, { cwd: repo, stdio: 'pipe' });
                 run(`npx wrangler deploy`, { cwd: repo, stdio: 'pipe' });
             }
             catch { }
@@ -2633,6 +2837,67 @@ async function pipePublish(args) {
     console.log(`  API: https://api.latinfo.dev/${sourceName.replace(/-/g, '/')}/`);
     console.log(`  CLI: latinfo ${sourceName.replace(/-/g, ' ')}`);
 }
+async function report(args) {
+    const message = args.join(' ').trim();
+    if (!message) {
+        console.error('Usage: latinfo report <message>');
+        console.error('Example: latinfo report "search returns empty results for banco"');
+        process.exit(1);
+    }
+    const config = loadConfig();
+    if (!config?.api_key) {
+        console.error('Not logged in. Run: latinfo login');
+        process.exit(1);
+    }
+    const res = await fetch(`${API_URL}/feedback`, {
+        method: 'POST',
+        headers: { 'Authorization': `Bearer ${config.api_key}`, 'Content-Type': 'application/json' },
+        body: JSON.stringify({
+            message,
+            cli_version: VERSION,
+            os: process.platform,
+        }),
+    });
+    if (!res.ok) {
+        const err = await res.json();
+        console.error(`Error: ${err.message || err.error}`);
+        process.exit(1);
+    }
+    console.log('Report sent. Thank you — we will look into it.');
+}
+async function issues() {
+    const config = loadConfig();
+    const adminSecret = process.env.ADMIN_SECRET ||
+        (() => { try {
+            return JSON.parse(fs_1.default.readFileSync(path_1.default.join(getRepoPath(), '.dev.vars'), 'utf-8').split('\n').find(l => l.startsWith('ADMIN_SECRET='))?.split('=')[1] || '');
+        }
+        catch {
+            return '';
+        } })();
+    if (!adminSecret) {
+        console.error('ADMIN_SECRET not found. Set it in .dev.vars or ADMIN_SECRET env var.');
+        process.exit(1);
+    }
+    const status = process.argv.includes('--resolved') ? 'resolved' : 'open';
+    const res = await fetch(`${API_URL}/admin/feedback?status=${status}`, {
+        headers: { 'Authorization': `Bearer ${adminSecret}` },
+    });
+    if (!res.ok) {
+        console.error(`Error: ${res.status}`);
+        process.exit(1);
+    }
+    const rows = await res.json();
+    if (rows.length === 0) {
+        console.log(`No ${status} issues.`);
+        return;
+    }
+    for (const r of rows) {
+        const date = r.created_at.slice(0, 10);
+        const meta = [r.cli_version, r.os].filter(Boolean).join(', ');
+        console.log(`[#${r.id}] ${date} @${r.github_username}${meta ? ` (${meta})` : ''}`);
+        console.log(`  ${r.message}\n`);
+    }
+}
 async function pipeStatus(args) {
     const [sourceName] = args;
     if (sourceName) {
@@ -2669,6 +2934,357 @@ async function pipeStatus(args) {
         }
     }
 }
+async function pipeLocal(args) {
+    const [sourceName] = args;
+    if (!sourceName) {
+        console.error('Usage: latinfo pipe local <source-name>');
+        process.exit(1);
+    }
+    const repo = getRepoPath();
+    const { execSync: run } = await Promise.resolve().then(() => __importStar(require('child_process')));
+    const yamlPath = path_1.default.join(repo, 'sources', `${sourceName}.yaml`);
+    const yamlContent = fs_1.default.existsSync(yamlPath) ? fs_1.default.readFileSync(yamlPath, 'utf-8') : '';
+    const importScriptMatch = yamlContent.match(/import_script:\s*(.+)/);
+    const customScript = importScriptMatch ? path_1.default.join(repo, importScriptMatch[1].trim()) : null;
+    const defaultScript = path_1.default.join(repo, 'src', 'imports', `${sourceName}.ts`);
+    const scriptPath = customScript && fs_1.default.existsSync(customScript) ? customScript
+        : fs_1.default.existsSync(defaultScript) ? defaultScript : null;
+    const easypipePath = path_1.default.join(repo, 'src', 'imports', 'easypipe.ts');
+    const cmd = scriptPath
+        ? `npx tsx ${scriptPath} --no-upload`
+        : `npx tsx ${easypipePath} ${yamlPath} --no-upload`;
+    const TEMP_DIR = `/tmp/${sourceName}-import`;
+    const MAX_SHARD_MB = 250;
+    const BENCH_SAMPLES = 500;
+    const IMPORT_RUNS = 1;
+    let passed = true;
+    const errors = [];
+    // ── Run import N times ───────────────────────────────────────────────────
+    for (let run_i = 1; run_i <= IMPORT_RUNS; run_i++) {
+        console.log(`\n[pipe:local] ── Import run ${run_i}/${IMPORT_RUNS} ──`);
+        try {
+            run(cmd, { cwd: repo, stdio: 'inherit' });
+        }
+        catch {
+            errors.push(`Import run ${run_i} crashed`);
+            passed = false;
+            break;
+        }
+        // ── Validate files ─────────────────────────────────────────────────────
+        const expected = [
+            `${sourceName}-0.bin`,
+            `${sourceName}.idx`,
+            `${sourceName}-search.idx`,
+            `${sourceName}-search.mphf`,
+            `${sourceName}-search-0.dat`,
+        ];
+        for (const f of expected) {
+            const fp = path_1.default.join(TEMP_DIR, f);
+            if (!fs_1.default.existsSync(fp)) {
+                errors.push(`Missing: ${f}`);
+                passed = false;
+            }
+        }
+        // ── Shard sizes ────────────────────────────────────────────────────────
+        const bins = fs_1.default.existsSync(TEMP_DIR) ? fs_1.default.readdirSync(TEMP_DIR).filter(f => f.endsWith('.bin')) : [];
+        for (const b of bins) {
+            const mb = fs_1.default.statSync(path_1.default.join(TEMP_DIR, b)).size / 1_048_576;
+            if (mb > MAX_SHARD_MB) {
+                errors.push(`Shard ${b} is ${mb.toFixed(0)}MB > ${MAX_SHARD_MB}MB`);
+                passed = false;
+            }
+            else
+                console.log(`  ✓ ${b}: ${mb.toFixed(1)} MB`);
+        }
+        // ── V2 search index ────────────────────────────────────────────────────
+        const searchIdx = path_1.default.join(TEMP_DIR, `${sourceName}-search.idx`);
+        if (fs_1.default.existsSync(searchIdx)) {
+            const content = fs_1.default.readFileSync(searchIdx);
+            // V2 magic: first 4 bytes are "LSRY" (V1 = "LSRX")
+            const magic = content.subarray(0, 4).toString('ascii');
+            if (magic !== 'LSRY') {
+                errors.push(`Search index is V1 (magic=${magic}) — must use V2 (statusFieldIndex required)`);
+                passed = false;
+            }
+            else
+                console.log(`  ✓ Search index: V2`);
+        }
+        // ── MPHF ───────────────────────────────────────────────────────────────
+        const mphf = path_1.default.join(TEMP_DIR, `${sourceName}-search.mphf`);
+        if (fs_1.default.existsSync(mphf) && fs_1.default.statSync(mphf).size > 0) {
+            console.log(`  ✓ MPHF: ${(fs_1.default.statSync(mphf).size / 1024).toFixed(1)} KB`);
+        }
+        else {
+            errors.push('MPHF missing or empty — call buildMphfFromIdx()');
+            passed = false;
+        }
+        if (!passed)
+            break;
+        if (run_i < IMPORT_RUNS)
+            console.log(`  ✓ Run ${run_i} OK`);
+    }
+    if (!passed) {
+        console.error(`\n[pipe:local] FAILED:`);
+        for (const e of errors)
+            console.error(`  ✗ ${e}`);
+        process.exit(1);
+    }
+    // ── Load binary into memory (shared by bench + smoke test + quality check) ──
+    const idLen = parseInt(yamlContent.match(/length:\s*(\d+)/)?.[1] || '8');
+    const prefixLen = parseInt(yamlContent.match(/prefix_length:\s*(\d+)/)?.[1] || '5');
+    const idxPath = path_1.default.join(TEMP_DIR, `${sourceName}.idx`);
+    const binPath = path_1.default.join(TEMP_DIR, `${sourceName}-0.bin`);
+    let binBuf;
+    let index;
+    try {
+        if (!fs_1.default.existsSync(idxPath) || !fs_1.default.existsSync(binPath))
+            throw new Error('Missing .idx or .bin');
+        binBuf = fs_1.default.readFileSync(binPath);
+        const HEADER_SIZE = 16, ENTRY_SIZE = 16, MAGIC = [0x4c, 0x49, 0x44, 0x58];
+        const idxBuf = fs_1.default.readFileSync(idxPath);
+        for (let i = 0; i < 4; i++)
+            if (idxBuf[i] !== MAGIC[i])
+                throw new Error('Invalid index magic');
+        const entryCount = idxBuf.readUInt32LE(4);
+        index = [];
+        for (let i = 0; i < entryCount; i++) {
+            const off = HEADER_SIZE + i * ENTRY_SIZE;
+            index.push({
+                prefix: idxBuf.readUInt32LE(off),
+                shard: idxBuf.readUInt32LE(off + 4),
+                offset: idxBuf.readUInt32LE(off + 8),
+                length: idxBuf.readUInt32LE(off + 12),
+            });
+        }
+    }
+    catch (e) {
+        errors.push(`Cannot load binary: ${e.message}`);
+        passed = false;
+        index = [];
+        binBuf = Buffer.alloc(0);
+    }
+    // Lookup a record by ID — returns field array (after the ID) or null
+    function lookupRecord(id) {
+        const prefix = parseInt(id.substring(0, prefixLen));
+        let lo = 0, hi = index.length - 1, entry = null;
+        while (lo <= hi) {
+            const mid = (lo + hi) >>> 1;
+            if (index[mid].prefix === prefix) {
+                entry = index[mid];
+                break;
+            }
+            if (index[mid].prefix < prefix)
+                lo = mid + 1;
+            else
+                hi = mid - 1;
+        }
+        if (!entry)
+            return null;
+        const chunk = binBuf.subarray(entry.offset, entry.offset + entry.length);
+        let p = 0;
+        while (p < chunk.length) {
+            const rlen = chunk.readUInt16LE(p);
+            if (rlen < 2)
+                break;
+            const rid = chunk.subarray(p + 2, p + 2 + idLen).toString();
+            if (rid === id) {
+                // Parse fields: [uint8 len][bytes]...
+                const fields = [];
+                let fp = p + 2 + idLen;
+                while (fp < p + rlen) {
+                    const flen = chunk[fp++];
+                    fields.push(chunk.subarray(fp, fp + flen).toString('utf-8'));
+                    fp += flen;
+                }
+                return fields;
+            }
+            if (rid > id)
+                return null;
+            p += rlen;
+        }
+        return null;
+    }
+    // ── Smoke test: look up known ID and verify expected field ────────────────
+    console.log(`\n[pipe:local] ── Smoke test ──`);
+    const smokeId = yamlContent.match(/smoke_test:\s*\n\s+id:\s*"?([^"\n]+)"?/)?.[1]?.trim();
+    const smokeField = yamlContent.match(/expect_field:\s*(\w+)/)?.[1]?.trim();
+    if (!smokeId || smokeId === '""' || !smokeField) {
+        errors.push('smoke_test.id and smoke_test.expect_field are required in sources YAML');
+        passed = false;
+    }
+    else {
+        const fieldNames = [...yamlContent.matchAll(/- name:\s*(\w+)/g)].map(m => m[1]);
+        const fieldIdx = fieldNames.indexOf(smokeField);
+        if (fieldIdx === -1) {
+            errors.push(`smoke_test.expect_field "${smokeField}" not found in fields list`);
+            passed = false;
+        }
+        else {
+            const record = lookupRecord(smokeId);
+            if (!record) {
+                errors.push(`Smoke test FAILED: ID "${smokeId}" not found in binary — fetchData() may be returning wrong data`);
+                passed = false;
+            }
+            else {
+                // Field count check: binary record fields must match YAML fields
+                if (record.length !== fieldNames.length) {
+                    errors.push(`Field count mismatch: binary has ${record.length} fields but YAML lists ${fieldNames.length} (${fieldNames.join(', ')}). Fix YAML or import script.`);
+                    passed = false;
+                }
+                else {
+                    console.log(`  ✓ field count: ${record.length} (matches YAML)`);
+                }
+                // Field content validation: verify each field value matches its name
+                for (let fi = 0; fi < Math.min(record.length, fieldNames.length); fi++) {
+                    const fname = fieldNames[fi];
+                    const fval = (record[fi] ?? '').trim();
+                    console.log(`    ${fname} = "${fval.slice(0, 60)}${fval.length > 60 ? '...' : ''}"`);
+                    if (!fval)
+                        continue; // empty fields checked in data quality
+                    const isDate = /^\d{2}[\/\-]\d{2}[\/\-]\d{4}$/.test(fval) || /^\d{4}[\/\-]\d{2}[\/\-]\d{2}/.test(fval) || /^\d{8}$/.test(fval);
+                    const isNumeric = /^\d+$/.test(fval);
+                    const hasLetters = /[a-zA-ZÁÉÍÓÚÑáéíóúñ]/.test(fval);
+                    if (fname.startsWith('fecha') && !isDate && !fval.includes('/') && !fval.includes('-')) {
+                        errors.push(`Field "${fname}" = "${fval.slice(0, 40)}" — expected a date`);
+                        passed = false;
+                    }
+                    if (fname.endsWith('_count') && !isNumeric) {
+                        errors.push(`Field "${fname}" = "${fval.slice(0, 40)}" — expected a number`);
+                        passed = false;
+                    }
+                    if (fname === 'nombre' && !hasLetters) {
+                        errors.push(`Field "nombre" = "${fval.slice(0, 40)}" — expected letters (got numbers/dates)`);
+                        passed = false;
+                    }
+                    if ((fname.startsWith('estado') || fname === 'tipo_sancion') && isDate) {
+                        errors.push(`Field "${fname}" = "${fval.slice(0, 40)}" — looks like a date, not a status/type`);
+                        passed = false;
+                    }
+                    if (fname.includes('entidad') && isDate) {
+                        errors.push(`Field "${fname}" = "${fval.slice(0, 40)}" — looks like a date, not an entity name`);
+                        passed = false;
+                    }
+                }
+                const value = record[fieldIdx] ?? '';
+                if (!value.trim()) {
+                    errors.push(`Smoke test FAILED: field "${smokeField}" is empty for ID "${smokeId}" — check your field mapping`);
+                    passed = false;
+                }
+                else {
+                    console.log(`  ✓ ${smokeField} = "${value.trim()}"`);
+                }
+            }
+        }
+    }
+    // ── Data quality: check empty name/status rates ───────────────────────────
+    console.log(`\n[pipe:local] ── Data quality ──`);
+    try {
+        const fieldNames = [...yamlContent.matchAll(/- name:\s*(\w+)/g)].map(m => m[1]);
+        let total = 0, emptyName = 0, emptyStatus = 0;
+        let pos2 = 0;
+        while (pos2 < binBuf.length) {
+            const rlen = binBuf.readUInt16LE(pos2);
+            if (rlen < 2 + idLen)
+                break;
+            let fp = pos2 + 2 + idLen;
+            const fields = [];
+            while (fp < pos2 + rlen) {
+                const flen = binBuf[fp++];
+                fields.push(binBuf.subarray(fp, fp + flen).toString('utf-8'));
+                fp += flen;
+            }
+            total++;
+            if (!fields[0]?.trim())
+                emptyName++;
+            if (!fields[1]?.trim())
+                emptyStatus++;
+            pos2 += rlen;
+        }
+        const emptyNamePct = total > 0 ? (emptyName / total) * 100 : 0;
+        const emptyStatusPct = total > 0 ? (emptyStatus / total) * 100 : 0;
+        const nameLabel = fieldNames[0] || 'name';
+        const statusLabel = fieldNames[1] || 'status';
+        console.log(`  ${total.toLocaleString()} records scanned`);
+        if (emptyNamePct > 5) {
+            errors.push(`${emptyNamePct.toFixed(1)}% of records have empty "${nameLabel}" — check searchFieldIndex`);
+            passed = false;
+        }
+        else {
+            console.log(`  ✓ ${nameLabel}: ${emptyNamePct.toFixed(1)}% empty`);
+        }
+        if (emptyStatusPct > 50) {
+            errors.push(`${emptyStatusPct.toFixed(1)}% of records have empty "${statusLabel}" — check statusFieldIndex`);
+            passed = false;
+        }
+        else {
+            console.log(`  ✓ ${statusLabel}: ${emptyStatusPct.toFixed(1)}% empty`);
+        }
+    }
+    catch (e) {
+        errors.push(`Quality check failed: ${e.message}`);
+        passed = false;
+    }
+    // ── Local benchmark: N random lookups from binary ─────────────────────────
+    console.log(`\n[pipe:local] ── Local benchmark: ${BENCH_SAMPLES} random lookups ──`);
+    try {
+        const allIds = [];
+        let pos3 = 0;
+        while (pos3 < binBuf.length && allIds.length < BENCH_SAMPLES * 10) {
+            const rlen = binBuf.readUInt16LE(pos3);
+            if (rlen < 2 + idLen || pos3 + rlen > binBuf.length)
+                break;
+            allIds.push(binBuf.subarray(pos3 + 2, pos3 + 2 + idLen).toString());
+            pos3 += rlen;
+        }
+        if (allIds.length === 0)
+            throw new Error('Could not read any records from .bin');
+        for (let i = allIds.length - 1; i > 0; i--) {
+            const j = Math.floor(Math.random() * (i + 1));
+            [allIds[i], allIds[j]] = [allIds[j], allIds[i]];
+        }
+        const sample = allIds.slice(0, BENCH_SAMPLES);
+        const latencies = [];
+        let found = 0;
+        for (const id of sample) {
+            const t0 = performance.now();
+            if (lookupRecord(id))
+                found++;
+            latencies.push(performance.now() - t0);
+        }
+        latencies.sort((a, b) => a - b);
+        const p50 = latencies[Math.floor(latencies.length * 0.50)];
+        const p95 = latencies[Math.floor(latencies.length * 0.95)];
+        const p99 = latencies[Math.floor(latencies.length * 0.99)];
+        console.log(`  ${BENCH_SAMPLES} lookups: p50=${p50.toFixed(2)}ms  p95=${p95.toFixed(2)}ms  p99=${p99.toFixed(2)}ms`);
+        console.log(`  Hit rate: ${found}/${BENCH_SAMPLES} (${((found / BENCH_SAMPLES) * 100).toFixed(1)}%)`);
+        if (found < BENCH_SAMPLES * 0.99) {
+            errors.push(`Low hit rate: ${found}/${BENCH_SAMPLES}`);
+            passed = false;
+        }
+    }
+    catch (e) {
+        errors.push(`Benchmark failed: ${e.message}`);
+        passed = false;
+    }
+    // ── Final summary ─────────────────────────────────────────────────────────
+    if (errors.length > 0) {
+        console.error(`\n[pipe:local] FAILED:`);
+        for (const e of errors)
+            console.error(`  ✗ ${e}`);
+        process.exit(1);
+    }
+    // Mark test + validate as passed so pipe stage can proceed
+    const status = loadPipeStatus(sourceName);
+    const now = new Date().toISOString();
+    status.test = { passed: true, timestamp: now };
+    status.validate = { passed: true, timestamp: now };
+    savePipeStatus(status);
+    console.log(`\n[pipe:local] ✓ ALL CHECKS PASSED`);
+    console.log(`  shards ≤ ${MAX_SHARD_MB}MB  •  V2  •  MPHF  •  bench OK`);
+    console.log(`  Gates test + validate marked as passed.`);
+    console.log(`  Files ready in: ${TEMP_DIR}`);
+    console.log(`\n  Next: latinfo pipe stage ${sourceName}  (uploads to R2 + Linux Mint bench)`);
+}
 async function pipe(args) {
     requireAdmin();
     const [subcommand, ...subArgs] = args;
@@ -2706,6 +3322,9 @@ async function pipe(args) {
             for (const y of yamls)
                 console.log(`  ${y.replace('.yaml', '')}`);
             break;
+        case 'local':
+            await pipeLocal(subArgs);
+            break;
         case 'run':
             const { execSync: run } = await Promise.resolve().then(() => __importStar(require('child_process')));
             try {
@@ -2726,6 +3345,7 @@ COMMANDS
   create <country> <institution> <dataset> [flags]   Create source (YAML template)
   script <source> <file.ts>                          Upload import script
   deps <source> <pkg1> [pkg2] ...                    Add npm dependencies
+  local <source>                                     Full local test: 3x import + file checks + bench
   test <source>                                      Gate 1: test 100 records locally
   validate <source>                                  Gate 2: full import locally
   stage <source>                                     Gate 3: import + 500 bench on Linux Mint
@@ -2763,6 +3383,11 @@ SCRIPT REQUIREMENTS
     7. uploadToR2() for each file
     8. saveImportMeta()
 
+  RECOMMENDED: export async function checkFresh(lastMeta): Promise<boolean>
+    Called before the import to skip if data hasn't changed. Saves RAM + API calls.
+    REST APIs: fetch 1 record, compare max ID. CSVs: HEAD request, compare Last-Modified.
+    Without this, the import always runs regardless of whether the source updated.
+
   See SOURCES.md for full template. See src/imports/pe-osce-sanctioned.ts for example.
 
 NAMING
@@ -3242,6 +3867,12 @@ else {
         case 'ep':
             easypipe(args).catch(e => { console.error(e); process.exit(1); });
             break;
+        case 'report':
+            report(args).catch(e => { console.error(e); process.exit(1); });
+            break;
+        case 'issues':
+            issues().catch(e => { console.error(e); process.exit(1); });
+            break;
         case 'completion':
             completion();
             break;

package/dist/sdk.js

@@ -70,7 +70,7 @@ class Country {
         const tokenPostings = [];
         for (const r of resolved) {
             const lists = await Promise.all(r.entries.map(async (entry) => {
-                const byteLen = Math.min(entry.count, 50000) * idx.entrySize;
+                const byteLen = Math.min(entry.count, 2000) * idx.entrySize;
                 const cdnUrl = `https://data.latinfo.dev/${this.cfg.baseName}-search-${entry.shard}.dat`;
                 const res = await fetch(cdnUrl, {
                     headers: { Range: `bytes=${entry.offset}-${entry.offset + byteLen - 1}` },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "latinfo",
-  "version": "0.19.1",
+  "version": "0.20.0",
   "description": "Tax registry & procurement API for Latin America. Query RUC, DNI, NIT, licitaciones from Peru & Colombia. Offline MPHF search, full OCDS data, updated daily.",
   "homepage": "https://latinfo.dev",
   "repository": {